diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 1bcb6f08..c8750694 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -198,6 +198,12 @@ targets: pipelineName: hwlab-d601-v03-ci-image-publish serviceAccountName: hwlab-d601-v03-tekton-runner pipelineRunPrefix: hwlab-d601-v03-ci-poll + gitWorkspaceSecret: + name: hwlab-git-ssh + namespace: hwlab-ci + sourceRefFrom: gitMirror.githubTransport + privateKeySecretKey: ssh-privatekey + knownHostsSecretKey: known_hosts toolsImage: output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 imagePullPolicy: Always @@ -349,6 +355,12 @@ targets: pipelineName: hwlab-d518-v03-ci-image-publish serviceAccountName: hwlab-d518-v03-tekton-runner pipelineRunPrefix: hwlab-d518-v03-ci-poll + gitWorkspaceSecret: + name: hwlab-git-ssh + namespace: hwlab-ci + sourceRefFrom: gitMirror.githubTransport + privateKeySecretKey: ssh-privatekey + knownHostsSecretKey: known_hosts toolsImage: output: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 imagePullPolicy: Always diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 37642d16..36588a8f 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -132,6 +132,14 @@ type ControlPlaneGitMirrorGithubTransportSpec = tokenSourceKey: string; }; +interface ControlPlaneTektonGitWorkspaceSecretSpec { + name: string; + namespace: string; + sourceRefFrom: "gitMirror.githubTransport"; + privateKeySecretKey: string; + knownHostsSecretKey: string; +} + interface ControlPlaneNodeSpec { id: string; route: string; @@ -192,6 +200,7 @@ interface ControlPlaneTargetSpec { pipelineName: string; serviceAccountName: string; pipelineRunPrefix: string; + gitWorkspaceSecret: ControlPlaneTektonGitWorkspaceSecretSpec; toolsImage: { output: string; imagePullPolicy: "Always" | "IfNotPresent" | "Never"; @@ -285,6 +294,14 @@ function controlPlaneContext(nodeId: string, lane: string): { config: ControlPla return { config, node, target }; } +export function hwlabNodeControlPlaneCiGitWorkspaceSecret(nodeId: string, lane: string): { name: string; namespace: string } { + const { target } = controlPlaneContext(nodeId, lane); + return { + name: target.tekton.gitWorkspaceSecret.name, + namespace: target.tekton.gitWorkspaceSecret.namespace, + }; +} + export function hwlabNodeControlPlaneInfraHelp(): Record { return { ok: true, @@ -356,6 +373,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta const gitMirrorGithubTransport = record(gitMirror.githubTransport); const tekton = record(components.tekton); const ciNamespace = record(components.ciNamespace); + const ciGitWorkspaceSecret = record(ciNamespace.gitWorkspaceSecret); const registry = record(components.registry); const k3sNodeConfig = record(components.k3sNodeConfig); const k3sNodeConfigReady = node.k3s === null @@ -366,6 +384,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta && k3sNodeConfigReady && boolField(tekton, "installed") && boolField(ciNamespace, "exists") + && boolField(ciGitWorkspaceSecret, "ready") && boolField(gitMirror, "namespaceExists") && boolField(gitMirror, "readServiceExists") && boolField(gitMirror, "writeServiceExists") @@ -394,6 +413,7 @@ function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, ta k3sNodeConfigReady, tektonInstalled: boolField(tekton, "installed"), ciNamespaceExists: boolField(ciNamespace, "exists"), + ciGitWorkspaceSecretReady: boolField(ciGitWorkspaceSecret, "ready"), gitMirrorNamespaceExists: boolField(gitMirror, "namespaceExists"), gitMirrorReadServiceExists: boolField(gitMirror, "readServiceExists"), gitMirrorWriteServiceExists: boolField(gitMirror, "writeServiceExists"), @@ -792,7 +812,7 @@ function ciBuildBenchmarkPipelineRunManifest( ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, - { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }, + { name: "git-ssh", secret: { secretName: target.tekton.gitWorkspaceSecret.name } }, ], }, }; @@ -1512,6 +1532,42 @@ function gitMirrorGithubTransportSpec(raw: Record, path: string }; } +function tektonGitWorkspaceSecretSpec( + value: unknown, + path: string, + ciNamespace: string, + githubTransport: ControlPlaneGitMirrorGithubTransportSpec, +): ControlPlaneTektonGitWorkspaceSecretSpec { + const raw = asRecord(value, path); + const sourceRefFrom = stringField(raw, "sourceRefFrom", path); + if (sourceRefFrom !== "gitMirror.githubTransport") { + throw new Error(`${path}.sourceRefFrom must be gitMirror.githubTransport`); + } + if (githubTransport.mode !== "ssh") { + throw new Error(`${path}.sourceRefFrom=gitMirror.githubTransport requires gitMirror.githubTransport.mode=ssh`); + } + if (githubTransport.knownHostsSecretKey === null) { + throw new Error(`${path}.sourceRefFrom=gitMirror.githubTransport requires gitMirror.githubTransport.knownHostsSecretKey`); + } + const name = stringField(raw, "name", path); + const namespace = stringField(raw, "namespace", path); + const privateKeySecretKey = stringField(raw, "privateKeySecretKey", path); + const knownHostsSecretKey = stringField(raw, "knownHostsSecretKey", path); + validateKubernetesName(name, `${path}.name`); + validateKubernetesName(namespace, `${path}.namespace`); + validateSecretKey(privateKeySecretKey, `${path}.privateKeySecretKey`); + validateSecretKey(knownHostsSecretKey, `${path}.knownHostsSecretKey`); + if (namespace !== ciNamespace) throw new Error(`${path}.namespace must equal targets[].ciNamespace ${ciNamespace}`); + if (privateKeySecretKey !== "ssh-privatekey") throw new Error(`${path}.privateKeySecretKey must be ssh-privatekey for Tekton git SSH workspaces`); + return { + name, + namespace, + sourceRefFrom, + privateKeySecretKey, + knownHostsSecretKey, + }; +} + function secretSourceEncodingField(raw: Record, key: string, path: string): "plain" | "base64" { const value = stringField(raw, key, path); if (value !== "plain" && value !== "base64") throw new Error(`${path}.${key} must be plain or base64`); @@ -1534,12 +1590,13 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa const gitMirrorEgressProxy = gitMirror.egressProxy === undefined ? null : gitMirrorEgressProxySpec(asRecord(gitMirror.egressProxy, `${path}.gitMirror.egressProxy`), `${path}.gitMirror.egressProxy`); const githubTransport = gitMirrorGithubTransportSpec(asRecord(gitMirror.githubTransport, `${path}.gitMirror.githubTransport`), `${path}.gitMirror.githubTransport`); const sourceRepository = stringField(source, "repository", `${path}.source`); + const ciNamespace = stringField(raw, "ciNamespace", path); return { id: stringField(raw, "id", path), node, lane, enabled: booleanField(raw, "enabled", path), - ciNamespace: stringField(raw, "ciNamespace", path), + ciNamespace, runtimeNamespace: stringField(raw, "runtimeNamespace", path), source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`) }, gitops: { branch: stringField(gitops, "branch", `${path}.gitops`), path: stringField(gitops, "path", `${path}.gitops`) }, @@ -1565,6 +1622,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`), serviceAccountName: stringField(tekton, "serviceAccountName", `${path}.tekton`), pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${path}.tekton`), + gitWorkspaceSecret: tektonGitWorkspaceSecretSpec(tekton.gitWorkspaceSecret, `${path}.tekton.gitWorkspaceSecret`, ciNamespace, githubTransport), toolsImage: toolsImageSpec(toolsImage, `${path}.tekton.toolsImage`), }, ciBuildBenchmarks: ciBuildBenchmarkProfileSpecs(raw.ciBuildBenchmarks, `${path}.ciBuildBenchmarks`), @@ -1609,6 +1667,8 @@ function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTa if (githubTokenSecret !== null) manifests.push(githubTokenSecret); const githubSshSecret = gitMirrorGithubSshSecret(target, labels); if (githubSshSecret !== null) manifests.push(githubSshSecret); + const ciGitWorkspaceSecret = tektonGitWorkspaceSecret(target, labels); + if (ciGitWorkspaceSecret !== null) manifests.push(ciGitWorkspaceSecret); if (target.gitMirror.cacheHostPath === null) { manifests.push({ apiVersion: "v1", @@ -1682,6 +1742,41 @@ function gitMirrorGithubSshSecret(target: ControlPlaneTargetSpec, labels: Record }; } +function tektonGitWorkspaceSecret(target: ControlPlaneTargetSpec, labels: Record): Record | null { + const transport = target.gitMirror.githubTransport; + if (transport.mode !== "ssh") return null; + const secret = target.tekton.gitWorkspaceSecret; + const material = gitMirrorGithubSshMaterial(transport); + if (material.knownHosts === null || material.knownHostsFingerprint === null) { + throw new Error(`targets.${target.id}.tekton.gitWorkspaceSecret requires known_hosts material from gitMirror.githubTransport`); + } + return { + apiVersion: "v1", + kind: "Secret", + metadata: { + name: secret.name, + namespace: secret.namespace, + labels: { ...labels, "app.kubernetes.io/name": "hwlab-tekton-git-workspace" }, + annotations: { + "unidesk.ai/source-ref-from": secret.sourceRefFrom, + "unidesk.ai/private-key-source-ref": transport.privateKeySourceRef, + "unidesk.ai/private-key-source-key": transport.privateKeySourceKey, + "unidesk.ai/private-key-target-key": secret.privateKeySecretKey, + "unidesk.ai/private-key-fingerprint": material.privateKeyFingerprint, + "unidesk.ai/known-hosts-source-ref": transport.knownHostsSourceRef ?? "", + "unidesk.ai/known-hosts-source-key": transport.knownHostsSourceKey ?? "", + "unidesk.ai/known-hosts-target-key": secret.knownHostsSecretKey, + "unidesk.ai/known-hosts-fingerprint": material.knownHostsFingerprint, + }, + }, + type: "kubernetes.io/ssh-auth", + stringData: { + [secret.privateKeySecretKey]: material.privateKey, + [secret.knownHostsSecretKey]: material.knownHosts, + }, + }; +} + function gitMirrorGithubSshMaterial(transport: Extract): { privateKey: string; knownHosts: string | null; privateKeyFingerprint: string; knownHostsFingerprint: string | null } { const privateSource = readControlPlaneSecretSource(transport.privateKeySourceRef, `gitMirror.githubTransport private key source ${transport.privateKeySourceRef} is missing; create the YAML-declared sourceRef with ${transport.privateKeySourceKey} before applying the control plane`); const privateValue = requiredEnvValue(privateSource.values, transport.privateKeySourceKey, transport.privateKeySourceRef); @@ -2296,6 +2391,7 @@ function planSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec) pipeline: target.tekton.pipelineName, pipelineRunPrefix: target.tekton.pipelineRunPrefix, serviceAccount: target.tekton.serviceAccountName, + gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), toolsImage: target.tekton.toolsImage, argoApplication: target.argo.applicationName, argoInstall: { @@ -2334,6 +2430,7 @@ function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetS pipeline: target.tekton.pipelineName, pipelineRunPrefix: target.tekton.pipelineRunPrefix, serviceAccount: target.tekton.serviceAccountName, + gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target), toolsImage: target.tekton.toolsImage, argoNamespace: target.argo.namespace, argoApplication: target.argo.applicationName, @@ -2448,6 +2545,23 @@ function gitMirrorGithubTransportSummary(transport: ControlPlaneGitMirrorGithubT }; } +function tektonGitWorkspaceSecretSummary(target: ControlPlaneTargetSpec): Record { + const transport = target.gitMirror.githubTransport; + const secret = target.tekton.gitWorkspaceSecret; + return { + name: secret.name, + namespace: secret.namespace, + sourceRefFrom: secret.sourceRefFrom, + privateKeySecretKey: secret.privateKeySecretKey, + privateKeySourceRef: transport.mode === "ssh" ? transport.privateKeySourceRef : null, + privateKeySourceKey: transport.mode === "ssh" ? transport.privateKeySourceKey : null, + knownHostsSecretKey: secret.knownHostsSecretKey, + knownHostsSourceRef: transport.mode === "ssh" ? transport.knownHostsSourceRef : null, + knownHostsSourceKey: transport.mode === "ssh" ? transport.knownHostsSourceKey : null, + valuesPrinted: false, + }; +} + function systemdExecArg(value: string): string { if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value; return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("$", "\\$").replaceAll("`", "\\`")}"`; @@ -2488,6 +2602,14 @@ github_token_source_key=${shQuote(target.gitMirror.githubTransport.mode === "htt gitmirror_egress_proxy_json=${shQuote(gitMirrorEgressProxyJson)} pipeline=${shQuote(target.tekton.pipelineName)} service_account=${shQuote(target.tekton.serviceAccountName)} +ci_git_secret=${shQuote(target.tekton.gitWorkspaceSecret.name)} +ci_git_private_key=${shQuote(target.tekton.gitWorkspaceSecret.privateKeySecretKey)} +ci_git_known_hosts_key=${shQuote(target.tekton.gitWorkspaceSecret.knownHostsSecretKey)} +ci_git_source_ref_from=${shQuote(target.tekton.gitWorkspaceSecret.sourceRefFrom)} +ci_git_private_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceRef : "")} +ci_git_private_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceKey : "")} +ci_git_known_hosts_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceRef ?? "" : "")} +ci_git_known_hosts_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceKey ?? "" : "")} argo_ns=${shQuote(target.argo.namespace)} argo_project=${shQuote(target.argo.projectName)} argo_app=${shQuote(target.argo.applicationName)} @@ -2558,6 +2680,48 @@ present = isinstance(encoded, str) and len(encoded) > 0 print(json.dumps({"mode": mode, "required": True, "ready": exists and present, "tokenSecretName": token_secret, "tokenSecretKey": token_key, "tokenSourceRef": token_source_ref, "tokenSourceKey": token_source_key, "tokenSecretExists": exists, "tokenKeyPresent": present, "tokenKeyBytes": len(encoded) if present else 0, "tokenFingerprint": fingerprint(encoded), "valuesPrinted": False})) PY ) +ci_git_workspace_json=$(python3 - "$ci_ns" "$ci_git_secret" "$ci_git_private_key" "$ci_git_known_hosts_key" "$ci_git_source_ref_from" "$ci_git_private_source_ref" "$ci_git_private_source_key" "$ci_git_known_hosts_source_ref" "$ci_git_known_hosts_source_key" <<'PY' +import hashlib, json, subprocess, sys +namespace, secret, private_key, known_hosts_key, source_ref_from, private_source_ref, private_source_key, known_hosts_source_ref, known_hosts_source_key = sys.argv[1:10] +proc = subprocess.run(["kubectl", "-n", namespace, "get", "secret", secret, "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +exists = proc.returncode == 0 +obj = {} +if exists: + try: + obj = json.loads(proc.stdout) + except Exception: + obj = {} +data = obj.get("data") or {} +annotations = obj.get("metadata", {}).get("annotations") or {} +def fingerprint(value): + return "sha256:" + hashlib.sha256(value.encode()).hexdigest()[:16] if value else None +private_encoded = data.get(private_key) if isinstance(data, dict) else None +known_hosts_encoded = data.get(known_hosts_key) if isinstance(data, dict) else None +private_present = isinstance(private_encoded, str) and len(private_encoded) > 0 +known_hosts_present = isinstance(known_hosts_encoded, str) and len(known_hosts_encoded) > 0 +print(json.dumps({ + "required": True, + "ready": exists and private_present and known_hosts_present, + "namespace": namespace, + "secretName": secret, + "sourceRefFrom": source_ref_from, + "privateKeySecretKey": private_key, + "privateKeySourceRef": private_source_ref, + "privateKeySourceKey": private_source_key, + "privateKeySecretExists": exists, + "privateKeyPresent": private_present, + "privateKeyBytes": len(private_encoded) if private_present else 0, + "privateKeyFingerprint": annotations.get("unidesk.ai/private-key-fingerprint") or fingerprint(private_encoded), + "knownHostsSecretKey": known_hosts_key, + "knownHostsSourceRef": known_hosts_source_ref, + "knownHostsSourceKey": known_hosts_source_key, + "knownHostsPresent": known_hosts_present, + "knownHostsBytes": len(known_hosts_encoded) if known_hosts_present else 0, + "knownHostsFingerprint": annotations.get("unidesk.ai/known-hosts-fingerprint") or fingerprint(known_hosts_encoded), + "valuesPrinted": False, +})) +PY +) registry_ready=false if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_ready=true; fi tools_repo_tag=\${tools_image#\${registry}/} @@ -2650,7 +2814,7 @@ print(json.dumps({"crds": crds, "deployments": deploy, "statefulSets": sts, "crd PY argo_fragment=$(cat /tmp/hwlab-node-status-fragments.json 2>/dev/null || printf '{}') cat </dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline")},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"egressProxy":$gitmirror_egress_proxy_json,"githubTransport":$github_transport_json,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} +{"observedAt":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","node":"$node","lane":"$lane","components":{"k3sNodeConfig":$k3s_fragment,"tekton":{"installed":$(kubectl get crd pipelines.tekton.dev pipelineruns.tekton.dev >/dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline"),"gitWorkspaceSecret":$ci_git_workspace_json},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"egressProxy":$gitmirror_egress_proxy_json,"githubTransport":$github_transport_json,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} JSON `; } @@ -2972,6 +3136,7 @@ function statusNext( k3sNodeConfig: Record, ): Record { const bootstrapMissing = !boolField(ciNamespace, "exists") + || !boolField(record(ciNamespace.gitWorkspaceSecret), "ready") || !boolField(gitMirror, "namespaceExists") || !boolField(gitMirror, "readServiceExists") || !boolField(gitMirror, "writeServiceExists") @@ -2980,6 +3145,7 @@ function statusNext( if (node.k3s !== null && !boolField(k3sNodeConfig, "ready")) blockers.push("k3s-node-config-not-applied"); if (!boolField(registry, "ready")) blockers.push("node-local-registry-not-ready"); if (!boolField(registry, "toolsImageReady")) blockers.push("tools-image-missing"); + if (!boolField(record(ciNamespace.gitWorkspaceSecret), "ready")) blockers.push("ci-git-workspace-secret-not-ready"); if (bootstrapMissing) blockers.push("control-plane-bootstrap-missing"); const gitMirrorGithubTransport = record(gitMirror.githubTransport); if (gitMirrorGithubTransport.required === true && !boolField(gitMirrorGithubTransport, "ready")) blockers.push("git-mirror-github-token-secret-not-ready"); diff --git a/scripts/src/hwlab-node/web-probe.ts b/scripts/src/hwlab-node/web-probe.ts index 84e60b17..2d97661b 100644 --- a/scripts/src/hwlab-node/web-probe.ts +++ b/scripts/src/hwlab-node/web-probe.ts @@ -12,7 +12,7 @@ import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; -import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; +import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneCiGitWorkspaceSecret, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; @@ -205,12 +205,13 @@ export function cleanupLocalNodeRuntimeRenderDir(spec: HwlabRuntimeLaneSpec, ren } export function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string): Record { + const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane); return { apiVersion: "tekton.dev/v1", kind: "PipelineRun", metadata: { name: pipelineRun, - namespace: "hwlab-ci", + namespace: gitWorkspaceSecret.namespace, labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/gitops-target": spec.lane, @@ -254,44 +255,51 @@ export function nodeRuntimePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourc ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, - { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }, + { name: "git-ssh", secret: { secretName: gitWorkspaceSecret.name } }, ], }, }; } export function createNodeRuntimePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun: string, timeoutSeconds: number, rerun: boolean): CommandResult { + const gitWorkspaceSecret = hwlabNodeControlPlaneCiGitWorkspaceSecret(spec.nodeId, spec.lane); const manifestB64 = Buffer.from(JSON.stringify(nodeRuntimePipelineRunManifest(spec, sourceCommit, pipelineRun)), "utf8").toString("base64"); const script = [ "set -eu", `manifest_b64=${shellQuote(manifestB64)}`, `pipeline_run=${shellQuote(pipelineRun)}`, + `ci_namespace=${shellQuote(gitWorkspaceSecret.namespace)}`, + `git_workspace_secret=${shellQuote(gitWorkspaceSecret.name)}`, `rerun=${rerun ? "true" : "false"}`, "manifest_path=\"/tmp/$pipeline_run.json\"", + "if ! kubectl -n \"$ci_namespace\" get secret \"$git_workspace_secret\" >/dev/null 2>&1; then", + " printf 'missing Tekton git workspace Secret %s/%s; run: bun scripts/cli.ts hwlab nodes control-plane infra apply --node %s --lane %s --confirm\\n' \"$ci_namespace\" \"$git_workspace_secret\" " + shellQuote(spec.nodeId) + " " + shellQuote(spec.lane) + " >&2", + " exit 44", + "fi", "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", - "existing_status=$(kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", + "existing_status=$(kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o 'jsonpath={.status.conditions[0].status}' 2>/dev/null || true)", "if [ \"$rerun\" = true ] && [ -n \"$existing_status\" ]; then", - " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", - " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", - " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", - " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "elif [ \"$existing_status\" = False ]; then", - " kubectl delete pipelinerun -n hwlab-ci \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", - " kubectl delete taskrun -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", - " kubectl delete pod -n hwlab-ci -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", - " kubectl -n hwlab-ci get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n hwlab-ci delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete taskrun -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl delete pod -n \"$ci_namespace\" -l tekton.dev/pipelineRun=\"$pipeline_run\" --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", + " kubectl -n \"$ci_namespace\" get pvc -o name | grep '^persistentvolumeclaim/pvc-' | xargs -r kubectl -n \"$ci_namespace\" delete --ignore-not-found=true --wait=false >/dev/null 2>&1 || true", "fi", "if kubectl create -f \"$manifest_path\"; then", " :", "else", " code=$?", - " if kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" >/dev/null 2>&1; then", + " if kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" >/dev/null 2>&1; then", " printf 'PipelineRun %s already exists; reusing existing object\\n' \"$pipeline_run\" >&2", " else", " exit \"$code\"", " fi", "fi", - "kubectl get pipelinerun -n hwlab-ci \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'", + "kubectl get pipelinerun -n \"$ci_namespace\" \"$pipeline_run\" -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'", ].join("\n"); return runNodeK3sScript(spec, script, timeoutSeconds); }