import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { rootPath } from "./config"; import { runCommand, type CommandResult } from "./command"; import { egressBenchmarkCompactResult, egressBenchmarkDryRun, egressBenchmarkStartScript, egressBenchmarkStatusScript, type EgressBenchmarkSpec } from "./egress-proxy-benchmark"; import { resolveEgressProxySourceRef } from "./egress-proxy-sources"; import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import { registryInfraManifest } from "./hwlab-node-control-plane-registry"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, type ArgoOptions, type CiBuildBenchmarkCachePolicy, type CiBuildBenchmarkProfileSpec, type CiBuildBenchmarkOptions, type ControlPlaneConfig, type ControlPlaneEgressProxySpec, type ControlPlaneGitMirrorEgressProxySpec, type ControlPlaneGitMirrorGithubTransportSpec, type ControlPlaneHostRouteEgressProxySpec, type ControlPlaneImagePolicy, type ControlPlaneK3sInstallSpec, type ControlPlaneK3sNodeSpec, type ControlPlaneK8sServiceEgressProxySpec, type ControlPlaneNodeSpec, type ControlPlaneRegistrySpec, type ControlPlaneRuntimeProxySpec, type ControlPlaneTargetSpec, type ControlPlaneTektonArgoObserverRbacSpec, type ControlPlaneTektonGitWorkspaceSecretSpec, type ControlPlaneTektonInstallManifestSpec, type ControlPlaneTektonInstallSpec, type ControlPlaneTektonRuntimeObserverRbacSpec, type DockerfileInlineSpec, type EgressBenchmarkOptions, type ImageRewriteSpec, type InfraOptions, type K3sInstallOptions, type TektonInstallOptions, type ToolsImageOptions, } from "./hwlab-node-control-plane-model"; import { applyNext, applyScript, argoApplyStartScript, argoDesiredManifest, argoApplicationSkeleton, argoProjectSkeleton, ciBuildBenchmarkFailureRows, ciBuildBenchmarkLiveOk, ciBuildBenchmarkServiceRows, ciBuildBenchmarkStartScript, ciBuildBenchmarkStatusScript, ciBuildBenchmarkTaskRows, controlPlaneEgressProxySummary, expectedSummary, gitMirrorGithubTransportSummary, gitMirrorRuntimeProxySpec, k3sInstallPlan, k3sInstallStatusScript, k3sInstallSubmitScript, k3sNodeConfigPlan, manifestObjectSummary, parseRemoteJson, planSummary, remoteJobLogs, remoteJobStateDir, remoteJobStatusScript, runTransHost, runTransK3s, runtimeHostProxyConfig, runtimeHostProxyEnv, runtimeProxyReady, shortDisplay, statusNext, statusScript, tektonInstallApplyStartScript, toolsImageBuildStartScript, toolsImageDockerfile, toolsImageStatus, } from "./hwlab-node-control-plane-runtime"; import type { RenderedCliResult } from "./output"; import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets"; export { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH } from "./hwlab-node-control-plane-model"; export function runHwlabNodeControlPlaneInfra(args: string[]): Record | RenderedCliResult { if (args[0] === "k3s") { const options = parseK3sInstallOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); return runK3sInstallCommand(config, node, target, options); } if (args[0] === "tools-image") { const options = parseToolsImageOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); return runToolsImageCommand(config, node, target, options); } if (args[0] === "tekton") { const options = parseTektonInstallOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); return runTektonInstallCommand(config, node, target, options); } if (args[0] === "argo") { const options = parseArgoOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); return runArgoCommand(config, node, target, options); } if (args[0] === "egress-benchmark") { const options = parseEgressBenchmarkOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); return runEgressBenchmarkCommand(config, node, target, options); } if (args[0] === "ci-build-benchmark") { const options = parseCiBuildBenchmarkOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); return runCiBuildBenchmarkCommand(config, node, target, options); } const options = parseInfraOptions(args); const { config, node, target } = controlPlaneContext(options.node, options.lane); if (options.action === "plan") return infraPlan(config, node, target, options); if (options.action === "status") return infraStatus(config, node, target, options); return infraApply(config, node, target, options); } function controlPlaneContext(nodeId: string, lane: string): { config: ControlPlaneConfig; node: ControlPlaneNodeSpec; target: ControlPlaneTargetSpec } { const config = readControlPlaneConfig(); const node = config.nodes[nodeId]; if (node === undefined) throw new Error(`unknown node ${nodeId}; known nodes: ${Object.keys(config.nodes).join(", ")}`); const target = config.targets.find((item) => item.node === nodeId && item.lane === lane); if (target === undefined) throw new Error(`no control-plane target for node=${nodeId} lane=${lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); if (!target.enabled) throw new Error(`control-plane target ${target.id} is disabled in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); return { config, node, target }; } export function hwlabNodeControlPlaneCiGitWorkspaceSecret(nodeId: string, lane: string): { name: string; namespace: string } { const { target } = controlPlaneContext(nodeId, lane); return { name: target.tekton.gitWorkspaceSecret.name, namespace: target.tekton.gitWorkspaceSecret.namespace, }; } export function hwlabNodeControlPlaneSourceWorkspaceBootstrap(nodeId: string, lane: string): HwlabNodeControlPlaneSourceWorkspaceBootstrapSpec { const { target } = controlPlaneContext(nodeId, lane); return { configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, targetId: target.id, node: target.node, lane: target.lane, ciNamespace: target.ciNamespace, serviceAccountName: target.tekton.serviceAccountName, toolsImage: target.tekton.toolsImage.output, imagePullPolicy: target.tekton.toolsImage.imagePullPolicy, gitReadUrl: target.gitMirror.readUrl, gitWriteUrl: target.gitMirror.writeUrl, gitMirrorNamespace: target.gitMirror.namespace, }; } export function hwlabNodeControlPlaneInfraHelp(): Record { return { ok: true, command: "hwlab nodes control-plane infra", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, description: "Plan/status/apply YAML-controlled HWLAB node-local k3s, CI/CD and git-mirror control-plane prerequisites. Cross-node PK01/Caddy/FRP/runtime rollout remains explicit semi-automatic CLI work.", usage: [ "bun scripts/cli.ts hwlab nodes control-plane infra plan --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra k3s plan --node JD01 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra k3s install --node JD01 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra k3s status --node JD01 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra tekton status --node JD01 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra tekton apply --node JD01 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra tekton apply --node JD01 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra tekton logs --node JD01 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark --node D601 --lane v03 --profile no-mirror --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark status --node D601 --lane v03 --profile no-mirror", "bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark --node D601 --lane v03 --profile no-mirror-full --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node D601 --lane v03 --profile no-mirror-full", "bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node D601 --lane v03 --profile no-mirror-full", ], g14Consistency: "D601 target fields mirror the existing G14 runtime lane control-plane vocabulary: source branch, gitops branch/path, Pipeline, PipelineRun prefix, ServiceAccount, Argo Application, and git-mirror read/write/sync/flush status concepts.", }; } function infraPlan(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: InfraOptions): Record { return { ok: true, command: "hwlab nodes control-plane infra plan", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "plan", mutation: false, target: planSummary(node, target), expected: expectedSummary(node, target), hostConfig: k3sNodeConfigPlan(node), imagePolicy: _config.imagePolicy, g14Consistency: { laneVocabulary: ["sourceBranch", "gitopsBranch", "catalogPath", "runtime.path", "runtime.namespace", "tekton.pipeline", "pipelineRunPrefix", "argo.application"], gitMirrorStatusVocabulary: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"], note: "D601 values differ only through YAML target fields; the control-plane model is intentionally aligned with G14 runtime lane semantics.", }, resources: manifestObjectSummary(renderInfraManifest(node, target)), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, dryRun: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --dry-run`, apply: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`, toolsImageBuild: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node ${node.id} --lane ${target.lane} --confirm`, argoApply: `bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node ${node.id} --lane ${target.lane} --confirm`, }, options: { timeoutSeconds: options.timeoutSeconds }, }; } function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: InfraOptions): Record { const script = statusScript(node, target); const result = runTransK3s(node.kubeRoute, script, options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const status = typeof parsed === "object" && parsed !== null ? parsed as Record : { parseError: "remote status did not return a JSON object", stdoutPreview: result.stdout.slice(0, 1000) }; const components = record(status.components); const argo = record(components.argo); const argoInstall = record(argo.install); const argoObserverRbac = record(argo.argoObserverRbac); const gitMirror = record(components.gitMirror); const gitMirrorGithubTransport = record(gitMirror.githubTransport); const tekton = record(components.tekton); const tektonInstall = record(tekton.install); const ciNamespace = record(components.ciNamespace); const ciGitWorkspaceSecret = record(ciNamespace.gitWorkspaceSecret); const runtimeNamespace = record(components.runtimeNamespace); const runtimeObserverRbac = record(runtimeNamespace.runtimeObserverRbac); const registry = record(components.registry); const k3sNodeConfig = record(components.k3sNodeConfig); const k3sNodeConfigReady = node.k3s === null || (boolField(k3sNodeConfig, "dropInMatches") && numberValue(k3sNodeConfig.liveCapacityPods) === node.k3s.kubelet.maxPods && numberValue(k3sNodeConfig.liveAllocatablePods) === node.k3s.kubelet.maxPods); const tektonRuntimeProxyReady = runtimeProxyReady(tektonInstall); const argoRuntimeProxyReady = runtimeProxyReady(argoInstall); const ok = result.exitCode === 0 && k3sNodeConfigReady && boolField(tekton, "installed") && boolField(tektonInstall, "crdsReady") && boolField(tektonInstall, "deploymentsReady") && tektonRuntimeProxyReady && boolField(ciNamespace, "exists") && boolField(ciGitWorkspaceSecret, "ready") && boolField(runtimeNamespace, "exists") && boolField(runtimeObserverRbac, "ready") && boolField(gitMirror, "namespaceExists") && boolField(gitMirror, "readServiceExists") && boolField(gitMirror, "writeServiceExists") && (gitMirrorGithubTransport.required !== true || boolField(gitMirrorGithubTransport, "ready")) && (boolField(gitMirror, "cachePvcExists") || boolField(gitMirror, "cacheHostPathReady")) && boolField(registry, "ready") && boolField(registry, "toolsImageReady") && boolField(argo, "installed") && boolField(argo, "projectExists") && boolField(argo, "applicationExists") && boolField(argoObserverRbac, "ready") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") && boolField(argoInstall, "statefulSetsReady") && argoRuntimeProxyReady; return { ok, command: "hwlab nodes control-plane infra status", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "status", mutation: false, expected: expectedSummary(node, target), status, readiness: { ok, k3sNodeConfigReady, tektonInstalled: boolField(tekton, "installed"), tektonCrdsReady: boolField(tektonInstall, "crdsReady"), tektonDeploymentsReady: boolField(tektonInstall, "deploymentsReady"), tektonRuntimeProxyReady, ciNamespaceExists: boolField(ciNamespace, "exists"), ciGitWorkspaceSecretReady: boolField(ciGitWorkspaceSecret, "ready"), runtimeNamespaceExists: boolField(runtimeNamespace, "exists"), runtimeObserverRbacReady: boolField(runtimeObserverRbac, "ready"), gitMirrorNamespaceExists: boolField(gitMirror, "namespaceExists"), gitMirrorReadServiceExists: boolField(gitMirror, "readServiceExists"), gitMirrorWriteServiceExists: boolField(gitMirror, "writeServiceExists"), gitMirrorGithubTransportReady: gitMirrorGithubTransport.required !== true || boolField(gitMirrorGithubTransport, "ready"), gitMirrorCachePvcExists: boolField(gitMirror, "cachePvcExists"), gitMirrorCacheHostPathReady: boolField(gitMirror, "cacheHostPathReady"), gitMirrorReadReady: boolField(gitMirror, "readDeploymentReady"), gitMirrorWriteReady: boolField(gitMirror, "writeDeploymentReady"), argoInstalled: boolField(argo, "installed"), argoProjectExists: boolField(argo, "projectExists"), argoApplicationExists: boolField(argo, "applicationExists"), argoObserverRbacReady: boolField(argoObserverRbac, "ready"), argoCrdsReady: boolField(argoInstall, "crdsReady"), argoDeploymentsReady: boolField(argoInstall, "deploymentsReady"), argoStatefulSetsReady: boolField(argoInstall, "statefulSetsReady"), argoRuntimeProxyReady, registryReady: boolField(registry, "ready"), toolsImageReady: boolField(registry, "toolsImageReady"), }, result: compactCommandResult(result), next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, tekton, argo, ciNamespace, runtimeNamespace, k3sNodeConfig), }; } function infraApply(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: InfraOptions): Record { if (options.confirm && options.dryRun) throw new Error("infra apply accepts only one of --dry-run or --confirm"); const dryRun = options.dryRun || !options.confirm; const manifest = renderInfraManifest(node, target); const yaml = `${manifest.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; const imageStatus = toolsImageStatus(node, target, options.timeoutSeconds); if (dryRun) { return { ok: true, command: "hwlab nodes control-plane infra apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "dry-run", mutation: false, expected: expectedSummary(node, target), hostConfig: k3sNodeConfigPlan(node), preflight: { registryReady: imageStatus.registryReady, toolsImageReady: imageStatus.toolsImageReady, toolsImage: target.tekton.toolsImage.output, result: imageStatus.result, }, resources: manifestObjectSummary(manifest), manifest: { objects: manifest.length, bytes: Buffer.byteLength(yaml), sha256: sha256Short(yaml) }, note: "dry-run renders D601 node-local control-plane bootstrap resources only; it does not trigger HWLAB runtime rollout and does not touch PK01/Caddy/FRP.", next: applyNext(node, target, imageStatus), }; } const script = applyScript(yaml, node, target); const result = runTransK3s(node.kubeRoute, script, options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { ok: result.exitCode === 0, command: "hwlab nodes control-plane infra apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "confirmed-apply", mutation: result.exitCode === 0, expected: expectedSummary(node, target), preflight: { registryReady: imageStatus.registryReady, toolsImageReady: imageStatus.toolsImageReady, toolsImage: target.tekton.toolsImage.output, warning: imageStatus.toolsImageReady ? null : "tools-image-missing; bootstrap objects were applied but readiness still requires a controlled image build/publish stage", result: imageStatus.result, }, resources: manifestObjectSummary(manifest), apply: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}` }, }; } function runK3sInstallCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: K3sInstallOptions): Record { const spec = node.k3s?.install ?? null; if (node.k3s === null || spec === null || !spec.enabled) { throw new Error(`nodes.${node.id}.k3s.install must be enabled in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); } const plan = k3sInstallPlan(node, target, spec); if (options.action === "plan" || (options.action === "install" && options.dryRun)) { return { ok: true, command: `hwlab nodes control-plane infra k3s ${options.action}`, configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: options.action === "install" ? "dry-run" : "plan", mutation: false, plan, next: { install: `bun scripts/cli.ts hwlab nodes control-plane infra k3s install --node ${node.id} --lane ${target.lane} --confirm`, status: `bun scripts/cli.ts hwlab nodes control-plane infra k3s status --node ${node.id} --lane ${target.lane}`, }, }; } if (options.action === "status") { const result = runTransHost(node.route, k3sInstallStatusScript(node, target, spec, options.tailLines), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const status = typeof parsed === "object" && parsed !== null ? parsed as Record : { stdoutPreview: result.stdout.slice(0, 2000) }; const checks = record(status.checks); const ok = result.exitCode === 0 && boolField(checks, "binarySha256Ok") && boolField(checks, "serviceActive") && boolField(checks, "nodeReady"); return { ok, command: "hwlab nodes control-plane infra k3s status", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "status", mutation: false, plan, status, result: compactCommandResult(result), next: ok ? { k3sRouteSmoke: `trans ${node.kubeRoute} kubectl get nodes -o wide` } : { install: `bun scripts/cli.ts hwlab nodes control-plane infra k3s install --node ${node.id} --lane ${target.lane} --confirm` }, }; } const result = runTransHost(node.route, k3sInstallSubmitScript(node, target, spec), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { ok: result.exitCode === 0, command: "hwlab nodes control-plane infra k3s install", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "confirmed-submit", mutation: result.exitCode === 0, plan, start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra k3s status --node ${node.id} --lane ${target.lane}` }, }; } function runToolsImageCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ToolsImageOptions): Record { if (options.action === "status") return toolsImageCommandStatus(node, target, options); if (options.action === "logs") return remoteJobLogs(node, target, "tools-image", options); return toolsImageBuild(node, target, options); } function runTektonInstallCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: TektonInstallOptions): Record { if (options.action === "status") return tektonInstallCommandStatus(node, target, options); if (options.action === "logs") return remoteJobLogs(node, target, "tekton", options); return tektonInstallApply(node, target, options); } function runArgoCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ArgoOptions): Record { if (options.action === "status") return argoCommandStatus(node, target, options); if (options.action === "logs") return remoteJobLogs(node, target, "argo", options); return argoApply(node, target, options); } function runEgressBenchmarkCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: EgressBenchmarkOptions): Record | RenderedCliResult { const spec = controlPlaneEgressBenchmarkSpec(node, target, options); if (options.action === "status" || options.action === "logs") { const result = runTransK3s(node.kubeRoute, egressBenchmarkStatusScript(spec, options.tailLines), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const status = record(record(parsed).status); const state = renderCell(status.state, "unknown"); return renderControlPlaneBenchmarkResult({ ok: result.exitCode === 0 && (options.action === "logs" || state !== "failed"), command: `hwlab nodes control-plane infra egress-benchmark ${options.action}`, configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mutation: false, job: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: egressBenchmarkCompactResult(result), }); } if (options.dryRun) { return renderControlPlaneBenchmarkResult({ ok: true, command: "hwlab nodes control-plane infra egress-benchmark", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "dry-run", mutation: false, plan: egressBenchmarkDryRun(spec), next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark --node ${node.id} --lane ${target.lane} --profile ${options.profile} --confirm` }, }); } const result = runTransK3s(node.kubeRoute, egressBenchmarkStartScript(spec), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return renderControlPlaneBenchmarkResult({ ok: result.exitCode === 0, command: "hwlab nodes control-plane infra egress-benchmark", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "async-job", mutation: result.exitCode === 0, start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: egressBenchmarkCompactResult(result), }); } function controlPlaneEgressBenchmarkSpec(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: EgressBenchmarkOptions): EgressBenchmarkSpec { const proxy = node.egressProxy; if (proxy === null) throw new Error(`nodes.${node.id}.egressProxy is required for egress-benchmark`); return { scope: "hwlab-control-plane", targetId: `${node.id}/${target.lane}`, route: node.kubeRoute, namespace: proxy.namespace, serviceName: proxy.serviceName, port: proxy.port, noProxy: proxy.noProxy, sourceType: proxy.sourceType, sourceRef: proxy.sourceRef, sourceConfigRef: proxy.sourceConfigRef, sourceFingerprint: proxy.sourceFingerprint, profile: options.profile, samples: options.samples, sampleTimeoutSeconds: options.sampleTimeoutSeconds, }; } function renderControlPlaneBenchmarkResult(result: Record): RenderedCliResult { const start = record(result.start); const job = record(result.job); const status = record(job.status); const plan = record(result.plan); const next = record(result.next); const rows = Array.isArray(status.rows) ? status.rows.map(record) : []; const statusText = renderCell(status.state, result.ok === false ? "failed" : "ok"); const logTail = typeof job.logTail === "string" ? job.logTail.trimEnd() : ""; const node = renderCell(result.node); const lane = renderCell(result.lane); const target = `${node}/${lane}`; const profile = renderCell(result.profile ?? plan.profile ?? status.profile, "no-mirror"); const statusCommand = renderCell(start.statusCommand, `bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark status --node ${node} --lane ${lane} --profile ${profile}`); const logsCommand = renderCell(start.logsCommand, `bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark logs --node ${node} --lane ${lane} --profile ${profile}`); const lines = [ "HWLAB CONTROL-PLANE EGRESS BENCHMARK", "", ...renderTable(["TARGET", "PROFILE", "MODE", "STATUS"], [[target, profile, renderCell(result.mode ?? optionsModeFromCommand(result.command)), statusText]]), "", rows.length === 0 ? "RESULTS\n-" : [ "RESULTS", ...renderTable(["TEST", "SUCCESS", "SAMPLES", "P50", "P95", "FAILURES"], rows.map((row) => [ renderCell(row.test), renderCell(row.success), renderCell(row.samples), renderCell(row.p50Ms), renderCell(row.p95Ms), JSON.stringify(row.failureFamilies ?? {}), ])), ].join("\n"), ...(logTail.length === 0 ? [] : ["", "LOG TAIL", logTail]), "", "NEXT", ` ${renderCell(next.confirm, "")}`, ` ${statusCommand}`, ` ${logsCommand}`, "", "Disclosure: default output is bounded; Secret/proxy source values are not printed.", ].filter((line) => line !== " "); return { ok: result.ok !== false, command: renderCell(result.command, "hwlab nodes control-plane infra egress-benchmark"), renderedText: lines.join("\n"), contentType: "text/plain" }; } function runCiBuildBenchmarkCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: CiBuildBenchmarkOptions): RenderedCliResult { const profile = ciBuildBenchmarkProfileForTarget(target, options.profile); const runtime = ciBuildBenchmarkRuntimeSpec(node, target, profile); if (options.action === "status" || options.action === "logs") { const result = runTransK3s(node.kubeRoute, ciBuildBenchmarkStatusScript(target, profile, options.tailLines, options.action === "logs"), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const job = typeof parsed === "object" && parsed !== null ? parsed as Record : { stdoutPreview: result.stdout.slice(0, 2000) }; return renderCiBuildBenchmarkResult({ ok: result.exitCode === 0 && ciBuildBenchmarkLiveOk(job, runtime.serviceIds, profile), command: `hwlab nodes control-plane infra ci-build-benchmark ${options.action}`, configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, profile: profile.profile, mode: options.action, mutation: false, benchmark: ciBuildBenchmarkDefinitionSummary(runtime, target, profile), job, result: compactCommandResult(result), }); } const head = resolveCiBuildBenchmarkSourceHead(runtime); if (head.sourceCommit === null) { throw new Error(`failed to resolve ${runtime.gitUrl} refs/heads/${runtime.sourceBranch}: ${head.result.stderr || head.result.stdout || `exit ${head.result.exitCode}`}`); } const pipelineRun = ciBuildBenchmarkPipelineRunName(profile.pipelineRunPrefix, head.sourceCommit); const catalogPath = ciBuildBenchmarkCatalogPath(profile, pipelineRun); const manifest = ciBuildBenchmarkPipelineRunManifest(runtime, target, profile, head.sourceCommit, pipelineRun, catalogPath); const plan = { ...ciBuildBenchmarkDefinitionSummary(runtime, target, profile), pipelineRun, sourceCommit: head.sourceCommit, catalogPath, manifest: manifestObjectSummary([manifest]), manifestSha256: sha256Short(JSON.stringify(manifest)), sourceHead: compactCommandResult(head.result), }; if (options.dryRun) { return renderCiBuildBenchmarkResult({ ok: true, command: "hwlab nodes control-plane infra ci-build-benchmark", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, profile: profile.profile, mode: "dry-run", mutation: false, plan, next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark --node ${node.id} --lane ${target.lane} --profile ${profile.profile} --confirm` }, }); } const result = runTransK3s(node.kubeRoute, ciBuildBenchmarkStartScript(target, profile, manifest, runtime.pipeline, pipelineRun, head.sourceCommit, catalogPath), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return renderCiBuildBenchmarkResult({ ok: result.exitCode === 0, command: "hwlab nodes control-plane infra ci-build-benchmark", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, profile: profile.profile, mode: "confirmed-start", mutation: result.exitCode === 0, benchmark: ciBuildBenchmarkDefinitionSummary(runtime, target, profile), start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), }); } function ciBuildBenchmarkProfileForTarget(target: ControlPlaneTargetSpec, profileName: string): CiBuildBenchmarkProfileSpec { const profile = target.ciBuildBenchmarks.find((item) => item.profile === profileName); if (profile === undefined) { const known = target.ciBuildBenchmarks.map((item) => item.profile).join(", ") || ""; throw new Error(`ci-build-benchmark profile ${profileName} is not declared for target ${target.id}; known profiles: ${known}`); } return profile; } function ciBuildBenchmarkRuntimeSpec(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec): HwlabRuntimeLaneSpec { if (!isHwlabRuntimeLane(target.lane)) throw new Error(`target ${target.id}.lane=${target.lane} is not a runtime lane in config/hwlab-node-lanes.yaml`); const runtime = hwlabRuntimeLaneSpecForNode(target.lane, node.id); if (runtime.nodeId !== node.id || runtime.lane !== target.lane) throw new Error(`runtime lane mismatch for ${node.id}/${target.lane}`); if (!profile.runtimeLaneConfigRef.startsWith("config/hwlab-node-lanes.yaml#")) { throw new Error(`targets.${target.id}.ciBuildBenchmarks.${profile.profile}.runtimeLaneConfigRef must point at config/hwlab-node-lanes.yaml`); } return runtime; } function resolveCiBuildBenchmarkSourceHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } { const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], rootPath(), { timeoutMs: 45_000 }); if (result.exitCode !== 0 || result.timedOut) return { sourceCommit: null, result }; const match = /[0-9a-f]{40}/iu.exec(`${result.stdout}\n${result.stderr}`); return { sourceCommit: match?.[0].toLowerCase() ?? null, result }; } function ciBuildBenchmarkPipelineRunName(prefix: string, sourceCommit: string): string { const suffix = `${sourceCommit.slice(0, 12)}-${Date.now().toString(36)}`; return `${prefix}-${suffix}`.slice(0, 63).replace(/-+$/u, ""); } function ciBuildBenchmarkCatalogPath(profile: CiBuildBenchmarkProfileSpec, pipelineRun: string): string { return profile.catalogPathTemplate.replace(/\{profile\}/gu, profile.profile).replace(/\{pipelineRun\}/gu, pipelineRun); } function ciBuildBenchmarkBuildCacheMode(profile: CiBuildBenchmarkProfileSpec): "disabled" | "registry" { return profile.cachePolicy.forbidBuildkitCache ? "disabled" : "registry"; } function ciBuildBenchmarkDefinitionSummary(runtime: HwlabRuntimeLaneSpec, target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec): Record { return { targetId: target.id, profile: profile.profile, runtimeLaneConfigRef: profile.runtimeLaneConfigRef, namespace: target.ciNamespace, pipeline: runtime.pipeline, serviceAccountName: runtime.serviceAccountName, sourceBranch: runtime.sourceBranch, gitReadUrl: runtime.gitReadUrl, gitWriteUrl: runtime.gitWriteUrl, gitopsBranch: runtime.gitopsBranch, runtimePath: runtime.runtimePath, registryPrefix: runtime.registryPrefix, baseImage: runtime.baseImage, services: runtime.serviceIds, imageTagMode: profile.imageTagMode, buildCacheMode: ciBuildBenchmarkBuildCacheMode(profile), cachePolicy: profile.cachePolicy, requiredTimings: profile.requiredTimings, failureFamilies: profile.failureFamilies, }; } function ciBuildBenchmarkPipelineRunManifest( runtime: HwlabRuntimeLaneSpec, target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec, sourceCommit: string, pipelineRun: string, catalogPath: string, ): Record { return { apiVersion: "tekton.dev/v1", kind: "PipelineRun", metadata: { name: pipelineRun, namespace: target.ciNamespace, labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/gitops-target": runtime.lane, "hwlab.pikastech.local/source-commit": sourceCommit, "hwlab.pikastech.local/trigger": "unidesk-ci-build-benchmark", "unidesk.ai/benchmark": "ci-build", "unidesk.ai/benchmark-profile": profile.profile, }, annotations: { "hwlab.pikastech.local/node": runtime.nodeId, "hwlab.pikastech.local/source-branch": runtime.sourceBranch, "hwlab.pikastech.local/gitops-branch": runtime.gitopsBranch, "hwlab.pikastech.local/runtime-path": runtime.runtimePath, "hwlab.pikastech.local/network-profile": runtime.networkProfileId, "hwlab.pikastech.local/download-profile": runtime.downloadProfileId, "unidesk.ai/issue": "pikasTech/unidesk#1010", "unidesk.ai/cache-policy": JSON.stringify(profile.cachePolicy), "unidesk.ai/build-cache-mode": ciBuildBenchmarkBuildCacheMode(profile), "unidesk.ai/catalog-path": catalogPath, "unidesk.ai/runtime-lane-config-ref": profile.runtimeLaneConfigRef, "unidesk.ai/required-timings": profile.requiredTimings.join(","), }, }, spec: { pipelineRef: { name: runtime.pipeline }, timeouts: { pipeline: `${profile.pipelineTimeoutSeconds}s` }, taskRunTemplate: { serviceAccountName: runtime.serviceAccountName, podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 }, }, }, params: [ { name: "git-url", value: runtime.gitUrl }, { name: "git-read-url", value: runtime.gitReadUrl }, { name: "git-write-url", value: runtime.gitWriteUrl }, { name: "source-branch", value: runtime.sourceBranch }, { name: "gitops-branch", value: runtime.gitopsBranch }, { name: "lane", value: runtime.lane }, { name: "catalog-path", value: catalogPath }, { name: "image-tag-mode", value: profile.imageTagMode }, { name: "runtime-path", value: runtime.runtimePath }, { name: "revision", value: sourceCommit }, { name: "registry-prefix", value: runtime.registryPrefix }, { name: "services", value: runtime.serviceIds.join(",") }, { name: "base-image", value: runtime.baseImage }, { name: "build-cache-mode", value: ciBuildBenchmarkBuildCacheMode(profile) }, ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, { name: "git-ssh", secret: { secretName: target.tekton.gitWorkspaceSecret.name } }, ], }, }; } function renderCiBuildBenchmarkResult(result: Record): RenderedCliResult { const start = record(result.start); const job = record(result.job); const plan = record(result.plan); const benchmark = record(result.benchmark ?? plan); const next = record(result.next); const pipelineRun = record(job.pipelineRun); const taskRows = ciBuildBenchmarkTaskRows(job); const serviceRows = ciBuildBenchmarkServiceRows(job, benchmark.services); const failures = ciBuildBenchmarkFailureRows(job, serviceRows, benchmark); const profile = renderCell(result.profile ?? benchmark.profile ?? plan.profile, "unknown"); const node = renderCell(result.node); const lane = renderCell(result.lane); const target = `${node}/${lane}`; const statusText = renderCell(job.state ?? pipelineRun.status ?? start.state, result.ok === false ? "failed" : "ok"); const pipelineRunName = renderCell(job.pipelineRunName ?? pipelineRun.name ?? start.pipelineRun ?? plan.pipelineRun); const sourceCommit = renderCell(pipelineRun.sourceCommit ?? start.sourceCommit ?? plan.sourceCommit); const statusCommand = renderCell(start.statusCommand, `bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node ${node} --lane ${lane} --profile ${profile}`); const logsCommand = renderCell(start.logsCommand, `bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node ${node} --lane ${lane} --profile ${profile}`); const logTail = typeof job.logTail === "string" ? job.logTail.trimEnd() : ""; const lines = [ "HWLAB K3S CI BUILD BENCHMARK", "", ...renderTable(["TARGET", "PROFILE", "MODE", "STATUS", "PIPELINERUN", "SOURCE"], [[target, profile, renderCell(result.mode ?? optionsModeFromCommand(result.command)), statusText, pipelineRunName, shortDisplay(sourceCommit)]]), "", "POLICY", ...renderTable(["FIELD", "VALUE"], [ ["pipeline", renderCell(benchmark.pipeline)], ["catalogPath", renderCell(start.catalogPath ?? plan.catalogPath ?? pipelineRun.catalogPath)], ["services", String((Array.isArray(benchmark.services) ? benchmark.services : []).length)], ["buildCacheMode", renderCell(benchmark.buildCacheMode)], ["cachePolicy", JSON.stringify(benchmark.cachePolicy ?? {})], ["requiredTimings", Array.isArray(benchmark.requiredTimings) ? benchmark.requiredTimings.join(",") : "-"], ]), "", serviceRows.length === 0 ? "SERVICES\n-" : [ "SERVICES", ...renderTable(["SERVICE", "TASK", "STATUS", "DURATION", "FAILURE"], serviceRows.map((row) => [ renderCell(row.service), renderCell(row.task), renderCell(row.status), renderCell(row.duration), renderCell(row.failure), ])), ].join("\n"), "", taskRows.length === 0 ? "TIMINGS\n-" : [ "TIMINGS", ...renderTable(["TASK", "STATUS", "DURATION", "START", "END"], taskRows.map((row) => [ renderCell(row.task), renderCell(row.status), renderCell(row.duration), renderCell(row.start), renderCell(row.end), ])), ].join("\n"), ...(failures.length === 0 ? [] : ["", "FAILURE FAMILIES", ...renderTable(["FAMILY", "COUNT", "SCOPE"], failures.map((row) => [renderCell(row.family), renderCell(row.count), renderCell(row.scope)]))]), ...(logTail.length === 0 ? [] : ["", "LOG TAIL", logTail]), "", "NEXT", ` ${renderCell(next.confirm, "")}`, ` ${statusCommand}`, ` ${logsCommand}`, "", "Disclosure: output is bounded; Git/proxy/Secret values are not expanded. A succeeded PipelineRun with missing build- TaskRuns is reported as cache-hit-forbidden.", ].filter((line) => line !== " "); return { ok: result.ok !== false, command: renderCell(result.command, "hwlab nodes control-plane infra ci-build-benchmark"), renderedText: lines.join("\n"), contentType: "text/plain" }; } function toolsImageCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ToolsImageOptions): Record { const registry = toolsImageStatus(node, target, options.timeoutSeconds); const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "tools-image", options.tailLines), options.timeoutSeconds); const jobStatus = parseRemoteJson(jobResult.stdout); const ok = registry.registryReady && registry.toolsImageReady; return { ok, command: "hwlab nodes control-plane infra tools-image status", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mutation: false, image: target.tekton.toolsImage.output, imageSource: target.tekton.toolsImage, registry, job: typeof jobStatus === "object" && jobStatus !== null ? jobStatus : { parseError: "remote job status did not return JSON", stdoutPreview: jobResult.stdout.slice(0, 1000) }, result: compactCommandResult(jobResult), next: ok ? { infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}` } : { build: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node ${node.id} --lane ${target.lane} --confirm`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node ${node.id} --lane ${target.lane}` }, }; } function toolsImageBuild(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ToolsImageOptions): Record { if (options.confirm && options.dryRun) throw new Error("tools-image build accepts only one of --dry-run or --confirm"); const dryRun = options.dryRun || !options.confirm; const dockerfile = toolsImageDockerfile(target); const buildPlan = { outputImage: target.tekton.toolsImage.output, sourceKind: target.tekton.toolsImage.sourceKind, dockerfileInline: target.tekton.toolsImage.dockerfileInline, buildArgs: target.tekton.toolsImage.buildArgs, buildNetwork: target.tekton.toolsImage.buildNetwork, publicBaseImages: target.tekton.toolsImage.publicBaseImages, nodeLocalRegistryOutputOnly: true, egressProxy: controlPlaneEgressProxySummary(node.egressProxy), stateDir: remoteJobStateDir(target, "tools-image"), }; if (dryRun) { return { ok: true, command: "hwlab nodes control-plane infra tools-image build", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "dry-run", mutation: false, buildPlan, dockerfile: { bytes: Buffer.byteLength(dockerfile), sha256: sha256Short(dockerfile), preview: dockerfile }, next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node ${node.id} --lane ${target.lane} --confirm` }, }; } const result = runTransK3s(node.kubeRoute, toolsImageBuildStartScript(node, target, dockerfile), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { ok: result.exitCode === 0, command: "hwlab nodes control-plane infra tools-image build", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "confirmed-start", mutation: result.exitCode === 0, buildPlan, start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node ${node.id} --lane ${target.lane}`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node ${node.id} --lane ${target.lane}`, infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, }, }; } function tektonInstallCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: TektonInstallOptions): Record { const result = runTransK3s(node.kubeRoute, statusScript(node, target), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; const tekton = record(record(status.components).tekton); const install = record(tekton.install); const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "tekton", options.tailLines), options.timeoutSeconds); const jobStatus = parseRemoteJson(jobResult.stdout); const ok = boolField(tekton, "installed") && boolField(install, "crdsReady") && boolField(install, "deploymentsReady") && runtimeProxyReady(install); return { ok, command: "hwlab nodes control-plane infra tekton status", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mutation: false, expected: { install: target.tekton.install, }, readiness: { installed: boolField(tekton, "installed"), crdsReady: boolField(install, "crdsReady"), deploymentsReady: boolField(install, "deploymentsReady"), runtimeProxyReady: runtimeProxyReady(install), }, tekton, job: typeof jobStatus === "object" && jobStatus !== null ? jobStatus : { parseError: "remote job status did not return JSON", stdoutPreview: jobResult.stdout.slice(0, 1000) }, result: { k3s: compactCommandResult(result), job: compactCommandResult(jobResult) }, next: ok ? { infraApply: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm` } : { apply: `bun scripts/cli.ts hwlab nodes control-plane infra tekton apply --node ${node.id} --lane ${target.lane} --confirm`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra tekton logs --node ${node.id} --lane ${target.lane}` }, }; } function tektonInstallApply(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: TektonInstallOptions): Record { if (options.confirm && options.dryRun) throw new Error("tekton apply accepts only one of --dry-run or --confirm"); if (!target.tekton.install.enabled) throw new Error(`targets.${target.id}.tekton.install.enabled=false`); const dryRun = options.dryRun || !options.confirm; const applyPlan = { version: target.tekton.install.version, sourceKind: target.tekton.install.sourceKind, fieldManager: target.tekton.install.fieldManager, manifests: target.tekton.install.manifests, requiredCrds: target.tekton.install.requiredCrds, expectedDeploymentNamespaces: target.tekton.install.expectedDeploymentNamespaces, readinessTimeoutSeconds: target.tekton.install.readinessTimeoutSeconds, stateDir: remoteJobStateDir(target, "tekton"), egressProxy: controlPlaneEgressProxySummary(node.egressProxy), runtimeProxy: runtimeHostProxyConfig(node, target.tekton.install.runtimeProxy), }; if (dryRun) { return { ok: true, command: "hwlab nodes control-plane infra tekton apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "dry-run", mutation: false, applyPlan, next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra tekton apply --node ${node.id} --lane ${target.lane} --confirm` }, }; } const result = runTransK3s(node.kubeRoute, tektonInstallApplyStartScript(node, target), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { ok: result.exitCode === 0, command: "hwlab nodes control-plane infra tekton apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "confirmed-start", mutation: result.exitCode === 0, applyPlan, start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra tekton status --node ${node.id} --lane ${target.lane}`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra tekton logs --node ${node.id} --lane ${target.lane}`, infraApply: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`, }, }; } function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ArgoOptions): Record { const result = runTransK3s(node.kubeRoute, statusScript(node, target), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; const argo = record(record(status.components).argo); const argoInstall = record(argo.install); const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "argo", options.tailLines), options.timeoutSeconds); const jobStatus = parseRemoteJson(jobResult.stdout); const ok = boolField(argo, "installed") && boolField(argo, "projectExists") && boolField(argo, "applicationExists") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") && boolField(argoInstall, "statefulSetsReady") && runtimeProxyReady(argoInstall); return { ok, command: "hwlab nodes control-plane infra argo status", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mutation: false, expected: { namespace: target.argo.namespace, project: target.argo.projectName, application: target.argo.applicationName, install: target.argo.install, }, readiness: { installed: boolField(argo, "installed"), projectExists: boolField(argo, "projectExists"), applicationExists: boolField(argo, "applicationExists"), crdsReady: boolField(argoInstall, "crdsReady"), deploymentsReady: boolField(argoInstall, "deploymentsReady"), statefulSetsReady: boolField(argoInstall, "statefulSetsReady"), runtimeProxyReady: runtimeProxyReady(argoInstall), }, argo, job: typeof jobStatus === "object" && jobStatus !== null ? jobStatus : { parseError: "remote job status did not return JSON", stdoutPreview: jobResult.stdout.slice(0, 1000) }, result: { k3s: compactCommandResult(result), job: compactCommandResult(jobResult) }, next: ok ? { infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}` } : { apply: `bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node ${node.id} --lane ${target.lane} --confirm`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node ${node.id} --lane ${target.lane}` }, }; } function argoApply(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ArgoOptions): Record { if (options.confirm && options.dryRun) throw new Error("argo apply accepts only one of --dry-run or --confirm"); if (!target.argo.install.enabled) throw new Error(`targets.${target.id}.argo.install.enabled=false`); const dryRun = options.dryRun || !options.confirm; const desired = argoDesiredManifest(target); const desiredYaml = `${desired.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; const applyPlan = { namespace: target.argo.namespace, version: target.argo.install.version, manifestUrl: target.argo.install.manifestUrl, fieldManager: target.argo.install.fieldManager, imageRewrites: target.argo.install.imageRewrites, preloadImages: target.argo.install.preloadImages, requiredCrds: target.argo.install.requiredCrds, desired: manifestObjectSummary(desired), desiredSha256: sha256Short(desiredYaml), stateDir: remoteJobStateDir(target, "argo"), egressProxy: controlPlaneEgressProxySummary(node.egressProxy), runtimeProxy: runtimeHostProxyConfig(node, target.argo.install.runtimeProxy), }; if (dryRun) { return { ok: true, command: "hwlab nodes control-plane infra argo apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "dry-run", mutation: false, applyPlan, desiredYaml: { objects: desired.length, bytes: Buffer.byteLength(desiredYaml), sha256: sha256Short(desiredYaml), preview: desiredYaml }, next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node ${node.id} --lane ${target.lane} --confirm` }, }; } const result = runTransK3s(node.kubeRoute, argoApplyStartScript(node, target, desiredYaml), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { ok: result.exitCode === 0, command: "hwlab nodes control-plane infra argo apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "confirmed-start", mutation: result.exitCode === 0, applyPlan, start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra argo status --node ${node.id} --lane ${target.lane}`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node ${node.id} --lane ${target.lane}`, infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, }, }; } function parseInfraOptions(args: string[]): InfraOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra usage: infra plan|status|apply --node NODE --lane vNN [--dry-run|--confirm]"); if (actionRaw !== "plan" && actionRaw !== "status" && actionRaw !== "apply") throw new Error(`unsupported infra action ${actionRaw}; expected plan|status|apply`); const node = requiredOption(args, "--node"); const lane = requiredOption(args, "--lane"); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("infra accepts only one of --confirm or --dry-run"); return { action: actionRaw, node, lane, confirm, dryRun: actionRaw === "apply" ? explicitDryRun || !confirm : true, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 300), }; } function parseK3sInstallOptions(args: string[]): K3sInstallOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") { throw new Error("infra k3s usage: k3s plan|install|status --node NODE --lane vNN [--dry-run|--confirm]"); } if (actionRaw !== "plan" && actionRaw !== "install" && actionRaw !== "status") { throw new Error(`unsupported k3s action ${actionRaw}; expected plan|install|status`); } const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("k3s install accepts only one of --confirm or --dry-run"); return { action: actionRaw, node: requiredOption(args, "--node"), lane: requiredOption(args, "--lane"), confirm, dryRun: actionRaw === "install" ? explicitDryRun || !confirm : true, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 60), tailLines: positiveIntegerOption(args, "--tail-lines", 120, 1000), }; } function parseToolsImageOptions(args: string[]): ToolsImageOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra tools-image usage: tools-image status|build|logs --node NODE --lane vNN [--dry-run|--confirm]"); if (actionRaw !== "status" && actionRaw !== "build" && actionRaw !== "logs") throw new Error(`unsupported tools-image action ${actionRaw}; expected status|build|logs`); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("tools-image accepts only one of --confirm or --dry-run"); return { action: actionRaw, node: requiredOption(args, "--node"), lane: requiredOption(args, "--lane"), confirm, dryRun: actionRaw === "build" ? explicitDryRun || !confirm : true, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 60), tailLines: positiveIntegerOption(args, "--tail-lines", 120, 1000), }; } function parseTektonInstallOptions(args: string[]): TektonInstallOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra tekton usage: tekton status|apply|logs --node NODE --lane vNN [--dry-run|--confirm]"); if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "logs") throw new Error(`unsupported tekton action ${actionRaw}; expected status|apply|logs`); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("tekton accepts only one of --confirm or --dry-run"); return { action: actionRaw, node: requiredOption(args, "--node"), lane: requiredOption(args, "--lane"), confirm, dryRun: actionRaw === "apply" ? explicitDryRun || !confirm : true, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 60), tailLines: positiveIntegerOption(args, "--tail-lines", 120, 1000), }; } function parseArgoOptions(args: string[]): ArgoOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra argo usage: argo status|apply|logs --node NODE --lane vNN [--dry-run|--confirm]"); if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "logs") throw new Error(`unsupported argo action ${actionRaw}; expected status|apply|logs`); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("argo accepts only one of --confirm or --dry-run"); return { action: actionRaw, node: requiredOption(args, "--node"), lane: requiredOption(args, "--lane"), confirm, dryRun: actionRaw === "apply" ? explicitDryRun || !confirm : true, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 60), tailLines: positiveIntegerOption(args, "--tail-lines", 120, 1000), }; } function parseEgressBenchmarkOptions(args: string[]): EgressBenchmarkOptions { const first = args[0]; const action: EgressBenchmarkAction = first === "status" || first === "logs" ? first : "benchmark"; const effectiveArgs = action === "benchmark" ? args : args.slice(1); if (first === "--help" || first === "-h" || first === "help") { throw new Error("infra egress-benchmark usage: egress-benchmark [status|logs] --node NODE --lane vNN --profile no-mirror [--dry-run|--confirm]"); } const profileRaw = optionValue(effectiveArgs, "--profile") ?? "no-mirror"; if (profileRaw !== "no-mirror") throw new Error("egress-benchmark --profile currently supports no-mirror"); const confirm = effectiveArgs.includes("--confirm"); const explicitDryRun = effectiveArgs.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("egress-benchmark accepts only one of --confirm or --dry-run"); return { action, node: requiredOption(effectiveArgs, "--node"), lane: requiredOption(effectiveArgs, "--lane"), profile: profileRaw, confirm, dryRun: action === "benchmark" ? explicitDryRun || !confirm : false, samples: positiveIntegerOption(effectiveArgs, "--samples", 5, 20), sampleTimeoutSeconds: positiveIntegerOption(effectiveArgs, "--sample-timeout-seconds", 240, 900), timeoutSeconds: positiveIntegerOption(effectiveArgs, "--timeout-seconds", 60, 60), tailLines: positiveIntegerOption(effectiveArgs, "--tail-lines", 80, 1000), }; } function parseCiBuildBenchmarkOptions(args: string[]): CiBuildBenchmarkOptions { const first = args[0]; const action: CiBuildBenchmarkAction = first === "status" || first === "logs" ? first : "benchmark"; const effectiveArgs = action === "benchmark" ? args : args.slice(1); if (first === "--help" || first === "-h" || first === "help") { throw new Error("infra ci-build-benchmark usage: ci-build-benchmark [status|logs] --node NODE --lane vNN --profile PROFILE [--dry-run|--confirm]"); } const confirm = effectiveArgs.includes("--confirm"); const explicitDryRun = effectiveArgs.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("ci-build-benchmark accepts only one of --confirm or --dry-run"); return { action, node: requiredOption(effectiveArgs, "--node"), lane: requiredOption(effectiveArgs, "--lane"), profile: requiredOption(effectiveArgs, "--profile"), confirm, dryRun: action === "benchmark" ? explicitDryRun || !confirm : false, timeoutSeconds: positiveIntegerOption(effectiveArgs, "--timeout-seconds", 60, 60), tailLines: positiveIntegerOption(effectiveArgs, "--tail-lines", 120, 1000), }; } function readControlPlaneConfig(): ControlPlaneConfig { const parsed = asRecord(Bun.YAML.parse(readFileSync(rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH), "utf8")) as unknown, HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); const version = numberField(parsed, "version", HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); const kind = stringField(parsed, "kind", HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); if (kind !== "hwlab-node-control-plane") throw new Error(`${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.kind must be hwlab-node-control-plane`); const metadataRaw = asRecord(parsed.metadata, "metadata"); const imagePolicy = imagePolicySpec(asRecord(parsed.imagePolicy, "imagePolicy")); const nodes = Object.fromEntries(Object.entries(asRecord(parsed.nodes, "nodes")).map(([id, raw]) => [id, nodeSpec(id, asRecord(raw, `nodes.${id}`))])); const targetsRaw = parsed.targets; if (!Array.isArray(targetsRaw)) throw new Error(`${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.targets must be an array`); const targets = targetsRaw.map((raw, index) => targetSpec(asRecord(raw, `targets[${index}]`), index)); for (const target of targets) { if (nodes[target.node] === undefined) throw new Error(`targets.${target.id}.node references missing node ${target.node}`); validateTargetImagePolicy(target, imagePolicy); } return { version, kind, metadata: { owner: stringField(metadataRaw, "owner", "metadata"), relatedIssues: numberArrayField(metadataRaw, "relatedIssues", "metadata"), }, imagePolicy, nodes, targets, }; } function imagePolicySpec(raw: Record): ControlPlaneImagePolicy { const requiredSourceKinds = stringArrayField(raw, "requiredSourceKinds", "imagePolicy").map((item) => { if (item !== "dockerfile" && item !== "docker-compose") throw new Error("imagePolicy.requiredSourceKinds must contain only dockerfile or docker-compose"); return item; }); return { requireReproducibleBuildSource: booleanField(raw, "requireReproducibleBuildSource", "imagePolicy"), forbidPrivateOrNodeLocalImagesAsInputs: booleanField(raw, "forbidPrivateOrNodeLocalImagesAsInputs", "imagePolicy"), allowNodeLocalRegistryAsBuildOutput: booleanField(raw, "allowNodeLocalRegistryAsBuildOutput", "imagePolicy"), requiredSourceKinds, }; } function toolsImageSpec(raw: Record, path: string): ControlPlaneTargetSpec["tekton"]["toolsImage"] { const sourceKind = stringField(raw, "sourceKind", path); if (sourceKind !== "dockerfile" && sourceKind !== "docker-compose") throw new Error(`${path}.sourceKind must be dockerfile or docker-compose`); const imagePullPolicy = optionalStringField(raw, "imagePullPolicy", path) ?? "IfNotPresent"; if (imagePullPolicy !== "Always" && imagePullPolicy !== "IfNotPresent" && imagePullPolicy !== "Never") throw new Error(`${path}.imagePullPolicy must be Always, IfNotPresent, or Never`); const publicBaseImages = stringArrayField(raw, "publicBaseImages", path); if (publicBaseImages.length === 0) throw new Error(`${path}.publicBaseImages must list at least one public base image`); for (const image of publicBaseImages) validatePublicBaseImage(image, `${path}.publicBaseImages`); const dockerfile = optionalStringField(raw, "dockerfile", path); const dockerfileInline = raw.dockerfileInline === undefined ? undefined : dockerfileInlineSpec(asRecord(raw.dockerfileInline, `${path}.dockerfileInline`), `${path}.dockerfileInline`); const composeFile = optionalStringField(raw, "composeFile", path); if (sourceKind === "dockerfile" && dockerfile === undefined && dockerfileInline === undefined) throw new Error(`${path}.dockerfile or ${path}.dockerfileInline is required when sourceKind=dockerfile`); if (dockerfile !== undefined && dockerfileInline !== undefined) throw new Error(`${path} must use only one of dockerfile or dockerfileInline`); if (sourceKind === "docker-compose" && composeFile === undefined) throw new Error(`${path}.composeFile is required when sourceKind=docker-compose`); const buildArgsRaw = raw.buildArgs === undefined ? {} : asRecord(raw.buildArgs, `${path}.buildArgs`); const buildArgs = stringRecordField(buildArgsRaw, `${path}.buildArgs`); for (const image of Object.values(buildArgs)) { if (looksLikeImageReference(image)) validatePublicBaseImage(image, `${path}.buildArgs`); } return { output: stringField(raw, "output", path), imagePullPolicy, sourceKind, context: stringField(raw, "context", path), dockerfile, dockerfileInline, composeFile, buildArgs, buildNetwork: optionalStringField(raw, "buildNetwork", path) ?? null, publicBaseImages, buildOwner: stringField(raw, "buildOwner", path), buildMode: stringField(raw, "buildMode", path), }; } function tektonInstallManifestSpec(raw: Record, path: string): ControlPlaneTektonInstallManifestSpec { const name = stringField(raw, "name", path); if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(name)) throw new Error(`${path}.name must be a lowercase identifier`); const url = stringField(raw, "url", path); validateHttpsUrl(url, `${path}.url`); return { name, url }; } function tektonInstallSpec(raw: Record, path: string): ControlPlaneTektonInstallSpec { const sourceKind = stringField(raw, "sourceKind", path); if (sourceKind !== "url") throw new Error(`${path}.sourceKind must be url`); const manifestsRaw = raw.manifests; if (!Array.isArray(manifestsRaw) || manifestsRaw.length === 0) throw new Error(`${path}.manifests must be a non-empty array`); const manifests = manifestsRaw.map((item, index) => tektonInstallManifestSpec(asRecord(item, `${path}.manifests[${index}]`), `${path}.manifests[${index}]`)); const manifestNames = new Set(); for (const manifest of manifests) { if (manifestNames.has(manifest.name)) throw new Error(`${path}.manifests contains duplicate name ${manifest.name}`); manifestNames.add(manifest.name); } const requiredCrds = stringArrayField(raw, "requiredCrds", path); for (const crd of requiredCrds) { if (!/^[a-z0-9.-]+$/u.test(crd)) throw new Error(`${path}.requiredCrds contains invalid CRD name ${crd}`); } const expectedDeploymentNamespaces = stringArrayField(raw, "expectedDeploymentNamespaces", path); for (const namespace of expectedDeploymentNamespaces) validateKubernetesName(namespace, `${path}.expectedDeploymentNamespaces`); return { enabled: booleanField(raw, "enabled", path), sourceKind, version: stringField(raw, "version", path), fieldManager: stringField(raw, "fieldManager", path), manifests, requiredCrds, expectedDeploymentNamespaces, readinessTimeoutSeconds: positiveConfigIntegerField(raw, "readinessTimeoutSeconds", path), runtimeProxy: runtimeProxySpec(raw.runtimeProxy ?? { enabled: false }, `${path}.runtimeProxy`), }; } function runtimeProxySpec(value: unknown, path: string): ControlPlaneRuntimeProxySpec { const raw = asRecord(value, path); const enabled = booleanField(raw, "enabled", path); if (!enabled) { return { enabled, mode: "host-route", configRef: optionalStringField(raw, "configRef", path) ?? null, hostNetwork: false, injectEnv: false, deployments: [], statefulSets: [], }; } const mode = stringField(raw, "mode", path); if (mode !== "host-route") throw new Error(`${path}.mode must be host-route`); const configRef = stringField(raw, "configRef", path); if (!/^nodes\.[A-Za-z0-9_.-]+\.egressProxy$/u.test(configRef)) { throw new Error(`${path}.configRef must point at nodes..egressProxy`); } const hostNetwork = booleanField(raw, "hostNetwork", path); const injectEnv = booleanField(raw, "injectEnv", path); if (!hostNetwork && !injectEnv) throw new Error(`${path} must enable at least one of hostNetwork or injectEnv`); const deployments = raw.deployments === undefined ? [] : stringArrayField(raw, "deployments", path); const statefulSets = raw.statefulSets === undefined ? [] : stringArrayField(raw, "statefulSets", path); for (const name of [...deployments, ...statefulSets]) validateKubernetesName(name, path); if (deployments.length + statefulSets.length === 0) throw new Error(`${path} must declare deployments or statefulSets when enabled=true`); return { enabled, mode, configRef, hostNetwork, injectEnv, deployments, statefulSets }; } function dockerfileInlineSpec(raw: Record, path: string): DockerfileInlineSpec { const filename = stringField(raw, "filename", path); if (!/^[A-Za-z0-9._/-]+$/u.test(filename) || filename.includes("..")) throw new Error(`${path}.filename has an unsupported format`); const lines = stringArrayField(raw, "lines", path); if (lines.length === 0) throw new Error(`${path}.lines must not be empty`); return { filename, lines }; } function argoInstallSpec(raw: Record, path: string): ControlPlaneTargetSpec["argo"]["install"] { const sourceKind = stringField(raw, "sourceKind", path); if (sourceKind !== "url") throw new Error(`${path}.sourceKind must be url`); const imagePullPolicy = optionalStringField(raw, "imagePullPolicy", path) ?? "IfNotPresent"; if (imagePullPolicy !== "Always" && imagePullPolicy !== "IfNotPresent" && imagePullPolicy !== "Never") throw new Error(`${path}.imagePullPolicy must be Always, IfNotPresent, or Never`); const imageRewritesRaw = raw.imageRewrites === undefined ? [] : raw.imageRewrites; if (!Array.isArray(imageRewritesRaw)) throw new Error(`${path}.imageRewrites must be an array`); const imageRewrites = imageRewritesRaw.map((item, index) => imageRewriteSpec(asRecord(item, `${path}.imageRewrites[${index}]`), `${path}.imageRewrites[${index}]`)); const manifestUrl = stringField(raw, "manifestUrl", path); validateHttpsUrl(manifestUrl, `${path}.manifestUrl`); return { enabled: booleanField(raw, "enabled", path), sourceKind, version: stringField(raw, "version", path), manifestUrl, fieldManager: stringField(raw, "fieldManager", path), imagePullPolicy, preloadImages: stringArrayField(raw, "preloadImages", path), imageRewrites, requiredCrds: stringArrayField(raw, "requiredCrds", path), expectedDeployments: stringArrayField(raw, "expectedDeployments", path), expectedStatefulSets: stringArrayField(raw, "expectedStatefulSets", path), readinessTimeoutSeconds: positiveConfigIntegerField(raw, "readinessTimeoutSeconds", path), runtimeProxy: runtimeProxySpec(raw.runtimeProxy ?? { enabled: false }, `${path}.runtimeProxy`), }; } function ciBuildBenchmarkProfileSpecs(raw: unknown, path: string): readonly CiBuildBenchmarkProfileSpec[] { if (raw === undefined) return []; if (!Array.isArray(raw)) throw new Error(`${path} must be an array`); const profiles = raw.map((item, index) => ciBuildBenchmarkProfileSpec(asRecord(item, `${path}[${index}]`), `${path}[${index}]`)); const names = new Set(); for (const profile of profiles) { if (names.has(profile.profile)) throw new Error(`${path} contains duplicate profile ${profile.profile}`); names.add(profile.profile); } return profiles; } function ciBuildBenchmarkProfileSpec(raw: Record, path: string): CiBuildBenchmarkProfileSpec { const profile = stringField(raw, "profile", path); validateBenchmarkProfileName(profile, `${path}.profile`); const pipelineRunPrefix = stringField(raw, "pipelineRunPrefix", path); validateKubernetesName(pipelineRunPrefix, `${path}.pipelineRunPrefix`); if (pipelineRunPrefix.length > 40) throw new Error(`${path}.pipelineRunPrefix must leave room for source and nonce suffix`); const catalogPathTemplate = stringField(raw, "catalogPathTemplate", path); validateBenchmarkCatalogPathTemplate(catalogPathTemplate, `${path}.catalogPathTemplate`); const imageTagMode = stringField(raw, "imageTagMode", path); if (imageTagMode !== "full") throw new Error(`${path}.imageTagMode currently must be full`); const timings = asRecord(raw.timings, `${path}.timings`); return { profile, runtimeLaneConfigRef: stringField(raw, "runtimeLaneConfigRef", path), pipelineRunPrefix, catalogPathTemplate, imageTagMode, pipelineTimeoutSeconds: positiveConfigIntegerField(raw, "pipelineTimeoutSeconds", path), cachePolicy: ciBuildBenchmarkCachePolicy(asRecord(raw.cachePolicy, `${path}.cachePolicy`), `${path}.cachePolicy`), requiredTimings: stringArrayField(timings, "requiredStages", `${path}.timings`), failureFamilies: stringArrayField(raw, "failureFamilies", path), }; } function ciBuildBenchmarkCachePolicy(raw: Record, path: string): CiBuildBenchmarkCachePolicy { return { noPipelineRunReuse: booleanField(raw, "noPipelineRunReuse", path), forceFullBuild: booleanField(raw, "forceFullBuild", path), forbidGitopsCatalogReuse: booleanField(raw, "forbidGitopsCatalogReuse", path), forbidDependencyCache: booleanField(raw, "forbidDependencyCache", path), forbidBuildkitCache: booleanField(raw, "forbidBuildkitCache", path), forbidRegistryMirror: booleanField(raw, "forbidRegistryMirror", path), forbidLocalPreheatedImages: booleanField(raw, "forbidLocalPreheatedImages", path), }; } function imageRewriteSpec(raw: Record, path: string): ImageRewriteSpec { const rewrite = { source: stringField(raw, "source", path), pullImage: stringField(raw, "pullImage", path), target: stringField(raw, "target", path), }; validatePublicBaseImage(rewrite.source, `${path}.source`); validatePublicBaseImage(rewrite.pullImage, `${path}.pullImage`); if (!isNodeLocalImage(rewrite.target)) throw new Error(`${path}.target must use a node-local registry output image`); return rewrite; } function validateTargetImagePolicy(target: ControlPlaneTargetSpec, imagePolicy: ControlPlaneImagePolicy): void { if (imagePolicy.requireReproducibleBuildSource && !imagePolicy.requiredSourceKinds.includes(target.tekton.toolsImage.sourceKind)) { throw new Error(`targets.${target.id}.tekton.toolsImage.sourceKind is not allowed by imagePolicy.requiredSourceKinds`); } if (imagePolicy.forbidPrivateOrNodeLocalImagesAsInputs) { for (const image of target.tekton.toolsImage.publicBaseImages) validatePublicBaseImage(image, `targets.${target.id}.tekton.toolsImage.publicBaseImages`); } if (!imagePolicy.allowNodeLocalRegistryAsBuildOutput && isNodeLocalImage(target.tekton.toolsImage.output)) { throw new Error(`targets.${target.id}.tekton.toolsImage.output uses node-local registry but imagePolicy.allowNodeLocalRegistryAsBuildOutput=false`); } } function validatePublicBaseImage(image: string, path: string): void { if (isNodeLocalImage(image)) { throw new Error(`${path} contains non-public base image ${image}`); } const publicPrefixes = ["docker.io/", "registry.k8s.io/", "ghcr.io/", "quay.io/", "gcr.io/", "public.ecr.aws/", "mcr.microsoft.com/", "cgr.dev/", "docker.m.daocloud.io/", "quay.m.daocloud.io/", "ghcr.m.daocloud.io/"]; if (!publicPrefixes.some((prefix) => image.startsWith(prefix))) throw new Error(`${path} image ${image} must use an explicit public registry prefix`); } function looksLikeImageReference(value: string): boolean { return /^(docker\.io|registry\.k8s\.io|ghcr\.io|quay\.io|gcr\.io|public\.ecr\.aws|mcr\.microsoft\.com|cgr\.dev|docker\.m\.daocloud\.io|quay\.m\.daocloud\.io|ghcr\.m\.daocloud\.io)\//u.test(value) || /^(127\.|localhost(?::|\/)|0\.0\.0\.0|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)/u.test(value); } function isNodeLocalImage(image: string): boolean { return /^(127\.|localhost(?::|\/)|0\.0\.0\.0|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)/u.test(image); } function nodeSpec(id: string, raw: Record): ControlPlaneNodeSpec { const registry = asRecord(raw.registry, `nodes.${id}.registry`); const egressProxy = raw.egressProxy === undefined ? null : egressProxySpec(asRecord(raw.egressProxy, `nodes.${id}.egressProxy`), `nodes.${id}.egressProxy`); const k3s = raw.k3s === undefined ? null : k3sNodeSpec(asRecord(raw.k3s, `nodes.${id}.k3s`), `nodes.${id}.k3s`); return { id, route: stringField(raw, "route", `nodes.${id}`), kubeRoute: stringField(raw, "kubeRoute", `nodes.${id}`), k3s, registry: registrySpec(registry, `nodes.${id}.registry`), egressProxy, }; } function registrySpec(raw: Record, path: string): ControlPlaneRegistrySpec { const mode = stringField(raw, "mode", path); const endpoint = stringField(raw, "endpoint", path); if (mode === "host-docker") return { mode, endpoint }; if (mode !== "k8s-workload") throw new Error(`${path}.mode must be host-docker or k8s-workload`); const imagePullPolicy = optionalStringField(raw, "imagePullPolicy", path) ?? "IfNotPresent"; if (imagePullPolicy !== "Always" && imagePullPolicy !== "IfNotPresent" && imagePullPolicy !== "Never") { throw new Error(`${path}.imagePullPolicy must be Always, IfNotPresent, or Never`); } const namespace = stringField(raw, "namespace", path); const deploymentName = stringField(raw, "deploymentName", path); const serviceName = stringField(raw, "serviceName", path); const pvcName = stringField(raw, "pvcName", path); validateKubernetesName(namespace, `${path}.namespace`); validateKubernetesName(deploymentName, `${path}.deploymentName`); validateKubernetesName(serviceName, `${path}.serviceName`); validateKubernetesName(pvcName, `${path}.pvcName`); const image = stringField(raw, "image", path); validatePublicBaseImage(image, `${path}.image`); const hostNetwork = booleanField(raw, "hostNetwork", path); const listenHost = stringField(raw, "listenHost", path); if (listenHost !== "127.0.0.1" && listenHost !== "0.0.0.0") throw new Error(`${path}.listenHost must be 127.0.0.1 or 0.0.0.0`); return { mode, endpoint, namespace, deploymentName, serviceName, pvcName, storage: stringField(raw, "storage", path), image, imagePullPolicy, containerPort: numberField(raw, "containerPort", path), listenHost, listenPort: numberField(raw, "listenPort", path), hostNetwork, }; } function k3sNodeSpec(raw: Record, path: string): ControlPlaneK3sNodeSpec { const kubelet = asRecord(raw.kubelet, `${path}.kubelet`); const serviceName = stringField(raw, "serviceName", path); if (!/^[A-Za-z0-9_.@-]+$/u.test(serviceName)) throw new Error(`${path}.serviceName has an unsupported systemd unit name`); const dropInPath = stringField(raw, "dropInPath", path); if (!dropInPath.startsWith("/etc/systemd/system/") || !dropInPath.endsWith(".conf") || dropInPath.includes("..")) { throw new Error(`${path}.dropInPath must be an absolute /etc/systemd/system/*.conf path`); } const nodeStatusName = stringField(raw, "nodeStatusName", path); if (!/^[A-Za-z0-9_.-]+$/u.test(nodeStatusName)) throw new Error(`${path}.nodeStatusName has an unsupported Kubernetes node name`); const execStartPre = execStartPreField(raw.execStartPre, `${path}.execStartPre`); const install = raw.install === undefined ? null : k3sInstallSpec(asRecord(raw.install, `${path}.install`), `${path}.install`); const serverArgs = stringArrayField(raw, "serverArgs", path); if (serverArgs.length === 0 || serverArgs[0] !== "server") throw new Error(`${path}.serverArgs must start with k3s server`); for (const [index, arg] of serverArgs.entries()) { if (arg.includes("\n") || arg.includes("\r") || arg.length === 0) throw new Error(`${path}.serverArgs[${index}] must be a single non-empty argv token`); } const maxPods = positiveConfigIntegerField(kubelet, "maxPods", `${path}.kubelet`); const expectedMaxPodsArg = `max-pods=${maxPods}`; let hasExpectedMaxPodsArg = false; for (let index = 0; index < serverArgs.length - 1; index += 1) { if (serverArgs[index] === "--kubelet-arg" && serverArgs[index + 1] === expectedMaxPodsArg) hasExpectedMaxPodsArg = true; } if (!hasExpectedMaxPodsArg) throw new Error(`${path}.serverArgs must include --kubelet-arg ${expectedMaxPodsArg}`); return { serviceName, dropInPath, nodeStatusName, execStartPre, install, serverArgs, kubelet: { maxPods }, }; } function k3sInstallSpec(raw: Record, path: string): ControlPlaneK3sInstallSpec { const localRegistryRaw = asRecord(raw.localRegistry, `${path}.localRegistry`); const stateRaw = asRecord(raw.state, `${path}.state`); const downloadsRaw = asRecord(raw.downloads, `${path}.downloads`); const installScriptUrl = stringField(raw, "installScriptUrl", path); const binaryUrl = stringField(raw, "binaryUrl", path); const sha256Url = stringField(raw, "sha256Url", path); validateHttpsUrl(installScriptUrl, `${path}.installScriptUrl`); validateHttpsUrl(binaryUrl, `${path}.binaryUrl`); validateHttpsUrl(sha256Url, `${path}.sha256Url`); const expectedSha256 = stringField(raw, "expectedSha256", path).toLowerCase(); if (!/^[0-9a-f]{64}$/u.test(expectedSha256)) throw new Error(`${path}.expectedSha256 must be a 64-character sha256 hex digest`); const hostProxyConfigRef = stringField(raw, "hostProxyConfigRef", path); if (!hostProxyConfigRef.startsWith("config/platform-infra/host-proxy.yaml#targets.")) { throw new Error(`${path}.hostProxyConfigRef must point at config/platform-infra/host-proxy.yaml#targets.`); } return { enabled: booleanField(raw, "enabled", path), channel: stringField(raw, "channel", path), version: stringField(raw, "version", path), installScriptUrl, binaryUrl, sha256Url, expectedSha256, hostProxyConfigRef, proxyEnvPath: absoluteConfigPathField(raw, "proxyEnvPath", path), registriesYamlPath: absoluteConfigPathField(raw, "registriesYamlPath", path), localRegistry: { containerName: stringField(localRegistryRaw, "containerName", `${path}.localRegistry`), image: stringField(localRegistryRaw, "image", `${path}.localRegistry`), canonicalImage: stringField(localRegistryRaw, "canonicalImage", `${path}.localRegistry`), bind: stringField(localRegistryRaw, "bind", `${path}.localRegistry`), }, state: { dir: absoluteConfigPathField(stateRaw, "dir", `${path}.state`), logPath: absoluteConfigPathField(stateRaw, "logPath", `${path}.state`), statusPath: absoluteConfigPathField(stateRaw, "statusPath", `${path}.state`), }, downloads: { connectTimeoutSeconds: positiveConfigIntegerField(downloadsRaw, "connectTimeoutSeconds", `${path}.downloads`), maxTimeSeconds: positiveConfigIntegerField(downloadsRaw, "maxTimeSeconds", `${path}.downloads`), retry: positiveConfigIntegerField(downloadsRaw, "retry", `${path}.downloads`), retryDelaySeconds: positiveConfigIntegerField(downloadsRaw, "retryDelaySeconds", `${path}.downloads`), }, }; } function execStartPreField(raw: unknown, path: string): readonly (readonly string[])[] { if (raw === undefined) return []; if (!Array.isArray(raw)) throw new Error(`${path} must be an array of argv arrays`); return raw.map((item, index) => { if (!Array.isArray(item)) throw new Error(`${path}[${index}] must be an argv array`); const command = item.map((value, tokenIndex) => { if (typeof value !== "string") throw new Error(`${path}[${index}][${tokenIndex}] must be a string`); if (value.length === 0 || value.includes("\n") || value.includes("\r")) throw new Error(`${path}[${index}][${tokenIndex}] must be a single non-empty argv token`); return value; }); if (command.length === 0) throw new Error(`${path}[${index}] must not be empty`); const executable = command[0].startsWith("-") ? command[0].slice(1) : command[0]; if (!executable.startsWith("/") || executable.includes("..")) throw new Error(`${path}[${index}][0] must be an absolute executable path, optionally prefixed with -`); return command; }); } function egressProxySpec(raw: Record, path: string): ControlPlaneEgressProxySpec { const mode = stringField(raw, "mode", path); if (mode === "host-route") { const hostProxyConfigRef = stringField(raw, "hostProxyConfigRef", path); if (!hostProxyConfigRef.startsWith("config/platform-infra/host-proxy.yaml#targets.")) { throw new Error(`${path}.hostProxyConfigRef must point at config/platform-infra/host-proxy.yaml#targets.`); } const proxyUrl = stringField(raw, "proxyUrl", path); const parsed = new URL(proxyUrl); if ( parsed.protocol !== "http:" || parsed.port.length === 0 || parsed.username.length > 0 || parsed.password.length > 0 || parsed.hash.length > 0 || (parsed.pathname !== "/" && parsed.pathname.length > 0) || parsed.search.length > 0 || !hostRouteProxyHostAllowed(parsed.hostname) ) { throw new Error(`${path}.proxyUrl must be a credential-free http://: URL`); } return { mode, clientName: stringField(raw, "clientName", path), hostProxyConfigRef, proxyEnvPath: absoluteConfigPathField(raw, "proxyEnvPath", path), proxyUrl, noProxy: stringArrayField(raw, "noProxy", path), }; } if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip or host-route`); const sourceConfigRef = optionalStringField(raw, "sourceConfigRef", path) ?? null; const source = sourceConfigRef === null ? null : resolveEgressProxySourceRef(sourceConfigRef, `${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.${path}.sourceConfigRef`); const sourceType = source?.sourceType ?? stringField(raw, "sourceType", path); if (sourceType !== "subscription-url" && sourceType !== "master-shadowsocks") throw new Error(`${path}.sourceType must be subscription-url or master-shadowsocks`); if (raw.sourceType !== undefined && raw.sourceType !== sourceType) throw new Error(`${path}.sourceType must match sourceConfigRef value ${sourceType}`); const preferredOutbound = source?.preferredOutbound ?? (raw.preferredOutbound === undefined ? null : preferredOutboundField(raw, "preferredOutbound", path)); if (source !== null && source.preferredOutbound !== null && raw.preferredOutbound !== undefined && raw.preferredOutbound !== source.preferredOutbound) { throw new Error(`${path}.preferredOutbound must match sourceConfigRef value ${source.preferredOutbound}`); } if (source !== null) { if (raw.sourceRef !== undefined && raw.sourceRef !== source.sourceRef) throw new Error(`${path}.sourceRef must match sourceConfigRef value ${source.sourceRef}`); if (raw.sourceKey !== undefined && raw.sourceKey !== source.sourceKey) throw new Error(`${path}.sourceKey must match sourceConfigRef value ${source.sourceKey}`); } const sourceRef = source?.sourceRef ?? stringField(raw, "sourceRef", path); const sourceKey = source?.sourceKey ?? stringField(raw, "sourceKey", path); validateSourceRef(sourceRef, `${path}.sourceRef`); validateEnvKey(sourceKey, `${path}.sourceKey`); return { mode, clientName: stringField(raw, "clientName", path), namespace: stringField(raw, "namespace", path), serviceName: stringField(raw, "serviceName", path), port: positiveConfigIntegerField(raw, "port", path), sourceConfigRef, sourceFingerprint: source?.fingerprint ?? null, sourceRef, sourceKey, sourceType, preferredOutbound, noProxy: stringArrayField(raw, "noProxy", path), }; } function preferredOutboundField(raw: Record, key: string, path: string): "vless-reality" | "hysteria2" { const value = stringField(raw, key, path); if (value !== "vless-reality" && value !== "hysteria2") throw new Error(`${path}.${key} must be vless-reality or hysteria2`); return value; } function gitMirrorEgressProxySpec(raw: Record, path: string): ControlPlaneGitMirrorEgressProxySpec { const mode = stringField(raw, "mode", path); if (mode !== "node-global" && mode !== "host-route" && mode !== "direct") throw new Error(`${path}.mode must be node-global, host-route, or direct`); const podHostNetwork = raw.podHostNetwork === undefined ? false : booleanField(raw, "podHostNetwork", path); const injectPodEnv = raw.injectPodEnv === undefined ? false : booleanField(raw, "injectPodEnv", path); if (mode === "host-route" && !podHostNetwork && !injectPodEnv) throw new Error(`${path} must enable podHostNetwork or injectPodEnv when mode=host-route`); return { mode, required: raw.required === undefined ? mode !== "direct" : booleanField(raw, "required", path), podHostNetwork, injectPodEnv, }; } function gitMirrorGithubTransportSpec(raw: Record, path: string): ControlPlaneGitMirrorGithubTransportSpec { const mode = stringField(raw, "mode", path); if (mode === "ssh") { const privateKeySecretKey = stringField(raw, "privateKeySecretKey", path); const privateKeySourceRef = stringField(raw, "privateKeySourceRef", path); const privateKeySourceKey = stringField(raw, "privateKeySourceKey", path); const privateKeySourceEncoding = secretSourceEncodingField(raw, "privateKeySourceEncoding", path); const knownHostsSecretKey = optionalStringField(raw, "knownHostsSecretKey", path) ?? null; const knownHostsSourceRef = optionalStringField(raw, "knownHostsSourceRef", path) ?? null; const knownHostsSourceKey = optionalStringField(raw, "knownHostsSourceKey", path) ?? null; const knownHostsSourceEncoding = raw.knownHostsSourceEncoding === undefined ? null : secretSourceEncodingField(raw, "knownHostsSourceEncoding", path); const knownHostsFields = [knownHostsSecretKey, knownHostsSourceRef, knownHostsSourceKey, knownHostsSourceEncoding]; if (knownHostsFields.some((value) => value !== null) && knownHostsFields.some((value) => value === null)) { throw new Error(`${path}.knownHostsSecretKey/sourceRef/sourceKey/sourceEncoding must be declared together`); } validateSecretKey(privateKeySecretKey, `${path}.privateKeySecretKey`); if (privateKeySecretKey !== "ssh-privatekey") throw new Error(`${path}.privateKeySecretKey must be ssh-privatekey for kubernetes.io/ssh-auth`); validateSourceRef(privateKeySourceRef, `${path}.privateKeySourceRef`); validateEnvKey(privateKeySourceKey, `${path}.privateKeySourceKey`); if (knownHostsSecretKey !== null) validateSecretKey(knownHostsSecretKey, `${path}.knownHostsSecretKey`); if (knownHostsSourceRef !== null) validateSourceRef(knownHostsSourceRef, `${path}.knownHostsSourceRef`); if (knownHostsSourceKey !== null) validateEnvKey(knownHostsSourceKey, `${path}.knownHostsSourceKey`); return { mode, privateKeySecretKey, privateKeySourceRef, privateKeySourceKey, privateKeySourceEncoding, knownHostsSecretKey, knownHostsSourceRef, knownHostsSourceKey, knownHostsSourceEncoding, }; } if (mode !== "https") throw new Error(`${path}.mode must be ssh or https`); const tokenSecretName = stringField(raw, "tokenSecretName", path); const tokenSecretKey = stringField(raw, "tokenSecretKey", path); const tokenSourceRef = stringField(raw, "tokenSourceRef", path); const tokenSourceKey = stringField(raw, "tokenSourceKey", path); validateKubernetesName(tokenSecretName, `${path}.tokenSecretName`); validateSecretKey(tokenSecretKey, `${path}.tokenSecretKey`); validateSourceRef(tokenSourceRef, `${path}.tokenSourceRef`); validateEnvKey(tokenSourceKey, `${path}.tokenSourceKey`); return { mode, username: stringField(raw, "username", path), tokenSecretName, tokenSecretKey, tokenSourceRef, tokenSourceKey, }; } function tektonGitWorkspaceSecretSpec( value: unknown, path: string, ciNamespace: string, githubTransport: ControlPlaneGitMirrorGithubTransportSpec, ): ControlPlaneTektonGitWorkspaceSecretSpec { const raw = asRecord(value, path); const sourceRefFrom = stringField(raw, "sourceRefFrom", path); if (sourceRefFrom !== "gitMirror.githubTransport") { throw new Error(`${path}.sourceRefFrom must be gitMirror.githubTransport`); } if (githubTransport.mode !== "ssh") { throw new Error(`${path}.sourceRefFrom=gitMirror.githubTransport requires gitMirror.githubTransport.mode=ssh`); } if (githubTransport.knownHostsSecretKey === null) { throw new Error(`${path}.sourceRefFrom=gitMirror.githubTransport requires gitMirror.githubTransport.knownHostsSecretKey`); } const name = stringField(raw, "name", path); const namespace = stringField(raw, "namespace", path); const privateKeySecretKey = stringField(raw, "privateKeySecretKey", path); const knownHostsSecretKey = stringField(raw, "knownHostsSecretKey", path); validateKubernetesName(name, `${path}.name`); validateKubernetesName(namespace, `${path}.namespace`); validateSecretKey(privateKeySecretKey, `${path}.privateKeySecretKey`); validateSecretKey(knownHostsSecretKey, `${path}.knownHostsSecretKey`); if (namespace !== ciNamespace) throw new Error(`${path}.namespace must equal targets[].ciNamespace ${ciNamespace}`); if (privateKeySecretKey !== "ssh-privatekey") throw new Error(`${path}.privateKeySecretKey must be ssh-privatekey for Tekton git SSH workspaces`); return { name, namespace, sourceRefFrom, privateKeySecretKey, knownHostsSecretKey, }; } function tektonRuntimeObserverRbacSpec( value: unknown, path: string, runtimeNamespace: string, ): ControlPlaneTektonRuntimeObserverRbacSpec { const raw = asRecord(value, path); const namespace = stringField(raw, "namespace", path); const roleName = stringField(raw, "roleName", path); const roleBindingName = stringField(raw, "roleBindingName", path); validateKubernetesName(namespace, `${path}.namespace`); validateKubernetesName(roleName, `${path}.roleName`); validateKubernetesName(roleBindingName, `${path}.roleBindingName`); if (namespace !== runtimeNamespace) throw new Error(`${path}.namespace must equal targets[].runtimeNamespace ${runtimeNamespace}`); return { namespace, roleName, roleBindingName }; } function tektonArgoObserverRbacSpec( value: unknown, path: string, argoNamespace: string, ): ControlPlaneTektonArgoObserverRbacSpec { const raw = asRecord(value, path); const namespace = stringField(raw, "namespace", path); const roleName = stringField(raw, "roleName", path); const roleBindingName = stringField(raw, "roleBindingName", path); validateKubernetesName(namespace, `${path}.namespace`); validateKubernetesName(roleName, `${path}.roleName`); validateKubernetesName(roleBindingName, `${path}.roleBindingName`); if (namespace !== argoNamespace) throw new Error(`${path}.namespace must equal targets[].argo.namespace ${argoNamespace}`); return { namespace, roleName, roleBindingName }; } function secretSourceEncodingField(raw: Record, key: string, path: string): "plain" | "base64" { const value = stringField(raw, key, path); if (value !== "plain" && value !== "base64") throw new Error(`${path}.${key} must be plain or base64`); return value; } function targetSpec(raw: Record, index: number): ControlPlaneTargetSpec { const path = `targets[${index}]`; const source = asRecord(raw.source, `${path}.source`); const gitops = asRecord(raw.gitops, `${path}.gitops`); const gitMirror = asRecord(raw.gitMirror, `${path}.gitMirror`); const tekton = asRecord(raw.tekton, `${path}.tekton`); const argo = asRecord(raw.argo, `${path}.argo`); const toolsImage = asRecord(tekton.toolsImage, `${path}.tekton.toolsImage`); const node = stringField(raw, "node", path); const lane = stringField(raw, "lane", path); const gitMirrorNamespace = stringField(gitMirror, "namespace", `${path}.gitMirror`); const serviceReadName = stringField(gitMirror, "serviceReadName", `${path}.gitMirror`); const serviceWriteName = stringField(gitMirror, "serviceWriteName", `${path}.gitMirror`); const gitMirrorEgressProxy = gitMirror.egressProxy === undefined ? null : gitMirrorEgressProxySpec(asRecord(gitMirror.egressProxy, `${path}.gitMirror.egressProxy`), `${path}.gitMirror.egressProxy`); const githubTransport = gitMirrorGithubTransportSpec(asRecord(gitMirror.githubTransport, `${path}.gitMirror.githubTransport`), `${path}.gitMirror.githubTransport`); const sourceRepository = stringField(source, "repository", `${path}.source`); const ciNamespace = stringField(raw, "ciNamespace", path); const runtimeNamespace = stringField(raw, "runtimeNamespace", path); const argoNamespace = stringField(argo, "namespace", `${path}.argo`); return { id: stringField(raw, "id", path), node, lane, enabled: booleanField(raw, "enabled", path), ciNamespace, runtimeNamespace, source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`) }, gitops: { branch: stringField(gitops, "branch", `${path}.gitops`), path: stringField(gitops, "path", `${path}.gitops`) }, gitMirror: { namespace: gitMirrorNamespace, serviceReadName, serviceWriteName, cachePvcName: stringField(gitMirror, "cachePvcName", `${path}.gitMirror`), cachePvcStorage: stringField(gitMirror, "cachePvcStorage", `${path}.gitMirror`), cacheHostPath: nullableStringField(gitMirror, "cacheHostPath", `${path}.gitMirror`), servicePort: numberField(gitMirror, "servicePort", `${path}.gitMirror`), readContainerPort: numberField(gitMirror, "readContainerPort", `${path}.gitMirror`), writeContainerPort: numberField(gitMirror, "writeContainerPort", `${path}.gitMirror`), deploymentReplicas: nonNegativeIntegerField(gitMirror, "deploymentReplicas", `${path}.gitMirror`), secretName: stringField(gitMirror, "secretName", `${path}.gitMirror`), syncConfigMapName: stringField(gitMirror, "syncConfigMapName", `${path}.gitMirror`), syncJobPrefix: stringField(gitMirror, "syncJobPrefix", `${path}.gitMirror`), flushJobPrefix: stringField(gitMirror, "flushJobPrefix", `${path}.gitMirror`), readUrl: optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, writeUrl: optionalStringField(gitMirror, "writeUrl", `${path}.gitMirror`) ?? `http://${serviceWriteName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, egressProxy: gitMirrorEgressProxy, githubTransport, }, tekton: { install: tektonInstallSpec(asRecord(tekton.install, `${path}.tekton.install`), `${path}.tekton.install`), pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`), serviceAccountName: stringField(tekton, "serviceAccountName", `${path}.tekton`), pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${path}.tekton`), gitWorkspaceSecret: tektonGitWorkspaceSecretSpec(tekton.gitWorkspaceSecret, `${path}.tekton.gitWorkspaceSecret`, ciNamespace, githubTransport), runtimeObserverRbac: tektonRuntimeObserverRbacSpec(tekton.runtimeObserverRbac, `${path}.tekton.runtimeObserverRbac`, runtimeNamespace), argoObserverRbac: tektonArgoObserverRbacSpec(tekton.argoObserverRbac, `${path}.tekton.argoObserverRbac`, argoNamespace), toolsImage: toolsImageSpec(toolsImage, `${path}.tekton.toolsImage`), }, ciBuildBenchmarks: ciBuildBenchmarkProfileSpecs(raw.ciBuildBenchmarks, `${path}.ciBuildBenchmarks`), argo: { namespace: argoNamespace, projectName: stringField(argo, "projectName", `${path}.argo`), applicationName: stringField(argo, "applicationName", `${path}.argo`), applicationFile: stringField(argo, "applicationFile", `${path}.argo`), install: argoInstallSpec(asRecord(argo.install, `${path}.argo.install`), `${path}.argo.install`), }, }; } function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record[] { const labels = { "app.kubernetes.io/part-of": "hwlab-node-control-plane", "hwlab.pikastech.local/node": target.node, "hwlab.pikastech.local/lane": target.lane, }; const manifests: Record[] = []; const namespaces = new Set(); const addNamespace = (name: string): void => { if (namespaces.has(name)) return; namespaces.add(name); manifests.push({ apiVersion: "v1", kind: "Namespace", metadata: { name, labels } }); }; addNamespace(target.ciNamespace); addNamespace(target.runtimeNamespace); addNamespace(target.gitMirror.namespace); if (_node.registry.mode === "k8s-workload") addNamespace(_node.registry.namespace); manifests.push(...registryInfraManifest(_node.registry, labels)); manifests.push( { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: target.tekton.serviceAccountName, namespace: target.ciNamespace, labels } }, tektonRuntimeObserverRole(target, labels), tektonRuntimeObserverRoleBinding(target, labels), tektonArgoObserverRole(target, labels), tektonArgoObserverRoleBinding(target, labels), { apiVersion: "v1", kind: "ConfigMap", metadata: { name: target.gitMirror.syncConfigMapName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, data: { "repositories.json": JSON.stringify([{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], null, 2), "server.js": gitMirrorServerJs(), "status.sh": gitMirrorStatusShell(), "sync.sh": gitMirrorSyncShell(_node, target), "flush.sh": gitMirrorFlushShell(_node, target), }, }, ); const githubTokenSecret = gitMirrorGithubTokenSecret(target, labels); if (githubTokenSecret !== null) manifests.push(githubTokenSecret); const githubSshSecret = gitMirrorGithubSshSecret(target, labels); if (githubSshSecret !== null) manifests.push(githubSshSecret); const ciGitWorkspaceSecret = tektonGitWorkspaceSecret(target, labels); if (ciGitWorkspaceSecret !== null) manifests.push(ciGitWorkspaceSecret); if (target.gitMirror.cacheHostPath === null) { manifests.push({ apiVersion: "v1", kind: "PersistentVolumeClaim", metadata: { name: target.gitMirror.cachePvcName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: target.gitMirror.cachePvcStorage } } }, }); } manifests.push( service(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), service(target.gitMirror.serviceWriteName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), gitMirrorDeployment(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, _node, target, "read"), gitMirrorDeployment(target.gitMirror.serviceWriteName, target.gitMirror.namespace, labels, _node, target, "write"), { apiVersion: "tekton.dev/v1", kind: "Pipeline", metadata: { name: target.tekton.pipelineName, namespace: target.ciNamespace, labels }, spec: { params: [ { name: "source-commit", type: "string" }, { name: "source-branch", type: "string", default: target.source.branch }, { name: "gitops-branch", type: "string", default: target.gitops.branch }, ], tasks: [{ name: "bootstrap-placeholder", taskSpec: { steps: [{ name: "notice", image: target.tekton.toolsImage.output, script: "#!/bin/sh\nset -eu\necho d601-hwlab-v03-pipeline-placeholder\n" }] } }], }, }, { apiVersion: "v1", kind: "Namespace", metadata: { name: target.argo.namespace, labels } }, { apiVersion: "v1", kind: "ConfigMap", metadata: { name: `${target.argo.applicationName}-desired`, namespace: target.argo.namespace, labels }, data: { "project.yaml": Bun.YAML.stringify(argoProjectSkeleton(target)), [target.argo.applicationFile]: Bun.YAML.stringify(argoApplicationSkeleton(target)), "note.txt": "Argo CD CRDs/controller are installed by the node control-plane bootstrap path when available; this ConfigMap preserves the desired Application until Argo is ready.\n", }, }, ); return manifests; } function tektonRuntimeObserverRole(target: ControlPlaneTargetSpec, labels: Record): Record { const rbac = target.tekton.runtimeObserverRbac; return { apiVersion: "rbac.authorization.k8s.io/v1", kind: "Role", metadata: { name: rbac.roleName, namespace: rbac.namespace, labels: { ...labels, "app.kubernetes.io/name": "hwlab-runtime-observer" }, }, rules: [ { apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"], }, ], }; } function tektonRuntimeObserverRoleBinding(target: ControlPlaneTargetSpec, labels: Record): Record { const rbac = target.tekton.runtimeObserverRbac; return { apiVersion: "rbac.authorization.k8s.io/v1", kind: "RoleBinding", metadata: { name: rbac.roleBindingName, namespace: rbac.namespace, labels: { ...labels, "app.kubernetes.io/name": "hwlab-runtime-observer" }, }, subjects: [ { kind: "ServiceAccount", name: target.tekton.serviceAccountName, namespace: target.ciNamespace, }, ], roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: rbac.roleName, }, }; } function tektonArgoObserverRole(target: ControlPlaneTargetSpec, labels: Record): Record { const rbac = target.tekton.argoObserverRbac; return { apiVersion: "rbac.authorization.k8s.io/v1", kind: "Role", metadata: { name: rbac.roleName, namespace: rbac.namespace, labels: { ...labels, "app.kubernetes.io/name": "hwlab-argo-observer" }, }, rules: [ { apiGroups: ["argoproj.io"], resources: ["applications"], verbs: ["get", "list", "watch"], }, ], }; } function tektonArgoObserverRoleBinding(target: ControlPlaneTargetSpec, labels: Record): Record { const rbac = target.tekton.argoObserverRbac; return { apiVersion: "rbac.authorization.k8s.io/v1", kind: "RoleBinding", metadata: { name: rbac.roleBindingName, namespace: rbac.namespace, labels: { ...labels, "app.kubernetes.io/name": "hwlab-argo-observer" }, }, subjects: [ { kind: "ServiceAccount", name: target.tekton.serviceAccountName, namespace: target.ciNamespace, }, ], roleRef: { apiGroup: "rbac.authorization.k8s.io", kind: "Role", name: rbac.roleName, }, }; } function gitMirrorGithubSshSecret(target: ControlPlaneTargetSpec, labels: Record): Record | null { const transport = target.gitMirror.githubTransport; if (transport.mode !== "ssh") return null; const material = gitMirrorGithubSshMaterial(transport); return { apiVersion: "v1", kind: "Secret", metadata: { name: target.gitMirror.secretName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" }, annotations: { "unidesk.ai/private-key-source-ref": transport.privateKeySourceRef, "unidesk.ai/private-key-source-key": transport.privateKeySourceKey, "unidesk.ai/private-key-target-key": transport.privateKeySecretKey, "unidesk.ai/private-key-fingerprint": material.privateKeyFingerprint, ...(transport.knownHostsSecretKey === null ? {} : { "unidesk.ai/known-hosts-source-ref": transport.knownHostsSourceRef ?? "", "unidesk.ai/known-hosts-source-key": transport.knownHostsSourceKey ?? "", "unidesk.ai/known-hosts-target-key": transport.knownHostsSecretKey, "unidesk.ai/known-hosts-fingerprint": material.knownHostsFingerprint ?? "", }), }, }, type: "kubernetes.io/ssh-auth", stringData: { [transport.privateKeySecretKey]: material.privateKey, ...(transport.knownHostsSecretKey === null || material.knownHosts === null ? {} : { [transport.knownHostsSecretKey]: material.knownHosts }), }, }; } function tektonGitWorkspaceSecret(target: ControlPlaneTargetSpec, labels: Record): Record | null { const transport = target.gitMirror.githubTransport; if (transport.mode !== "ssh") return null; const secret = target.tekton.gitWorkspaceSecret; const material = gitMirrorGithubSshMaterial(transport); if (material.knownHosts === null || material.knownHostsFingerprint === null) { throw new Error(`targets.${target.id}.tekton.gitWorkspaceSecret requires known_hosts material from gitMirror.githubTransport`); } return { apiVersion: "v1", kind: "Secret", metadata: { name: secret.name, namespace: secret.namespace, labels: { ...labels, "app.kubernetes.io/name": "hwlab-tekton-git-workspace" }, annotations: { "unidesk.ai/source-ref-from": secret.sourceRefFrom, "unidesk.ai/private-key-source-ref": transport.privateKeySourceRef, "unidesk.ai/private-key-source-key": transport.privateKeySourceKey, "unidesk.ai/private-key-target-key": secret.privateKeySecretKey, "unidesk.ai/private-key-fingerprint": material.privateKeyFingerprint, "unidesk.ai/known-hosts-source-ref": transport.knownHostsSourceRef ?? "", "unidesk.ai/known-hosts-source-key": transport.knownHostsSourceKey ?? "", "unidesk.ai/known-hosts-target-key": secret.knownHostsSecretKey, "unidesk.ai/known-hosts-fingerprint": material.knownHostsFingerprint, }, }, type: "kubernetes.io/ssh-auth", stringData: { [secret.privateKeySecretKey]: material.privateKey, [secret.knownHostsSecretKey]: material.knownHosts, }, }; } function gitMirrorGithubSshMaterial(transport: Extract): { privateKey: string; knownHosts: string | null; privateKeyFingerprint: string; knownHostsFingerprint: string | null } { const privateSource = readControlPlaneSecretSource(transport.privateKeySourceRef, `gitMirror.githubTransport private key source ${transport.privateKeySourceRef} is missing; create the YAML-declared sourceRef with ${transport.privateKeySourceKey} before applying the control plane`); const privateValue = requiredEnvValue(privateSource.values, transport.privateKeySourceKey, transport.privateKeySourceRef); const privateKey = decodeSecretSourceValue(privateValue, transport.privateKeySourceEncoding, "gitMirror.githubTransport.privateKeySourceKey"); if (!/-----BEGIN [A-Z ]*PRIVATE KEY-----/u.test(privateKey)) throw new Error(`${transport.privateKeySourceRef}.${transport.privateKeySourceKey} does not contain private key material`); let knownHosts: string | null = null; let knownHostsFingerprint: string | null = null; if (transport.knownHostsSourceRef !== null && transport.knownHostsSourceKey !== null && transport.knownHostsSourceEncoding !== null) { const knownHostsSource = readControlPlaneSecretSource(transport.knownHostsSourceRef, `gitMirror.githubTransport known_hosts source ${transport.knownHostsSourceRef} is missing; create the YAML-declared sourceRef with ${transport.knownHostsSourceKey} before applying the control plane`); const knownHostsValue = requiredEnvValue(knownHostsSource.values, transport.knownHostsSourceKey, transport.knownHostsSourceRef); knownHosts = decodeSecretSourceValue(knownHostsValue, transport.knownHostsSourceEncoding, "gitMirror.githubTransport.knownHostsSourceKey"); if (!/(^|\n)(github\.com|\[github\.com\]:22)\s+(ssh-ed25519|ssh-rsa|ecdsa-sha2-nistp256)\s+/u.test(knownHosts)) { throw new Error(`${transport.knownHostsSourceRef}.${transport.knownHostsSourceKey} must contain github.com known_hosts rows`); } knownHostsFingerprint = fingerprintSecretValues({ knownHosts }, ["knownHosts"]); } return { privateKey: privateKey.endsWith("\n") ? privateKey : `${privateKey}\n`, knownHosts: knownHosts === null ? null : (knownHosts.endsWith("\n") ? knownHosts : `${knownHosts}\n`), privateKeyFingerprint: fingerprintSecretValues({ privateKey }, ["privateKey"]), knownHostsFingerprint, }; } function readControlPlaneSecretSource(sourceRef: string, missingMessage: string): ReturnType { return readEnvSourceFile({ root: rootPath(".state", "secrets"), sourceRef, missingMessage: () => missingMessage, }); } function decodeSecretSourceValue(value: string, encoding: "plain" | "base64", path: string): string { if (encoding === "plain") return value; try { const compact = value.replace(/\s+/gu, ""); if (compact.length === 0) throw new Error("empty base64 value"); return Buffer.from(compact, "base64").toString("utf8"); } catch (error) { throw new Error(`${path} must be valid base64: ${error instanceof Error ? error.message : String(error)}`); } } function gitMirrorGithubTokenSecret(target: ControlPlaneTargetSpec, labels: Record): Record | null { const transport = target.gitMirror.githubTransport; if (transport.mode !== "https") return null; const token = gitMirrorGithubHttpsToken(transport); return { apiVersion: "v1", kind: "Secret", metadata: { name: transport.tokenSecretName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" }, annotations: { "unidesk.ai/source-ref": transport.tokenSourceRef, "unidesk.ai/source-key": transport.tokenSourceKey, "unidesk.ai/target-key": transport.tokenSecretKey, "unidesk.ai/fingerprint": token.fingerprint, }, }, type: "Opaque", stringData: { [transport.tokenSecretKey]: token.value }, }; } function gitMirrorGithubHttpsToken(transport: Extract): { value: string; fingerprint: string } { const absolute = transport.tokenSourceRef.startsWith("/"); const source = readEnvSourceFile({ root: absolute ? "/" : rootPath("."), sourceRef: absolute ? transport.tokenSourceRef.slice(1) : transport.tokenSourceRef, missingMessage: () => `gitMirror.githubTransport token source ${transport.tokenSourceRef} is missing; create the YAML-declared sourceRef with ${transport.tokenSourceKey} before applying the control plane`, }); const value = requiredEnvValue(source.values, transport.tokenSourceKey, transport.tokenSourceRef); return { value, fingerprint: fingerprintSecretValues({ [transport.tokenSourceKey]: value }, [transport.tokenSourceKey]), }; } function service(name: string, namespace: string, labels: Record, port: number): Record { return { apiVersion: "v1", kind: "Service", metadata: { name, namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": name }, ports: [{ name: "http", port, targetPort: "http" }] }, }; } function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { return sha256Short(JSON.stringify({ repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], ports: { read: target.gitMirror.readContainerPort, write: target.gitMirror.writeContainerPort }, githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport), runtimeProxy: runtimeHostProxyConfig(node, gitMirrorRuntimeProxySpec(node, target)), server: gitMirrorServerJs(), status: gitMirrorStatusShell(), sync: gitMirrorSyncShell(node, target), flush: gitMirrorFlushShell(node, target), })); } function gitMirrorDeployment(name: string, namespace: string, labels: Record, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, mode: "read" | "write"): Record { const runtimeProxy = gitMirrorRuntimeProxySpec(node, target); const containerPort = mode === "read" ? target.gitMirror.readContainerPort : target.gitMirror.writeContainerPort; const podSpec: Record = { containers: [{ name: "git-mirror", image: target.tekton.toolsImage.output, command: ["node", "/etc/git-mirror/server.js"], env: [ { name: "PORT", value: String(containerPort) }, { name: "GIT_PROJECT_ROOT", value: "/cache" }, { name: "GIT_MIRROR_MODE", value: mode }, ...runtimeHostProxyEnv(node, runtimeProxy), ], ports: [{ name: "http", containerPort }], volumeMounts: [{ name: "cache", mountPath: "/cache" }, { name: "config", mountPath: "/etc/git-mirror" }], }], volumes: [ { name: "cache", ...(target.gitMirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: target.gitMirror.cachePvcName } } : { hostPath: { path: target.gitMirror.cacheHostPath, type: "DirectoryOrCreate" } }) }, { name: "config", configMap: { name: target.gitMirror.syncConfigMapName, defaultMode: 0o755 } }, ], }; if (runtimeProxy.enabled && runtimeProxy.hostNetwork) { podSpec.hostNetwork = true; podSpec.dnsPolicy = "ClusterFirstWithHostNet"; } return { apiVersion: "apps/v1", kind: "Deployment", metadata: { name, namespace, labels: { ...labels, "app.kubernetes.io/name": name, "hwlab.pikastech.local/git-mirror-mode": mode } }, spec: { replicas: target.gitMirror.deploymentReplicas, selector: { matchLabels: { "app.kubernetes.io/name": name } }, ...(runtimeProxy.enabled && runtimeProxy.hostNetwork ? { strategy: { type: "RollingUpdate", rollingUpdate: { maxSurge: 0, maxUnavailable: 1 } } } : {}), template: { metadata: { labels: { ...labels, "app.kubernetes.io/name": name, "hwlab.pikastech.local/git-mirror-mode": mode }, annotations: { "checksum/config": gitMirrorConfigHash(node, target) }, }, spec: podSpec, }, }, }; } function gitMirrorServerJs(): string { return String.raw`const http = require('node:http'); const { spawn } = require('node:child_process'); const projectRoot = process.env.GIT_PROJECT_ROOT || '/cache'; const port = Number.parseInt(process.env.PORT || '8080', 10); function sendHealth(res) { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ ok: true, mode: process.env.GIT_MIRROR_MODE || null, projectRoot })); } function parseHeaders(buffer) { const crlf = buffer.indexOf('\r\n\r\n'); const lf = buffer.indexOf('\n\n'); const headerEnd = crlf >= 0 ? crlf + 4 : (lf >= 0 ? lf + 2 : -1); if (headerEnd < 0) return null; const headerText = buffer.slice(0, headerEnd).toString('latin1').trim(); const rest = buffer.slice(headerEnd); const headers = {}; let status = 200; for (const line of headerText.split(/\r?\n/u)) { const index = line.indexOf(':'); if (index < 0) continue; const key = line.slice(0, index).trim(); const value = line.slice(index + 1).trim(); if (key.toLowerCase() === 'status') { const parsed = Number.parseInt(value.split(' ')[0] || '', 10); if (Number.isInteger(parsed)) status = parsed; } else { headers[key] = value; } } return { status, headers, rest }; } function cgiHeaderEnv(req) { const env = {}; for (const [key, value] of Object.entries(req.headers)) { if (value === undefined) continue; const joined = Array.isArray(value) ? value.join(', ') : String(value); const normalized = key.toUpperCase().replace(/-/g, '_'); if (normalized === 'CONTENT_TYPE') env.CONTENT_TYPE = joined; else if (normalized === 'CONTENT_LENGTH') env.CONTENT_LENGTH = joined; else env['HTTP_' + normalized] = joined; } return env; } function handleGit(req, res) { const url = new URL(req.url || '/', 'http://git-mirror.local'); const env = { ...process.env, ...cgiHeaderEnv(req), GIT_PROJECT_ROOT: projectRoot, GIT_HTTP_EXPORT_ALL: '1', PATH_INFO: decodeURIComponent(url.pathname), REQUEST_METHOD: req.method || 'GET', QUERY_STRING: url.search.slice(1), CONTENT_TYPE: req.headers['content-type'] || '', CONTENT_LENGTH: req.headers['content-length'] || '', REMOTE_USER: 'git', }; const child = spawn('git', ['http-backend'], { env }); let pending = Buffer.alloc(0); let headersSent = false; child.stderr.on('data', (chunk) => process.stderr.write(chunk)); child.on('error', (error) => { if (!headersSent) { res.writeHead(500, { 'content-type': 'text/plain' }); headersSent = true; } res.end(String(error && error.message || error)); }); child.stdout.on('data', (chunk) => { if (headersSent) { res.write(chunk); return; } pending = Buffer.concat([pending, chunk]); const parsed = parseHeaders(pending); if (!parsed) return; headersSent = true; res.writeHead(parsed.status, parsed.headers); if (parsed.rest.length) res.write(parsed.rest); }); child.on('close', (code) => { if (!headersSent) { res.writeHead(code === 0 ? 200 : 500, { 'content-type': 'text/plain' }); headersSent = true; if (pending.length) res.write(pending); } res.end(); }); req.pipe(child.stdin); } http.createServer((req, res) => { if ((req.url || '').startsWith('/healthz')) return sendHealth(res); return handleGit(req, res); }).listen(port, '0.0.0.0', () => { console.log(JSON.stringify({ event: 'git-mirror-http-started', port, projectRoot, mode: process.env.GIT_MIRROR_MODE || null })); }); `; } function gitMirrorStatusShell(): string { return String.raw`#!/bin/sh set -eu node <<'NODE' const { execFileSync } = require('node:child_process'); const { readFileSync, existsSync } = require('node:fs'); const repositories = JSON.parse(readFileSync('/etc/git-mirror/repositories.json', 'utf8')); function readJson(path) { try { return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : null; } catch { return null; } } function rev(repo, ref) { try { return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } } const items = {}; for (const spec of repositories) { const repoPath = '/cache/' + spec.repository + '.git'; const localSource = rev(repoPath, 'refs/heads/' + spec.sourceBranch); const githubSource = rev(repoPath, 'refs/mirror-stage/heads/' + spec.sourceBranch); const localGitops = rev(repoPath, 'refs/heads/' + spec.gitopsBranch); const githubGitops = rev(repoPath, 'refs/mirror-stage/heads/' + spec.gitopsBranch); items[spec.key] = { repository: spec.repository, sourceBranch: spec.sourceBranch, localSource, githubSource, gitopsBranch: spec.gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush: Boolean(localGitops && (!githubGitops || localGitops !== githubGitops)), }; } const first = items[repositories[0]?.key] || {}; const pendingFlush = Object.values(items).some((item) => Boolean(item.pendingFlush)); console.log(JSON.stringify({ localSource: first.localSource || null, githubSource: first.githubSource || null, localGitops: first.localGitops || null, githubGitops: first.githubGitops || null, refSources: { localSource: 'refs/heads/' + (first.sourceBranch || ''), githubSource: 'refs/mirror-stage/heads/' + (first.sourceBranch || ''), localGitops: 'refs/heads/' + (first.gitopsBranch || ''), githubGitops: 'refs/mirror-stage/heads/' + (first.gitopsBranch || ''), githubFieldsAreMirrorStageCache: true }, pendingFlush, flushNeeded: pendingFlush, githubInSync: Object.values(items).every((item) => item.sourceInSync === true && item.gitopsInSync === true), repositories: items, lastSync: readJson('/cache/HWLAB.last-sync.json'), lastFlush: readJson('/cache/HWLAB.last-flush.json'), })); NODE `; } function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { const gitMirrorProxy = target.gitMirror.egressProxy; const transport = target.gitMirror.githubTransport; const useDirect = gitMirrorProxy?.mode === "direct"; const proxyRequired = gitMirrorProxy?.required === true; if (!useDirect && gitMirrorProxy?.mode !== "node-global" && gitMirrorProxy?.mode !== "host-route") throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode must be node-global, host-route, or direct`); if (!useDirect && proxyRequired && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is required by targets.${target.id}.gitMirror.egressProxy`); if (!useDirect && node.egressProxy === null) throw new Error(`nodes.${node.id}.egressProxy is missing; git-mirror GitHub transport cannot use node-global proxy`); if (!useDirect && gitMirrorProxy?.mode === "node-global" && node.egressProxy?.mode !== "k8s-service-cluster-ip") { throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode=node-global requires nodes.${node.id}.egressProxy.mode=k8s-service-cluster-ip`); } if (!useDirect && gitMirrorProxy?.mode === "host-route" && node.egressProxy?.mode !== "host-route") { throw new Error(`targets.${target.id}.gitMirror.egressProxy.mode=host-route requires nodes.${node.id}.egressProxy.mode=host-route`); } const hostRouteUrl = !useDirect && node.egressProxy?.mode === "host-route" ? new URL(node.egressProxy.proxyUrl) : null; const proxyHost = useDirect || node.egressProxy === null ? "" : node.egressProxy.mode === "host-route" ? hostRouteUrl!.hostname : `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`; const proxyPort = useDirect || node.egressProxy === null ? 0 : node.egressProxy.mode === "host-route" ? Number(hostRouteUrl!.port) : node.egressProxy.port; const proxyUrl = useDirect || node.egressProxy === null ? "" : node.egressProxy.mode === "host-route" ? node.egressProxy.proxyUrl : `http://${proxyHost}:${proxyPort}`; const noProxy = useDirect || node.egressProxy === null ? "" : node.egressProxy.noProxy.join(","); const proxySource = useDirect || node.egressProxy === null ? "sourceType=direct sourceRef=- sourceFingerprint=-" : node.egressProxy.mode === "host-route" ? `sourceType=host-route hostProxyConfigRef=${node.egressProxy.hostProxyConfigRef} sourceFingerprint=-` : `sourceType=${node.egressProxy.sourceType} sourceRef=${node.egressProxy.sourceRef} sourceFingerprint=${node.egressProxy.sourceFingerprint ?? "-"}`; const proxySummary = useDirect ? `git-mirror-egress-proxy mode=direct required=false transport=${transport.mode} source=yaml` : transport.mode === "https" ? `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=${node.egressProxy?.mode} required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=https proxy=HTTP_PROXY authSecret=${transport.tokenSecretName} authKey=${transport.tokenSecretKey} authSourceRef=${transport.tokenSourceRef} ${proxySource} source=yaml` : `git-mirror-egress-proxy client=${node.egressProxy?.clientName} mode=${node.egressProxy?.mode} required=${proxyRequired ? "true" : "false"} host=${proxyHost} port=${proxyPort} transport=ssh ssh=GIT_SSH-wrapper ${proxySource} source=yaml`; const proxyCommand = useDirect ? "" : `ProxyCommand=node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p`; const common = [ `printf '%s\\n' ${shQuote(proxySummary)} >&2`, useDirect ? "unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy" : `export HTTP_PROXY=${shQuote(proxyUrl)}`, useDirect ? "export NO_PROXY='*'" : `export HTTPS_PROXY=${shQuote(proxyUrl)}`, useDirect ? "export no_proxy='*'" : `export ALL_PROXY=${shQuote(proxyUrl)}`, useDirect ? "" : `export http_proxy=${shQuote(proxyUrl)}`, useDirect ? "" : `export https_proxy=${shQuote(proxyUrl)}`, useDirect ? "" : `export all_proxy=${shQuote(proxyUrl)}`, useDirect ? "" : `export NO_PROXY=${shQuote(noProxy)}`, useDirect ? "" : `export no_proxy=${shQuote(noProxy)}`, `repository=${shQuote(target.source.repository)}`, `source_branch=${shQuote(target.source.branch)}`, `gitops_branch=${shQuote(target.gitops.branch)}`, "repo=\"/cache/${repository}.git\"", ].filter(Boolean); const proxyConnectBlock = useDirect ? [] : [ "cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'", "#!/usr/bin/env node", "const net = require('node:net');", "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", "const targetPort = Number.parseInt(targetPortRaw || '', 10);", "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) {", " console.error('hwlab git-mirror proxy-connect: invalid ProxyCommand arguments');", " process.exit(64);", "}", "let settled = false;", "let tunnelEstablished = false;", "function finish(code, message) {", " if (settled) return;", " settled = true;", " if (message) console.error('hwlab git-mirror proxy-connect: ' + message);", " process.exit(code);", "}", "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", "let buffer = Buffer.alloc(0);", "socket.setTimeout(15000, () => { socket.destroy(); finish(65, 'timeout connecting via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); });", "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", "socket.on('error', (error) => finish(tunnelEstablished ? 69 : 66, (tunnelEstablished ? 'tunnel socket error: ' : 'tcp error connecting to proxy: ') + (error && error.message ? error.message : String(error))));", "socket.on('close', () => { if (!tunnelEstablished) finish(68, 'proxy closed before CONNECT completed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); else finish(0); });", "function onData(chunk) {", " buffer = Buffer.concat([buffer, chunk]);", " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", " if (headerEnd === -1 && buffer.length < 8192) return;", " if (headerEnd === -1) { socket.destroy(); finish(68, 'proxy response header exceeded 8192 bytes before CONNECT status via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort); return; }", " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", " const statusLine = head.split('\\r\\n', 1)[0] || '';", " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) {", " const safeStatus = statusLine.replace(/[^\\x20-\\x7e]/g, '?').slice(0, 160);", " socket.destroy();", " finish(67, 'proxy CONNECT failed via ' + proxyHost + ':' + proxyPort + ' to ' + targetHost + ':' + targetPort + ': ' + safeStatus);", " return;", " }", " socket.off('data', onData);", " socket.setTimeout(0);", " tunnelEstablished = true;", " const rest = buffer.slice(headerEnd + 4);", " if (rest.length) process.stdout.write(rest);", " process.stdin.on('error', () => {});", " process.stdout.on('error', () => {});", " process.stdin.pipe(socket);", " socket.pipe(process.stdout);", "}", "socket.on('data', onData);", "NODE_PROXY", "chmod 0700 /tmp/hwlab-github-proxy-connect.cjs", ]; if (transport.mode === "https") { return [ ...common, "if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo 'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env' >&2; exit 64; fi", "cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'", "#!/bin/sh", "case \"$1\" in", " *Username*) printf '%s\\n' \"${GITHUB_USERNAME:-x-access-token}\" ;;", " *Password*) printf '%s\\n' \"$GITHUB_TOKEN\" ;;", " *) printf '\\n' ;;", "esac", "SH_ASKPASS", "chmod 0700 /tmp/hwlab-git-askpass.sh", `export GITHUB_USERNAME=${shQuote(transport.username)}`, "export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh", "export GIT_TERMINAL_PROMPT=0", "unset GIT_SSH", "unset GIT_SSH_COMMAND", "remote=\"https://github.com/${repository}.git\"", ].join("\n"); } const privateKeyPath = transport.mode === "ssh" ? `/git-ssh/${transport.privateKeySecretKey}` : "/git-ssh/ssh-privatekey"; const knownHostsCopy = transport.mode === "ssh" && transport.knownHostsSecretKey !== null ? [`cp ${shQuote(`/git-ssh/${transport.knownHostsSecretKey}`)} /root/.ssh/known_hosts`, "chmod 0600 /root/.ssh/known_hosts"] : []; return [ "mkdir -p /root/.ssh", `cp ${shQuote(privateKeyPath)} /root/.ssh/id_rsa`, "chmod 0400 /root/.ssh/id_rsa", ...knownHostsCopy, ...common, ...proxyConnectBlock, "cat > /tmp/hwlab-git-ssh-proxy.sh <<'SH_PROXY'", "#!/bin/sh", useDirect ? `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 "$@"` : `exec ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ${shQuote(proxyCommand)} "$@"`, "SH_PROXY", "chmod 0700 /tmp/hwlab-git-ssh-proxy.sh", "export GIT_SSH=/tmp/hwlab-git-ssh-proxy.sh", "unset GIT_SSH_COMMAND", "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", ].filter(Boolean).join("\n"); } function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { return [ "#!/bin/sh", "set -eu", "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)", gitMirrorProxyPrelude(node, target), "mkdir -p \"$(dirname \"$repo\")\"", "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", " git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", "else", " rm -rf \"$repo\"", " git init --bare \"$repo\"", " git --git-dir=\"$repo\" remote add origin \"$remote\"", "fi", "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", "git --git-dir=\"$repo\" config http.uploadpack true", "git --git-dir=\"$repo\" config http.receivepack true", "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", "source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", "git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", "discarded_stale_gitops=false", "if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then", " github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", " local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", " if [ -z \"$local_gitops\" ] && [ -n \"$github_gitops\" ]; then", " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", " elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && git --git-dir=\"$repo\" merge-base --is-ancestor \"$local_gitops\" \"$github_gitops\"; then", " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", " elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && [ \"${UNIDESK_GIT_MIRROR_DISCARD_STALE_GITOPS:-false}\" = \"true\" ]; then", " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", " discarded_stale_gitops=true", " printf '%s\\n' \"git-mirror sync: discarded stale local gitops ref local=${local_gitops} github=${github_gitops}\" >&2", " fi", "fi", "git --git-dir=\"$repo\" update-server-info", "export repository source_branch gitops_branch started_at discarded_stale_gitops", "node <<'NODE' | tee /cache/HWLAB.last-sync.json", "const { execFileSync } = require('node:child_process');", "const repository = process.env.repository;", "const sourceBranch = process.env.source_branch;", "const gitopsBranch = process.env.gitops_branch;", "const repoPath = `/cache/${repository}.git`;", "function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }", "const localSource = rev(`refs/heads/${sourceBranch}`);", "const githubSource = rev(`refs/mirror-stage/heads/${sourceBranch}`);", "const localGitops = rev(`refs/heads/${gitopsBranch}`);", "const githubGitops = rev(`refs/mirror-stage/heads/${gitopsBranch}`);", "const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));", "console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), localSource, githubSource, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush, discardedStaleGitops: process.env.discarded_stale_gitops === 'true' }));", "NODE", "cat /cache/HWLAB.last-sync.json", "", ].join("\n"); } function gitMirrorFlushShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { return [ "#!/bin/sh", "set -eu", "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)", gitMirrorProxyPrelude(node, target), "test -d \"$repo/objects\"", "git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", "local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", "push_status=skipped", "push_exit=0", "fetch_status=skipped", "fetch_exit=0", "fetch_attempt=0", "fetch_max_attempts=5", "if [ -n \"$local_gitops\" ]; then", " set +e", " timeout 240 git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"", " push_exit=$?", " set -e", " if [ \"$push_exit\" = \"0\" ]; then", " push_status=succeeded", " fetch_retry_delay=1", " while [ \"$fetch_attempt\" -lt \"$fetch_max_attempts\" ]; do", " fetch_attempt=$((fetch_attempt + 1))", " echo \"git-mirror post-push fetch attempt ${fetch_attempt}/${fetch_max_attempts}\" >&2", " set +e", " timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"", " fetch_exit=$?", " set -e", " if [ \"$fetch_exit\" = \"0\" ]; then fetch_status=succeeded; break; fi", " fetch_status=failed", " if [ \"$fetch_attempt\" -lt \"$fetch_max_attempts\" ]; then", " echo \"git-mirror post-push fetch retry ${fetch_attempt}/${fetch_max_attempts} failed exit=${fetch_exit}; backoff=${fetch_retry_delay}s\" >&2", " sleep \"$fetch_retry_delay\"", " if [ \"$fetch_retry_delay\" -lt 16 ]; then fetch_retry_delay=$((fetch_retry_delay * 2)); fi", " fi", " done", " else", " push_status=failed", " fi", "fi", "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", "status=succeeded", "partial_success=", "degraded_reason=", "exit_code=0", "if [ \"$push_status\" = \"failed\" ]; then", " status=failed", " degraded_reason=git-mirror-push-failed", " exit_code=$push_exit", "elif [ \"$push_status\" = \"succeeded\" ] && [ \"$fetch_status\" = \"failed\" ]; then", " status=partial-success", " partial_success=push-succeeded-fetch-failed", " degraded_reason=git-mirror-post-push-fetch-failed", " exit_code=44", "fi", "export repository gitops_branch started_at local_gitops github_gitops pending push_status push_exit fetch_status fetch_exit fetch_attempt fetch_max_attempts status partial_success degraded_reason", "node <<'NODE' | tee /cache/HWLAB.last-flush.json", "const payload = { event: 'git-mirror-flush', repo: process.env.repository, status: process.env.status || 'failed', partialSuccess: process.env.partial_success || null, degradedReason: process.env.degraded_reason || null, startedAt: process.env.started_at, flushedAt: new Date().toISOString(), gitopsBranch: process.env.gitops_branch, localGitops: process.env.local_gitops || null, githubGitops: process.env.github_gitops || null, pendingFlush: process.env.pending === 'true', stages: { push: process.env.push_status || null, pushExitCode: Number.parseInt(process.env.push_exit || '0', 10), postPushFetch: process.env.fetch_status || null, postPushFetchExitCode: Number.parseInt(process.env.fetch_exit || '0', 10), postPushFetchAttempts: Number.parseInt(process.env.fetch_attempt || '0', 10), postPushFetchMaxAttempts: Number.parseInt(process.env.fetch_max_attempts || '0', 10) } };", "console.log(JSON.stringify(payload));", "NODE", "cat /cache/HWLAB.last-flush.json", "if [ \"$exit_code\" != \"0\" ]; then exit \"$exit_code\"; fi", "", ].join("\n"); } function validateBenchmarkProfileName(value: string, path: string): void { if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 63) throw new Error(`${path} must be a DNS-label style benchmark profile`); } function validateBenchmarkCatalogPathTemplate(value: string, path: string): void { if (!value.includes("{profile}") || !value.includes("{pipelineRun}")) throw new Error(`${path} must include {profile} and {pipelineRun}`); if (value.startsWith("/") || value.includes("\n") || value.includes("\r")) throw new Error(`${path} must be a relative repo path template`); const rendered = value.replace(/\{profile\}/gu, "profile").replace(/\{pipelineRun\}/gu, "pipeline-run"); if (rendered.split("/").some((segment) => segment === ".." || segment.length === 0)) throw new Error(`${path} must not contain empty or parent path segments`); if (!rendered.endsWith(".json")) throw new Error(`${path} must render to a JSON catalog path`); } function asRecord(value: unknown, path: string): Record { if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`); return value as Record; } function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function renderTable(headers: string[], rows: string[][]): string[] { const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0))); const render = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd(); return [render(headers), ...rows.map(render)]; } function renderCell(value: unknown, fallback = "-"): string { if (value === undefined || value === null || value === "") return fallback; return String(value); } function optionsModeFromCommand(command: unknown): string { const value = String(command ?? ""); if (value.endsWith(" status") || value.endsWith(" logs")) return "status"; return "benchmark"; } function stringField(obj: Record, key: string, path: string): string { const value = obj[key]; if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return value; } function optionalStringField(obj: Record, key: string, path: string): string | undefined { const value = obj[key]; if (value === undefined) return undefined; if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return value; } function nullableStringField(obj: Record, key: string, path: string): string | null { const value = obj[key]; if (value === undefined || value === null) return null; if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string when set`); return value; } function absoluteConfigPathField(obj: Record, key: string, path: string): string { const value = stringField(obj, key, path); if (!value.startsWith("/") || value.includes("\0") || value.includes("..")) throw new Error(`${path}.${key} must be an absolute path without '..'`); return value; } function validateKubernetesName(value: string, path: string): void { if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 253) throw new Error(`${path} must be a Kubernetes resource name`); } function validateSecretKey(value: string, path: string): void { if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a Kubernetes Secret key`); } function validateEnvKey(value: string, path: string): void { if (!/^[A-Z0-9_]+$/u.test(value)) throw new Error(`${path} must be an env key`); } function validateSourceRef(value: string, path: string): void { if (!/^[A-Za-z0-9_./-]+$/u.test(value) || value.includes("..")) throw new Error(`${path} has an unsupported sourceRef format`); } function numberField(obj: Record, key: string, path: string): number { const value = obj[key]; if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`); return value; } function positiveConfigIntegerField(obj: Record, key: string, path: string): number { const value = obj[key]; if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`); return value; } function nonNegativeIntegerField(obj: Record, key: string, path: string): number { const value = obj[key]; if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`); return value; } function numberArrayField(obj: Record, key: string, path: string): number[] { const value = obj[key]; if (!Array.isArray(value) || value.some((item) => typeof item !== "number" || !Number.isInteger(item))) throw new Error(`${path}.${key} must be an array of integers`); return [...value] as number[]; } function stringArrayField(obj: Record, key: string, path: string): string[] { const value = obj[key]; if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) throw new Error(`${path}.${key} must be an array of non-empty strings`); return [...value] as string[]; } function stringRecordField(obj: Record, path: string): Record { const result: Record = {}; for (const [key, value] of Object.entries(obj)) { if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) throw new Error(`${path}.${key} has an unsupported key format`); if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); result[key] = value; } return result; } function booleanField(obj: Record, key: string, path: string): boolean { const value = obj[key]; if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); return value; } function boolField(obj: Record, key: string): boolean { return obj[key] === true; } function numberValue(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } function requiredOption(args: string[], name: string): string { const index = args.indexOf(name); if (index === -1) throw new Error(`${name} is required`); const value = args[index + 1]; if (value === undefined || value.startsWith("--") || value.length === 0) throw new Error(`${name} requires a value`); return value; } function optionValue(args: string[], name: string): string | null { const index = args.indexOf(name); if (index === -1) return null; const value = args[index + 1]; if (value === undefined || value.startsWith("--") || value.length === 0) throw new Error(`${name} requires a value`); return value; } function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const raw = optionValue(args, name); if (raw === null) return defaultValue; const value = Number(raw); if (!Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`); return Math.min(value, maxValue); } function compactCommandResult(result: CommandResult): Record { return { exitCode: result.exitCode, timedOut: result.timedOut, stdoutBytes: Buffer.byteLength(result.stdout), stderrBytes: Buffer.byteLength(result.stderr), stdoutTail: result.stdout.slice(-2000), stderrTail: result.stderr.slice(-2000), }; } function shQuote(value: string): string { return `'${value.replace(/'/gu, `'"'"'`)}'`; } function validateHttpsUrl(value: string, path: string): void { let parsed: URL; try { parsed = new URL(value); } catch { throw new Error(`${path} must be a valid URL`); } if (parsed.protocol !== "https:") throw new Error(`${path} must use https://`); } function hostRouteProxyHostAllowed(value: string): boolean { if (value === "127.0.0.1" || value === "localhost") return true; const parts = value.split(".").map((part) => Number(part)); if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false; return parts[0] === 10 || (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) || (parts[0] === 192 && parts[1] === 168); } function sha256Short(text: string): string { return `sha256:${createHash("sha256").update(text).digest("hex")}`; }