import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; import { rootPath } from "./config"; import { runCommand, type CommandResult } from "./command"; export const HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH = "config/hwlab-node-control-plane.yaml"; type InfraAction = "plan" | "status" | "apply"; type ToolsImageAction = "status" | "build" | "logs"; type ArgoAction = "status" | "apply" | "logs"; interface InfraOptions { action: InfraAction; node: string; lane: string; dryRun: boolean; confirm: boolean; timeoutSeconds: number; } interface ToolsImageOptions { action: ToolsImageAction; node: string; lane: string; dryRun: boolean; confirm: boolean; timeoutSeconds: number; tailLines: number; } interface ArgoOptions { action: ArgoAction; node: string; lane: string; dryRun: boolean; confirm: boolean; timeoutSeconds: number; tailLines: number; } interface ControlPlaneEgressProxySpec { mode: "k8s-service-cluster-ip"; namespace: string; serviceName: string; port: number; noProxy: readonly string[]; } interface ControlPlaneNodeSpec { id: string; route: string; kubeRoute: string; k3s: ControlPlaneK3sNodeSpec | null; registry: { endpoint: string }; egressProxy: ControlPlaneEgressProxySpec | null; } interface ControlPlaneK3sNodeSpec { serviceName: string; dropInPath: string; nodeStatusName: string; execStartPre: readonly (readonly string[])[]; serverArgs: readonly string[]; kubelet: { maxPods: number }; } interface DockerfileInlineSpec { filename: string; lines: readonly string[]; } interface ImageRewriteSpec { source: string; pullImage: string; target: string; } interface ControlPlaneTargetSpec { id: string; node: string; lane: string; enabled: boolean; ciNamespace: string; runtimeNamespace: string; source: { repository: string; branch: string }; gitops: { branch: string; path: string }; gitMirror: { namespace: string; serviceReadName: string; serviceWriteName: string; cachePvcName: string; cachePvcStorage: string; cacheHostPath: string | null; servicePort: number; deploymentReplicas: number; secretName: string; syncConfigMapName: string; syncJobPrefix: string; flushJobPrefix: string; readUrl: string; writeUrl: string; }; tekton: { pipelineName: string; serviceAccountName: string; pipelineRunPrefix: string; toolsImage: { output: string; sourceKind: "dockerfile" | "docker-compose"; context: string; dockerfile?: string; dockerfileInline?: DockerfileInlineSpec; composeFile?: string; buildArgs: Readonly>; buildNetwork: string | null; publicBaseImages: readonly string[]; buildOwner: string; buildMode: string; }; }; argo: { namespace: string; projectName: string; applicationName: string; applicationFile: string; install: { enabled: boolean; sourceKind: "url"; version: string; manifestUrl: string; fieldManager: string; imagePullPolicy: "Always" | "IfNotPresent" | "Never"; preloadImages: readonly string[]; imageRewrites: readonly ImageRewriteSpec[]; requiredCrds: readonly string[]; expectedDeployments: readonly string[]; expectedStatefulSets: readonly string[]; readinessTimeoutSeconds: number; }; }; } interface ControlPlaneImagePolicy { requireReproducibleBuildSource: boolean; forbidPrivateOrNodeLocalImagesAsInputs: boolean; allowNodeLocalRegistryAsBuildOutput: boolean; requiredSourceKinds: readonly ("dockerfile" | "docker-compose")[]; } interface ControlPlaneConfig { version: number; kind: string; metadata: { owner: string; relatedIssues: readonly number[] }; imagePolicy: ControlPlaneImagePolicy; nodes: Record; targets: readonly ControlPlaneTargetSpec[]; } export function runHwlabNodeControlPlaneInfra(args: string[]): Record { if (args[0] === "tools-image") { const options = parseToolsImageOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); return runToolsImageCommand(config, node, target, options); } if (args[0] === "argo") { const options = parseArgoOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); return runArgoCommand(config, node, target, options); } const options = parseInfraOptions(args); const { config, node, target } = controlPlaneContext(options.node, options.lane); if (options.action === "plan") return infraPlan(config, node, target, options); if (options.action === "status") return infraStatus(config, node, target, options); return infraApply(config, node, target, options); } function controlPlaneContext(nodeId: string, lane: string): { config: ControlPlaneConfig; node: ControlPlaneNodeSpec; target: ControlPlaneTargetSpec } { const config = readControlPlaneConfig(); const node = config.nodes[nodeId]; if (node === undefined) throw new Error(`unknown node ${nodeId}; known nodes: ${Object.keys(config.nodes).join(", ")}`); const target = config.targets.find((item) => item.node === nodeId && item.lane === lane); if (target === undefined) throw new Error(`no control-plane target for node=${nodeId} lane=${lane} in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); if (!target.enabled) throw new Error(`control-plane target ${target.id} is disabled in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); return { config, node, target }; } export function hwlabNodeControlPlaneInfraHelp(): Record { return { ok: true, command: "hwlab nodes control-plane infra", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, description: "Plan/status/apply YAML-controlled HWLAB node-local k3s, CI/CD and git-mirror control-plane prerequisites. Cross-node PK01/Caddy/FRP/runtime rollout remains explicit semi-automatic CLI work.", usage: [ "bun scripts/cli.ts hwlab nodes control-plane infra plan --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra apply --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node D601 --lane v03", ], g14Consistency: "D601 target fields mirror the existing G14 runtime lane control-plane vocabulary: source branch, gitops branch/path, Pipeline, PipelineRun prefix, ServiceAccount, Argo Application, and git-mirror read/write/sync/flush status concepts.", }; } function infraPlan(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: InfraOptions): Record { return { ok: true, command: "hwlab nodes control-plane infra plan", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "plan", mutation: false, target: planSummary(node, target), expected: expectedSummary(node, target), hostConfig: k3sNodeConfigPlan(node), imagePolicy: _config.imagePolicy, g14Consistency: { laneVocabulary: ["sourceBranch", "gitopsBranch", "catalogPath", "runtime.path", "runtime.namespace", "tekton.pipeline", "pipelineRunPrefix", "argo.application"], gitMirrorStatusVocabulary: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"], note: "D601 values differ only through YAML target fields; the control-plane model is intentionally aligned with G14 runtime lane semantics.", }, resources: manifestObjectSummary(renderInfraManifest(node, target)), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, dryRun: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --dry-run`, apply: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`, toolsImageBuild: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node ${node.id} --lane ${target.lane} --confirm`, argoApply: `bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node ${node.id} --lane ${target.lane} --confirm`, }, options: { timeoutSeconds: options.timeoutSeconds }, }; } function infraStatus(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: InfraOptions): Record { const script = statusScript(node, target); const result = runTransK3s(node.kubeRoute, script, options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const status = typeof parsed === "object" && parsed !== null ? parsed as Record : { parseError: "remote status did not return a JSON object", stdoutPreview: result.stdout.slice(0, 1000) }; const components = record(status.components); const argo = record(components.argo); const argoInstall = record(argo.install); const gitMirror = record(components.gitMirror); const tekton = record(components.tekton); const ciNamespace = record(components.ciNamespace); const registry = record(components.registry); const k3sNodeConfig = record(components.k3sNodeConfig); const k3sNodeConfigReady = node.k3s === null || (boolField(k3sNodeConfig, "dropInMatches") && numberValue(k3sNodeConfig.liveCapacityPods) === node.k3s.kubelet.maxPods && numberValue(k3sNodeConfig.liveAllocatablePods) === node.k3s.kubelet.maxPods); const ok = result.exitCode === 0 && k3sNodeConfigReady && boolField(tekton, "installed") && boolField(ciNamespace, "exists") && boolField(gitMirror, "namespaceExists") && boolField(gitMirror, "readServiceExists") && boolField(gitMirror, "writeServiceExists") && (boolField(gitMirror, "cachePvcExists") || boolField(gitMirror, "cacheHostPathReady")) && boolField(registry, "ready") && boolField(registry, "toolsImageReady") && boolField(argo, "installed") && boolField(argo, "projectExists") && boolField(argo, "applicationExists") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") && boolField(argoInstall, "statefulSetsReady"); return { ok, command: "hwlab nodes control-plane infra status", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "status", mutation: false, expected: expectedSummary(node, target), status, readiness: { ok, k3sNodeConfigReady, tektonInstalled: boolField(tekton, "installed"), ciNamespaceExists: boolField(ciNamespace, "exists"), gitMirrorNamespaceExists: boolField(gitMirror, "namespaceExists"), gitMirrorReadServiceExists: boolField(gitMirror, "readServiceExists"), gitMirrorWriteServiceExists: boolField(gitMirror, "writeServiceExists"), gitMirrorCachePvcExists: boolField(gitMirror, "cachePvcExists"), gitMirrorCacheHostPathReady: boolField(gitMirror, "cacheHostPathReady"), gitMirrorReadReady: boolField(gitMirror, "readDeploymentReady"), gitMirrorWriteReady: boolField(gitMirror, "writeDeploymentReady"), argoInstalled: boolField(argo, "installed"), argoProjectExists: boolField(argo, "projectExists"), argoApplicationExists: boolField(argo, "applicationExists"), argoCrdsReady: boolField(argoInstall, "crdsReady"), argoDeploymentsReady: boolField(argoInstall, "deploymentsReady"), argoStatefulSetsReady: boolField(argoInstall, "statefulSetsReady"), registryReady: boolField(registry, "ready"), toolsImageReady: boolField(registry, "toolsImageReady"), }, result: compactCommandResult(result), next: ok ? { runtimePreparation: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${node.id} --lane ${target.lane}` } : statusNext(node, target, registry, gitMirror, argo, ciNamespace, k3sNodeConfig), }; } function infraApply(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: InfraOptions): Record { if (options.confirm && options.dryRun) throw new Error("infra apply accepts only one of --dry-run or --confirm"); const dryRun = options.dryRun || !options.confirm; const manifest = renderInfraManifest(node, target); const yaml = `${manifest.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; const imageStatus = toolsImageStatus(node, target, options.timeoutSeconds); if (dryRun) { return { ok: true, command: "hwlab nodes control-plane infra apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "dry-run", mutation: false, expected: expectedSummary(node, target), hostConfig: k3sNodeConfigPlan(node), preflight: { registryReady: imageStatus.registryReady, toolsImageReady: imageStatus.toolsImageReady, toolsImage: target.tekton.toolsImage.output, result: imageStatus.result, }, resources: manifestObjectSummary(manifest), manifest: { objects: manifest.length, bytes: Buffer.byteLength(yaml), sha256: sha256Short(yaml) }, note: "dry-run renders D601 node-local control-plane bootstrap resources only; it does not trigger HWLAB runtime rollout and does not touch PK01/Caddy/FRP.", next: applyNext(node, target, imageStatus), }; } const script = applyScript(yaml, node, target); const result = runTransK3s(node.kubeRoute, script, options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { ok: result.exitCode === 0, command: "hwlab nodes control-plane infra apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "confirmed-apply", mutation: result.exitCode === 0, expected: expectedSummary(node, target), preflight: { registryReady: imageStatus.registryReady, toolsImageReady: imageStatus.toolsImageReady, toolsImage: target.tekton.toolsImage.output, warning: imageStatus.toolsImageReady ? null : "tools-image-missing; bootstrap objects were applied but readiness still requires a controlled image build/publish stage", result: imageStatus.result, }, resources: manifestObjectSummary(manifest), apply: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}` }, }; } function runToolsImageCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ToolsImageOptions): Record { if (options.action === "status") return toolsImageCommandStatus(node, target, options); if (options.action === "logs") return remoteJobLogs(node, target, "tools-image", options); return toolsImageBuild(node, target, options); } function runArgoCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ArgoOptions): Record { if (options.action === "status") return argoCommandStatus(node, target, options); if (options.action === "logs") return remoteJobLogs(node, target, "argo", options); return argoApply(node, target, options); } function toolsImageCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ToolsImageOptions): Record { const registry = toolsImageStatus(node, target, options.timeoutSeconds); const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "tools-image", options.tailLines), options.timeoutSeconds); const jobStatus = parseRemoteJson(jobResult.stdout); const ok = registry.registryReady && registry.toolsImageReady; return { ok, command: "hwlab nodes control-plane infra tools-image status", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mutation: false, image: target.tekton.toolsImage.output, imageSource: target.tekton.toolsImage, registry, job: typeof jobStatus === "object" && jobStatus !== null ? jobStatus : { parseError: "remote job status did not return JSON", stdoutPreview: jobResult.stdout.slice(0, 1000) }, result: compactCommandResult(jobResult), next: ok ? { infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}` } : { build: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node ${node.id} --lane ${target.lane} --confirm`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node ${node.id} --lane ${target.lane}` }, }; } function toolsImageBuild(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ToolsImageOptions): Record { if (options.confirm && options.dryRun) throw new Error("tools-image build accepts only one of --dry-run or --confirm"); const dryRun = options.dryRun || !options.confirm; const dockerfile = toolsImageDockerfile(target); const buildPlan = { outputImage: target.tekton.toolsImage.output, sourceKind: target.tekton.toolsImage.sourceKind, dockerfileInline: target.tekton.toolsImage.dockerfileInline, buildArgs: target.tekton.toolsImage.buildArgs, buildNetwork: target.tekton.toolsImage.buildNetwork, publicBaseImages: target.tekton.toolsImage.publicBaseImages, nodeLocalRegistryOutputOnly: true, egressProxy: node.egressProxy, stateDir: remoteJobStateDir(target, "tools-image"), }; if (dryRun) { return { ok: true, command: "hwlab nodes control-plane infra tools-image build", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "dry-run", mutation: false, buildPlan, dockerfile: { bytes: Buffer.byteLength(dockerfile), sha256: sha256Short(dockerfile), preview: dockerfile }, next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node ${node.id} --lane ${target.lane} --confirm` }, }; } const result = runTransK3s(node.kubeRoute, toolsImageBuildStartScript(node, target, dockerfile), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { ok: result.exitCode === 0, command: "hwlab nodes control-plane infra tools-image build", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "confirmed-start", mutation: result.exitCode === 0, buildPlan, start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node ${node.id} --lane ${target.lane}`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node ${node.id} --lane ${target.lane}`, infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, }, }; } function argoCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ArgoOptions): Record { const result = runTransK3s(node.kubeRoute, statusScript(node, target), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; const argo = record(record(status.components).argo); const argoInstall = record(argo.install); const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "argo", options.tailLines), options.timeoutSeconds); const jobStatus = parseRemoteJson(jobResult.stdout); const ok = boolField(argo, "installed") && boolField(argo, "projectExists") && boolField(argo, "applicationExists") && boolField(argoInstall, "crdsReady") && boolField(argoInstall, "deploymentsReady") && boolField(argoInstall, "statefulSetsReady"); return { ok, command: "hwlab nodes control-plane infra argo status", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mutation: false, expected: { namespace: target.argo.namespace, project: target.argo.projectName, application: target.argo.applicationName, install: target.argo.install, }, readiness: { installed: boolField(argo, "installed"), projectExists: boolField(argo, "projectExists"), applicationExists: boolField(argo, "applicationExists"), crdsReady: boolField(argoInstall, "crdsReady"), deploymentsReady: boolField(argoInstall, "deploymentsReady"), statefulSetsReady: boolField(argoInstall, "statefulSetsReady"), }, argo, job: typeof jobStatus === "object" && jobStatus !== null ? jobStatus : { parseError: "remote job status did not return JSON", stdoutPreview: jobResult.stdout.slice(0, 1000) }, result: { k3s: compactCommandResult(result), job: compactCommandResult(jobResult) }, next: ok ? { infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}` } : { apply: `bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node ${node.id} --lane ${target.lane} --confirm`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node ${node.id} --lane ${target.lane}` }, }; } function argoApply(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ArgoOptions): Record { if (options.confirm && options.dryRun) throw new Error("argo apply accepts only one of --dry-run or --confirm"); if (!target.argo.install.enabled) throw new Error(`targets.${target.id}.argo.install.enabled=false`); const dryRun = options.dryRun || !options.confirm; const desired = argoDesiredManifest(target); const desiredYaml = `${desired.map((item) => Bun.YAML.stringify(item).trim()).join("\n---\n")}\n`; const applyPlan = { namespace: target.argo.namespace, version: target.argo.install.version, manifestUrl: target.argo.install.manifestUrl, fieldManager: target.argo.install.fieldManager, imageRewrites: target.argo.install.imageRewrites, preloadImages: target.argo.install.preloadImages, requiredCrds: target.argo.install.requiredCrds, desired: manifestObjectSummary(desired), desiredSha256: sha256Short(desiredYaml), stateDir: remoteJobStateDir(target, "argo"), egressProxy: node.egressProxy, }; if (dryRun) { return { ok: true, command: "hwlab nodes control-plane infra argo apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "dry-run", mutation: false, applyPlan, desiredYaml: { objects: desired.length, bytes: Buffer.byteLength(desiredYaml), sha256: sha256Short(desiredYaml), preview: desiredYaml }, next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node ${node.id} --lane ${target.lane} --confirm` }, }; } const result = runTransK3s(node.kubeRoute, argoApplyStartScript(node, target, desiredYaml), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { ok: result.exitCode === 0, command: "hwlab nodes control-plane infra argo apply", configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mode: "confirmed-start", mutation: result.exitCode === 0, applyPlan, start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), next: { status: `bun scripts/cli.ts hwlab nodes control-plane infra argo status --node ${node.id} --lane ${target.lane}`, logs: `bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node ${node.id} --lane ${target.lane}`, infraStatus: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, }, }; } function parseInfraOptions(args: string[]): InfraOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra usage: infra plan|status|apply --node NODE --lane vNN [--dry-run|--confirm]"); if (actionRaw !== "plan" && actionRaw !== "status" && actionRaw !== "apply") throw new Error(`unsupported infra action ${actionRaw}; expected plan|status|apply`); const node = requiredOption(args, "--node"); const lane = requiredOption(args, "--lane"); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("infra accepts only one of --confirm or --dry-run"); return { action: actionRaw, node, lane, confirm, dryRun: actionRaw === "apply" ? explicitDryRun || !confirm : true, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 60), }; } function parseToolsImageOptions(args: string[]): ToolsImageOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra tools-image usage: tools-image status|build|logs --node NODE --lane vNN [--dry-run|--confirm]"); if (actionRaw !== "status" && actionRaw !== "build" && actionRaw !== "logs") throw new Error(`unsupported tools-image action ${actionRaw}; expected status|build|logs`); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("tools-image accepts only one of --confirm or --dry-run"); return { action: actionRaw, node: requiredOption(args, "--node"), lane: requiredOption(args, "--lane"), confirm, dryRun: actionRaw === "build" ? explicitDryRun || !confirm : true, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 60), tailLines: positiveIntegerOption(args, "--tail-lines", 120, 1000), }; } function parseArgoOptions(args: string[]): ArgoOptions { const [actionRaw] = args; if (actionRaw === undefined || actionRaw === "--help" || actionRaw === "-h" || actionRaw === "help") throw new Error("infra argo usage: argo status|apply|logs --node NODE --lane vNN [--dry-run|--confirm]"); if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "logs") throw new Error(`unsupported argo action ${actionRaw}; expected status|apply|logs`); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("argo accepts only one of --confirm or --dry-run"); return { action: actionRaw, node: requiredOption(args, "--node"), lane: requiredOption(args, "--lane"), confirm, dryRun: actionRaw === "apply" ? explicitDryRun || !confirm : true, timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 60, 60), tailLines: positiveIntegerOption(args, "--tail-lines", 120, 1000), }; } function readControlPlaneConfig(): ControlPlaneConfig { const parsed = asRecord(Bun.YAML.parse(readFileSync(rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH), "utf8")) as unknown, HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); const version = numberField(parsed, "version", HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); const kind = stringField(parsed, "kind", HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); if (kind !== "hwlab-node-control-plane") throw new Error(`${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.kind must be hwlab-node-control-plane`); const metadataRaw = asRecord(parsed.metadata, "metadata"); const imagePolicy = imagePolicySpec(asRecord(parsed.imagePolicy, "imagePolicy")); const nodes = Object.fromEntries(Object.entries(asRecord(parsed.nodes, "nodes")).map(([id, raw]) => [id, nodeSpec(id, asRecord(raw, `nodes.${id}`))])); const targetsRaw = parsed.targets; if (!Array.isArray(targetsRaw)) throw new Error(`${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}.targets must be an array`); const targets = targetsRaw.map((raw, index) => targetSpec(asRecord(raw, `targets[${index}]`), index)); for (const target of targets) { if (nodes[target.node] === undefined) throw new Error(`targets.${target.id}.node references missing node ${target.node}`); validateTargetImagePolicy(target, imagePolicy); } return { version, kind, metadata: { owner: stringField(metadataRaw, "owner", "metadata"), relatedIssues: numberArrayField(metadataRaw, "relatedIssues", "metadata"), }, imagePolicy, nodes, targets, }; } function imagePolicySpec(raw: Record): ControlPlaneImagePolicy { const requiredSourceKinds = stringArrayField(raw, "requiredSourceKinds", "imagePolicy").map((item) => { if (item !== "dockerfile" && item !== "docker-compose") throw new Error("imagePolicy.requiredSourceKinds must contain only dockerfile or docker-compose"); return item; }); return { requireReproducibleBuildSource: booleanField(raw, "requireReproducibleBuildSource", "imagePolicy"), forbidPrivateOrNodeLocalImagesAsInputs: booleanField(raw, "forbidPrivateOrNodeLocalImagesAsInputs", "imagePolicy"), allowNodeLocalRegistryAsBuildOutput: booleanField(raw, "allowNodeLocalRegistryAsBuildOutput", "imagePolicy"), requiredSourceKinds, }; } function toolsImageSpec(raw: Record, path: string): ControlPlaneTargetSpec["tekton"]["toolsImage"] { const sourceKind = stringField(raw, "sourceKind", path); if (sourceKind !== "dockerfile" && sourceKind !== "docker-compose") throw new Error(`${path}.sourceKind must be dockerfile or docker-compose`); const publicBaseImages = stringArrayField(raw, "publicBaseImages", path); if (publicBaseImages.length === 0) throw new Error(`${path}.publicBaseImages must list at least one public base image`); for (const image of publicBaseImages) validatePublicBaseImage(image, `${path}.publicBaseImages`); const dockerfile = optionalStringField(raw, "dockerfile", path); const dockerfileInline = raw.dockerfileInline === undefined ? undefined : dockerfileInlineSpec(asRecord(raw.dockerfileInline, `${path}.dockerfileInline`), `${path}.dockerfileInline`); const composeFile = optionalStringField(raw, "composeFile", path); if (sourceKind === "dockerfile" && dockerfile === undefined && dockerfileInline === undefined) throw new Error(`${path}.dockerfile or ${path}.dockerfileInline is required when sourceKind=dockerfile`); if (dockerfile !== undefined && dockerfileInline !== undefined) throw new Error(`${path} must use only one of dockerfile or dockerfileInline`); if (sourceKind === "docker-compose" && composeFile === undefined) throw new Error(`${path}.composeFile is required when sourceKind=docker-compose`); const buildArgsRaw = raw.buildArgs === undefined ? {} : asRecord(raw.buildArgs, `${path}.buildArgs`); const buildArgs = stringRecordField(buildArgsRaw, `${path}.buildArgs`); for (const image of Object.values(buildArgs)) { if (looksLikeImageReference(image)) validatePublicBaseImage(image, `${path}.buildArgs`); } return { output: stringField(raw, "output", path), sourceKind, context: stringField(raw, "context", path), dockerfile, dockerfileInline, composeFile, buildArgs, buildNetwork: optionalStringField(raw, "buildNetwork", path) ?? null, publicBaseImages, buildOwner: stringField(raw, "buildOwner", path), buildMode: stringField(raw, "buildMode", path), }; } function dockerfileInlineSpec(raw: Record, path: string): DockerfileInlineSpec { const filename = stringField(raw, "filename", path); if (!/^[A-Za-z0-9._/-]+$/u.test(filename) || filename.includes("..")) throw new Error(`${path}.filename has an unsupported format`); const lines = stringArrayField(raw, "lines", path); if (lines.length === 0) throw new Error(`${path}.lines must not be empty`); return { filename, lines }; } function argoInstallSpec(raw: Record, path: string): ControlPlaneTargetSpec["argo"]["install"] { const sourceKind = stringField(raw, "sourceKind", path); if (sourceKind !== "url") throw new Error(`${path}.sourceKind must be url`); const imagePullPolicy = optionalStringField(raw, "imagePullPolicy", path) ?? "IfNotPresent"; if (imagePullPolicy !== "Always" && imagePullPolicy !== "IfNotPresent" && imagePullPolicy !== "Never") throw new Error(`${path}.imagePullPolicy must be Always, IfNotPresent, or Never`); const imageRewritesRaw = raw.imageRewrites === undefined ? [] : raw.imageRewrites; if (!Array.isArray(imageRewritesRaw)) throw new Error(`${path}.imageRewrites must be an array`); const imageRewrites = imageRewritesRaw.map((item, index) => imageRewriteSpec(asRecord(item, `${path}.imageRewrites[${index}]`), `${path}.imageRewrites[${index}]`)); const manifestUrl = stringField(raw, "manifestUrl", path); validateHttpsUrl(manifestUrl, `${path}.manifestUrl`); return { enabled: booleanField(raw, "enabled", path), sourceKind, version: stringField(raw, "version", path), manifestUrl, fieldManager: stringField(raw, "fieldManager", path), imagePullPolicy, preloadImages: stringArrayField(raw, "preloadImages", path), imageRewrites, requiredCrds: stringArrayField(raw, "requiredCrds", path), expectedDeployments: stringArrayField(raw, "expectedDeployments", path), expectedStatefulSets: stringArrayField(raw, "expectedStatefulSets", path), readinessTimeoutSeconds: positiveConfigIntegerField(raw, "readinessTimeoutSeconds", path), }; } function imageRewriteSpec(raw: Record, path: string): ImageRewriteSpec { const rewrite = { source: stringField(raw, "source", path), pullImage: stringField(raw, "pullImage", path), target: stringField(raw, "target", path), }; validatePublicBaseImage(rewrite.source, `${path}.source`); validatePublicBaseImage(rewrite.pullImage, `${path}.pullImage`); if (!isNodeLocalImage(rewrite.target)) throw new Error(`${path}.target must use a node-local registry output image`); return rewrite; } function validateTargetImagePolicy(target: ControlPlaneTargetSpec, imagePolicy: ControlPlaneImagePolicy): void { if (imagePolicy.requireReproducibleBuildSource && !imagePolicy.requiredSourceKinds.includes(target.tekton.toolsImage.sourceKind)) { throw new Error(`targets.${target.id}.tekton.toolsImage.sourceKind is not allowed by imagePolicy.requiredSourceKinds`); } if (imagePolicy.forbidPrivateOrNodeLocalImagesAsInputs) { for (const image of target.tekton.toolsImage.publicBaseImages) validatePublicBaseImage(image, `targets.${target.id}.tekton.toolsImage.publicBaseImages`); } if (!imagePolicy.allowNodeLocalRegistryAsBuildOutput && isNodeLocalImage(target.tekton.toolsImage.output)) { throw new Error(`targets.${target.id}.tekton.toolsImage.output uses node-local registry but imagePolicy.allowNodeLocalRegistryAsBuildOutput=false`); } } function validatePublicBaseImage(image: string, path: string): void { if (isNodeLocalImage(image)) { throw new Error(`${path} contains non-public base image ${image}`); } const publicPrefixes = ["docker.io/", "registry.k8s.io/", "ghcr.io/", "quay.io/", "gcr.io/", "public.ecr.aws/", "mcr.microsoft.com/", "cgr.dev/", "docker.m.daocloud.io/", "quay.m.daocloud.io/", "ghcr.m.daocloud.io/"]; if (!publicPrefixes.some((prefix) => image.startsWith(prefix))) throw new Error(`${path} image ${image} must use an explicit public registry prefix`); } function looksLikeImageReference(value: string): boolean { return /^(docker\.io|registry\.k8s\.io|ghcr\.io|quay\.io|gcr\.io|public\.ecr\.aws|mcr\.microsoft\.com|cgr\.dev|docker\.m\.daocloud\.io|quay\.m\.daocloud\.io|ghcr\.m\.daocloud\.io)\//u.test(value) || /^(127\.|localhost(?::|\/)|0\.0\.0\.0|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)/u.test(value); } function isNodeLocalImage(image: string): boolean { return /^(127\.|localhost(?::|\/)|0\.0\.0\.0|10\.|192\.168\.|172\.(1[6-9]|2[0-9]|3[01])\.)/u.test(image); } function nodeSpec(id: string, raw: Record): ControlPlaneNodeSpec { const registry = asRecord(raw.registry, `nodes.${id}.registry`); const egressProxy = raw.egressProxy === undefined ? null : egressProxySpec(asRecord(raw.egressProxy, `nodes.${id}.egressProxy`), `nodes.${id}.egressProxy`); const k3s = raw.k3s === undefined ? null : k3sNodeSpec(asRecord(raw.k3s, `nodes.${id}.k3s`), `nodes.${id}.k3s`); return { id, route: stringField(raw, "route", `nodes.${id}`), kubeRoute: stringField(raw, "kubeRoute", `nodes.${id}`), k3s, registry: { endpoint: stringField(registry, "endpoint", `nodes.${id}.registry`) }, egressProxy, }; } function k3sNodeSpec(raw: Record, path: string): ControlPlaneK3sNodeSpec { const kubelet = asRecord(raw.kubelet, `${path}.kubelet`); const serviceName = stringField(raw, "serviceName", path); if (!/^[A-Za-z0-9_.@-]+$/u.test(serviceName)) throw new Error(`${path}.serviceName has an unsupported systemd unit name`); const dropInPath = stringField(raw, "dropInPath", path); if (!dropInPath.startsWith("/etc/systemd/system/") || !dropInPath.endsWith(".conf") || dropInPath.includes("..")) { throw new Error(`${path}.dropInPath must be an absolute /etc/systemd/system/*.conf path`); } const nodeStatusName = stringField(raw, "nodeStatusName", path); if (!/^[A-Za-z0-9_.-]+$/u.test(nodeStatusName)) throw new Error(`${path}.nodeStatusName has an unsupported Kubernetes node name`); const execStartPre = execStartPreField(raw.execStartPre, `${path}.execStartPre`); const serverArgs = stringArrayField(raw, "serverArgs", path); if (serverArgs.length === 0 || serverArgs[0] !== "server") throw new Error(`${path}.serverArgs must start with k3s server`); for (const [index, arg] of serverArgs.entries()) { if (arg.includes("\n") || arg.includes("\r") || arg.length === 0) throw new Error(`${path}.serverArgs[${index}] must be a single non-empty argv token`); } const maxPods = positiveConfigIntegerField(kubelet, "maxPods", `${path}.kubelet`); const expectedMaxPodsArg = `max-pods=${maxPods}`; let hasExpectedMaxPodsArg = false; for (let index = 0; index < serverArgs.length - 1; index += 1) { if (serverArgs[index] === "--kubelet-arg" && serverArgs[index + 1] === expectedMaxPodsArg) hasExpectedMaxPodsArg = true; } if (!hasExpectedMaxPodsArg) throw new Error(`${path}.serverArgs must include --kubelet-arg ${expectedMaxPodsArg}`); return { serviceName, dropInPath, nodeStatusName, execStartPre, serverArgs, kubelet: { maxPods }, }; } function execStartPreField(raw: unknown, path: string): readonly (readonly string[])[] { if (raw === undefined) return []; if (!Array.isArray(raw)) throw new Error(`${path} must be an array of argv arrays`); return raw.map((item, index) => { if (!Array.isArray(item)) throw new Error(`${path}[${index}] must be an argv array`); const command = item.map((value, tokenIndex) => { if (typeof value !== "string") throw new Error(`${path}[${index}][${tokenIndex}] must be a string`); if (value.length === 0 || value.includes("\n") || value.includes("\r")) throw new Error(`${path}[${index}][${tokenIndex}] must be a single non-empty argv token`); return value; }); if (command.length === 0) throw new Error(`${path}[${index}] must not be empty`); const executable = command[0].startsWith("-") ? command[0].slice(1) : command[0]; if (!executable.startsWith("/") || executable.includes("..")) throw new Error(`${path}[${index}][0] must be an absolute executable path, optionally prefixed with -`); return command; }); } function egressProxySpec(raw: Record, path: string): ControlPlaneEgressProxySpec { const mode = stringField(raw, "mode", path); if (mode !== "k8s-service-cluster-ip") throw new Error(`${path}.mode must be k8s-service-cluster-ip`); return { mode, namespace: stringField(raw, "namespace", path), serviceName: stringField(raw, "serviceName", path), port: positiveConfigIntegerField(raw, "port", path), noProxy: stringArrayField(raw, "noProxy", path), }; } function targetSpec(raw: Record, index: number): ControlPlaneTargetSpec { const path = `targets[${index}]`; const source = asRecord(raw.source, `${path}.source`); const gitops = asRecord(raw.gitops, `${path}.gitops`); const gitMirror = asRecord(raw.gitMirror, `${path}.gitMirror`); const tekton = asRecord(raw.tekton, `${path}.tekton`); const argo = asRecord(raw.argo, `${path}.argo`); const toolsImage = asRecord(tekton.toolsImage, `${path}.tekton.toolsImage`); const node = stringField(raw, "node", path); const lane = stringField(raw, "lane", path); const gitMirrorNamespace = stringField(gitMirror, "namespace", `${path}.gitMirror`); const serviceReadName = stringField(gitMirror, "serviceReadName", `${path}.gitMirror`); const serviceWriteName = stringField(gitMirror, "serviceWriteName", `${path}.gitMirror`); const sourceRepository = stringField(source, "repository", `${path}.source`); return { id: stringField(raw, "id", path), node, lane, enabled: booleanField(raw, "enabled", path), ciNamespace: stringField(raw, "ciNamespace", path), runtimeNamespace: stringField(raw, "runtimeNamespace", path), source: { repository: sourceRepository, branch: stringField(source, "branch", `${path}.source`) }, gitops: { branch: stringField(gitops, "branch", `${path}.gitops`), path: stringField(gitops, "path", `${path}.gitops`) }, gitMirror: { namespace: gitMirrorNamespace, serviceReadName, serviceWriteName, cachePvcName: stringField(gitMirror, "cachePvcName", `${path}.gitMirror`), cachePvcStorage: stringField(gitMirror, "cachePvcStorage", `${path}.gitMirror`), cacheHostPath: optionalStringField(gitMirror, "cacheHostPath", `${path}.gitMirror`) ?? null, servicePort: numberField(gitMirror, "servicePort", `${path}.gitMirror`), deploymentReplicas: nonNegativeIntegerField(gitMirror, "deploymentReplicas", `${path}.gitMirror`), secretName: stringField(gitMirror, "secretName", `${path}.gitMirror`), syncConfigMapName: stringField(gitMirror, "syncConfigMapName", `${path}.gitMirror`), syncJobPrefix: stringField(gitMirror, "syncJobPrefix", `${path}.gitMirror`), flushJobPrefix: stringField(gitMirror, "flushJobPrefix", `${path}.gitMirror`), readUrl: optionalStringField(gitMirror, "readUrl", `${path}.gitMirror`) ?? `http://${serviceReadName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, writeUrl: optionalStringField(gitMirror, "writeUrl", `${path}.gitMirror`) ?? `http://${serviceWriteName}.${gitMirrorNamespace}.svc.cluster.local/${sourceRepository}.git`, }, tekton: { pipelineName: stringField(tekton, "pipelineName", `${path}.tekton`), serviceAccountName: stringField(tekton, "serviceAccountName", `${path}.tekton`), pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${path}.tekton`), toolsImage: toolsImageSpec(toolsImage, `${path}.tekton.toolsImage`), }, argo: { namespace: stringField(argo, "namespace", `${path}.argo`), projectName: stringField(argo, "projectName", `${path}.argo`), applicationName: stringField(argo, "applicationName", `${path}.argo`), applicationFile: stringField(argo, "applicationFile", `${path}.argo`), install: argoInstallSpec(asRecord(argo.install, `${path}.argo.install`), `${path}.argo.install`), }, }; } function renderInfraManifest(_node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record[] { const labels = { "app.kubernetes.io/part-of": "hwlab-node-control-plane", "hwlab.pikastech.local/node": target.node, "hwlab.pikastech.local/lane": target.lane, }; const manifests: Record[] = [ { apiVersion: "v1", kind: "Namespace", metadata: { name: target.ciNamespace, labels } }, ]; if (target.gitMirror.namespace !== target.ciNamespace) { manifests.push({ apiVersion: "v1", kind: "Namespace", metadata: { name: target.gitMirror.namespace, labels } }); } manifests.push( { apiVersion: "v1", kind: "ServiceAccount", metadata: { name: target.tekton.serviceAccountName, namespace: target.ciNamespace, labels } }, { apiVersion: "v1", kind: "ConfigMap", metadata: { name: target.gitMirror.syncConfigMapName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, data: { "repositories.json": JSON.stringify([{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], null, 2), "server.js": gitMirrorServerJs(), "status.sh": gitMirrorStatusShell(), "sync.sh": gitMirrorSyncShell(_node, target), "flush.sh": gitMirrorFlushShell(_node, target), }, }, ); if (target.gitMirror.cacheHostPath === null) { manifests.push({ apiVersion: "v1", kind: "PersistentVolumeClaim", metadata: { name: target.gitMirror.cachePvcName, namespace: target.gitMirror.namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: target.gitMirror.cachePvcStorage } } }, }); } manifests.push( service(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), service(target.gitMirror.serviceWriteName, target.gitMirror.namespace, labels, target.gitMirror.servicePort), gitMirrorDeployment(target.gitMirror.serviceReadName, target.gitMirror.namespace, labels, _node, target, "read"), gitMirrorDeployment(target.gitMirror.serviceWriteName, target.gitMirror.namespace, labels, _node, target, "write"), { apiVersion: "tekton.dev/v1", kind: "Pipeline", metadata: { name: target.tekton.pipelineName, namespace: target.ciNamespace, labels }, spec: { params: [ { name: "source-commit", type: "string" }, { name: "source-branch", type: "string", default: target.source.branch }, { name: "gitops-branch", type: "string", default: target.gitops.branch }, ], tasks: [{ name: "bootstrap-placeholder", taskSpec: { steps: [{ name: "notice", image: target.tekton.toolsImage.output, script: "#!/bin/sh\nset -eu\necho d601-hwlab-v03-pipeline-placeholder\n" }] } }], }, }, { apiVersion: "v1", kind: "Namespace", metadata: { name: target.argo.namespace, labels } }, { apiVersion: "v1", kind: "ConfigMap", metadata: { name: `${target.argo.applicationName}-desired`, namespace: target.argo.namespace, labels }, data: { "project.yaml": Bun.YAML.stringify(argoProjectSkeleton(target)), [target.argo.applicationFile]: Bun.YAML.stringify(argoApplicationSkeleton(target)), "note.txt": "Argo CD CRDs/controller are installed by the node control-plane bootstrap path when available; this ConfigMap preserves the desired Application until Argo is ready.\n", }, }, ); return manifests; } function service(name: string, namespace: string, labels: Record, port: number): Record { return { apiVersion: "v1", kind: "Service", metadata: { name, namespace, labels: { ...labels, "app.kubernetes.io/name": "git-mirror" } }, spec: { type: "ClusterIP", selector: { "app.kubernetes.io/name": name }, ports: [{ name: "http", port, targetPort: "http" }] }, }; } function gitMirrorConfigHash(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { return sha256Short(JSON.stringify({ repositories: [{ key: target.id, repository: target.source.repository, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch }], server: gitMirrorServerJs(), status: gitMirrorStatusShell(), sync: gitMirrorSyncShell(node, target), flush: gitMirrorFlushShell(node, target), })); } function gitMirrorDeployment(name: string, namespace: string, labels: Record, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, mode: "read" | "write"): Record { return { apiVersion: "apps/v1", kind: "Deployment", metadata: { name, namespace, labels: { ...labels, "app.kubernetes.io/name": name, "hwlab.pikastech.local/git-mirror-mode": mode } }, spec: { replicas: target.gitMirror.deploymentReplicas, selector: { matchLabels: { "app.kubernetes.io/name": name } }, template: { metadata: { labels: { ...labels, "app.kubernetes.io/name": name, "hwlab.pikastech.local/git-mirror-mode": mode }, annotations: { "checksum/config": gitMirrorConfigHash(node, target) }, }, spec: { containers: [{ name: "git-mirror", image: target.tekton.toolsImage.output, command: ["node", "/etc/git-mirror/server.js"], env: [ { name: "PORT", value: String(target.gitMirror.servicePort) }, { name: "GIT_PROJECT_ROOT", value: "/cache" }, { name: "GIT_MIRROR_MODE", value: mode }, ], ports: [{ name: "http", containerPort: target.gitMirror.servicePort }], volumeMounts: [{ name: "cache", mountPath: "/cache" }, { name: "config", mountPath: "/etc/git-mirror" }], }], volumes: [ { name: "cache", ...(target.gitMirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: target.gitMirror.cachePvcName } } : { hostPath: { path: target.gitMirror.cacheHostPath, type: "DirectoryOrCreate" } }) }, { name: "config", configMap: { name: target.gitMirror.syncConfigMapName, defaultMode: 0o755 } }, ], }, }, }, }; } function gitMirrorServerJs(): string { return String.raw`const http = require('node:http'); const { spawn } = require('node:child_process'); const projectRoot = process.env.GIT_PROJECT_ROOT || '/cache'; const port = Number.parseInt(process.env.PORT || '8080', 10); function sendHealth(res) { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ ok: true, mode: process.env.GIT_MIRROR_MODE || null, projectRoot })); } function parseHeaders(buffer) { const crlf = buffer.indexOf('\r\n\r\n'); const lf = buffer.indexOf('\n\n'); const headerEnd = crlf >= 0 ? crlf + 4 : (lf >= 0 ? lf + 2 : -1); if (headerEnd < 0) return null; const headerText = buffer.slice(0, headerEnd).toString('latin1').trim(); const rest = buffer.slice(headerEnd); const headers = {}; let status = 200; for (const line of headerText.split(/\r?\n/u)) { const index = line.indexOf(':'); if (index < 0) continue; const key = line.slice(0, index).trim(); const value = line.slice(index + 1).trim(); if (key.toLowerCase() === 'status') { const parsed = Number.parseInt(value.split(' ')[0] || '', 10); if (Number.isInteger(parsed)) status = parsed; } else { headers[key] = value; } } return { status, headers, rest }; } function cgiHeaderEnv(req) { const env = {}; for (const [key, value] of Object.entries(req.headers)) { if (value === undefined) continue; const joined = Array.isArray(value) ? value.join(', ') : String(value); const normalized = key.toUpperCase().replace(/-/g, '_'); if (normalized === 'CONTENT_TYPE') env.CONTENT_TYPE = joined; else if (normalized === 'CONTENT_LENGTH') env.CONTENT_LENGTH = joined; else env['HTTP_' + normalized] = joined; } return env; } function handleGit(req, res) { const url = new URL(req.url || '/', 'http://git-mirror.local'); const env = { ...process.env, ...cgiHeaderEnv(req), GIT_PROJECT_ROOT: projectRoot, GIT_HTTP_EXPORT_ALL: '1', PATH_INFO: decodeURIComponent(url.pathname), REQUEST_METHOD: req.method || 'GET', QUERY_STRING: url.search.slice(1), CONTENT_TYPE: req.headers['content-type'] || '', CONTENT_LENGTH: req.headers['content-length'] || '', REMOTE_USER: 'git', }; const child = spawn('git', ['http-backend'], { env }); let pending = Buffer.alloc(0); let headersSent = false; child.stderr.on('data', (chunk) => process.stderr.write(chunk)); child.on('error', (error) => { if (!headersSent) { res.writeHead(500, { 'content-type': 'text/plain' }); headersSent = true; } res.end(String(error && error.message || error)); }); child.stdout.on('data', (chunk) => { if (headersSent) { res.write(chunk); return; } pending = Buffer.concat([pending, chunk]); const parsed = parseHeaders(pending); if (!parsed) return; headersSent = true; res.writeHead(parsed.status, parsed.headers); if (parsed.rest.length) res.write(parsed.rest); }); child.on('close', (code) => { if (!headersSent) { res.writeHead(code === 0 ? 200 : 500, { 'content-type': 'text/plain' }); headersSent = true; if (pending.length) res.write(pending); } res.end(); }); req.pipe(child.stdin); } http.createServer((req, res) => { if ((req.url || '').startsWith('/healthz')) return sendHealth(res); return handleGit(req, res); }).listen(port, '0.0.0.0', () => { console.log(JSON.stringify({ event: 'git-mirror-http-started', port, projectRoot, mode: process.env.GIT_MIRROR_MODE || null })); }); `; } function gitMirrorStatusShell(): string { return String.raw`#!/bin/sh set -eu node <<'NODE' const { execFileSync } = require('node:child_process'); const { readFileSync, existsSync } = require('node:fs'); const repositories = JSON.parse(readFileSync('/etc/git-mirror/repositories.json', 'utf8')); function readJson(path) { try { return existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : null; } catch { return null; } } function rev(repo, ref) { try { return execFileSync('git', ['--git-dir=' + repo, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } } const items = {}; for (const spec of repositories) { const repoPath = '/cache/' + spec.repository + '.git'; const localSource = rev(repoPath, 'refs/heads/' + spec.sourceBranch); const githubSource = rev(repoPath, 'refs/mirror-stage/heads/' + spec.sourceBranch); const localGitops = rev(repoPath, 'refs/heads/' + spec.gitopsBranch); const githubGitops = rev(repoPath, 'refs/mirror-stage/heads/' + spec.gitopsBranch); items[spec.key] = { repository: spec.repository, sourceBranch: spec.sourceBranch, localSource, githubSource, gitopsBranch: spec.gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush: Boolean(localGitops && (!githubGitops || localGitops !== githubGitops)), }; } const first = items[repositories[0]?.key] || {}; const pendingFlush = Object.values(items).some((item) => Boolean(item.pendingFlush)); console.log(JSON.stringify({ localSource: first.localSource || null, githubSource: first.githubSource || null, localGitops: first.localGitops || null, githubGitops: first.githubGitops || null, refSources: { localSource: 'refs/heads/' + (first.sourceBranch || ''), githubSource: 'refs/mirror-stage/heads/' + (first.sourceBranch || ''), localGitops: 'refs/heads/' + (first.gitopsBranch || ''), githubGitops: 'refs/mirror-stage/heads/' + (first.gitopsBranch || ''), githubFieldsAreMirrorStageCache: true }, pendingFlush, flushNeeded: pendingFlush, githubInSync: Object.values(items).every((item) => item.sourceInSync === true && item.gitopsInSync === true), repositories: items, lastSync: readJson('/cache/HWLAB.last-sync.json'), lastFlush: readJson('/cache/HWLAB.last-flush.json'), })); NODE `; } function gitMirrorProxyPrelude(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { const proxyHost = node.egressProxy?.serviceName === undefined ? "127.0.0.1" : `${node.egressProxy.serviceName}.${node.egressProxy.namespace}.svc.cluster.local`; const proxyPort = node.egressProxy?.port ?? 10808; return [ "mkdir -p /root/.ssh", "cp /git-ssh/ssh-privatekey /root/.ssh/id_rsa", "chmod 0400 /root/.ssh/id_rsa", "cat > /tmp/hwlab-github-proxy-connect.cjs <<'NODE_PROXY'", "#!/usr/bin/env node", "const net = require('node:net');", "const [proxyHost, proxyPortRaw, targetHost, targetPortRaw] = process.argv.slice(2);", "const proxyPort = Number.parseInt(proxyPortRaw || '', 10);", "const targetPort = Number.parseInt(targetPortRaw || '', 10);", "if (!proxyHost || !Number.isInteger(proxyPort) || !targetHost || !Number.isInteger(targetPort)) process.exit(64);", "const socket = net.createConnection({ host: proxyHost, port: proxyPort });", "let buffer = Buffer.alloc(0);", "socket.setTimeout(10000, () => { socket.destroy(); process.exit(65); });", "socket.on('connect', () => socket.write('CONNECT ' + targetHost + ':' + targetPort + ' HTTP/1.1\\r\\nHost: ' + targetHost + ':' + targetPort + '\\r\\nProxy-Connection: Keep-Alive\\r\\n\\r\\n'));", "socket.on('error', () => process.exit(66));", "function onData(chunk) {", " buffer = Buffer.concat([buffer, chunk]);", " const headerEnd = buffer.indexOf('\\r\\n\\r\\n');", " if (headerEnd === -1 && buffer.length < 8192) return;", " const head = buffer.slice(0, headerEnd + 4).toString('latin1');", " const statusLine = head.split('\\r\\n', 1)[0] || '';", " const statusCode = Number.parseInt(statusLine.split(' ')[1] || '', 10);", " if (!statusLine.startsWith('HTTP/1.') || !Number.isInteger(statusCode) || statusCode < 200 || statusCode > 299) { socket.destroy(); process.exit(67); }", " socket.off('data', onData);", " socket.setTimeout(0);", " const rest = buffer.slice(headerEnd + 4);", " if (rest.length) process.stdout.write(rest);", " process.stdin.pipe(socket);", " socket.pipe(process.stdout);", "}", "socket.on('data', onData);", "socket.on('close', () => process.exit(0));", "NODE_PROXY", "chmod 0700 /tmp/hwlab-github-proxy-connect.cjs", `export GIT_SSH_COMMAND=${shQuote(`ssh -i /root/.ssh/id_rsa -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/root/.ssh/known_hosts -o ConnectTimeout=15 -o ServerAliveInterval=5 -o ServerAliveCountMax=1 -o ProxyCommand='node /tmp/hwlab-github-proxy-connect.cjs ${proxyHost} ${proxyPort} %h %p'`)}`, `repository=${shQuote(target.source.repository)}`, `source_branch=${shQuote(target.source.branch)}`, `gitops_branch=${shQuote(target.gitops.branch)}`, "repo=\"/cache/${repository}.git\"", "remote=\"ssh://git@ssh.github.com:443/${repository}.git\"", ].join("\n"); } function gitMirrorSyncShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { return [ "#!/bin/sh", "set -eu", "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)", gitMirrorProxyPrelude(node, target), "mkdir -p \"$(dirname \"$repo\")\"", "if [ -d \"$repo/objects\" ] && [ -f \"$repo/HEAD\" ]; then", " git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", "else", " rm -rf \"$repo\"", " git init --bare \"$repo\"", " git --git-dir=\"$repo\" remote add origin \"$remote\"", "fi", "git --git-dir=\"$repo\" config uploadpack.allowReachableSHA1InWant true", "git --git-dir=\"$repo\" config uploadpack.allowAnySHA1InWant true", "git --git-dir=\"$repo\" config http.uploadpack true", "git --git-dir=\"$repo\" config http.receivepack true", "timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${source_branch}:refs/mirror-stage/heads/${source_branch}\"", "source_sha=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${source_branch}^{commit}\")", "git --git-dir=\"$repo\" update-ref \"refs/heads/${source_branch}\" \"$source_sha\"", "if timeout 240 git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"; then", " github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", " local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", " if [ -z \"$local_gitops\" ] && [ -n \"$github_gitops\" ]; then", " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", " elif [ -n \"$local_gitops\" ] && [ -n \"$github_gitops\" ] && [ \"$local_gitops\" != \"$github_gitops\" ] && git --git-dir=\"$repo\" merge-base --is-ancestor \"$local_gitops\" \"$github_gitops\"; then", " git --git-dir=\"$repo\" update-ref \"refs/heads/${gitops_branch}\" \"$github_gitops\"", " fi", "fi", "git --git-dir=\"$repo\" update-server-info", "export repository source_branch gitops_branch started_at", "node <<'NODE' | tee /cache/HWLAB.last-sync.json", "const { execFileSync } = require('node:child_process');", "const repository = process.env.repository;", "const sourceBranch = process.env.source_branch;", "const gitopsBranch = process.env.gitops_branch;", "const repoPath = `/cache/${repository}.git`;", "function rev(ref) { try { return execFileSync('git', ['--git-dir=' + repoPath, 'rev-parse', '--verify', ref + '^{commit}'], { encoding: 'utf8' }).trim(); } catch { return null; } }", "const localSource = rev(`refs/heads/${sourceBranch}`);", "const githubSource = rev(`refs/mirror-stage/heads/${sourceBranch}`);", "const localGitops = rev(`refs/heads/${gitopsBranch}`);", "const githubGitops = rev(`refs/mirror-stage/heads/${gitopsBranch}`);", "const pendingFlush = Boolean(localGitops && (!githubGitops || localGitops !== githubGitops));", "console.log(JSON.stringify({ event: 'git-mirror-sync', repo: repository, status: 'succeeded', startedAt: process.env.started_at, syncedAt: new Date().toISOString(), localSource, githubSource, gitopsBranch, localGitops, githubGitops, sourceInSync: Boolean(localSource && githubSource && localSource === githubSource), gitopsInSync: Boolean(localGitops && githubGitops && localGitops === githubGitops), pendingFlush }));", "NODE", "cat /cache/HWLAB.last-sync.json", "", ].join("\n"); } function gitMirrorFlushShell(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { return [ "#!/bin/sh", "set -eu", "started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)", gitMirrorProxyPrelude(node, target), "test -d \"$repo/objects\"", "git --git-dir=\"$repo\" remote set-url origin \"$remote\" || git --git-dir=\"$repo\" remote add origin \"$remote\"", "local_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", "push_status=skipped", "push_exit=0", "fetch_status=skipped", "fetch_exit=0", "if [ -n \"$local_gitops\" ]; then", " set +e", " git --git-dir=\"$repo\" -c remote.origin.mirror=false push origin \"refs/heads/${gitops_branch}:refs/heads/${gitops_branch}\"", " push_exit=$?", " set -e", " if [ \"$push_exit\" = \"0\" ]; then", " push_status=succeeded", " set +e", " git --git-dir=\"$repo\" fetch origin \"+refs/heads/${gitops_branch}:refs/mirror-stage/heads/${gitops_branch}\"", " fetch_exit=$?", " set -e", " if [ \"$fetch_exit\" = \"0\" ]; then fetch_status=succeeded; else fetch_status=failed; fi", " else", " push_status=failed", " fi", "fi", "github_gitops=$(git --git-dir=\"$repo\" rev-parse --verify \"refs/mirror-stage/heads/${gitops_branch}^{commit}\" 2>/dev/null || true)", "pending=false; if [ -n \"$local_gitops\" ] && { [ -z \"$github_gitops\" ] || [ \"$local_gitops\" != \"$github_gitops\" ]; }; then pending=true; fi", "status=succeeded", "partial_success=", "degraded_reason=", "exit_code=0", "if [ \"$push_status\" = \"failed\" ]; then", " status=failed", " degraded_reason=git-mirror-push-failed", " exit_code=$push_exit", "elif [ \"$push_status\" = \"succeeded\" ] && [ \"$fetch_status\" = \"failed\" ]; then", " status=partial-success", " partial_success=push-succeeded-fetch-failed", " degraded_reason=git-mirror-post-push-fetch-failed", " exit_code=44", "fi", "export repository gitops_branch started_at local_gitops github_gitops pending push_status push_exit fetch_status fetch_exit status partial_success degraded_reason", "node <<'NODE' | tee /cache/HWLAB.last-flush.json", "const payload = { event: 'git-mirror-flush', repo: process.env.repository, status: process.env.status || 'failed', partialSuccess: process.env.partial_success || null, degradedReason: process.env.degraded_reason || null, startedAt: process.env.started_at, flushedAt: new Date().toISOString(), gitopsBranch: process.env.gitops_branch, localGitops: process.env.local_gitops || null, githubGitops: process.env.github_gitops || null, pendingFlush: process.env.pending === 'true', stages: { push: process.env.push_status || null, pushExitCode: Number.parseInt(process.env.push_exit || '0', 10), postPushFetch: process.env.fetch_status || null, postPushFetchExitCode: Number.parseInt(process.env.fetch_exit || '0', 10) } };", "console.log(JSON.stringify(payload));", "NODE", "cat /cache/HWLAB.last-flush.json", "if [ \"$exit_code\" != \"0\" ]; then exit \"$exit_code\"; fi", "", ].join("\n"); } function argoDesiredManifest(target: ControlPlaneTargetSpec): Record[] { return [argoProjectSkeleton(target), argoApplicationSkeleton(target)]; } function argoProjectSkeleton(target: ControlPlaneTargetSpec): Record { return { apiVersion: "argoproj.io/v1alpha1", kind: "AppProject", metadata: { name: target.argo.projectName, namespace: target.argo.namespace }, spec: { sourceRepos: [target.gitMirror.readUrl], destinations: [{ server: "https://kubernetes.default.svc", namespace: target.runtimeNamespace }], clusterResourceWhitelist: [{ group: "*", kind: "*" }], namespaceResourceWhitelist: [{ group: "*", kind: "*" }], }, }; } function argoApplicationSkeleton(target: ControlPlaneTargetSpec): Record { return { apiVersion: "argoproj.io/v1alpha1", kind: "Application", metadata: { name: target.argo.applicationName, namespace: target.argo.namespace }, spec: { project: target.argo.projectName, source: { repoURL: target.gitMirror.readUrl, targetRevision: target.gitops.branch, path: target.gitops.path }, destination: { server: "https://kubernetes.default.svc", namespace: target.runtimeNamespace }, syncPolicy: { automated: { prune: true, selfHeal: true } }, }, }; } function planSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record { return { id: target.id, node: node.id, kubeRoute: node.kubeRoute, lane: target.lane, enabled: target.enabled, ciNamespace: target.ciNamespace, runtimeNamespace: target.runtimeNamespace, k3sNodeConfig: k3sNodeConfigPlan(node), registry: node.registry.endpoint, egressProxy: node.egressProxy, sourceBranch: target.source.branch, gitopsBranch: target.gitops.branch, gitopsPath: target.gitops.path, gitMirrorNamespace: target.gitMirror.namespace, readUrl: target.gitMirror.readUrl, writeUrl: target.gitMirror.writeUrl, pipeline: target.tekton.pipelineName, pipelineRunPrefix: target.tekton.pipelineRunPrefix, serviceAccount: target.tekton.serviceAccountName, toolsImage: target.tekton.toolsImage, argoApplication: target.argo.applicationName, argoInstall: { enabled: target.argo.install.enabled, version: target.argo.install.version, manifestUrl: target.argo.install.manifestUrl, imageRewrites: target.argo.install.imageRewrites, }, }; } function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record { return { sourceRepo: target.source.repository, branch: target.source.branch, gitopsBranch: target.gitops.branch, runtimePath: target.gitops.path, runtimeNamespace: target.runtimeNamespace, namespace: target.ciNamespace, k3sNodeConfig: k3sNodeConfigPlan(node), gitMirror: { namespace: target.gitMirror.namespace, readUrl: target.gitMirror.readUrl, writeUrl: target.gitMirror.writeUrl, cachePvc: target.gitMirror.cachePvcName, cachePvcStorage: target.gitMirror.cachePvcStorage, cacheHostPath: target.gitMirror.cacheHostPath, servicePort: target.gitMirror.servicePort, deploymentReplicas: target.gitMirror.deploymentReplicas, syncConfigMap: target.gitMirror.syncConfigMapName, statusSummaryKeys: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"], }, pipeline: target.tekton.pipelineName, pipelineRunPrefix: target.tekton.pipelineRunPrefix, serviceAccount: target.tekton.serviceAccountName, toolsImage: target.tekton.toolsImage, argoNamespace: target.argo.namespace, argoApplication: target.argo.applicationName, argoInstall: { enabled: target.argo.install.enabled, sourceKind: target.argo.install.sourceKind, version: target.argo.install.version, manifestUrl: target.argo.install.manifestUrl, preloadImages: target.argo.install.preloadImages, imageRewrites: target.argo.install.imageRewrites, requiredCrds: target.argo.install.requiredCrds, expectedDeployments: target.argo.install.expectedDeployments, expectedStatefulSets: target.argo.install.expectedStatefulSets, }, registry: node.registry.endpoint, imagePolicy: { noPrivateInputImages: true, buildInput: { sourceKind: target.tekton.toolsImage.sourceKind, context: target.tekton.toolsImage.context, dockerfile: target.tekton.toolsImage.dockerfile ?? null, dockerfileInline: target.tekton.toolsImage.dockerfileInline ?? null, composeFile: target.tekton.toolsImage.composeFile ?? null, publicBaseImages: target.tekton.toolsImage.publicBaseImages }, outputImage: target.tekton.toolsImage.output, }, }; } function k3sNodeConfigPlan(node: ControlPlaneNodeSpec): Record { if (node.k3s === null) return { managed: false }; const dropIn = k3sDropInContent(node.k3s); return { managed: true, serviceName: node.k3s.serviceName, dropInPath: node.k3s.dropInPath, nodeStatusName: node.k3s.nodeStatusName, desiredMaxPods: node.k3s.kubelet.maxPods, dropInSha256: sha256Short(dropIn), execStartPreCount: node.k3s.execStartPre.length, serverArgCount: node.k3s.serverArgs.length, }; } function k3sDropInContent(spec: ControlPlaneK3sNodeSpec): string { return [ "# Managed by UniDesk. Source: config/hwlab-node-control-plane.yaml nodes..k3s", "[Service]", ...spec.execStartPre.map((command) => `ExecStartPre=${command.map(systemdExecArg).join(" ")}`), "ExecStart=", `ExecStart=${["/usr/local/bin/k3s", ...spec.serverArgs].map(systemdExecArg).join(" ")}`, "", ].join("\n"); } function systemdExecArg(value: string): string { if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value; return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("$", "\\$").replaceAll("`", "\\`")}"`; } function statusScript(nodeSpec: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { const requiredCrds = shellJsonArray(target.argo.install.requiredCrds); const argoDeployments = shellJsonArray(target.argo.install.expectedDeployments); const argoStatefulSets = shellJsonArray(target.argo.install.expectedStatefulSets); const k3s = nodeSpec.k3s; const k3sDropIn = k3s === null ? "" : k3sDropInContent(k3s); return ` set +e node=${shQuote(target.node)} lane=${shQuote(target.lane)} ci_ns=${shQuote(target.ciNamespace)} runtime_ns=${shQuote(target.runtimeNamespace)} gitmirror_ns=${shQuote(target.gitMirror.namespace)} read_deploy=${shQuote(target.gitMirror.serviceReadName)} write_deploy=${shQuote(target.gitMirror.serviceWriteName)} read_svc=${shQuote(target.gitMirror.serviceReadName)} write_svc=${shQuote(target.gitMirror.serviceWriteName)} cache_pvc=${shQuote(target.gitMirror.cachePvcName)} cache_host_path=${shQuote(target.gitMirror.cacheHostPath ?? "")} pipeline=${shQuote(target.tekton.pipelineName)} service_account=${shQuote(target.tekton.serviceAccountName)} argo_ns=${shQuote(target.argo.namespace)} argo_project=${shQuote(target.argo.projectName)} argo_app=${shQuote(target.argo.applicationName)} registry=${shQuote(nodeSpec.registry.endpoint)} tools_image=${shQuote(target.tekton.toolsImage.output)} required_crds_json=${shQuote(requiredCrds)} argo_deployments_json=${shQuote(argoDeployments)} argo_statefulsets_json=${shQuote(argoStatefulSets)} k3s_managed=${k3s === null ? "false" : "true"} k3s_service=${shQuote(k3s?.serviceName ?? "")} k3s_dropin=${shQuote(k3s?.dropInPath ?? "")} k3s_node=${shQuote(k3s?.nodeStatusName ?? "")} k3s_desired_max_pods=${shQuote(String(k3s?.kubelet.maxPods ?? ""))} k3s_expected_sha=${shQuote(k3s === null ? "" : sha256Short(k3sDropIn))} exists_ns() { kubectl get ns "$1" >/dev/null 2>&1 && printf true || printf false; } exists_res() { kubectl -n "$1" get "$2" "$3" >/dev/null 2>&1 && printf true || printf false; } deploy_ready() { desired=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; } sts_ready() { desired=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; } endpoint_ready() { endpoints=$(kubectl -n "$1" get endpoints "$2" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n "$endpoints" ] && printf true || printf false; } registry_ready=false if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_ready=true; fi tools_repo_tag=\${tools_image#\${registry}/} tools_repo=\${tools_repo_tag%:*} tools_tag=\${tools_repo_tag##*:} tools_image_ready=false if [ "$tools_repo" != "$tools_repo_tag" ] && command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 "http://$registry/v2/$tools_repo/manifests/$tools_tag" >/tmp/hwlab-tools-image.out 2>/tmp/hwlab-tools-image.err && tools_image_ready=true; fi cache_host_path_ready=false if [ -n "$cache_host_path" ] && kubectl -n "$gitmirror_ns" exec deploy/"$read_deploy" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_ready=true; fi k3s_fragment=$(python3 - "$k3s_managed" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_expected_sha" <<'PY' import hashlib, json, re, subprocess, sys managed = sys.argv[1] == "true" service, dropin, node_name, desired_raw, expected_sha = sys.argv[2:7] def run(args): return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) def to_int(value): try: return int(value) except Exception: return None if not managed: print(json.dumps({"managed": False, "ready": True})) raise SystemExit(0) desired = to_int(desired_raw) node_json = run(["kubectl", "get", "node", node_name, "-o", "json"]) capacity = None allocatable = None node_ready = False if node_json.returncode == 0: data = json.loads(node_json.stdout) capacity = to_int(data.get("status", {}).get("capacity", {}).get("pods")) allocatable = to_int(data.get("status", {}).get("allocatable", {}).get("pods")) for condition in data.get("status", {}).get("conditions", []): if condition.get("type") == "Ready": node_ready = condition.get("status") == "True" unit = run(["systemctl", "cat", service]) unit_text = unit.stdout if unit.returncode == 0 else "" dropin_read = run(["cat", dropin]) dropin_exists = dropin_read.returncode == 0 dropin_text = dropin_read.stdout if dropin_exists else "" dropin_sha = "sha256:" + hashlib.sha256(dropin_text.encode()).hexdigest() if dropin_exists else None matches = re.findall(r"max-pods=([0-9]+)", unit_text + "\\n" + dropin_text) configured = to_int(matches[-1]) if matches else None dropin_matches = dropin_sha == expected_sha ready = dropin_matches and capacity == desired and allocatable == desired source = "managed-dropin" if dropin_matches else ("systemd-or-config" if configured is not None else "kubelet-default") print(json.dumps({ "managed": True, "ready": ready, "serviceName": service, "dropInPath": dropin, "dropInExists": dropin_exists, "dropInSha256": dropin_sha, "expectedDropInSha256": expected_sha, "dropInMatches": dropin_matches, "configuredMaxPods": configured, "desiredMaxPods": desired, "liveNodeName": node_name, "liveCapacityPods": capacity, "liveAllocatablePods": allocatable, "nodeReady": node_ready, "restartRequired": not ready, "source": source, "unitReadable": unit.returncode == 0, })) PY ) python3 - "$required_crds_json" "$argo_deployments_json" "$argo_statefulsets_json" <<'PY' >/tmp/hwlab-node-status-fragments.json import json, subprocess, sys required_crds=json.loads(sys.argv[1]) deployments=json.loads(sys.argv[2]) statefulsets=json.loads(sys.argv[3]) ns="${target.argo.namespace}" def run(args): return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) def exists(args): return run(args).returncode == 0 def ready(kind, name): data = run(["kubectl", "-n", ns, "get", kind, name, "-o", "json"]) if data.returncode != 0: return {"name": name, "exists": False, "ready": False, "desired": None, "readyReplicas": None} obj=json.loads(data.stdout) desired=int(obj.get("spec", {}).get("replicas") or 0) ready_replicas=int(obj.get("status", {}).get("readyReplicas") or 0) return {"name": name, "exists": True, "ready": desired > 0 and ready_replicas == desired, "desired": desired, "readyReplicas": ready_replicas} crds=[{"name": name, "exists": exists(["kubectl", "get", "crd", name])} for name in required_crds] deploy=[ready("deployment", name) for name in deployments] sts=[ready("statefulset", name) for name in statefulsets] print(json.dumps({"crds": crds, "deployments": deploy, "statefulSets": sts, "crdsReady": all(item["exists"] for item in crds), "deploymentsReady": all(item["ready"] for item in deploy) if deploy else True, "statefulSetsReady": all(item["ready"] for item in sts) if sts else True})) PY argo_fragment=$(cat /tmp/hwlab-node-status-fragments.json 2>/dev/null || printf '{}') cat </dev/null 2>&1 && printf true || printf false),"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook)},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline")},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns")}}} JSON `; } function applyScript(yaml: string, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string { const encoded = Buffer.from(yaml, "utf8").toString("base64"); return ` set +e manifest=$(mktemp /tmp/hwlab-node-infra.XXXXXX.yaml) printf %s ${shQuote(encoded)} | base64 -d >"$manifest" kubectl apply --server-side --field-manager=unidesk-hwlab-node-control-plane -f "$manifest" >/tmp/hwlab-node-infra-apply.out 2>/tmp/hwlab-node-infra-apply.err kubectl_rc=$? ${k3sApplyScriptFragment(node.k3s, target)} python3 - "$kubectl_rc" "$k3s_report_file" <<'PY' import json, pathlib, sys k3s_report = {} try: k3s_report = json.loads(pathlib.Path(sys.argv[2]).read_text(errors='replace')) except Exception as exc: k3s_report = {"managed": None, "ok": False, "parseError": str(exc)} out=pathlib.Path('/tmp/hwlab-node-infra-apply.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.out').exists() else '' err=pathlib.Path('/tmp/hwlab-node-infra-apply.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.err').exists() else '' print(json.dumps({'k3sNodeConfig': k3s_report, 'kubernetesApply': {'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'runtimeRolloutTriggered': False, 'pk01Touched': False}}, ensure_ascii=False)) PY rm -f "$manifest" if [ "$kubectl_rc" != 0 ]; then exit "$kubectl_rc"; fi exit "$k3s_rc" `; } function k3sApplyScriptFragment(spec: ControlPlaneK3sNodeSpec | null, target: ControlPlaneTargetSpec): string { if (spec === null) { return ` k3s_report_file=$(mktemp /tmp/hwlab-node-k3s.XXXXXX.json) printf '{"managed":false,"ok":true,"mutation":false}\\n' >"$k3s_report_file" k3s_rc=0 `; } const content = k3sDropInContent(spec); const encoded = Buffer.from(content, "utf8").toString("base64"); return ` k3s_report_file=$(mktemp /tmp/hwlab-node-k3s.XXXXXX.json) k3s_service=${shQuote(spec.serviceName)} k3s_dropin=${shQuote(spec.dropInPath)} k3s_node=${shQuote(spec.nodeStatusName)} k3s_namespace=${shQuote(target.ciNamespace)} k3s_image=${shQuote(target.tekton.toolsImage.output)} k3s_desired_max_pods=${shQuote(String(spec.kubelet.maxPods))} k3s_expected_sha=${shQuote(sha256Short(content))} k3s_before_capacity=$(kubectl get node "$k3s_node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true) k3s_before_allocatable=$(kubectl get node "$k3s_node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true) capacity_restart=false if [ "$k3s_before_capacity" != "$k3s_desired_max_pods" ] || [ "$k3s_before_allocatable" != "$k3s_desired_max_pods" ]; then capacity_restart=true; fi k3s_current_dropin_sha= if [ -f "$k3s_dropin" ]; then k3s_current_dropin_sha=$(sha256sum "$k3s_dropin" | awk '{print "sha256:"$1}'); fi if [ "$k3s_current_dropin_sha" = "$k3s_expected_sha" ] && [ "$capacity_restart" = false ]; then python3 - "$k3s_current_dropin_sha" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_before_capacity" "$k3s_before_allocatable" <<'PY' >"$k3s_report_file" import json, sys dropin_sha, expected_sha, service, dropin, node_name, desired, before_capacity, before_allocatable = sys.argv[1:9] print(json.dumps({ "managed": True, "ok": True, "mutation": False, "applyMode": "noop", "completionPending": False, "serviceName": service, "dropInPath": dropin, "dropInSha256": dropin_sha, "expectedDropInSha256": expected_sha, "dropInMatches": dropin_sha == expected_sha, "nodeName": node_name, "desiredMaxPods": int(desired), "beforeCapacityPods": int(before_capacity) if before_capacity.isdigit() else None, "beforeAllocatablePods": int(before_allocatable) if before_allocatable.isdigit() else None, }, ensure_ascii=False)) PY k3s_rc=0 else k3s_job="hwlab-node-k3s-config-$(date +%s)" k3s_job_manifest=$(mktemp /tmp/hwlab-node-k3s-job.XXXXXX.json) k3s_host_script=$(mktemp /tmp/hwlab-node-k3s-host.XXXXXX.sh) k3s_job_apply_stdout=/tmp/hwlab-node-k3s-job-apply.out k3s_job_apply_stderr=/tmp/hwlab-node-k3s-job-apply.err k3s_docker_stdout=/tmp/hwlab-node-k3s-docker.out k3s_docker_stderr=/tmp/hwlab-node-k3s-docker.err k3s_host_report="/tmp/$k3s_job-report.json" rm -f "$k3s_host_report" python3 - "$k3s_job_manifest" "$k3s_host_script" "$k3s_job" "$k3s_namespace" "$k3s_image" "$k3s_dropin" ${shQuote(encoded)} "$k3s_service" "$k3s_desired_max_pods" "$k3s_expected_sha" "$capacity_restart" "$k3s_host_report" <<'PY' import json, os, shlex, sys manifest_path, host_script_path, job, namespace, image, dropin, encoded, service, desired, expected_sha, capacity_restart, report_path = sys.argv[1:13] script = f"""#!/bin/sh set -eu expected=/tmp/unidesk-k3s-dropin.conf printf %s {shlex.quote(encoded)} | base64 -d > "$expected" host_dropin=/host{shlex.quote(dropin)} host_report=/host{shlex.quote(report_path)} mkdir -p "$(dirname "$host_dropin")" before_sha= if [ -f "$host_dropin" ]; then before_sha=$(sha256sum "$host_dropin" | awk '{{print "sha256:"$1}}'); fi changed=false if ! cmp -s "$expected" "$host_dropin" 2>/dev/null; then cp "$expected" "$host_dropin" chown 0:0 "$host_dropin" 2>/dev/null || true chmod 0644 "$host_dropin" changed=true fi nsenter_path=$(command -v nsenter || true) host_systemctl() {{ if command -v chroot >/dev/null 2>&1 && [ -x /host/usr/bin/systemctl ]; then chroot /host /usr/bin/systemctl "$@" return $? fi if [ -n "$nsenter_path" ]; then "$nsenter_path" -t 1 -m -u -i -n -p -- /usr/bin/systemctl "$@" return $? fi return 127 }} daemon_reload_rc=0 restart_rc=0 restarted=false if command -v chroot >/dev/null 2>&1 || [ -n "$nsenter_path" ]; then host_systemctl daemon-reload || daemon_reload_rc=$? if [ "$changed" = true ] || [ {shlex.quote(capacity_restart)} = true ]; then restarted=true host_systemctl restart {shlex.quote(service)} || restart_rc=$? fi else daemon_reload_rc=127 restart_rc=127 fi after_sha= if [ -f "$host_dropin" ]; then after_sha=$(sha256sum "$host_dropin" | awk '{{print "sha256:"$1}}'); fi service_active=unknown if command -v chroot >/dev/null 2>&1 || [ -n "$nsenter_path" ]; then service_active=$(host_systemctl is-active {shlex.quote(service)} 2>/dev/null || true); fi python3 - "$changed" "$restarted" "$daemon_reload_rc" "$restart_rc" "$before_sha" "$after_sha" "$service_active" "$nsenter_path" <<'REPORT' >"$host_report" import json, sys changed, restarted = sys.argv[1] == "true", sys.argv[2] == "true" daemon_reload_rc, restart_rc = int(sys.argv[3] or "0"), int(sys.argv[4] or "0") print(json.dumps({{ "jobChanged": changed, "jobRestarted": restarted, "daemonReloadExitCode": daemon_reload_rc, "restartExitCode": restart_rc, "beforeDropInSha256": sys.argv[5] or None, "dropInSha256": sys.argv[6] or None, "expectedDropInSha256": {json.dumps(expected_sha)}, "dropInMatches": sys.argv[6] == {json.dumps(expected_sha)}, "serviceActiveText": sys.argv[7] or None, "nsenterPresent": bool(sys.argv[8]), }})) REPORT chmod 0644 "$host_report" 2>/dev/null || true cat "$host_report" """ with open(host_script_path, "w", encoding="utf-8") as handle: handle.write(script) os.chmod(host_script_path, 0o755) manifest = { "apiVersion": "batch/v1", "kind": "Job", "metadata": {"name": job, "namespace": namespace, "labels": {"app.kubernetes.io/part-of": "hwlab-node-control-plane", "unidesk.ai/operation": "k3s-node-config"}}, "spec": { "backoffLimit": 0, "ttlSecondsAfterFinished": 300, "template": { "metadata": {"labels": {"app.kubernetes.io/part-of": "hwlab-node-control-plane", "unidesk.ai/operation": "k3s-node-config"}}, "spec": { "restartPolicy": "Never", "hostPID": True, "hostNetwork": True, "containers": [{ "name": "apply-k3s-node-config", "image": image, "imagePullPolicy": "IfNotPresent", "securityContext": {"privileged": True}, "command": ["/bin/sh", "-lc", script], "volumeMounts": [{"name": "host-root", "mountPath": "/host"}], }], "volumes": [{"name": "host-root", "hostPath": {"path": "/", "type": "Directory"}}], }, }, }, } with open(manifest_path, "w", encoding="utf-8") as handle: json.dump(manifest, handle) PY k3s_render_rc=$? if [ "$k3s_render_rc" != 0 ]; then python3 - "$k3s_render_rc" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" <<'PY' >"$k3s_report_file" import json, sys render_rc = int(sys.argv[1] or "1") expected_sha, service, dropin, node_name, desired = sys.argv[2:7] print(json.dumps({ "managed": True, "ok": False, "mutation": False, "renderExitCode": render_rc, "serviceName": service, "dropInPath": dropin, "expectedDropInSha256": expected_sha, "nodeName": node_name, "desiredMaxPods": int(desired), }, ensure_ascii=False)) PY k3s_rc=$k3s_render_rc else kubectl apply -f "$k3s_job_manifest" >"$k3s_job_apply_stdout" 2>"$k3s_job_apply_stderr" k3s_job_apply_rc=$? k3s_apply_mode=kubernetes-job k3s_docker_rc=127 if [ "$k3s_job_apply_rc" != 0 ] && command -v docker >/dev/null 2>&1; then k3s_apply_mode=docker-host-fallback docker run --rm --privileged --pid=host --network=host -v /:/host --entrypoint /bin/sh "$k3s_image" "/host$k3s_host_script" >"$k3s_docker_stdout" 2>"$k3s_docker_stderr" k3s_docker_rc=$? fi k3s_submit_rc=$k3s_job_apply_rc if [ "$k3s_job_apply_rc" != 0 ] && [ "$k3s_docker_rc" = 0 ]; then k3s_submit_rc=0; fi python3 - "$k3s_submit_rc" "$k3s_job_apply_rc" "$k3s_docker_rc" "$k3s_apply_mode" "$k3s_before_capacity" "$k3s_before_allocatable" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_job" "$k3s_namespace" "$k3s_host_report" "$k3s_job_apply_stdout" "$k3s_job_apply_stderr" "$k3s_docker_stdout" "$k3s_docker_stderr" <<'PY' >"$k3s_report_file" import json, pathlib, sys submit_rc, job_apply_rc, docker_rc = [int(value or "0") for value in sys.argv[1:4]] apply_mode = sys.argv[4] before_capacity, before_allocatable = sys.argv[5:7] expected_sha, service, dropin, node_name, desired, job_name, namespace, host_report = sys.argv[7:15] def read(path): return pathlib.Path(path).read_text(errors='replace') if pathlib.Path(path).exists() else '' try: host_report_data = json.loads(read(host_report) or "{}") except Exception: host_report_data = {} apply_ok = submit_rc == 0 print(json.dumps({ "managed": True, "ok": apply_ok, "mutation": apply_ok, "completionPending": apply_ok and apply_mode == "kubernetes-job", "applyMode": apply_mode, "jobName": job_name, "namespace": namespace, "jobApplyExitCode": job_apply_rc, "dockerFallbackExitCode": docker_rc, "serviceName": service, "dropInPath": dropin, "dropInSha256": host_report_data.get("dropInSha256"), "expectedDropInSha256": expected_sha, "dropInMatches": host_report_data.get("dropInSha256") == expected_sha if host_report_data else None, "daemonReloadExitCode": host_report_data.get("daemonReloadExitCode"), "restartExitCode": host_report_data.get("restartExitCode"), "serviceActive": host_report_data.get("serviceActiveText") == "active" if host_report_data else None, "nodeName": node_name, "desiredMaxPods": int(desired), "beforeCapacityPods": int(before_capacity) if before_capacity.isdigit() else None, "beforeAllocatablePods": int(before_allocatable) if before_allocatable.isdigit() else None, "hostReportPath": host_report, "statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra status --node {node_name.upper()} --lane ${target.lane}", "jobCompletionCommand": f"kubectl -n {namespace} wait --for=condition=complete job/{job_name} --timeout=120s", "jobLogsCommand": f"kubectl -n {namespace} logs job/{job_name} --tail=120", "jobApplyStdoutPreview": read(sys.argv[15])[-1000:], "jobApplyStderrPreview": read(sys.argv[16])[-1000:], "dockerStdoutPreview": read(sys.argv[17])[-1000:], "dockerStderrPreview": read(sys.argv[18])[-1000:], }, ensure_ascii=False)) PY k3s_rc=$k3s_submit_rc fi rm -f "$k3s_job_manifest" "$k3s_host_script" fi `; } function toolsImageStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, timeoutSeconds: number): { registryReady: boolean; toolsImageReady: boolean; result: Record; } { const result = runTransK3s(node.kubeRoute, registryStatusScript(node.registry.endpoint, target.tekton.toolsImage.output), timeoutSeconds); const parsed = parseRemoteJson(result.stdout); const status = typeof parsed === "object" && parsed !== null ? parsed as Record : {}; return { registryReady: boolField(status, "registryReady"), toolsImageReady: boolField(status, "toolsImageReady"), result: { status, command: compactCommandResult(result), }, }; } function applyNext(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, imageStatus: { registryReady: boolean; toolsImageReady: boolean }): Record { if (!imageStatus.registryReady) { return { status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, blockedBy: "node-local-registry-not-ready", }; } if (!imageStatus.toolsImageReady) { return { status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, blockedBy: "tools-image-missing", applyBootstrap: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`, buildToolsImage: "准备受控 D601 tools-image build/publish 入口后提升 control-plane readiness。", }; } return { apply: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm` }; } function statusNext( node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, registry: Record, gitMirror: Record, argo: Record, ciNamespace: Record, k3sNodeConfig: Record, ): Record { const bootstrapMissing = !boolField(ciNamespace, "exists") || !boolField(gitMirror, "namespaceExists") || !boolField(gitMirror, "readServiceExists") || !boolField(gitMirror, "writeServiceExists") || (!boolField(gitMirror, "cachePvcExists") && !boolField(gitMirror, "cacheHostPathReady")); const blockers: string[] = []; if (node.k3s !== null && !boolField(k3sNodeConfig, "ready")) blockers.push("k3s-node-config-not-applied"); if (!boolField(registry, "ready")) blockers.push("node-local-registry-not-ready"); if (!boolField(registry, "toolsImageReady")) blockers.push("tools-image-missing"); if (bootstrapMissing) blockers.push("control-plane-bootstrap-missing"); const argoInstall = record(argo.install); if (!boolField(argo, "installed")) blockers.push("argocd-not-installed"); else if (!boolField(argoInstall, "crdsReady")) blockers.push("argocd-crds-not-ready"); else if (!boolField(argoInstall, "deploymentsReady")) blockers.push("argocd-deployments-not-ready"); else if (!boolField(argoInstall, "statefulSetsReady")) blockers.push("argocd-statefulsets-not-ready"); else if (!boolField(argo, "projectExists")) blockers.push("argocd-project-missing"); else if (!boolField(argo, "applicationExists")) blockers.push("argocd-application-missing"); const next: Record = { status: `bun scripts/cli.ts hwlab nodes control-plane infra status --node ${node.id} --lane ${target.lane}`, dryRun: `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --dry-run`, }; if (blockers.length > 0) { next.blockedBy = blockers[0]; next.blockers = blockers; } if (!boolField(registry, "toolsImageReady")) { next.buildToolsImage = "准备受控 D601 tools-image build/publish 入口后提升 control-plane readiness。"; } if (!boolField(argo, "installed")) { next.installArgo = "准备受控 D601 Argo CD 安装入口后再进入 runtime rollout。"; } if (node.k3s !== null && !boolField(k3sNodeConfig, "ready")) { next.applyK3sNodeConfig = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; } if (bootstrapMissing) next.applyBootstrap = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; else next.reapplyBootstrap = `bun scripts/cli.ts hwlab nodes control-plane infra apply --node ${node.id} --lane ${target.lane} --confirm`; return next; } function registryStatusScript(registryEndpoint: string, toolsImage: string): string { return ` set +e registry=${shQuote(registryEndpoint)} tools_image=${shQuote(toolsImage)} registry_ready=false if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_ready=true; fi tools_repo_tag=\${tools_image#\${registry}/} tools_repo=\${tools_repo_tag%:*} tools_tag=\${tools_repo_tag##*:} tools_image_ready=false if [ "$tools_repo" != "$tools_repo_tag" ] && command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 "http://$registry/v2/$tools_repo/manifests/$tools_tag" >/tmp/hwlab-tools-image.out 2>/tmp/hwlab-tools-image.err && tools_image_ready=true; fi cat < ["--build-arg", `${key}=${value}`]); const proxyArgs = node.egressProxy === null ? [] : ["--build-arg", "HTTP_PROXY", "--build-arg", "HTTPS_PROXY", "--build-arg", "ALL_PROXY", "--build-arg", "NO_PROXY", "--build-arg", "http_proxy", "--build-arg", "https_proxy", "--build-arg", "all_proxy", "--build-arg", "no_proxy"]; const networkArgs = target.tekton.toolsImage.buildNetwork === null ? [] : ["--network", target.tekton.toolsImage.buildNetwork]; const dockerBuildArgs = [...networkArgs, "--pull", ...buildArgs, ...proxyArgs, "-f", "$dockerfile", "-t", "$image", "$context_dir"].join(" "); return ` set -eu state_dir=${shQuote(stateDir)} mkdir -p "$state_dir" if [ -s "$state_dir/pid" ] && kill -0 "$(cat "$state_dir/pid")" >/dev/null 2>&1; then printf '{"started":false,"reason":"job-already-running","pid":%s,"stateDir":"%s"}\\n' "$(cat "$state_dir/pid")" "$state_dir" exit 0 fi cat >"$state_dir/job.sh" <<'JOB' #!/bin/sh set -eu state_dir=${shQuote(stateDir)} image=${shQuote(target.tekton.toolsImage.output)} context_dir="$state_dir/context" dockerfile="$state_dir/${target.tekton.toolsImage.dockerfileInline?.filename ?? "Dockerfile"}" log="$state_dir/job.log" status="$state_dir/status.json" write_status() { state="$1"; shift message="$1"; shift || true python3 - "$status" "$state" "$message" "$image" <<'PY' import json, pathlib, sys, time path=pathlib.Path(sys.argv[1]) payload={"state":sys.argv[2],"message":sys.argv[3],"image":sys.argv[4],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} path.write_text(json.dumps(payload, ensure_ascii=False) + "\\n") PY } run_job() { write_status running starting rm -rf "$context_dir" mkdir -p "$context_dir" printf %s ${shQuote(dockerfileEncoded)} | base64 -d >"$dockerfile" ${proxyExportBlock(node)} docker build ${dockerBuildArgs} || return "$?" docker run --rm "$image" sh -lc 'node --version && npm --version && bun --version && git --version && python3 --version && docker --version && ssh -V' || return "$?" docker push "$image" || return "$?" image_id="$(docker image inspect "$image" --format '{{.Id}}' 2>/dev/null || true)" digest="$(docker image inspect "$image" --format '{{join .RepoDigests ","}}' 2>/dev/null || true)" python3 - "$status" "$image" "$image_id" "$digest" <<'PY' import json, pathlib, sys, time path=pathlib.Path(sys.argv[1]) path.write_text(json.dumps({"state":"succeeded","message":"image-built-and-pushed","image":sys.argv[2],"imageId":sys.argv[3] or None,"repoDigests":[item for item in sys.argv[4].split(",") if item],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, ensure_ascii=False) + "\\n") PY } run_job >>"$log" 2>&1 || { rc=$? write_status failed "exit-$rc" exit "$rc" } JOB chmod +x "$state_dir/job.sh" : >"$state_dir/job.log" nohup "$state_dir/job.sh" >/dev/null 2>&1 & pid=$! printf '%s' "$pid" >"$state_dir/pid" printf '{"started":true,"pid":%s,"stateDir":"%s","statusCommand":"bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node %s --lane %s","logsCommand":"bun scripts/cli.ts hwlab nodes control-plane infra tools-image logs --node %s --lane %s"}\\n' "$pid" "$state_dir" ${shQuote(node.id)} ${shQuote(target.lane)} ${shQuote(node.id)} ${shQuote(target.lane)} `; } function argoApplyStartScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, desiredYaml: string): string { const stateDir = remoteJobStateDir(target, "argo"); const desiredEncoded = Buffer.from(desiredYaml, "utf8").toString("base64"); const rewritesEncoded = Buffer.from(JSON.stringify(target.argo.install.imageRewrites), "utf8").toString("base64"); const preloadEncoded = Buffer.from(JSON.stringify(target.argo.install.preloadImages), "utf8").toString("base64"); return ` set -eu state_dir=${shQuote(stateDir)} mkdir -p "$state_dir" if [ -s "$state_dir/pid" ] && kill -0 "$(cat "$state_dir/pid")" >/dev/null 2>&1; then printf '{"started":false,"reason":"job-already-running","pid":%s,"stateDir":"%s"}\\n' "$(cat "$state_dir/pid")" "$state_dir" exit 0 fi cat >"$state_dir/job.sh" <<'JOB' #!/bin/sh set -eu state_dir=${shQuote(stateDir)} namespace=${shQuote(target.argo.namespace)} manifest_url=${shQuote(target.argo.install.manifestUrl)} field_manager=${shQuote(target.argo.install.fieldManager)} readiness_timeout=${shQuote(String(target.argo.install.readinessTimeoutSeconds))} log="$state_dir/job.log" status="$state_dir/status.json" install_yaml="$state_dir/install.yaml" rendered_yaml="$state_dir/install.rendered.yaml" desired_yaml="$state_dir/desired.yaml" write_status() { state="$1"; shift message="$1"; shift || true python3 - "$status" "$state" "$message" <<'PY' import json, pathlib, sys, time path=pathlib.Path(sys.argv[1]) path.write_text(json.dumps({"state":sys.argv[2],"message":sys.argv[3],"updatedAt":time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())}, ensure_ascii=False) + "\\n") PY } { write_status running starting ${proxyExportBlock(node)} printf %s ${shQuote(desiredEncoded)} | base64 -d >"$desired_yaml" printf %s ${shQuote(rewritesEncoded)} | base64 -d >"$state_dir/image-rewrites.json" printf %s ${shQuote(preloadEncoded)} | base64 -d >"$state_dir/preload-images.json" kubectl create namespace "$namespace" --dry-run=client -o yaml | kubectl apply --server-side --field-manager="$field_manager" -f - || exit "$?" python3 - "$state_dir/preload-images.json" "$state_dir/image-rewrites.json" <<'PY' >"$state_dir/pull-images.sh" import json, pathlib, shlex, sys preload=json.loads(pathlib.Path(sys.argv[1]).read_text()) rewrites=json.loads(pathlib.Path(sys.argv[2]).read_text()) print("#!/bin/sh") print("set -eu") seen=set() for item in rewrites: pull=item["pullImage"] target=item["target"] if target in seen: continue seen.add(target) print("docker pull " + shlex.quote(pull)) print("docker tag " + shlex.quote(pull) + " " + shlex.quote(target)) print("docker push " + shlex.quote(target)) for image in preload: if image not in seen and image.startswith("127.0.0.1:5000/"): print("docker image inspect " + shlex.quote(image) + " >/dev/null") PY chmod +x "$state_dir/pull-images.sh" "$state_dir/pull-images.sh" || exit "$?" curl -fsSL --max-time 60 "$manifest_url" >"$install_yaml" || exit "$?" python3 - "$install_yaml" "$state_dir/image-rewrites.json" "$rendered_yaml" ${shQuote(target.argo.install.imagePullPolicy)} <<'PY' import json, pathlib, sys text=pathlib.Path(sys.argv[1]).read_text() rewrites=json.loads(pathlib.Path(sys.argv[2]).read_text()) for item in rewrites: text=text.replace(item["source"], item["target"]) policy=sys.argv[4] text=text.replace("imagePullPolicy: Always", "imagePullPolicy: " + policy) pathlib.Path(sys.argv[3]).write_text(text) PY kubectl apply --server-side --field-manager="$field_manager" -n "$namespace" -f "$rendered_yaml" || exit "$?" deadline=$(( $(date +%s) + readiness_timeout )) while [ "$(date +%s)" -lt "$deadline" ]; do kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && break sleep 5 done kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null || exit "$?" kubectl apply --server-side --field-manager="$field_manager" -f "$desired_yaml" || exit "$?" write_status succeeded argocd-install-applied } >>"$log" 2>&1 || { rc=$? write_status failed "exit-$rc" exit "$rc" } JOB chmod +x "$state_dir/job.sh" : >"$state_dir/job.log" nohup "$state_dir/job.sh" >/dev/null 2>&1 & pid=$! printf '%s' "$pid" >"$state_dir/pid" printf '{"started":true,"pid":%s,"stateDir":"%s","statusCommand":"bun scripts/cli.ts hwlab nodes control-plane infra argo status --node %s --lane %s","logsCommand":"bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node %s --lane %s"}\\n' "$pid" "$state_dir" ${shQuote(node.id)} ${shQuote(target.lane)} ${shQuote(node.id)} ${shQuote(target.lane)} `; } function remoteJobStatusScript(target: ControlPlaneTargetSpec, name: "tools-image" | "argo", tailLines: number): string { const stateDir = remoteJobStateDir(target, name); return ` set +e state_dir=${shQuote(stateDir)} status_file="$state_dir/status.json" log_file="$state_dir/job.log" pid_file="$state_dir/pid" running=false pid=null if [ -s "$pid_file" ]; then pid_raw="$(cat "$pid_file" 2>/dev/null || true)" if [ -n "$pid_raw" ] && kill -0 "$pid_raw" >/dev/null 2>&1; then running=true; pid="$pid_raw"; else pid="$pid_raw"; fi fi python3 - "$state_dir" "$status_file" "$log_file" "$running" "$pid" ${shQuote(String(tailLines))} <<'PY' import json, pathlib, sys state_dir=pathlib.Path(sys.argv[1]) status_path=pathlib.Path(sys.argv[2]) log_path=pathlib.Path(sys.argv[3]) running=sys.argv[4] == "true" pid=None if sys.argv[5] in ("", "null") else sys.argv[5] tail_lines=int(sys.argv[6]) status=None if status_path.exists(): try: status=json.loads(status_path.read_text()) except Exception as error: status={"parseError": str(error), "raw": status_path.read_text(errors="replace")[-1000:]} log_tail="" if log_path.exists(): lines=log_path.read_text(errors="replace").splitlines() log_tail="\\n".join(lines[-tail_lines:]) print(json.dumps({"stateDir": str(state_dir), "pid": pid, "running": running, "status": status, "logBytes": log_path.stat().st_size if log_path.exists() else 0, "logTail": log_tail}, ensure_ascii=False)) PY `; } function remoteJobLogs(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, name: "tools-image" | "argo", options: ToolsImageOptions | ArgoOptions): Record { const result = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, name, options.tailLines), options.timeoutSeconds); const parsed = parseRemoteJson(result.stdout); return { ok: result.exitCode === 0, command: `hwlab nodes control-plane infra ${name === "tools-image" ? "tools-image" : "argo"} logs`, configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, node: node.id, lane: target.lane, mutation: false, job: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, result: compactCommandResult(result), }; } function manifestObjectSummary(manifest: readonly Record[]): Record[] { return manifest.map((item) => { const metadata = record(item.metadata); return { kind: item.kind ?? null, namespace: metadata.namespace ?? null, name: metadata.name ?? null }; }); } function runTransK3s(kubeRoute: string, script: string, timeoutSeconds: number): CommandResult { return runCommand(["/root/.local/bin/trans", kubeRoute, "sh", "--", script], rootPath(), { timeoutMs: timeoutSeconds * 1000 }); } function proxyExportBlock(node: ControlPlaneNodeSpec): string { const proxy = node.egressProxy; if (proxy === null) return " : # no egress proxy configured\n"; const noProxy = [...new Set(["localhost", "127.0.0.1", "::1", "127.0.0.1:5000", "localhost:5000", ...proxy.noProxy])]; return ` proxy_ip="$(kubectl -n ${shQuote(proxy.namespace)} get svc ${shQuote(proxy.serviceName)} -o 'jsonpath={.spec.clusterIP}' 2>/dev/null || true)" if [ -z "$proxy_ip" ]; then echo "egress proxy service missing: ${proxy.namespace}/${proxy.serviceName}" >&2; exit 41; fi export HTTP_PROXY="http://$proxy_ip:${proxy.port}" export HTTPS_PROXY="$HTTP_PROXY" export ALL_PROXY="$HTTP_PROXY" export http_proxy="$HTTP_PROXY" export https_proxy="$HTTP_PROXY" export all_proxy="$HTTP_PROXY" export NO_PROXY=${shQuote(noProxy.join(","))} export no_proxy="$NO_PROXY" `; } function remoteJobStateDir(target: ControlPlaneTargetSpec, name: "tools-image" | "argo"): string { return `/tmp/unidesk-hwlab-node-control-plane/${target.id}/${name}`; } function shellJsonArray(items: readonly string[]): string { return JSON.stringify([...items]); } function parseRemoteJson(text: string): unknown { const trimmed = text.trim(); if (trimmed.length === 0) return null; try { return JSON.parse(trimmed); } catch { const start = trimmed.indexOf("{"); const end = trimmed.lastIndexOf("}"); if (start >= 0 && end > start) { try { return JSON.parse(trimmed.slice(start, end + 1)); } catch {} } } return null; } function asRecord(value: unknown, path: string): Record { if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`); return value as Record; } function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function stringField(obj: Record, key: string, path: string): string { const value = obj[key]; if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return value; } function optionalStringField(obj: Record, key: string, path: string): string | undefined { const value = obj[key]; if (value === undefined) return undefined; if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); return value; } function numberField(obj: Record, key: string, path: string): number { const value = obj[key]; if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`); return value; } function positiveConfigIntegerField(obj: Record, key: string, path: string): number { const value = obj[key]; if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`); return value; } function nonNegativeIntegerField(obj: Record, key: string, path: string): number { const value = obj[key]; if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`); return value; } function numberArrayField(obj: Record, key: string, path: string): number[] { const value = obj[key]; if (!Array.isArray(value) || value.some((item) => typeof item !== "number" || !Number.isInteger(item))) throw new Error(`${path}.${key} must be an array of integers`); return [...value] as number[]; } function stringArrayField(obj: Record, key: string, path: string): string[] { const value = obj[key]; if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) throw new Error(`${path}.${key} must be an array of non-empty strings`); return [...value] as string[]; } function stringRecordField(obj: Record, path: string): Record { const result: Record = {}; for (const [key, value] of Object.entries(obj)) { if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) throw new Error(`${path}.${key} has an unsupported key format`); if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); result[key] = value; } return result; } function booleanField(obj: Record, key: string, path: string): boolean { const value = obj[key]; if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); return value; } function boolField(obj: Record, key: string): boolean { return obj[key] === true; } function numberValue(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } function requiredOption(args: string[], name: string): string { const index = args.indexOf(name); if (index === -1) throw new Error(`${name} is required`); const value = args[index + 1]; if (value === undefined || value.startsWith("--") || value.length === 0) throw new Error(`${name} requires a value`); return value; } function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const index = args.indexOf(name); if (index === -1) return defaultValue; const raw = args[index + 1]; const value = Number(raw); if (!Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`); return Math.min(value, maxValue); } function compactCommandResult(result: CommandResult): Record { return { exitCode: result.exitCode, timedOut: result.timedOut, stdoutBytes: Buffer.byteLength(result.stdout), stderrBytes: Buffer.byteLength(result.stderr), stdoutTail: result.stdout.slice(-2000), stderrTail: result.stderr.slice(-2000), }; } function shQuote(value: string): string { return `'${value.replace(/'/gu, `'"'"'`)}'`; } function validateHttpsUrl(value: string, path: string): void { let parsed: URL; try { parsed = new URL(value); } catch { throw new Error(`${path} must be a valid URL`); } if (parsed.protocol !== "https:") throw new Error(`${path} must use https://`); } function sha256Short(text: string): string { return `sha256:${createHash("sha256").update(text).digest("hex")}`; }