// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/hwlab-node-impl.ts. // Moved mechanically from scripts/src/hwlab-node-impl.ts:4000-5260 for #903. // SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: YAML-first node/lane operations, including Workbench observability control commands. import { createHash, randomBytes } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand, type CommandResult } from "../command"; import { startJob } from "../jobs"; import { classifySshTcpPoolFailure } from "../ssh"; import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes"; import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source"; import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source"; import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source"; import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect"; import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render"; import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper"; import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render"; import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd"; import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help"; import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary"; import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql"; import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport"; import type { RenderedCliResult } from "../output"; import type { NodeRuntimeGitMirrorTargetSpec, NodeRuntimeRenderResult } from "./entry"; import { isCommandSuccess, nodeRuntimeGitopsRoot, nodeRuntimePipelineRunName, resolveNodeRuntimeLaneHead, runNodeK3sArgs, runtimeLaneCicdRepoEnsureScript, shortSha } from "./cleanup"; import { HWLAB_CI_NAMESPACE } from "./entry"; import { nodeRuntimeExpected, nodeRuntimeLocalPostgresExpectedAbsent, parseNodeScopedDelegatedOptions } from "./plan"; import { runTransHostScript } from "./public-exposure"; import { compactRuntimeCommand, runNodeHostScriptAsync } from "./runtime-common"; import { compactNodeRuntimeGitMirrorStatus, nodeRuntimeGitMirrorStatus } from "./status"; import { keyValueLinesFromText, numericField, optionValue, record, shellQuote } from "./utils"; import { externalPostgresBridgeStatus, externalPostgresSecretStatus, getNodeRuntimePipelineRun, isLocalPostgresObject, nodeRuntimeCodeAgentRuntimeStatus, nodeRuntimeRenderOverlay } from "./web-probe"; import { webObserveShort, webObserveText } from "./web-probe-observe"; import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes"; export function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush"): string { const prefix = action === "sync" ? mirror.syncJobPrefix : mirror.flushJobPrefix; return `${prefix}-${Date.now().toString(36)}`.slice(0, 63); } export function nodeRuntimeGitMirrorJobManifest( mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string, options: { discardStaleGitops?: boolean } = {}, ): Record { const volumes: Record[] = [ { name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) }, { name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } }, ]; const volumeMounts: Record[] = [ { name: "cache", mountPath: "/cache" }, { name: "script", mountPath: "/script", readOnly: true }, ]; if (mirror.githubTransport.mode === "ssh") { volumes.splice(1, 0, { name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } }); volumeMounts.splice(1, 0, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true }); } return { apiVersion: "batch/v1", kind: "Job", metadata: { name: jobName, namespace: mirror.namespace, labels: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/part-of": "hwlab-node-control-plane", "app.kubernetes.io/component": `${action}-controller`, "hwlab.pikastech.local/node": mirror.node, "hwlab.pikastech.local/lane": mirror.lane, "hwlab.pikastech.local/trigger": "manual-cli", }, }, spec: { backoffLimit: 0, activeDeadlineSeconds: 600, ttlSecondsAfterFinished: 3600, template: { metadata: { labels: { "app.kubernetes.io/name": "git-mirror", "app.kubernetes.io/part-of": "hwlab-node-control-plane", "app.kubernetes.io/component": `${action}-controller`, "hwlab.pikastech.local/node": mirror.node, "hwlab.pikastech.local/lane": mirror.lane, }, }, spec: { ...(mirror.egressProxy.mode === "host-route" ? { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet" } : {}), restartPolicy: "Never", volumes, containers: [{ name: action, image: mirror.toolsImage, imagePullPolicy: mirror.toolsImagePullPolicy, command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"], env: [ ...nodeRuntimeGitMirrorProxyEnv(mirror), ...nodeRuntimeGitMirrorGithubTransportEnv(mirror), ...(options.discardStaleGitops === true ? [{ name: "UNIDESK_GIT_MIRROR_DISCARD_STALE_GITOPS", value: "true" }] : []), ], volumeMounts, }], }, }, }, }; } export function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record[] { const proxy = mirror.egressProxy; if (proxy.mode === "direct") return []; const proxyUrl = proxy.mode === "host-route" ? proxy.proxyUrl : `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`; const noProxy = proxy.noProxy.join(","); return [ { name: "HTTP_PROXY", value: proxyUrl }, { name: "HTTPS_PROXY", value: proxyUrl }, { name: "ALL_PROXY", value: proxyUrl }, { name: "http_proxy", value: proxyUrl }, { name: "https_proxy", value: proxyUrl }, { name: "all_proxy", value: proxyUrl }, { name: "NO_PROXY", value: noProxy }, { name: "no_proxy", value: noProxy }, ]; } export function nodeRuntimeGitMirrorGithubTransportEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record[] { if (mirror.githubTransport.mode !== "https") return []; return [{ name: "GITHUB_TOKEN", valueFrom: { secretKeyRef: { name: mirror.githubTransport.tokenSecretName, key: mirror.githubTransport.tokenSecretKey } }, }]; } export function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGitMirrorTargetSpec): Record { const transport = mirror.githubTransport; if (transport.mode === "ssh") { return { mode: "ssh", secretName: mirror.secretName, privateKeySecretKey: transport.privateKeySecretKey, privateKeySourceRef: transport.privateKeySourceRef, privateKeySourceKey: transport.privateKeySourceKey, privateKeySourceEncoding: transport.privateKeySourceEncoding, knownHostsSecretKey: transport.knownHostsSecretKey, knownHostsSourceRef: transport.knownHostsSourceRef, knownHostsSourceKey: transport.knownHostsSourceKey, knownHostsSourceEncoding: transport.knownHostsSourceEncoding, valuesPrinted: false, }; } return { mode: "https", username: transport.username, tokenSecretName: transport.tokenSecretName, tokenSecretKey: transport.tokenSecretKey, tokenSourceRef: transport.tokenSourceRef, tokenSourceKey: transport.tokenSourceKey, valuesPrinted: false, }; } export function nodeRuntimeControlPlaneStatus(scoped: ReturnType): Record { const spec = scoped.spec; const probeTimeoutSeconds = Math.max(1, Math.min(60, scoped.timeoutSeconds)); const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit"); const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run"); const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null; const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null; const pipelineRun = pipelineRunOverride ?? (sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit)); const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], probeTimeoutSeconds); const namespaceExists = namespace.exitCode === 0; const postgresObjects = namespaceExists ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], probeTimeoutSeconds) : null; const localPostgresObjects = postgresObjects === null ? [] : postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec)); const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], probeTimeoutSeconds); const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], probeTimeoutSeconds); const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], probeTimeoutSeconds); const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u); const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun); const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True" ? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun) : null; const workloads = namespaceExists ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], probeTimeoutSeconds) : null; const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); const workloadReadinessProbe = namespaceExists ? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], probeTimeoutSeconds) : null; const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? ""); const bridge = externalPostgresBridgeStatus(spec, namespaceExists); const secrets = externalPostgresSecretStatus(spec, namespaceExists); const codeAgentRuntime = nodeRuntimeCodeAgentRuntimeStatus(spec, namespaceExists); const publicProbes = nodeRuntimePublicProbeStatus(spec); const gitMirror = nodeRuntimeGitMirrorStatus(scoped); const gitMirrorCompact = compactNodeRuntimeGitMirrorStatus(gitMirror); const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec); const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0; const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready); const localPostgresExpectedAbsent = nodeRuntimeLocalPostgresExpectedAbsent(spec); const localPostgresReady = localPostgresExpectedAbsent ? localPostgresObjects.length === 0 : localPostgresObjects.length > 0; const runtimeReady = namespaceExists && localPostgresReady && workloadsReady && (activeExternalPostgres === undefined || (bridge.ready && secrets.ready)) && codeAgentRuntime.ready === true; const codeAgentRuntimeDegradedReason = typeof codeAgentRuntime.degradedReason === "string" ? codeAgentRuntime.degradedReason : null; const runtimeDegradedReason = !namespaceExists ? "runtime-namespace-missing" : !localPostgresReady ? "runtime-local-postgres-not-ready" : !workloadsReady ? "runtime-workloads-not-ready" : activeExternalPostgres !== undefined && bridge.ready !== true ? "external-postgres-bridge-not-ready" : activeExternalPostgres !== undefined && secrets.ready !== true ? "external-postgres-secrets-not-ready" : codeAgentRuntime.ready !== true ? codeAgentRuntimeDegradedReason ?? "code-agent-runtime-not-ready" : "runtime-not-ready"; const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy"; const argoDiagnostics = argo.exitCode === 0 && !argoReady ? nodeRuntimeArgoDiagnostics(spec, probeTimeoutSeconds) : null; const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True"; const pipelineRunDegradedReason = typeof pipelineRunDiagnostics?.degradedReason === "string" ? pipelineRunDiagnostics.degradedReason : pipelineRunProbe === null || pipelineRunProbe.exists !== true ? "pipelinerun-not-found" : "pipelinerun-not-succeeded"; const publicReady = publicProbes.ready === true; const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true; const gitMirrorDegradedReason = gitMirrorCompact.sourceSnapshotReady === false ? "source-snapshot-missing" : gitMirrorCompact.pendingFlush === true ? "git-mirror-pending-flush" : "git-mirror-not-in-sync"; const targetGitopsRevision = nodeRuntimeTargetGitopsRevision(gitMirrorCompact); const argoDegradedReason = nodeRuntimeArgoDegradedReason({ argoCommandOk: argo.exitCode === 0, repoURL, expectedRepoURL: spec.argoRepoUrl, targetRevision, expectedTargetRevision: spec.gitopsBranch, path, expectedPath: spec.runtimePath, syncRevision, syncStatus, health, targetGitopsRevision, runtimeReady, publicReady, }); const degradedReason = nodeRuntimeStatusDegradedReason({ controlPlaneReady, pipelineRunReady, pipelineRunDegradedReason, gitMirrorReady, gitMirrorDegradedReason, argoReady, argoDegradedReason, runtimeReady, runtimeDegradedReason, publicReady, }); const fullStatus = { ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady && gitMirrorReady, command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`, mode: "node-scoped-runtime-status", mutation: false, node: scoped.node, lane: scoped.lane, sourceCommit, pipelineRun, expected: nodeRuntimeExpected(spec), sourceHead: head === null ? { ok: sourceCommit !== null, value: sourceCommit, source: "option" } : { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) }, controlPlane: { ready: controlPlaneReady, serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) }, pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) }, }, argo: { ready: argoReady, application: spec.app, repoURL, expectedRepoURL: spec.argoRepoUrl, targetRevision, path, syncRevision, targetGitopsRevision, syncStatus, health, diagnostics: argoDiagnostics, result: compactRuntimeCommand(argo), }, pipelineRun: pipelineRunProbe, pipelineRunDiagnostics, runtime: { ready: runtimeReady, namespace: spec.runtimeNamespace, namespaceExists, localPostgresObjects, localPostgresAbsent: namespaceExists && localPostgresObjects.length === 0, localPostgresExpectedAbsent, localPostgresReady, workloadNames, workloadCount: workloadNames.length, workloadReadiness, workloadsReady, workloadReadinessResult: workloadReadinessProbe === null ? null : compactRuntimeCommand(workloadReadinessProbe), workloadResult: workloads === null ? null : compactRuntimeCommand(workloads), externalPostgresBridge: bridge, externalPostgresSecrets: secrets, codeAgentRuntime, }, publicProbes, gitMirror: { ...gitMirror, compact: gitMirrorCompact, ready: gitMirrorReady, }, probes: { namespace: compactRuntimeCommand(namespace), postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects), }, degradedReason, next: { plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`, triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`, }, }; const summary = summarizeNodeRuntimeControlPlaneStatus(fullStatus, scoped); if (scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw")) return { ...fullStatus, summary }; return summary; } export function parseNodeRuntimeWorkloadReadiness(text: string): Array> { return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => { const [ref = "", counts = ""] = line.split(/\t/u); const [readyRaw = "", currentRaw = "", desiredRaw = ""] = counts.split("/"); const readyReplicas = nullableInteger(readyRaw); const currentReplicas = nullableInteger(currentRaw); const desiredReplicas = nullableInteger(desiredRaw); const ready = desiredReplicas !== null ? (readyReplicas ?? 0) >= desiredReplicas : readyReplicas !== null && currentReplicas !== null && readyReplicas >= currentReplicas; return { ref, readyReplicas, currentReplicas, desiredReplicas, ready, }; }); } export function nullableInteger(value: string): number | null { if (!/^[0-9]+$/u.test(value)) return null; return Number(value); } export function nodeRuntimeTargetGitopsRevision(gitMirrorCompact: Record): string | null { return typeof gitMirrorCompact.localGitops === "string" && /^[0-9a-f]{40}$/iu.test(gitMirrorCompact.localGitops) ? gitMirrorCompact.localGitops : typeof gitMirrorCompact.githubGitops === "string" && /^[0-9a-f]{40}$/iu.test(gitMirrorCompact.githubGitops) ? gitMirrorCompact.githubGitops : null; } export function nodeRuntimeArgoDegradedReason(input: { argoCommandOk: boolean; repoURL: string; expectedRepoURL: string; targetRevision: string; expectedTargetRevision: string; path: string; expectedPath: string; syncRevision: string; syncStatus: string; health: string; targetGitopsRevision: string | null; runtimeReady: boolean; publicReady: boolean; }): string | null { if (!input.argoCommandOk) return "argo-application-not-readable"; if (input.repoURL !== input.expectedRepoURL || input.targetRevision !== input.expectedTargetRevision || input.path !== input.expectedPath) { return "argo-application-spec-drift"; } const argoAtTarget = input.targetGitopsRevision !== null && input.syncRevision === input.targetGitopsRevision; if (argoAtTarget && input.syncStatus === "Synced" && input.health !== "Healthy") return "argo-health-progressing"; if (argoAtTarget && input.syncStatus !== "Synced" && input.runtimeReady && input.publicReady) return "argo-health-progressing"; if (argoAtTarget) return "argo-target-revision-progressing"; if (input.targetGitopsRevision !== null) return "argo-revision-not-observed"; return "argo-not-synced-healthy"; } export function nodeRuntimeStatusDegradedReason(input: { controlPlaneReady: boolean; pipelineRunReady: boolean; pipelineRunDegradedReason: string; gitMirrorReady: boolean; gitMirrorDegradedReason: string; argoReady: boolean; argoDegradedReason: string | null; runtimeReady: boolean; runtimeDegradedReason: string; publicReady: boolean; }): string | undefined { if (!input.controlPlaneReady) return "control-plane-not-ready"; if (!input.pipelineRunReady) return input.pipelineRunDegradedReason; if (!input.gitMirrorReady) return input.gitMirrorDegradedReason; if (!input.argoReady) return input.argoDegradedReason ?? "argo-not-synced-healthy"; if (!input.runtimeReady) return input.runtimeDegradedReason; if (!input.publicReady) return "public-probe-not-ready"; return undefined; } export function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record { const web = publicHttpProbe("web", spec.publicWebUrl); const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live")); const ready = web.ok === true && apiHealth.ok === true; const targetHost = nodeRuntimeTargetHostPublicProbeStatus(spec); return { ready, web, apiHealth, targetHost, diagnostic: nodeRuntimePublicProbeDiagnostic(ready, targetHost), }; } export function publicHttpProbe(kind: string, url: string): Record { const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "/dev/null", "-w", "%{http_code}", url], repoRoot, { timeoutMs: 15_000 }); const httpStatus = nullableInteger(result.stdout.trim().slice(-3)); return { kind, url, ok: result.exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400, httpStatus, result: compactRuntimeCommand(result), }; } export function nodeRuntimeTargetHostPublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record { const webUrl = spec.publicWebUrl; const apiHealthUrl = joinUrlPath(spec.publicApiUrl, "/health/live"); const script = [ "set -eu", `web_url=${shellQuote(webUrl)}`, `api_url=${shellQuote(apiHealthUrl)}`, "probe() {", " name=\"$1\"", " url=\"$2\"", " err_file=$(mktemp)", " set +e", " http_status=$(curl -k -sS --connect-timeout 5 --max-time 12 -o /dev/null -w '%{http_code}' \"$url\" 2>\"$err_file\")", " rc=$?", " set -e", " error_text=$(tr '\\r\\n\\t' ' ' <\"$err_file\" | tail -c 600)", " rm -f \"$err_file\"", " printf '%sUrl\\t%s\\n' \"$name\" \"$url\"", " printf '%sExitCode\\t%s\\n' \"$name\" \"$rc\"", " printf '%sHttpStatus\\t%s\\n' \"$name\" \"$http_status\"", " printf '%sError\\t%s\\n' \"$name\" \"$error_text\"", "}", "probe web \"$web_url\"", "probe apiHealth \"$api_url\"", ].join("\n"); const result = runTransHostScript(spec.nodeId, script, "", 35); const fields = keyValueLinesFromText(result.stdout); const web = targetHostPublicHttpProbeFromFields("web", fields, webUrl, result.exitCode === 0); const apiHealth = targetHostPublicHttpProbeFromFields("apiHealth", fields, apiHealthUrl, result.exitCode === 0); return { node: spec.nodeId, ready: web.ok === true && apiHealth.ok === true, probeAvailable: result.exitCode === 0 && result.timedOut !== true, web, apiHealth, result: compactRuntimeCommand(result), }; } export function targetHostPublicHttpProbeFromFields(kind: string, fields: Record, fallbackUrl: string, transportOk: boolean): Record { const exitCode = numericField(fields[`${kind}ExitCode`]); const httpStatus = numericField(fields[`${kind}HttpStatus`]); const error = fields[`${kind}Error`] ?? ""; return { kind, url: fields[`${kind}Url`] ?? fallbackUrl, ok: transportOk && exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400, httpStatus, exitCode, error: error.length > 0 ? error.slice(0, 600) : null, }; } export function nodeRuntimePublicProbeDiagnostic(publicReady: boolean, targetHost: Record): Record { const targetWeb = record(targetHost.web); const targetApiHealth = record(targetHost.apiHealth); const targetReady = targetHost.ready === true; if (!publicReady) { return { kind: "public-entry-probe-failed", affectsUserEntry: true, targetHostReady: targetReady, message: "control-plane public probe failed; treat this as a public endpoint readiness failure before using web-probe closeout evidence.", nextAction: "run web-probe run for the same node/lane after checking publicProbe.web and publicProbe.apiHealth", }; } if (targetHost.probeAvailable !== true) { return { kind: "target-host-public-probe-unavailable", affectsUserEntry: false, targetHostReady: false, message: "control-plane public probe passed, but the target host diagnostic probe could not run; this does not invalidate user-entry evidence.", nextAction: "use control-plane publicProbe and web-probe evidence for closeout; inspect target host SSH/trans health separately if host-side CLI must call the public URL", }; } if (!targetReady) { return { kind: "target-host-public-egress-mismatch", affectsUserEntry: false, targetHostReady: false, failed: { web: targetWeb.ok === true ? null : { httpStatus: targetWeb.httpStatus ?? null, exitCode: targetWeb.exitCode ?? null, error: targetWeb.error ?? null }, apiHealth: targetApiHealth.ok === true ? null : { httpStatus: targetApiHealth.httpStatus ?? null, exitCode: targetApiHealth.exitCode ?? null, error: targetApiHealth.error ?? null }, }, message: "control-plane public probe passed, but the target host cannot reach the same public URLs; classify this as target-host egress/hairpin diagnostics, not a public endpoint failure.", nextAction: "use control-plane publicProbe and web-probe evidence for issue closeout; track host-side public URL access separately if hwlab-cli must run on that host", }; } return { kind: "public-entry-and-target-host-ok", affectsUserEntry: false, targetHostReady: true, message: "control-plane public probe and target-host public URL probe both passed.", nextAction: null, }; } export function joinUrlPath(baseUrl: string, suffix: string): string { return `${baseUrl.replace(/\/+$/u, "")}/${suffix.replace(/^\/+/u, "")}`; } export function compactNodeRuntimeTaskRunDiagnostic(value: unknown): string { const item = record(value); const pipelineTask = webObserveText(item.pipelineTask ?? item.pipelineTaskName); const taskRun = webObserveText(item.taskRun ?? item.name); const reason = webObserveText(item.reason ?? item.status ?? item.message); const left = [pipelineTask, taskRun].filter(Boolean).join("/"); return [left, reason ? `(${webObserveShort(reason, 36)})` : ""].filter(Boolean).join(""); } export function nodeRuntimePipelinePendingTaskRunSummaries( spec: HwlabRuntimeLaneSpec, pendingTaskRuns: Array>, pods: Array>, ): Array> { return pendingTaskRuns.slice(0, 16).map((taskRun) => { const taskRunName = stringOrNull(taskRun.name); const podName = stringOrNull(taskRun.podName); const pod = pods.find((item) => item.name === podName || (taskRunName !== null && item.taskRun === taskRunName)) ?? {}; const containers = Array.isArray(pod.containers) ? pod.containers.map(record) : []; const initContainers = Array.isArray(pod.initContainers) ? pod.initContainers.map(record) : []; const waitingContainers = [...initContainers, ...containers].filter((container) => container.state === "waiting"); const runningContainers = [...initContainers, ...containers].filter((container) => container.state === "running"); return { name: taskRunName, taskRun: taskRunName, pipelineTask: taskRun.pipelineTask ?? null, taskRef: taskRun.taskRef ?? null, status: taskRun.status ?? null, reason: taskRun.reason ?? null, message: diagnosticText(taskRun.message), pod: podName, podPhase: pod.phase ?? null, scheduled: pod.scheduled ?? null, scheduledReason: pod.scheduledReason ?? null, scheduledMessage: diagnosticText(pod.scheduledMessage), waitingContainers, runningContainers, taskRunCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["get", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName, "-o", "yaml"]), taskRunDescribeCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName]), podDescribeCommand: podName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "pod", "-n", HWLAB_CI_NAMESPACE, podName]), podLogsCommand: podName === null ? null : nodeRuntimePipelineLogsCommand(spec, podName, null), }; }); } export function summarizeNodeRuntimeControlPlaneStatus(status: Record, scoped: ReturnType): Record { const pipelineRun = record(status.pipelineRun); const pipelineRunDiagnostics = record(status.pipelineRunDiagnostics); const argo = record(status.argo); const runtime = record(status.runtime); const publicProbes = record(status.publicProbes); const gitMirror = record(status.gitMirror); const gitMirrorCompact = record(gitMirror.compact); const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : []; const readyWorkloads = workloadReadiness.filter((item) => item.ready === true).length; const notReadyWorkloads = workloadReadiness.filter((item) => item.ready !== true).map((item) => item.ref).filter(Boolean); const workloadCount = typeof runtime.workloadCount === "number" ? runtime.workloadCount : workloadReadiness.length; const webProbe = record(publicProbes.web); const apiProbe = record(publicProbes.apiHealth); const targetHostProbe = record(publicProbes.targetHost); const targetHostWebProbe = record(targetHostProbe.web); const targetHostApiProbe = record(targetHostProbe.apiHealth); const publicProbeDiagnostic = record(publicProbes.diagnostic); return { ok: status.ok === true, command: status.command, mode: "node-scoped-runtime-status-summary", mutation: false, node: status.node, lane: status.lane, sourceCommit: status.sourceCommit, degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null, pipelineRun: { name: pipelineRun.name ?? null, exists: pipelineRun.exists === true, status: pipelineRun.status ?? null, reason: pipelineRun.reason ?? null, message: pipelineRun.message ?? null, createdAt: pipelineRun.createdAt ?? null, ready: pipelineRun.status === "True", diagnostics: Object.keys(pipelineRunDiagnostics).length === 0 ? null : { degradedReason: pipelineRunDiagnostics.degradedReason ?? null, taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null, podCount: pipelineRunDiagnostics.podCount ?? null, failedTaskRunCount: pipelineRunDiagnostics.failedTaskRunCount ?? null, failedTaskRuns: pipelineRunDiagnostics.failedTaskRuns ?? [], failureSummary: pipelineRunDiagnostics.failureSummary ?? null, pendingTaskRuns: pipelineRunDiagnostics.pendingTaskRuns ?? [], unscheduledPods: pipelineRunDiagnostics.unscheduledPods ?? [], schedulingMessages: pipelineRunDiagnostics.schedulingMessages ?? [], next: pipelineRunDiagnostics.next ?? null, }, }, argo: { application: argo.application ?? null, ready: argo.ready === true, syncRevision: argo.syncRevision ?? null, targetGitopsRevision: argo.targetGitopsRevision ?? null, revisionObserved: typeof argo.targetGitopsRevision === "string" && argo.syncRevision === argo.targetGitopsRevision, syncStatus: argo.syncStatus ?? null, health: argo.health ?? null, diagnostics: compactArgoDiagnostics(record(argo.diagnostics)), }, runtime: { namespace: runtime.namespace ?? null, namespaceExists: runtime.namespaceExists === true, ready: runtime.ready === true, workloadCount, workloadReady: `${readyWorkloads}/${workloadReadiness.length}`, notReadyWorkloads, localPostgresAbsent: runtime.localPostgresAbsent === true, localPostgresExpectedAbsent: runtime.localPostgresExpectedAbsent === true, localPostgresReady: runtime.localPostgresReady === true, externalPostgresReady: runtime.externalPostgresBridge === undefined && runtime.externalPostgresSecrets === undefined ? null : record(runtime.externalPostgresBridge).ready === true && record(runtime.externalPostgresSecrets).ready === true, codeAgentRuntimeReady: record(runtime.codeAgentRuntime).required === true ? record(runtime.codeAgentRuntime).ready === true : null, codeAgentRuntimeReason: typeof record(runtime.codeAgentRuntime).degradedReason === "string" ? record(runtime.codeAgentRuntime).degradedReason : null, }, publicProbe: { ready: publicProbes.ready === true, web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null }, apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null }, targetHost: Object.keys(targetHostProbe).length === 0 ? null : { node: targetHostProbe.node ?? status.node ?? null, ready: targetHostProbe.ready === true, probeAvailable: targetHostProbe.probeAvailable === true, web: { url: targetHostWebProbe.url ?? null, ok: targetHostWebProbe.ok === true, httpStatus: targetHostWebProbe.httpStatus ?? null, exitCode: targetHostWebProbe.exitCode ?? null, error: targetHostWebProbe.error ?? null, }, apiHealth: { url: targetHostApiProbe.url ?? null, ok: targetHostApiProbe.ok === true, httpStatus: targetHostApiProbe.httpStatus ?? null, exitCode: targetHostApiProbe.exitCode ?? null, error: targetHostApiProbe.error ?? null, }, }, diagnostic: Object.keys(publicProbeDiagnostic).length === 0 ? null : publicProbeDiagnostic, }, gitMirror: { ready: gitMirror.ready === true, localSource: gitMirrorCompact.localSource ?? null, githubSource: gitMirrorCompact.githubSource ?? null, sourceAuthority: gitMirrorCompact.sourceAuthority ?? null, sourceStageRef: gitMirrorCompact.sourceStageRef ?? null, sourceSnapshot: gitMirrorCompact.sourceSnapshot ?? null, sourceSnapshotReady: gitMirrorCompact.sourceSnapshotReady === true, localGitops: gitMirrorCompact.localGitops ?? null, githubGitops: gitMirrorCompact.githubGitops ?? null, pendingFlush: gitMirrorCompact.pendingFlush === true, flushNeeded: gitMirrorCompact.flushNeeded === true, githubInSync: gitMirrorCompact.githubInSync === true, }, nextAction: nodeRuntimeStatusNextAction(status, scoped), next: { full: `${nodeRuntimeStatusCommand(scoped)} --full`, plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`, triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`, gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`, webProbe: `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}`, }, }; } function nodeRuntimeArgoDiagnostics(spec: HwlabRuntimeLaneSpec, timeoutSeconds: number): Record { const resourceTemplate = `{{range .status.resources}}{{.kind}}{{"\\t"}}{{.namespace}}{{"\\t"}}{{.name}}{{"\\t"}}{{.status}}{{"\\t"}}{{with .health}}{{.status}}{{"\\t"}}{{printf "%.500s" .message}}{{else}}{{"\\t"}}{{end}}{{"\\n"}}{{end}}`; const operationTemplate = `{{with .status.operationState}}{{.phase}}{{"\\t"}}{{printf "%.500s" .message}}{{"\\t"}}{{.startedAt}}{{"\\t"}}{{.finishedAt}}{{"\\t"}}{{with .syncResult}}{{.revision}}{{"\\t"}}{{with .source}}{{.repoURL}}{{end}}{{end}}{{"\\n"}}{{with .syncResult}}{{range .resources}}{{.group}}{{"\\t"}}{{.kind}}{{"\\t"}}{{.namespace}}{{"\\t"}}{{.name}}{{"\\t"}}{{.status}}{{"\\t"}}{{printf "%.500s" .message}}{{"\\t"}}{{.hookPhase}}{{"\\t"}}{{.syncPhase}}{{"\\n"}}{{end}}{{end}}{{end}}`; const conditionTemplate = `{{range .status.conditions}}{{.type}}{{"\\t"}}{{printf "%.500s" .message}}{{"\\t"}}{{.lastTransitionTime}}{{"\\n"}}{{end}}`; const eventTemplate = `{{range .items}}{{.type}}{{"\\t"}}{{.reason}}{{"\\t"}}{{printf "%.500s" .message}}{{"\\t"}}{{.count}}{{"\\t"}}{{.lastTimestamp}}{{"\\n"}}{{end}}`; const resourceResult = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", `go-template=${resourceTemplate}`], timeoutSeconds); const operationResult = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", `go-template=${operationTemplate}`], timeoutSeconds); const conditionResult = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", `go-template=${conditionTemplate}`], timeoutSeconds); const eventsResult = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "events", "--field-selector", `involvedObject.name=${spec.app}`, "--sort-by=.lastTimestamp", "-o", `go-template=${eventTemplate}`], timeoutSeconds); const resources = argoResourceRows(resourceResult.stdout); const problemResources = resources.filter((item) => { const sync = typeof item.status === "string" ? item.status : null; const health = typeof item.healthStatus === "string" ? item.healthStatus : null; return (sync !== null && sync !== "Synced") || (health !== null && health !== "Healthy"); }).slice(0, 12); const operation = argoOperationRows(operationResult.stdout); const conditions = argoConditionRows(conditionResult.stdout).slice(-8); const events = argoEventRows(eventsResult.stdout).slice(-8); return { ok: resourceResult.exitCode === 0 && operationResult.exitCode === 0 && conditionResult.exitCode === 0, application: spec.app, resourceCount: resources.length, problemResourceCount: problemResources.length, problemResources, operationState: operation.state, operationResources: operation.resources.slice(0, 12), conditions, events, result: compactRuntimeCommand(operationResult), resourcesResult: compactRuntimeCommand(resourceResult), conditionsResult: compactRuntimeCommand(conditionResult), eventsResult: compactRuntimeCommand(eventsResult), valuesPrinted: false, }; } function compactArgoDiagnostics(diagnostics: Record): Record | null { if (Object.keys(diagnostics).length === 0) return null; return { ok: diagnostics.ok === true, problemResourceCount: diagnostics.problemResourceCount ?? null, problemResources: Array.isArray(diagnostics.problemResources) ? diagnostics.problemResources.slice(0, 8) : [], operationState: diagnostics.operationState ?? null, operationResources: Array.isArray(diagnostics.operationResources) ? diagnostics.operationResources.slice(0, 8) : [], conditions: Array.isArray(diagnostics.conditions) ? diagnostics.conditions.slice(0, 6) : [], events: Array.isArray(diagnostics.events) ? diagnostics.events.slice(0, 6) : [], valuesPrinted: false, }; } function shortDiagnosticText(value: unknown): string | null { if (typeof value !== "string" || value.length === 0) return null; return webObserveShort(value.replace(/\s+/gu, " ").trim(), 500); } function argoResourceRows(text: string): Record[] { return text.split(/\r?\n/u).map((line) => { const [kind = "", namespace = "", name = "", status = "", healthStatus = "", healthMessage = ""] = line.split("\t"); if (kind.length === 0 && name.length === 0) return null; return { kind: kind || null, namespace: namespace || null, name: name || null, status: status || null, healthStatus: healthStatus || null, healthMessage: shortDiagnosticText(healthMessage), }; }).filter((item): item is Record => item !== null); } function argoOperationRows(text: string): { state: Record; resources: Record[] } { const lines = text.split(/\r?\n/u).filter((line) => line.length > 0); const [phase = "", message = "", startedAt = "", finishedAt = "", syncResultRevision = "", syncResultSource = ""] = (lines.shift() ?? "").split("\t"); return { state: { phase: phase || null, message: shortDiagnosticText(message), startedAt: startedAt || null, finishedAt: finishedAt || null, syncResultRevision: syncResultRevision || null, syncResultSource: syncResultSource || null, }, resources: lines.map((line) => { const [group = "", kind = "", namespace = "", name = "", status = "", resourceMessage = "", hookPhase = "", syncPhase = ""] = line.split("\t"); return { group: group || null, kind: kind || null, namespace: namespace || null, name: name || null, status: status || null, message: shortDiagnosticText(resourceMessage), hookPhase: hookPhase || null, syncPhase: syncPhase || null, }; }).filter((item) => item.status !== "Synced" || item.message !== null || item.hookPhase !== null), }; } function argoConditionRows(text: string): Record[] { return text.split(/\r?\n/u).map((line) => { const [type = "", message = "", lastTransitionTime = ""] = line.split("\t"); if (type.length === 0) return null; return { type, message: shortDiagnosticText(message), lastTransitionTime: lastTransitionTime || null }; }).filter((item): item is Record => item !== null); } function argoEventRows(text: string): Record[] { return text.split(/\r?\n/u).map((line) => { const [type = "", reason = "", message = "", count = "", lastTimestamp = ""] = line.split("\t"); if (type.length === 0 && reason.length === 0) return null; return { type: type || null, reason: reason || null, message: shortDiagnosticText(message), count: numericField(count), lastTimestamp: lastTimestamp || null }; }).filter((item): item is Record => item !== null); } export function nodeRuntimeStatusNextAction(status: Record, scoped: ReturnType): string { const reason = typeof status.degradedReason === "string" ? status.degradedReason : null; if (reason === null) return `${nodeRuntimeStatusCommand(scoped)} --full`; if (reason === "control-plane-not-ready" || reason === "runtime-namespace-missing") { return `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`; } if (reason === "runtime-not-ready") return `${nodeRuntimeStatusCommand(scoped)} --full`; if (reason === "argo-not-synced-healthy") { return `bun scripts/cli.ts hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane} --confirm`; } if (reason === "argo-revision-not-observed" || reason === "argo-target-revision-progressing" || reason === "argo-health-progressing") { return `${nodeRuntimeStatusCommand(scoped)} --full`; } if (reason === "pipelinerun-not-succeeded") { return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`; } if (reason === "node-runtime-ci-taskrun-pending") { const next = record(record(status.pipelineRunDiagnostics).next); const pendingTaskRun = typeof next.pendingTaskRun === "string" ? next.pendingTaskRun : null; return pendingTaskRun ?? `${nodeRuntimeStatusCommand(scoped)} --full`; } if (reason === "node-runtime-ci-step-publish-failed") { return `bun scripts/cli.ts platform-infra sub2api status --target ${scoped.node}`; } if (reason === "node-runtime-ci-taskrun-failed") { const next = record(record(status.pipelineRunDiagnostics).next); const failedStepLogs = typeof next.failedStepLogs === "string" ? next.failedStepLogs : null; return failedStepLogs ?? `${nodeRuntimeStatusCommand(scoped)} --full`; } if (reason === "node-runtime-ci-pod-capacity-exhausted" || reason === "node-runtime-ci-pod-unschedulable") { return `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane} --min-age-minutes 60 --limit 20 --dry-run`; } if (reason === "public-probe-not-ready") { return `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}`; } if (reason === "source-snapshot-missing") { return `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`; } if (reason === "git-mirror-pending-flush") { return `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`; } if (reason === "git-mirror-not-in-sync") { return `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane} --full`; } return `${nodeRuntimeStatusCommand(scoped)} --full`; } export function nodeRuntimeStatusCommand(scoped: ReturnType): string { const sourceCommit = optionValue(scoped.originalArgs, "--source-commit"); const pipelineRun = optionValue(scoped.originalArgs, "--pipeline-run"); return [ `bun scripts/cli.ts hwlab nodes control-plane status --node ${shellQuote(scoped.node)} --lane ${shellQuote(scoped.lane)}`, sourceCommit === undefined ? "" : `--source-commit ${shellQuote(sourceCommit)}`, pipelineRun === undefined ? "" : `--pipeline-run ${shellQuote(pipelineRun)}`, ].filter(Boolean).join(" "); } export function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record { const taskRunTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineTask"}}{{"\\t"}}{{if .spec.taskRef}}{{.spec.taskRef.name}}{{end}}{{"\\t"}}{{with index .status.conditions 0}}{{.status}}{{"\\t"}}{{.reason}}{{else}}{{"\\t"}}{{end}}{{"\\t"}}{{.status.podName}}{{"\\t"}}{{with index .status.conditions 0}}{{printf "%.600s" .message}}{{end}}{{"\\n"}}{{end}}`; const podTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/taskRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "PodScheduled"}}{{.status}}|{{.reason}}|{{printf "%.600s" .message}}{{end}}{{end}}{{"\\t"}}{{range .status.initContainerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\t"}}{{range .status.containerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\n"}}{{end}}`; const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${taskRunTemplate}`], 60); const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${podTemplate}`], 60); const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRunsFromTsv(taskRunsResult.stdout) : []; const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPodsFromTsv(podsResult.stdout) : []; const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False"); const failedTaskRuns = taskRuns.filter((item) => item.status === "False"); const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods); const pendingTaskRunSummaries = nodeRuntimePipelinePendingTaskRunSummaries(spec, pendingTaskRuns, pods); const stepPublishFailures = failedTaskRunSummaries.filter((item) => item.container === "step-publish" || item.step === "publish" || item.step === "step-publish"); const unscheduledPods = pods.filter((item) => item.scheduled === false); const schedulingMessages = unscheduledPods .map((item) => typeof item.scheduledMessage === "string" ? item.scheduledMessage : "") .filter((message) => message.length > 0); const tooManyPods = schedulingMessages.some((message) => /too many pods/iu.test(message)); const failureSummary = failedTaskRunSummaries.length > 0 ? { failedTaskRunCount: failedTaskRuns.length, failedStepCount: failedTaskRunSummaries.length, stepPublishFailureCount: stepPublishFailures.length, firstFailure: failedTaskRunSummaries[0] ?? null, stepFailures: failedTaskRunSummaries.slice(0, 8), nextAction: stepPublishFailures.length > 0 ? "step-publish failed in a service build; first distinguish platform-infra Sub2API/proxy health from a single upstream transient, then choose controlled rerun or artifact-publish/envRecipe retry fix." : failedTaskRunSummaries.length > 0 ? "Inspect the failed TaskRun and bounded pod log command before rerunning the control-plane trigger." : null, } : null; return { ok: taskRunsResult.exitCode === 0 && podsResult.exitCode === 0, pipelineRun, taskRuns, pods, taskRunCount: taskRuns.length, podCount: pods.length, failedTaskRunCount: failedTaskRuns.length, failedTaskRuns: failedTaskRunSummaries, stepPublishFailures, failureSummary, pendingTaskRuns: pendingTaskRunSummaries, pendingTaskRunCount: pendingTaskRunSummaries.length, unscheduledPods, schedulingMessages, degradedReason: tooManyPods ? "node-runtime-ci-pod-capacity-exhausted" : unscheduledPods.length > 0 ? "node-runtime-ci-pod-unschedulable" : stepPublishFailures.length > 0 ? "node-runtime-ci-step-publish-failed" : failedTaskRunSummaries.length > 0 ? "node-runtime-ci-taskrun-failed" : pendingTaskRuns.length > 0 ? "node-runtime-ci-taskrun-pending" : undefined, query: { taskRuns: compactRuntimeCommand(taskRunsResult), pods: compactRuntimeCommand(podsResult), }, next: tooManyPods || unscheduledPods.length > 0 ? { cleanupRuns: `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 60 --limit 20 --dry-run` } : stepPublishFailures.length > 0 ? { sub2apiStatus: `bun scripts/cli.ts platform-infra sub2api status --target ${spec.nodeId}`, sub2apiValidate: `bun scripts/cli.ts platform-infra sub2api validate --target ${spec.nodeId}`, failedStepLogs: stepPublishFailures[0]?.logCommand ?? null, rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm --rerun`, } : failedTaskRunSummaries.length > 0 ? { failedStepLogs: failedTaskRunSummaries[0]?.logCommand ?? null, failedTaskRun: failedTaskRunSummaries[0]?.taskRunCommand ?? null, status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`, } : pendingTaskRunSummaries.length > 0 ? { pendingTaskRun: pendingTaskRunSummaries[0]?.taskRunDescribeCommand ?? pendingTaskRunSummaries[0]?.taskRunCommand ?? null, pendingPod: pendingTaskRunSummaries[0]?.podDescribeCommand ?? null, pendingPodLogs: pendingTaskRunSummaries[0]?.podLogsCommand ?? null, status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`, } : undefined, }; } export function nodeRuntimePipelineDiagnosticTaskRunsFromTsv(text: string): Array> { return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => { const parts = line.split("\t"); const [name = "", pipelineTask = "", taskRef = "", status = "", reason = "", podName = "", ...messageParts] = parts; const message = messageParts.join("\t"); return { name: stringOrNull(name), pipelineTask: stringOrNull(pipelineTask), taskRef: stringOrNull(taskRef), status: stringOrNull(status), reason: stringOrNull(reason), message: diagnosticText(message), podName: stringOrNull(podName), steps: [], failedSteps: [], }; }); } export function nodeRuntimePipelineDiagnosticPodsFromTsv(text: string): Array> { return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => { const [name = "", taskRun = "", phase = "", nodeName = "", scheduledRaw = "", initRaw = "", containersRaw = ""] = line.split("\t"); const [scheduledStatus = "", scheduledReason = "", scheduledMessage = ""] = scheduledRaw.split("|"); const initContainers = nodeRuntimePipelineContainerStatusesFromCompact(initRaw); const containers = nodeRuntimePipelineContainerStatusesFromCompact(containersRaw); const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true); return { name: stringOrNull(name), taskRun: stringOrNull(taskRun), phase: stringOrNull(phase), nodeName: stringOrNull(nodeName), scheduled: scheduledStatus.length === 0 ? null : scheduledStatus === "True", scheduledReason: stringOrNull(scheduledReason), scheduledMessage: diagnosticText(scheduledMessage), initContainers, containers, failedContainers, }; }); } export function nodeRuntimePipelineContainerStatusesFromCompact(text: string): Array> { return text.split(",").map((item) => item.trim()).filter(Boolean).map((item) => { const [name = "", stateOrExit = "", reason = ""] = item.split(":"); const exitCode = /^[0-9]+$/u.test(stateOrExit) ? Number(stateOrExit) : null; const state = exitCode !== null ? "terminated" : stateOrExit === "waiting" ? "waiting" : stateOrExit === "running" ? "running" : null; return { name: stringOrNull(name), ready: null, restartCount: null, state, failed: exitCode !== null && exitCode !== 0, exitCode, reason: stringOrNull(reason), message: null, startedAt: null, finishedAt: null, }; }); } export function nodeRuntimePipelineDiagnosticTaskRuns(json: Record): Array> { const items = Array.isArray(json.items) ? json.items.map(record) : []; return items.map((item) => { const metadata = record(item.metadata); const labels = record(metadata.labels); const spec = record(item.spec); const taskRef = record(spec.taskRef); const status = record(item.status); const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : []; const condition = conditions[0] ?? {}; const steps = nodeRuntimePipelineDiagnosticSteps(status.steps); const failedSteps = steps.filter((step) => step.failed === true); return { name: metadata.name ?? null, pipelineTask: labels["tekton.dev/pipelineTask"] ?? null, taskRef: taskRef.name ?? null, status: condition.status ?? null, reason: condition.reason ?? null, message: diagnosticText(condition.message), podName: status.podName ?? null, steps, failedSteps, }; }); } export function nodeRuntimePipelineDiagnosticPods(json: Record): Array> { const items = Array.isArray(json.items) ? json.items.map(record) : []; return items.map((item) => { const metadata = record(item.metadata); const labels = record(metadata.labels); const spec = record(item.spec); const status = record(item.status); const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : []; const scheduled = conditions.find((condition) => condition.type === "PodScheduled"); const initContainers = nodeRuntimeContainerStatusSummaries(status.initContainerStatuses); const containers = nodeRuntimeContainerStatusSummaries(status.containerStatuses); const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true); return { name: metadata.name ?? null, taskRun: labels["tekton.dev/taskRun"] ?? null, phase: status.phase ?? null, nodeName: spec.nodeName ?? null, scheduled: scheduled === undefined ? null : scheduled.status === "True", scheduledReason: scheduled?.reason ?? null, scheduledMessage: diagnosticText(scheduled?.message), initContainers, containers, failedContainers, }; }); } export function nodeRuntimePipelineDiagnosticSteps(value: unknown): Array> { const steps = Array.isArray(value) ? value.map(record) : []; return steps.map((step) => { const terminated = record(step.terminated); const running = record(step.running); const waiting = record(step.waiting); const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null; const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null; const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null; const state = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null; return { name: step.name ?? null, container: step.container ?? null, state, failed: exitCode !== null && exitCode !== 0, exitCode, reason: terminatedReason ?? waitingReason, message: diagnosticText(terminated.message ?? waiting.message), startedAt: terminated.startedAt ?? running.startedAt ?? null, finishedAt: terminated.finishedAt ?? null, }; }); } export function nodeRuntimeContainerStatusSummaries(value: unknown): Array> { const containers = Array.isArray(value) ? value.map(record) : []; return containers.map((container) => { const state = record(container.state); const terminated = record(state.terminated); const waiting = record(state.waiting); const running = record(state.running); const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null; const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null; const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null; const stateName = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null; return { name: container.name ?? null, ready: container.ready === true, restartCount: typeof container.restartCount === "number" ? container.restartCount : null, state: stateName, failed: exitCode !== null && exitCode !== 0, exitCode, reason: terminatedReason ?? waitingReason, message: diagnosticText(terminated.message ?? waiting.message), startedAt: terminated.startedAt ?? running.startedAt ?? null, finishedAt: terminated.finishedAt ?? null, }; }); } export function nodeRuntimePipelineFailedTaskRunSummaries( spec: HwlabRuntimeLaneSpec, failedTaskRuns: Array>, pods: Array>, ): Array> { const summaries: Array> = []; for (const taskRun of failedTaskRuns) { const taskRunName = stringOrNull(taskRun.name); const podName = stringOrNull(taskRun.podName); const pod = pods.find((item) => item.name === podName || (taskRunName !== null && item.taskRun === taskRunName)) ?? {}; const failedSteps = Array.isArray(taskRun.failedSteps) ? taskRun.failedSteps.map(record) : []; const failedContainers = Array.isArray(pod.failedContainers) ? pod.failedContainers.map(record) : []; const failures = failedSteps.length > 0 ? failedSteps : failedContainers.map((container) => ({ name: typeof container.name === "string" && container.name.startsWith("step-") ? container.name.slice("step-".length) : container.name ?? null, container: container.name ?? null, state: container.state ?? null, exitCode: container.exitCode ?? null, reason: container.reason ?? null, message: container.message ?? null, })); const effectiveFailures = failures.length > 0 ? failures : [{ name: null, container: null, state: null, exitCode: null, reason: taskRun.reason ?? null, message: taskRun.message ?? null }]; for (const failure of effectiveFailures) { const stepName = stringOrNull(failure.name); const containerName = stringOrNull(failure.container) ?? (stepName === null ? null : `step-${stepName}`); summaries.push({ taskRun: taskRunName, pipelineTask: taskRun.pipelineTask ?? null, taskRef: taskRun.taskRef ?? null, taskRunStatus: taskRun.status ?? null, taskRunReason: taskRun.reason ?? null, taskRunMessage: diagnosticText(taskRun.message), pod: podName, podPhase: pod.phase ?? null, nodeName: pod.nodeName ?? null, step: stepName, container: containerName, containerState: failure.state ?? null, terminationReason: failure.reason ?? null, exitCode: typeof failure.exitCode === "number" ? failure.exitCode : null, message: diagnosticText(failure.message), logCommand: podName === null ? null : nodeRuntimePipelineLogsCommand(spec, podName, containerName), taskRunCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["get", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName, "-o", "yaml"]), taskRunDescribeCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName]), podDescribeCommand: podName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "pod", "-n", HWLAB_CI_NAMESPACE, podName]), }); } } return summaries.slice(0, 16); } export function nodeRuntimePipelineFailureSummary(value: unknown): Record | null { const recordValue = record(value); const direct = record(recordValue.failureSummary); if (Object.keys(direct).length > 0) return direct; const diagnostics = record(recordValue.diagnostics); const fromDiagnostics = record(diagnostics.failureSummary); return Object.keys(fromDiagnostics).length > 0 ? fromDiagnostics : null; } export function nodeRuntimePipelineLogsCommand(spec: HwlabRuntimeLaneSpec, podName: string, containerName: string | null): string { return nodeRuntimeK3sCommand(spec, [ "logs", "--namespace", HWLAB_CI_NAMESPACE, "--pod", podName, ...(containerName === null ? ["--all-containers"] : ["--container", containerName]), "--tail", "200", ]); } export function nodeRuntimeK3sCommand(spec: HwlabRuntimeLaneSpec, args: string[]): string { return ["trans", spec.nodeKubeRoute, ...args].map(shellQuote).join(" "); } export function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } export function diagnosticText(value: unknown): string | null { if (typeof value !== "string") return null; const trimmed = value.trim(); if (trimmed.length === 0) return null; return trimmed .replace(/postgres(?:ql)?:\/\/[^\s"'`]+/giu, "postgres://") .replace(/\b([A-Za-z0-9_.-]*(?:TOKEN|PASSWORD|SECRET|API_KEY|DATABASE_URL)[A-Za-z0-9_.-]*)=([^\s"'`]+)/giu, "$1=") .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gu, "Bearer ") .slice(0, 600); } export function nodeRuntimeRenderToken(): string { return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-"); } export function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds); } export function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] { const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds)); const retryCount = Math.max(0, Math.floor(retries)); const maxAttempts = retryCount + 1; const safeContext = context.replace(/[^A-Za-z0-9_.-]/gu, "-"); return [ `yaml_registry=${shellQuote(registry)}`, `yaml_fetch_timeout=${shellQuote(String(timeoutSeconds))}`, `yaml_fetch_retries=${shellQuote(String(retryCount))}`, `yaml_max_attempts=${shellQuote(String(maxAttempts))}`, `yaml_dependency_context=${shellQuote(safeContext)}`, "yaml_dependency_log() { echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"'\"$1\"'\",\"manager\":\"'\"${2:-}\"'\"}' >&2; }", "yaml_prepare_node_package_proxy() {", " yaml_http_proxy=\"${HTTP_PROXY:-${http_proxy:-${ALL_PROXY:-${all_proxy:-}}}}\"", " yaml_https_proxy=\"${HTTPS_PROXY:-${https_proxy:-${yaml_http_proxy}}}\"", " yaml_all_proxy=\"${ALL_PROXY:-${all_proxy:-${yaml_http_proxy}}}\"", " if [ -n \"$yaml_http_proxy\" ]; then export HTTP_PROXY=\"$yaml_http_proxy\" http_proxy=\"$yaml_http_proxy\" npm_config_proxy=\"$yaml_http_proxy\"; fi", " if [ -n \"$yaml_https_proxy\" ]; then export HTTPS_PROXY=\"$yaml_https_proxy\" https_proxy=\"$yaml_https_proxy\" npm_config_https_proxy=\"$yaml_https_proxy\"; fi", " if [ -n \"$yaml_all_proxy\" ]; then export ALL_PROXY=\"$yaml_all_proxy\" all_proxy=\"$yaml_all_proxy\"; fi", " export npm_config_registry=\"$yaml_registry\"", " export BUN_CONFIG_REGISTRY=\"$yaml_registry\"", " export npm_config_noproxy=\"${NO_PROXY:-${no_proxy:-}}\"", " export npm_config_fetch_retries=\"$yaml_fetch_retries\"", " export npm_config_fetch_retry_mintimeout=2000", " export npm_config_fetch_retry_maxtimeout=16000", " export npm_config_fetch_timeout=$((yaml_fetch_timeout * 1000))", "}", "yaml_run_dependency_manager() {", " yaml_manager=\"$1\"; shift", " yaml_attempt=1", " yaml_delay=2", " while [ \"$yaml_attempt\" -le \"$yaml_max_attempts\" ]; do", " echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"attempt\",\"manager\":\"'\"$yaml_manager\"'\",\"attempt\":\"'\"$yaml_attempt/$yaml_max_attempts\"'\",\"proxy\":\"'\"${yaml_https_proxy:-$yaml_http_proxy}\"'\"}' >&2", " if timeout \"$yaml_fetch_timeout\" \"$@\"; then return 0; fi", " if [ \"$yaml_attempt\" -ge \"$yaml_max_attempts\" ]; then return 1; fi", " echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"retrying\",\"manager\":\"'\"$yaml_manager\"'\",\"attempt\":\"'\"$yaml_attempt/$yaml_max_attempts\"'\",\"sleepSeconds\":'\"$yaml_delay\"'}' >&2", " sleep \"$yaml_delay\"", " yaml_delay=$((yaml_delay * 2))", " yaml_attempt=$((yaml_attempt + 1))", " done", " return 1", "}", "yaml_npm_debug_log_tail() {", " yaml_npm_log_dir=\"${HOME:-/tmp}/.npm/_logs\"", " if [ ! -d \"$yaml_npm_log_dir\" ]; then return 0; fi", " yaml_npm_log=\"$(find \"$yaml_npm_log_dir\" -type f -name '*debug*.log' | sort | tail -n 1 || true)\"", " if [ -n \"$yaml_npm_log\" ] && [ -f \"$yaml_npm_log\" ]; then", " echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"npm-debug-log\",\"path\":\"'\"$yaml_npm_log\"'\"}' >&2", " tail -n 80 \"$yaml_npm_log\" >&2 || true", " fi", "}", "if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then", " yaml_prepare_node_package_proxy", "fi", "if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 && command -v bun >/dev/null 2>&1; then", " rm -rf node_modules/yaml", " if yaml_run_dependency_manager bun bun add --no-save --ignore-scripts --registry \"$yaml_registry\" yaml@2.8.3; then", " yaml_dependency_log installed bun", " else", " yaml_dependency_log bun-failed bun", " fi", "fi", "if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then", " rm -rf node_modules/yaml", " command -v npm >/dev/null 2>&1 || { yaml_dependency_log failed missing-tool; exit 31; }", " if yaml_run_dependency_manager npm npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev --registry \"$yaml_registry\" yaml@2.8.3; then", " yaml_dependency_log installed npm", " else", " yaml_dependency_log npm-failed npm", " yaml_npm_debug_log_tail", " exit 34", " fi", "fi", "node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 || { yaml_dependency_log failed unresolved; exit 34; }", ]; } export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult { const token = nodeRuntimeRenderToken(); const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`; const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`; const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64"); const script = [ "set -eu", runtimeLaneCicdRepoEnsureScript(spec), `source_commit=${shellQuote(sourceCommit)}`, `render_dir=${shellQuote(renderDir)}`, `worktree_dir=${shellQuote(worktreeDir)}`, `overlay_b64=${shellQuote(overlay)}`, "cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }", "trap cleanup_render_worktree EXIT", `test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = "$source_commit"`, "rm -rf \"$render_dir\" \"$worktree_dir\"", "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", "git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"", "git -C \"$worktree_dir\" checkout --detach \"$source_commit\"", "cd \"$worktree_dir\"", ...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "control-plane-render"), "node - \"$overlay_b64\" <<'NODE'", "const fs = require('fs');", "const YAML = require('yaml');", "const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));", "const path = 'deploy/deploy.yaml';", "const doc = YAML.parse(fs.readFileSync(path, 'utf8'));", "doc.nodes = doc.nodes || {};", "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", "doc.lanes = doc.lanes || {};", "const lane = doc.lanes[overlay.lane] || {};", "const downloadStack = {", " ...(lane.envRecipe?.downloadStack || {}),", " httpProxy: overlay.dockerProxyHttp,", " httpsProxy: overlay.dockerProxyHttps,", " noProxy: overlay.dockerNoProxyList,", "};", "doc.lanes[overlay.lane] = {", " ...lane,", " node: overlay.nodeId,", " sourceBranch: overlay.sourceBranch,", " gitopsBranch: overlay.gitopsBranch,", " namespace: overlay.runtimeNamespace,", " endpoint: overlay.publicApiUrl,", " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", " artifactCatalog: overlay.catalogPath,", " runtimePath: overlay.runtimePath,", " imageTagMode: 'full',", " sourceRepo: overlay.gitUrl,", " externalPostgres: overlay.externalPostgres,", " observability: overlay.observability,", " envRecipe: { ...(lane.envRecipe || {}), downloadStack },", "};", "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;", "if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;", "if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;", "fs.writeFileSync(path, YAML.stringify(doc));", "NODE", "if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi", [ "node scripts/run-bun.mjs \"$render_script\"", `--lane ${shellQuote(spec.lane)}`, `--node ${shellQuote(spec.nodeId)}`, `--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`, `--catalog-path ${shellQuote(spec.catalogPath)}`, "--image-tag-mode full", `--source-revision ${shellQuote(sourceCommit)}`, `--source-repo ${shellQuote(spec.gitUrl)}`, `--source-branch ${shellQuote(spec.sourceBranch)}`, `--gitops-branch ${shellQuote(spec.gitopsBranch)}`, `--git-read-url ${shellQuote(spec.gitReadUrl)}`, `--git-write-url ${shellQuote(spec.gitWriteUrl)}`, `--registry-prefix ${shellQuote(spec.registryPrefix)}`, `--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`, `--web-endpoint ${shellQuote(spec.publicWebUrl)}`, `--out ${shellQuote(renderDir)}`, ].join(" "), ...nodeRuntimePipelinePostprocessScript(), ].join("\n"); return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" }; } export function nodeRuntimePipelinePostprocessScript(): string[] { return [ "node - \"$render_dir\" \"$overlay_b64\" <<'NODE'", "const fs = require('fs');", "const path = require('path');", "const vm = require('node:vm');", "const renderDir = process.argv[2];", "const overlay = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));", "const pipelinePath = path.join(renderDir, overlay.tektonDir, 'pipeline.yaml');", "let text = fs.readFileSync(pipelinePath, 'utf8');", "let YAML = null;", "try { YAML = require('yaml'); } catch {}", "const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');", "const shellSingle = (value) => `'${String(value).replaceAll(\"'\", `'\"'\"'`)}'`;", "const yamlString = (value) => JSON.stringify(String(value));", "const proxyEnv = {", " HTTP_PROXY: overlay.proxyHttp,", " HTTPS_PROXY: overlay.proxyHttps,", " ALL_PROXY: overlay.proxyAll,", " NO_PROXY: overlay.noProxy,", " http_proxy: overlay.proxyHttp,", " https_proxy: overlay.proxyHttps,", " all_proxy: overlay.proxyAll,", " no_proxy: overlay.noProxy,", "};", "const dockerProxyEnv = {", " HWLAB_NODE_PROXY_URL: overlay.dockerProxyHttp,", " HWLAB_NODE_ALL_PROXY_URL: overlay.dockerProxyAll,", " HWLAB_NODE_NO_PROXY: overlay.dockerNoProxy,", "};", "const stepEnv = { ...proxyEnv, ...dockerProxyEnv, ...(overlay.stepEnv || {}) };", "function prepareSourceDependencyScript() {", " return `prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"", "echo '{\"event\":\"prepare-source-dependencies\",\"status\":\"not-required\",\"dependency\":\"yaml\",\"manager\":\"inline-service-id-parser\"}' >&2", "ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"`;", "}", "function validatePrepareSourceDependencyScript(script) {", " const marker = 'NODE_UNIDESK_YAML_DEPENDENCY';", " let offset = 0;", " while (true) {", " const markerStart = script.indexOf(\"node <<'\" + marker + \"'\", offset);", " if (markerStart === -1) return;", " const bodyStart = script.indexOf('\\n', markerStart);", " if (bodyStart === -1) throw new Error('prepare-source dependency heredoc body missing newline');", " const bodyEnd = script.indexOf('\\n' + marker, bodyStart + 1);", " if (bodyEnd === -1) throw new Error('prepare-source dependency heredoc terminator missing');", " const body = script.slice(bodyStart + 1, bodyEnd);", " try { new vm.Script(body, { filename: 'NODE_UNIDESK_YAML_DEPENDENCY.js' }); } catch (error) { throw new Error(`generated prepare-source yaml dependency script is invalid: ${error.message}`); }", " offset = bodyEnd + marker.length + 1;", " }", "}", "function deployYamlOverlayScript() {", " const runtimeOverlay = JSON.stringify({", " nodeId: overlay.nodeId,", " lane: overlay.lane,", " sourceBranch: overlay.sourceBranch,", " gitopsBranch: overlay.gitopsBranch,", " gitopsRoot: overlay.gitopsRoot,", " runtimePath: overlay.runtimePath,", " runtimeRenderDir: overlay.runtimeRenderDir,", " runtimeNamespace: overlay.runtimeNamespace,", " catalogPath: overlay.catalogPath,", " gitUrl: overlay.gitUrl,", " publicWebUrl: overlay.publicWebUrl,", " publicApiUrl: overlay.publicApiUrl,", " externalPostgres: overlay.externalPostgres,", " runtimeStore: overlay.runtimeStore,", " codeAgentRuntime: overlay.codeAgentRuntime,", " observability: overlay.observability,", " deployYamlGitMirror: overlay.deployYamlGitMirror,", " runtimeImageRewrites: overlay.runtimeImageRewrites,", " dockerProxyHttp: overlay.dockerProxyHttp,", " dockerProxyHttps: overlay.dockerProxyHttps,", " dockerNoProxyList: overlay.dockerNoProxyList,", " npmRegistry: overlay.npmRegistry,", " npmFetchTimeoutMs: overlay.npmFetchTimeoutMs,", " npmRetries: overlay.npmRetries,", " });", " return `node - <<'NODE_UNIDESK_DEPLOY_YAML_OVERLAY'", "const fs = require('fs');", "const YAML = require('yaml');", "const overlay = ${runtimeOverlay};", "const file = 'deploy/deploy.yaml';", "if (!fs.existsSync(file)) {", " console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: false, reason: 'deploy-yaml-missing', file }));", " process.exit(45);", "}", "const doc = YAML.parse(fs.readFileSync(file, 'utf8'));", "doc.nodes = doc.nodes || {};", "doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };", "doc.lanes = doc.lanes || {};", "const lane = doc.lanes[overlay.lane] || {};", "const envRecipe = lane.envRecipe || {};", "const downloadStack = {", " ...(envRecipe.downloadStack || {}),", " httpProxy: overlay.dockerProxyHttp,", " httpsProxy: overlay.dockerProxyHttps,", " noProxy: overlay.dockerNoProxyList,", "};", "if (overlay.npmRegistry) downloadStack.npmRegistry = overlay.npmRegistry;", "if (overlay.npmFetchTimeoutMs) downloadStack.npmFetchTimeoutMs = overlay.npmFetchTimeoutMs;", "doc.lanes[overlay.lane] = {", " ...lane,", " node: overlay.nodeId,", " sourceBranch: overlay.sourceBranch,", " gitopsBranch: overlay.gitopsBranch,", " namespace: overlay.runtimeNamespace,", " endpoint: overlay.publicApiUrl,", " publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },", " artifactCatalog: overlay.catalogPath,", " runtimePath: overlay.runtimePath,", " imageTagMode: 'full',", " sourceRepo: overlay.gitUrl,", " externalPostgres: overlay.externalPostgres,", " observability: overlay.observability,", " envRecipe: { ...envRecipe, downloadStack },", "};", "if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;", "if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;", "if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;", "fs.writeFileSync(file, YAML.stringify(doc));", "console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));", "NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;", "}", "function runtimePathOverlayScript() {", " const sourcePath = `${String(overlay.gitopsRoot || '').replace(/\\/+$/u, '')}/${overlay.runtimeRenderDir}`;", " const targetPath = String(overlay.runtimePath || '');", " if (!targetPath || sourcePath === targetPath) return '';", " return [", " `if [ ! -d ${shellSingle(targetPath)} ]; then`,", " ` if [ ! -d ${shellSingle(sourcePath)} ]; then echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: false, reason: 'source-runtime-path-missing' }))} >&2; exit 46; fi`,", " ` mkdir -p \"$(dirname ${shellSingle(targetPath)})\"`,", " ` cp -a ${shellSingle(sourcePath)} ${shellSingle(targetPath)}` ,", " ` echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: true }))} >&2`,", " `fi`,", " ].join('\\n');", "}", "function stepEnvBootstrapScript() {", " const entries = Object.entries(overlay.stepEnv || {}).filter(([, value]) => value !== undefined && value !== null && String(value).length > 0);", " const lines = ['# unidesk-step-env-bootstrap'];", " for (const [name, value] of entries) lines.push(`export ${name}=${shellSingle(value)}`);", " if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'HOME')) lines.push('mkdir -p \"$HOME\"');", " if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'XDG_CONFIG_HOME')) lines.push('mkdir -p \"$XDG_CONFIG_HOME\"');", " lines.push('ci_node_deps=\"${HWLAB_CI_NODE_DEPS:-/opt/hwlab-ci-node-deps/node_modules}\"');", " lines.push('if [ -d \"$ci_node_deps\" ]; then');", " lines.push(' if [ -d /workspace/source/repo ]; then ci_node_deps_target=/workspace/source/repo/node_modules; else ci_node_deps_target=./node_modules; fi');", " lines.push(' mkdir -p \"$ci_node_deps_target\"');", " lines.push(' for ci_node_dep in yaml; do if [ -d \"$ci_node_deps/$ci_node_dep\" ] && [ ! -e \"$ci_node_deps_target/$ci_node_dep\" ]; then ln -s \"$ci_node_deps/$ci_node_dep\" \"$ci_node_deps_target/$ci_node_dep\"; fi; done');", " lines.push('fi');", " return lines.join('\\n');", "}", "function runtimeGitopsPostprocessScript() {", " const runtimeOverlay = JSON.stringify({", " gitopsRoot: overlay.gitopsRoot,", " runtimePath: overlay.runtimePath,", " runtimeRenderDir: overlay.runtimeRenderDir,", " runtimeNamespace: overlay.runtimeNamespace,", " externalPostgres: overlay.externalPostgres,", " codeAgentRuntime: overlay.codeAgentRuntime,", " publicExposure: overlay.publicExposure,", " observability: overlay.observability,", " runtimeImageRewrites: overlay.runtimeImageRewrites,", " gitReadUrl: overlay.gitReadUrl,", " publicWebUrl: overlay.publicWebUrl,", " publicApiUrl: overlay.publicApiUrl,", " });", " return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS'", "const fs = require('fs');", "const path = require('path');", "const crypto = require('crypto');", "const YAML = require('yaml');", "const overlay = ${runtimeOverlay};", "const runtimePath = String(overlay.runtimePath || '');", "const renderDir = String(overlay.runtimeRenderDir || '');", "const legacyRuntimePath = runtimePath ? path.posix.join(path.posix.dirname(path.posix.dirname(runtimePath)), path.posix.basename(runtimePath)) : '';", "const candidates = [...new Set([", " runtimePath,", " renderDir,", " overlay.gitopsRoot && renderDir ? path.posix.join(String(overlay.gitopsRoot), renderDir) : '',", " legacyRuntimePath,", "].filter(Boolean))];", "const sourcePath = candidates.find((candidate) => fs.existsSync(candidate)) || runtimePath;", "if (!runtimePath || !fs.existsSync(sourcePath)) {", " console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: false, reason: 'runtime-path-missing', runtimePath, sourcePath }));", " process.exit(47);", "}", "if (sourcePath !== runtimePath) {", " fs.rmSync(runtimePath, { recursive: true, force: true });", " fs.mkdirSync(path.dirname(runtimePath), { recursive: true });", " fs.cpSync(sourcePath, runtimePath, { recursive: true });", " fs.rmSync(sourcePath, { recursive: true, force: true });", "}", "for (const candidate of candidates) {", " if (candidate !== runtimePath && candidate !== sourcePath && candidate.endsWith('/' + path.posix.basename(runtimePath))) fs.rmSync(candidate, { recursive: true, force: true });", "}", "function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }", "function writeYaml(file, doc) { fs.writeFileSync(file, YAML.stringify(doc).trimEnd() + '\\\\n'); }", "function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }", "function normalizeList(items) { return { apiVersion: 'v1', kind: 'List', items }; }", "function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }", "function yamlFiles(dir) {", " const files = [];", " for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {", " const file = path.join(dir, entry.name);", " if (entry.isDirectory()) files.push(...yamlFiles(file));", " else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);", " }", " return files;", "}", "function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }", "function writeYamlDocuments(file, docs) { fs.writeFileSync(file, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\\\n---\\\\n') + '\\\\n'); }", "function podSpecFor(item) {", " if (!isObject(item) || !isObject(item.spec)) return null;", " if (item.kind === 'Pod') return item.spec;", " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", " if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;", " return null;", "}", "function templateMetadataFor(item) {", " if (!isObject(item) || !isObject(item.spec)) return null;", " if (item.kind === 'Pod') return item.metadata || null;", " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template ? item.spec.template.metadata : null;", " if (item.kind === 'Job') return item.spec.template ? item.spec.template.metadata : null;", " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.metadata : null;", " return null;", "}", "function stripMonitoringMetadata(metadata) {", " if (!isObject(metadata)) return false;", " let changed = false;", " if (isObject(metadata.labels) && metadata.labels['hwlab.pikastech.local/monitoring'] !== undefined && metadata.labels['hwlab.pikastech.local/monitoring'] !== 'disabled') {", " metadata.labels['hwlab.pikastech.local/monitoring'] = 'disabled';", " changed = true;", " }", " if (isObject(metadata.annotations) && metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'] !== undefined) {", " delete metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'];", " changed = true;", " }", " return changed;", "}", "function isPrometheusOperatorResource(item) { return item && item.apiVersion && String(item.apiVersion).startsWith('monitoring.coreos.com/') && ['ServiceMonitor', 'PrometheusRule', 'PodMonitor', 'Probe'].includes(item.kind); }", "function stripPrometheusOperatorResources(doc) {", " if (!(overlay.observability && overlay.observability.prometheusOperator === false)) return { docs: [doc], changed: false };", " if (doc && doc.kind === 'List' && Array.isArray(doc.items)) {", " const items = doc.items.filter((item) => !isPrometheusOperatorResource(item));", " return { docs: items.length > 0 ? [{ ...doc, items }] : [], changed: items.length !== doc.items.length };", " }", " return isPrometheusOperatorResource(doc) ? { docs: [], changed: true } : { docs: [doc], changed: false };", "}", "function containerHasVolumeMount(container, name) { return isObject(container) && Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === name); }", "function removeMetricsSidecar(podSpec) {", " if (!isObject(podSpec)) return false;", " let changed = false;", " if (Array.isArray(podSpec.containers)) {", " const next = podSpec.containers.filter((container) => !(isObject(container) && (container.name === 'hwlab-metrics' || (Array.isArray(container.command) && container.command.includes('/metrics/metrics-sidecar.mjs')))));", " if (next.length !== podSpec.containers.length) { podSpec.containers = next; changed = true; }", " }", " for (const group of ['containers', 'initContainers']) {", " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", " if (!isObject(container) || !Array.isArray(container.volumeMounts)) continue;", " const nextMounts = container.volumeMounts.filter((mount) => !(mount && mount.name === 'hwlab-metrics-sidecar'));", " if (nextMounts.length !== container.volumeMounts.length) { container.volumeMounts = nextMounts; changed = true; }", " }", " }", " if (Array.isArray(podSpec.volumes)) {", " const nextVolumes = podSpec.volumes.filter((volume) => !(volume && volume.name === 'hwlab-metrics-sidecar'));", " if (nextVolumes.length !== podSpec.volumes.length) { podSpec.volumes = nextVolumes; changed = true; }", " }", " return changed;", "}", "function envValue(container, name) {", " if (!isObject(container) || !Array.isArray(container.env)) return undefined;", " const item = container.env.find((env) => env && env.name === name);", " return item ? item.value : undefined;", "}", "function setEnvValue(container, name, value) {", " if (!isObject(container) || typeof value !== 'string') return false;", " container.env = Array.isArray(container.env) ? container.env : [];", " let item = container.env.find((env) => env && env.name === name);", " if (!item) { item = { name }; container.env.push(item); }", " const changed = item.value !== value || item.valueFrom !== undefined;", " item.value = value;", " delete item.valueFrom;", " return changed;", "}", "function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }", "function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }", "function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }", "function publicExposureCloudWebEnvEntries() {", " const exposure = overlay.publicExposure;", " if (!exposure || !Array.isArray(exposure.extraProxies)) return [];", " return exposure.extraProxies.filter((proxy) => proxy && proxy.cloudWebEnvName && proxy.publicBaseUrl).map((proxy) => ({ name: String(proxy.cloudWebEnvName), value: String(proxy.publicBaseUrl) }));", "}", "function startupProbeFrom(probe) {", " const next = JSON.parse(JSON.stringify(probe));", " next.periodSeconds = 10;", " next.timeoutSeconds = Math.max(Number(next.timeoutSeconds || 0), 2);", " next.failureThreshold = 30;", " next.successThreshold = 1;", " delete next.initialDelaySeconds;", " return next;", "}", "function addEnvReuseStartupProbe(podSpec) {", " if (!isObject(podSpec) || !Array.isArray(podSpec.containers)) return false;", " let changed = false;", " for (const container of podSpec.containers) {", " if (!isObject(container) || !isEnvReuseContainer(container) || container.startupProbe) continue;", " const sourceProbe = container.readinessProbe || container.livenessProbe;", " if (!sourceProbe) continue;", " container.startupProbe = startupProbeFrom(sourceProbe);", " changed = true;", " }", " return changed;", "}", "function rewriteRuntimeImage(image) {", " if (typeof image !== 'string') return image;", " const match = (overlay.runtimeImageRewrites || []).find((item) => item && item.source === image);", " return match ? match.target : image;", "}", "function patchRuntimeImages(podSpec) {", " if (!isObject(podSpec)) return false;", " let changed = false;", " for (const group of ['containers', 'initContainers']) {", " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", " if (!isObject(container) || typeof container.image !== 'string') continue;", " const nextImage = rewriteRuntimeImage(container.image);", " if (nextImage !== container.image) { container.image = nextImage; changed = true; }", " }", " }", " return changed;", "}", "function rewriteEnvValue(value) {", " if (typeof value !== 'string') return value;", " return value.replaceAll('http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git', overlay.gitReadUrl);", "}", "function patchGitReadUrlEnv(podSpec) {", " if (!isObject(podSpec)) return false;", " let changed = false;", " for (const group of ['containers', 'initContainers']) {", " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", " if (!isObject(container) || !Array.isArray(container.env)) continue;", " for (const env of container.env) {", " if (!isObject(env) || typeof env.value !== 'string') continue;", " const nextValue = rewriteEnvValue(env.value);", " if (nextValue !== env.value) { env.value = nextValue; changed = true; }", " }", " }", " }", " return changed;", "}", "function patchRuntimeEnv(item, podSpec) {", " if (!isObject(podSpec)) return { publicEndpointChanged: false, dbSslModeChanged: false, codeAgentRuntimeChanged: false, cloudWebRuntimeChanged: false };", " let publicEndpointChanged = false;", " let dbSslModeChanged = false;", " let codeAgentRuntimeChanged = false;", " let cloudWebRuntimeChanged = false;", " const pg = overlay.externalPostgres;", " const codeAgentRuntime = overlay.codeAgentRuntime;", " const cloudWebEnvEntries = publicExposureCloudWebEnvEntries();", " for (const group of ['containers', 'initContainers']) {", " for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {", " if (!isObject(container)) continue;", " if (envValue(container, 'HWLAB_PUBLIC_ENDPOINT') !== undefined) publicEndpointChanged = setEnvValue(container, 'HWLAB_PUBLIC_ENDPOINT', expectedPublicEndpoint(item)) || publicEndpointChanged;", " if (pg && pg.sslmode && envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE') !== undefined) dbSslModeChanged = setEnvValue(container, 'HWLAB_CLOUD_DB_SSL_MODE', pg.sslmode) || dbSslModeChanged;", " if (workloadName(item) === 'hwlab-cloud-web' && container.name === 'hwlab-cloud-web') {", " for (const env of cloudWebEnvEntries) cloudWebRuntimeChanged = setEnvValue(container, env.name, env.value) || cloudWebRuntimeChanged;", " }", " if (codeAgentRuntime && codeAgentRuntime.enabled && workloadName(item) === 'hwlab-cloud-api' && container.name === 'hwlab-cloud-api') {", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_ADAPTER', String(codeAgentRuntime.adapter)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'AGENTRUN_MGR_URL', String(codeAgentRuntime.managerUrl)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvFromSecret(container, 'AGENTRUN_API_KEY', String(codeAgentRuntime.apiKeySecretName), String(codeAgentRuntime.apiKeySecretKey)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE', String(codeAgentRuntime.runnerNamespace)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE', String(codeAgentRuntime.secretNamespace)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_REPO_URL', String(codeAgentRuntime.repoUrl)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID', String(codeAgentRuntime.providerId)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE', String(codeAgentRuntime.defaultProviderProfile)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR', String(codeAgentRuntime.codexStdioSupervisor)) || codeAgentRuntimeChanged;", " if (codeAgentRuntime.kafkaShadowProducer) {", " const kafka = codeAgentRuntime.kafkaShadowProducer;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED', String(kafka.enabled)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_SHADOW_CONSUME_ENABLED', String(kafka.consumeEnabled)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_BOOTSTRAP_SERVERS', String(kafka.bootstrapServers)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_COMMAND_TOPIC', String(kafka.commandTopic)) || codeAgentRuntimeChanged;", " codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_CLIENT_ID', String(kafka.clientId)) || codeAgentRuntimeChanged;", " }", " }", " }", " }", " return { publicEndpointChanged, dbSslModeChanged, codeAgentRuntimeChanged, cloudWebRuntimeChanged };", "}", "function patchRuntimeWorkloads() {", " let observabilityChanged = false;", " let startupProbeChanged = false;", " let imageRewriteChanged = false;", " let gitReadUrlChanged = false;", " let publicEndpointChanged = false;", " let dbSslModeChanged = false;", " let codeAgentRuntimeChanged = false;", " let cloudWebRuntimeChanged = false;", " for (const file of yamlFiles(runtimePath)) {", " if (path.basename(file) === 'kustomization.yaml') continue;", " const docs = readYamlDocuments(file);", " let changed = false;", " const nextDocs = [];", " for (const doc of docs) {", " const stripped = stripPrometheusOperatorResources(doc);", " changed = stripped.changed || changed;", " observabilityChanged = observabilityChanged || stripped.changed;", " for (const nextDoc of stripped.docs) {", " nextDocs.push(nextDoc);", " for (const item of listItems(nextDoc).filter(Boolean)) {", " if (!isObject(item)) continue;", " if (overlay.observability && overlay.observability.prometheusOperator === false) {", " const metadataChanged = stripMonitoringMetadata(item.metadata);", " const templateChanged = stripMonitoringMetadata(templateMetadataFor(item));", " const sidecarChanged = removeMetricsSidecar(podSpecFor(item));", " const monitoringChanged = metadataChanged || templateChanged || sidecarChanged;", " changed = monitoringChanged || changed;", " observabilityChanged = observabilityChanged || monitoringChanged;", " }", " const probeChanged = addEnvReuseStartupProbe(podSpecFor(item));", " changed = probeChanged || changed;", " startupProbeChanged = startupProbeChanged || probeChanged;", " const imageChanged = patchRuntimeImages(podSpecFor(item));", " changed = imageChanged || changed;", " imageRewriteChanged = imageRewriteChanged || imageChanged;", " const gitUrlChanged = patchGitReadUrlEnv(podSpecFor(item));", " changed = gitUrlChanged || changed;", " gitReadUrlChanged = gitReadUrlChanged || gitUrlChanged;", " const envChanged = patchRuntimeEnv(item, podSpecFor(item));", " changed = envChanged.publicEndpointChanged || envChanged.dbSslModeChanged || envChanged.codeAgentRuntimeChanged || envChanged.cloudWebRuntimeChanged || changed;", " publicEndpointChanged = publicEndpointChanged || envChanged.publicEndpointChanged;", " dbSslModeChanged = dbSslModeChanged || envChanged.dbSslModeChanged;", " codeAgentRuntimeChanged = codeAgentRuntimeChanged || envChanged.codeAgentRuntimeChanged;", " cloudWebRuntimeChanged = cloudWebRuntimeChanged || envChanged.cloudWebRuntimeChanged;", " }", " }", " }", " if (changed) {", " if (nextDocs.length === 0) fs.rmSync(file, { force: true });", " else writeYamlDocuments(file, nextDocs);", " }", " }", " return { observabilityChanged, startupProbeChanged, imageRewriteChanged, gitReadUrlChanged, publicEndpointChanged, dbSslModeChanged, codeAgentRuntimeChanged, cloudWebRuntimeChanged };", "}", "function patchKustomization() {", " const file = path.join(runtimePath, 'kustomization.yaml');", " if (!fs.existsSync(file)) return false;", " const doc = readYaml(file) || {};", " const resources = Array.isArray(doc.resources) ? doc.resources : [];", " const next = resources.filter((item) => {", " if (!(overlay.observability && overlay.observability.prometheusOperator === false)) return true;", " const resource = String(item);", " if (resource === 'observability.yaml') return false;", " return !(/\\.ya?ml$/u.test(resource) && !fs.existsSync(path.join(runtimePath, resource)));", " });", " let changed = false;", " if (next.length !== resources.length) { doc.resources = next; writeYaml(file, doc); changed = true; }", " const observabilityFile = path.join(runtimePath, 'observability.yaml');", " if (overlay.observability && overlay.observability.prometheusOperator === false && fs.existsSync(observabilityFile)) { fs.rmSync(observabilityFile, { force: true }); changed = true; }", " return changed;", "}", "function patchExternalPostgres() {", " const pg = overlay.externalPostgres;", " if (!pg || !pg.serviceName) return false;", " const access = pg.runtimeAccess || { endpointAddress: pg.endpointAddress, port: pg.port };", " const file = path.join(runtimePath, 'external-postgres.yaml');", " if (!fs.existsSync(file)) return false;", " const doc = readYaml(file);", " const items = listItems(doc).filter(Boolean);", " let changed = false;", " const endpointIsIpv4 = /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/.test(String(access.endpointAddress));", " const endpointSliceName = String(pg.serviceName) + '-host';", " for (const item of items) {", " if (!item || typeof item !== 'object') continue;", " item.metadata = item.metadata || {};", " item.metadata.namespace = overlay.runtimeNamespace;", " item.metadata.labels = item.metadata.labels || {};", " item.metadata.labels['app.kubernetes.io/name'] = pg.serviceName;", " if (item.kind === 'Service') {", " item.metadata.name = pg.serviceName;", " item.spec = item.spec || {};", " if (endpointIsIpv4) {", " item.spec.type = 'ClusterIP';", " delete item.spec.externalName;", " } else {", " item.spec.type = 'ExternalName';", " item.spec.externalName = String(access.endpointAddress);", " delete item.spec.clusterIP;", " delete item.spec.clusterIPs;", " delete item.spec.ipFamilies;", " delete item.spec.ipFamilyPolicy;", " delete item.spec.internalTrafficPolicy;", " }", " item.spec.ports = [{ name: 'postgres', port: access.port, targetPort: access.port, protocol: 'TCP' }];", " delete item.spec.selector;", " changed = true;", " }", " if (item.kind === 'EndpointSlice') {", " if (!endpointIsIpv4) { item.__delete = true; changed = true; continue; }", " item.metadata.name = endpointSliceName;", " item.metadata.labels['kubernetes.io/service-name'] = pg.serviceName;", " item.addressType = 'IPv4';", " item.ports = [{ name: 'postgres', port: access.port, protocol: 'TCP' }];", " item.endpoints = [{ addresses: [access.endpointAddress], conditions: { ready: true } }];", " changed = true;", " }", " }", " if (changed) writeYaml(file, normalizeList(items.filter((item) => !item.__delete)));", " return changed;", "}", "function patchHealthContract() {", " const file = path.join(runtimePath, 'health-contract.yaml');", " if (!fs.existsSync(file)) return false;", " const doc = readYaml(file);", " const items = listItems(doc).filter(Boolean);", " let changed = false;", " const pg = overlay.externalPostgres;", " for (const item of items) {", " if (!item || item.kind !== 'ConfigMap') continue;", " item.data = item.data || {};", " if (item.data.endpoint !== overlay.publicWebUrl) { item.data.endpoint = overlay.publicWebUrl; changed = true; }", " const cloudApi = 'GET /health/live through ' + overlay.publicApiUrl;", " if (item.data['cloud-api'] !== cloudApi) { item.data['cloud-api'] = cloudApi; changed = true; }", " const cloudWeb = 'GET /health/live on ' + overlay.publicWebUrl + '; consumes cloud-api only';", " if (item.data['cloud-web'] !== cloudWeb) { item.data['cloud-web'] = cloudWeb; changed = true; }", " if (pg && pg.sslmode && typeof item.data['cloud-api-db'] === 'string') {", " const next = item.data['cloud-api-db'].replace(/HWLAB_CLOUD_DB_SSL_MODE=[A-Za-z0-9_-]+/g, 'HWLAB_CLOUD_DB_SSL_MODE=' + pg.sslmode);", " if (next !== item.data['cloud-api-db']) { item.data['cloud-api-db'] = next; changed = true; }", " }", " }", " if (changed) writeYaml(file, normalizeList(items));", " return changed;", "}", "function publicExposureFrpcProxies(exposure) { return [exposure.webProxy, exposure.apiProxy, ...(Array.isArray(exposure.extraProxies) ? exposure.extraProxies : [])].filter(Boolean); }", "function renderPublicExposureFrpcProxyToml(proxy) {", " return [", " '[[proxies]]',", " 'name = ' + JSON.stringify(String(proxy.name)),", " 'type = \"tcp\"',", " 'localIP = ' + JSON.stringify(String(proxy.localIP)),", " 'localPort = ' + Number(proxy.localPort),", " 'remotePort = ' + Number(proxy.remotePort),", " '',", " ];", "}", "function renderPublicExposureFrpcToml(exposure) {", " return [", " 'serverAddr = ' + JSON.stringify(String(exposure.serverAddr)),", " 'serverPort = ' + Number(exposure.serverPort),", " 'loginFailExit = true',", " 'auth.token = \"{{ .Envs.HWLAB_FRP_TOKEN }}\"',", " '',", " ...publicExposureFrpcProxies(exposure).flatMap(renderPublicExposureFrpcProxyToml),", " ].join('\\\\n');", "}", "function setEnvFromSecret(container, name, secretName, secretKey) {", " if (!isObject(container)) return false;", " container.env = Array.isArray(container.env) ? container.env : [];", " let item = container.env.find((env) => env && env.name === name);", " if (!item) { item = { name }; container.env.push(item); }", " const nextValueFrom = { secretKeyRef: { name: secretName, key: secretKey } };", " const changed = item.value !== undefined || JSON.stringify(item.valueFrom || {}) !== JSON.stringify(nextValueFrom);", " delete item.value;", " item.valueFrom = nextValueFrom;", " return changed;", "}", "function patchPublicExposure() {", " const exposure = overlay.publicExposure;", " if (!exposure || !exposure.enabled) return { configured: false, changed: false };", " const file = path.join(runtimePath, 'node-frpc.yaml');", " if (!fs.existsSync(file)) {", " console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'node-frpc-yaml-missing', filePath: file, hostname: exposure.hostname }));", " process.exit(49);", " }", " const docs = readYamlDocuments(file);", " const configName = String(overlay.runtimeNamespace) + '-frpc-config';", " const deploymentName = String(overlay.runtimeNamespace) + '-frpc';", " const configKey = String(exposure.secretKey || 'frpc.toml');", " const tokenKey = String(exposure.tokenKey || 'token');", " const toml = renderPublicExposureFrpcToml(exposure);", " const tomlSha256 = crypto.createHash('sha256').update(toml).digest('hex');", " let changed = false;", " let foundConfigMap = false;", " let foundDeployment = false;", " for (const doc of docs) {", " for (const item of listItems(doc).filter(Boolean)) {", " if (!isObject(item)) continue;", " item.metadata = item.metadata || {};", " if (item.kind === 'ConfigMap' && item.metadata.name === configName) {", " foundConfigMap = true;", " item.data = item.data || {};", " if (item.data[configKey] !== toml) { item.data[configKey] = toml; changed = true; }", " }", " if (item.kind === 'Deployment' && item.metadata.name === deploymentName) {", " foundDeployment = true;", " item.spec = item.spec || {};", " const nextStrategy = { type: 'Recreate' };", " if (JSON.stringify(item.spec.strategy || {}) !== JSON.stringify(nextStrategy)) { item.spec.strategy = nextStrategy; changed = true; }", " const templateMetadata = templateMetadataFor(item);", " if (templateMetadata) {", " templateMetadata.annotations = isObject(templateMetadata.annotations) ? templateMetadata.annotations : {};", " if (templateMetadata.annotations['unidesk.ai/frpc-config-sha256'] !== tomlSha256) { templateMetadata.annotations['unidesk.ai/frpc-config-sha256'] = tomlSha256; changed = true; }", " }", " const podSpec = podSpecFor(item);", " for (const container of Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : []) {", " if (!isObject(container)) continue;", " if (container.name === 'frpc' || String(container.image || '').includes('frpc')) changed = setEnvFromSecret(container, 'HWLAB_FRP_TOKEN', exposure.secretName, tokenKey) || changed;", " }", " }", " }", " }", " if (!foundConfigMap || !foundDeployment) {", " console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'frpc-resource-missing', filePath: file, configName, deploymentName, foundConfigMap, foundDeployment }));", " process.exit(50);", " }", " if (changed) writeYamlDocuments(file, docs);", " console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: true, applied: true, changed, filePath: file, hostname: exposure.hostname, serverAddr: exposure.serverAddr, serverPort: exposure.serverPort, webProxy: exposure.webProxy.name, apiProxy: exposure.apiProxy.name, extraProxyCount: Array.isArray(exposure.extraProxies) ? exposure.extraProxies.length : 0, configSha256: tomlSha256 }));", " return { configured: true, changed, foundConfigMap, foundDeployment };", "}", "const runtimeWorkloadsChanged = patchRuntimeWorkloads();", "const kustomizationChanged = patchKustomization();", "const externalPostgresChanged = patchExternalPostgres();", "const healthContractChanged = patchHealthContract();", "const publicExposureChanged = patchPublicExposure();", "console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: true, runtimePath, sourcePath, pathRelocated: sourcePath !== runtimePath, observabilityPrometheusOperator: overlay.observability ? overlay.observability.prometheusOperator : null, runtimeImageRewriteCount: (overlay.runtimeImageRewrites || []).length, kustomizationChanged, observabilityWorkloadsChanged: runtimeWorkloadsChanged.observabilityChanged, startupProbeChanged: runtimeWorkloadsChanged.startupProbeChanged, runtimeImageRewriteChanged: runtimeWorkloadsChanged.imageRewriteChanged, gitReadUrlChanged: runtimeWorkloadsChanged.gitReadUrlChanged, publicEndpointChanged: runtimeWorkloadsChanged.publicEndpointChanged, dbSslModeChanged: runtimeWorkloadsChanged.dbSslModeChanged, codeAgentRuntimeChanged: runtimeWorkloadsChanged.codeAgentRuntimeChanged, cloudWebRuntimeChanged: runtimeWorkloadsChanged.cloudWebRuntimeChanged, externalPostgresChanged, healthContractChanged, publicExposureChanged }));", "NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS`;", "}", "function runtimeGitopsVerifyScript() {", " const runtimeOverlay = JSON.stringify({", " runtimePath: overlay.runtimePath,", " runtimeNamespace: overlay.runtimeNamespace,", " externalPostgres: overlay.externalPostgres,", " codeAgentRuntime: overlay.codeAgentRuntime,", " publicExposure: overlay.publicExposure,", " observability: overlay.observability,", " runtimeImageRewrites: overlay.runtimeImageRewrites,", " gitReadUrl: overlay.gitReadUrl,", " publicWebUrl: overlay.publicWebUrl,", " publicApiUrl: overlay.publicApiUrl,", " });", " return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_VERIFY'", "const fs = require('fs');", "const path = require('path');", "const crypto = require('crypto');", "const YAML = require('yaml');", "const overlay = ${runtimeOverlay};", "const runtimePath = String(overlay.runtimePath || '');", "function fail(reason, extra = {}) {", " console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: false, reason, runtimePath, ...extra }));", " process.exit(48);", "}", "if (!runtimePath || !fs.existsSync(runtimePath)) fail('runtime-path-missing');", "function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }", "function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }", "function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }", "function yamlFiles(dir) {", " const files = [];", " for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {", " const file = path.join(dir, entry.name);", " if (entry.isDirectory()) files.push(...yamlFiles(file));", " else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);", " }", " return files;", "}", "function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }", "function allItemsFromFile(file) { return readYamlDocuments(file).flatMap((doc) => listItems(doc).filter(Boolean)); }", "function podSpecFor(item) {", " if (!isObject(item) || !isObject(item.spec)) return null;", " if (item.kind === 'Pod') return item.spec;", " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", " if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;", " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;", " return null;", "}", "function templateMetadataFor(item) {", " if (!isObject(item) || !isObject(item.spec)) return null;", " if (item.kind === 'Pod') return item.metadata || null;", " if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template ? item.spec.template.metadata : null;", " if (item.kind === 'Job') return item.spec.template ? item.spec.template.metadata : null;", " if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.metadata : null;", " return null;", "}", "function envValue(container, name) {", " if (!isObject(container) || !Array.isArray(container.env)) return undefined;", " const item = container.env.find((env) => env && env.name === name);", " return item ? item.value : undefined;", "}", "function envSecretRef(container, name) {", " if (!isObject(container) || !Array.isArray(container.env)) return {};", " const item = container.env.find((env) => env && env.name === name);", " return item && item.valueFrom && item.valueFrom.secretKeyRef ? item.valueFrom.secretKeyRef : {};", "}", "function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }", "function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }", "function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }", "function publicExposureCloudWebEnvEntries() {", " const exposure = overlay.publicExposure;", " if (!exposure || !Array.isArray(exposure.extraProxies)) return [];", " return exposure.extraProxies.filter((proxy) => proxy && proxy.cloudWebEnvName && proxy.publicBaseUrl).map((proxy) => ({ name: String(proxy.cloudWebEnvName), value: String(proxy.publicBaseUrl) }));", "}", "function isPrometheusOperatorResource(item) { return item && item.apiVersion && String(item.apiVersion).startsWith('monitoring.coreos.com/') && ['ServiceMonitor', 'PrometheusRule', 'PodMonitor', 'Probe'].includes(item.kind); }", "function workloadRef(item, file, container) { return { file, kind: item && item.kind, name: item && item.metadata && item.metadata.name, container: container && container.name }; }", "function workloadChecks() {", " const monitoringResources = [];", " const metricsRefs = [];", " const missingStartupProbes = [];", " const publicRuntimeImages = [];", " const staleGitReadUrls = [];", " const wrongPublicEndpoints = [];", " const wrongDbSslModes = [];", " const wrongCodeAgentRuntimeEnvs = [];", " const wrongCloudWebRuntimeEnvs = [];", " const rewriteSources = new Set((overlay.runtimeImageRewrites || []).map((item) => item && item.source).filter(Boolean));", " const codeAgentRuntime = overlay.codeAgentRuntime;", " const cloudWebEnvEntries = publicExposureCloudWebEnvEntries();", " function checkCloudWebRuntimeValue(item, file, container, envName, expected) {", " const value = envValue(container, envName);", " if (value !== expected) wrongCloudWebRuntimeEnvs.push({ ...workloadRef(item, file, container), envName, expected, value: value ?? null });", " }", " function checkCodeAgentRuntimeValue(item, file, container, envName, expected) {", " const value = envValue(container, envName);", " if (value !== expected) wrongCodeAgentRuntimeEnvs.push({ ...workloadRef(item, file, container), envName, kind: 'value', expected, value: value ?? null });", " }", " function checkCodeAgentRuntimeSecret(item, file, container, envName, expectedSecret, expectedKey) {", " const secretRef = envSecretRef(container, envName);", " if (secretRef.name !== expectedSecret || secretRef.key !== expectedKey) wrongCodeAgentRuntimeEnvs.push({ ...workloadRef(item, file, container), envName, kind: 'secretKeyRef', expectedSecret, expectedKey, secret: secretRef.name ?? null, key: secretRef.key ?? null });", " }", " for (const file of yamlFiles(runtimePath)) {", " if (path.basename(file) === 'kustomization.yaml') continue;", " for (const doc of readYamlDocuments(file)) {", " for (const item of listItems(doc).filter(Boolean)) {", " if (isPrometheusOperatorResource(item)) monitoringResources.push(workloadRef(item, file, null));", " const podSpec = podSpecFor(item);", " if (!isObject(podSpec)) continue;", " for (const container of Array.isArray(podSpec.containers) ? podSpec.containers : []) {", " if (!isObject(container)) continue;", " if (container.name === 'hwlab-metrics' || (Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === 'hwlab-metrics-sidecar'))) metricsRefs.push(workloadRef(item, file, container));", " if (isEnvReuseContainer(container) && (container.readinessProbe || container.livenessProbe) && !container.startupProbe) missingStartupProbes.push(workloadRef(item, file, container));", " if (typeof container.image === 'string' && rewriteSources.has(container.image)) publicRuntimeImages.push({ ...workloadRef(item, file, container), image: container.image });", " if (Array.isArray(container.env) && container.env.some((env) => env && typeof env.value === 'string' && env.value.includes('git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git') && env.value !== overlay.gitReadUrl)) staleGitReadUrls.push(workloadRef(item, file, container));", " const publicEndpoint = envValue(container, 'HWLAB_PUBLIC_ENDPOINT');", " if (publicEndpoint !== undefined && publicEndpoint !== expectedPublicEndpoint(item)) wrongPublicEndpoints.push({ ...workloadRef(item, file, container), value: publicEndpoint, expected: expectedPublicEndpoint(item) });", " const dbSslMode = envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE');", " if (overlay.externalPostgres && overlay.externalPostgres.sslmode && dbSslMode !== undefined && dbSslMode !== overlay.externalPostgres.sslmode) wrongDbSslModes.push({ ...workloadRef(item, file, container), value: dbSslMode, expected: overlay.externalPostgres.sslmode });", " if (workloadName(item) === 'hwlab-cloud-web' && container.name === 'hwlab-cloud-web') {", " for (const env of cloudWebEnvEntries) checkCloudWebRuntimeValue(item, file, container, env.name, env.value);", " }", " if (codeAgentRuntime && codeAgentRuntime.enabled && workloadName(item) === 'hwlab-cloud-api' && container.name === 'hwlab-cloud-api') {", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_ADAPTER', String(codeAgentRuntime.adapter));", " checkCodeAgentRuntimeValue(item, file, container, 'AGENTRUN_MGR_URL', String(codeAgentRuntime.managerUrl));", " checkCodeAgentRuntimeSecret(item, file, container, 'AGENTRUN_API_KEY', String(codeAgentRuntime.apiKeySecretName), String(codeAgentRuntime.apiKeySecretKey));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE', String(codeAgentRuntime.runnerNamespace));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE', String(codeAgentRuntime.secretNamespace));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_REPO_URL', String(codeAgentRuntime.repoUrl));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID', String(codeAgentRuntime.providerId));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE', String(codeAgentRuntime.defaultProviderProfile));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR', String(codeAgentRuntime.codexStdioSupervisor));", " if (codeAgentRuntime.kafkaShadowProducer) {", " const kafka = codeAgentRuntime.kafkaShadowProducer;", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED', String(kafka.enabled));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_SHADOW_CONSUME_ENABLED', String(kafka.consumeEnabled));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_BOOTSTRAP_SERVERS', String(kafka.bootstrapServers));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_COMMAND_TOPIC', String(kafka.commandTopic));", " checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_CLIENT_ID', String(kafka.clientId));", " }", " }", " }", " if (Array.isArray(podSpec.volumes) && podSpec.volumes.some((volume) => volume && volume.name === 'hwlab-metrics-sidecar')) metricsRefs.push(workloadRef(item, file, { name: 'volume/hwlab-metrics-sidecar' }));", " }", " }", " }", " return { monitoringResources, metricsRefs, missingStartupProbes, publicRuntimeImages, staleGitReadUrls, wrongPublicEndpoints, wrongDbSslModes, wrongCodeAgentRuntimeEnvs, wrongCloudWebRuntimeEnvs };", "}", "const checks = [];", "const workloadCheck = workloadChecks();", "const kustomizationPath = path.join(runtimePath, 'kustomization.yaml');", "if (overlay.observability && overlay.observability.prometheusOperator === false) {", " if (!fs.existsSync(kustomizationPath)) fail('kustomization-missing');", " const resources = readYaml(kustomizationPath).resources || [];", " if (resources.includes('observability.yaml')) fail('observability-resource-still-rendered', { file: kustomizationPath });", " if (workloadCheck.monitoringResources.length > 0) fail('prometheus-operator-resource-still-rendered', { refs: workloadCheck.monitoringResources.slice(0, 12), count: workloadCheck.monitoringResources.length });", " if (workloadCheck.metricsRefs.length > 0) fail('observability-sidecar-still-rendered', { refs: workloadCheck.metricsRefs.slice(0, 12), count: workloadCheck.metricsRefs.length });", " checks.push('observability-disabled');", "}", "if (workloadCheck.missingStartupProbes.length > 0) fail('env-reuse-startup-probe-missing', { refs: workloadCheck.missingStartupProbes.slice(0, 12), count: workloadCheck.missingStartupProbes.length });", "checks.push('env-reuse-startup-probes');", "if (workloadCheck.publicRuntimeImages.length > 0) fail('runtime-image-rewrite-missing', { refs: workloadCheck.publicRuntimeImages.slice(0, 12), count: workloadCheck.publicRuntimeImages.length });", "if ((overlay.runtimeImageRewrites || []).length > 0) checks.push('runtime-image-rewrites');", "if (workloadCheck.staleGitReadUrls.length > 0) fail('runtime-git-read-url-stale', { refs: workloadCheck.staleGitReadUrls.slice(0, 12), count: workloadCheck.staleGitReadUrls.length, expected: overlay.gitReadUrl });", "checks.push('runtime-git-read-url');", "if (workloadCheck.wrongPublicEndpoints.length > 0) fail('runtime-public-endpoint-mismatch', { refs: workloadCheck.wrongPublicEndpoints.slice(0, 12), count: workloadCheck.wrongPublicEndpoints.length });", "checks.push('runtime-public-endpoint');", "if (workloadCheck.wrongDbSslModes.length > 0) fail('runtime-db-ssl-mode-mismatch', { refs: workloadCheck.wrongDbSslModes.slice(0, 12), count: workloadCheck.wrongDbSslModes.length });", "if (overlay.externalPostgres && overlay.externalPostgres.sslmode) checks.push('runtime-db-ssl-mode');", "if (workloadCheck.wrongCodeAgentRuntimeEnvs.length > 0) fail('code-agent-runtime-env-mismatch', { refs: workloadCheck.wrongCodeAgentRuntimeEnvs.slice(0, 12), count: workloadCheck.wrongCodeAgentRuntimeEnvs.length });", "if (overlay.codeAgentRuntime && overlay.codeAgentRuntime.enabled) checks.push('code-agent-runtime-env');", "if (workloadCheck.wrongCloudWebRuntimeEnvs.length > 0) fail('cloud-web-runtime-env-mismatch', { refs: workloadCheck.wrongCloudWebRuntimeEnvs.slice(0, 12), count: workloadCheck.wrongCloudWebRuntimeEnvs.length });", "if (publicExposureCloudWebEnvEntries().length > 0) checks.push('cloud-web-runtime-env');", "const pg = overlay.externalPostgres;", "if (pg && pg.serviceName) {", " const access = pg.runtimeAccess || { endpointAddress: pg.endpointAddress, port: pg.port };", " const file = path.join(runtimePath, 'external-postgres.yaml');", " if (!fs.existsSync(file)) fail('external-postgres-missing');", " const items = listItems(readYaml(file)).filter(Boolean);", " const service = items.find((item) => item && item.kind === 'Service' && item.metadata && item.metadata.name === pg.serviceName);", " const endpointSlice = items.find((item) => item && item.kind === 'EndpointSlice' && item.metadata && item.metadata.name === String(pg.serviceName) + '-host');", " if (!service) fail('external-postgres-service-missing', { expected: pg.serviceName });", " const endpointIsIpv4 = /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/.test(String(access.endpointAddress));", " const servicePort = service.spec && Array.isArray(service.spec.ports) && service.spec.ports[0] ? service.spec.ports[0].port : null;", " if (!endpointIsIpv4) {", " if (service.spec && service.spec.type !== 'ExternalName') fail('external-postgres-service-type-mismatch', { value: service.spec.type, expected: 'ExternalName' });", " if (!service.spec || String(service.spec.externalName) !== String(access.endpointAddress)) fail('external-postgres-external-name-mismatch', { value: service.spec && service.spec.externalName, expected: access.endpointAddress });", " if (Number(servicePort) !== Number(access.port)) fail('external-postgres-port-mismatch', { servicePort, expectedPort: access.port });", " if (endpointSlice) fail('external-postgres-endpointslice-unexpected', { name: String(pg.serviceName) + '-host' });", " checks.push('external-postgres-bridge');", " } else {", " if (!endpointSlice) fail('external-postgres-endpointslice-missing', { expected: String(pg.serviceName) + '-host' });", " const endpointPort = Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;", " const endpointAddress = Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;", " if (Number(servicePort) !== Number(access.port) || Number(endpointPort) !== Number(access.port)) fail('external-postgres-port-mismatch', { servicePort, endpointPort, expectedPort: access.port });", " if (String(endpointAddress) !== String(access.endpointAddress)) fail('external-postgres-address-mismatch', { endpointAddress, expectedEndpointAddress: access.endpointAddress });", " checks.push('external-postgres-bridge');", " }", "}", "const exposure = overlay.publicExposure;", "if (exposure && exposure.enabled) {", " const file = path.join(runtimePath, 'node-frpc.yaml');", " if (!fs.existsSync(file)) fail('public-exposure-frpc-missing');", " const items = allItemsFromFile(file);", " const configName = String(overlay.runtimeNamespace) + '-frpc-config';", " const deploymentName = String(overlay.runtimeNamespace) + '-frpc';", " const configKey = String(exposure.secretKey || 'frpc.toml');", " const tokenKey = String(exposure.tokenKey || 'token');", " const configMap = items.find((item) => item && item.kind === 'ConfigMap' && item.metadata && item.metadata.name === configName);", " const deployment = items.find((item) => item && item.kind === 'Deployment' && item.metadata && item.metadata.name === deploymentName);", " if (!configMap) fail('public-exposure-frpc-configmap-missing', { expected: configName });", " if (!deployment) fail('public-exposure-frpc-deployment-missing', { expected: deploymentName });", " const toml = configMap.data && configMap.data[configKey];", " if (typeof toml !== 'string') fail('public-exposure-frpc-config-missing', { expectedKey: configKey });", " const expectedConfigSha256 = crypto.createHash('sha256').update(toml).digest('hex');", " const expectedProxyTokens = [exposure.webProxy, exposure.apiProxy, ...(Array.isArray(exposure.extraProxies) ? exposure.extraProxies : [])].flatMap((proxy) => [String(proxy.name), String(proxy.remotePort)]);", " for (const expected of [String(exposure.serverAddr), String(exposure.serverPort), ...expectedProxyTokens, 'HWLAB_FRP_TOKEN']) {", " if (!toml.includes(expected)) fail('public-exposure-frpc-config-mismatch', { expected });", " }", " const podSpec = podSpecFor(deployment);", " const templateMetadata = templateMetadataFor(deployment);", " const actualConfigSha256 = templateMetadata && templateMetadata.annotations ? templateMetadata.annotations['unidesk.ai/frpc-config-sha256'] : null;", " if (actualConfigSha256 !== expectedConfigSha256) fail('public-exposure-frpc-config-rollout-hash-mismatch', { expected: expectedConfigSha256, actual: actualConfigSha256 });", " const containers = Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : [];", " const strategyType = deployment.spec && deployment.spec.strategy && deployment.spec.strategy.type;", " if (strategyType !== 'Recreate') fail('public-exposure-frpc-strategy-mismatch', { expected: 'Recreate', actual: strategyType || null });", " const frpc = containers.find((container) => container && (container.name === 'frpc' || String(container.image || '').includes('frpc')));", " const env = frpc && Array.isArray(frpc.env) ? frpc.env.find((item) => item && item.name === 'HWLAB_FRP_TOKEN') : null;", " const secretRef = env && env.valueFrom && env.valueFrom.secretKeyRef;", " if (!secretRef || secretRef.name !== exposure.secretName || secretRef.key !== tokenKey) fail('public-exposure-frpc-token-env-mismatch', { expectedSecret: exposure.secretName, expectedKey: tokenKey });", " checks.push('public-exposure-frpc');", "}", "console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: true, runtimePath, checks }));", "NODE_UNIDESK_RUNTIME_GITOPS_VERIFY`;", "}", "function patchScript(script) {", " let result = String(script || '');", " const bootstrap = stepEnvBootstrapScript();", " if (bootstrap && !result.includes('unidesk-step-env-bootstrap')) result = `${bootstrap}\\n${result}`;", " const inlineDeployServiceIdParser = `const deployText = fs.readFileSync(deployPath, 'utf8');\\nconst deploy = { services: [...deployText.matchAll(/(?:^|\\\\n)\\\\s*serviceId:\\\\s*[\\\"']?([^\\\"'\\\\n#]+)[\\\"']?/g)].map((match) => ({ serviceId: match[1].trim() })) };`;", " result = result.split('import { readStructuredFile } from \"./scripts/src/structured-config.mjs\";\\n').join('');", " result = result.split('const deploy = await readStructuredFile(process.cwd(), deployPath);').join(inlineDeployServiceIdParser);", " if (result.includes('npm run gitops:ts:check')) {", " result = result.replace(/\\n[ \\t]*npm run gitops:ts:check\\n/g, '\\n echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-replaces-tsc-gate\"}\\' >&2\\n');", " }", " const isNodeContractCheck = result.includes('\\\"event\\\":\\\"unidesk-node-contract-check\\\"');", " if (isNodeContractCheck) {", " result = result.replace(/(^|\\n)([ \\t]*)node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*(?=\\n|$)/g, '$1$2echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-disabled-gitops-render\"}\\' >&2');", " }", " const prepareSourceDependencyPattern = new RegExp(String.raw`prepare_source_dependencies_started_ms=\"\\$\\(ci_now_ms\\)\"\\nif node -e 'require\\.resolve\\(\"yaml\"\\)'[\\s\\S]*?\\nci_timing_emit prepare-source-dependencies succeeded \"\\$prepare_source_dependencies_started_ms\"`, 'g');", " if (result.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"')) {", " result = result.replace(prepareSourceDependencyPattern, prepareSourceDependencyScript());", " }", " const artifactPublishNeedle = 'node scripts/artifact-publish.mjs --publish';", " if (result.includes(artifactPublishNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {", " result = result.replace(artifactPublishNeedle, `${deployYamlOverlayScript()}\\n${artifactPublishNeedle}`);", " }", " const gitopsRenderNeedle = 'node scripts/run-bun.mjs scripts/gitops-render.mjs';", " if (!isNodeContractCheck && result.includes(gitopsRenderNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {", " result = result.replace(gitopsRenderNeedle, `${deployYamlOverlayScript()}\\n${gitopsRenderNeedle}`);", " }", " result = result.replaceAll('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808', `node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost || '127.0.0.1'} ${overlay.gitSshProxyPort || 10808}`);", " result = result.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);", " result = result.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);", " result = result.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);", " result = result.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", " result = result.replace(/--gitops-root \"\\$gitops_root\"/g, `--gitops-root ${JSON.stringify(overlay.gitopsRoot)}`);", " result = result.replace(/--out \"\\$gitops_root\"/g, `--out ${JSON.stringify(overlay.gitopsRoot)}`);", " const legacyRuntimePath = (() => {", " const runtimePath = String(overlay.runtimePath || '');", " const parts = runtimePath.split('/').filter(Boolean);", " if (parts.length < 2) return '';", " const leaf = parts.at(-1);", " const parent = parts.slice(0, -2).join('/');", " return parent && leaf ? `${parent}/${leaf}` : '';", " })();", " if (legacyRuntimePath && legacyRuntimePath !== overlay.runtimePath) {", " result = result.replace('runtime_path=\"$(params.runtime-path)\"', `runtime_path=\"$(params.runtime-path)\"\\nunidesk_legacy_runtime_path=${shellSingle(legacyRuntimePath)}`);", " result = result.replace('rm -rf \"$runtime_path\" \"$catalog_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi', 'rm -rf \"$runtime_path\" \"$catalog_path\" \"$unidesk_legacy_runtime_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi');", " result = result.replace('git add \"$catalog_path\" \"$runtime_path\"', 'git add \"$catalog_path\" \"$runtime_path\"\\n if [ -n \"${unidesk_legacy_runtime_path:-}\" ]; then git add -A \"$unidesk_legacy_runtime_path\" || true; fi');", " }", " if (bootstrap && result.includes('git config --global') && !result.includes('unidesk-step-env-bootstrap')) {", " result = result.replace(/\\n([ \\t]*)git config --global/g, (match, indent) => `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`);", " }", " result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--use-deploy-images/g, (match) => {", " let next = match;", " if (!next.includes('--gitops-root ')) next = next.replace(' --use-deploy-images', ` --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);", " if (!next.includes('--out ')) next = next.replace(' --use-deploy-images', ` --out ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);", " return next;", " });", " result = result.replace(/(node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*--use-deploy-images[^\\n]*)/g, (match) => {", " if (match.includes('--check')) return runtimeGitopsVerifyScript();", " return `${match}\\n${[runtimePathOverlayScript(), runtimeGitopsPostprocessScript()].filter(Boolean).join('\\n')}`;", " });", " result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--out \"\\$render_check_dir\"/g, (match) => match.includes('--gitops-root ') ? match : `${match} --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --node ${shellSingle(overlay.nodeId)} --git-read-url ${shellSingle(overlay.gitReadUrl)} --git-write-url ${shellSingle(overlay.gitWriteUrl)} --runtime-endpoint ${shellSingle(overlay.publicApiUrl)} --web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", " validatePrepareSourceDependencyScript(result);", " return result;", "}", "function patchManifestObject(doc) {", " if (!doc || typeof doc !== 'object') return false;", " if (doc.kind !== 'Pipeline' || !doc.spec) return false;", " const defaults = {", " 'git-url': overlay.gitUrl,", " 'git-read-url': overlay.gitReadUrl,", " 'git-write-url': overlay.gitWriteUrl,", " 'catalog-path': overlay.catalogPath,", " 'runtime-path': overlay.runtimePath,", " 'registry-prefix': overlay.registryPrefix,", " };", " for (const param of doc.spec?.params || []) {", " if (Object.prototype.hasOwnProperty.call(defaults, param.name)) param.default = defaults[param.name];", " }", " doc.metadata = doc.metadata || {};", " if (typeof overlay.pipelineName === 'string' && overlay.pipelineName.length > 0) doc.metadata.name = overlay.pipelineName;", " doc.metadata.annotations = doc.metadata.annotations || {};", " doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId;", " doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId;", " for (const task of doc.spec?.tasks || []) {", " for (const sidecar of task.taskSpec?.sidecars || []) {", " if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage;", " }", " for (const step of task.taskSpec?.steps || []) {", " if (step.image === overlay.toolsImage && overlay.toolsImagePullPolicy) step.imagePullPolicy = overlay.toolsImagePullPolicy;", " if (Array.isArray(step.env)) {", " for (const env of step.env) {", " if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name];", " }", " }", " step.env = Array.isArray(step.env) ? step.env : [];", " const existingEnv = new Set(step.env.map((env) => env.name));", " for (const [name, value] of Object.entries(stepEnv)) {", " if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value });", " }", " if (typeof step.script === 'string') step.script = patchScript(step.script);", " }", " }", " return true;", "}", "function patchStructuredPipeline() {", " try {", " const doc = JSON.parse(text);", " if (!patchManifestObject(doc)) return false;", " text = JSON.stringify(doc, null, 2) + '\\n';", " return true;", " } catch {}", " if (YAML) {", " try {", " const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null);", " const changed = docs.some((doc) => patchManifestObject(doc));", " if (!changed) return false;", " text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n';", " return true;", " } catch {}", " }", " return false;", "}", "function patchGitMirrorTransportYaml() {", " const mirror = overlay.gitMirror || {};", " const transport = mirror.githubTransport || {};", " if (transport.mode !== 'https') return;", " if (!YAML) throw new Error('yaml module is required to patch git-mirror githubTransport=https');", " const requireString = (pathName, value) => {", " if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror githubTransport=https');", " return value;", " };", " const repository = requireString('gitMirror.sourceRepository', mirror.sourceRepository);", " const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);", " const tokenSecretName = requireString('gitMirror.githubTransport.tokenSecretName', transport.tokenSecretName);", " const tokenSecretKey = requireString('gitMirror.githubTransport.tokenSecretKey', transport.tokenSecretKey);", " const tokenSourceRef = requireString('gitMirror.githubTransport.tokenSourceRef', transport.tokenSourceRef);", " const username = typeof transport.username === 'string' && transport.username ? transport.username : 'x-access-token';", " const remoteUrl = `https://github.com/${repository}.git`;", " const proxySummary = `transport=https proxy=HTTP_PROXY authSecret=${tokenSecretName} authKey=${tokenSecretKey} authSourceRef=${tokenSourceRef} source=yaml`;", " const askpassBlock = [", " 'if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo \\'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env\\' >&2; exit 64; fi',", " \"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'\",", " '#!/bin/sh',", " 'case \"$1\" in',", " ' *Username*) printf \\'%s\\\\n\\' \"${GITHUB_USERNAME:-x-access-token}\" ;;',", " ' *Password*) printf \\'%s\\\\n\\' \"$GITHUB_TOKEN\" ;;',", " \" *) printf '\\\\n' ;;\",", " 'esac',", " 'SH_ASKPASS',", " 'chmod 0700 /tmp/hwlab-git-askpass.sh',", " `export GITHUB_USERNAME=${shellSingle(username)}`,", " 'export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh',", " 'export GIT_TERMINAL_PROMPT=0',", " 'unset GIT_SSH',", " 'unset GIT_SSH_COMMAND',", " ].join('\\n');", " function patchScript(script, key) {", " let next = String(script || '');", " if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);", " let remoteReplaced = false;", " next = next.replace(/repo_url=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `repo_url=${shellSingle(remoteUrl)}`; });", " next = next.replace(/remote=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `remote=${shellSingle(remoteUrl)}`; });", " if (!remoteReplaced) throw new Error(`generated git-mirror ${key} missing remote url assignment for githubTransport=https`);", " next = next.replace(/transport=ssh ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);", " next = next.replace(/ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);", " next = next.replace(/mkdir -p ([^\\n]*?) \\/root\\/\\.ssh/g, 'mkdir -p $1');", " next = next.replace(/\\n[ \\t]*mkdir -p \\/root\\/\\.ssh\\n/g, '\\n');", " next = next.replace(/\\n[ \\t]*cp \\/git-ssh\\/ssh-privatekey[^\\n]*\\n[ \\t]*chmod 0?400[^\\n]*\\n/g, '\\n');", " next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) <<'NODE_PROXY'[\\s\\S]*?\\nNODE_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs)\\n/g, '\\n');", " next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-git-ssh-proxy\\.sh <<'SH_PROXY'[\\s\\S]*?\\nSH_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-git-ssh-proxy\\.sh\\n/g, '\\n');", " next = next.replace(/\\n[ \\t]*export GIT_SSH=.*\\n/g, '\\n');", " next = next.replace(/\\n[ \\t]*export GIT_SSH_COMMAND=.*\\n/g, '\\n');", " next = next.replace(/\\n[ \\t]*unset GIT_SSH_COMMAND\\n/g, '\\n');", " if (!next.includes('hwlab git-mirror https auth: missing GITHUB_TOKEN secret env')) {", " const noProxyExport = /\\nexport no_proxy=[^\\n]*\\n/;", " if (noProxyExport.test(next)) next = next.replace(noProxyExport, (match) => `${match}${askpassBlock}\\n`);", " else if (next.includes('\\nset -eu\\n')) next = next.replace('\\nset -eu\\n', `\\nset -eu\\n${askpassBlock}\\n`);", " else next = `${askpassBlock}\\n${next}`;", " }", " if (next.includes('/git-ssh') || next.includes('ssh://git@') || next.includes('GIT_SSH=')) throw new Error(`generated git-mirror ${key} still contains ssh transport after githubTransport=https patch`);", " if (!next.includes('GIT_ASKPASS') || !next.includes('GITHUB_TOKEN')) throw new Error(`generated git-mirror ${key} missing https auth after githubTransport=https patch`);", " return next;", " }", " const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');", " if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);", " const docs = YAML.parseAllDocuments(fs.readFileSync(gitMirrorFile, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);", " const manifests = [];", " for (const doc of docs) {", " if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);", " else manifests.push(doc);", " }", " let changed = false;", " for (const doc of manifests) {", " if (!doc || typeof doc !== 'object' || doc.kind !== 'ConfigMap') continue;", " if (!doc.metadata || doc.metadata.name !== configMapName) continue;", " doc.data = doc.data || {};", " doc.data['sync.sh'] = patchScript(doc.data['sync.sh'], 'sync.sh');", " doc.data['flush.sh'] = patchScript(doc.data['flush.sh'], 'flush.sh');", " changed = true;", " }", " if (!changed) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);", " fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');", "}", "function patchGitMirrorHostRouteYaml() {", " const mirror = overlay.gitMirror || {};", " const proxy = mirror.egressProxy || {};", " if (proxy.mode !== 'host-route') return;", " if (!YAML) throw new Error('yaml module is required to patch git-mirror host-route proxy');", " const requireString = (pathName, value) => {", " if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror host-route proxy');", " return value;", " };", " const proxyUrl = requireString('gitMirror.egressProxy.proxyUrl', proxy.proxyUrl);", " const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);", " let proxyEndpoint;", " try { proxyEndpoint = new URL(proxyUrl); } catch (error) { throw new Error(`overlay.gitMirror.egressProxy.proxyUrl is invalid: ${error.message}`); }", " if (proxyEndpoint.protocol !== 'http:') throw new Error('overlay.gitMirror.egressProxy.proxyUrl must use http:// for host-route');", " const proxyHost = proxyEndpoint.hostname;", " const proxyPort = Number(proxyEndpoint.port || '80');", " if (!proxyHost || !Number.isInteger(proxyPort) || proxyPort < 1 || proxyPort > 65535) throw new Error('overlay.gitMirror.egressProxy.proxyUrl must include a valid host and port');", " const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.join(',') : '';", " const envEntries = [", " ['HTTP_PROXY', proxyUrl],", " ['HTTPS_PROXY', proxyUrl],", " ['ALL_PROXY', proxyUrl],", " ['http_proxy', proxyUrl],", " ['https_proxy', proxyUrl],", " ['all_proxy', proxyUrl],", " ['NO_PROXY', noProxy],", " ['no_proxy', noProxy],", " ];", " function readDocs(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }", " function flattenDocs(docs) {", " const manifests = [];", " for (const doc of docs) {", " if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);", " else manifests.push(doc);", " }", " return manifests;", " }", " function setContainerEnv(container, name, value) {", " if (!container || typeof container !== 'object') return false;", " container.env = Array.isArray(container.env) ? container.env : [];", " let item = container.env.find((env) => env && env.name === name);", " if (!item) { item = { name }; container.env.push(item); }", " const changed = item.value !== value || item.valueFrom !== undefined;", " item.value = value;", " delete item.valueFrom;", " return changed;", " }", " function patchProxyScript(script, key) {", " let next = String(script || '');", " if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);", " next = next.replace(/node \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) \\S+ \\d+/g, `node /tmp/hwlab-github-proxy-connect.mjs ${proxyHost} ${proxyPort}`);", " const missing = [];", " for (const [name, value] of envEntries) {", " let replaced = false;", " const exportPattern = new RegExp(`(^|\\\\n)([ \\\\t]*export ${escapeRegExp(name)}=)[^\\\\n]*`, 'g');", " next = next.replace(exportPattern, (_match, prefix, left) => { replaced = true; return `${prefix}${left}${shellSingle(value)}`; });", " if (!replaced) missing.push([name, value]);", " }", " if (missing.length > 0) {", " const block = missing.map(([name, value]) => `export ${name}=${shellSingle(value)}`).join('\\\\n');", " if (next.includes('\\\\nset -eu\\\\n')) next = next.replace('\\\\nset -eu\\\\n', `\\\\nset -eu\\\\n${block}\\\\n`);", " else next = `${block}\\\\n${next}`;", " }", " next = next.replace(/mode=node-global/g, 'mode=host-route');", " return next;", " }", " const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');", " if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);", " const docs = readDocs(gitMirrorFile);", " const manifests = flattenDocs(docs);", " let configMapChanged = false;", " let workloadChanged = false;", " const workloadNames = new Set([mirror.serviceReadName, mirror.serviceWriteName].filter((value) => typeof value === 'string' && value.length > 0));", " for (const doc of manifests) {", " if (!doc || typeof doc !== 'object') continue;", " if (doc.kind === 'ConfigMap' && doc.metadata && doc.metadata.name === configMapName) {", " doc.data = doc.data || {};", " doc.data['sync.sh'] = patchProxyScript(doc.data['sync.sh'], 'sync.sh');", " doc.data['flush.sh'] = patchProxyScript(doc.data['flush.sh'], 'flush.sh');", " configMapChanged = true;", " continue;", " }", " if (!['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(doc.kind)) continue;", " const name = doc.metadata && doc.metadata.name ? String(doc.metadata.name) : '';", " const labels = doc.metadata && doc.metadata.labels && typeof doc.metadata.labels === 'object' ? doc.metadata.labels : {};", " if (!workloadNames.has(name) && labels['app.kubernetes.io/name'] !== 'git-mirror' && !name.includes('git-mirror')) continue;", " doc.spec = doc.spec || {};", " doc.spec.template = doc.spec.template || {};", " doc.spec.template.metadata = doc.spec.template.metadata || {};", " doc.spec.template.metadata.annotations = doc.spec.template.metadata.annotations || {};", " doc.spec.template.metadata.annotations['unidesk.ai/git-mirror-egress-proxy'] = 'host-route';", " doc.spec.template.metadata.annotations['unidesk.ai/git-mirror-host-proxy-config-ref'] = String(proxy.hostProxyConfigRef || '');", " doc.spec.template.spec = doc.spec.template.spec || {};", " if (proxy.podHostNetwork) {", " doc.spec.template.spec.hostNetwork = true;", " doc.spec.template.spec.dnsPolicy = 'ClusterFirstWithHostNet';", " } else {", " delete doc.spec.template.spec.hostNetwork;", " delete doc.spec.template.spec.dnsPolicy;", " }", " for (const group of ['containers', 'initContainers']) {", " for (const container of Array.isArray(doc.spec.template.spec[group]) ? doc.spec.template.spec[group] : []) {", " for (const [envName, envValue] of envEntries) setContainerEnv(container, envName, envValue);", " }", " }", " workloadChanged = true;", " }", " if (!configMapChanged) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);", " if (!workloadChanged) throw new Error(`generated git-mirror workload for host-route proxy was not found in ${gitMirrorFile}`);", " fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');", "}", "const structured = patchStructuredPipeline();", "function replaceParamDefault(name, value) {", " const namePattern = escapeRegExp(name);", " text = text.replace(new RegExp(`(- default: )[^\\n]*(\\n\\\\s+name: ${namePattern}\\n\\\\s+type: string)`, 'g'), `$1${yamlString(value)}$2`);", " text = text.replace(new RegExp(`(- name: ${namePattern}\\n\\\\s+type: string\\n\\\\s+default: )[^\\n]*`, 'g'), `$1${yamlString(value)}`);", "}", "replaceParamDefault('git-url', overlay.gitUrl);", "replaceParamDefault('git-read-url', overlay.gitReadUrl);", "replaceParamDefault('git-write-url', overlay.gitWriteUrl);", "replaceParamDefault('catalog-path', overlay.catalogPath);", "replaceParamDefault('runtime-path', overlay.runtimePath);", "replaceParamDefault('registry-prefix', overlay.registryPrefix);", "text = text.replace(/hwlab\\.pikastech\\.local\\/download-profile: [^\\n]+/g, `hwlab.pikastech.local/download-profile: ${overlay.downloadProfileId}`);", "text = text.replace(/hwlab\\.pikastech\\.local\\/network-profile: [^\\n]+/g, `hwlab.pikastech.local/network-profile: ${overlay.networkProfileId}`);", "if (overlay.buildkitSidecarImage) text = text.replace(/moby\\/buildkit:rootless/g, overlay.buildkitSidecarImage);", "text = text.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);", "text = text.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);", "text = text.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);", "text = text.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);", "for (const [name, value] of Object.entries(stepEnv)) {", " if (value === undefined) continue;", " text = text.replace(new RegExp(`(- name: ${escapeRegExp(name)}\\\\n\\\\s+value: )[^\\\\n]+`, 'g'), `$1${yamlString(value)}`);", " text = text.replace(new RegExp(`(\\\"name\\\": ${JSON.stringify(name)},\\\\n\\\\s+\\\"value\\\": )\\\"[^\\\"]*\\\"`, 'g'), `$1${yamlString(value)}`);", "}", "const bootstrap = stepEnvBootstrapScript();", "if (bootstrap) {", " text = text.replace(/\\n([ \\t]*)git config --global/g, (match, indent, offset, fullText) => {", " const previous = fullText.slice(Math.max(0, offset - 500), offset);", " if (previous.includes('unidesk-step-env-bootstrap')) return match;", " return `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`;", " });", "}", "if (overlay.gitSshProxyHost && overlay.gitSshProxyPort) {", " text = text.split('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808').join(`node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost} ${overlay.gitSshProxyPort}`);", "}", "const quotedRoot = JSON.stringify(overlay.gitopsRoot);", "const escapedQuotedRoot = quotedRoot.replaceAll('\"', '\\\\\"');", "const replacements = [", " ['--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images' : `--registry-prefix \"$(params.registry-prefix)\" --gitops-root ${quotedRoot} --use-deploy-images`],", " ['--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images' : `--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --gitops-root ${escapedQuotedRoot} --use-deploy-images`],", "];", "let changed = false;", "for (const [needle, replacement] of replacements) {", " if (!text.includes(needle)) continue;", " text = text.split(needle).join(replacement);", " changed = true;", "}", "if (!structured && !changed && !text.includes(`--gitops-root ${quotedRoot}`) && !text.includes(`--gitops-root ${escapedQuotedRoot}`)) { throw new Error(`generated pipeline missing expected gitops-render invocation in ${pipelinePath}`); }", "if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && text.includes('npm ci --ignore-scripts --no-audit --prefer-offline')) { throw new Error(`generated pipeline still uses full npm ci prepare-source dependency install in ${pipelinePath}`); }", "if (text.includes('readStructuredFile(process.cwd(), deployPath)')) { throw new Error(`generated pipeline still imports YAML parser for prepare-source catalog check in ${pipelinePath}`); }", "if (text.includes('/yaml/-/yaml-') || text.includes('bun add --no-save --ignore-scripts') || text.includes('npm install --package-lock=false --no-save')) { throw new Error(`generated pipeline still downloads yaml during prepare-source in ${pipelinePath}`); }", "if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }", "fs.writeFileSync(pipelinePath, text);", "patchGitMirrorHostRouteYaml();", "patchGitMirrorTransportYaml();", "function patchArgoYaml(filePath) {", " if (!YAML || !fs.existsSync(filePath)) return;", " const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);", " let changed = false;", " for (const doc of docs) {", " if (!doc || typeof doc !== 'object') continue;", " if (doc.kind === 'AppProject') {", " doc.spec = doc.spec || {};", " doc.spec.sourceRepos = [overlay.argoRepoUrl];", " changed = true;", " }", " if (doc.kind === 'Application') {", " doc.spec = doc.spec || {};", " doc.spec.source = { ...(doc.spec.source || {}), repoURL: overlay.argoRepoUrl, targetRevision: overlay.gitopsBranch, path: overlay.runtimePath };", " changed = true;", " }", " }", " if (changed) fs.writeFileSync(filePath, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');", "}", "patchArgoYaml(path.join(renderDir, 'argocd', 'project.yaml'));", "patchArgoYaml(path.join(renderDir, 'argocd', overlay.argoApplicationFile));", "NODE", ]; }