mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ac4b9b60ad421dba393669cd1141a2c4c742cf44
pikasTech-unidesk/scripts/src/hwlab-node/render.ts
T
Codex ac4b9b60ad fix: project code agent stdio supervisor
2026-06-28 03:05:59 +00:00

2265 lines
140 KiB
TypeScript
Raw Blame History

// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/hwlab-node-impl.ts.
// Moved mechanically from scripts/src/hwlab-node-impl.ts:4000-5260 for #903.
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
import { createHash, randomBytes } from "node:crypto";
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { repoRoot, rootPath, type Config } from "../config";
import { runCommand, type CommandResult } from "../command";
import { startJob } from "../jobs";
import { classifySshTcpPoolFailure } from "../ssh";
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
import type { RenderedCliResult } from "../output";
import type { NodeRuntimeGitMirrorTargetSpec, NodeRuntimeRenderResult } from "./entry";
import { isCommandSuccess, nodeRuntimeGitopsRoot, nodeRuntimePipelineRunName, resolveNodeRuntimeLaneHead, runNodeK3sArgs, runtimeLaneCicdRepoEnsureScript, shortSha } from "./cleanup";
import { HWLAB_CI_NAMESPACE } from "./entry";
import { nodeRuntimeExpected, nodeRuntimeLocalPostgresExpectedAbsent, parseNodeScopedDelegatedOptions } from "./plan";
import { runTransHostScript } from "./public-exposure";
import { compactRuntimeCommand, runNodeHostScriptAsync } from "./runtime-common";
import { compactNodeRuntimeGitMirrorStatus, nodeRuntimeGitMirrorStatus } from "./status";
import { keyValueLinesFromText, numericField, optionValue, record, shellQuote } from "./utils";
import { externalPostgresBridgeStatus, externalPostgresSecretStatus, getNodeRuntimePipelineRun, isLocalPostgresObject, nodeRuntimeCodeAgentRuntimeStatus, nodeRuntimeRenderOverlay } from "./web-probe";
import { webObserveShort, webObserveText } from "./web-probe-observe";
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
export function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush"): string {
const prefix = action === "sync" ? mirror.syncJobPrefix : mirror.flushJobPrefix;
return `${prefix}-${Date.now().toString(36)}`.slice(0, 63);
}
export function nodeRuntimeGitMirrorJobManifest(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush", jobName: string): Record<string, unknown> {
const volumes: Record<string, unknown>[] = [
{ name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) },
{ name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } },
];
const volumeMounts: Record<string, unknown>[] = [
{ name: "cache", mountPath: "/cache" },
{ name: "script", mountPath: "/script", readOnly: true },
];
if (mirror.githubTransport.mode === "ssh") {
volumes.splice(1, 0, { name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } });
volumeMounts.splice(1, 0, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true });
}
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: {
name: jobName,
namespace: mirror.namespace,
labels: {
"app.kubernetes.io/name": "git-mirror",
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
"app.kubernetes.io/component": `${action}-controller`,
"hwlab.pikastech.local/node": mirror.node,
"hwlab.pikastech.local/lane": mirror.lane,
"hwlab.pikastech.local/trigger": "manual-cli",
},
},
spec: {
backoffLimit: 0,
activeDeadlineSeconds: 600,
ttlSecondsAfterFinished: 3600,
template: {
metadata: {
labels: {
"app.kubernetes.io/name": "git-mirror",
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
"app.kubernetes.io/component": `${action}-controller`,
"hwlab.pikastech.local/node": mirror.node,
"hwlab.pikastech.local/lane": mirror.lane,
},
},
spec: {
restartPolicy: "Never",
volumes,
containers: [{
name: action,
image: mirror.toolsImage,
imagePullPolicy: mirror.toolsImagePullPolicy,
command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"],
env: [...nodeRuntimeGitMirrorProxyEnv(mirror), ...nodeRuntimeGitMirrorGithubTransportEnv(mirror)],
volumeMounts,
}],
},
},
},
};
}
export function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
const proxy = mirror.egressProxy;
if (proxy.mode === "direct") return [];
const proxyUrl = `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`;
const noProxy = proxy.noProxy.join(",");
return [
{ name: "HTTP_PROXY", value: proxyUrl },
{ name: "HTTPS_PROXY", value: proxyUrl },
{ name: "ALL_PROXY", value: proxyUrl },
{ name: "http_proxy", value: proxyUrl },
{ name: "https_proxy", value: proxyUrl },
{ name: "all_proxy", value: proxyUrl },
{ name: "NO_PROXY", value: noProxy },
{ name: "no_proxy", value: noProxy },
];
}
export function nodeRuntimeGitMirrorGithubTransportEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
if (mirror.githubTransport.mode !== "https") return [];
return [{
name: "GITHUB_TOKEN",
valueFrom: { secretKeyRef: { name: mirror.githubTransport.tokenSecretName, key: mirror.githubTransport.tokenSecretKey } },
}];
}
export function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown> {
const transport = mirror.githubTransport;
if (transport.mode === "ssh") {
return {
mode: "ssh",
secretName: mirror.secretName,
privateKeySecretKey: transport.privateKeySecretKey,
privateKeySourceRef: transport.privateKeySourceRef,
privateKeySourceKey: transport.privateKeySourceKey,
privateKeySourceEncoding: transport.privateKeySourceEncoding,
knownHostsSecretKey: transport.knownHostsSecretKey,
knownHostsSourceRef: transport.knownHostsSourceRef,
knownHostsSourceKey: transport.knownHostsSourceKey,
knownHostsSourceEncoding: transport.knownHostsSourceEncoding,
valuesPrinted: false,
};
}
return {
mode: "https",
username: transport.username,
tokenSecretName: transport.tokenSecretName,
tokenSecretKey: transport.tokenSecretKey,
tokenSourceRef: transport.tokenSourceRef,
tokenSourceKey: transport.tokenSourceKey,
valuesPrinted: false,
};
}
export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run");
const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null;
const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null;
const pipelineRun = pipelineRunOverride ?? (sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit));
const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], 60);
const namespaceExists = namespace.exitCode === 0;
const postgresObjects = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], 60)
: null;
const localPostgresObjects = postgresObjects === null
? []
: postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec));
const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], 60);
const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], 60);
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60);
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
: null;
const workloads = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60)
: null;
const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
const workloadReadinessProbe = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], 60)
: null;
const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? "");
const bridge = externalPostgresBridgeStatus(spec, namespaceExists);
const secrets = externalPostgresSecretStatus(spec, namespaceExists);
const codeAgentRuntime = nodeRuntimeCodeAgentRuntimeStatus(spec, namespaceExists);
const publicProbes = nodeRuntimePublicProbeStatus(spec);
const gitMirror = nodeRuntimeGitMirrorStatus(scoped);
const gitMirrorCompact = compactNodeRuntimeGitMirrorStatus(gitMirror);
const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec);
const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0;
const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready);
const localPostgresExpectedAbsent = nodeRuntimeLocalPostgresExpectedAbsent(spec);
const localPostgresReady = localPostgresExpectedAbsent ? localPostgresObjects.length === 0 : localPostgresObjects.length > 0;
const runtimeReady = namespaceExists && localPostgresReady && workloadsReady && (activeExternalPostgres === undefined || (bridge.ready && secrets.ready)) && codeAgentRuntime.ready === true;
const codeAgentRuntimeDegradedReason = typeof codeAgentRuntime.degradedReason === "string" ? codeAgentRuntime.degradedReason : null;
const runtimeDegradedReason = !namespaceExists
? "runtime-namespace-missing"
: !localPostgresReady
? "runtime-local-postgres-not-ready"
: !workloadsReady
? "runtime-workloads-not-ready"
: activeExternalPostgres !== undefined && bridge.ready !== true
? "external-postgres-bridge-not-ready"
: activeExternalPostgres !== undefined && secrets.ready !== true
? "external-postgres-secrets-not-ready"
: codeAgentRuntime.ready !== true
? codeAgentRuntimeDegradedReason ?? "code-agent-runtime-not-ready"
: "runtime-not-ready";
const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
const pipelineRunDegradedReason = typeof pipelineRunDiagnostics?.degradedReason === "string"
? pipelineRunDiagnostics.degradedReason
: "pipelinerun-not-succeeded";
const publicReady = publicProbes.ready === true;
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
const fullStatus = {
ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady && gitMirrorReady,
command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
mode: "node-scoped-runtime-status",
mutation: false,
node: scoped.node,
lane: scoped.lane,
sourceCommit,
pipelineRun,
expected: nodeRuntimeExpected(spec),
sourceHead: head === null
? { ok: sourceCommit !== null, value: sourceCommit, source: "option" }
: { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) },
controlPlane: {
ready: controlPlaneReady,
serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) },
pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) },
},
argo: {
ready: argoReady,
application: spec.app,
repoURL,
expectedRepoURL: spec.argoRepoUrl,
targetRevision,
path,
syncRevision,
syncStatus,
health,
result: compactRuntimeCommand(argo),
},
pipelineRun: pipelineRunProbe,
pipelineRunDiagnostics,
runtime: {
ready: runtimeReady,
namespace: spec.runtimeNamespace,
namespaceExists,
localPostgresObjects,
localPostgresAbsent: namespaceExists && localPostgresObjects.length === 0,
localPostgresExpectedAbsent,
localPostgresReady,
workloadNames,
workloadCount: workloadNames.length,
workloadReadiness,
workloadsReady,
workloadReadinessResult: workloadReadinessProbe === null ? null : compactRuntimeCommand(workloadReadinessProbe),
workloadResult: workloads === null ? null : compactRuntimeCommand(workloads),
externalPostgresBridge: bridge,
externalPostgresSecrets: secrets,
codeAgentRuntime,
},
publicProbes,
gitMirror: {
...gitMirror,
compact: gitMirrorCompact,
ready: gitMirrorReady,
},
probes: {
namespace: compactRuntimeCommand(namespace),
postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects),
},
degradedReason: controlPlaneReady
? runtimeReady
? argoReady
? pipelineRunReady
? publicReady
? gitMirrorReady ? undefined : "git-mirror-pending-flush"
: "public-probe-not-ready"
: pipelineRunDegradedReason
: "argo-not-synced-healthy"
: runtimeDegradedReason
: "control-plane-not-ready",
next: {
plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
},
};
const summary = summarizeNodeRuntimeControlPlaneStatus(fullStatus, scoped);
if (scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw")) return { ...fullStatus, summary };
return summary;
}
export function parseNodeRuntimeWorkloadReadiness(text: string): Array<Record<string, unknown>> {
return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => {
const [ref = "", counts = ""] = line.split(/\t/u);
const [readyRaw = "", currentRaw = "", desiredRaw = ""] = counts.split("/");
const readyReplicas = nullableInteger(readyRaw);
const currentReplicas = nullableInteger(currentRaw);
const desiredReplicas = nullableInteger(desiredRaw);
const ready = desiredReplicas !== null
? (readyReplicas ?? 0) >= desiredReplicas
: readyReplicas !== null && currentReplicas !== null && readyReplicas >= currentReplicas;
return {
ref,
readyReplicas,
currentReplicas,
desiredReplicas,
ready,
};
});
}
export function nullableInteger(value: string): number | null {
if (!/^[0-9]+$/u.test(value)) return null;
return Number(value);
}
export function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const web = publicHttpProbe("web", spec.publicWebUrl);
const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live"));
const ready = web.ok === true && apiHealth.ok === true;
const targetHost = nodeRuntimeTargetHostPublicProbeStatus(spec);
return {
ready,
web,
apiHealth,
targetHost,
diagnostic: nodeRuntimePublicProbeDiagnostic(ready, targetHost),
};
}
export function publicHttpProbe(kind: string, url: string): Record<string, unknown> {
const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "/dev/null", "-w", "%{http_code}", url], repoRoot, { timeoutMs: 15_000 });
const httpStatus = nullableInteger(result.stdout.trim().slice(-3));
return {
kind,
url,
ok: result.exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400,
httpStatus,
result: compactRuntimeCommand(result),
};
}
export function nodeRuntimeTargetHostPublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const webUrl = spec.publicWebUrl;
const apiHealthUrl = joinUrlPath(spec.publicApiUrl, "/health/live");
const script = [
"set -eu",
`web_url=${shellQuote(webUrl)}`,
`api_url=${shellQuote(apiHealthUrl)}`,
"probe() {",
" name=\"$1\"",
" url=\"$2\"",
" err_file=$(mktemp)",
" set +e",
" http_status=$(curl -k -sS --connect-timeout 5 --max-time 12 -o /dev/null -w '%{http_code}' \"$url\" 2>\"$err_file\")",
" rc=$?",
" set -e",
" error_text=$(tr '\\r\\n\\t' ' ' <\"$err_file\" | tail -c 600)",
" rm -f \"$err_file\"",
" printf '%sUrl\\t%s\\n' \"$name\" \"$url\"",
" printf '%sExitCode\\t%s\\n' \"$name\" \"$rc\"",
" printf '%sHttpStatus\\t%s\\n' \"$name\" \"$http_status\"",
" printf '%sError\\t%s\\n' \"$name\" \"$error_text\"",
"}",
"probe web \"$web_url\"",
"probe apiHealth \"$api_url\"",
].join("\n");
const result = runTransHostScript(spec.nodeId, script, "", 35);
const fields = keyValueLinesFromText(result.stdout);
const web = targetHostPublicHttpProbeFromFields("web", fields, webUrl, result.exitCode === 0);
const apiHealth = targetHostPublicHttpProbeFromFields("apiHealth", fields, apiHealthUrl, result.exitCode === 0);
return {
node: spec.nodeId,
ready: web.ok === true && apiHealth.ok === true,
probeAvailable: result.exitCode === 0 && result.timedOut !== true,
web,
apiHealth,
result: compactRuntimeCommand(result),
};
}
export function targetHostPublicHttpProbeFromFields(kind: string, fields: Record<string, string>, fallbackUrl: string, transportOk: boolean): Record<string, unknown> {
const exitCode = numericField(fields[`${kind}ExitCode`]);
const httpStatus = numericField(fields[`${kind}HttpStatus`]);
const error = fields[`${kind}Error`] ?? "";
return {
kind,
url: fields[`${kind}Url`] ?? fallbackUrl,
ok: transportOk && exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400,
httpStatus,
exitCode,
error: error.length > 0 ? error.slice(0, 600) : null,
};
}
export function nodeRuntimePublicProbeDiagnostic(publicReady: boolean, targetHost: Record<string, unknown>): Record<string, unknown> {
const targetWeb = record(targetHost.web);
const targetApiHealth = record(targetHost.apiHealth);
const targetReady = targetHost.ready === true;
if (!publicReady) {
return {
kind: "public-entry-probe-failed",
affectsUserEntry: true,
targetHostReady: targetReady,
message: "control-plane public probe failed; treat this as a public endpoint readiness failure before using web-probe closeout evidence.",
nextAction: "run web-probe run for the same node/lane after checking publicProbe.web and publicProbe.apiHealth",
};
}
if (targetHost.probeAvailable !== true) {
return {
kind: "target-host-public-probe-unavailable",
affectsUserEntry: false,
targetHostReady: false,
message: "control-plane public probe passed, but the target host diagnostic probe could not run; this does not invalidate user-entry evidence.",
nextAction: "use control-plane publicProbe and web-probe evidence for closeout; inspect target host SSH/trans health separately if host-side CLI must call the public URL",
};
}
if (!targetReady) {
return {
kind: "target-host-public-egress-mismatch",
affectsUserEntry: false,
targetHostReady: false,
failed: {
web: targetWeb.ok === true ? null : { httpStatus: targetWeb.httpStatus ?? null, exitCode: targetWeb.exitCode ?? null, error: targetWeb.error ?? null },
apiHealth: targetApiHealth.ok === true ? null : { httpStatus: targetApiHealth.httpStatus ?? null, exitCode: targetApiHealth.exitCode ?? null, error: targetApiHealth.error ?? null },
},
message: "control-plane public probe passed, but the target host cannot reach the same public URLs; classify this as target-host egress/hairpin diagnostics, not a public endpoint failure.",
nextAction: "use control-plane publicProbe and web-probe evidence for issue closeout; track host-side public URL access separately if hwlab-cli must run on that host",
};
}
return {
kind: "public-entry-and-target-host-ok",
affectsUserEntry: false,
targetHostReady: true,
message: "control-plane public probe and target-host public URL probe both passed.",
nextAction: null,
};
}
export function joinUrlPath(baseUrl: string, suffix: string): string {
return `${baseUrl.replace(/\/+$/u, "")}/${suffix.replace(/^\/+/u, "")}`;
}
export function compactNodeRuntimeTaskRunDiagnostic(value: unknown): string {
const item = record(value);
const pipelineTask = webObserveText(item.pipelineTask ?? item.pipelineTaskName);
const taskRun = webObserveText(item.taskRun ?? item.name);
const reason = webObserveText(item.reason ?? item.status ?? item.message);
const left = [pipelineTask, taskRun].filter(Boolean).join("/");
return [left, reason ? `(${webObserveShort(reason, 36)})` : ""].filter(Boolean).join("");
}
export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const pipelineRun = record(status.pipelineRun);
const pipelineRunDiagnostics = record(status.pipelineRunDiagnostics);
const argo = record(status.argo);
const runtime = record(status.runtime);
const publicProbes = record(status.publicProbes);
const gitMirror = record(status.gitMirror);
const gitMirrorCompact = record(gitMirror.compact);
const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : [];
const readyWorkloads = workloadReadiness.filter((item) => item.ready === true).length;
const notReadyWorkloads = workloadReadiness.filter((item) => item.ready !== true).map((item) => item.ref).filter(Boolean);
const workloadCount = typeof runtime.workloadCount === "number" ? runtime.workloadCount : workloadReadiness.length;
const webProbe = record(publicProbes.web);
const apiProbe = record(publicProbes.apiHealth);
const targetHostProbe = record(publicProbes.targetHost);
const targetHostWebProbe = record(targetHostProbe.web);
const targetHostApiProbe = record(targetHostProbe.apiHealth);
const publicProbeDiagnostic = record(publicProbes.diagnostic);
return {
ok: status.ok === true,
command: status.command,
mode: "node-scoped-runtime-status-summary",
mutation: false,
node: status.node,
lane: status.lane,
sourceCommit: status.sourceCommit,
degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null,
pipelineRun: {
name: pipelineRun.name ?? null,
exists: pipelineRun.exists === true,
status: pipelineRun.status ?? null,
reason: pipelineRun.reason ?? null,
message: pipelineRun.message ?? null,
createdAt: pipelineRun.createdAt ?? null,
ready: pipelineRun.status === "True",
diagnostics: Object.keys(pipelineRunDiagnostics).length === 0 ? null : {
degradedReason: pipelineRunDiagnostics.degradedReason ?? null,
taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null,
podCount: pipelineRunDiagnostics.podCount ?? null,
failedTaskRunCount: pipelineRunDiagnostics.failedTaskRunCount ?? null,
failedTaskRuns: pipelineRunDiagnostics.failedTaskRuns ?? [],
failureSummary: pipelineRunDiagnostics.failureSummary ?? null,
pendingTaskRuns: pipelineRunDiagnostics.pendingTaskRuns ?? [],
unscheduledPods: pipelineRunDiagnostics.unscheduledPods ?? [],
schedulingMessages: pipelineRunDiagnostics.schedulingMessages ?? [],
next: pipelineRunDiagnostics.next ?? null,
},
},
argo: {
application: argo.application ?? null,
ready: argo.ready === true,
syncRevision: argo.syncRevision ?? null,
syncStatus: argo.syncStatus ?? null,
health: argo.health ?? null,
},
runtime: {
namespace: runtime.namespace ?? null,
namespaceExists: runtime.namespaceExists === true,
ready: runtime.ready === true,
workloadCount,
workloadReady: `${readyWorkloads}/${workloadReadiness.length}`,
notReadyWorkloads,
localPostgresAbsent: runtime.localPostgresAbsent === true,
localPostgresExpectedAbsent: runtime.localPostgresExpectedAbsent === true,
localPostgresReady: runtime.localPostgresReady === true,
externalPostgresReady: runtime.externalPostgresBridge === undefined && runtime.externalPostgresSecrets === undefined
? null
: record(runtime.externalPostgresBridge).ready === true && record(runtime.externalPostgresSecrets).ready === true,
codeAgentRuntimeReady: record(runtime.codeAgentRuntime).required === true ? record(runtime.codeAgentRuntime).ready === true : null,
codeAgentRuntimeReason: typeof record(runtime.codeAgentRuntime).degradedReason === "string" ? record(runtime.codeAgentRuntime).degradedReason : null,
},
publicProbe: {
ready: publicProbes.ready === true,
web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null },
apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null },
targetHost: Object.keys(targetHostProbe).length === 0 ? null : {
node: targetHostProbe.node ?? status.node ?? null,
ready: targetHostProbe.ready === true,
probeAvailable: targetHostProbe.probeAvailable === true,
web: {
url: targetHostWebProbe.url ?? null,
ok: targetHostWebProbe.ok === true,
httpStatus: targetHostWebProbe.httpStatus ?? null,
exitCode: targetHostWebProbe.exitCode ?? null,
error: targetHostWebProbe.error ?? null,
},
apiHealth: {
url: targetHostApiProbe.url ?? null,
ok: targetHostApiProbe.ok === true,
httpStatus: targetHostApiProbe.httpStatus ?? null,
exitCode: targetHostApiProbe.exitCode ?? null,
error: targetHostApiProbe.error ?? null,
},
},
diagnostic: Object.keys(publicProbeDiagnostic).length === 0 ? null : publicProbeDiagnostic,
},
gitMirror: {
ready: gitMirror.ready === true,
localSource: gitMirrorCompact.localSource ?? null,
githubSource: gitMirrorCompact.githubSource ?? null,
localGitops: gitMirrorCompact.localGitops ?? null,
githubGitops: gitMirrorCompact.githubGitops ?? null,
pendingFlush: gitMirrorCompact.pendingFlush === true,
flushNeeded: gitMirrorCompact.flushNeeded === true,
githubInSync: gitMirrorCompact.githubInSync === true,
},
nextAction: nodeRuntimeStatusNextAction(status, scoped),
next: {
full: `${nodeRuntimeStatusCommand(scoped)} --full`,
plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
webProbe: `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}`,
},
};
}
export function nodeRuntimeStatusNextAction(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
const reason = typeof status.degradedReason === "string" ? status.degradedReason : null;
if (reason === null) return `${nodeRuntimeStatusCommand(scoped)} --full`;
if (reason === "control-plane-not-ready" || reason === "runtime-namespace-missing") {
return `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`;
}
if (reason === "runtime-not-ready") return `${nodeRuntimeStatusCommand(scoped)} --full`;
if (reason === "argo-not-synced-healthy") {
return `bun scripts/cli.ts hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane} --confirm`;
}
if (reason === "pipelinerun-not-succeeded") {
return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`;
}
if (reason === "node-runtime-ci-step-publish-failed") {
return `bun scripts/cli.ts platform-infra sub2api status --target ${scoped.node}`;
}
if (reason === "node-runtime-ci-taskrun-failed") {
const next = record(record(status.pipelineRunDiagnostics).next);
const failedStepLogs = typeof next.failedStepLogs === "string" ? next.failedStepLogs : null;
return failedStepLogs ?? `${nodeRuntimeStatusCommand(scoped)} --full`;
}
if (reason === "node-runtime-ci-pod-capacity-exhausted" || reason === "node-runtime-ci-pod-unschedulable") {
return `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane} --min-age-minutes 60 --limit 20 --dry-run`;
}
if (reason === "public-probe-not-ready") {
return `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}`;
}
if (reason === "git-mirror-pending-flush") {
return `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`;
}
return `${nodeRuntimeStatusCommand(scoped)} --full`;
}
export function nodeRuntimeStatusCommand(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
const sourceCommit = optionValue(scoped.originalArgs, "--source-commit");
const pipelineRun = optionValue(scoped.originalArgs, "--pipeline-run");
return [
`bun scripts/cli.ts hwlab nodes control-plane status --node ${shellQuote(scoped.node)} --lane ${shellQuote(scoped.lane)}`,
sourceCommit === undefined ? "" : `--source-commit ${shellQuote(sourceCommit)}`,
pipelineRun === undefined ? "" : `--pipeline-run ${shellQuote(pipelineRun)}`,
].filter(Boolean).join(" ");
}
export function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
const taskRunTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineTask"}}{{"\\t"}}{{if .spec.taskRef}}{{.spec.taskRef.name}}{{end}}{{"\\t"}}{{with index .status.conditions 0}}{{.status}}{{"\\t"}}{{.reason}}{{else}}{{"\\t"}}{{end}}{{"\\t"}}{{.status.podName}}{{"\\t"}}{{with index .status.conditions 0}}{{printf "%.600s" .message}}{{end}}{{"\\n"}}{{end}}`;
const podTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/taskRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "PodScheduled"}}{{.status}}|{{.reason}}|{{printf "%.600s" .message}}{{end}}{{end}}{{"\\t"}}{{range .status.initContainerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\t"}}{{range .status.containerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\n"}}{{end}}`;
const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${taskRunTemplate}`], 60);
const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${podTemplate}`], 60);
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRunsFromTsv(taskRunsResult.stdout) : [];
const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPodsFromTsv(podsResult.stdout) : [];
const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False");
const failedTaskRuns = taskRuns.filter((item) => item.status === "False");
const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods);
const stepPublishFailures = failedTaskRunSummaries.filter((item) => item.container === "step-publish" || item.step === "publish" || item.step === "step-publish");
const unscheduledPods = pods.filter((item) => item.scheduled === false);
const schedulingMessages = unscheduledPods
.map((item) => typeof item.scheduledMessage === "string" ? item.scheduledMessage : "")
.filter((message) => message.length > 0);
const tooManyPods = schedulingMessages.some((message) => /too many pods/iu.test(message));
const failureSummary = failedTaskRunSummaries.length > 0
? {
failedTaskRunCount: failedTaskRuns.length,
failedStepCount: failedTaskRunSummaries.length,
stepPublishFailureCount: stepPublishFailures.length,
firstFailure: failedTaskRunSummaries[0] ?? null,
stepFailures: failedTaskRunSummaries.slice(0, 8),
nextAction: stepPublishFailures.length > 0
? "step-publish failed in a service build; first distinguish platform-infra Sub2API/proxy health from a single upstream transient, then choose controlled rerun or artifact-publish/envRecipe retry fix."
: failedTaskRunSummaries.length > 0
? "Inspect the failed TaskRun and bounded pod log command before rerunning the control-plane trigger."
: null,
}
: null;
return {
ok: taskRunsResult.exitCode === 0 && podsResult.exitCode === 0,
pipelineRun,
taskRuns,
pods,
taskRunCount: taskRuns.length,
podCount: pods.length,
failedTaskRunCount: failedTaskRuns.length,
failedTaskRuns: failedTaskRunSummaries,
stepPublishFailures,
failureSummary,
pendingTaskRuns,
unscheduledPods,
schedulingMessages,
degradedReason: tooManyPods
? "node-runtime-ci-pod-capacity-exhausted"
: unscheduledPods.length > 0
? "node-runtime-ci-pod-unschedulable"
: stepPublishFailures.length > 0
? "node-runtime-ci-step-publish-failed"
: failedTaskRunSummaries.length > 0
? "node-runtime-ci-taskrun-failed"
: pendingTaskRuns.length > 0
? "node-runtime-ci-taskrun-pending"
: undefined,
query: {
taskRuns: compactRuntimeCommand(taskRunsResult),
pods: compactRuntimeCommand(podsResult),
},
next: tooManyPods || unscheduledPods.length > 0
? { cleanupRuns: `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 60 --limit 20 --dry-run` }
: stepPublishFailures.length > 0
? {
sub2apiStatus: `bun scripts/cli.ts platform-infra sub2api status --target ${spec.nodeId}`,
sub2apiValidate: `bun scripts/cli.ts platform-infra sub2api validate --target ${spec.nodeId}`,
failedStepLogs: stepPublishFailures[0]?.logCommand ?? null,
rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm --rerun`,
}
: failedTaskRunSummaries.length > 0
? {
failedStepLogs: failedTaskRunSummaries[0]?.logCommand ?? null,
failedTaskRun: failedTaskRunSummaries[0]?.taskRunCommand ?? null,
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
}
: undefined,
};
}
export function nodeRuntimePipelineDiagnosticTaskRunsFromTsv(text: string): Array<Record<string, unknown>> {
return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => {
const parts = line.split("\t");
const [name = "", pipelineTask = "", taskRef = "", status = "", reason = "", podName = "", ...messageParts] = parts;
const message = messageParts.join("\t");
return {
name: stringOrNull(name),
pipelineTask: stringOrNull(pipelineTask),
taskRef: stringOrNull(taskRef),
status: stringOrNull(status),
reason: stringOrNull(reason),
message: diagnosticText(message),
podName: stringOrNull(podName),
steps: [],
failedSteps: [],
};
});
}
export function nodeRuntimePipelineDiagnosticPodsFromTsv(text: string): Array<Record<string, unknown>> {
return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => {
const [name = "", taskRun = "", phase = "", nodeName = "", scheduledRaw = "", initRaw = "", containersRaw = ""] = line.split("\t");
const [scheduledStatus = "", scheduledReason = "", scheduledMessage = ""] = scheduledRaw.split("|");
const initContainers = nodeRuntimePipelineContainerStatusesFromCompact(initRaw);
const containers = nodeRuntimePipelineContainerStatusesFromCompact(containersRaw);
const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true);
return {
name: stringOrNull(name),
taskRun: stringOrNull(taskRun),
phase: stringOrNull(phase),
nodeName: stringOrNull(nodeName),
scheduled: scheduledStatus.length === 0 ? null : scheduledStatus === "True",
scheduledReason: stringOrNull(scheduledReason),
scheduledMessage: diagnosticText(scheduledMessage),
initContainers,
containers,
failedContainers,
};
});
}
export function nodeRuntimePipelineContainerStatusesFromCompact(text: string): Array<Record<string, unknown>> {
return text.split(",").map((item) => item.trim()).filter(Boolean).map((item) => {
const [name = "", stateOrExit = "", reason = ""] = item.split(":");
const exitCode = /^[0-9]+$/u.test(stateOrExit) ? Number(stateOrExit) : null;
const state = exitCode !== null ? "terminated" : stateOrExit === "waiting" ? "waiting" : stateOrExit === "running" ? "running" : null;
return {
name: stringOrNull(name),
ready: null,
restartCount: null,
state,
failed: exitCode !== null && exitCode !== 0,
exitCode,
reason: stringOrNull(reason),
message: null,
startedAt: null,
finishedAt: null,
};
});
}
export function nodeRuntimePipelineDiagnosticTaskRuns(json: Record<string, unknown>): Array<Record<string, unknown>> {
const items = Array.isArray(json.items) ? json.items.map(record) : [];
return items.map((item) => {
const metadata = record(item.metadata);
const labels = record(metadata.labels);
const spec = record(item.spec);
const taskRef = record(spec.taskRef);
const status = record(item.status);
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
const condition = conditions[0] ?? {};
const steps = nodeRuntimePipelineDiagnosticSteps(status.steps);
const failedSteps = steps.filter((step) => step.failed === true);
return {
name: metadata.name ?? null,
pipelineTask: labels["tekton.dev/pipelineTask"] ?? null,
taskRef: taskRef.name ?? null,
status: condition.status ?? null,
reason: condition.reason ?? null,
message: diagnosticText(condition.message),
podName: status.podName ?? null,
steps,
failedSteps,
};
});
}
export function nodeRuntimePipelineDiagnosticPods(json: Record<string, unknown>): Array<Record<string, unknown>> {
const items = Array.isArray(json.items) ? json.items.map(record) : [];
return items.map((item) => {
const metadata = record(item.metadata);
const labels = record(metadata.labels);
const spec = record(item.spec);
const status = record(item.status);
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
const scheduled = conditions.find((condition) => condition.type === "PodScheduled");
const initContainers = nodeRuntimeContainerStatusSummaries(status.initContainerStatuses);
const containers = nodeRuntimeContainerStatusSummaries(status.containerStatuses);
const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true);
return {
name: metadata.name ?? null,
taskRun: labels["tekton.dev/taskRun"] ?? null,
phase: status.phase ?? null,
nodeName: spec.nodeName ?? null,
scheduled: scheduled === undefined ? null : scheduled.status === "True",
scheduledReason: scheduled?.reason ?? null,
scheduledMessage: diagnosticText(scheduled?.message),
initContainers,
containers,
failedContainers,
};
});
}
export function nodeRuntimePipelineDiagnosticSteps(value: unknown): Array<Record<string, unknown>> {
const steps = Array.isArray(value) ? value.map(record) : [];
return steps.map((step) => {
const terminated = record(step.terminated);
const running = record(step.running);
const waiting = record(step.waiting);
const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
const state = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
return {
name: step.name ?? null,
container: step.container ?? null,
state,
failed: exitCode !== null && exitCode !== 0,
exitCode,
reason: terminatedReason ?? waitingReason,
message: diagnosticText(terminated.message ?? waiting.message),
startedAt: terminated.startedAt ?? running.startedAt ?? null,
finishedAt: terminated.finishedAt ?? null,
};
});
}
export function nodeRuntimeContainerStatusSummaries(value: unknown): Array<Record<string, unknown>> {
const containers = Array.isArray(value) ? value.map(record) : [];
return containers.map((container) => {
const state = record(container.state);
const terminated = record(state.terminated);
const waiting = record(state.waiting);
const running = record(state.running);
const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
const stateName = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
return {
name: container.name ?? null,
ready: container.ready === true,
restartCount: typeof container.restartCount === "number" ? container.restartCount : null,
state: stateName,
failed: exitCode !== null && exitCode !== 0,
exitCode,
reason: terminatedReason ?? waitingReason,
message: diagnosticText(terminated.message ?? waiting.message),
startedAt: terminated.startedAt ?? running.startedAt ?? null,
finishedAt: terminated.finishedAt ?? null,
};
});
}
export function nodeRuntimePipelineFailedTaskRunSummaries(
spec: HwlabRuntimeLaneSpec,
failedTaskRuns: Array<Record<string, unknown>>,
pods: Array<Record<string, unknown>>,
): Array<Record<string, unknown>> {
const summaries: Array<Record<string, unknown>> = [];
for (const taskRun of failedTaskRuns) {
const taskRunName = stringOrNull(taskRun.name);
const podName = stringOrNull(taskRun.podName);
const pod = pods.find((item) => item.name === podName || (taskRunName !== null && item.taskRun === taskRunName)) ?? {};
const failedSteps = Array.isArray(taskRun.failedSteps) ? taskRun.failedSteps.map(record) : [];
const failedContainers = Array.isArray(pod.failedContainers) ? pod.failedContainers.map(record) : [];
const failures = failedSteps.length > 0
? failedSteps
: failedContainers.map((container) => ({
name: typeof container.name === "string" && container.name.startsWith("step-") ? container.name.slice("step-".length) : container.name ?? null,
container: container.name ?? null,
state: container.state ?? null,
exitCode: container.exitCode ?? null,
reason: container.reason ?? null,
message: container.message ?? null,
}));
const effectiveFailures = failures.length > 0 ? failures : [{ name: null, container: null, state: null, exitCode: null, reason: taskRun.reason ?? null, message: taskRun.message ?? null }];
for (const failure of effectiveFailures) {
const stepName = stringOrNull(failure.name);
const containerName = stringOrNull(failure.container) ?? (stepName === null ? null : `step-${stepName}`);
summaries.push({
taskRun: taskRunName,
pipelineTask: taskRun.pipelineTask ?? null,
taskRef: taskRun.taskRef ?? null,
taskRunStatus: taskRun.status ?? null,
taskRunReason: taskRun.reason ?? null,
taskRunMessage: diagnosticText(taskRun.message),
pod: podName,
podPhase: pod.phase ?? null,
nodeName: pod.nodeName ?? null,
step: stepName,
container: containerName,
containerState: failure.state ?? null,
terminationReason: failure.reason ?? null,
exitCode: typeof failure.exitCode === "number" ? failure.exitCode : null,
message: diagnosticText(failure.message),
logCommand: podName === null ? null : nodeRuntimePipelineLogsCommand(spec, podName, containerName),
taskRunCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["get", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName, "-o", "yaml"]),
taskRunDescribeCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName]),
podDescribeCommand: podName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "pod", "-n", HWLAB_CI_NAMESPACE, podName]),
});
}
}
return summaries.slice(0, 16);
}
export function nodeRuntimePipelineFailureSummary(value: unknown): Record<string, unknown> | null {
const recordValue = record(value);
const direct = record(recordValue.failureSummary);
if (Object.keys(direct).length > 0) return direct;
const diagnostics = record(recordValue.diagnostics);
const fromDiagnostics = record(diagnostics.failureSummary);
return Object.keys(fromDiagnostics).length > 0 ? fromDiagnostics : null;
}
export function nodeRuntimePipelineLogsCommand(spec: HwlabRuntimeLaneSpec, podName: string, containerName: string | null): string {
return nodeRuntimeK3sCommand(spec, [
"logs",
"--namespace", HWLAB_CI_NAMESPACE,
"--pod", podName,
...(containerName === null ? ["--all-containers"] : ["--container", containerName]),
"--tail", "200",
]);
}
export function nodeRuntimeK3sCommand(spec: HwlabRuntimeLaneSpec, args: string[]): string {
return ["trans", spec.nodeKubeRoute, ...args].map(shellQuote).join(" ");
}
export function stringOrNull(value: unknown): string | null {
return typeof value === "string" && value.length > 0 ? value : null;
}
export function diagnosticText(value: unknown): string | null {
if (typeof value !== "string") return null;
const trimmed = value.trim();
if (trimmed.length === 0) return null;
return trimmed
.replace(/postgres(?:ql)?:\/\/[^\s"'`]+/giu, "postgres://<redacted>")
.replace(/\b([A-Za-z0-9_.-]*(?:TOKEN|PASSWORD|SECRET|API_KEY|DATABASE_URL)[A-Za-z0-9_.-]*)=([^\s"'`]+)/giu, "$1=<redacted>")
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gu, "Bearer <redacted>")
.slice(0, 600);
}
export function nodeRuntimeRenderToken(): string {
return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-");
}
export function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds);
return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds);
}
export function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean {
return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId;
}
export function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] {
const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds));
const retryCount = Math.max(0, Math.floor(retries));
const maxAttempts = retryCount + 1;
const safeContext = context.replace(/[^A-Za-z0-9_.-]/gu, "-");
return [
`yaml_registry=${shellQuote(registry)}`,
`yaml_fetch_timeout=${shellQuote(String(timeoutSeconds))}`,
`yaml_fetch_retries=${shellQuote(String(retryCount))}`,
`yaml_max_attempts=${shellQuote(String(maxAttempts))}`,
`yaml_dependency_context=${shellQuote(safeContext)}`,
"yaml_dependency_log() { echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"'\"$1\"'\",\"manager\":\"'\"${2:-}\"'\"}' >&2; }",
"yaml_prepare_node_package_proxy() {",
" yaml_http_proxy=\"${HTTP_PROXY:-${http_proxy:-${ALL_PROXY:-${all_proxy:-}}}}\"",
" yaml_https_proxy=\"${HTTPS_PROXY:-${https_proxy:-${yaml_http_proxy}}}\"",
" yaml_all_proxy=\"${ALL_PROXY:-${all_proxy:-${yaml_http_proxy}}}\"",
" if [ -n \"$yaml_http_proxy\" ]; then export HTTP_PROXY=\"$yaml_http_proxy\" http_proxy=\"$yaml_http_proxy\" npm_config_proxy=\"$yaml_http_proxy\"; fi",
" if [ -n \"$yaml_https_proxy\" ]; then export HTTPS_PROXY=\"$yaml_https_proxy\" https_proxy=\"$yaml_https_proxy\" npm_config_https_proxy=\"$yaml_https_proxy\"; fi",
" if [ -n \"$yaml_all_proxy\" ]; then export ALL_PROXY=\"$yaml_all_proxy\" all_proxy=\"$yaml_all_proxy\"; fi",
" export npm_config_registry=\"$yaml_registry\"",
" export BUN_CONFIG_REGISTRY=\"$yaml_registry\"",
" export npm_config_noproxy=\"${NO_PROXY:-${no_proxy:-}}\"",
" export npm_config_fetch_retries=\"$yaml_fetch_retries\"",
" export npm_config_fetch_retry_mintimeout=2000",
" export npm_config_fetch_retry_maxtimeout=16000",
" export npm_config_fetch_timeout=$((yaml_fetch_timeout * 1000))",
"}",
"yaml_run_dependency_manager() {",
" yaml_manager=\"$1\"; shift",
" yaml_attempt=1",
" yaml_delay=2",
" while [ \"$yaml_attempt\" -le \"$yaml_max_attempts\" ]; do",
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"attempt\",\"manager\":\"'\"$yaml_manager\"'\",\"attempt\":\"'\"$yaml_attempt/$yaml_max_attempts\"'\",\"proxy\":\"'\"${yaml_https_proxy:-$yaml_http_proxy}\"'\"}' >&2",
" if timeout \"$yaml_fetch_timeout\" \"$@\"; then return 0; fi",
" if [ \"$yaml_attempt\" -ge \"$yaml_max_attempts\" ]; then return 1; fi",
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"retrying\",\"manager\":\"'\"$yaml_manager\"'\",\"attempt\":\"'\"$yaml_attempt/$yaml_max_attempts\"'\",\"sleepSeconds\":'\"$yaml_delay\"'}' >&2",
" sleep \"$yaml_delay\"",
" yaml_delay=$((yaml_delay * 2))",
" yaml_attempt=$((yaml_attempt + 1))",
" done",
" return 1",
"}",
"yaml_npm_debug_log_tail() {",
" yaml_npm_log_dir=\"${HOME:-/tmp}/.npm/_logs\"",
" if [ ! -d \"$yaml_npm_log_dir\" ]; then return 0; fi",
" yaml_npm_log=\"$(find \"$yaml_npm_log_dir\" -type f -name '*debug*.log' | sort | tail -n 1 || true)\"",
" if [ -n \"$yaml_npm_log\" ] && [ -f \"$yaml_npm_log\" ]; then",
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"npm-debug-log\",\"path\":\"'\"$yaml_npm_log\"'\"}' >&2",
" tail -n 80 \"$yaml_npm_log\" >&2 || true",
" fi",
"}",
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
" yaml_prepare_node_package_proxy",
"fi",
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 && command -v bun >/dev/null 2>&1; then",
" rm -rf node_modules/yaml",
" if yaml_run_dependency_manager bun bun add --no-save --ignore-scripts --registry \"$yaml_registry\" yaml@2.8.3; then",
" yaml_dependency_log installed bun",
" else",
" yaml_dependency_log bun-failed bun",
" fi",
"fi",
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
" rm -rf node_modules/yaml",
" command -v npm >/dev/null 2>&1 || { yaml_dependency_log failed missing-tool; exit 31; }",
" if yaml_run_dependency_manager npm npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev --registry \"$yaml_registry\" yaml@2.8.3; then",
" yaml_dependency_log installed npm",
" else",
" yaml_dependency_log npm-failed npm",
" yaml_npm_debug_log_tail",
" exit 34",
" fi",
"fi",
"node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 || { yaml_dependency_log failed unresolved; exit 34; }",
];
}
export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
const token = nodeRuntimeRenderToken();
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
const script = [
"set -eu",
runtimeLaneCicdRepoEnsureScript(spec),
`source_commit=${shellQuote(sourceCommit)}`,
`render_dir=${shellQuote(renderDir)}`,
`worktree_dir=${shellQuote(worktreeDir)}`,
`overlay_b64=${shellQuote(overlay)}`,
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
"trap cleanup_render_worktree EXIT",
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = "$source_commit"`,
"rm -rf \"$render_dir\" \"$worktree_dir\"",
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
"cd \"$worktree_dir\"",
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "control-plane-render"),
"node - \"$overlay_b64\" <<'NODE'",
"const fs = require('fs');",
"const YAML = require('yaml');",
"const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
"const path = 'deploy/deploy.yaml';",
"const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
"doc.nodes = doc.nodes || {};",
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
"doc.lanes = doc.lanes || {};",
"const lane = doc.lanes[overlay.lane] || {};",
"const downloadStack = {",
" ...(lane.envRecipe?.downloadStack || {}),",
" httpProxy: overlay.dockerProxyHttp,",
" httpsProxy: overlay.dockerProxyHttps,",
" noProxy: overlay.dockerNoProxyList,",
"};",
"doc.lanes[overlay.lane] = {",
" ...lane,",
" node: overlay.nodeId,",
" sourceBranch: overlay.sourceBranch,",
" gitopsBranch: overlay.gitopsBranch,",
" namespace: overlay.runtimeNamespace,",
" endpoint: overlay.publicApiUrl,",
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
" artifactCatalog: overlay.catalogPath,",
" runtimePath: overlay.runtimePath,",
" imageTagMode: 'full',",
" sourceRepo: overlay.gitUrl,",
" externalPostgres: overlay.externalPostgres,",
" observability: overlay.observability,",
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
"};",
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
"fs.writeFileSync(path, YAML.stringify(doc));",
"NODE",
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
[
"node scripts/run-bun.mjs \"$render_script\"",
`--lane ${shellQuote(spec.lane)}`,
`--node ${shellQuote(spec.nodeId)}`,
`--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
`--catalog-path ${shellQuote(spec.catalogPath)}`,
"--image-tag-mode full",
`--source-revision ${shellQuote(sourceCommit)}`,
`--source-repo ${shellQuote(spec.gitUrl)}`,
`--source-branch ${shellQuote(spec.sourceBranch)}`,
`--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
`--git-read-url ${shellQuote(spec.gitReadUrl)}`,
`--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
`--registry-prefix ${shellQuote(spec.registryPrefix)}`,
`--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
`--out ${shellQuote(renderDir)}`,
].join(" "),
...nodeRuntimePipelinePostprocessScript(),
].join("\n");
return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" };
}
export function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
const token = nodeRuntimeRenderToken();
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
const script = [
"set -eu",
`source_url=${shellQuote(spec.gitUrl)}`,
`source_branch=${shellQuote(spec.sourceBranch)}`,
`source_commit=${shellQuote(sourceCommit)}`,
`render_dir=${shellQuote(renderDir)}`,
`worktree_dir=${shellQuote(worktreeDir)}`,
`overlay_b64=${shellQuote(overlay)}`,
`git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
"run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
"rm -rf \"$render_dir\" \"$worktree_dir\"",
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
"echo \"phase=local-git-clone-worktree\" >&2",
"run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"",
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
"cd \"$worktree_dir\"",
"echo \"phase=local-install-yaml\" >&2",
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"),
"node - \"$overlay_b64\" <<'NODE'",
"const fs = require('fs');",
"const YAML = require('yaml');",
"const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
"const path = 'deploy/deploy.yaml';",
"const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
"doc.nodes = doc.nodes || {};",
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
"doc.lanes = doc.lanes || {};",
"const lane = doc.lanes[overlay.lane] || {};",
"const downloadStack = {",
" ...(lane.envRecipe?.downloadStack || {}),",
" httpProxy: overlay.dockerProxyHttp,",
" httpsProxy: overlay.dockerProxyHttps,",
" noProxy: overlay.dockerNoProxyList,",
"};",
"doc.lanes[overlay.lane] = {",
" ...lane,",
" node: overlay.nodeId,",
" sourceBranch: overlay.sourceBranch,",
" gitopsBranch: overlay.gitopsBranch,",
" namespace: overlay.runtimeNamespace,",
" endpoint: overlay.publicApiUrl,",
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
" artifactCatalog: overlay.catalogPath,",
" runtimePath: overlay.runtimePath,",
" imageTagMode: 'full',",
" sourceRepo: overlay.gitUrl,",
" externalPostgres: overlay.externalPostgres,",
" observability: overlay.observability,",
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
"};",
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
"if (overlay.gitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.gitMirror;",
"fs.writeFileSync(path, YAML.stringify(doc));",
"NODE",
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
"echo \"phase=local-gitops-render\" >&2",
[
"node scripts/run-bun.mjs \"$render_script\"",
`--lane ${shellQuote(spec.lane)}`,
`--node ${shellQuote(spec.nodeId)}`,
`--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
`--catalog-path ${shellQuote(spec.catalogPath)}`,
"--image-tag-mode full",
`--source-revision ${shellQuote(sourceCommit)}`,
`--source-repo ${shellQuote(spec.gitUrl)}`,
`--source-branch ${shellQuote(spec.sourceBranch)}`,
`--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
`--git-read-url ${shellQuote(spec.gitReadUrl)}`,
`--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
`--registry-prefix ${shellQuote(spec.registryPrefix)}`,
`--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
`--out ${shellQuote(renderDir)}`,
].join(" "),
...nodeRuntimePipelinePostprocessScript(),
].join("\n");
return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" };
}
export function nodeRuntimePipelinePostprocessScript(): string[] {
return [
"node - \"$render_dir\" \"$overlay_b64\" <<'NODE'",
"const fs = require('fs');",
"const path = require('path');",
"const vm = require('node:vm');",
"const renderDir = process.argv[2];",
"const overlay = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));",
"const pipelinePath = path.join(renderDir, overlay.tektonDir, 'pipeline.yaml');",
"let text = fs.readFileSync(pipelinePath, 'utf8');",
"let YAML = null;",
"try { YAML = require('yaml'); } catch {}",
"const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');",
"const shellSingle = (value) => `'${String(value).replaceAll(\"'\", `'\"'\"'`)}'`;",
"const yamlString = (value) => JSON.stringify(String(value));",
"const proxyEnv = {",
" HTTP_PROXY: overlay.proxyHttp,",
" HTTPS_PROXY: overlay.proxyHttps,",
" ALL_PROXY: overlay.proxyAll,",
" NO_PROXY: overlay.noProxy,",
" http_proxy: overlay.proxyHttp,",
" https_proxy: overlay.proxyHttps,",
" all_proxy: overlay.proxyAll,",
" no_proxy: overlay.noProxy,",
"};",
"const dockerProxyEnv = {",
" HWLAB_NODE_PROXY_URL: overlay.dockerProxyHttp,",
" HWLAB_NODE_ALL_PROXY_URL: overlay.dockerProxyAll,",
" HWLAB_NODE_NO_PROXY: overlay.dockerNoProxy,",
"};",
"const stepEnv = { ...proxyEnv, ...dockerProxyEnv, ...(overlay.stepEnv || {}) };",
"function prepareSourceDependencyScript() {",
" return `prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"",
"echo '{\"event\":\"prepare-source-dependencies\",\"status\":\"not-required\",\"dependency\":\"yaml\",\"manager\":\"inline-service-id-parser\"}' >&2",
"ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"`;",
"}",
"function validatePrepareSourceDependencyScript(script) {",
" const marker = 'NODE_UNIDESK_YAML_DEPENDENCY';",
" let offset = 0;",
" while (true) {",
" const markerStart = script.indexOf(\"node <<'\" + marker + \"'\", offset);",
" if (markerStart === -1) return;",
" const bodyStart = script.indexOf('\\n', markerStart);",
" if (bodyStart === -1) throw new Error('prepare-source dependency heredoc body missing newline');",
" const bodyEnd = script.indexOf('\\n' + marker, bodyStart + 1);",
" if (bodyEnd === -1) throw new Error('prepare-source dependency heredoc terminator missing');",
" const body = script.slice(bodyStart + 1, bodyEnd);",
" try { new vm.Script(body, { filename: 'NODE_UNIDESK_YAML_DEPENDENCY.js' }); } catch (error) { throw new Error(`generated prepare-source yaml dependency script is invalid: ${error.message}`); }",
" offset = bodyEnd + marker.length + 1;",
" }",
"}",
"function deployYamlOverlayScript() {",
" const runtimeOverlay = JSON.stringify({",
" nodeId: overlay.nodeId,",
" lane: overlay.lane,",
" sourceBranch: overlay.sourceBranch,",
" gitopsBranch: overlay.gitopsBranch,",
" gitopsRoot: overlay.gitopsRoot,",
" runtimePath: overlay.runtimePath,",
" runtimeRenderDir: overlay.runtimeRenderDir,",
" runtimeNamespace: overlay.runtimeNamespace,",
" catalogPath: overlay.catalogPath,",
" gitUrl: overlay.gitUrl,",
" publicWebUrl: overlay.publicWebUrl,",
" publicApiUrl: overlay.publicApiUrl,",
" externalPostgres: overlay.externalPostgres,",
" runtimeStore: overlay.runtimeStore,",
" codeAgentRuntime: overlay.codeAgentRuntime,",
" observability: overlay.observability,",
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
" dockerProxyHttp: overlay.dockerProxyHttp,",
" dockerProxyHttps: overlay.dockerProxyHttps,",
" dockerNoProxyList: overlay.dockerNoProxyList,",
" npmRegistry: overlay.npmRegistry,",
" npmFetchTimeoutMs: overlay.npmFetchTimeoutMs,",
" npmRetries: overlay.npmRetries,",
" });",
" return `node - <<'NODE_UNIDESK_DEPLOY_YAML_OVERLAY'",
"const fs = require('fs');",
"const YAML = require('yaml');",
"const overlay = ${runtimeOverlay};",
"const file = 'deploy/deploy.yaml';",
"if (!fs.existsSync(file)) {",
" console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: false, reason: 'deploy-yaml-missing', file }));",
" process.exit(45);",
"}",
"const doc = YAML.parse(fs.readFileSync(file, 'utf8'));",
"doc.nodes = doc.nodes || {};",
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
"doc.lanes = doc.lanes || {};",
"const lane = doc.lanes[overlay.lane] || {};",
"const envRecipe = lane.envRecipe || {};",
"const downloadStack = {",
" ...(envRecipe.downloadStack || {}),",
" httpProxy: overlay.dockerProxyHttp,",
" httpsProxy: overlay.dockerProxyHttps,",
" noProxy: overlay.dockerNoProxyList,",
"};",
"if (overlay.npmRegistry) downloadStack.npmRegistry = overlay.npmRegistry;",
"if (overlay.npmFetchTimeoutMs) downloadStack.npmFetchTimeoutMs = overlay.npmFetchTimeoutMs;",
"doc.lanes[overlay.lane] = {",
" ...lane,",
" node: overlay.nodeId,",
" sourceBranch: overlay.sourceBranch,",
" gitopsBranch: overlay.gitopsBranch,",
" namespace: overlay.runtimeNamespace,",
" endpoint: overlay.publicApiUrl,",
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
" artifactCatalog: overlay.catalogPath,",
" runtimePath: overlay.runtimePath,",
" imageTagMode: 'full',",
" sourceRepo: overlay.gitUrl,",
" externalPostgres: overlay.externalPostgres,",
" observability: overlay.observability,",
" envRecipe: { ...envRecipe, downloadStack },",
"};",
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
"fs.writeFileSync(file, YAML.stringify(doc));",
"console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));",
"NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;",
"}",
"function runtimePathOverlayScript() {",
" const sourcePath = `${String(overlay.gitopsRoot || '').replace(/\\/+$/u, '')}/${overlay.runtimeRenderDir}`;",
" const targetPath = String(overlay.runtimePath || '');",
" if (!targetPath || sourcePath === targetPath) return '';",
" return [",
" `if [ ! -d ${shellSingle(targetPath)} ]; then`,",
" ` if [ ! -d ${shellSingle(sourcePath)} ]; then echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: false, reason: 'source-runtime-path-missing' }))} >&2; exit 46; fi`,",
" ` mkdir -p \"$(dirname ${shellSingle(targetPath)})\"`,",
" ` cp -a ${shellSingle(sourcePath)} ${shellSingle(targetPath)}` ,",
" ` echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: true }))} >&2`,",
" `fi`,",
" ].join('\\n');",
"}",
"function stepEnvBootstrapScript() {",
" const entries = Object.entries(overlay.stepEnv || {}).filter(([, value]) => value !== undefined && value !== null && String(value).length > 0);",
" const lines = ['# unidesk-step-env-bootstrap'];",
" for (const [name, value] of entries) lines.push(`export ${name}=${shellSingle(value)}`);",
" if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'HOME')) lines.push('mkdir -p \"$HOME\"');",
" if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'XDG_CONFIG_HOME')) lines.push('mkdir -p \"$XDG_CONFIG_HOME\"');",
" lines.push('ci_node_deps=\"${HWLAB_CI_NODE_DEPS:-/opt/hwlab-ci-node-deps/node_modules}\"');",
" lines.push('if [ -d \"$ci_node_deps\" ]; then');",
" lines.push(' if [ -d /workspace/source/repo ]; then ci_node_deps_target=/workspace/source/repo/node_modules; else ci_node_deps_target=./node_modules; fi');",
" lines.push(' mkdir -p \"$ci_node_deps_target\"');",
" lines.push(' for ci_node_dep in yaml; do if [ -d \"$ci_node_deps/$ci_node_dep\" ] && [ ! -e \"$ci_node_deps_target/$ci_node_dep\" ]; then ln -s \"$ci_node_deps/$ci_node_dep\" \"$ci_node_deps_target/$ci_node_dep\"; fi; done');",
" lines.push('fi');",
" return lines.join('\\n');",
"}",
"function runtimeGitopsPostprocessScript() {",
" const runtimeOverlay = JSON.stringify({",
" gitopsRoot: overlay.gitopsRoot,",
" runtimePath: overlay.runtimePath,",
" runtimeRenderDir: overlay.runtimeRenderDir,",
" runtimeNamespace: overlay.runtimeNamespace,",
" externalPostgres: overlay.externalPostgres,",
" codeAgentRuntime: overlay.codeAgentRuntime,",
" publicExposure: overlay.publicExposure,",
" observability: overlay.observability,",
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
" gitReadUrl: overlay.gitReadUrl,",
" publicWebUrl: overlay.publicWebUrl,",
" publicApiUrl: overlay.publicApiUrl,",
" });",
" return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS'",
"const fs = require('fs');",
"const path = require('path');",
"const crypto = require('crypto');",
"const YAML = require('yaml');",
"const overlay = ${runtimeOverlay};",
"const runtimePath = String(overlay.runtimePath || '');",
"const renderDir = String(overlay.runtimeRenderDir || '');",
"const legacyRuntimePath = runtimePath ? path.posix.join(path.posix.dirname(path.posix.dirname(runtimePath)), path.posix.basename(runtimePath)) : '';",
"const candidates = [...new Set([",
" runtimePath,",
" renderDir,",
" overlay.gitopsRoot && renderDir ? path.posix.join(String(overlay.gitopsRoot), renderDir) : '',",
" legacyRuntimePath,",
"].filter(Boolean))];",
"const sourcePath = candidates.find((candidate) => fs.existsSync(candidate)) || runtimePath;",
"if (!runtimePath || !fs.existsSync(sourcePath)) {",
" console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: false, reason: 'runtime-path-missing', runtimePath, sourcePath }));",
" process.exit(47);",
"}",
"if (sourcePath !== runtimePath) {",
" fs.rmSync(runtimePath, { recursive: true, force: true });",
" fs.mkdirSync(path.dirname(runtimePath), { recursive: true });",
" fs.cpSync(sourcePath, runtimePath, { recursive: true });",
" fs.rmSync(sourcePath, { recursive: true, force: true });",
"}",
"for (const candidate of candidates) {",
" if (candidate !== runtimePath && candidate !== sourcePath && candidate.endsWith('/' + path.posix.basename(runtimePath))) fs.rmSync(candidate, { recursive: true, force: true });",
"}",
"function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }",
"function writeYaml(file, doc) { fs.writeFileSync(file, YAML.stringify(doc).trimEnd() + '\\\\n'); }",
"function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }",
"function normalizeList(items) { return { apiVersion: 'v1', kind: 'List', items }; }",
"function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }",
"function yamlFiles(dir) {",
" const files = [];",
" for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {",
" const file = path.join(dir, entry.name);",
" if (entry.isDirectory()) files.push(...yamlFiles(file));",
" else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);",
" }",
" return files;",
"}",
"function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
"function writeYamlDocuments(file, docs) { fs.writeFileSync(file, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\\\n---\\\\n') + '\\\\n'); }",
"function podSpecFor(item) {",
" if (!isObject(item) || !isObject(item.spec)) return null;",
" if (item.kind === 'Pod') return item.spec;",
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
" if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;",
" return null;",
"}",
"function templateMetadataFor(item) {",
" if (!isObject(item) || !isObject(item.spec)) return null;",
" if (item.kind === 'Pod') return item.metadata || null;",
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template ? item.spec.template.metadata : null;",
" if (item.kind === 'Job') return item.spec.template ? item.spec.template.metadata : null;",
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.metadata : null;",
" return null;",
"}",
"function stripMonitoringMetadata(metadata) {",
" if (!isObject(metadata)) return false;",
" let changed = false;",
" if (isObject(metadata.labels) && metadata.labels['hwlab.pikastech.local/monitoring'] !== undefined && metadata.labels['hwlab.pikastech.local/monitoring'] !== 'disabled') {",
" metadata.labels['hwlab.pikastech.local/monitoring'] = 'disabled';",
" changed = true;",
" }",
" if (isObject(metadata.annotations) && metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'] !== undefined) {",
" delete metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'];",
" changed = true;",
" }",
" return changed;",
"}",
"function containerHasVolumeMount(container, name) { return isObject(container) && Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === name); }",
"function removeMetricsSidecar(podSpec) {",
" if (!isObject(podSpec)) return false;",
" let changed = false;",
" if (Array.isArray(podSpec.containers)) {",
" const next = podSpec.containers.filter((container) => !(isObject(container) && (container.name === 'hwlab-metrics' || (Array.isArray(container.command) && container.command.includes('/metrics/metrics-sidecar.mjs')))));",
" if (next.length !== podSpec.containers.length) { podSpec.containers = next; changed = true; }",
" }",
" for (const group of ['containers', 'initContainers']) {",
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
" if (!isObject(container) || !Array.isArray(container.volumeMounts)) continue;",
" const nextMounts = container.volumeMounts.filter((mount) => !(mount && mount.name === 'hwlab-metrics-sidecar'));",
" if (nextMounts.length !== container.volumeMounts.length) { container.volumeMounts = nextMounts; changed = true; }",
" }",
" }",
" if (Array.isArray(podSpec.volumes)) {",
" const nextVolumes = podSpec.volumes.filter((volume) => !(volume && volume.name === 'hwlab-metrics-sidecar'));",
" if (nextVolumes.length !== podSpec.volumes.length) { podSpec.volumes = nextVolumes; changed = true; }",
" }",
" return changed;",
"}",
"function envValue(container, name) {",
" if (!isObject(container) || !Array.isArray(container.env)) return undefined;",
" const item = container.env.find((env) => env && env.name === name);",
" return item ? item.value : undefined;",
"}",
"function setEnvValue(container, name, value) {",
" if (!isObject(container) || typeof value !== 'string') return false;",
" container.env = Array.isArray(container.env) ? container.env : [];",
" let item = container.env.find((env) => env && env.name === name);",
" if (!item) { item = { name }; container.env.push(item); }",
" const changed = item.value !== value || item.valueFrom !== undefined;",
" item.value = value;",
" delete item.valueFrom;",
" return changed;",
"}",
"function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }",
"function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }",
"function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }",
"function startupProbeFrom(probe) {",
" const next = JSON.parse(JSON.stringify(probe));",
" next.periodSeconds = 10;",
" next.timeoutSeconds = Math.max(Number(next.timeoutSeconds || 0), 2);",
" next.failureThreshold = 30;",
" next.successThreshold = 1;",
" delete next.initialDelaySeconds;",
" return next;",
"}",
"function addEnvReuseStartupProbe(podSpec) {",
" if (!isObject(podSpec) || !Array.isArray(podSpec.containers)) return false;",
" let changed = false;",
" for (const container of podSpec.containers) {",
" if (!isObject(container) || !isEnvReuseContainer(container) || container.startupProbe) continue;",
" const sourceProbe = container.readinessProbe || container.livenessProbe;",
" if (!sourceProbe) continue;",
" container.startupProbe = startupProbeFrom(sourceProbe);",
" changed = true;",
" }",
" return changed;",
"}",
"function rewriteRuntimeImage(image) {",
" if (typeof image !== 'string') return image;",
" const match = (overlay.runtimeImageRewrites || []).find((item) => item && item.source === image);",
" return match ? match.target : image;",
"}",
"function patchRuntimeImages(podSpec) {",
" if (!isObject(podSpec)) return false;",
" let changed = false;",
" for (const group of ['containers', 'initContainers']) {",
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
" if (!isObject(container) || typeof container.image !== 'string') continue;",
" const nextImage = rewriteRuntimeImage(container.image);",
" if (nextImage !== container.image) { container.image = nextImage; changed = true; }",
" }",
" }",
" return changed;",
"}",
"function rewriteEnvValue(value) {",
" if (typeof value !== 'string') return value;",
" return value.replaceAll('http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git', overlay.gitReadUrl);",
"}",
"function patchGitReadUrlEnv(podSpec) {",
" if (!isObject(podSpec)) return false;",
" let changed = false;",
" for (const group of ['containers', 'initContainers']) {",
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
" if (!isObject(container) || !Array.isArray(container.env)) continue;",
" for (const env of container.env) {",
" if (!isObject(env) || typeof env.value !== 'string') continue;",
" const nextValue = rewriteEnvValue(env.value);",
" if (nextValue !== env.value) { env.value = nextValue; changed = true; }",
" }",
" }",
" }",
" return changed;",
"}",
"function patchRuntimeEnv(item, podSpec) {",
" if (!isObject(podSpec)) return { publicEndpointChanged: false, dbSslModeChanged: false, codeAgentRuntimeChanged: false };",
" let publicEndpointChanged = false;",
" let dbSslModeChanged = false;",
" let codeAgentRuntimeChanged = false;",
" const pg = overlay.externalPostgres;",
" const codeAgentRuntime = overlay.codeAgentRuntime;",
" for (const group of ['containers', 'initContainers']) {",
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
" if (!isObject(container)) continue;",
" if (envValue(container, 'HWLAB_PUBLIC_ENDPOINT') !== undefined) publicEndpointChanged = setEnvValue(container, 'HWLAB_PUBLIC_ENDPOINT', expectedPublicEndpoint(item)) || publicEndpointChanged;",
" if (pg && pg.sslmode && envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE') !== undefined) dbSslModeChanged = setEnvValue(container, 'HWLAB_CLOUD_DB_SSL_MODE', pg.sslmode) || dbSslModeChanged;",
" if (codeAgentRuntime && codeAgentRuntime.enabled && workloadName(item) === 'hwlab-cloud-api' && container.name === 'hwlab-cloud-api') {",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_ADAPTER', String(codeAgentRuntime.adapter)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'AGENTRUN_MGR_URL', String(codeAgentRuntime.managerUrl)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvFromSecret(container, 'AGENTRUN_API_KEY', String(codeAgentRuntime.apiKeySecretName), String(codeAgentRuntime.apiKeySecretKey)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE', String(codeAgentRuntime.runnerNamespace)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE', String(codeAgentRuntime.secretNamespace)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_REPO_URL', String(codeAgentRuntime.repoUrl)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID', String(codeAgentRuntime.providerId)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE', String(codeAgentRuntime.defaultProviderProfile)) || codeAgentRuntimeChanged;",
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR', String(codeAgentRuntime.codexStdioSupervisor)) || codeAgentRuntimeChanged;",
" }",
" }",
" }",
" return { publicEndpointChanged, dbSslModeChanged, codeAgentRuntimeChanged };",
"}",
"function patchRuntimeWorkloads() {",
" let observabilityChanged = false;",
" let startupProbeChanged = false;",
" let imageRewriteChanged = false;",
" let gitReadUrlChanged = false;",
" let publicEndpointChanged = false;",
" let dbSslModeChanged = false;",
" let codeAgentRuntimeChanged = false;",
" for (const file of yamlFiles(runtimePath)) {",
" if (path.basename(file) === 'kustomization.yaml') continue;",
" const docs = readYamlDocuments(file);",
" let changed = false;",
" for (const doc of docs) {",
" for (const item of listItems(doc).filter(Boolean)) {",
" if (!isObject(item)) continue;",
" if (overlay.observability && overlay.observability.prometheusOperator === false) {",
" const metadataChanged = stripMonitoringMetadata(item.metadata);",
" const templateChanged = stripMonitoringMetadata(templateMetadataFor(item));",
" const sidecarChanged = removeMetricsSidecar(podSpecFor(item));",
" const monitoringChanged = metadataChanged || templateChanged || sidecarChanged;",
" changed = monitoringChanged || changed;",
" observabilityChanged = observabilityChanged || monitoringChanged;",
" }",
" const probeChanged = addEnvReuseStartupProbe(podSpecFor(item));",
" changed = probeChanged || changed;",
" startupProbeChanged = startupProbeChanged || probeChanged;",
" const imageChanged = patchRuntimeImages(podSpecFor(item));",
" changed = imageChanged || changed;",
" imageRewriteChanged = imageRewriteChanged || imageChanged;",
" const gitUrlChanged = patchGitReadUrlEnv(podSpecFor(item));",
" changed = gitUrlChanged || changed;",
" gitReadUrlChanged = gitReadUrlChanged || gitUrlChanged;",
" const envChanged = patchRuntimeEnv(item, podSpecFor(item));",
" changed = envChanged.publicEndpointChanged || envChanged.dbSslModeChanged || envChanged.codeAgentRuntimeChanged || changed;",
" publicEndpointChanged = publicEndpointChanged || envChanged.publicEndpointChanged;",
" dbSslModeChanged = dbSslModeChanged || envChanged.dbSslModeChanged;",
" codeAgentRuntimeChanged = codeAgentRuntimeChanged || envChanged.codeAgentRuntimeChanged;",
" }",
" }",
" if (changed) writeYamlDocuments(file, docs);",
" }",
" return { observabilityChanged, startupProbeChanged, imageRewriteChanged, gitReadUrlChanged, publicEndpointChanged, dbSslModeChanged, codeAgentRuntimeChanged };",
"}",
"function patchKustomization() {",
" const file = path.join(runtimePath, 'kustomization.yaml');",
" if (!fs.existsSync(file)) return false;",
" const doc = readYaml(file) || {};",
" const resources = Array.isArray(doc.resources) ? doc.resources : [];",
" const next = resources.filter((item) => !(overlay.observability && overlay.observability.prometheusOperator === false && item === 'observability.yaml'));",
" let changed = false;",
" if (next.length !== resources.length) { doc.resources = next; writeYaml(file, doc); changed = true; }",
" const observabilityFile = path.join(runtimePath, 'observability.yaml');",
" if (overlay.observability && overlay.observability.prometheusOperator === false && fs.existsSync(observabilityFile)) { fs.rmSync(observabilityFile, { force: true }); changed = true; }",
" return changed;",
"}",
"function patchExternalPostgres() {",
" const pg = overlay.externalPostgres;",
" if (!pg || !pg.serviceName) return false;",
" const access = pg.runtimeAccess || { endpointAddress: pg.endpointAddress, port: pg.port };",
" const file = path.join(runtimePath, 'external-postgres.yaml');",
" if (!fs.existsSync(file)) return false;",
" const doc = readYaml(file);",
" const items = listItems(doc).filter(Boolean);",
" let changed = false;",
" const endpointIsIpv4 = /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/.test(String(access.endpointAddress));",
" const endpointSliceName = String(pg.serviceName) + '-host';",
" for (const item of items) {",
" if (!item || typeof item !== 'object') continue;",
" item.metadata = item.metadata || {};",
" item.metadata.namespace = overlay.runtimeNamespace;",
" item.metadata.labels = item.metadata.labels || {};",
" item.metadata.labels['app.kubernetes.io/name'] = pg.serviceName;",
" if (item.kind === 'Service') {",
" item.metadata.name = pg.serviceName;",
" item.spec = item.spec || {};",
" if (endpointIsIpv4) {",
" item.spec.type = 'ClusterIP';",
" delete item.spec.externalName;",
" } else {",
" item.spec.type = 'ExternalName';",
" item.spec.externalName = String(access.endpointAddress);",
" delete item.spec.clusterIP;",
" delete item.spec.clusterIPs;",
" delete item.spec.ipFamilies;",
" delete item.spec.ipFamilyPolicy;",
" delete item.spec.internalTrafficPolicy;",
" }",
" item.spec.ports = [{ name: 'postgres', port: access.port, targetPort: access.port, protocol: 'TCP' }];",
" delete item.spec.selector;",
" changed = true;",
" }",
" if (item.kind === 'EndpointSlice') {",
" if (!endpointIsIpv4) { item.__delete = true; changed = true; continue; }",
" item.metadata.name = endpointSliceName;",
" item.metadata.labels['kubernetes.io/service-name'] = pg.serviceName;",
" item.addressType = 'IPv4';",
" item.ports = [{ name: 'postgres', port: access.port, protocol: 'TCP' }];",
" item.endpoints = [{ addresses: [access.endpointAddress], conditions: { ready: true } }];",
" changed = true;",
" }",
" }",
" if (changed) writeYaml(file, normalizeList(items.filter((item) => !item.__delete)));",
" return changed;",
"}",
"function patchHealthContract() {",
" const file = path.join(runtimePath, 'health-contract.yaml');",
" if (!fs.existsSync(file)) return false;",
" const doc = readYaml(file);",
" const items = listItems(doc).filter(Boolean);",
" let changed = false;",
" const pg = overlay.externalPostgres;",
" for (const item of items) {",
" if (!item || item.kind !== 'ConfigMap') continue;",
" item.data = item.data || {};",
" if (item.data.endpoint !== overlay.publicWebUrl) { item.data.endpoint = overlay.publicWebUrl; changed = true; }",
" const cloudApi = 'GET /health/live through ' + overlay.publicApiUrl;",
" if (item.data['cloud-api'] !== cloudApi) { item.data['cloud-api'] = cloudApi; changed = true; }",
" const cloudWeb = 'GET /health/live on ' + overlay.publicWebUrl + '; consumes cloud-api only';",
" if (item.data['cloud-web'] !== cloudWeb) { item.data['cloud-web'] = cloudWeb; changed = true; }",
" if (pg && pg.sslmode && typeof item.data['cloud-api-db'] === 'string') {",
" const next = item.data['cloud-api-db'].replace(/HWLAB_CLOUD_DB_SSL_MODE=[A-Za-z0-9_-]+/g, 'HWLAB_CLOUD_DB_SSL_MODE=' + pg.sslmode);",
" if (next !== item.data['cloud-api-db']) { item.data['cloud-api-db'] = next; changed = true; }",
" }",
" }",
" if (changed) writeYaml(file, normalizeList(items));",
" return changed;",
"}",
"function renderPublicExposureFrpcToml(exposure) {",
" return [",
" 'serverAddr = ' + JSON.stringify(String(exposure.serverAddr)),",
" 'serverPort = ' + Number(exposure.serverPort),",
" 'loginFailExit = true',",
" 'auth.token = \"{{ .Envs.HWLAB_FRP_TOKEN }}\"',",
" '',",
" '[[proxies]]',",
" 'name = ' + JSON.stringify(String(exposure.webProxy.name)),",
" 'type = \"tcp\"',",
" 'localIP = ' + JSON.stringify(String(exposure.webProxy.localIP)),",
" 'localPort = ' + Number(exposure.webProxy.localPort),",
" 'remotePort = ' + Number(exposure.webProxy.remotePort),",
" '',",
" '[[proxies]]',",
" 'name = ' + JSON.stringify(String(exposure.apiProxy.name)),",
" 'type = \"tcp\"',",
" 'localIP = ' + JSON.stringify(String(exposure.apiProxy.localIP)),",
" 'localPort = ' + Number(exposure.apiProxy.localPort),",
" 'remotePort = ' + Number(exposure.apiProxy.remotePort),",
" '',",
" ].join('\\\\n');",
"}",
"function setEnvFromSecret(container, name, secretName, secretKey) {",
" if (!isObject(container)) return false;",
" container.env = Array.isArray(container.env) ? container.env : [];",
" let item = container.env.find((env) => env && env.name === name);",
" if (!item) { item = { name }; container.env.push(item); }",
" const nextValueFrom = { secretKeyRef: { name: secretName, key: secretKey } };",
" const changed = item.value !== undefined || JSON.stringify(item.valueFrom || {}) !== JSON.stringify(nextValueFrom);",
" delete item.value;",
" item.valueFrom = nextValueFrom;",
" return changed;",
"}",
"function patchPublicExposure() {",
" const exposure = overlay.publicExposure;",
" if (!exposure || !exposure.enabled) return { configured: false, changed: false };",
" const file = path.join(runtimePath, 'node-frpc.yaml');",
" if (!fs.existsSync(file)) {",
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'node-frpc-yaml-missing', filePath: file, hostname: exposure.hostname }));",
" process.exit(49);",
" }",
" const docs = readYamlDocuments(file);",
" const configName = String(overlay.runtimeNamespace) + '-frpc-config';",
" const deploymentName = String(overlay.runtimeNamespace) + '-frpc';",
" const configKey = String(exposure.secretKey || 'frpc.toml');",
" const tokenKey = String(exposure.tokenKey || 'token');",
" const toml = renderPublicExposureFrpcToml(exposure);",
" let changed = false;",
" let foundConfigMap = false;",
" let foundDeployment = false;",
" for (const doc of docs) {",
" for (const item of listItems(doc).filter(Boolean)) {",
" if (!isObject(item)) continue;",
" item.metadata = item.metadata || {};",
" if (item.kind === 'ConfigMap' && item.metadata.name === configName) {",
" foundConfigMap = true;",
" item.data = item.data || {};",
" if (item.data[configKey] !== toml) { item.data[configKey] = toml; changed = true; }",
" }",
" if (item.kind === 'Deployment' && item.metadata.name === deploymentName) {",
" foundDeployment = true;",
" item.spec = item.spec || {};",
" const nextStrategy = { type: 'Recreate' };",
" if (JSON.stringify(item.spec.strategy || {}) !== JSON.stringify(nextStrategy)) { item.spec.strategy = nextStrategy; changed = true; }",
" const podSpec = podSpecFor(item);",
" for (const container of Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : []) {",
" if (!isObject(container)) continue;",
" if (container.name === 'frpc' || String(container.image || '').includes('frpc')) changed = setEnvFromSecret(container, 'HWLAB_FRP_TOKEN', exposure.secretName, tokenKey) || changed;",
" }",
" }",
" }",
" }",
" if (!foundConfigMap || !foundDeployment) {",
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'frpc-resource-missing', filePath: file, configName, deploymentName, foundConfigMap, foundDeployment }));",
" process.exit(50);",
" }",
" if (changed) writeYamlDocuments(file, docs);",
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: true, applied: true, changed, filePath: file, hostname: exposure.hostname, serverAddr: exposure.serverAddr, serverPort: exposure.serverPort, webProxy: exposure.webProxy.name, apiProxy: exposure.apiProxy.name, configSha256: crypto.createHash('sha256').update(toml).digest('hex') }));",
" return { configured: true, changed, foundConfigMap, foundDeployment };",
"}",
"const kustomizationChanged = patchKustomization();",
"const runtimeWorkloadsChanged = patchRuntimeWorkloads();",
"const externalPostgresChanged = patchExternalPostgres();",
"const healthContractChanged = patchHealthContract();",
"const publicExposureChanged = patchPublicExposure();",
"console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: true, runtimePath, sourcePath, pathRelocated: sourcePath !== runtimePath, observabilityPrometheusOperator: overlay.observability ? overlay.observability.prometheusOperator : null, runtimeImageRewriteCount: (overlay.runtimeImageRewrites || []).length, kustomizationChanged, observabilityWorkloadsChanged: runtimeWorkloadsChanged.observabilityChanged, startupProbeChanged: runtimeWorkloadsChanged.startupProbeChanged, runtimeImageRewriteChanged: runtimeWorkloadsChanged.imageRewriteChanged, gitReadUrlChanged: runtimeWorkloadsChanged.gitReadUrlChanged, publicEndpointChanged: runtimeWorkloadsChanged.publicEndpointChanged, dbSslModeChanged: runtimeWorkloadsChanged.dbSslModeChanged, codeAgentRuntimeChanged: runtimeWorkloadsChanged.codeAgentRuntimeChanged, externalPostgresChanged, healthContractChanged, publicExposureChanged }));",
"NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS`;",
"}",
"function runtimeGitopsVerifyScript() {",
" const runtimeOverlay = JSON.stringify({",
" runtimePath: overlay.runtimePath,",
" runtimeNamespace: overlay.runtimeNamespace,",
" externalPostgres: overlay.externalPostgres,",
" codeAgentRuntime: overlay.codeAgentRuntime,",
" publicExposure: overlay.publicExposure,",
" observability: overlay.observability,",
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
" gitReadUrl: overlay.gitReadUrl,",
" publicWebUrl: overlay.publicWebUrl,",
" publicApiUrl: overlay.publicApiUrl,",
" });",
" return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_VERIFY'",
"const fs = require('fs');",
"const path = require('path');",
"const YAML = require('yaml');",
"const overlay = ${runtimeOverlay};",
"const runtimePath = String(overlay.runtimePath || '');",
"function fail(reason, extra = {}) {",
" console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: false, reason, runtimePath, ...extra }));",
" process.exit(48);",
"}",
"if (!runtimePath || !fs.existsSync(runtimePath)) fail('runtime-path-missing');",
"function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }",
"function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }",
"function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }",
"function yamlFiles(dir) {",
" const files = [];",
" for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {",
" const file = path.join(dir, entry.name);",
" if (entry.isDirectory()) files.push(...yamlFiles(file));",
" else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);",
" }",
" return files;",
"}",
"function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
"function allItemsFromFile(file) { return readYamlDocuments(file).flatMap((doc) => listItems(doc).filter(Boolean)); }",
"function podSpecFor(item) {",
" if (!isObject(item) || !isObject(item.spec)) return null;",
" if (item.kind === 'Pod') return item.spec;",
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
" if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;",
" return null;",
"}",
"function envValue(container, name) {",
" if (!isObject(container) || !Array.isArray(container.env)) return undefined;",
" const item = container.env.find((env) => env && env.name === name);",
" return item ? item.value : undefined;",
"}",
"function envSecretRef(container, name) {",
" if (!isObject(container) || !Array.isArray(container.env)) return {};",
" const item = container.env.find((env) => env && env.name === name);",
" return item && item.valueFrom && item.valueFrom.secretKeyRef ? item.valueFrom.secretKeyRef : {};",
"}",
"function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }",
"function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }",
"function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }",
"function workloadRef(item, file, container) { return { file, kind: item && item.kind, name: item && item.metadata && item.metadata.name, container: container && container.name }; }",
"function workloadChecks() {",
" const metricsRefs = [];",
" const missingStartupProbes = [];",
" const publicRuntimeImages = [];",
" const staleGitReadUrls = [];",
" const wrongPublicEndpoints = [];",
" const wrongDbSslModes = [];",
" const wrongCodeAgentRuntimeEnvs = [];",
" const rewriteSources = new Set((overlay.runtimeImageRewrites || []).map((item) => item && item.source).filter(Boolean));",
" const codeAgentRuntime = overlay.codeAgentRuntime;",
" function checkCodeAgentRuntimeValue(item, file, container, envName, expected) {",
" const value = envValue(container, envName);",
" if (value !== expected) wrongCodeAgentRuntimeEnvs.push({ ...workloadRef(item, file, container), envName, kind: 'value', expected, value: value ?? null });",
" }",
" function checkCodeAgentRuntimeSecret(item, file, container, envName, expectedSecret, expectedKey) {",
" const secretRef = envSecretRef(container, envName);",
" if (secretRef.name !== expectedSecret || secretRef.key !== expectedKey) wrongCodeAgentRuntimeEnvs.push({ ...workloadRef(item, file, container), envName, kind: 'secretKeyRef', expectedSecret, expectedKey, secret: secretRef.name ?? null, key: secretRef.key ?? null });",
" }",
" for (const file of yamlFiles(runtimePath)) {",
" if (path.basename(file) === 'kustomization.yaml') continue;",
" for (const doc of readYamlDocuments(file)) {",
" for (const item of listItems(doc).filter(Boolean)) {",
" const podSpec = podSpecFor(item);",
" if (!isObject(podSpec)) continue;",
" for (const container of Array.isArray(podSpec.containers) ? podSpec.containers : []) {",
" if (!isObject(container)) continue;",
" if (container.name === 'hwlab-metrics' || (Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === 'hwlab-metrics-sidecar'))) metricsRefs.push(workloadRef(item, file, container));",
" if (isEnvReuseContainer(container) && (container.readinessProbe || container.livenessProbe) && !container.startupProbe) missingStartupProbes.push(workloadRef(item, file, container));",
" if (typeof container.image === 'string' && rewriteSources.has(container.image)) publicRuntimeImages.push({ ...workloadRef(item, file, container), image: container.image });",
" if (Array.isArray(container.env) && container.env.some((env) => env && typeof env.value === 'string' && env.value.includes('git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git') && env.value !== overlay.gitReadUrl)) staleGitReadUrls.push(workloadRef(item, file, container));",
" const publicEndpoint = envValue(container, 'HWLAB_PUBLIC_ENDPOINT');",
" if (publicEndpoint !== undefined && publicEndpoint !== expectedPublicEndpoint(item)) wrongPublicEndpoints.push({ ...workloadRef(item, file, container), value: publicEndpoint, expected: expectedPublicEndpoint(item) });",
" const dbSslMode = envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE');",
" if (overlay.externalPostgres && overlay.externalPostgres.sslmode && dbSslMode !== undefined && dbSslMode !== overlay.externalPostgres.sslmode) wrongDbSslModes.push({ ...workloadRef(item, file, container), value: dbSslMode, expected: overlay.externalPostgres.sslmode });",
" if (codeAgentRuntime && codeAgentRuntime.enabled && workloadName(item) === 'hwlab-cloud-api' && container.name === 'hwlab-cloud-api') {",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_ADAPTER', String(codeAgentRuntime.adapter));",
" checkCodeAgentRuntimeValue(item, file, container, 'AGENTRUN_MGR_URL', String(codeAgentRuntime.managerUrl));",
" checkCodeAgentRuntimeSecret(item, file, container, 'AGENTRUN_API_KEY', String(codeAgentRuntime.apiKeySecretName), String(codeAgentRuntime.apiKeySecretKey));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE', String(codeAgentRuntime.runnerNamespace));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE', String(codeAgentRuntime.secretNamespace));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_REPO_URL', String(codeAgentRuntime.repoUrl));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID', String(codeAgentRuntime.providerId));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE', String(codeAgentRuntime.defaultProviderProfile));",
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR', String(codeAgentRuntime.codexStdioSupervisor));",
" }",
" }",
" if (Array.isArray(podSpec.volumes) && podSpec.volumes.some((volume) => volume && volume.name === 'hwlab-metrics-sidecar')) metricsRefs.push(workloadRef(item, file, { name: 'volume/hwlab-metrics-sidecar' }));",
" }",
" }",
" }",
" return { metricsRefs, missingStartupProbes, publicRuntimeImages, staleGitReadUrls, wrongPublicEndpoints, wrongDbSslModes, wrongCodeAgentRuntimeEnvs };",
"}",
"const checks = [];",
"const workloadCheck = workloadChecks();",
"const kustomizationPath = path.join(runtimePath, 'kustomization.yaml');",
"if (overlay.observability && overlay.observability.prometheusOperator === false) {",
" if (!fs.existsSync(kustomizationPath)) fail('kustomization-missing');",
" const resources = readYaml(kustomizationPath).resources || [];",
" if (resources.includes('observability.yaml')) fail('observability-resource-still-rendered', { file: kustomizationPath });",
" if (workloadCheck.metricsRefs.length > 0) fail('observability-sidecar-still-rendered', { refs: workloadCheck.metricsRefs.slice(0, 12), count: workloadCheck.metricsRefs.length });",
" checks.push('observability-disabled');",
"}",
"if (workloadCheck.missingStartupProbes.length > 0) fail('env-reuse-startup-probe-missing', { refs: workloadCheck.missingStartupProbes.slice(0, 12), count: workloadCheck.missingStartupProbes.length });",
"checks.push('env-reuse-startup-probes');",
"if (workloadCheck.publicRuntimeImages.length > 0) fail('runtime-image-rewrite-missing', { refs: workloadCheck.publicRuntimeImages.slice(0, 12), count: workloadCheck.publicRuntimeImages.length });",
"if ((overlay.runtimeImageRewrites || []).length > 0) checks.push('runtime-image-rewrites');",
"if (workloadCheck.staleGitReadUrls.length > 0) fail('runtime-git-read-url-stale', { refs: workloadCheck.staleGitReadUrls.slice(0, 12), count: workloadCheck.staleGitReadUrls.length, expected: overlay.gitReadUrl });",
"checks.push('runtime-git-read-url');",
"if (workloadCheck.wrongPublicEndpoints.length > 0) fail('runtime-public-endpoint-mismatch', { refs: workloadCheck.wrongPublicEndpoints.slice(0, 12), count: workloadCheck.wrongPublicEndpoints.length });",
"checks.push('runtime-public-endpoint');",
"if (workloadCheck.wrongDbSslModes.length > 0) fail('runtime-db-ssl-mode-mismatch', { refs: workloadCheck.wrongDbSslModes.slice(0, 12), count: workloadCheck.wrongDbSslModes.length });",
"if (overlay.externalPostgres && overlay.externalPostgres.sslmode) checks.push('runtime-db-ssl-mode');",
"if (workloadCheck.wrongCodeAgentRuntimeEnvs.length > 0) fail('code-agent-runtime-env-mismatch', { refs: workloadCheck.wrongCodeAgentRuntimeEnvs.slice(0, 12), count: workloadCheck.wrongCodeAgentRuntimeEnvs.length });",
"if (overlay.codeAgentRuntime && overlay.codeAgentRuntime.enabled) checks.push('code-agent-runtime-env');",
"const pg = overlay.externalPostgres;",
"if (pg && pg.serviceName) {",
" const access = pg.runtimeAccess || { endpointAddress: pg.endpointAddress, port: pg.port };",
" const file = path.join(runtimePath, 'external-postgres.yaml');",
" if (!fs.existsSync(file)) fail('external-postgres-missing');",
" const items = listItems(readYaml(file)).filter(Boolean);",
" const service = items.find((item) => item && item.kind === 'Service' && item.metadata && item.metadata.name === pg.serviceName);",
" const endpointSlice = items.find((item) => item && item.kind === 'EndpointSlice' && item.metadata && item.metadata.name === String(pg.serviceName) + '-host');",
" if (!service) fail('external-postgres-service-missing', { expected: pg.serviceName });",
" const endpointIsIpv4 = /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/.test(String(access.endpointAddress));",
" const servicePort = service.spec && Array.isArray(service.spec.ports) && service.spec.ports[0] ? service.spec.ports[0].port : null;",
" if (!endpointIsIpv4) {",
" if (service.spec && service.spec.type !== 'ExternalName') fail('external-postgres-service-type-mismatch', { value: service.spec.type, expected: 'ExternalName' });",
" if (!service.spec || String(service.spec.externalName) !== String(access.endpointAddress)) fail('external-postgres-external-name-mismatch', { value: service.spec && service.spec.externalName, expected: access.endpointAddress });",
" if (Number(servicePort) !== Number(access.port)) fail('external-postgres-port-mismatch', { servicePort, expectedPort: access.port });",
" if (endpointSlice) fail('external-postgres-endpointslice-unexpected', { name: String(pg.serviceName) + '-host' });",
" checks.push('external-postgres-bridge');",
" } else {",
" if (!endpointSlice) fail('external-postgres-endpointslice-missing', { expected: String(pg.serviceName) + '-host' });",
" const endpointPort = Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;",
" const endpointAddress = Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;",
" if (Number(servicePort) !== Number(access.port) || Number(endpointPort) !== Number(access.port)) fail('external-postgres-port-mismatch', { servicePort, endpointPort, expectedPort: access.port });",
" if (String(endpointAddress) !== String(access.endpointAddress)) fail('external-postgres-address-mismatch', { endpointAddress, expectedEndpointAddress: access.endpointAddress });",
" checks.push('external-postgres-bridge');",
" }",
"}",
"const exposure = overlay.publicExposure;",
"if (exposure && exposure.enabled) {",
" const file = path.join(runtimePath, 'node-frpc.yaml');",
" if (!fs.existsSync(file)) fail('public-exposure-frpc-missing');",
" const items = allItemsFromFile(file);",
" const configName = String(overlay.runtimeNamespace) + '-frpc-config';",
" const deploymentName = String(overlay.runtimeNamespace) + '-frpc';",
" const configKey = String(exposure.secretKey || 'frpc.toml');",
" const tokenKey = String(exposure.tokenKey || 'token');",
" const configMap = items.find((item) => item && item.kind === 'ConfigMap' && item.metadata && item.metadata.name === configName);",
" const deployment = items.find((item) => item && item.kind === 'Deployment' && item.metadata && item.metadata.name === deploymentName);",
" if (!configMap) fail('public-exposure-frpc-configmap-missing', { expected: configName });",
" if (!deployment) fail('public-exposure-frpc-deployment-missing', { expected: deploymentName });",
" const toml = configMap.data && configMap.data[configKey];",
" if (typeof toml !== 'string') fail('public-exposure-frpc-config-missing', { expectedKey: configKey });",
" for (const expected of [String(exposure.serverAddr), String(exposure.serverPort), String(exposure.webProxy.name), String(exposure.webProxy.remotePort), String(exposure.apiProxy.name), String(exposure.apiProxy.remotePort), 'HWLAB_FRP_TOKEN']) {",
" if (!toml.includes(expected)) fail('public-exposure-frpc-config-mismatch', { expected });",
" }",
" const podSpec = podSpecFor(deployment);",
" const containers = Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : [];",
" const strategyType = deployment.spec && deployment.spec.strategy && deployment.spec.strategy.type;",
" if (strategyType !== 'Recreate') fail('public-exposure-frpc-strategy-mismatch', { expected: 'Recreate', actual: strategyType || null });",
" const frpc = containers.find((container) => container && (container.name === 'frpc' || String(container.image || '').includes('frpc')));",
" const env = frpc && Array.isArray(frpc.env) ? frpc.env.find((item) => item && item.name === 'HWLAB_FRP_TOKEN') : null;",
" const secretRef = env && env.valueFrom && env.valueFrom.secretKeyRef;",
" if (!secretRef || secretRef.name !== exposure.secretName || secretRef.key !== tokenKey) fail('public-exposure-frpc-token-env-mismatch', { expectedSecret: exposure.secretName, expectedKey: tokenKey });",
" checks.push('public-exposure-frpc');",
"}",
"console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: true, runtimePath, checks }));",
"NODE_UNIDESK_RUNTIME_GITOPS_VERIFY`;",
"}",
"function patchScript(script) {",
" let result = String(script || '');",
" const bootstrap = stepEnvBootstrapScript();",
" if (bootstrap && !result.includes('unidesk-step-env-bootstrap')) result = `${bootstrap}\\n${result}`;",
" const inlineDeployServiceIdParser = `const deployText = fs.readFileSync(deployPath, 'utf8');\\nconst deploy = { services: [...deployText.matchAll(/(?:^|\\\\n)\\\\s*serviceId:\\\\s*[\\\"']?([^\\\"'\\\\n#]+)[\\\"']?/g)].map((match) => ({ serviceId: match[1].trim() })) };`;",
" result = result.split('import { readStructuredFile } from \"./scripts/src/structured-config.mjs\";\\n').join('');",
" result = result.split('const deploy = await readStructuredFile(process.cwd(), deployPath);').join(inlineDeployServiceIdParser);",
" if (result.includes('npm run gitops:ts:check')) {",
" result = result.replace(/\\n[ \\t]*npm run gitops:ts:check\\n/g, '\\n echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-replaces-tsc-gate\"}\\' >&2\\n');",
" }",
" const isNodeContractCheck = result.includes('\\\"event\\\":\\\"unidesk-node-contract-check\\\"');",
" if (isNodeContractCheck) {",
" result = result.replace(/(^|\\n)([ \\t]*)node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*(?=\\n|$)/g, '$1$2echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-disabled-gitops-render\"}\\' >&2');",
" }",
" const prepareSourceDependencyPattern = new RegExp(String.raw`prepare_source_dependencies_started_ms=\"\\$\\(ci_now_ms\\)\"\\nif node -e 'require\\.resolve\\(\"yaml\"\\)'[\\s\\S]*?\\nci_timing_emit prepare-source-dependencies succeeded \"\\$prepare_source_dependencies_started_ms\"`, 'g');",
" if (result.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"')) {",
" result = result.replace(prepareSourceDependencyPattern, prepareSourceDependencyScript());",
" }",
" const artifactPublishNeedle = 'node scripts/artifact-publish.mjs --publish';",
" if (result.includes(artifactPublishNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {",
" result = result.replace(artifactPublishNeedle, `${deployYamlOverlayScript()}\\n${artifactPublishNeedle}`);",
" }",
" const gitopsRenderNeedle = 'node scripts/run-bun.mjs scripts/gitops-render.mjs';",
" if (!isNodeContractCheck && result.includes(gitopsRenderNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {",
" result = result.replace(gitopsRenderNeedle, `${deployYamlOverlayScript()}\\n${gitopsRenderNeedle}`);",
" }",
" result = result.replaceAll('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808', `node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost || '127.0.0.1'} ${overlay.gitSshProxyPort || 10808}`);",
" result = result.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);",
" result = result.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);",
" result = result.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);",
" result = result.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
" result = result.replace(/--gitops-root \"\\$gitops_root\"/g, `--gitops-root ${JSON.stringify(overlay.gitopsRoot)}`);",
" result = result.replace(/--out \"\\$gitops_root\"/g, `--out ${JSON.stringify(overlay.gitopsRoot)}`);",
" const legacyRuntimePath = (() => {",
" const runtimePath = String(overlay.runtimePath || '');",
" const parts = runtimePath.split('/').filter(Boolean);",
" if (parts.length < 2) return '';",
" const leaf = parts.at(-1);",
" const parent = parts.slice(0, -2).join('/');",
" return parent && leaf ? `${parent}/${leaf}` : '';",
" })();",
" if (legacyRuntimePath && legacyRuntimePath !== overlay.runtimePath) {",
" result = result.replace('runtime_path=\"$(params.runtime-path)\"', `runtime_path=\"$(params.runtime-path)\"\\nunidesk_legacy_runtime_path=${shellSingle(legacyRuntimePath)}`);",
" result = result.replace('rm -rf \"$runtime_path\" \"$catalog_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi', 'rm -rf \"$runtime_path\" \"$catalog_path\" \"$unidesk_legacy_runtime_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi');",
" result = result.replace('git add \"$catalog_path\" \"$runtime_path\"', 'git add \"$catalog_path\" \"$runtime_path\"\\n if [ -n \"${unidesk_legacy_runtime_path:-}\" ]; then git add -A \"$unidesk_legacy_runtime_path\" || true; fi');",
" }",
" if (bootstrap && result.includes('git config --global') && !result.includes('unidesk-step-env-bootstrap')) {",
" result = result.replace(/\\n([ \\t]*)git config --global/g, (match, indent) => `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`);",
" }",
" result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--use-deploy-images/g, (match) => {",
" let next = match;",
" if (!next.includes('--gitops-root ')) next = next.replace(' --use-deploy-images', ` --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);",
" if (!next.includes('--out ')) next = next.replace(' --use-deploy-images', ` --out ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);",
" return next;",
" });",
" result = result.replace(/(node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*--use-deploy-images[^\\n]*)/g, (match) => {",
" if (match.includes('--check')) return runtimeGitopsVerifyScript();",
" return `${match}\\n${[runtimePathOverlayScript(), runtimeGitopsPostprocessScript()].filter(Boolean).join('\\n')}`;",
" });",
" result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--out \"\\$render_check_dir\"/g, (match) => match.includes('--gitops-root ') ? match : `${match} --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --node ${shellSingle(overlay.nodeId)} --git-read-url ${shellSingle(overlay.gitReadUrl)} --git-write-url ${shellSingle(overlay.gitWriteUrl)} --runtime-endpoint ${shellSingle(overlay.publicApiUrl)} --web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
" validatePrepareSourceDependencyScript(result);",
" return result;",
"}",
"function patchManifestObject(doc) {",
" if (!doc || typeof doc !== 'object') return false;",
" if (doc.kind !== 'Pipeline' || !doc.spec) return false;",
" const defaults = {",
" 'git-url': overlay.gitUrl,",
" 'git-read-url': overlay.gitReadUrl,",
" 'git-write-url': overlay.gitWriteUrl,",
" 'catalog-path': overlay.catalogPath,",
" 'runtime-path': overlay.runtimePath,",
" 'registry-prefix': overlay.registryPrefix,",
" };",
" for (const param of doc.spec?.params || []) {",
" if (Object.prototype.hasOwnProperty.call(defaults, param.name)) param.default = defaults[param.name];",
" }",
" doc.metadata = doc.metadata || {};",
" if (typeof overlay.pipelineName === 'string' && overlay.pipelineName.length > 0) doc.metadata.name = overlay.pipelineName;",
" doc.metadata.annotations = doc.metadata.annotations || {};",
" doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId;",
" doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId;",
" for (const task of doc.spec?.tasks || []) {",
" for (const sidecar of task.taskSpec?.sidecars || []) {",
" if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage;",
" }",
" for (const step of task.taskSpec?.steps || []) {",
" if (step.image === overlay.toolsImage && overlay.toolsImagePullPolicy) step.imagePullPolicy = overlay.toolsImagePullPolicy;",
" if (Array.isArray(step.env)) {",
" for (const env of step.env) {",
" if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name];",
" }",
" }",
" step.env = Array.isArray(step.env) ? step.env : [];",
" const existingEnv = new Set(step.env.map((env) => env.name));",
" for (const [name, value] of Object.entries(stepEnv)) {",
" if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value });",
" }",
" if (typeof step.script === 'string') step.script = patchScript(step.script);",
" }",
" }",
" return true;",
"}",
"function patchStructuredPipeline() {",
" try {",
" const doc = JSON.parse(text);",
" if (!patchManifestObject(doc)) return false;",
" text = JSON.stringify(doc, null, 2) + '\\n';",
" return true;",
" } catch {}",
" if (YAML) {",
" try {",
" const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null);",
" const changed = docs.some((doc) => patchManifestObject(doc));",
" if (!changed) return false;",
" text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n';",
" return true;",
" } catch {}",
" }",
" return false;",
"}",
"function patchGitMirrorTransportYaml() {",
" const mirror = overlay.gitMirror || {};",
" const transport = mirror.githubTransport || {};",
" if (transport.mode !== 'https') return;",
" if (!YAML) throw new Error('yaml module is required to patch git-mirror githubTransport=https');",
" const requireString = (pathName, value) => {",
" if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror githubTransport=https');",
" return value;",
" };",
" const repository = requireString('gitMirror.sourceRepository', mirror.sourceRepository);",
" const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);",
" const tokenSecretName = requireString('gitMirror.githubTransport.tokenSecretName', transport.tokenSecretName);",
" const tokenSecretKey = requireString('gitMirror.githubTransport.tokenSecretKey', transport.tokenSecretKey);",
" const tokenSourceRef = requireString('gitMirror.githubTransport.tokenSourceRef', transport.tokenSourceRef);",
" const username = typeof transport.username === 'string' && transport.username ? transport.username : 'x-access-token';",
" const remoteUrl = `https://github.com/${repository}.git`;",
" const proxySummary = `transport=https proxy=HTTP_PROXY authSecret=${tokenSecretName} authKey=${tokenSecretKey} authSourceRef=${tokenSourceRef} source=yaml`;",
" const askpassBlock = [",
" 'if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo \\'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env\\' >&2; exit 64; fi',",
" \"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'\",",
" '#!/bin/sh',",
" 'case \"$1\" in',",
" ' *Username*) printf \\'%s\\\\n\\' \"${GITHUB_USERNAME:-x-access-token}\" ;;',",
" ' *Password*) printf \\'%s\\\\n\\' \"$GITHUB_TOKEN\" ;;',",
" \" *) printf '\\\\n' ;;\",",
" 'esac',",
" 'SH_ASKPASS',",
" 'chmod 0700 /tmp/hwlab-git-askpass.sh',",
" `export GITHUB_USERNAME=${shellSingle(username)}`,",
" 'export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh',",
" 'export GIT_TERMINAL_PROMPT=0',",
" 'unset GIT_SSH',",
" 'unset GIT_SSH_COMMAND',",
" ].join('\\n');",
" function patchScript(script, key) {",
" let next = String(script || '');",
" if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);",
" let remoteReplaced = false;",
" next = next.replace(/repo_url=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `repo_url=${shellSingle(remoteUrl)}`; });",
" next = next.replace(/remote=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `remote=${shellSingle(remoteUrl)}`; });",
" if (!remoteReplaced) throw new Error(`generated git-mirror ${key} missing remote url assignment for githubTransport=https`);",
" next = next.replace(/transport=ssh ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);",
" next = next.replace(/ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);",
" next = next.replace(/mkdir -p ([^\\n]*?) \\/root\\/\\.ssh/g, 'mkdir -p $1');",
" next = next.replace(/\\n[ \\t]*mkdir -p \\/root\\/\\.ssh\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*cp \\/git-ssh\\/ssh-privatekey[^\\n]*\\n[ \\t]*chmod 0?400[^\\n]*\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) <<'NODE_PROXY'[\\s\\S]*?\\nNODE_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs)\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-git-ssh-proxy\\.sh <<'SH_PROXY'[\\s\\S]*?\\nSH_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-git-ssh-proxy\\.sh\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*export GIT_SSH=.*\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*export GIT_SSH_COMMAND=.*\\n/g, '\\n');",
" next = next.replace(/\\n[ \\t]*unset GIT_SSH_COMMAND\\n/g, '\\n');",
" if (!next.includes('hwlab git-mirror https auth: missing GITHUB_TOKEN secret env')) {",
" const noProxyExport = /\\nexport no_proxy=[^\\n]*\\n/;",
" if (noProxyExport.test(next)) next = next.replace(noProxyExport, (match) => `${match}${askpassBlock}\\n`);",
" else if (next.includes('\\nset -eu\\n')) next = next.replace('\\nset -eu\\n', `\\nset -eu\\n${askpassBlock}\\n`);",
" else next = `${askpassBlock}\\n${next}`;",
" }",
" if (next.includes('/git-ssh') || next.includes('ssh://git@') || next.includes('GIT_SSH=')) throw new Error(`generated git-mirror ${key} still contains ssh transport after githubTransport=https patch`);",
" if (!next.includes('GIT_ASKPASS') || !next.includes('GITHUB_TOKEN')) throw new Error(`generated git-mirror ${key} missing https auth after githubTransport=https patch`);",
" return next;",
" }",
" const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');",
" if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);",
" const docs = YAML.parseAllDocuments(fs.readFileSync(gitMirrorFile, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);",
" const manifests = [];",
" for (const doc of docs) {",
" if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);",
" else manifests.push(doc);",
" }",
" let changed = false;",
" for (const doc of manifests) {",
" if (!doc || typeof doc !== 'object' || doc.kind !== 'ConfigMap') continue;",
" if (!doc.metadata || doc.metadata.name !== configMapName) continue;",
" doc.data = doc.data || {};",
" doc.data['sync.sh'] = patchScript(doc.data['sync.sh'], 'sync.sh');",
" doc.data['flush.sh'] = patchScript(doc.data['flush.sh'], 'flush.sh');",
" changed = true;",
" }",
" if (!changed) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);",
" fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
"}",
"const structured = patchStructuredPipeline();",
"function replaceParamDefault(name, value) {",
" const namePattern = escapeRegExp(name);",
" text = text.replace(new RegExp(`(- default: )[^\\n]*(\\n\\\\s+name: ${namePattern}\\n\\\\s+type: string)`, 'g'), `$1${yamlString(value)}$2`);",
" text = text.replace(new RegExp(`(- name: ${namePattern}\\n\\\\s+type: string\\n\\\\s+default: )[^\\n]*`, 'g'), `$1${yamlString(value)}`);",
"}",
"replaceParamDefault('git-url', overlay.gitUrl);",
"replaceParamDefault('git-read-url', overlay.gitReadUrl);",
"replaceParamDefault('git-write-url', overlay.gitWriteUrl);",
"replaceParamDefault('catalog-path', overlay.catalogPath);",
"replaceParamDefault('runtime-path', overlay.runtimePath);",
"replaceParamDefault('registry-prefix', overlay.registryPrefix);",
"text = text.replace(/hwlab\\.pikastech\\.local\\/download-profile: [^\\n]+/g, `hwlab.pikastech.local/download-profile: ${overlay.downloadProfileId}`);",
"text = text.replace(/hwlab\\.pikastech\\.local\\/network-profile: [^\\n]+/g, `hwlab.pikastech.local/network-profile: ${overlay.networkProfileId}`);",
"if (overlay.buildkitSidecarImage) text = text.replace(/moby\\/buildkit:rootless/g, overlay.buildkitSidecarImage);",
"text = text.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);",
"text = text.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);",
"text = text.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);",
"text = text.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
"for (const [name, value] of Object.entries(stepEnv)) {",
" if (value === undefined) continue;",
" text = text.replace(new RegExp(`(- name: ${escapeRegExp(name)}\\\\n\\\\s+value: )[^\\\\n]+`, 'g'), `$1${yamlString(value)}`);",
" text = text.replace(new RegExp(`(\\\"name\\\": ${JSON.stringify(name)},\\\\n\\\\s+\\\"value\\\": )\\\"[^\\\"]*\\\"`, 'g'), `$1${yamlString(value)}`);",
"}",
"const bootstrap = stepEnvBootstrapScript();",
"if (bootstrap) {",
" text = text.replace(/\\n([ \\t]*)git config --global/g, (match, indent, offset, fullText) => {",
" const previous = fullText.slice(Math.max(0, offset - 500), offset);",
" if (previous.includes('unidesk-step-env-bootstrap')) return match;",
" return `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`;",
" });",
"}",
"if (overlay.gitSshProxyHost && overlay.gitSshProxyPort) {",
" text = text.split('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808').join(`node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost} ${overlay.gitSshProxyPort}`);",
"}",
"const quotedRoot = JSON.stringify(overlay.gitopsRoot);",
"const escapedQuotedRoot = quotedRoot.replaceAll('\"', '\\\\\"');",
"const replacements = [",
" ['--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images' : `--registry-prefix \"$(params.registry-prefix)\" --gitops-root ${quotedRoot} --use-deploy-images`],",
" ['--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images' : `--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --gitops-root ${escapedQuotedRoot} --use-deploy-images`],",
"];",
"let changed = false;",
"for (const [needle, replacement] of replacements) {",
" if (!text.includes(needle)) continue;",
" text = text.split(needle).join(replacement);",
" changed = true;",
"}",
"if (!structured && !changed && !text.includes(`--gitops-root ${quotedRoot}`) && !text.includes(`--gitops-root ${escapedQuotedRoot}`)) { throw new Error(`generated pipeline missing expected gitops-render invocation in ${pipelinePath}`); }",
"if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && text.includes('npm ci --ignore-scripts --no-audit --prefer-offline')) { throw new Error(`generated pipeline still uses full npm ci prepare-source dependency install in ${pipelinePath}`); }",
"if (text.includes('readStructuredFile(process.cwd(), deployPath)')) { throw new Error(`generated pipeline still imports YAML parser for prepare-source catalog check in ${pipelinePath}`); }",
"if (text.includes('/yaml/-/yaml-') || text.includes('bun add --no-save --ignore-scripts') || text.includes('npm install --package-lock=false --no-save')) { throw new Error(`generated pipeline still downloads yaml during prepare-source in ${pipelinePath}`); }",
"if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }",
"fs.writeFileSync(pipelinePath, text);",
"patchGitMirrorTransportYaml();",
"function patchArgoYaml(filePath) {",
" if (!YAML || !fs.existsSync(filePath)) return;",
" const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);",
" let changed = false;",
" for (const doc of docs) {",
" if (!doc || typeof doc !== 'object') continue;",
" if (doc.kind === 'AppProject') {",
" doc.spec = doc.spec || {};",
" doc.spec.sourceRepos = [overlay.argoRepoUrl];",
" changed = true;",
" }",
" if (doc.kind === 'Application') {",
" doc.spec = doc.spec || {};",
" doc.spec.source = { ...(doc.spec.source || {}), repoURL: overlay.argoRepoUrl, targetRevision: overlay.gitopsBranch, path: overlay.runtimePath };",
" changed = true;",
" }",
" }",
" if (changed) fs.writeFileSync(filePath, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
"}",
"patchArgoYaml(path.join(renderDir, 'argocd', 'project.yaml'));",
"patchArgoYaml(path.join(renderDir, 'argocd', overlay.argoApplicationFile));",
"NODE",
];
}
Reference in New Issue View Git Blame Copy Permalink