2452 lines
154 KiB
TypeScript
2452 lines
154 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/hwlab-node-impl.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-node-impl.ts:4000-5260 for #903.
|
|
|
|
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// Responsibility: YAML-first node/lane operations, including Workbench observability control commands.
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand, type CommandResult } from "../command";
|
|
import { startJob } from "../jobs";
|
|
import { classifySshTcpPoolFailure } from "../ssh";
|
|
import { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, hwlabNodeControlPlaneInfraHelp, runHwlabNodeControlPlaneInfra } from "../hwlab-node-control-plane";
|
|
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, hwlabRuntimeLaneSpecForNode, hwlabRuntimeNodeIds, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec, type HwlabRuntimeObservabilityRecordingRuleSpec, type HwlabRuntimeObservabilitySpec, type HwlabRuntimeObservabilityWarningAlertSpec, type HwlabRuntimePublicExposureSpec, type HwlabRuntimeWebProbeAlertThresholdsSpec, type HwlabRuntimeWebProbeProjectManagementSpec } from "../hwlab-node-lanes";
|
|
import { nodeWebProbeScriptRunnerSource } from "../hwlab-node-web-probe-runner-source";
|
|
import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
|
|
import { nodeWebObserveRunnerSource } from "../hwlab-node-web-observe-runner-source";
|
|
import { nodeWebObserveCollectViewNodeScript, parseNodeWebProbeObserveCollectView, type NodeWebProbeObserveCollectView } from "../hwlab-node-web-observe-collect";
|
|
import { withWebObserveCollectRendered, withWebObserveCommandRendered, withWebObserveStatusRendered } from "../hwlab-node-web-observe-render";
|
|
import { buildWebObserveWrapperForObserveOptions, webObserveWrapperStateDirFromStatus } from "../hwlab-node-web-observe-wrapper";
|
|
import { renderWebObserveWrapperContract } from "../hwlab-node-web-observe-wrapper-render";
|
|
import { runWebProbeSentinelCommand, type WebProbeSentinelOptions } from "../hwlab-node-web-sentinel-cicd";
|
|
import { hwlabNodeHelp, hwlabNodeObservabilityHelp, hwlabNodeWebProbeHelp } from "../hwlab-node-help";
|
|
import { compactWebProbeResult, compactWebProbeScriptResult } from "../hwlab-node-web-probe-summary";
|
|
import { nodeObservabilityRecordingRuleExpression, nodeObservabilityRecordingRuleSummaries, nodeObservabilityWarningAlertExpression, nodeObservabilityWarningAlertSummaries } from "../hwlab-node-observability-promql";
|
|
import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab-node-transport";
|
|
import type { RenderedCliResult } from "../output";
|
|
|
|
import type { NodeRuntimeGitMirrorTargetSpec, NodeRuntimeRenderResult } from "./entry";
|
|
import { isCommandSuccess, nodeRuntimeGitopsRoot, nodeRuntimePipelineRunName, resolveNodeRuntimeLaneHead, runNodeK3sArgs, runtimeLaneCicdRepoEnsureScript, shortSha } from "./cleanup";
|
|
import { HWLAB_CI_NAMESPACE } from "./entry";
|
|
import { nodeRuntimeExpected, nodeRuntimeLocalPostgresExpectedAbsent, parseNodeScopedDelegatedOptions } from "./plan";
|
|
import { runTransHostScript } from "./public-exposure";
|
|
import { compactRuntimeCommand, runNodeHostScriptAsync } from "./runtime-common";
|
|
import { compactNodeRuntimeGitMirrorStatus, nodeRuntimeGitMirrorStatus } from "./status";
|
|
import { keyValueLinesFromText, numericField, optionValue, record, shellQuote } from "./utils";
|
|
import { externalPostgresBridgeStatus, externalPostgresSecretStatus, getNodeRuntimePipelineRun, isLocalPostgresObject, nodeRuntimeCodeAgentRuntimeStatus, nodeRuntimeRenderOverlay } from "./web-probe";
|
|
import { webObserveShort, webObserveText } from "./web-probe-observe";
|
|
import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes";
|
|
|
|
export function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush"): string {
|
|
const prefix = action === "sync" ? mirror.syncJobPrefix : mirror.flushJobPrefix;
|
|
return `${prefix}-${Date.now().toString(36)}`.slice(0, 63);
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorJobManifest(
|
|
mirror: NodeRuntimeGitMirrorTargetSpec,
|
|
action: "sync" | "flush",
|
|
jobName: string,
|
|
options: { discardStaleGitops?: boolean } = {},
|
|
): Record<string, unknown> {
|
|
const volumes: Record<string, unknown>[] = [
|
|
{ name: "cache", ...(mirror.cacheHostPath === null ? { persistentVolumeClaim: { claimName: mirror.cachePvcName } } : { hostPath: { path: mirror.cacheHostPath, type: "DirectoryOrCreate" } }) },
|
|
{ name: "script", configMap: { name: mirror.syncConfigMapName, defaultMode: 0o755 } },
|
|
];
|
|
const volumeMounts: Record<string, unknown>[] = [
|
|
{ name: "cache", mountPath: "/cache" },
|
|
{ name: "script", mountPath: "/script", readOnly: true },
|
|
];
|
|
if (mirror.githubTransport.mode === "ssh") {
|
|
volumes.splice(1, 0, { name: "git-ssh", secret: { secretName: mirror.secretName, defaultMode: 0o400 } });
|
|
volumeMounts.splice(1, 0, { name: "git-ssh", mountPath: "/git-ssh", readOnly: true });
|
|
}
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name: jobName,
|
|
namespace: mirror.namespace,
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
|
"app.kubernetes.io/component": `${action}-controller`,
|
|
"hwlab.pikastech.local/node": mirror.node,
|
|
"hwlab.pikastech.local/lane": mirror.lane,
|
|
"hwlab.pikastech.local/trigger": "manual-cli",
|
|
},
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
activeDeadlineSeconds: 600,
|
|
ttlSecondsAfterFinished: 3600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/name": "git-mirror",
|
|
"app.kubernetes.io/part-of": "hwlab-node-control-plane",
|
|
"app.kubernetes.io/component": `${action}-controller`,
|
|
"hwlab.pikastech.local/node": mirror.node,
|
|
"hwlab.pikastech.local/lane": mirror.lane,
|
|
},
|
|
},
|
|
spec: {
|
|
...(mirror.egressProxy.mode === "host-route" ? { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet" } : {}),
|
|
restartPolicy: "Never",
|
|
volumes,
|
|
containers: [{
|
|
name: action,
|
|
image: mirror.toolsImage,
|
|
imagePullPolicy: mirror.toolsImagePullPolicy,
|
|
command: [action === "sync" ? "/script/sync.sh" : "/script/flush.sh"],
|
|
env: [
|
|
...nodeRuntimeGitMirrorProxyEnv(mirror),
|
|
...nodeRuntimeGitMirrorGithubTransportEnv(mirror),
|
|
...(options.discardStaleGitops === true ? [{ name: "UNIDESK_GIT_MIRROR_DISCARD_STALE_GITOPS", value: "true" }] : []),
|
|
],
|
|
volumeMounts,
|
|
}],
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorProxyEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
|
|
const proxy = mirror.egressProxy;
|
|
if (proxy.mode === "direct") return [];
|
|
const proxyUrl = proxy.mode === "host-route" ? proxy.proxyUrl : `http://${proxy.serviceName}.${proxy.namespace}.svc.cluster.local:${proxy.port}`;
|
|
const noProxy = proxy.noProxy.join(",");
|
|
return [
|
|
{ name: "HTTP_PROXY", value: proxyUrl },
|
|
{ name: "HTTPS_PROXY", value: proxyUrl },
|
|
{ name: "ALL_PROXY", value: proxyUrl },
|
|
{ name: "http_proxy", value: proxyUrl },
|
|
{ name: "https_proxy", value: proxyUrl },
|
|
{ name: "all_proxy", value: proxyUrl },
|
|
{ name: "NO_PROXY", value: noProxy },
|
|
{ name: "no_proxy", value: noProxy },
|
|
];
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorGithubTransportEnv(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown>[] {
|
|
if (mirror.githubTransport.mode !== "https") return [];
|
|
return [{
|
|
name: "GITHUB_TOKEN",
|
|
valueFrom: { secretKeyRef: { name: mirror.githubTransport.tokenSecretName, key: mirror.githubTransport.tokenSecretKey } },
|
|
}];
|
|
}
|
|
|
|
export function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGitMirrorTargetSpec): Record<string, unknown> {
|
|
const transport = mirror.githubTransport;
|
|
if (transport.mode === "ssh") {
|
|
return {
|
|
mode: "ssh",
|
|
secretName: mirror.secretName,
|
|
privateKeySecretKey: transport.privateKeySecretKey,
|
|
privateKeySourceRef: transport.privateKeySourceRef,
|
|
privateKeySourceKey: transport.privateKeySourceKey,
|
|
privateKeySourceEncoding: transport.privateKeySourceEncoding,
|
|
knownHostsSecretKey: transport.knownHostsSecretKey,
|
|
knownHostsSourceRef: transport.knownHostsSourceRef,
|
|
knownHostsSourceKey: transport.knownHostsSourceKey,
|
|
knownHostsSourceEncoding: transport.knownHostsSourceEncoding,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
return {
|
|
mode: "https",
|
|
username: transport.username,
|
|
tokenSecretName: transport.tokenSecretName,
|
|
tokenSecretKey: transport.tokenSecretKey,
|
|
tokenSourceRef: transport.tokenSourceRef,
|
|
tokenSourceKey: transport.tokenSourceKey,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const spec = scoped.spec;
|
|
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
|
|
const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run");
|
|
const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null;
|
|
const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null;
|
|
const pipelineRun = pipelineRunOverride ?? (sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit));
|
|
const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], 60);
|
|
const namespaceExists = namespace.exitCode === 0;
|
|
const postgresObjects = namespaceExists
|
|
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], 60)
|
|
: null;
|
|
const localPostgresObjects = postgresObjects === null
|
|
? []
|
|
: postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec));
|
|
const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], 60);
|
|
const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], 60);
|
|
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60);
|
|
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
|
|
const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
|
|
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
|
|
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
|
|
: null;
|
|
const workloads = namespaceExists
|
|
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60)
|
|
: null;
|
|
const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
const workloadReadinessProbe = namespaceExists
|
|
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], 60)
|
|
: null;
|
|
const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? "");
|
|
const bridge = externalPostgresBridgeStatus(spec, namespaceExists);
|
|
const secrets = externalPostgresSecretStatus(spec, namespaceExists);
|
|
const codeAgentRuntime = nodeRuntimeCodeAgentRuntimeStatus(spec, namespaceExists);
|
|
const publicProbes = nodeRuntimePublicProbeStatus(spec);
|
|
const gitMirror = nodeRuntimeGitMirrorStatus(scoped);
|
|
const gitMirrorCompact = compactNodeRuntimeGitMirrorStatus(gitMirror);
|
|
const activeExternalPostgres = hwlabRuntimeActiveExternalPostgres(spec);
|
|
const controlPlaneReady = serviceAccount.exitCode === 0 && pipeline.exitCode === 0 && argo.exitCode === 0;
|
|
const workloadsReady = workloadReadiness.length > 0 && workloadReadiness.every((item) => item.ready);
|
|
const localPostgresExpectedAbsent = nodeRuntimeLocalPostgresExpectedAbsent(spec);
|
|
const localPostgresReady = localPostgresExpectedAbsent ? localPostgresObjects.length === 0 : localPostgresObjects.length > 0;
|
|
const runtimeReady = namespaceExists && localPostgresReady && workloadsReady && (activeExternalPostgres === undefined || (bridge.ready && secrets.ready)) && codeAgentRuntime.ready === true;
|
|
const codeAgentRuntimeDegradedReason = typeof codeAgentRuntime.degradedReason === "string" ? codeAgentRuntime.degradedReason : null;
|
|
const runtimeDegradedReason = !namespaceExists
|
|
? "runtime-namespace-missing"
|
|
: !localPostgresReady
|
|
? "runtime-local-postgres-not-ready"
|
|
: !workloadsReady
|
|
? "runtime-workloads-not-ready"
|
|
: activeExternalPostgres !== undefined && bridge.ready !== true
|
|
? "external-postgres-bridge-not-ready"
|
|
: activeExternalPostgres !== undefined && secrets.ready !== true
|
|
? "external-postgres-secrets-not-ready"
|
|
: codeAgentRuntime.ready !== true
|
|
? codeAgentRuntimeDegradedReason ?? "code-agent-runtime-not-ready"
|
|
: "runtime-not-ready";
|
|
const argoReady = argo.exitCode === 0 && repoURL === spec.argoRepoUrl && targetRevision === spec.gitopsBranch && path === spec.runtimePath && syncStatus === "Synced" && health === "Healthy";
|
|
const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
|
|
const pipelineRunDegradedReason = typeof pipelineRunDiagnostics?.degradedReason === "string"
|
|
? pipelineRunDiagnostics.degradedReason
|
|
: "pipelinerun-not-succeeded";
|
|
const publicReady = publicProbes.ready === true;
|
|
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
|
|
const fullStatus = {
|
|
ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady && gitMirrorReady,
|
|
command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
|
|
mode: "node-scoped-runtime-status",
|
|
mutation: false,
|
|
node: scoped.node,
|
|
lane: scoped.lane,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
expected: nodeRuntimeExpected(spec),
|
|
sourceHead: head === null
|
|
? { ok: sourceCommit !== null, value: sourceCommit, source: "option" }
|
|
: { ok: head.sourceCommit !== null, value: head.sourceCommit, probe: compactRuntimeCommand(head.result) },
|
|
controlPlane: {
|
|
ready: controlPlaneReady,
|
|
serviceAccount: { exists: serviceAccount.exitCode === 0, result: compactRuntimeCommand(serviceAccount) },
|
|
pipeline: { exists: pipeline.exitCode === 0, result: compactRuntimeCommand(pipeline) },
|
|
},
|
|
argo: {
|
|
ready: argoReady,
|
|
application: spec.app,
|
|
repoURL,
|
|
expectedRepoURL: spec.argoRepoUrl,
|
|
targetRevision,
|
|
path,
|
|
syncRevision,
|
|
syncStatus,
|
|
health,
|
|
result: compactRuntimeCommand(argo),
|
|
},
|
|
pipelineRun: pipelineRunProbe,
|
|
pipelineRunDiagnostics,
|
|
runtime: {
|
|
ready: runtimeReady,
|
|
namespace: spec.runtimeNamespace,
|
|
namespaceExists,
|
|
localPostgresObjects,
|
|
localPostgresAbsent: namespaceExists && localPostgresObjects.length === 0,
|
|
localPostgresExpectedAbsent,
|
|
localPostgresReady,
|
|
workloadNames,
|
|
workloadCount: workloadNames.length,
|
|
workloadReadiness,
|
|
workloadsReady,
|
|
workloadReadinessResult: workloadReadinessProbe === null ? null : compactRuntimeCommand(workloadReadinessProbe),
|
|
workloadResult: workloads === null ? null : compactRuntimeCommand(workloads),
|
|
externalPostgresBridge: bridge,
|
|
externalPostgresSecrets: secrets,
|
|
codeAgentRuntime,
|
|
},
|
|
publicProbes,
|
|
gitMirror: {
|
|
...gitMirror,
|
|
compact: gitMirrorCompact,
|
|
ready: gitMirrorReady,
|
|
},
|
|
probes: {
|
|
namespace: compactRuntimeCommand(namespace),
|
|
postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects),
|
|
},
|
|
degradedReason: controlPlaneReady
|
|
? runtimeReady
|
|
? argoReady
|
|
? pipelineRunReady
|
|
? publicReady
|
|
? gitMirrorReady ? undefined : "git-mirror-pending-flush"
|
|
: "public-probe-not-ready"
|
|
: pipelineRunDegradedReason
|
|
: "argo-not-synced-healthy"
|
|
: runtimeDegradedReason
|
|
: "control-plane-not-ready",
|
|
next: {
|
|
plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
|
|
apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
},
|
|
};
|
|
const summary = summarizeNodeRuntimeControlPlaneStatus(fullStatus, scoped);
|
|
if (scoped.originalArgs.includes("--full") || scoped.originalArgs.includes("--raw")) return { ...fullStatus, summary };
|
|
return summary;
|
|
}
|
|
|
|
export function parseNodeRuntimeWorkloadReadiness(text: string): Array<Record<string, unknown>> {
|
|
return text.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => {
|
|
const [ref = "", counts = ""] = line.split(/\t/u);
|
|
const [readyRaw = "", currentRaw = "", desiredRaw = ""] = counts.split("/");
|
|
const readyReplicas = nullableInteger(readyRaw);
|
|
const currentReplicas = nullableInteger(currentRaw);
|
|
const desiredReplicas = nullableInteger(desiredRaw);
|
|
const ready = desiredReplicas !== null
|
|
? (readyReplicas ?? 0) >= desiredReplicas
|
|
: readyReplicas !== null && currentReplicas !== null && readyReplicas >= currentReplicas;
|
|
return {
|
|
ref,
|
|
readyReplicas,
|
|
currentReplicas,
|
|
desiredReplicas,
|
|
ready,
|
|
};
|
|
});
|
|
}
|
|
|
|
export function nullableInteger(value: string): number | null {
|
|
if (!/^[0-9]+$/u.test(value)) return null;
|
|
return Number(value);
|
|
}
|
|
|
|
export function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const web = publicHttpProbe("web", spec.publicWebUrl);
|
|
const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live"));
|
|
const ready = web.ok === true && apiHealth.ok === true;
|
|
const targetHost = nodeRuntimeTargetHostPublicProbeStatus(spec);
|
|
return {
|
|
ready,
|
|
web,
|
|
apiHealth,
|
|
targetHost,
|
|
diagnostic: nodeRuntimePublicProbeDiagnostic(ready, targetHost),
|
|
};
|
|
}
|
|
|
|
export function publicHttpProbe(kind: string, url: string): Record<string, unknown> {
|
|
const result = runCommand(["curl", "-k", "-sS", "--connect-timeout", "5", "--max-time", "12", "-o", "/dev/null", "-w", "%{http_code}", url], repoRoot, { timeoutMs: 15_000 });
|
|
const httpStatus = nullableInteger(result.stdout.trim().slice(-3));
|
|
return {
|
|
kind,
|
|
url,
|
|
ok: result.exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400,
|
|
httpStatus,
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeTargetHostPublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const webUrl = spec.publicWebUrl;
|
|
const apiHealthUrl = joinUrlPath(spec.publicApiUrl, "/health/live");
|
|
const script = [
|
|
"set -eu",
|
|
`web_url=${shellQuote(webUrl)}`,
|
|
`api_url=${shellQuote(apiHealthUrl)}`,
|
|
"probe() {",
|
|
" name=\"$1\"",
|
|
" url=\"$2\"",
|
|
" err_file=$(mktemp)",
|
|
" set +e",
|
|
" http_status=$(curl -k -sS --connect-timeout 5 --max-time 12 -o /dev/null -w '%{http_code}' \"$url\" 2>\"$err_file\")",
|
|
" rc=$?",
|
|
" set -e",
|
|
" error_text=$(tr '\\r\\n\\t' ' ' <\"$err_file\" | tail -c 600)",
|
|
" rm -f \"$err_file\"",
|
|
" printf '%sUrl\\t%s\\n' \"$name\" \"$url\"",
|
|
" printf '%sExitCode\\t%s\\n' \"$name\" \"$rc\"",
|
|
" printf '%sHttpStatus\\t%s\\n' \"$name\" \"$http_status\"",
|
|
" printf '%sError\\t%s\\n' \"$name\" \"$error_text\"",
|
|
"}",
|
|
"probe web \"$web_url\"",
|
|
"probe apiHealth \"$api_url\"",
|
|
].join("\n");
|
|
const result = runTransHostScript(spec.nodeId, script, "", 35);
|
|
const fields = keyValueLinesFromText(result.stdout);
|
|
const web = targetHostPublicHttpProbeFromFields("web", fields, webUrl, result.exitCode === 0);
|
|
const apiHealth = targetHostPublicHttpProbeFromFields("apiHealth", fields, apiHealthUrl, result.exitCode === 0);
|
|
return {
|
|
node: spec.nodeId,
|
|
ready: web.ok === true && apiHealth.ok === true,
|
|
probeAvailable: result.exitCode === 0 && result.timedOut !== true,
|
|
web,
|
|
apiHealth,
|
|
result: compactRuntimeCommand(result),
|
|
};
|
|
}
|
|
|
|
export function targetHostPublicHttpProbeFromFields(kind: string, fields: Record<string, string>, fallbackUrl: string, transportOk: boolean): Record<string, unknown> {
|
|
const exitCode = numericField(fields[`${kind}ExitCode`]);
|
|
const httpStatus = numericField(fields[`${kind}HttpStatus`]);
|
|
const error = fields[`${kind}Error`] ?? "";
|
|
return {
|
|
kind,
|
|
url: fields[`${kind}Url`] ?? fallbackUrl,
|
|
ok: transportOk && exitCode === 0 && httpStatus !== null && httpStatus >= 200 && httpStatus < 400,
|
|
httpStatus,
|
|
exitCode,
|
|
error: error.length > 0 ? error.slice(0, 600) : null,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimePublicProbeDiagnostic(publicReady: boolean, targetHost: Record<string, unknown>): Record<string, unknown> {
|
|
const targetWeb = record(targetHost.web);
|
|
const targetApiHealth = record(targetHost.apiHealth);
|
|
const targetReady = targetHost.ready === true;
|
|
if (!publicReady) {
|
|
return {
|
|
kind: "public-entry-probe-failed",
|
|
affectsUserEntry: true,
|
|
targetHostReady: targetReady,
|
|
message: "control-plane public probe failed; treat this as a public endpoint readiness failure before using web-probe closeout evidence.",
|
|
nextAction: "run web-probe run for the same node/lane after checking publicProbe.web and publicProbe.apiHealth",
|
|
};
|
|
}
|
|
if (targetHost.probeAvailable !== true) {
|
|
return {
|
|
kind: "target-host-public-probe-unavailable",
|
|
affectsUserEntry: false,
|
|
targetHostReady: false,
|
|
message: "control-plane public probe passed, but the target host diagnostic probe could not run; this does not invalidate user-entry evidence.",
|
|
nextAction: "use control-plane publicProbe and web-probe evidence for closeout; inspect target host SSH/trans health separately if host-side CLI must call the public URL",
|
|
};
|
|
}
|
|
if (!targetReady) {
|
|
return {
|
|
kind: "target-host-public-egress-mismatch",
|
|
affectsUserEntry: false,
|
|
targetHostReady: false,
|
|
failed: {
|
|
web: targetWeb.ok === true ? null : { httpStatus: targetWeb.httpStatus ?? null, exitCode: targetWeb.exitCode ?? null, error: targetWeb.error ?? null },
|
|
apiHealth: targetApiHealth.ok === true ? null : { httpStatus: targetApiHealth.httpStatus ?? null, exitCode: targetApiHealth.exitCode ?? null, error: targetApiHealth.error ?? null },
|
|
},
|
|
message: "control-plane public probe passed, but the target host cannot reach the same public URLs; classify this as target-host egress/hairpin diagnostics, not a public endpoint failure.",
|
|
nextAction: "use control-plane publicProbe and web-probe evidence for issue closeout; track host-side public URL access separately if hwlab-cli must run on that host",
|
|
};
|
|
}
|
|
return {
|
|
kind: "public-entry-and-target-host-ok",
|
|
affectsUserEntry: false,
|
|
targetHostReady: true,
|
|
message: "control-plane public probe and target-host public URL probe both passed.",
|
|
nextAction: null,
|
|
};
|
|
}
|
|
|
|
export function joinUrlPath(baseUrl: string, suffix: string): string {
|
|
return `${baseUrl.replace(/\/+$/u, "")}/${suffix.replace(/^\/+/u, "")}`;
|
|
}
|
|
|
|
export function compactNodeRuntimeTaskRunDiagnostic(value: unknown): string {
|
|
const item = record(value);
|
|
const pipelineTask = webObserveText(item.pipelineTask ?? item.pipelineTaskName);
|
|
const taskRun = webObserveText(item.taskRun ?? item.name);
|
|
const reason = webObserveText(item.reason ?? item.status ?? item.message);
|
|
const left = [pipelineTask, taskRun].filter(Boolean).join("/");
|
|
return [left, reason ? `(${webObserveShort(reason, 36)})` : ""].filter(Boolean).join("");
|
|
}
|
|
|
|
export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
|
|
const pipelineRun = record(status.pipelineRun);
|
|
const pipelineRunDiagnostics = record(status.pipelineRunDiagnostics);
|
|
const argo = record(status.argo);
|
|
const runtime = record(status.runtime);
|
|
const publicProbes = record(status.publicProbes);
|
|
const gitMirror = record(status.gitMirror);
|
|
const gitMirrorCompact = record(gitMirror.compact);
|
|
const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : [];
|
|
const readyWorkloads = workloadReadiness.filter((item) => item.ready === true).length;
|
|
const notReadyWorkloads = workloadReadiness.filter((item) => item.ready !== true).map((item) => item.ref).filter(Boolean);
|
|
const workloadCount = typeof runtime.workloadCount === "number" ? runtime.workloadCount : workloadReadiness.length;
|
|
const webProbe = record(publicProbes.web);
|
|
const apiProbe = record(publicProbes.apiHealth);
|
|
const targetHostProbe = record(publicProbes.targetHost);
|
|
const targetHostWebProbe = record(targetHostProbe.web);
|
|
const targetHostApiProbe = record(targetHostProbe.apiHealth);
|
|
const publicProbeDiagnostic = record(publicProbes.diagnostic);
|
|
return {
|
|
ok: status.ok === true,
|
|
command: status.command,
|
|
mode: "node-scoped-runtime-status-summary",
|
|
mutation: false,
|
|
node: status.node,
|
|
lane: status.lane,
|
|
sourceCommit: status.sourceCommit,
|
|
degradedReason: typeof status.degradedReason === "string" ? status.degradedReason : null,
|
|
pipelineRun: {
|
|
name: pipelineRun.name ?? null,
|
|
exists: pipelineRun.exists === true,
|
|
status: pipelineRun.status ?? null,
|
|
reason: pipelineRun.reason ?? null,
|
|
message: pipelineRun.message ?? null,
|
|
createdAt: pipelineRun.createdAt ?? null,
|
|
ready: pipelineRun.status === "True",
|
|
diagnostics: Object.keys(pipelineRunDiagnostics).length === 0 ? null : {
|
|
degradedReason: pipelineRunDiagnostics.degradedReason ?? null,
|
|
taskRunCount: pipelineRunDiagnostics.taskRunCount ?? null,
|
|
podCount: pipelineRunDiagnostics.podCount ?? null,
|
|
failedTaskRunCount: pipelineRunDiagnostics.failedTaskRunCount ?? null,
|
|
failedTaskRuns: pipelineRunDiagnostics.failedTaskRuns ?? [],
|
|
failureSummary: pipelineRunDiagnostics.failureSummary ?? null,
|
|
pendingTaskRuns: pipelineRunDiagnostics.pendingTaskRuns ?? [],
|
|
unscheduledPods: pipelineRunDiagnostics.unscheduledPods ?? [],
|
|
schedulingMessages: pipelineRunDiagnostics.schedulingMessages ?? [],
|
|
next: pipelineRunDiagnostics.next ?? null,
|
|
},
|
|
},
|
|
argo: {
|
|
application: argo.application ?? null,
|
|
ready: argo.ready === true,
|
|
syncRevision: argo.syncRevision ?? null,
|
|
syncStatus: argo.syncStatus ?? null,
|
|
health: argo.health ?? null,
|
|
},
|
|
runtime: {
|
|
namespace: runtime.namespace ?? null,
|
|
namespaceExists: runtime.namespaceExists === true,
|
|
ready: runtime.ready === true,
|
|
workloadCount,
|
|
workloadReady: `${readyWorkloads}/${workloadReadiness.length}`,
|
|
notReadyWorkloads,
|
|
localPostgresAbsent: runtime.localPostgresAbsent === true,
|
|
localPostgresExpectedAbsent: runtime.localPostgresExpectedAbsent === true,
|
|
localPostgresReady: runtime.localPostgresReady === true,
|
|
externalPostgresReady: runtime.externalPostgresBridge === undefined && runtime.externalPostgresSecrets === undefined
|
|
? null
|
|
: record(runtime.externalPostgresBridge).ready === true && record(runtime.externalPostgresSecrets).ready === true,
|
|
codeAgentRuntimeReady: record(runtime.codeAgentRuntime).required === true ? record(runtime.codeAgentRuntime).ready === true : null,
|
|
codeAgentRuntimeReason: typeof record(runtime.codeAgentRuntime).degradedReason === "string" ? record(runtime.codeAgentRuntime).degradedReason : null,
|
|
},
|
|
publicProbe: {
|
|
ready: publicProbes.ready === true,
|
|
web: { url: webProbe.url ?? null, ok: webProbe.ok === true, httpStatus: webProbe.httpStatus ?? null },
|
|
apiHealth: { url: apiProbe.url ?? null, ok: apiProbe.ok === true, httpStatus: apiProbe.httpStatus ?? null },
|
|
targetHost: Object.keys(targetHostProbe).length === 0 ? null : {
|
|
node: targetHostProbe.node ?? status.node ?? null,
|
|
ready: targetHostProbe.ready === true,
|
|
probeAvailable: targetHostProbe.probeAvailable === true,
|
|
web: {
|
|
url: targetHostWebProbe.url ?? null,
|
|
ok: targetHostWebProbe.ok === true,
|
|
httpStatus: targetHostWebProbe.httpStatus ?? null,
|
|
exitCode: targetHostWebProbe.exitCode ?? null,
|
|
error: targetHostWebProbe.error ?? null,
|
|
},
|
|
apiHealth: {
|
|
url: targetHostApiProbe.url ?? null,
|
|
ok: targetHostApiProbe.ok === true,
|
|
httpStatus: targetHostApiProbe.httpStatus ?? null,
|
|
exitCode: targetHostApiProbe.exitCode ?? null,
|
|
error: targetHostApiProbe.error ?? null,
|
|
},
|
|
},
|
|
diagnostic: Object.keys(publicProbeDiagnostic).length === 0 ? null : publicProbeDiagnostic,
|
|
},
|
|
gitMirror: {
|
|
ready: gitMirror.ready === true,
|
|
localSource: gitMirrorCompact.localSource ?? null,
|
|
githubSource: gitMirrorCompact.githubSource ?? null,
|
|
localGitops: gitMirrorCompact.localGitops ?? null,
|
|
githubGitops: gitMirrorCompact.githubGitops ?? null,
|
|
pendingFlush: gitMirrorCompact.pendingFlush === true,
|
|
flushNeeded: gitMirrorCompact.flushNeeded === true,
|
|
githubInSync: gitMirrorCompact.githubInSync === true,
|
|
},
|
|
nextAction: nodeRuntimeStatusNextAction(status, scoped),
|
|
next: {
|
|
full: `${nodeRuntimeStatusCommand(scoped)} --full`,
|
|
plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
|
|
triggerCurrent: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`,
|
|
gitMirrorFlush: `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`,
|
|
webProbe: `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimeStatusNextAction(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
|
|
const reason = typeof status.degradedReason === "string" ? status.degradedReason : null;
|
|
if (reason === null) return `${nodeRuntimeStatusCommand(scoped)} --full`;
|
|
if (reason === "control-plane-not-ready" || reason === "runtime-namespace-missing") {
|
|
return `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`;
|
|
}
|
|
if (reason === "runtime-not-ready") return `${nodeRuntimeStatusCommand(scoped)} --full`;
|
|
if (reason === "argo-not-synced-healthy") {
|
|
return `bun scripts/cli.ts hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane} --confirm`;
|
|
}
|
|
if (reason === "pipelinerun-not-succeeded") {
|
|
return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`;
|
|
}
|
|
if (reason === "node-runtime-ci-step-publish-failed") {
|
|
return `bun scripts/cli.ts platform-infra sub2api status --target ${scoped.node}`;
|
|
}
|
|
if (reason === "node-runtime-ci-taskrun-failed") {
|
|
const next = record(record(status.pipelineRunDiagnostics).next);
|
|
const failedStepLogs = typeof next.failedStepLogs === "string" ? next.failedStepLogs : null;
|
|
return failedStepLogs ?? `${nodeRuntimeStatusCommand(scoped)} --full`;
|
|
}
|
|
if (reason === "node-runtime-ci-pod-capacity-exhausted" || reason === "node-runtime-ci-pod-unschedulable") {
|
|
return `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${scoped.node} --lane ${scoped.lane} --min-age-minutes 60 --limit 20 --dry-run`;
|
|
}
|
|
if (reason === "public-probe-not-ready") {
|
|
return `bun scripts/cli.ts web-probe run --node ${scoped.node} --lane ${scoped.lane}`;
|
|
}
|
|
if (reason === "git-mirror-pending-flush") {
|
|
return `bun scripts/cli.ts hwlab nodes git-mirror flush --node ${scoped.node} --lane ${scoped.lane} --confirm --wait`;
|
|
}
|
|
return `${nodeRuntimeStatusCommand(scoped)} --full`;
|
|
}
|
|
|
|
export function nodeRuntimeStatusCommand(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): string {
|
|
const sourceCommit = optionValue(scoped.originalArgs, "--source-commit");
|
|
const pipelineRun = optionValue(scoped.originalArgs, "--pipeline-run");
|
|
return [
|
|
`bun scripts/cli.ts hwlab nodes control-plane status --node ${shellQuote(scoped.node)} --lane ${shellQuote(scoped.lane)}`,
|
|
sourceCommit === undefined ? "" : `--source-commit ${shellQuote(sourceCommit)}`,
|
|
pipelineRun === undefined ? "" : `--pipeline-run ${shellQuote(pipelineRun)}`,
|
|
].filter(Boolean).join(" ");
|
|
}
|
|
|
|
export function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pipelineRun: string): Record<string, unknown> {
|
|
const taskRunTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/pipelineTask"}}{{"\\t"}}{{if .spec.taskRef}}{{.spec.taskRef.name}}{{end}}{{"\\t"}}{{with index .status.conditions 0}}{{.status}}{{"\\t"}}{{.reason}}{{else}}{{"\\t"}}{{end}}{{"\\t"}}{{.status.podName}}{{"\\t"}}{{with index .status.conditions 0}}{{printf "%.600s" .message}}{{end}}{{"\\n"}}{{end}}`;
|
|
const podTemplate = `{{range .items}}{{.metadata.name}}{{"\\t"}}{{index .metadata.labels "tekton.dev/taskRun"}}{{"\\t"}}{{.status.phase}}{{"\\t"}}{{.spec.nodeName}}{{"\\t"}}{{range .status.conditions}}{{if eq .type "PodScheduled"}}{{.status}}|{{.reason}}|{{printf "%.600s" .message}}{{end}}{{end}}{{"\\t"}}{{range .status.initContainerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\t"}}{{range .status.containerStatuses}}{{.name}}:{{if .state.terminated}}{{.state.terminated.exitCode}}:{{.state.terminated.reason}}{{else if .state.waiting}}waiting:{{.state.waiting.reason}}{{else if .state.running}}running:{{.state.running.startedAt}}{{end}},{{end}}{{"\\n"}}{{end}}`;
|
|
const taskRunsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "taskrun", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${taskRunTemplate}`], 60);
|
|
const podsResult = runNodeK3sArgs(spec, ["kubectl", "-n", HWLAB_CI_NAMESPACE, "get", "pod", "-l", `tekton.dev/pipelineRun=${pipelineRun}`, "-o", `go-template=${podTemplate}`], 60);
|
|
const taskRuns = isCommandSuccess(taskRunsResult) ? nodeRuntimePipelineDiagnosticTaskRunsFromTsv(taskRunsResult.stdout) : [];
|
|
const pods = isCommandSuccess(podsResult) ? nodeRuntimePipelineDiagnosticPodsFromTsv(podsResult.stdout) : [];
|
|
const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False");
|
|
const failedTaskRuns = taskRuns.filter((item) => item.status === "False");
|
|
const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods);
|
|
const stepPublishFailures = failedTaskRunSummaries.filter((item) => item.container === "step-publish" || item.step === "publish" || item.step === "step-publish");
|
|
const unscheduledPods = pods.filter((item) => item.scheduled === false);
|
|
const schedulingMessages = unscheduledPods
|
|
.map((item) => typeof item.scheduledMessage === "string" ? item.scheduledMessage : "")
|
|
.filter((message) => message.length > 0);
|
|
const tooManyPods = schedulingMessages.some((message) => /too many pods/iu.test(message));
|
|
const failureSummary = failedTaskRunSummaries.length > 0
|
|
? {
|
|
failedTaskRunCount: failedTaskRuns.length,
|
|
failedStepCount: failedTaskRunSummaries.length,
|
|
stepPublishFailureCount: stepPublishFailures.length,
|
|
firstFailure: failedTaskRunSummaries[0] ?? null,
|
|
stepFailures: failedTaskRunSummaries.slice(0, 8),
|
|
nextAction: stepPublishFailures.length > 0
|
|
? "step-publish failed in a service build; first distinguish platform-infra Sub2API/proxy health from a single upstream transient, then choose controlled rerun or artifact-publish/envRecipe retry fix."
|
|
: failedTaskRunSummaries.length > 0
|
|
? "Inspect the failed TaskRun and bounded pod log command before rerunning the control-plane trigger."
|
|
: null,
|
|
}
|
|
: null;
|
|
return {
|
|
ok: taskRunsResult.exitCode === 0 && podsResult.exitCode === 0,
|
|
pipelineRun,
|
|
taskRuns,
|
|
pods,
|
|
taskRunCount: taskRuns.length,
|
|
podCount: pods.length,
|
|
failedTaskRunCount: failedTaskRuns.length,
|
|
failedTaskRuns: failedTaskRunSummaries,
|
|
stepPublishFailures,
|
|
failureSummary,
|
|
pendingTaskRuns,
|
|
unscheduledPods,
|
|
schedulingMessages,
|
|
degradedReason: tooManyPods
|
|
? "node-runtime-ci-pod-capacity-exhausted"
|
|
: unscheduledPods.length > 0
|
|
? "node-runtime-ci-pod-unschedulable"
|
|
: stepPublishFailures.length > 0
|
|
? "node-runtime-ci-step-publish-failed"
|
|
: failedTaskRunSummaries.length > 0
|
|
? "node-runtime-ci-taskrun-failed"
|
|
: pendingTaskRuns.length > 0
|
|
? "node-runtime-ci-taskrun-pending"
|
|
: undefined,
|
|
query: {
|
|
taskRuns: compactRuntimeCommand(taskRunsResult),
|
|
pods: compactRuntimeCommand(podsResult),
|
|
},
|
|
next: tooManyPods || unscheduledPods.length > 0
|
|
? { cleanupRuns: `bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node ${spec.nodeId} --lane ${spec.lane} --min-age-minutes 60 --limit 20 --dry-run` }
|
|
: stepPublishFailures.length > 0
|
|
? {
|
|
sub2apiStatus: `bun scripts/cli.ts platform-infra sub2api status --target ${spec.nodeId}`,
|
|
sub2apiValidate: `bun scripts/cli.ts platform-infra sub2api validate --target ${spec.nodeId}`,
|
|
failedStepLogs: stepPublishFailures[0]?.logCommand ?? null,
|
|
rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${spec.nodeId} --lane ${spec.lane} --confirm --rerun`,
|
|
}
|
|
: failedTaskRunSummaries.length > 0
|
|
? {
|
|
failedStepLogs: failedTaskRunSummaries[0]?.logCommand ?? null,
|
|
failedTaskRun: failedTaskRunSummaries[0]?.taskRunCommand ?? null,
|
|
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
|
|
}
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
export function nodeRuntimePipelineDiagnosticTaskRunsFromTsv(text: string): Array<Record<string, unknown>> {
|
|
return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => {
|
|
const parts = line.split("\t");
|
|
const [name = "", pipelineTask = "", taskRef = "", status = "", reason = "", podName = "", ...messageParts] = parts;
|
|
const message = messageParts.join("\t");
|
|
return {
|
|
name: stringOrNull(name),
|
|
pipelineTask: stringOrNull(pipelineTask),
|
|
taskRef: stringOrNull(taskRef),
|
|
status: stringOrNull(status),
|
|
reason: stringOrNull(reason),
|
|
message: diagnosticText(message),
|
|
podName: stringOrNull(podName),
|
|
steps: [],
|
|
failedSteps: [],
|
|
};
|
|
});
|
|
}
|
|
|
|
export function nodeRuntimePipelineDiagnosticPodsFromTsv(text: string): Array<Record<string, unknown>> {
|
|
return text.split(/\r?\n/u).map((line) => line.trimEnd()).filter(Boolean).map((line) => {
|
|
const [name = "", taskRun = "", phase = "", nodeName = "", scheduledRaw = "", initRaw = "", containersRaw = ""] = line.split("\t");
|
|
const [scheduledStatus = "", scheduledReason = "", scheduledMessage = ""] = scheduledRaw.split("|");
|
|
const initContainers = nodeRuntimePipelineContainerStatusesFromCompact(initRaw);
|
|
const containers = nodeRuntimePipelineContainerStatusesFromCompact(containersRaw);
|
|
const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true);
|
|
return {
|
|
name: stringOrNull(name),
|
|
taskRun: stringOrNull(taskRun),
|
|
phase: stringOrNull(phase),
|
|
nodeName: stringOrNull(nodeName),
|
|
scheduled: scheduledStatus.length === 0 ? null : scheduledStatus === "True",
|
|
scheduledReason: stringOrNull(scheduledReason),
|
|
scheduledMessage: diagnosticText(scheduledMessage),
|
|
initContainers,
|
|
containers,
|
|
failedContainers,
|
|
};
|
|
});
|
|
}
|
|
|
|
export function nodeRuntimePipelineContainerStatusesFromCompact(text: string): Array<Record<string, unknown>> {
|
|
return text.split(",").map((item) => item.trim()).filter(Boolean).map((item) => {
|
|
const [name = "", stateOrExit = "", reason = ""] = item.split(":");
|
|
const exitCode = /^[0-9]+$/u.test(stateOrExit) ? Number(stateOrExit) : null;
|
|
const state = exitCode !== null ? "terminated" : stateOrExit === "waiting" ? "waiting" : stateOrExit === "running" ? "running" : null;
|
|
return {
|
|
name: stringOrNull(name),
|
|
ready: null,
|
|
restartCount: null,
|
|
state,
|
|
failed: exitCode !== null && exitCode !== 0,
|
|
exitCode,
|
|
reason: stringOrNull(reason),
|
|
message: null,
|
|
startedAt: null,
|
|
finishedAt: null,
|
|
};
|
|
});
|
|
}
|
|
|
|
export function nodeRuntimePipelineDiagnosticTaskRuns(json: Record<string, unknown>): Array<Record<string, unknown>> {
|
|
const items = Array.isArray(json.items) ? json.items.map(record) : [];
|
|
return items.map((item) => {
|
|
const metadata = record(item.metadata);
|
|
const labels = record(metadata.labels);
|
|
const spec = record(item.spec);
|
|
const taskRef = record(spec.taskRef);
|
|
const status = record(item.status);
|
|
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
|
|
const condition = conditions[0] ?? {};
|
|
const steps = nodeRuntimePipelineDiagnosticSteps(status.steps);
|
|
const failedSteps = steps.filter((step) => step.failed === true);
|
|
return {
|
|
name: metadata.name ?? null,
|
|
pipelineTask: labels["tekton.dev/pipelineTask"] ?? null,
|
|
taskRef: taskRef.name ?? null,
|
|
status: condition.status ?? null,
|
|
reason: condition.reason ?? null,
|
|
message: diagnosticText(condition.message),
|
|
podName: status.podName ?? null,
|
|
steps,
|
|
failedSteps,
|
|
};
|
|
});
|
|
}
|
|
|
|
export function nodeRuntimePipelineDiagnosticPods(json: Record<string, unknown>): Array<Record<string, unknown>> {
|
|
const items = Array.isArray(json.items) ? json.items.map(record) : [];
|
|
return items.map((item) => {
|
|
const metadata = record(item.metadata);
|
|
const labels = record(metadata.labels);
|
|
const spec = record(item.spec);
|
|
const status = record(item.status);
|
|
const conditions = Array.isArray(status.conditions) ? status.conditions.map(record) : [];
|
|
const scheduled = conditions.find((condition) => condition.type === "PodScheduled");
|
|
const initContainers = nodeRuntimeContainerStatusSummaries(status.initContainerStatuses);
|
|
const containers = nodeRuntimeContainerStatusSummaries(status.containerStatuses);
|
|
const failedContainers = [...initContainers, ...containers].filter((container) => container.failed === true);
|
|
return {
|
|
name: metadata.name ?? null,
|
|
taskRun: labels["tekton.dev/taskRun"] ?? null,
|
|
phase: status.phase ?? null,
|
|
nodeName: spec.nodeName ?? null,
|
|
scheduled: scheduled === undefined ? null : scheduled.status === "True",
|
|
scheduledReason: scheduled?.reason ?? null,
|
|
scheduledMessage: diagnosticText(scheduled?.message),
|
|
initContainers,
|
|
containers,
|
|
failedContainers,
|
|
};
|
|
});
|
|
}
|
|
|
|
export function nodeRuntimePipelineDiagnosticSteps(value: unknown): Array<Record<string, unknown>> {
|
|
const steps = Array.isArray(value) ? value.map(record) : [];
|
|
return steps.map((step) => {
|
|
const terminated = record(step.terminated);
|
|
const running = record(step.running);
|
|
const waiting = record(step.waiting);
|
|
const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
|
|
const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
|
|
const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
|
|
const state = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
|
|
return {
|
|
name: step.name ?? null,
|
|
container: step.container ?? null,
|
|
state,
|
|
failed: exitCode !== null && exitCode !== 0,
|
|
exitCode,
|
|
reason: terminatedReason ?? waitingReason,
|
|
message: diagnosticText(terminated.message ?? waiting.message),
|
|
startedAt: terminated.startedAt ?? running.startedAt ?? null,
|
|
finishedAt: terminated.finishedAt ?? null,
|
|
};
|
|
});
|
|
}
|
|
|
|
export function nodeRuntimeContainerStatusSummaries(value: unknown): Array<Record<string, unknown>> {
|
|
const containers = Array.isArray(value) ? value.map(record) : [];
|
|
return containers.map((container) => {
|
|
const state = record(container.state);
|
|
const terminated = record(state.terminated);
|
|
const waiting = record(state.waiting);
|
|
const running = record(state.running);
|
|
const exitCode = typeof terminated.exitCode === "number" ? terminated.exitCode : null;
|
|
const terminatedReason = typeof terminated.reason === "string" ? terminated.reason : null;
|
|
const waitingReason = typeof waiting.reason === "string" ? waiting.reason : null;
|
|
const stateName = Object.keys(terminated).length > 0 ? "terminated" : Object.keys(waiting).length > 0 ? "waiting" : Object.keys(running).length > 0 ? "running" : null;
|
|
return {
|
|
name: container.name ?? null,
|
|
ready: container.ready === true,
|
|
restartCount: typeof container.restartCount === "number" ? container.restartCount : null,
|
|
state: stateName,
|
|
failed: exitCode !== null && exitCode !== 0,
|
|
exitCode,
|
|
reason: terminatedReason ?? waitingReason,
|
|
message: diagnosticText(terminated.message ?? waiting.message),
|
|
startedAt: terminated.startedAt ?? running.startedAt ?? null,
|
|
finishedAt: terminated.finishedAt ?? null,
|
|
};
|
|
});
|
|
}
|
|
|
|
export function nodeRuntimePipelineFailedTaskRunSummaries(
|
|
spec: HwlabRuntimeLaneSpec,
|
|
failedTaskRuns: Array<Record<string, unknown>>,
|
|
pods: Array<Record<string, unknown>>,
|
|
): Array<Record<string, unknown>> {
|
|
const summaries: Array<Record<string, unknown>> = [];
|
|
for (const taskRun of failedTaskRuns) {
|
|
const taskRunName = stringOrNull(taskRun.name);
|
|
const podName = stringOrNull(taskRun.podName);
|
|
const pod = pods.find((item) => item.name === podName || (taskRunName !== null && item.taskRun === taskRunName)) ?? {};
|
|
const failedSteps = Array.isArray(taskRun.failedSteps) ? taskRun.failedSteps.map(record) : [];
|
|
const failedContainers = Array.isArray(pod.failedContainers) ? pod.failedContainers.map(record) : [];
|
|
const failures = failedSteps.length > 0
|
|
? failedSteps
|
|
: failedContainers.map((container) => ({
|
|
name: typeof container.name === "string" && container.name.startsWith("step-") ? container.name.slice("step-".length) : container.name ?? null,
|
|
container: container.name ?? null,
|
|
state: container.state ?? null,
|
|
exitCode: container.exitCode ?? null,
|
|
reason: container.reason ?? null,
|
|
message: container.message ?? null,
|
|
}));
|
|
const effectiveFailures = failures.length > 0 ? failures : [{ name: null, container: null, state: null, exitCode: null, reason: taskRun.reason ?? null, message: taskRun.message ?? null }];
|
|
for (const failure of effectiveFailures) {
|
|
const stepName = stringOrNull(failure.name);
|
|
const containerName = stringOrNull(failure.container) ?? (stepName === null ? null : `step-${stepName}`);
|
|
summaries.push({
|
|
taskRun: taskRunName,
|
|
pipelineTask: taskRun.pipelineTask ?? null,
|
|
taskRef: taskRun.taskRef ?? null,
|
|
taskRunStatus: taskRun.status ?? null,
|
|
taskRunReason: taskRun.reason ?? null,
|
|
taskRunMessage: diagnosticText(taskRun.message),
|
|
pod: podName,
|
|
podPhase: pod.phase ?? null,
|
|
nodeName: pod.nodeName ?? null,
|
|
step: stepName,
|
|
container: containerName,
|
|
containerState: failure.state ?? null,
|
|
terminationReason: failure.reason ?? null,
|
|
exitCode: typeof failure.exitCode === "number" ? failure.exitCode : null,
|
|
message: diagnosticText(failure.message),
|
|
logCommand: podName === null ? null : nodeRuntimePipelineLogsCommand(spec, podName, containerName),
|
|
taskRunCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["get", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName, "-o", "yaml"]),
|
|
taskRunDescribeCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName]),
|
|
podDescribeCommand: podName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "pod", "-n", HWLAB_CI_NAMESPACE, podName]),
|
|
});
|
|
}
|
|
}
|
|
return summaries.slice(0, 16);
|
|
}
|
|
|
|
export function nodeRuntimePipelineFailureSummary(value: unknown): Record<string, unknown> | null {
|
|
const recordValue = record(value);
|
|
const direct = record(recordValue.failureSummary);
|
|
if (Object.keys(direct).length > 0) return direct;
|
|
const diagnostics = record(recordValue.diagnostics);
|
|
const fromDiagnostics = record(diagnostics.failureSummary);
|
|
return Object.keys(fromDiagnostics).length > 0 ? fromDiagnostics : null;
|
|
}
|
|
|
|
export function nodeRuntimePipelineLogsCommand(spec: HwlabRuntimeLaneSpec, podName: string, containerName: string | null): string {
|
|
return nodeRuntimeK3sCommand(spec, [
|
|
"logs",
|
|
"--namespace", HWLAB_CI_NAMESPACE,
|
|
"--pod", podName,
|
|
...(containerName === null ? ["--all-containers"] : ["--container", containerName]),
|
|
"--tail", "200",
|
|
]);
|
|
}
|
|
|
|
export function nodeRuntimeK3sCommand(spec: HwlabRuntimeLaneSpec, args: string[]): string {
|
|
return ["trans", spec.nodeKubeRoute, ...args].map(shellQuote).join(" ");
|
|
}
|
|
|
|
export function stringOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
export function diagnosticText(value: unknown): string | null {
|
|
if (typeof value !== "string") return null;
|
|
const trimmed = value.trim();
|
|
if (trimmed.length === 0) return null;
|
|
return trimmed
|
|
.replace(/postgres(?:ql)?:\/\/[^\s"'`]+/giu, "postgres://<redacted>")
|
|
.replace(/\b([A-Za-z0-9_.-]*(?:TOKEN|PASSWORD|SECRET|API_KEY|DATABASE_URL)[A-Za-z0-9_.-]*)=([^\s"'`]+)/giu, "$1=<redacted>")
|
|
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gu, "Bearer <redacted>")
|
|
.slice(0, 600);
|
|
}
|
|
|
|
export function nodeRuntimeRenderToken(): string {
|
|
return `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(16).slice(2, 10)}`.replace(/[^A-Za-z0-9_.-]/gu, "-");
|
|
}
|
|
|
|
export function renderNodeRuntimeControlPlane(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
|
|
if (shouldRenderNodeRuntimeControlPlaneLocally(spec)) return renderNodeRuntimeControlPlaneLocal(spec, sourceCommit, timeoutSeconds);
|
|
return renderNodeRuntimeControlPlaneOnNode(spec, sourceCommit, timeoutSeconds);
|
|
}
|
|
|
|
export function shouldRenderNodeRuntimeControlPlaneLocally(spec: HwlabRuntimeLaneSpec): boolean {
|
|
return hwlabRuntimeLaneSpec(spec.lane).nodeId !== spec.nodeId;
|
|
}
|
|
|
|
export function yamlDependencyInstallScript(registry: string, fetchTimeoutSeconds: number, retries: number, context: string): string[] {
|
|
const timeoutSeconds = Math.max(15, Math.ceil(fetchTimeoutSeconds));
|
|
const retryCount = Math.max(0, Math.floor(retries));
|
|
const maxAttempts = retryCount + 1;
|
|
const safeContext = context.replace(/[^A-Za-z0-9_.-]/gu, "-");
|
|
return [
|
|
`yaml_registry=${shellQuote(registry)}`,
|
|
`yaml_fetch_timeout=${shellQuote(String(timeoutSeconds))}`,
|
|
`yaml_fetch_retries=${shellQuote(String(retryCount))}`,
|
|
`yaml_max_attempts=${shellQuote(String(maxAttempts))}`,
|
|
`yaml_dependency_context=${shellQuote(safeContext)}`,
|
|
"yaml_dependency_log() { echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"'\"$1\"'\",\"manager\":\"'\"${2:-}\"'\"}' >&2; }",
|
|
"yaml_prepare_node_package_proxy() {",
|
|
" yaml_http_proxy=\"${HTTP_PROXY:-${http_proxy:-${ALL_PROXY:-${all_proxy:-}}}}\"",
|
|
" yaml_https_proxy=\"${HTTPS_PROXY:-${https_proxy:-${yaml_http_proxy}}}\"",
|
|
" yaml_all_proxy=\"${ALL_PROXY:-${all_proxy:-${yaml_http_proxy}}}\"",
|
|
" if [ -n \"$yaml_http_proxy\" ]; then export HTTP_PROXY=\"$yaml_http_proxy\" http_proxy=\"$yaml_http_proxy\" npm_config_proxy=\"$yaml_http_proxy\"; fi",
|
|
" if [ -n \"$yaml_https_proxy\" ]; then export HTTPS_PROXY=\"$yaml_https_proxy\" https_proxy=\"$yaml_https_proxy\" npm_config_https_proxy=\"$yaml_https_proxy\"; fi",
|
|
" if [ -n \"$yaml_all_proxy\" ]; then export ALL_PROXY=\"$yaml_all_proxy\" all_proxy=\"$yaml_all_proxy\"; fi",
|
|
" export npm_config_registry=\"$yaml_registry\"",
|
|
" export BUN_CONFIG_REGISTRY=\"$yaml_registry\"",
|
|
" export npm_config_noproxy=\"${NO_PROXY:-${no_proxy:-}}\"",
|
|
" export npm_config_fetch_retries=\"$yaml_fetch_retries\"",
|
|
" export npm_config_fetch_retry_mintimeout=2000",
|
|
" export npm_config_fetch_retry_maxtimeout=16000",
|
|
" export npm_config_fetch_timeout=$((yaml_fetch_timeout * 1000))",
|
|
"}",
|
|
"yaml_run_dependency_manager() {",
|
|
" yaml_manager=\"$1\"; shift",
|
|
" yaml_attempt=1",
|
|
" yaml_delay=2",
|
|
" while [ \"$yaml_attempt\" -le \"$yaml_max_attempts\" ]; do",
|
|
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"attempt\",\"manager\":\"'\"$yaml_manager\"'\",\"attempt\":\"'\"$yaml_attempt/$yaml_max_attempts\"'\",\"proxy\":\"'\"${yaml_https_proxy:-$yaml_http_proxy}\"'\"}' >&2",
|
|
" if timeout \"$yaml_fetch_timeout\" \"$@\"; then return 0; fi",
|
|
" if [ \"$yaml_attempt\" -ge \"$yaml_max_attempts\" ]; then return 1; fi",
|
|
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"retrying\",\"manager\":\"'\"$yaml_manager\"'\",\"attempt\":\"'\"$yaml_attempt/$yaml_max_attempts\"'\",\"sleepSeconds\":'\"$yaml_delay\"'}' >&2",
|
|
" sleep \"$yaml_delay\"",
|
|
" yaml_delay=$((yaml_delay * 2))",
|
|
" yaml_attempt=$((yaml_attempt + 1))",
|
|
" done",
|
|
" return 1",
|
|
"}",
|
|
"yaml_npm_debug_log_tail() {",
|
|
" yaml_npm_log_dir=\"${HOME:-/tmp}/.npm/_logs\"",
|
|
" if [ ! -d \"$yaml_npm_log_dir\" ]; then return 0; fi",
|
|
" yaml_npm_log=\"$(find \"$yaml_npm_log_dir\" -type f -name '*debug*.log' | sort | tail -n 1 || true)\"",
|
|
" if [ -n \"$yaml_npm_log\" ] && [ -f \"$yaml_npm_log\" ]; then",
|
|
" echo '{\"event\":\"yaml-dependency\",\"context\":\"'\"$yaml_dependency_context\"'\",\"status\":\"npm-debug-log\",\"path\":\"'\"$yaml_npm_log\"'\"}' >&2",
|
|
" tail -n 80 \"$yaml_npm_log\" >&2 || true",
|
|
" fi",
|
|
"}",
|
|
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
|
|
" yaml_prepare_node_package_proxy",
|
|
"fi",
|
|
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 && command -v bun >/dev/null 2>&1; then",
|
|
" rm -rf node_modules/yaml",
|
|
" if yaml_run_dependency_manager bun bun add --no-save --ignore-scripts --registry \"$yaml_registry\" yaml@2.8.3; then",
|
|
" yaml_dependency_log installed bun",
|
|
" else",
|
|
" yaml_dependency_log bun-failed bun",
|
|
" fi",
|
|
"fi",
|
|
"if ! node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1; then",
|
|
" rm -rf node_modules/yaml",
|
|
" command -v npm >/dev/null 2>&1 || { yaml_dependency_log failed missing-tool; exit 31; }",
|
|
" if yaml_run_dependency_manager npm npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev --registry \"$yaml_registry\" yaml@2.8.3; then",
|
|
" yaml_dependency_log installed npm",
|
|
" else",
|
|
" yaml_dependency_log npm-failed npm",
|
|
" yaml_npm_debug_log_tail",
|
|
" exit 34",
|
|
" fi",
|
|
"fi",
|
|
"node -e 'require.resolve(\"yaml\")' >/dev/null 2>&1 || { yaml_dependency_log failed unresolved; exit 34; }",
|
|
];
|
|
}
|
|
|
|
export function renderNodeRuntimeControlPlaneOnNode(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
|
|
const token = nodeRuntimeRenderToken();
|
|
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
|
|
const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
|
|
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
runtimeLaneCicdRepoEnsureScript(spec),
|
|
`source_commit=${shellQuote(sourceCommit)}`,
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
`worktree_dir=${shellQuote(worktreeDir)}`,
|
|
`overlay_b64=${shellQuote(overlay)}`,
|
|
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
|
|
"trap cleanup_render_worktree EXIT",
|
|
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = "$source_commit"`,
|
|
"rm -rf \"$render_dir\" \"$worktree_dir\"",
|
|
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
|
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
|
|
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
|
|
"cd \"$worktree_dir\"",
|
|
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "control-plane-render"),
|
|
"node - \"$overlay_b64\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
|
|
"const path = 'deploy/deploy.yaml';",
|
|
"const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
|
|
"doc.nodes = doc.nodes || {};",
|
|
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
|
|
"doc.lanes = doc.lanes || {};",
|
|
"const lane = doc.lanes[overlay.lane] || {};",
|
|
"const downloadStack = {",
|
|
" ...(lane.envRecipe?.downloadStack || {}),",
|
|
" httpProxy: overlay.dockerProxyHttp,",
|
|
" httpsProxy: overlay.dockerProxyHttps,",
|
|
" noProxy: overlay.dockerNoProxyList,",
|
|
"};",
|
|
"doc.lanes[overlay.lane] = {",
|
|
" ...lane,",
|
|
" node: overlay.nodeId,",
|
|
" sourceBranch: overlay.sourceBranch,",
|
|
" gitopsBranch: overlay.gitopsBranch,",
|
|
" namespace: overlay.runtimeNamespace,",
|
|
" endpoint: overlay.publicApiUrl,",
|
|
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
|
|
" artifactCatalog: overlay.catalogPath,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" imageTagMode: 'full',",
|
|
" sourceRepo: overlay.gitUrl,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" observability: overlay.observability,",
|
|
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
|
|
"};",
|
|
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
|
|
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
|
|
"if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;",
|
|
"fs.writeFileSync(path, YAML.stringify(doc));",
|
|
"NODE",
|
|
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
|
|
[
|
|
"node scripts/run-bun.mjs \"$render_script\"",
|
|
`--lane ${shellQuote(spec.lane)}`,
|
|
`--node ${shellQuote(spec.nodeId)}`,
|
|
`--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
|
|
`--catalog-path ${shellQuote(spec.catalogPath)}`,
|
|
"--image-tag-mode full",
|
|
`--source-revision ${shellQuote(sourceCommit)}`,
|
|
`--source-repo ${shellQuote(spec.gitUrl)}`,
|
|
`--source-branch ${shellQuote(spec.sourceBranch)}`,
|
|
`--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
|
|
`--git-read-url ${shellQuote(spec.gitReadUrl)}`,
|
|
`--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
|
|
`--registry-prefix ${shellQuote(spec.registryPrefix)}`,
|
|
`--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
|
|
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
|
|
`--out ${shellQuote(renderDir)}`,
|
|
].join(" "),
|
|
...nodeRuntimePipelinePostprocessScript(),
|
|
].join("\n");
|
|
return { result: runNodeHostScriptAsync(spec, script, timeoutSeconds, `${spec.nodeId.toLowerCase()}-${spec.lane}-render`), renderDir, worktreeDir, location: "node-host" };
|
|
}
|
|
|
|
export function renderNodeRuntimeControlPlaneLocal(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): NodeRuntimeRenderResult {
|
|
const token = nodeRuntimeRenderToken();
|
|
const renderDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-control-plane-${shortSha(sourceCommit)}-${token}`;
|
|
const worktreeDir = `/tmp/hwlab-${spec.nodeId.toLowerCase()}-${spec.lane}-source-${shortSha(sourceCommit)}-${token}`;
|
|
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
|
|
const gitTimeoutSeconds = Math.max(30, spec.downloadProfile.git.timeoutSeconds);
|
|
const script = [
|
|
"set -eu",
|
|
`source_url=${shellQuote(spec.gitUrl)}`,
|
|
`source_branch=${shellQuote(spec.sourceBranch)}`,
|
|
`source_commit=${shellQuote(sourceCommit)}`,
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
`worktree_dir=${shellQuote(worktreeDir)}`,
|
|
`overlay_b64=${shellQuote(overlay)}`,
|
|
`git_timeout=${shellQuote(String(gitTimeoutSeconds))}`,
|
|
"run_git() { if command -v timeout >/dev/null 2>&1; then timeout \"$git_timeout\" git -c protocol.version=2 \"$@\"; else git -c protocol.version=2 \"$@\"; fi; }",
|
|
"rm -rf \"$render_dir\" \"$worktree_dir\"",
|
|
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
|
"echo \"phase=local-git-clone-worktree\" >&2",
|
|
"run_git clone --depth 1 --single-branch --branch \"$source_branch\" \"$source_url\" \"$worktree_dir\"",
|
|
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
|
|
"cd \"$worktree_dir\"",
|
|
"echo \"phase=local-install-yaml\" >&2",
|
|
...yamlDependencyInstallScript(spec.downloadProfile.npm.registry, spec.downloadProfile.npm.fetchTimeoutSeconds, spec.downloadProfile.npm.retries, "local-control-plane-render"),
|
|
"node - \"$overlay_b64\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = JSON.parse(Buffer.from(process.argv[2], 'base64').toString('utf8'));",
|
|
"const path = 'deploy/deploy.yaml';",
|
|
"const doc = YAML.parse(fs.readFileSync(path, 'utf8'));",
|
|
"doc.nodes = doc.nodes || {};",
|
|
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
|
|
"doc.lanes = doc.lanes || {};",
|
|
"const lane = doc.lanes[overlay.lane] || {};",
|
|
"const downloadStack = {",
|
|
" ...(lane.envRecipe?.downloadStack || {}),",
|
|
" httpProxy: overlay.dockerProxyHttp,",
|
|
" httpsProxy: overlay.dockerProxyHttps,",
|
|
" noProxy: overlay.dockerNoProxyList,",
|
|
"};",
|
|
"doc.lanes[overlay.lane] = {",
|
|
" ...lane,",
|
|
" node: overlay.nodeId,",
|
|
" sourceBranch: overlay.sourceBranch,",
|
|
" gitopsBranch: overlay.gitopsBranch,",
|
|
" namespace: overlay.runtimeNamespace,",
|
|
" endpoint: overlay.publicApiUrl,",
|
|
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
|
|
" artifactCatalog: overlay.catalogPath,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" imageTagMode: 'full',",
|
|
" sourceRepo: overlay.gitUrl,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" observability: overlay.observability,",
|
|
" envRecipe: { ...(lane.envRecipe || {}), downloadStack },",
|
|
"};",
|
|
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
|
|
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
|
|
"if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;",
|
|
"fs.writeFileSync(path, YAML.stringify(doc));",
|
|
"NODE",
|
|
"if [ -f scripts/gitops-render.mjs ]; then render_script=scripts/gitops-render.mjs; else echo 'render script missing: scripts/gitops-render.mjs' >&2; exit 43; fi",
|
|
"echo \"phase=local-gitops-render\" >&2",
|
|
[
|
|
"node scripts/run-bun.mjs \"$render_script\"",
|
|
`--lane ${shellQuote(spec.lane)}`,
|
|
`--node ${shellQuote(spec.nodeId)}`,
|
|
`--gitops-root ${shellQuote(nodeRuntimeGitopsRoot(spec))}`,
|
|
`--catalog-path ${shellQuote(spec.catalogPath)}`,
|
|
"--image-tag-mode full",
|
|
`--source-revision ${shellQuote(sourceCommit)}`,
|
|
`--source-repo ${shellQuote(spec.gitUrl)}`,
|
|
`--source-branch ${shellQuote(spec.sourceBranch)}`,
|
|
`--gitops-branch ${shellQuote(spec.gitopsBranch)}`,
|
|
`--git-read-url ${shellQuote(spec.gitReadUrl)}`,
|
|
`--git-write-url ${shellQuote(spec.gitWriteUrl)}`,
|
|
`--registry-prefix ${shellQuote(spec.registryPrefix)}`,
|
|
`--runtime-endpoint ${shellQuote(spec.publicApiUrl)}`,
|
|
`--web-endpoint ${shellQuote(spec.publicWebUrl)}`,
|
|
`--out ${shellQuote(renderDir)}`,
|
|
].join(" "),
|
|
...nodeRuntimePipelinePostprocessScript(),
|
|
].join("\n");
|
|
return { result: runCommand(["bash", "-lc", script], repoRoot, { timeoutMs: timeoutSeconds * 1000 }), renderDir, worktreeDir, location: "local" };
|
|
}
|
|
|
|
export function nodeRuntimePipelinePostprocessScript(): string[] {
|
|
return [
|
|
"node - \"$render_dir\" \"$overlay_b64\" <<'NODE'",
|
|
"const fs = require('fs');",
|
|
"const path = require('path');",
|
|
"const vm = require('node:vm');",
|
|
"const renderDir = process.argv[2];",
|
|
"const overlay = JSON.parse(Buffer.from(process.argv[3], 'base64').toString('utf8'));",
|
|
"const pipelinePath = path.join(renderDir, overlay.tektonDir, 'pipeline.yaml');",
|
|
"let text = fs.readFileSync(pipelinePath, 'utf8');",
|
|
"let YAML = null;",
|
|
"try { YAML = require('yaml'); } catch {}",
|
|
"const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');",
|
|
"const shellSingle = (value) => `'${String(value).replaceAll(\"'\", `'\"'\"'`)}'`;",
|
|
"const yamlString = (value) => JSON.stringify(String(value));",
|
|
"const proxyEnv = {",
|
|
" HTTP_PROXY: overlay.proxyHttp,",
|
|
" HTTPS_PROXY: overlay.proxyHttps,",
|
|
" ALL_PROXY: overlay.proxyAll,",
|
|
" NO_PROXY: overlay.noProxy,",
|
|
" http_proxy: overlay.proxyHttp,",
|
|
" https_proxy: overlay.proxyHttps,",
|
|
" all_proxy: overlay.proxyAll,",
|
|
" no_proxy: overlay.noProxy,",
|
|
"};",
|
|
"const dockerProxyEnv = {",
|
|
" HWLAB_NODE_PROXY_URL: overlay.dockerProxyHttp,",
|
|
" HWLAB_NODE_ALL_PROXY_URL: overlay.dockerProxyAll,",
|
|
" HWLAB_NODE_NO_PROXY: overlay.dockerNoProxy,",
|
|
"};",
|
|
"const stepEnv = { ...proxyEnv, ...dockerProxyEnv, ...(overlay.stepEnv || {}) };",
|
|
"function prepareSourceDependencyScript() {",
|
|
" return `prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"",
|
|
"echo '{\"event\":\"prepare-source-dependencies\",\"status\":\"not-required\",\"dependency\":\"yaml\",\"manager\":\"inline-service-id-parser\"}' >&2",
|
|
"ci_timing_emit prepare-source-dependencies succeeded \"$prepare_source_dependencies_started_ms\"`;",
|
|
"}",
|
|
"function validatePrepareSourceDependencyScript(script) {",
|
|
" const marker = 'NODE_UNIDESK_YAML_DEPENDENCY';",
|
|
" let offset = 0;",
|
|
" while (true) {",
|
|
" const markerStart = script.indexOf(\"node <<'\" + marker + \"'\", offset);",
|
|
" if (markerStart === -1) return;",
|
|
" const bodyStart = script.indexOf('\\n', markerStart);",
|
|
" if (bodyStart === -1) throw new Error('prepare-source dependency heredoc body missing newline');",
|
|
" const bodyEnd = script.indexOf('\\n' + marker, bodyStart + 1);",
|
|
" if (bodyEnd === -1) throw new Error('prepare-source dependency heredoc terminator missing');",
|
|
" const body = script.slice(bodyStart + 1, bodyEnd);",
|
|
" try { new vm.Script(body, { filename: 'NODE_UNIDESK_YAML_DEPENDENCY.js' }); } catch (error) { throw new Error(`generated prepare-source yaml dependency script is invalid: ${error.message}`); }",
|
|
" offset = bodyEnd + marker.length + 1;",
|
|
" }",
|
|
"}",
|
|
"function deployYamlOverlayScript() {",
|
|
" const runtimeOverlay = JSON.stringify({",
|
|
" nodeId: overlay.nodeId,",
|
|
" lane: overlay.lane,",
|
|
" sourceBranch: overlay.sourceBranch,",
|
|
" gitopsBranch: overlay.gitopsBranch,",
|
|
" gitopsRoot: overlay.gitopsRoot,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" runtimeRenderDir: overlay.runtimeRenderDir,",
|
|
" runtimeNamespace: overlay.runtimeNamespace,",
|
|
" catalogPath: overlay.catalogPath,",
|
|
" gitUrl: overlay.gitUrl,",
|
|
" publicWebUrl: overlay.publicWebUrl,",
|
|
" publicApiUrl: overlay.publicApiUrl,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" runtimeStore: overlay.runtimeStore,",
|
|
" codeAgentRuntime: overlay.codeAgentRuntime,",
|
|
" observability: overlay.observability,",
|
|
" deployYamlGitMirror: overlay.deployYamlGitMirror,",
|
|
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
|
|
" dockerProxyHttp: overlay.dockerProxyHttp,",
|
|
" dockerProxyHttps: overlay.dockerProxyHttps,",
|
|
" dockerNoProxyList: overlay.dockerNoProxyList,",
|
|
" npmRegistry: overlay.npmRegistry,",
|
|
" npmFetchTimeoutMs: overlay.npmFetchTimeoutMs,",
|
|
" npmRetries: overlay.npmRetries,",
|
|
" });",
|
|
" return `node - <<'NODE_UNIDESK_DEPLOY_YAML_OVERLAY'",
|
|
"const fs = require('fs');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = ${runtimeOverlay};",
|
|
"const file = 'deploy/deploy.yaml';",
|
|
"if (!fs.existsSync(file)) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: false, reason: 'deploy-yaml-missing', file }));",
|
|
" process.exit(45);",
|
|
"}",
|
|
"const doc = YAML.parse(fs.readFileSync(file, 'utf8'));",
|
|
"doc.nodes = doc.nodes || {};",
|
|
"doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };",
|
|
"doc.lanes = doc.lanes || {};",
|
|
"const lane = doc.lanes[overlay.lane] || {};",
|
|
"const envRecipe = lane.envRecipe || {};",
|
|
"const downloadStack = {",
|
|
" ...(envRecipe.downloadStack || {}),",
|
|
" httpProxy: overlay.dockerProxyHttp,",
|
|
" httpsProxy: overlay.dockerProxyHttps,",
|
|
" noProxy: overlay.dockerNoProxyList,",
|
|
"};",
|
|
"if (overlay.npmRegistry) downloadStack.npmRegistry = overlay.npmRegistry;",
|
|
"if (overlay.npmFetchTimeoutMs) downloadStack.npmFetchTimeoutMs = overlay.npmFetchTimeoutMs;",
|
|
"doc.lanes[overlay.lane] = {",
|
|
" ...lane,",
|
|
" node: overlay.nodeId,",
|
|
" sourceBranch: overlay.sourceBranch,",
|
|
" gitopsBranch: overlay.gitopsBranch,",
|
|
" namespace: overlay.runtimeNamespace,",
|
|
" endpoint: overlay.publicApiUrl,",
|
|
" publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },",
|
|
" artifactCatalog: overlay.catalogPath,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" imageTagMode: 'full',",
|
|
" sourceRepo: overlay.gitUrl,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" observability: overlay.observability,",
|
|
" envRecipe: { ...envRecipe, downloadStack },",
|
|
"};",
|
|
"if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;",
|
|
"if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;",
|
|
"if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;",
|
|
"fs.writeFileSync(file, YAML.stringify(doc));",
|
|
"console.error(JSON.stringify({ event: 'unidesk-deploy-yaml-overlay', ok: true, lane: overlay.lane, httpProxy: overlay.dockerProxyHttp, noProxyCount: overlay.dockerNoProxyList.length }));",
|
|
"NODE_UNIDESK_DEPLOY_YAML_OVERLAY`;",
|
|
"}",
|
|
"function runtimePathOverlayScript() {",
|
|
" const sourcePath = `${String(overlay.gitopsRoot || '').replace(/\\/+$/u, '')}/${overlay.runtimeRenderDir}`;",
|
|
" const targetPath = String(overlay.runtimePath || '');",
|
|
" if (!targetPath || sourcePath === targetPath) return '';",
|
|
" return [",
|
|
" `if [ ! -d ${shellSingle(targetPath)} ]; then`,",
|
|
" ` if [ ! -d ${shellSingle(sourcePath)} ]; then echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: false, reason: 'source-runtime-path-missing' }))} >&2; exit 46; fi`,",
|
|
" ` mkdir -p \"$(dirname ${shellSingle(targetPath)})\"`,",
|
|
" ` cp -a ${shellSingle(sourcePath)} ${shellSingle(targetPath)}` ,",
|
|
" ` echo ${shellSingle(JSON.stringify({ event: 'unidesk-runtime-path-overlay', ok: true }))} >&2`,",
|
|
" `fi`,",
|
|
" ].join('\\n');",
|
|
"}",
|
|
"function stepEnvBootstrapScript() {",
|
|
" const entries = Object.entries(overlay.stepEnv || {}).filter(([, value]) => value !== undefined && value !== null && String(value).length > 0);",
|
|
" const lines = ['# unidesk-step-env-bootstrap'];",
|
|
" for (const [name, value] of entries) lines.push(`export ${name}=${shellSingle(value)}`);",
|
|
" if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'HOME')) lines.push('mkdir -p \"$HOME\"');",
|
|
" if (Object.prototype.hasOwnProperty.call(overlay.stepEnv || {}, 'XDG_CONFIG_HOME')) lines.push('mkdir -p \"$XDG_CONFIG_HOME\"');",
|
|
" lines.push('ci_node_deps=\"${HWLAB_CI_NODE_DEPS:-/opt/hwlab-ci-node-deps/node_modules}\"');",
|
|
" lines.push('if [ -d \"$ci_node_deps\" ]; then');",
|
|
" lines.push(' if [ -d /workspace/source/repo ]; then ci_node_deps_target=/workspace/source/repo/node_modules; else ci_node_deps_target=./node_modules; fi');",
|
|
" lines.push(' mkdir -p \"$ci_node_deps_target\"');",
|
|
" lines.push(' for ci_node_dep in yaml; do if [ -d \"$ci_node_deps/$ci_node_dep\" ] && [ ! -e \"$ci_node_deps_target/$ci_node_dep\" ]; then ln -s \"$ci_node_deps/$ci_node_dep\" \"$ci_node_deps_target/$ci_node_dep\"; fi; done');",
|
|
" lines.push('fi');",
|
|
" return lines.join('\\n');",
|
|
"}",
|
|
"function runtimeGitopsPostprocessScript() {",
|
|
" const runtimeOverlay = JSON.stringify({",
|
|
" gitopsRoot: overlay.gitopsRoot,",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" runtimeRenderDir: overlay.runtimeRenderDir,",
|
|
" runtimeNamespace: overlay.runtimeNamespace,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" codeAgentRuntime: overlay.codeAgentRuntime,",
|
|
" publicExposure: overlay.publicExposure,",
|
|
" observability: overlay.observability,",
|
|
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
|
|
" gitReadUrl: overlay.gitReadUrl,",
|
|
" publicWebUrl: overlay.publicWebUrl,",
|
|
" publicApiUrl: overlay.publicApiUrl,",
|
|
" });",
|
|
" return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS'",
|
|
"const fs = require('fs');",
|
|
"const path = require('path');",
|
|
"const crypto = require('crypto');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = ${runtimeOverlay};",
|
|
"const runtimePath = String(overlay.runtimePath || '');",
|
|
"const renderDir = String(overlay.runtimeRenderDir || '');",
|
|
"const legacyRuntimePath = runtimePath ? path.posix.join(path.posix.dirname(path.posix.dirname(runtimePath)), path.posix.basename(runtimePath)) : '';",
|
|
"const candidates = [...new Set([",
|
|
" runtimePath,",
|
|
" renderDir,",
|
|
" overlay.gitopsRoot && renderDir ? path.posix.join(String(overlay.gitopsRoot), renderDir) : '',",
|
|
" legacyRuntimePath,",
|
|
"].filter(Boolean))];",
|
|
"const sourcePath = candidates.find((candidate) => fs.existsSync(candidate)) || runtimePath;",
|
|
"if (!runtimePath || !fs.existsSync(sourcePath)) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: false, reason: 'runtime-path-missing', runtimePath, sourcePath }));",
|
|
" process.exit(47);",
|
|
"}",
|
|
"if (sourcePath !== runtimePath) {",
|
|
" fs.rmSync(runtimePath, { recursive: true, force: true });",
|
|
" fs.mkdirSync(path.dirname(runtimePath), { recursive: true });",
|
|
" fs.cpSync(sourcePath, runtimePath, { recursive: true });",
|
|
" fs.rmSync(sourcePath, { recursive: true, force: true });",
|
|
"}",
|
|
"for (const candidate of candidates) {",
|
|
" if (candidate !== runtimePath && candidate !== sourcePath && candidate.endsWith('/' + path.posix.basename(runtimePath))) fs.rmSync(candidate, { recursive: true, force: true });",
|
|
"}",
|
|
"function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }",
|
|
"function writeYaml(file, doc) { fs.writeFileSync(file, YAML.stringify(doc).trimEnd() + '\\\\n'); }",
|
|
"function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }",
|
|
"function normalizeList(items) { return { apiVersion: 'v1', kind: 'List', items }; }",
|
|
"function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }",
|
|
"function yamlFiles(dir) {",
|
|
" const files = [];",
|
|
" for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {",
|
|
" const file = path.join(dir, entry.name);",
|
|
" if (entry.isDirectory()) files.push(...yamlFiles(file));",
|
|
" else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);",
|
|
" }",
|
|
" return files;",
|
|
"}",
|
|
"function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
|
|
"function writeYamlDocuments(file, docs) { fs.writeFileSync(file, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\\\n---\\\\n') + '\\\\n'); }",
|
|
"function podSpecFor(item) {",
|
|
" if (!isObject(item) || !isObject(item.spec)) return null;",
|
|
" if (item.kind === 'Pod') return item.spec;",
|
|
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
|
|
" if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
|
|
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;",
|
|
" return null;",
|
|
"}",
|
|
"function templateMetadataFor(item) {",
|
|
" if (!isObject(item) || !isObject(item.spec)) return null;",
|
|
" if (item.kind === 'Pod') return item.metadata || null;",
|
|
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template ? item.spec.template.metadata : null;",
|
|
" if (item.kind === 'Job') return item.spec.template ? item.spec.template.metadata : null;",
|
|
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.metadata : null;",
|
|
" return null;",
|
|
"}",
|
|
"function stripMonitoringMetadata(metadata) {",
|
|
" if (!isObject(metadata)) return false;",
|
|
" let changed = false;",
|
|
" if (isObject(metadata.labels) && metadata.labels['hwlab.pikastech.local/monitoring'] !== undefined && metadata.labels['hwlab.pikastech.local/monitoring'] !== 'disabled') {",
|
|
" metadata.labels['hwlab.pikastech.local/monitoring'] = 'disabled';",
|
|
" changed = true;",
|
|
" }",
|
|
" if (isObject(metadata.annotations) && metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'] !== undefined) {",
|
|
" delete metadata.annotations['hwlab.pikastech.local/metrics-sidecar-sha256'];",
|
|
" changed = true;",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function containerHasVolumeMount(container, name) { return isObject(container) && Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === name); }",
|
|
"function removeMetricsSidecar(podSpec) {",
|
|
" if (!isObject(podSpec)) return false;",
|
|
" let changed = false;",
|
|
" if (Array.isArray(podSpec.containers)) {",
|
|
" const next = podSpec.containers.filter((container) => !(isObject(container) && (container.name === 'hwlab-metrics' || (Array.isArray(container.command) && container.command.includes('/metrics/metrics-sidecar.mjs')))));",
|
|
" if (next.length !== podSpec.containers.length) { podSpec.containers = next; changed = true; }",
|
|
" }",
|
|
" for (const group of ['containers', 'initContainers']) {",
|
|
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
|
|
" if (!isObject(container) || !Array.isArray(container.volumeMounts)) continue;",
|
|
" const nextMounts = container.volumeMounts.filter((mount) => !(mount && mount.name === 'hwlab-metrics-sidecar'));",
|
|
" if (nextMounts.length !== container.volumeMounts.length) { container.volumeMounts = nextMounts; changed = true; }",
|
|
" }",
|
|
" }",
|
|
" if (Array.isArray(podSpec.volumes)) {",
|
|
" const nextVolumes = podSpec.volumes.filter((volume) => !(volume && volume.name === 'hwlab-metrics-sidecar'));",
|
|
" if (nextVolumes.length !== podSpec.volumes.length) { podSpec.volumes = nextVolumes; changed = true; }",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function envValue(container, name) {",
|
|
" if (!isObject(container) || !Array.isArray(container.env)) return undefined;",
|
|
" const item = container.env.find((env) => env && env.name === name);",
|
|
" return item ? item.value : undefined;",
|
|
"}",
|
|
"function setEnvValue(container, name, value) {",
|
|
" if (!isObject(container) || typeof value !== 'string') return false;",
|
|
" container.env = Array.isArray(container.env) ? container.env : [];",
|
|
" let item = container.env.find((env) => env && env.name === name);",
|
|
" if (!item) { item = { name }; container.env.push(item); }",
|
|
" const changed = item.value !== value || item.valueFrom !== undefined;",
|
|
" item.value = value;",
|
|
" delete item.valueFrom;",
|
|
" return changed;",
|
|
"}",
|
|
"function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }",
|
|
"function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }",
|
|
"function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }",
|
|
"function publicExposureCloudWebEnvEntries() {",
|
|
" const exposure = overlay.publicExposure;",
|
|
" if (!exposure || !Array.isArray(exposure.extraProxies)) return [];",
|
|
" return exposure.extraProxies.filter((proxy) => proxy && proxy.cloudWebEnvName && proxy.publicBaseUrl).map((proxy) => ({ name: String(proxy.cloudWebEnvName), value: String(proxy.publicBaseUrl) }));",
|
|
"}",
|
|
"function startupProbeFrom(probe) {",
|
|
" const next = JSON.parse(JSON.stringify(probe));",
|
|
" next.periodSeconds = 10;",
|
|
" next.timeoutSeconds = Math.max(Number(next.timeoutSeconds || 0), 2);",
|
|
" next.failureThreshold = 30;",
|
|
" next.successThreshold = 1;",
|
|
" delete next.initialDelaySeconds;",
|
|
" return next;",
|
|
"}",
|
|
"function addEnvReuseStartupProbe(podSpec) {",
|
|
" if (!isObject(podSpec) || !Array.isArray(podSpec.containers)) return false;",
|
|
" let changed = false;",
|
|
" for (const container of podSpec.containers) {",
|
|
" if (!isObject(container) || !isEnvReuseContainer(container) || container.startupProbe) continue;",
|
|
" const sourceProbe = container.readinessProbe || container.livenessProbe;",
|
|
" if (!sourceProbe) continue;",
|
|
" container.startupProbe = startupProbeFrom(sourceProbe);",
|
|
" changed = true;",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function rewriteRuntimeImage(image) {",
|
|
" if (typeof image !== 'string') return image;",
|
|
" const match = (overlay.runtimeImageRewrites || []).find((item) => item && item.source === image);",
|
|
" return match ? match.target : image;",
|
|
"}",
|
|
"function patchRuntimeImages(podSpec) {",
|
|
" if (!isObject(podSpec)) return false;",
|
|
" let changed = false;",
|
|
" for (const group of ['containers', 'initContainers']) {",
|
|
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
|
|
" if (!isObject(container) || typeof container.image !== 'string') continue;",
|
|
" const nextImage = rewriteRuntimeImage(container.image);",
|
|
" if (nextImage !== container.image) { container.image = nextImage; changed = true; }",
|
|
" }",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function rewriteEnvValue(value) {",
|
|
" if (typeof value !== 'string') return value;",
|
|
" return value.replaceAll('http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git', overlay.gitReadUrl);",
|
|
"}",
|
|
"function patchGitReadUrlEnv(podSpec) {",
|
|
" if (!isObject(podSpec)) return false;",
|
|
" let changed = false;",
|
|
" for (const group of ['containers', 'initContainers']) {",
|
|
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
|
|
" if (!isObject(container) || !Array.isArray(container.env)) continue;",
|
|
" for (const env of container.env) {",
|
|
" if (!isObject(env) || typeof env.value !== 'string') continue;",
|
|
" const nextValue = rewriteEnvValue(env.value);",
|
|
" if (nextValue !== env.value) { env.value = nextValue; changed = true; }",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" return changed;",
|
|
"}",
|
|
"function patchRuntimeEnv(item, podSpec) {",
|
|
" if (!isObject(podSpec)) return { publicEndpointChanged: false, dbSslModeChanged: false, codeAgentRuntimeChanged: false, cloudWebRuntimeChanged: false };",
|
|
" let publicEndpointChanged = false;",
|
|
" let dbSslModeChanged = false;",
|
|
" let codeAgentRuntimeChanged = false;",
|
|
" let cloudWebRuntimeChanged = false;",
|
|
" const pg = overlay.externalPostgres;",
|
|
" const codeAgentRuntime = overlay.codeAgentRuntime;",
|
|
" const cloudWebEnvEntries = publicExposureCloudWebEnvEntries();",
|
|
" for (const group of ['containers', 'initContainers']) {",
|
|
" for (const container of Array.isArray(podSpec[group]) ? podSpec[group] : []) {",
|
|
" if (!isObject(container)) continue;",
|
|
" if (envValue(container, 'HWLAB_PUBLIC_ENDPOINT') !== undefined) publicEndpointChanged = setEnvValue(container, 'HWLAB_PUBLIC_ENDPOINT', expectedPublicEndpoint(item)) || publicEndpointChanged;",
|
|
" if (pg && pg.sslmode && envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE') !== undefined) dbSslModeChanged = setEnvValue(container, 'HWLAB_CLOUD_DB_SSL_MODE', pg.sslmode) || dbSslModeChanged;",
|
|
" if (workloadName(item) === 'hwlab-cloud-web' && container.name === 'hwlab-cloud-web') {",
|
|
" for (const env of cloudWebEnvEntries) cloudWebRuntimeChanged = setEnvValue(container, env.name, env.value) || cloudWebRuntimeChanged;",
|
|
" }",
|
|
" if (codeAgentRuntime && codeAgentRuntime.enabled && workloadName(item) === 'hwlab-cloud-api' && container.name === 'hwlab-cloud-api') {",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_ADAPTER', String(codeAgentRuntime.adapter)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'AGENTRUN_MGR_URL', String(codeAgentRuntime.managerUrl)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvFromSecret(container, 'AGENTRUN_API_KEY', String(codeAgentRuntime.apiKeySecretName), String(codeAgentRuntime.apiKeySecretKey)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE', String(codeAgentRuntime.runnerNamespace)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE', String(codeAgentRuntime.secretNamespace)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_REPO_URL', String(codeAgentRuntime.repoUrl)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID', String(codeAgentRuntime.providerId)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE', String(codeAgentRuntime.defaultProviderProfile)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR', String(codeAgentRuntime.codexStdioSupervisor)) || codeAgentRuntimeChanged;",
|
|
" if (codeAgentRuntime.kafkaShadowProducer) {",
|
|
" const kafka = codeAgentRuntime.kafkaShadowProducer;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED', String(kafka.enabled)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_SHADOW_CONSUME_ENABLED', String(kafka.consumeEnabled)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_BOOTSTRAP_SERVERS', String(kafka.bootstrapServers)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_COMMAND_TOPIC', String(kafka.commandTopic)) || codeAgentRuntimeChanged;",
|
|
" codeAgentRuntimeChanged = setEnvValue(container, 'HWLAB_KAFKA_CLIENT_ID', String(kafka.clientId)) || codeAgentRuntimeChanged;",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" return { publicEndpointChanged, dbSslModeChanged, codeAgentRuntimeChanged, cloudWebRuntimeChanged };",
|
|
"}",
|
|
"function patchRuntimeWorkloads() {",
|
|
" let observabilityChanged = false;",
|
|
" let startupProbeChanged = false;",
|
|
" let imageRewriteChanged = false;",
|
|
" let gitReadUrlChanged = false;",
|
|
" let publicEndpointChanged = false;",
|
|
" let dbSslModeChanged = false;",
|
|
" let codeAgentRuntimeChanged = false;",
|
|
" let cloudWebRuntimeChanged = false;",
|
|
" for (const file of yamlFiles(runtimePath)) {",
|
|
" if (path.basename(file) === 'kustomization.yaml') continue;",
|
|
" const docs = readYamlDocuments(file);",
|
|
" let changed = false;",
|
|
" for (const doc of docs) {",
|
|
" for (const item of listItems(doc).filter(Boolean)) {",
|
|
" if (!isObject(item)) continue;",
|
|
" if (overlay.observability && overlay.observability.prometheusOperator === false) {",
|
|
" const metadataChanged = stripMonitoringMetadata(item.metadata);",
|
|
" const templateChanged = stripMonitoringMetadata(templateMetadataFor(item));",
|
|
" const sidecarChanged = removeMetricsSidecar(podSpecFor(item));",
|
|
" const monitoringChanged = metadataChanged || templateChanged || sidecarChanged;",
|
|
" changed = monitoringChanged || changed;",
|
|
" observabilityChanged = observabilityChanged || monitoringChanged;",
|
|
" }",
|
|
" const probeChanged = addEnvReuseStartupProbe(podSpecFor(item));",
|
|
" changed = probeChanged || changed;",
|
|
" startupProbeChanged = startupProbeChanged || probeChanged;",
|
|
" const imageChanged = patchRuntimeImages(podSpecFor(item));",
|
|
" changed = imageChanged || changed;",
|
|
" imageRewriteChanged = imageRewriteChanged || imageChanged;",
|
|
" const gitUrlChanged = patchGitReadUrlEnv(podSpecFor(item));",
|
|
" changed = gitUrlChanged || changed;",
|
|
" gitReadUrlChanged = gitReadUrlChanged || gitUrlChanged;",
|
|
" const envChanged = patchRuntimeEnv(item, podSpecFor(item));",
|
|
" changed = envChanged.publicEndpointChanged || envChanged.dbSslModeChanged || envChanged.codeAgentRuntimeChanged || envChanged.cloudWebRuntimeChanged || changed;",
|
|
" publicEndpointChanged = publicEndpointChanged || envChanged.publicEndpointChanged;",
|
|
" dbSslModeChanged = dbSslModeChanged || envChanged.dbSslModeChanged;",
|
|
" codeAgentRuntimeChanged = codeAgentRuntimeChanged || envChanged.codeAgentRuntimeChanged;",
|
|
" cloudWebRuntimeChanged = cloudWebRuntimeChanged || envChanged.cloudWebRuntimeChanged;",
|
|
" }",
|
|
" }",
|
|
" if (changed) writeYamlDocuments(file, docs);",
|
|
" }",
|
|
" return { observabilityChanged, startupProbeChanged, imageRewriteChanged, gitReadUrlChanged, publicEndpointChanged, dbSslModeChanged, codeAgentRuntimeChanged, cloudWebRuntimeChanged };",
|
|
"}",
|
|
"function patchKustomization() {",
|
|
" const file = path.join(runtimePath, 'kustomization.yaml');",
|
|
" if (!fs.existsSync(file)) return false;",
|
|
" const doc = readYaml(file) || {};",
|
|
" const resources = Array.isArray(doc.resources) ? doc.resources : [];",
|
|
" const next = resources.filter((item) => !(overlay.observability && overlay.observability.prometheusOperator === false && item === 'observability.yaml'));",
|
|
" let changed = false;",
|
|
" if (next.length !== resources.length) { doc.resources = next; writeYaml(file, doc); changed = true; }",
|
|
" const observabilityFile = path.join(runtimePath, 'observability.yaml');",
|
|
" if (overlay.observability && overlay.observability.prometheusOperator === false && fs.existsSync(observabilityFile)) { fs.rmSync(observabilityFile, { force: true }); changed = true; }",
|
|
" return changed;",
|
|
"}",
|
|
"function patchExternalPostgres() {",
|
|
" const pg = overlay.externalPostgres;",
|
|
" if (!pg || !pg.serviceName) return false;",
|
|
" const access = pg.runtimeAccess || { endpointAddress: pg.endpointAddress, port: pg.port };",
|
|
" const file = path.join(runtimePath, 'external-postgres.yaml');",
|
|
" if (!fs.existsSync(file)) return false;",
|
|
" const doc = readYaml(file);",
|
|
" const items = listItems(doc).filter(Boolean);",
|
|
" let changed = false;",
|
|
" const endpointIsIpv4 = /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/.test(String(access.endpointAddress));",
|
|
" const endpointSliceName = String(pg.serviceName) + '-host';",
|
|
" for (const item of items) {",
|
|
" if (!item || typeof item !== 'object') continue;",
|
|
" item.metadata = item.metadata || {};",
|
|
" item.metadata.namespace = overlay.runtimeNamespace;",
|
|
" item.metadata.labels = item.metadata.labels || {};",
|
|
" item.metadata.labels['app.kubernetes.io/name'] = pg.serviceName;",
|
|
" if (item.kind === 'Service') {",
|
|
" item.metadata.name = pg.serviceName;",
|
|
" item.spec = item.spec || {};",
|
|
" if (endpointIsIpv4) {",
|
|
" item.spec.type = 'ClusterIP';",
|
|
" delete item.spec.externalName;",
|
|
" } else {",
|
|
" item.spec.type = 'ExternalName';",
|
|
" item.spec.externalName = String(access.endpointAddress);",
|
|
" delete item.spec.clusterIP;",
|
|
" delete item.spec.clusterIPs;",
|
|
" delete item.spec.ipFamilies;",
|
|
" delete item.spec.ipFamilyPolicy;",
|
|
" delete item.spec.internalTrafficPolicy;",
|
|
" }",
|
|
" item.spec.ports = [{ name: 'postgres', port: access.port, targetPort: access.port, protocol: 'TCP' }];",
|
|
" delete item.spec.selector;",
|
|
" changed = true;",
|
|
" }",
|
|
" if (item.kind === 'EndpointSlice') {",
|
|
" if (!endpointIsIpv4) { item.__delete = true; changed = true; continue; }",
|
|
" item.metadata.name = endpointSliceName;",
|
|
" item.metadata.labels['kubernetes.io/service-name'] = pg.serviceName;",
|
|
" item.addressType = 'IPv4';",
|
|
" item.ports = [{ name: 'postgres', port: access.port, protocol: 'TCP' }];",
|
|
" item.endpoints = [{ addresses: [access.endpointAddress], conditions: { ready: true } }];",
|
|
" changed = true;",
|
|
" }",
|
|
" }",
|
|
" if (changed) writeYaml(file, normalizeList(items.filter((item) => !item.__delete)));",
|
|
" return changed;",
|
|
"}",
|
|
"function patchHealthContract() {",
|
|
" const file = path.join(runtimePath, 'health-contract.yaml');",
|
|
" if (!fs.existsSync(file)) return false;",
|
|
" const doc = readYaml(file);",
|
|
" const items = listItems(doc).filter(Boolean);",
|
|
" let changed = false;",
|
|
" const pg = overlay.externalPostgres;",
|
|
" for (const item of items) {",
|
|
" if (!item || item.kind !== 'ConfigMap') continue;",
|
|
" item.data = item.data || {};",
|
|
" if (item.data.endpoint !== overlay.publicWebUrl) { item.data.endpoint = overlay.publicWebUrl; changed = true; }",
|
|
" const cloudApi = 'GET /health/live through ' + overlay.publicApiUrl;",
|
|
" if (item.data['cloud-api'] !== cloudApi) { item.data['cloud-api'] = cloudApi; changed = true; }",
|
|
" const cloudWeb = 'GET /health/live on ' + overlay.publicWebUrl + '; consumes cloud-api only';",
|
|
" if (item.data['cloud-web'] !== cloudWeb) { item.data['cloud-web'] = cloudWeb; changed = true; }",
|
|
" if (pg && pg.sslmode && typeof item.data['cloud-api-db'] === 'string') {",
|
|
" const next = item.data['cloud-api-db'].replace(/HWLAB_CLOUD_DB_SSL_MODE=[A-Za-z0-9_-]+/g, 'HWLAB_CLOUD_DB_SSL_MODE=' + pg.sslmode);",
|
|
" if (next !== item.data['cloud-api-db']) { item.data['cloud-api-db'] = next; changed = true; }",
|
|
" }",
|
|
" }",
|
|
" if (changed) writeYaml(file, normalizeList(items));",
|
|
" return changed;",
|
|
"}",
|
|
"function publicExposureFrpcProxies(exposure) { return [exposure.webProxy, exposure.apiProxy, ...(Array.isArray(exposure.extraProxies) ? exposure.extraProxies : [])].filter(Boolean); }",
|
|
"function renderPublicExposureFrpcProxyToml(proxy) {",
|
|
" return [",
|
|
" '[[proxies]]',",
|
|
" 'name = ' + JSON.stringify(String(proxy.name)),",
|
|
" 'type = \"tcp\"',",
|
|
" 'localIP = ' + JSON.stringify(String(proxy.localIP)),",
|
|
" 'localPort = ' + Number(proxy.localPort),",
|
|
" 'remotePort = ' + Number(proxy.remotePort),",
|
|
" '',",
|
|
" ];",
|
|
"}",
|
|
"function renderPublicExposureFrpcToml(exposure) {",
|
|
" return [",
|
|
" 'serverAddr = ' + JSON.stringify(String(exposure.serverAddr)),",
|
|
" 'serverPort = ' + Number(exposure.serverPort),",
|
|
" 'loginFailExit = true',",
|
|
" 'auth.token = \"{{ .Envs.HWLAB_FRP_TOKEN }}\"',",
|
|
" '',",
|
|
" ...publicExposureFrpcProxies(exposure).flatMap(renderPublicExposureFrpcProxyToml),",
|
|
" ].join('\\\\n');",
|
|
"}",
|
|
"function setEnvFromSecret(container, name, secretName, secretKey) {",
|
|
" if (!isObject(container)) return false;",
|
|
" container.env = Array.isArray(container.env) ? container.env : [];",
|
|
" let item = container.env.find((env) => env && env.name === name);",
|
|
" if (!item) { item = { name }; container.env.push(item); }",
|
|
" const nextValueFrom = { secretKeyRef: { name: secretName, key: secretKey } };",
|
|
" const changed = item.value !== undefined || JSON.stringify(item.valueFrom || {}) !== JSON.stringify(nextValueFrom);",
|
|
" delete item.value;",
|
|
" item.valueFrom = nextValueFrom;",
|
|
" return changed;",
|
|
"}",
|
|
"function patchPublicExposure() {",
|
|
" const exposure = overlay.publicExposure;",
|
|
" if (!exposure || !exposure.enabled) return { configured: false, changed: false };",
|
|
" const file = path.join(runtimePath, 'node-frpc.yaml');",
|
|
" if (!fs.existsSync(file)) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'node-frpc-yaml-missing', filePath: file, hostname: exposure.hostname }));",
|
|
" process.exit(49);",
|
|
" }",
|
|
" const docs = readYamlDocuments(file);",
|
|
" const configName = String(overlay.runtimeNamespace) + '-frpc-config';",
|
|
" const deploymentName = String(overlay.runtimeNamespace) + '-frpc';",
|
|
" const configKey = String(exposure.secretKey || 'frpc.toml');",
|
|
" const tokenKey = String(exposure.tokenKey || 'token');",
|
|
" const toml = renderPublicExposureFrpcToml(exposure);",
|
|
" const tomlSha256 = crypto.createHash('sha256').update(toml).digest('hex');",
|
|
" let changed = false;",
|
|
" let foundConfigMap = false;",
|
|
" let foundDeployment = false;",
|
|
" for (const doc of docs) {",
|
|
" for (const item of listItems(doc).filter(Boolean)) {",
|
|
" if (!isObject(item)) continue;",
|
|
" item.metadata = item.metadata || {};",
|
|
" if (item.kind === 'ConfigMap' && item.metadata.name === configName) {",
|
|
" foundConfigMap = true;",
|
|
" item.data = item.data || {};",
|
|
" if (item.data[configKey] !== toml) { item.data[configKey] = toml; changed = true; }",
|
|
" }",
|
|
" if (item.kind === 'Deployment' && item.metadata.name === deploymentName) {",
|
|
" foundDeployment = true;",
|
|
" item.spec = item.spec || {};",
|
|
" const nextStrategy = { type: 'Recreate' };",
|
|
" if (JSON.stringify(item.spec.strategy || {}) !== JSON.stringify(nextStrategy)) { item.spec.strategy = nextStrategy; changed = true; }",
|
|
" const templateMetadata = templateMetadataFor(item);",
|
|
" if (templateMetadata) {",
|
|
" templateMetadata.annotations = isObject(templateMetadata.annotations) ? templateMetadata.annotations : {};",
|
|
" if (templateMetadata.annotations['unidesk.ai/frpc-config-sha256'] !== tomlSha256) { templateMetadata.annotations['unidesk.ai/frpc-config-sha256'] = tomlSha256; changed = true; }",
|
|
" }",
|
|
" const podSpec = podSpecFor(item);",
|
|
" for (const container of Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : []) {",
|
|
" if (!isObject(container)) continue;",
|
|
" if (container.name === 'frpc' || String(container.image || '').includes('frpc')) changed = setEnvFromSecret(container, 'HWLAB_FRP_TOKEN', exposure.secretName, tokenKey) || changed;",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" if (!foundConfigMap || !foundDeployment) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: false, reason: 'frpc-resource-missing', filePath: file, configName, deploymentName, foundConfigMap, foundDeployment }));",
|
|
" process.exit(50);",
|
|
" }",
|
|
" if (changed) writeYamlDocuments(file, docs);",
|
|
" console.error(JSON.stringify({ event: 'unidesk-public-exposure-postprocess', ok: true, applied: true, changed, filePath: file, hostname: exposure.hostname, serverAddr: exposure.serverAddr, serverPort: exposure.serverPort, webProxy: exposure.webProxy.name, apiProxy: exposure.apiProxy.name, extraProxyCount: Array.isArray(exposure.extraProxies) ? exposure.extraProxies.length : 0, configSha256: tomlSha256 }));",
|
|
" return { configured: true, changed, foundConfigMap, foundDeployment };",
|
|
"}",
|
|
"const kustomizationChanged = patchKustomization();",
|
|
"const runtimeWorkloadsChanged = patchRuntimeWorkloads();",
|
|
"const externalPostgresChanged = patchExternalPostgres();",
|
|
"const healthContractChanged = patchHealthContract();",
|
|
"const publicExposureChanged = patchPublicExposure();",
|
|
"console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-postprocess', ok: true, runtimePath, sourcePath, pathRelocated: sourcePath !== runtimePath, observabilityPrometheusOperator: overlay.observability ? overlay.observability.prometheusOperator : null, runtimeImageRewriteCount: (overlay.runtimeImageRewrites || []).length, kustomizationChanged, observabilityWorkloadsChanged: runtimeWorkloadsChanged.observabilityChanged, startupProbeChanged: runtimeWorkloadsChanged.startupProbeChanged, runtimeImageRewriteChanged: runtimeWorkloadsChanged.imageRewriteChanged, gitReadUrlChanged: runtimeWorkloadsChanged.gitReadUrlChanged, publicEndpointChanged: runtimeWorkloadsChanged.publicEndpointChanged, dbSslModeChanged: runtimeWorkloadsChanged.dbSslModeChanged, codeAgentRuntimeChanged: runtimeWorkloadsChanged.codeAgentRuntimeChanged, cloudWebRuntimeChanged: runtimeWorkloadsChanged.cloudWebRuntimeChanged, externalPostgresChanged, healthContractChanged, publicExposureChanged }));",
|
|
"NODE_UNIDESK_RUNTIME_GITOPS_POSTPROCESS`;",
|
|
"}",
|
|
"function runtimeGitopsVerifyScript() {",
|
|
" const runtimeOverlay = JSON.stringify({",
|
|
" runtimePath: overlay.runtimePath,",
|
|
" runtimeNamespace: overlay.runtimeNamespace,",
|
|
" externalPostgres: overlay.externalPostgres,",
|
|
" codeAgentRuntime: overlay.codeAgentRuntime,",
|
|
" publicExposure: overlay.publicExposure,",
|
|
" observability: overlay.observability,",
|
|
" runtimeImageRewrites: overlay.runtimeImageRewrites,",
|
|
" gitReadUrl: overlay.gitReadUrl,",
|
|
" publicWebUrl: overlay.publicWebUrl,",
|
|
" publicApiUrl: overlay.publicApiUrl,",
|
|
" });",
|
|
" return `node - <<'NODE_UNIDESK_RUNTIME_GITOPS_VERIFY'",
|
|
"const fs = require('fs');",
|
|
"const path = require('path');",
|
|
"const crypto = require('crypto');",
|
|
"const YAML = require('yaml');",
|
|
"const overlay = ${runtimeOverlay};",
|
|
"const runtimePath = String(overlay.runtimePath || '');",
|
|
"function fail(reason, extra = {}) {",
|
|
" console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: false, reason, runtimePath, ...extra }));",
|
|
" process.exit(48);",
|
|
"}",
|
|
"if (!runtimePath || !fs.existsSync(runtimePath)) fail('runtime-path-missing');",
|
|
"function readYaml(file) { return YAML.parse(fs.readFileSync(file, 'utf8')); }",
|
|
"function listItems(doc) { return doc && doc.kind === 'List' && Array.isArray(doc.items) ? doc.items : [doc]; }",
|
|
"function isObject(value) { return value && typeof value === 'object' && !Array.isArray(value); }",
|
|
"function yamlFiles(dir) {",
|
|
" const files = [];",
|
|
" for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {",
|
|
" const file = path.join(dir, entry.name);",
|
|
" if (entry.isDirectory()) files.push(...yamlFiles(file));",
|
|
" else if (entry.isFile() && /\\.ya?ml$/u.test(entry.name)) files.push(file);",
|
|
" }",
|
|
" return files;",
|
|
"}",
|
|
"function readYamlDocuments(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
|
|
"function allItemsFromFile(file) { return readYamlDocuments(file).flatMap((doc) => listItems(doc).filter(Boolean)); }",
|
|
"function podSpecFor(item) {",
|
|
" if (!isObject(item) || !isObject(item.spec)) return null;",
|
|
" if (item.kind === 'Pod') return item.spec;",
|
|
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
|
|
" if (item.kind === 'Job') return item.spec.template && item.spec.template.spec ? item.spec.template.spec : null;",
|
|
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.spec : null;",
|
|
" return null;",
|
|
"}",
|
|
"function templateMetadataFor(item) {",
|
|
" if (!isObject(item) || !isObject(item.spec)) return null;",
|
|
" if (item.kind === 'Pod') return item.metadata || null;",
|
|
" if (['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(item.kind)) return item.spec.template ? item.spec.template.metadata : null;",
|
|
" if (item.kind === 'Job') return item.spec.template ? item.spec.template.metadata : null;",
|
|
" if (item.kind === 'CronJob') return item.spec.jobTemplate && item.spec.jobTemplate.spec && item.spec.jobTemplate.spec.template ? item.spec.jobTemplate.spec.template.metadata : null;",
|
|
" return null;",
|
|
"}",
|
|
"function envValue(container, name) {",
|
|
" if (!isObject(container) || !Array.isArray(container.env)) return undefined;",
|
|
" const item = container.env.find((env) => env && env.name === name);",
|
|
" return item ? item.value : undefined;",
|
|
"}",
|
|
"function envSecretRef(container, name) {",
|
|
" if (!isObject(container) || !Array.isArray(container.env)) return {};",
|
|
" const item = container.env.find((env) => env && env.name === name);",
|
|
" return item && item.valueFrom && item.valueFrom.secretKeyRef ? item.valueFrom.secretKeyRef : {};",
|
|
"}",
|
|
"function isEnvReuseContainer(container) { return envValue(container, 'HWLAB_RUNTIME_MODE') === 'env-reuse-git-mirror-checkout' || envValue(container, 'HWLAB_BOOT_SH') !== undefined || envValue(container, 'HWLAB_BOOT_COMMIT') !== undefined; }",
|
|
"function workloadName(item) { return item && item.metadata && item.metadata.labels && item.metadata.labels['app.kubernetes.io/name'] ? String(item.metadata.labels['app.kubernetes.io/name']) : String(item && item.metadata && item.metadata.name || ''); }",
|
|
"function expectedPublicEndpoint(item) { return workloadName(item) === 'hwlab-cloud-web' ? overlay.publicWebUrl : overlay.publicApiUrl; }",
|
|
"function publicExposureCloudWebEnvEntries() {",
|
|
" const exposure = overlay.publicExposure;",
|
|
" if (!exposure || !Array.isArray(exposure.extraProxies)) return [];",
|
|
" return exposure.extraProxies.filter((proxy) => proxy && proxy.cloudWebEnvName && proxy.publicBaseUrl).map((proxy) => ({ name: String(proxy.cloudWebEnvName), value: String(proxy.publicBaseUrl) }));",
|
|
"}",
|
|
"function workloadRef(item, file, container) { return { file, kind: item && item.kind, name: item && item.metadata && item.metadata.name, container: container && container.name }; }",
|
|
"function workloadChecks() {",
|
|
" const metricsRefs = [];",
|
|
" const missingStartupProbes = [];",
|
|
" const publicRuntimeImages = [];",
|
|
" const staleGitReadUrls = [];",
|
|
" const wrongPublicEndpoints = [];",
|
|
" const wrongDbSslModes = [];",
|
|
" const wrongCodeAgentRuntimeEnvs = [];",
|
|
" const wrongCloudWebRuntimeEnvs = [];",
|
|
" const rewriteSources = new Set((overlay.runtimeImageRewrites || []).map((item) => item && item.source).filter(Boolean));",
|
|
" const codeAgentRuntime = overlay.codeAgentRuntime;",
|
|
" const cloudWebEnvEntries = publicExposureCloudWebEnvEntries();",
|
|
" function checkCloudWebRuntimeValue(item, file, container, envName, expected) {",
|
|
" const value = envValue(container, envName);",
|
|
" if (value !== expected) wrongCloudWebRuntimeEnvs.push({ ...workloadRef(item, file, container), envName, expected, value: value ?? null });",
|
|
" }",
|
|
" function checkCodeAgentRuntimeValue(item, file, container, envName, expected) {",
|
|
" const value = envValue(container, envName);",
|
|
" if (value !== expected) wrongCodeAgentRuntimeEnvs.push({ ...workloadRef(item, file, container), envName, kind: 'value', expected, value: value ?? null });",
|
|
" }",
|
|
" function checkCodeAgentRuntimeSecret(item, file, container, envName, expectedSecret, expectedKey) {",
|
|
" const secretRef = envSecretRef(container, envName);",
|
|
" if (secretRef.name !== expectedSecret || secretRef.key !== expectedKey) wrongCodeAgentRuntimeEnvs.push({ ...workloadRef(item, file, container), envName, kind: 'secretKeyRef', expectedSecret, expectedKey, secret: secretRef.name ?? null, key: secretRef.key ?? null });",
|
|
" }",
|
|
" for (const file of yamlFiles(runtimePath)) {",
|
|
" if (path.basename(file) === 'kustomization.yaml') continue;",
|
|
" for (const doc of readYamlDocuments(file)) {",
|
|
" for (const item of listItems(doc).filter(Boolean)) {",
|
|
" const podSpec = podSpecFor(item);",
|
|
" if (!isObject(podSpec)) continue;",
|
|
" for (const container of Array.isArray(podSpec.containers) ? podSpec.containers : []) {",
|
|
" if (!isObject(container)) continue;",
|
|
" if (container.name === 'hwlab-metrics' || (Array.isArray(container.volumeMounts) && container.volumeMounts.some((mount) => mount && mount.name === 'hwlab-metrics-sidecar'))) metricsRefs.push(workloadRef(item, file, container));",
|
|
" if (isEnvReuseContainer(container) && (container.readinessProbe || container.livenessProbe) && !container.startupProbe) missingStartupProbes.push(workloadRef(item, file, container));",
|
|
" if (typeof container.image === 'string' && rewriteSources.has(container.image)) publicRuntimeImages.push({ ...workloadRef(item, file, container), image: container.image });",
|
|
" if (Array.isArray(container.env) && container.env.some((env) => env && typeof env.value === 'string' && env.value.includes('git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git') && env.value !== overlay.gitReadUrl)) staleGitReadUrls.push(workloadRef(item, file, container));",
|
|
" const publicEndpoint = envValue(container, 'HWLAB_PUBLIC_ENDPOINT');",
|
|
" if (publicEndpoint !== undefined && publicEndpoint !== expectedPublicEndpoint(item)) wrongPublicEndpoints.push({ ...workloadRef(item, file, container), value: publicEndpoint, expected: expectedPublicEndpoint(item) });",
|
|
" const dbSslMode = envValue(container, 'HWLAB_CLOUD_DB_SSL_MODE');",
|
|
" if (overlay.externalPostgres && overlay.externalPostgres.sslmode && dbSslMode !== undefined && dbSslMode !== overlay.externalPostgres.sslmode) wrongDbSslModes.push({ ...workloadRef(item, file, container), value: dbSslMode, expected: overlay.externalPostgres.sslmode });",
|
|
" if (workloadName(item) === 'hwlab-cloud-web' && container.name === 'hwlab-cloud-web') {",
|
|
" for (const env of cloudWebEnvEntries) checkCloudWebRuntimeValue(item, file, container, env.name, env.value);",
|
|
" }",
|
|
" if (codeAgentRuntime && codeAgentRuntime.enabled && workloadName(item) === 'hwlab-cloud-api' && container.name === 'hwlab-cloud-api') {",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_ADAPTER', String(codeAgentRuntime.adapter));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'AGENTRUN_MGR_URL', String(codeAgentRuntime.managerUrl));",
|
|
" checkCodeAgentRuntimeSecret(item, file, container, 'AGENTRUN_API_KEY', String(codeAgentRuntime.apiKeySecretName), String(codeAgentRuntime.apiKeySecretKey));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE', String(codeAgentRuntime.runnerNamespace));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE', String(codeAgentRuntime.secretNamespace));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_REPO_URL', String(codeAgentRuntime.repoUrl));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID', String(codeAgentRuntime.providerId));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE', String(codeAgentRuntime.defaultProviderProfile));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR', String(codeAgentRuntime.codexStdioSupervisor));",
|
|
" if (codeAgentRuntime.kafkaShadowProducer) {",
|
|
" const kafka = codeAgentRuntime.kafkaShadowProducer;",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_SHADOW_PRODUCE_ENABLED', String(kafka.enabled));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_SHADOW_CONSUME_ENABLED', String(kafka.consumeEnabled));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_BOOTSTRAP_SERVERS', String(kafka.bootstrapServers));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_COMMAND_TOPIC', String(kafka.commandTopic));",
|
|
" checkCodeAgentRuntimeValue(item, file, container, 'HWLAB_KAFKA_CLIENT_ID', String(kafka.clientId));",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" if (Array.isArray(podSpec.volumes) && podSpec.volumes.some((volume) => volume && volume.name === 'hwlab-metrics-sidecar')) metricsRefs.push(workloadRef(item, file, { name: 'volume/hwlab-metrics-sidecar' }));",
|
|
" }",
|
|
" }",
|
|
" }",
|
|
" return { metricsRefs, missingStartupProbes, publicRuntimeImages, staleGitReadUrls, wrongPublicEndpoints, wrongDbSslModes, wrongCodeAgentRuntimeEnvs, wrongCloudWebRuntimeEnvs };",
|
|
"}",
|
|
"const checks = [];",
|
|
"const workloadCheck = workloadChecks();",
|
|
"const kustomizationPath = path.join(runtimePath, 'kustomization.yaml');",
|
|
"if (overlay.observability && overlay.observability.prometheusOperator === false) {",
|
|
" if (!fs.existsSync(kustomizationPath)) fail('kustomization-missing');",
|
|
" const resources = readYaml(kustomizationPath).resources || [];",
|
|
" if (resources.includes('observability.yaml')) fail('observability-resource-still-rendered', { file: kustomizationPath });",
|
|
" if (workloadCheck.metricsRefs.length > 0) fail('observability-sidecar-still-rendered', { refs: workloadCheck.metricsRefs.slice(0, 12), count: workloadCheck.metricsRefs.length });",
|
|
" checks.push('observability-disabled');",
|
|
"}",
|
|
"if (workloadCheck.missingStartupProbes.length > 0) fail('env-reuse-startup-probe-missing', { refs: workloadCheck.missingStartupProbes.slice(0, 12), count: workloadCheck.missingStartupProbes.length });",
|
|
"checks.push('env-reuse-startup-probes');",
|
|
"if (workloadCheck.publicRuntimeImages.length > 0) fail('runtime-image-rewrite-missing', { refs: workloadCheck.publicRuntimeImages.slice(0, 12), count: workloadCheck.publicRuntimeImages.length });",
|
|
"if ((overlay.runtimeImageRewrites || []).length > 0) checks.push('runtime-image-rewrites');",
|
|
"if (workloadCheck.staleGitReadUrls.length > 0) fail('runtime-git-read-url-stale', { refs: workloadCheck.staleGitReadUrls.slice(0, 12), count: workloadCheck.staleGitReadUrls.length, expected: overlay.gitReadUrl });",
|
|
"checks.push('runtime-git-read-url');",
|
|
"if (workloadCheck.wrongPublicEndpoints.length > 0) fail('runtime-public-endpoint-mismatch', { refs: workloadCheck.wrongPublicEndpoints.slice(0, 12), count: workloadCheck.wrongPublicEndpoints.length });",
|
|
"checks.push('runtime-public-endpoint');",
|
|
"if (workloadCheck.wrongDbSslModes.length > 0) fail('runtime-db-ssl-mode-mismatch', { refs: workloadCheck.wrongDbSslModes.slice(0, 12), count: workloadCheck.wrongDbSslModes.length });",
|
|
"if (overlay.externalPostgres && overlay.externalPostgres.sslmode) checks.push('runtime-db-ssl-mode');",
|
|
"if (workloadCheck.wrongCodeAgentRuntimeEnvs.length > 0) fail('code-agent-runtime-env-mismatch', { refs: workloadCheck.wrongCodeAgentRuntimeEnvs.slice(0, 12), count: workloadCheck.wrongCodeAgentRuntimeEnvs.length });",
|
|
"if (overlay.codeAgentRuntime && overlay.codeAgentRuntime.enabled) checks.push('code-agent-runtime-env');",
|
|
"if (workloadCheck.wrongCloudWebRuntimeEnvs.length > 0) fail('cloud-web-runtime-env-mismatch', { refs: workloadCheck.wrongCloudWebRuntimeEnvs.slice(0, 12), count: workloadCheck.wrongCloudWebRuntimeEnvs.length });",
|
|
"if (publicExposureCloudWebEnvEntries().length > 0) checks.push('cloud-web-runtime-env');",
|
|
"const pg = overlay.externalPostgres;",
|
|
"if (pg && pg.serviceName) {",
|
|
" const access = pg.runtimeAccess || { endpointAddress: pg.endpointAddress, port: pg.port };",
|
|
" const file = path.join(runtimePath, 'external-postgres.yaml');",
|
|
" if (!fs.existsSync(file)) fail('external-postgres-missing');",
|
|
" const items = listItems(readYaml(file)).filter(Boolean);",
|
|
" const service = items.find((item) => item && item.kind === 'Service' && item.metadata && item.metadata.name === pg.serviceName);",
|
|
" const endpointSlice = items.find((item) => item && item.kind === 'EndpointSlice' && item.metadata && item.metadata.name === String(pg.serviceName) + '-host');",
|
|
" if (!service) fail('external-postgres-service-missing', { expected: pg.serviceName });",
|
|
" const endpointIsIpv4 = /^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+$/.test(String(access.endpointAddress));",
|
|
" const servicePort = service.spec && Array.isArray(service.spec.ports) && service.spec.ports[0] ? service.spec.ports[0].port : null;",
|
|
" if (!endpointIsIpv4) {",
|
|
" if (service.spec && service.spec.type !== 'ExternalName') fail('external-postgres-service-type-mismatch', { value: service.spec.type, expected: 'ExternalName' });",
|
|
" if (!service.spec || String(service.spec.externalName) !== String(access.endpointAddress)) fail('external-postgres-external-name-mismatch', { value: service.spec && service.spec.externalName, expected: access.endpointAddress });",
|
|
" if (Number(servicePort) !== Number(access.port)) fail('external-postgres-port-mismatch', { servicePort, expectedPort: access.port });",
|
|
" if (endpointSlice) fail('external-postgres-endpointslice-unexpected', { name: String(pg.serviceName) + '-host' });",
|
|
" checks.push('external-postgres-bridge');",
|
|
" } else {",
|
|
" if (!endpointSlice) fail('external-postgres-endpointslice-missing', { expected: String(pg.serviceName) + '-host' });",
|
|
" const endpointPort = Array.isArray(endpointSlice.ports) && endpointSlice.ports[0] ? endpointSlice.ports[0].port : null;",
|
|
" const endpointAddress = Array.isArray(endpointSlice.endpoints) && endpointSlice.endpoints[0] && Array.isArray(endpointSlice.endpoints[0].addresses) ? endpointSlice.endpoints[0].addresses[0] : null;",
|
|
" if (Number(servicePort) !== Number(access.port) || Number(endpointPort) !== Number(access.port)) fail('external-postgres-port-mismatch', { servicePort, endpointPort, expectedPort: access.port });",
|
|
" if (String(endpointAddress) !== String(access.endpointAddress)) fail('external-postgres-address-mismatch', { endpointAddress, expectedEndpointAddress: access.endpointAddress });",
|
|
" checks.push('external-postgres-bridge');",
|
|
" }",
|
|
"}",
|
|
"const exposure = overlay.publicExposure;",
|
|
"if (exposure && exposure.enabled) {",
|
|
" const file = path.join(runtimePath, 'node-frpc.yaml');",
|
|
" if (!fs.existsSync(file)) fail('public-exposure-frpc-missing');",
|
|
" const items = allItemsFromFile(file);",
|
|
" const configName = String(overlay.runtimeNamespace) + '-frpc-config';",
|
|
" const deploymentName = String(overlay.runtimeNamespace) + '-frpc';",
|
|
" const configKey = String(exposure.secretKey || 'frpc.toml');",
|
|
" const tokenKey = String(exposure.tokenKey || 'token');",
|
|
" const configMap = items.find((item) => item && item.kind === 'ConfigMap' && item.metadata && item.metadata.name === configName);",
|
|
" const deployment = items.find((item) => item && item.kind === 'Deployment' && item.metadata && item.metadata.name === deploymentName);",
|
|
" if (!configMap) fail('public-exposure-frpc-configmap-missing', { expected: configName });",
|
|
" if (!deployment) fail('public-exposure-frpc-deployment-missing', { expected: deploymentName });",
|
|
" const toml = configMap.data && configMap.data[configKey];",
|
|
" if (typeof toml !== 'string') fail('public-exposure-frpc-config-missing', { expectedKey: configKey });",
|
|
" const expectedConfigSha256 = crypto.createHash('sha256').update(toml).digest('hex');",
|
|
" const expectedProxyTokens = [exposure.webProxy, exposure.apiProxy, ...(Array.isArray(exposure.extraProxies) ? exposure.extraProxies : [])].flatMap((proxy) => [String(proxy.name), String(proxy.remotePort)]);",
|
|
" for (const expected of [String(exposure.serverAddr), String(exposure.serverPort), ...expectedProxyTokens, 'HWLAB_FRP_TOKEN']) {",
|
|
" if (!toml.includes(expected)) fail('public-exposure-frpc-config-mismatch', { expected });",
|
|
" }",
|
|
" const podSpec = podSpecFor(deployment);",
|
|
" const templateMetadata = templateMetadataFor(deployment);",
|
|
" const actualConfigSha256 = templateMetadata && templateMetadata.annotations ? templateMetadata.annotations['unidesk.ai/frpc-config-sha256'] : null;",
|
|
" if (actualConfigSha256 !== expectedConfigSha256) fail('public-exposure-frpc-config-rollout-hash-mismatch', { expected: expectedConfigSha256, actual: actualConfigSha256 });",
|
|
" const containers = Array.isArray(podSpec && podSpec.containers) ? podSpec.containers : [];",
|
|
" const strategyType = deployment.spec && deployment.spec.strategy && deployment.spec.strategy.type;",
|
|
" if (strategyType !== 'Recreate') fail('public-exposure-frpc-strategy-mismatch', { expected: 'Recreate', actual: strategyType || null });",
|
|
" const frpc = containers.find((container) => container && (container.name === 'frpc' || String(container.image || '').includes('frpc')));",
|
|
" const env = frpc && Array.isArray(frpc.env) ? frpc.env.find((item) => item && item.name === 'HWLAB_FRP_TOKEN') : null;",
|
|
" const secretRef = env && env.valueFrom && env.valueFrom.secretKeyRef;",
|
|
" if (!secretRef || secretRef.name !== exposure.secretName || secretRef.key !== tokenKey) fail('public-exposure-frpc-token-env-mismatch', { expectedSecret: exposure.secretName, expectedKey: tokenKey });",
|
|
" checks.push('public-exposure-frpc');",
|
|
"}",
|
|
"console.error(JSON.stringify({ event: 'unidesk-runtime-gitops-verify', ok: true, runtimePath, checks }));",
|
|
"NODE_UNIDESK_RUNTIME_GITOPS_VERIFY`;",
|
|
"}",
|
|
"function patchScript(script) {",
|
|
" let result = String(script || '');",
|
|
" const bootstrap = stepEnvBootstrapScript();",
|
|
" if (bootstrap && !result.includes('unidesk-step-env-bootstrap')) result = `${bootstrap}\\n${result}`;",
|
|
" const inlineDeployServiceIdParser = `const deployText = fs.readFileSync(deployPath, 'utf8');\\nconst deploy = { services: [...deployText.matchAll(/(?:^|\\\\n)\\\\s*serviceId:\\\\s*[\\\"']?([^\\\"'\\\\n#]+)[\\\"']?/g)].map((match) => ({ serviceId: match[1].trim() })) };`;",
|
|
" result = result.split('import { readStructuredFile } from \"./scripts/src/structured-config.mjs\";\\n').join('');",
|
|
" result = result.split('const deploy = await readStructuredFile(process.cwd(), deployPath);').join(inlineDeployServiceIdParser);",
|
|
" if (result.includes('npm run gitops:ts:check')) {",
|
|
" result = result.replace(/\\n[ \\t]*npm run gitops:ts:check\\n/g, '\\n echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-replaces-tsc-gate\"}\\' >&2\\n');",
|
|
" }",
|
|
" const isNodeContractCheck = result.includes('\\\"event\\\":\\\"unidesk-node-contract-check\\\"');",
|
|
" if (isNodeContractCheck) {",
|
|
" result = result.replace(/(^|\\n)([ \\t]*)node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*(?=\\n|$)/g, '$1$2echo \\'{\"event\":\"unidesk-node-contract-check\",\"status\":\"skipped\",\"reason\":\"d601-yaml-render-check-disabled-gitops-render\"}\\' >&2');",
|
|
" }",
|
|
" const prepareSourceDependencyPattern = new RegExp(String.raw`prepare_source_dependencies_started_ms=\"\\$\\(ci_now_ms\\)\"\\nif node -e 'require\\.resolve\\(\"yaml\"\\)'[\\s\\S]*?\\nci_timing_emit prepare-source-dependencies succeeded \"\\$prepare_source_dependencies_started_ms\"`, 'g');",
|
|
" if (result.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"')) {",
|
|
" result = result.replace(prepareSourceDependencyPattern, prepareSourceDependencyScript());",
|
|
" }",
|
|
" const artifactPublishNeedle = 'node scripts/artifact-publish.mjs --publish';",
|
|
" if (result.includes(artifactPublishNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {",
|
|
" result = result.replace(artifactPublishNeedle, `${deployYamlOverlayScript()}\\n${artifactPublishNeedle}`);",
|
|
" }",
|
|
" const gitopsRenderNeedle = 'node scripts/run-bun.mjs scripts/gitops-render.mjs';",
|
|
" if (!isNodeContractCheck && result.includes(gitopsRenderNeedle) && !result.includes('unidesk-deploy-yaml-overlay')) {",
|
|
" result = result.replace(gitopsRenderNeedle, `${deployYamlOverlayScript()}\\n${gitopsRenderNeedle}`);",
|
|
" }",
|
|
" result = result.replaceAll('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808', `node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost || '127.0.0.1'} ${overlay.gitSshProxyPort || 10808}`);",
|
|
" result = result.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);",
|
|
" result = result.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);",
|
|
" result = result.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);",
|
|
" result = result.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
|
|
" result = result.replace(/--gitops-root \"\\$gitops_root\"/g, `--gitops-root ${JSON.stringify(overlay.gitopsRoot)}`);",
|
|
" result = result.replace(/--out \"\\$gitops_root\"/g, `--out ${JSON.stringify(overlay.gitopsRoot)}`);",
|
|
" const legacyRuntimePath = (() => {",
|
|
" const runtimePath = String(overlay.runtimePath || '');",
|
|
" const parts = runtimePath.split('/').filter(Boolean);",
|
|
" if (parts.length < 2) return '';",
|
|
" const leaf = parts.at(-1);",
|
|
" const parent = parts.slice(0, -2).join('/');",
|
|
" return parent && leaf ? `${parent}/${leaf}` : '';",
|
|
" })();",
|
|
" if (legacyRuntimePath && legacyRuntimePath !== overlay.runtimePath) {",
|
|
" result = result.replace('runtime_path=\"$(params.runtime-path)\"', `runtime_path=\"$(params.runtime-path)\"\\nunidesk_legacy_runtime_path=${shellSingle(legacyRuntimePath)}`);",
|
|
" result = result.replace('rm -rf \"$runtime_path\" \"$catalog_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi', 'rm -rf \"$runtime_path\" \"$catalog_path\" \"$unidesk_legacy_runtime_path\"; else rm -rf deploy/gitops/node \"$catalog_path\"; fi');",
|
|
" result = result.replace('git add \"$catalog_path\" \"$runtime_path\"', 'git add \"$catalog_path\" \"$runtime_path\"\\n if [ -n \"${unidesk_legacy_runtime_path:-}\" ]; then git add -A \"$unidesk_legacy_runtime_path\" || true; fi');",
|
|
" }",
|
|
" if (bootstrap && result.includes('git config --global') && !result.includes('unidesk-step-env-bootstrap')) {",
|
|
" result = result.replace(/\\n([ \\t]*)git config --global/g, (match, indent) => `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`);",
|
|
" }",
|
|
" result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--use-deploy-images/g, (match) => {",
|
|
" let next = match;",
|
|
" if (!next.includes('--gitops-root ')) next = next.replace(' --use-deploy-images', ` --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);",
|
|
" if (!next.includes('--out ')) next = next.replace(' --use-deploy-images', ` --out ${JSON.stringify(overlay.gitopsRoot)} --use-deploy-images`);",
|
|
" return next;",
|
|
" });",
|
|
" result = result.replace(/(node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs[^\\n]*--use-deploy-images[^\\n]*)/g, (match) => {",
|
|
" if (match.includes('--check')) return runtimeGitopsVerifyScript();",
|
|
" return `${match}\\n${[runtimePathOverlayScript(), runtimeGitopsPostprocessScript()].filter(Boolean).join('\\n')}`;",
|
|
" });",
|
|
" result = result.replace(/node scripts\\/run-bun\\.mjs scripts\\/gitops-render\\.mjs([^\\n]*?)--out \"\\$render_check_dir\"/g, (match) => match.includes('--gitops-root ') ? match : `${match} --gitops-root ${JSON.stringify(overlay.gitopsRoot)} --node ${shellSingle(overlay.nodeId)} --git-read-url ${shellSingle(overlay.gitReadUrl)} --git-write-url ${shellSingle(overlay.gitWriteUrl)} --runtime-endpoint ${shellSingle(overlay.publicApiUrl)} --web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
|
|
" validatePrepareSourceDependencyScript(result);",
|
|
" return result;",
|
|
"}",
|
|
"function patchManifestObject(doc) {",
|
|
" if (!doc || typeof doc !== 'object') return false;",
|
|
" if (doc.kind !== 'Pipeline' || !doc.spec) return false;",
|
|
" const defaults = {",
|
|
" 'git-url': overlay.gitUrl,",
|
|
" 'git-read-url': overlay.gitReadUrl,",
|
|
" 'git-write-url': overlay.gitWriteUrl,",
|
|
" 'catalog-path': overlay.catalogPath,",
|
|
" 'runtime-path': overlay.runtimePath,",
|
|
" 'registry-prefix': overlay.registryPrefix,",
|
|
" };",
|
|
" for (const param of doc.spec?.params || []) {",
|
|
" if (Object.prototype.hasOwnProperty.call(defaults, param.name)) param.default = defaults[param.name];",
|
|
" }",
|
|
" doc.metadata = doc.metadata || {};",
|
|
" if (typeof overlay.pipelineName === 'string' && overlay.pipelineName.length > 0) doc.metadata.name = overlay.pipelineName;",
|
|
" doc.metadata.annotations = doc.metadata.annotations || {};",
|
|
" doc.metadata.annotations['hwlab.pikastech.local/download-profile'] = overlay.downloadProfileId;",
|
|
" doc.metadata.annotations['hwlab.pikastech.local/network-profile'] = overlay.networkProfileId;",
|
|
" for (const task of doc.spec?.tasks || []) {",
|
|
" for (const sidecar of task.taskSpec?.sidecars || []) {",
|
|
" if (overlay.buildkitSidecarImage && typeof sidecar.image === 'string' && sidecar.image.includes('buildkit')) sidecar.image = overlay.buildkitSidecarImage;",
|
|
" }",
|
|
" for (const step of task.taskSpec?.steps || []) {",
|
|
" if (step.image === overlay.toolsImage && overlay.toolsImagePullPolicy) step.imagePullPolicy = overlay.toolsImagePullPolicy;",
|
|
" if (Array.isArray(step.env)) {",
|
|
" for (const env of step.env) {",
|
|
" if (Object.prototype.hasOwnProperty.call(stepEnv, env.name) && stepEnv[env.name] !== undefined) env.value = stepEnv[env.name];",
|
|
" }",
|
|
" }",
|
|
" step.env = Array.isArray(step.env) ? step.env : [];",
|
|
" const existingEnv = new Set(step.env.map((env) => env.name));",
|
|
" for (const [name, value] of Object.entries(stepEnv)) {",
|
|
" if (value !== undefined && !existingEnv.has(name)) step.env.push({ name, value });",
|
|
" }",
|
|
" if (typeof step.script === 'string') step.script = patchScript(step.script);",
|
|
" }",
|
|
" }",
|
|
" return true;",
|
|
"}",
|
|
"function patchStructuredPipeline() {",
|
|
" try {",
|
|
" const doc = JSON.parse(text);",
|
|
" if (!patchManifestObject(doc)) return false;",
|
|
" text = JSON.stringify(doc, null, 2) + '\\n';",
|
|
" return true;",
|
|
" } catch {}",
|
|
" if (YAML) {",
|
|
" try {",
|
|
" const docs = YAML.parseAllDocuments(text).map((document) => document.toJS()).filter((doc) => doc !== null);",
|
|
" const changed = docs.some((doc) => patchManifestObject(doc));",
|
|
" if (!changed) return false;",
|
|
" text = docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n';",
|
|
" return true;",
|
|
" } catch {}",
|
|
" }",
|
|
" return false;",
|
|
"}",
|
|
"function patchGitMirrorTransportYaml() {",
|
|
" const mirror = overlay.gitMirror || {};",
|
|
" const transport = mirror.githubTransport || {};",
|
|
" if (transport.mode !== 'https') return;",
|
|
" if (!YAML) throw new Error('yaml module is required to patch git-mirror githubTransport=https');",
|
|
" const requireString = (pathName, value) => {",
|
|
" if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror githubTransport=https');",
|
|
" return value;",
|
|
" };",
|
|
" const repository = requireString('gitMirror.sourceRepository', mirror.sourceRepository);",
|
|
" const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);",
|
|
" const tokenSecretName = requireString('gitMirror.githubTransport.tokenSecretName', transport.tokenSecretName);",
|
|
" const tokenSecretKey = requireString('gitMirror.githubTransport.tokenSecretKey', transport.tokenSecretKey);",
|
|
" const tokenSourceRef = requireString('gitMirror.githubTransport.tokenSourceRef', transport.tokenSourceRef);",
|
|
" const username = typeof transport.username === 'string' && transport.username ? transport.username : 'x-access-token';",
|
|
" const remoteUrl = `https://github.com/${repository}.git`;",
|
|
" const proxySummary = `transport=https proxy=HTTP_PROXY authSecret=${tokenSecretName} authKey=${tokenSecretKey} authSourceRef=${tokenSourceRef} source=yaml`;",
|
|
" const askpassBlock = [",
|
|
" 'if [ -z \"${GITHUB_TOKEN:-}\" ]; then echo \\'hwlab git-mirror https auth: missing GITHUB_TOKEN secret env\\' >&2; exit 64; fi',",
|
|
" \"cat > /tmp/hwlab-git-askpass.sh <<'SH_ASKPASS'\",",
|
|
" '#!/bin/sh',",
|
|
" 'case \"$1\" in',",
|
|
" ' *Username*) printf \\'%s\\\\n\\' \"${GITHUB_USERNAME:-x-access-token}\" ;;',",
|
|
" ' *Password*) printf \\'%s\\\\n\\' \"$GITHUB_TOKEN\" ;;',",
|
|
" \" *) printf '\\\\n' ;;\",",
|
|
" 'esac',",
|
|
" 'SH_ASKPASS',",
|
|
" 'chmod 0700 /tmp/hwlab-git-askpass.sh',",
|
|
" `export GITHUB_USERNAME=${shellSingle(username)}`,",
|
|
" 'export GIT_ASKPASS=/tmp/hwlab-git-askpass.sh',",
|
|
" 'export GIT_TERMINAL_PROMPT=0',",
|
|
" 'unset GIT_SSH',",
|
|
" 'unset GIT_SSH_COMMAND',",
|
|
" ].join('\\n');",
|
|
" function patchScript(script, key) {",
|
|
" let next = String(script || '');",
|
|
" if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);",
|
|
" let remoteReplaced = false;",
|
|
" next = next.replace(/repo_url=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `repo_url=${shellSingle(remoteUrl)}`; });",
|
|
" next = next.replace(/remote=(?:\"[^\"]*\"|'[^']*')/g, () => { remoteReplaced = true; return `remote=${shellSingle(remoteUrl)}`; });",
|
|
" if (!remoteReplaced) throw new Error(`generated git-mirror ${key} missing remote url assignment for githubTransport=https`);",
|
|
" next = next.replace(/transport=ssh ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);",
|
|
" next = next.replace(/ssh=GIT_SSH-wrapper source=yaml/g, proxySummary);",
|
|
" next = next.replace(/mkdir -p ([^\\n]*?) \\/root\\/\\.ssh/g, 'mkdir -p $1');",
|
|
" next = next.replace(/\\n[ \\t]*mkdir -p \\/root\\/\\.ssh\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*cp \\/git-ssh\\/ssh-privatekey[^\\n]*\\n[ \\t]*chmod 0?400[^\\n]*\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) <<'NODE_PROXY'[\\s\\S]*?\\nNODE_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs)\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*cat > \\/tmp\\/hwlab-git-ssh-proxy\\.sh <<'SH_PROXY'[\\s\\S]*?\\nSH_PROXY\\n[ \\t]*chmod [^\\n]*\\/tmp\\/hwlab-git-ssh-proxy\\.sh\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*export GIT_SSH=.*\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*export GIT_SSH_COMMAND=.*\\n/g, '\\n');",
|
|
" next = next.replace(/\\n[ \\t]*unset GIT_SSH_COMMAND\\n/g, '\\n');",
|
|
" if (!next.includes('hwlab git-mirror https auth: missing GITHUB_TOKEN secret env')) {",
|
|
" const noProxyExport = /\\nexport no_proxy=[^\\n]*\\n/;",
|
|
" if (noProxyExport.test(next)) next = next.replace(noProxyExport, (match) => `${match}${askpassBlock}\\n`);",
|
|
" else if (next.includes('\\nset -eu\\n')) next = next.replace('\\nset -eu\\n', `\\nset -eu\\n${askpassBlock}\\n`);",
|
|
" else next = `${askpassBlock}\\n${next}`;",
|
|
" }",
|
|
" if (next.includes('/git-ssh') || next.includes('ssh://git@') || next.includes('GIT_SSH=')) throw new Error(`generated git-mirror ${key} still contains ssh transport after githubTransport=https patch`);",
|
|
" if (!next.includes('GIT_ASKPASS') || !next.includes('GITHUB_TOKEN')) throw new Error(`generated git-mirror ${key} missing https auth after githubTransport=https patch`);",
|
|
" return next;",
|
|
" }",
|
|
" const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');",
|
|
" if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);",
|
|
" const docs = YAML.parseAllDocuments(fs.readFileSync(gitMirrorFile, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);",
|
|
" const manifests = [];",
|
|
" for (const doc of docs) {",
|
|
" if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);",
|
|
" else manifests.push(doc);",
|
|
" }",
|
|
" let changed = false;",
|
|
" for (const doc of manifests) {",
|
|
" if (!doc || typeof doc !== 'object' || doc.kind !== 'ConfigMap') continue;",
|
|
" if (!doc.metadata || doc.metadata.name !== configMapName) continue;",
|
|
" doc.data = doc.data || {};",
|
|
" doc.data['sync.sh'] = patchScript(doc.data['sync.sh'], 'sync.sh');",
|
|
" doc.data['flush.sh'] = patchScript(doc.data['flush.sh'], 'flush.sh');",
|
|
" changed = true;",
|
|
" }",
|
|
" if (!changed) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);",
|
|
" fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
|
|
"}",
|
|
"function patchGitMirrorHostRouteYaml() {",
|
|
" const mirror = overlay.gitMirror || {};",
|
|
" const proxy = mirror.egressProxy || {};",
|
|
" if (proxy.mode !== 'host-route') return;",
|
|
" if (!YAML) throw new Error('yaml module is required to patch git-mirror host-route proxy');",
|
|
" const requireString = (pathName, value) => {",
|
|
" if (typeof value !== 'string' || value.length === 0) throw new Error('overlay.' + pathName + ' is required for git-mirror host-route proxy');",
|
|
" return value;",
|
|
" };",
|
|
" const proxyUrl = requireString('gitMirror.egressProxy.proxyUrl', proxy.proxyUrl);",
|
|
" const configMapName = requireString('gitMirror.syncConfigMapName', mirror.syncConfigMapName);",
|
|
" let proxyEndpoint;",
|
|
" try { proxyEndpoint = new URL(proxyUrl); } catch (error) { throw new Error(`overlay.gitMirror.egressProxy.proxyUrl is invalid: ${error.message}`); }",
|
|
" if (proxyEndpoint.protocol !== 'http:') throw new Error('overlay.gitMirror.egressProxy.proxyUrl must use http:// for host-route');",
|
|
" const proxyHost = proxyEndpoint.hostname;",
|
|
" const proxyPort = Number(proxyEndpoint.port || '80');",
|
|
" if (!proxyHost || !Number.isInteger(proxyPort) || proxyPort < 1 || proxyPort > 65535) throw new Error('overlay.gitMirror.egressProxy.proxyUrl must include a valid host and port');",
|
|
" const noProxy = Array.isArray(proxy.noProxy) ? proxy.noProxy.join(',') : '';",
|
|
" const envEntries = [",
|
|
" ['HTTP_PROXY', proxyUrl],",
|
|
" ['HTTPS_PROXY', proxyUrl],",
|
|
" ['ALL_PROXY', proxyUrl],",
|
|
" ['http_proxy', proxyUrl],",
|
|
" ['https_proxy', proxyUrl],",
|
|
" ['all_proxy', proxyUrl],",
|
|
" ['NO_PROXY', noProxy],",
|
|
" ['no_proxy', noProxy],",
|
|
" ];",
|
|
" function readDocs(file) { return YAML.parseAllDocuments(fs.readFileSync(file, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null); }",
|
|
" function flattenDocs(docs) {",
|
|
" const manifests = [];",
|
|
" for (const doc of docs) {",
|
|
" if (doc && typeof doc === 'object' && doc.kind === 'List' && Array.isArray(doc.items)) manifests.push(...doc.items);",
|
|
" else manifests.push(doc);",
|
|
" }",
|
|
" return manifests;",
|
|
" }",
|
|
" function setContainerEnv(container, name, value) {",
|
|
" if (!container || typeof container !== 'object') return false;",
|
|
" container.env = Array.isArray(container.env) ? container.env : [];",
|
|
" let item = container.env.find((env) => env && env.name === name);",
|
|
" if (!item) { item = { name }; container.env.push(item); }",
|
|
" const changed = item.value !== value || item.valueFrom !== undefined;",
|
|
" item.value = value;",
|
|
" delete item.valueFrom;",
|
|
" return changed;",
|
|
" }",
|
|
" function patchProxyScript(script, key) {",
|
|
" let next = String(script || '');",
|
|
" if (next.length === 0) throw new Error(`generated git-mirror ConfigMap ${configMapName} missing ${key}`);",
|
|
" next = next.replace(/node \\/tmp\\/hwlab-github-proxy-connect\\.(?:mjs|cjs) \\S+ \\d+/g, `node /tmp/hwlab-github-proxy-connect.mjs ${proxyHost} ${proxyPort}`);",
|
|
" const missing = [];",
|
|
" for (const [name, value] of envEntries) {",
|
|
" let replaced = false;",
|
|
" const exportPattern = new RegExp(`(^|\\\\n)([ \\\\t]*export ${escapeRegExp(name)}=)[^\\\\n]*`, 'g');",
|
|
" next = next.replace(exportPattern, (_match, prefix, left) => { replaced = true; return `${prefix}${left}${shellSingle(value)}`; });",
|
|
" if (!replaced) missing.push([name, value]);",
|
|
" }",
|
|
" if (missing.length > 0) {",
|
|
" const block = missing.map(([name, value]) => `export ${name}=${shellSingle(value)}`).join('\\\\n');",
|
|
" if (next.includes('\\\\nset -eu\\\\n')) next = next.replace('\\\\nset -eu\\\\n', `\\\\nset -eu\\\\n${block}\\\\n`);",
|
|
" else next = `${block}\\\\n${next}`;",
|
|
" }",
|
|
" next = next.replace(/mode=node-global/g, 'mode=host-route');",
|
|
" return next;",
|
|
" }",
|
|
" const gitMirrorFile = path.join(renderDir, 'devops-infra', 'git-mirror.yaml');",
|
|
" if (!fs.existsSync(gitMirrorFile)) throw new Error(`generated git-mirror manifest missing: ${gitMirrorFile}`);",
|
|
" const docs = readDocs(gitMirrorFile);",
|
|
" const manifests = flattenDocs(docs);",
|
|
" let configMapChanged = false;",
|
|
" let workloadChanged = false;",
|
|
" const workloadNames = new Set([mirror.serviceReadName, mirror.serviceWriteName].filter((value) => typeof value === 'string' && value.length > 0));",
|
|
" for (const doc of manifests) {",
|
|
" if (!doc || typeof doc !== 'object') continue;",
|
|
" if (doc.kind === 'ConfigMap' && doc.metadata && doc.metadata.name === configMapName) {",
|
|
" doc.data = doc.data || {};",
|
|
" doc.data['sync.sh'] = patchProxyScript(doc.data['sync.sh'], 'sync.sh');",
|
|
" doc.data['flush.sh'] = patchProxyScript(doc.data['flush.sh'], 'flush.sh');",
|
|
" configMapChanged = true;",
|
|
" continue;",
|
|
" }",
|
|
" if (!['Deployment', 'StatefulSet', 'DaemonSet', 'ReplicaSet', 'ReplicationController'].includes(doc.kind)) continue;",
|
|
" const name = doc.metadata && doc.metadata.name ? String(doc.metadata.name) : '';",
|
|
" const labels = doc.metadata && doc.metadata.labels && typeof doc.metadata.labels === 'object' ? doc.metadata.labels : {};",
|
|
" if (!workloadNames.has(name) && labels['app.kubernetes.io/name'] !== 'git-mirror' && !name.includes('git-mirror')) continue;",
|
|
" doc.spec = doc.spec || {};",
|
|
" doc.spec.template = doc.spec.template || {};",
|
|
" doc.spec.template.metadata = doc.spec.template.metadata || {};",
|
|
" doc.spec.template.metadata.annotations = doc.spec.template.metadata.annotations || {};",
|
|
" doc.spec.template.metadata.annotations['unidesk.ai/git-mirror-egress-proxy'] = 'host-route';",
|
|
" doc.spec.template.metadata.annotations['unidesk.ai/git-mirror-host-proxy-config-ref'] = String(proxy.hostProxyConfigRef || '');",
|
|
" doc.spec.template.spec = doc.spec.template.spec || {};",
|
|
" if (proxy.podHostNetwork) {",
|
|
" doc.spec.template.spec.hostNetwork = true;",
|
|
" doc.spec.template.spec.dnsPolicy = 'ClusterFirstWithHostNet';",
|
|
" } else {",
|
|
" delete doc.spec.template.spec.hostNetwork;",
|
|
" delete doc.spec.template.spec.dnsPolicy;",
|
|
" }",
|
|
" for (const group of ['containers', 'initContainers']) {",
|
|
" for (const container of Array.isArray(doc.spec.template.spec[group]) ? doc.spec.template.spec[group] : []) {",
|
|
" for (const [envName, envValue] of envEntries) setContainerEnv(container, envName, envValue);",
|
|
" }",
|
|
" }",
|
|
" workloadChanged = true;",
|
|
" }",
|
|
" if (!configMapChanged) throw new Error(`generated git-mirror ConfigMap ${configMapName} was not found in ${gitMirrorFile}`);",
|
|
" if (!workloadChanged) throw new Error(`generated git-mirror workload for host-route proxy was not found in ${gitMirrorFile}`);",
|
|
" fs.writeFileSync(gitMirrorFile, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
|
|
"}",
|
|
"const structured = patchStructuredPipeline();",
|
|
"function replaceParamDefault(name, value) {",
|
|
" const namePattern = escapeRegExp(name);",
|
|
" text = text.replace(new RegExp(`(- default: )[^\\n]*(\\n\\\\s+name: ${namePattern}\\n\\\\s+type: string)`, 'g'), `$1${yamlString(value)}$2`);",
|
|
" text = text.replace(new RegExp(`(- name: ${namePattern}\\n\\\\s+type: string\\n\\\\s+default: )[^\\n]*`, 'g'), `$1${yamlString(value)}`);",
|
|
"}",
|
|
"replaceParamDefault('git-url', overlay.gitUrl);",
|
|
"replaceParamDefault('git-read-url', overlay.gitReadUrl);",
|
|
"replaceParamDefault('git-write-url', overlay.gitWriteUrl);",
|
|
"replaceParamDefault('catalog-path', overlay.catalogPath);",
|
|
"replaceParamDefault('runtime-path', overlay.runtimePath);",
|
|
"replaceParamDefault('registry-prefix', overlay.registryPrefix);",
|
|
"text = text.replace(/hwlab\\.pikastech\\.local\\/download-profile: [^\\n]+/g, `hwlab.pikastech.local/download-profile: ${overlay.downloadProfileId}`);",
|
|
"text = text.replace(/hwlab\\.pikastech\\.local\\/network-profile: [^\\n]+/g, `hwlab.pikastech.local/network-profile: ${overlay.networkProfileId}`);",
|
|
"if (overlay.buildkitSidecarImage) text = text.replace(/moby\\/buildkit:rootless/g, overlay.buildkitSidecarImage);",
|
|
"text = text.replace(/--git-read-url '[^']*'/g, `--git-read-url ${shellSingle(overlay.gitReadUrl)}`);",
|
|
"text = text.replace(/--git-write-url '[^']*'/g, `--git-write-url ${shellSingle(overlay.gitWriteUrl)}`);",
|
|
"text = text.replace(/--runtime-endpoint '[^']*'/g, `--runtime-endpoint ${shellSingle(overlay.publicApiUrl)}`);",
|
|
"text = text.replace(/--web-endpoint '[^']*'/g, `--web-endpoint ${shellSingle(overlay.publicWebUrl)}`);",
|
|
"for (const [name, value] of Object.entries(stepEnv)) {",
|
|
" if (value === undefined) continue;",
|
|
" text = text.replace(new RegExp(`(- name: ${escapeRegExp(name)}\\\\n\\\\s+value: )[^\\\\n]+`, 'g'), `$1${yamlString(value)}`);",
|
|
" text = text.replace(new RegExp(`(\\\"name\\\": ${JSON.stringify(name)},\\\\n\\\\s+\\\"value\\\": )\\\"[^\\\"]*\\\"`, 'g'), `$1${yamlString(value)}`);",
|
|
"}",
|
|
"const bootstrap = stepEnvBootstrapScript();",
|
|
"if (bootstrap) {",
|
|
" text = text.replace(/\\n([ \\t]*)git config --global/g, (match, indent, offset, fullText) => {",
|
|
" const previous = fullText.slice(Math.max(0, offset - 500), offset);",
|
|
" if (previous.includes('unidesk-step-env-bootstrap')) return match;",
|
|
" return `\\n${indent}${bootstrap.split('\\n').join(`\\n${indent}`)}\\n${indent}git config --global`;",
|
|
" });",
|
|
"}",
|
|
"if (overlay.gitSshProxyHost && overlay.gitSshProxyPort) {",
|
|
" text = text.split('node /tmp/hwlab-github-proxy-connect.mjs 127.0.0.1 10808').join(`node /tmp/hwlab-github-proxy-connect.mjs ${overlay.gitSshProxyHost} ${overlay.gitSshProxyPort}`);",
|
|
"}",
|
|
"const quotedRoot = JSON.stringify(overlay.gitopsRoot);",
|
|
"const escapedQuotedRoot = quotedRoot.replaceAll('\"', '\\\\\"');",
|
|
"const replacements = [",
|
|
" ['--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \"$(params.registry-prefix)\" --use-deploy-images' : `--registry-prefix \"$(params.registry-prefix)\" --gitops-root ${quotedRoot} --use-deploy-images`],",
|
|
" ['--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images', text.includes('--gitops-root ') ? '--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --use-deploy-images' : `--registry-prefix \\\\\"$(params.registry-prefix)\\\\\" --gitops-root ${escapedQuotedRoot} --use-deploy-images`],",
|
|
"];",
|
|
"let changed = false;",
|
|
"for (const [needle, replacement] of replacements) {",
|
|
" if (!text.includes(needle)) continue;",
|
|
" text = text.split(needle).join(replacement);",
|
|
" changed = true;",
|
|
"}",
|
|
"if (!structured && !changed && !text.includes(`--gitops-root ${quotedRoot}`) && !text.includes(`--gitops-root ${escapedQuotedRoot}`)) { throw new Error(`generated pipeline missing expected gitops-render invocation in ${pipelinePath}`); }",
|
|
"if (text.includes('prepare_source_dependencies_started_ms=\"$(ci_now_ms)\"') && text.includes('npm ci --ignore-scripts --no-audit --prefer-offline')) { throw new Error(`generated pipeline still uses full npm ci prepare-source dependency install in ${pipelinePath}`); }",
|
|
"if (text.includes('readStructuredFile(process.cwd(), deployPath)')) { throw new Error(`generated pipeline still imports YAML parser for prepare-source catalog check in ${pipelinePath}`); }",
|
|
"if (text.includes('/yaml/-/yaml-') || text.includes('bun add --no-save --ignore-scripts') || text.includes('npm install --package-lock=false --no-save')) { throw new Error(`generated pipeline still downloads yaml during prepare-source in ${pipelinePath}`); }",
|
|
"if (text.includes('npm run gitops:ts:check')) { throw new Error(`generated pipeline still uses npm gitops:ts:check gate in ${pipelinePath}`); }",
|
|
"fs.writeFileSync(pipelinePath, text);",
|
|
"patchGitMirrorHostRouteYaml();",
|
|
"patchGitMirrorTransportYaml();",
|
|
"function patchArgoYaml(filePath) {",
|
|
" if (!YAML || !fs.existsSync(filePath)) return;",
|
|
" const docs = YAML.parseAllDocuments(fs.readFileSync(filePath, 'utf8')).map((document) => document.toJS()).filter((doc) => doc !== null);",
|
|
" let changed = false;",
|
|
" for (const doc of docs) {",
|
|
" if (!doc || typeof doc !== 'object') continue;",
|
|
" if (doc.kind === 'AppProject') {",
|
|
" doc.spec = doc.spec || {};",
|
|
" doc.spec.sourceRepos = [overlay.argoRepoUrl];",
|
|
" changed = true;",
|
|
" }",
|
|
" if (doc.kind === 'Application') {",
|
|
" doc.spec = doc.spec || {};",
|
|
" doc.spec.source = { ...(doc.spec.source || {}), repoURL: overlay.argoRepoUrl, targetRevision: overlay.gitopsBranch, path: overlay.runtimePath };",
|
|
" changed = true;",
|
|
" }",
|
|
" }",
|
|
" if (changed) fs.writeFileSync(filePath, docs.map((doc) => YAML.stringify(doc).trimEnd()).join('\\n---\\n') + '\\n');",
|
|
"}",
|
|
"patchArgoYaml(path.join(renderDir, 'argocd', 'project.yaml'));",
|
|
"patchArgoYaml(path.join(renderDir, 'argocd', overlay.argoApplicationFile));",
|
|
"NODE",
|
|
];
|
|
}
|