From 82439833b42ecb61f964598bd7df0b1eeda041c7 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 4 Jul 2026 12:42:27 +0000 Subject: [PATCH] fix: keep hwlab runtime gitops guard on manual refresh --- .../cicd/hwlab-node-control-plane-refresh.mjs | 37 ++++ .../hwlab/runtime-gitops-pipeline-guard.mjs | 206 ++++++++++++++++++ scripts/src/cicd-controller-render.ts | 3 +- scripts/src/cicd-hwlab-refresh.ts | 2 +- scripts/src/hwlab-node/render.ts | 32 +++ scripts/src/hwlab-node/web-probe.ts | 1 + 6 files changed, 279 insertions(+), 2 deletions(-) create mode 100644 scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs diff --git a/scripts/native/cicd/hwlab-node-control-plane-refresh.mjs b/scripts/native/cicd/hwlab-node-control-plane-refresh.mjs index a590b6fd..0bc25814 100644 --- a/scripts/native/cicd/hwlab-node-control-plane-refresh.mjs +++ b/scripts/native/cicd/hwlab-node-control-plane-refresh.mjs @@ -144,6 +144,7 @@ async function applyPipeline() { if (typeof renderedPipelineName !== "string" || renderedPipelineName.length === 0) { throw new Error(`rendered Pipeline metadata.name missing: ${pipelinePath}`); } + const runtimeGitopsScripts = await applyRuntimeGitopsScriptsConfigMap(); const runtimeGitopsGuard = injectRuntimeGitopsGuard(pipeline); const render = summarizeRenderedPipeline(pipeline, renderedPipelineName, runtimeGitopsGuard); const pipelineName = requiredOverlayString("pipelineName"); @@ -158,10 +159,46 @@ async function applyPipeline() { ); return { render, + runtimeGitopsScripts, apply: summarizeAppliedPipeline(parseJsonObject(applyText), pipelineName, tektonNamespace), }; } +async function applyRuntimeGitopsScriptsConfigMap() { + const data = {}; + for (const name of ["runtime-gitops-observability.mjs", "runtime-gitops-postprocess.mjs", "runtime-gitops-verify.mjs"]) { + data[name] = readFileSync(`/etc/unidesk-cicd-branch-follower/${name}`, "utf8"); + } + const YAML = yamlModule(); + const applied = parseJsonObject(await kubeRequest( + "PATCH", + `/api/v1/namespaces/${encodeURIComponent(tektonNamespace)}/configmaps/${encodeURIComponent(runtimeGitopsConfigMapName)}?fieldManager=${encodeURIComponent(fieldManager)}&force=true`, + YAML.stringify({ + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: runtimeGitopsConfigMapName, + namespace: tektonNamespace, + labels: { + "app.kubernetes.io/name": "hwlab-runtime-gitops-scripts", + "app.kubernetes.io/part-of": "unidesk-cicd-branch-follower", + "hwlab.pikastech.local/node": overlay.nodeId, + "hwlab.pikastech.local/lane": overlay.lane, + }, + }, + data, + }), + "application/apply-patch+yaml", + )); + const metadata = recordOrNull(applied?.metadata); + return { + name: stringOrNull(metadata?.name) || runtimeGitopsConfigMapName, + namespace: stringOrNull(metadata?.namespace) || tektonNamespace, + resourceVersion: stringOrNull(metadata?.resourceVersion), + keyCount: Object.keys(data).length, + }; +} + function yamlModule() { const requireFromDeps = createRequire(path.join(depsDir, "package.json")); return requireFromDeps("yaml"); diff --git a/scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs b/scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs new file mode 100644 index 00000000..3169eb90 --- /dev/null +++ b/scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs @@ -0,0 +1,206 @@ +#!/usr/bin/env node +// SPEC: PJ2026-01060703 HWLAB runtime GitOps Pipeline guard. +// Responsibility: patch rendered Tekton Pipeline GitOps steps to keep runtime GitOps gates active on no-build paths. +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { createRequire } from "node:module"; + +const requireFromScript = createRequire(import.meta.url); +const requireFromCwd = createRequire(path.join(process.cwd(), "package.json")); +const YAML = requireYaml(); +const args = parseArgs(process.argv.slice(2)); +const overlay = readOverlay(); +const pipelinePath = requiredArg("pipeline"); +const scriptsConfigMapPath = args.scriptsConfigMap ?? args.scriptsConfigmap ?? null; +const scriptsDir = args.scriptsDir ?? null; +const namespaceArg = args.namespace ?? null; +const configMapName = args.configMapName ?? args.configmapName ?? `${requiredOverlayString("pipelineName")}-runtime-gitops-scripts`; + +if (!existsSync(pipelinePath)) throw new Error(`rendered Pipeline missing: ${pipelinePath}`); + +const docs = YAML.parseAllDocuments(readFileSync(pipelinePath, "utf8")).map((document) => document.toJS()).filter((doc) => doc !== null); +let patched = false; +let summary = null; +for (const doc of docs) { + if (!doc || typeof doc !== "object" || doc.kind !== "Pipeline") continue; + summary = injectRuntimeGitopsGuard(doc); + patched = true; +} +if (!patched || summary === null) throw new Error(`rendered Pipeline not found: ${pipelinePath}`); +writeFileSync(pipelinePath, `${docs.map((doc) => YAML.stringify(doc).trimEnd()).join("\n---\n")}\n`, "utf8"); + +let scriptsConfigMap = null; +if (scriptsConfigMapPath !== null) { + const namespace = namespaceArg || pipelineNamespace(docs) || "hwlab-ci"; + scriptsConfigMap = writeScriptsConfigMap(scriptsConfigMapPath, namespace); +} + +console.error(JSON.stringify({ + event: "unidesk-runtime-gitops-pipeline-guard", + ok: true, + pipelinePath, + runtimeGitopsGuard: summary, + scriptsConfigMap, +})); + +function injectRuntimeGitopsGuard(pipeline) { + const tasks = Array.isArray(pipeline?.spec?.tasks) ? pipeline.spec.tasks : []; + const task = tasks.find((item) => item && item.name === "gitops-promote"); + if (!task) throw new Error("runtime GitOps guard injection failed: gitops-promote task missing"); + task.taskSpec = objectOr(task.taskSpec); + const steps = Array.isArray(task.taskSpec.steps) ? task.taskSpec.steps : []; + const step = steps.find((item) => typeof item?.script === "string" && item.script.includes("scripts/gitops-render.mjs")); + if (!step) throw new Error("runtime GitOps guard injection failed: gitops-promote render step missing"); + + const originalScript = String(step.script); + let script = patchNoBuildNoRolloutSkip(originalScript); + const noBuildSkipPatched = script !== originalScript; + const beforeInjection = script; + script = injectRuntimeGitopsCommands(script); + if (hasNoBuildNoRolloutEarlyExit(script)) { + throw new Error("runtime GitOps guard injection failed: no-build-no-rollout-plan early exit remains after patch"); + } + const postprocessPresent = hasPostprocess(script); + const verifyPresent = hasVerify(script); + if (!postprocessPresent || !verifyPresent) throw new Error("runtime GitOps guard injection failed: postprocess/verify commands missing after patch"); + step.script = script; + ensureRuntimeGitopsScriptsMount(task.taskSpec, step); + return { + present: true, + configMapName, + stepName: typeof step.name === "string" ? step.name : null, + noBuildSkipPatched, + postprocessInjected: !hasPostprocess(beforeInjection) && postprocessPresent, + verifyInjected: !hasVerify(beforeInjection) && verifyPresent, + }; +} + +function patchNoBuildNoRolloutSkip(script) { + return String(script).replace( + /(^|\n)([ \t]*)echo '\{"event":"gitops-promote","status":"skipped","reason":"no-build-no-rollout-plan"\}'(?:\s*>\s*&2)?\n[ \t]*exit 0(?=\n|$)/u, + (_match, prefix, indent) => `${prefix}${indent}echo '{"event":"gitops-promote","status":"continuing","reason":"no-build-no-rollout-plan-gitops-verify"}' >&2`, + ); +} + +function hasNoBuildNoRolloutEarlyExit(script) { + return /echo '\{"event":"gitops-promote","status":"skipped","reason":"no-build-no-rollout-plan"\}'(?:\s*>\s*&2)?\n[ \t]*exit 0(?=\n|$)/u.test(String(script)); +} + +function injectRuntimeGitopsCommands(script) { + if (hasPostprocess(script) && hasVerify(script)) return script; + const overlayEnv = `UNIDESK_RUNTIME_GITOPS_OVERLAY_B64=${shellSingle(Buffer.from(JSON.stringify({ + runtimePath: overlay.runtimePath, + observability: overlay.observability, + }), "utf8").toString("base64"))}`; + const postprocess = `${overlayEnv} node /etc/unidesk-cicd-runtime-gitops/runtime-gitops-postprocess.mjs`; + const verify = `${overlayEnv} node /etc/unidesk-cicd-runtime-gitops/runtime-gitops-verify.mjs`; + return String(script).replace( + /(node scripts\/run-bun\.mjs scripts\/gitops-render\.mjs[^\n]*--use-deploy-images[^\n]*)/g, + (match) => { + if (match.includes("--check")) return hasVerify(script) ? match : `${match}\n${verify}`; + return hasPostprocess(script) ? match : `${match}\n${postprocess}`; + }, + ); +} + +function hasPostprocess(script) { + const value = String(script); + return value.includes("runtime-gitops-postprocess.mjs") || value.includes("unidesk-runtime-gitops-postprocess"); +} + +function hasVerify(script) { + const value = String(script); + return value.includes("runtime-gitops-verify.mjs") || value.includes("unidesk-runtime-gitops-verify"); +} + +function ensureRuntimeGitopsScriptsMount(taskSpec, step) { + const volumeName = "unidesk-runtime-gitops-scripts"; + taskSpec.volumes = Array.isArray(taskSpec.volumes) ? taskSpec.volumes : []; + if (!taskSpec.volumes.some((item) => item && item.name === volumeName)) { + taskSpec.volumes.push({ name: volumeName, configMap: { name: configMapName, defaultMode: 0o755 } }); + } + step.volumeMounts = Array.isArray(step.volumeMounts) ? step.volumeMounts : []; + if (!step.volumeMounts.some((item) => item && item.name === volumeName)) { + step.volumeMounts.push({ name: volumeName, mountPath: "/etc/unidesk-cicd-runtime-gitops", readOnly: true }); + } +} + +function writeScriptsConfigMap(targetPath, namespace) { + if (scriptsDir === null) throw new Error("--scripts-dir is required with --scripts-configmap"); + const data = {}; + for (const name of ["runtime-gitops-observability.mjs", "runtime-gitops-postprocess.mjs", "runtime-gitops-verify.mjs"]) { + data[name] = readFileSync(path.join(scriptsDir, name), "utf8"); + } + writeFileSync(targetPath, `${YAML.stringify({ + apiVersion: "v1", + kind: "ConfigMap", + metadata: { + name: configMapName, + namespace, + labels: { + "app.kubernetes.io/name": "hwlab-runtime-gitops-scripts", + "app.kubernetes.io/part-of": "unidesk-hwlab-control-plane", + "hwlab.pikastech.local/node": overlay.nodeId ?? null, + "hwlab.pikastech.local/lane": overlay.lane ?? null, + }, + }, + data, + }).trimEnd()}\n`, "utf8"); + return { name: configMapName, namespace, keyCount: Object.keys(data).length, path: targetPath }; +} + +function pipelineNamespace(docs) { + for (const doc of docs) { + const namespace = doc?.metadata?.namespace; + if (typeof namespace === "string" && namespace.length > 0) return namespace; + } + return null; +} + +function readOverlay() { + const encoded = process.env.UNIDESK_RUNTIME_GITOPS_OVERLAY_B64; + if (!encoded) throw new Error("UNIDESK_RUNTIME_GITOPS_OVERLAY_B64 is required"); + return JSON.parse(Buffer.from(encoded, "base64").toString("utf8")); +} + +function parseArgs(values) { + const out = {}; + for (let index = 0; index < values.length; index += 1) { + const value = values[index]; + if (!value.startsWith("--")) throw new Error(`unexpected argument: ${value}`); + const key = value.slice(2).replace(/-([a-z])/gu, (_match, char) => char.toUpperCase()); + const next = values[index + 1]; + if (next === undefined || next.startsWith("--")) throw new Error(`${value} requires a value`); + out[key] = next; + index += 1; + } + return out; +} + +function requiredArg(name) { + const value = args[name]; + if (typeof value !== "string" || value.length === 0) throw new Error(`--${name} is required`); + return value; +} + +function requiredOverlayString(name) { + const value = overlay[name]; + if (typeof value !== "string" || value.length === 0) throw new Error(`overlay.${name} is required`); + return value; +} + +function objectOr(value) { + return value && typeof value === "object" && !Array.isArray(value) ? value : {}; +} + +function shellSingle(value) { + return `'${String(value).replaceAll("'", "'\\''")}'`; +} + +function requireYaml() { + try { + return requireFromScript("yaml"); + } catch { + return requireFromCwd("yaml"); + } +} diff --git a/scripts/src/cicd-controller-render.ts b/scripts/src/cicd-controller-render.ts index 79e1c47a..5d30a7d3 100644 --- a/scripts/src/cicd-controller-render.ts +++ b/scripts/src/cicd-controller-render.ts @@ -203,7 +203,8 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec kind: "ClusterRole", metadata: { name: registry.controller.serviceAccountName, labels }, rules: [ - { apiGroups: [""], resources: ["pods", "pods/log", "configmaps", "events"], verbs: ["get", "list", "watch"] }, + { apiGroups: [""], resources: ["pods", "pods/log", "events"], verbs: ["get", "list", "watch"] }, + { apiGroups: [""], resources: ["configmaps"], verbs: ["get", "list", "watch", "create", "update", "patch"] }, { apiGroups: [""], resources: ["pods/exec"], verbs: ["create"] }, { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "delete"] }, { apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] }, diff --git a/scripts/src/cicd-hwlab-refresh.ts b/scripts/src/cicd-hwlab-refresh.ts index 5d1d678f..e141ea2f 100644 --- a/scripts/src/cicd-hwlab-refresh.ts +++ b/scripts/src/cicd-hwlab-refresh.ts @@ -83,7 +83,7 @@ export function nativeHwlabControlPlaneRefreshJobManifest( { name: "FIELD_MANAGER", value: spec.controlPlaneFieldManager }, { name: "TEKTON_NAMESPACE", value: tektonNamespace }, { name: "KUBE_REQUEST_TIMEOUT_SECONDS", value: String(timeoutSeconds) }, - { name: "RUNTIME_GITOPS_CONFIGMAP_NAME", value: registry.controller.configMapName }, + { name: "RUNTIME_GITOPS_CONFIGMAP_NAME", value: `${spec.pipeline}-runtime-gitops-scripts` }, { name: "HWLAB_RENDER_OVERLAY_B64", value: overlay }, ], }], diff --git a/scripts/src/hwlab-node/render.ts b/scripts/src/hwlab-node/render.ts index 836ec0bb..0018a31f 100644 --- a/scripts/src/hwlab-node/render.ts +++ b/scripts/src/hwlab-node/render.ts @@ -41,6 +41,9 @@ import { webObserveShort, webObserveText } from "./web-probe-observe"; import { hwlabRuntimeActiveExternalPostgres } from "../hwlab-node-lanes"; const runtimeGitopsObservabilityNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-observability.mjs"), "utf8").trimEnd(); +const runtimeGitopsPipelineGuardNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-pipeline-guard.mjs"), "utf8").trimEnd(); +const runtimeGitopsPostprocessNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-postprocess.mjs"), "utf8").trimEnd(); +const runtimeGitopsVerifyNativeScript = readFileSync(rootPath("scripts/native/hwlab/runtime-gitops-verify.mjs"), "utf8").trimEnd(); export function nodeRuntimeGitMirrorJobName(mirror: NodeRuntimeGitMirrorTargetSpec, action: "sync" | "flush"): string { const prefix = action === "sync" ? mirror.syncJobPrefix : mirror.flushJobPrefix; @@ -2648,5 +2651,34 @@ export function nodeRuntimePipelinePostprocessScript(): string[] { "patchArgoYaml(path.join(renderDir, 'argocd', 'project.yaml'));", "patchArgoYaml(path.join(renderDir, 'argocd', overlay.argoApplicationFile));", "NODE", + ...runtimeGitopsPipelineGuardScript(), + ]; +} + +function runtimeGitopsPipelineGuardScript(): string[] { + return [ + "runtime_gitops_guard_dir=\"$render_dir/.unidesk-runtime-gitops\"", + "mkdir -p \"$runtime_gitops_guard_dir\"", + ...writeRuntimeGitopsNativeScript("runtime-gitops-pipeline-guard.mjs", runtimeGitopsPipelineGuardNativeScript, "UNIDESK_RUNTIME_GITOPS_PIPELINE_GUARD_MJS"), + ...writeRuntimeGitopsNativeScript("runtime-gitops-observability.mjs", runtimeGitopsObservabilityNativeScript, "UNIDESK_RUNTIME_GITOPS_OBSERVABILITY_MJS"), + ...writeRuntimeGitopsNativeScript("runtime-gitops-postprocess.mjs", runtimeGitopsPostprocessNativeScript, "UNIDESK_RUNTIME_GITOPS_POSTPROCESS_MJS"), + ...writeRuntimeGitopsNativeScript("runtime-gitops-verify.mjs", runtimeGitopsVerifyNativeScript, "UNIDESK_RUNTIME_GITOPS_VERIFY_MJS"), + [ + "UNIDESK_RUNTIME_GITOPS_OVERLAY_B64=\"$overlay_b64\"", + "node \"$runtime_gitops_guard_dir/runtime-gitops-pipeline-guard.mjs\"", + "--pipeline \"$render_dir/$(node -e 'const o=JSON.parse(Buffer.from(process.argv[1],\"base64\").toString(\"utf8\")); process.stdout.write(o.tektonDir)' \"$overlay_b64\")/pipeline.yaml\"", + "--scripts-configmap \"$render_dir/$(node -e 'const o=JSON.parse(Buffer.from(process.argv[1],\"base64\").toString(\"utf8\")); process.stdout.write(o.tektonDir)' \"$overlay_b64\")/runtime-gitops-scripts.yaml\"", + `--namespace ${shellQuote(HWLAB_CI_NAMESPACE)}`, + "--scripts-dir \"$runtime_gitops_guard_dir\"", + ].join(" "), + ]; +} + +function writeRuntimeGitopsNativeScript(name: string, content: string, marker: string): string[] { + return [ + `cat > "$runtime_gitops_guard_dir/${name}" <<'${marker}'`, + content, + marker, + `chmod 0755 "$runtime_gitops_guard_dir/${name}"`, ]; } diff --git a/scripts/src/hwlab-node/web-probe.ts b/scripts/src/hwlab-node/web-probe.ts index d3c0302a..3ab8a65e 100644 --- a/scripts/src/hwlab-node/web-probe.ts +++ b/scripts/src/hwlab-node/web-probe.ts @@ -178,6 +178,7 @@ export function nodeRuntimeControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderD return [ `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`, `${renderDir}/${spec.tektonDir}/rbac.yaml`, + `${renderDir}/${spec.tektonDir}/runtime-gitops-scripts.yaml`, `${renderDir}/${spec.tektonDir}/pipeline.yaml`, `${renderDir}/argocd/project.yaml`, `${renderDir}/argocd/${spec.argoApplicationFile}`,