// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/hwlab-g14.ts. // Moved mechanically from scripts/src/hwlab-g14.ts:3346-3736 for #903. import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { createHash, randomBytes } from "node:crypto"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand } from "../command"; import { cancelJob, listJobs, readJob, startJob } from "../jobs"; import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes"; import type { CommandJsonResult } from "./types"; import { g14HostScript } from "./images"; import { compactCommandResult, g14K3s, isCommandSuccess, record, runG14K3sRemoteAsync, shellQuote, shortSha, textHash } from "./remote"; import { runtimeLaneCicdRepoEnsureScript } from "./source"; import { CI_NAMESPACE, G14_PROVIDER, V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS, V02_CONTROL_PLANE_REFRESH_TTL_MS, V02_LANE_SPEC, V02_POLLER, V02_RECONCILER } from "./types"; import { timestampMs } from "./v02-status"; export function runtimeLaneControlPlaneRenderScript(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token = runtimeLaneRenderToken()): string { const renderDir = runtimeLaneRenderDir(spec, sourceCommit, token); const worktreeDir = runtimeLaneRenderWorktreeDir(spec, sourceCommit, token); return [ "set -eu", runtimeLaneCicdRepoEnsureScript(spec), `source_commit=${shellQuote(sourceCommit)}`, `render_dir=${shellQuote(renderDir)}`, `worktree_dir=${shellQuote(worktreeDir)}`, `render_lane=${shellQuote(spec.lane)}`, "cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }", "trap cleanup_render_worktree EXIT", `test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = ${shellQuote(sourceCommit)}`, "cleanup_render_worktree", "rm -rf \"$render_dir\"", "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", "git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"", "git -C \"$worktree_dir\" checkout --detach \"$source_commit\"", "test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"", "cd \"$worktree_dir\"", "if [ ! -d node_modules/yaml ]; then", " if [ -f package-lock.json ]; then", " npm ci --ignore-scripts --no-audit --prefer-offline", " elif [ -f bun.lock ] || [ -f bun.lockb ]; then", " bun install --frozen-lockfile --ignore-scripts", " else", ` echo "${spec.version} control-plane render cannot install dependencies: no lockfile found" >&2`, " exit 42", " fi", "fi", "if [ -f scripts/gitops-render.mjs ]; then", " render_script=scripts/gitops-render.mjs", "elif [ \"$render_lane\" = v02 ] && [ -f scripts/g14-gitops-render.mjs ]; then", " render_script=scripts/g14-gitops-render.mjs", "else", ` echo "${spec.version} control-plane render script missing: scripts/gitops-render.mjs" >&2`, " exit 43", "fi", "if [ -f scripts/run-bun.mjs ]; then", ` node scripts/run-bun.mjs "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, "else", ` node "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, "fi", ].join("\n"); } export function v02ControlPlaneRenderScript(sourceCommit: string, token = v02RenderToken()): string { return runtimeLaneControlPlaneRenderScript(V02_LANE_SPEC, sourceCommit, token); } export interface V02RenderTempResult { result: CommandJsonResult; renderDir: string; worktreeDir: string; token: string; } export function runRuntimeLaneRenderToTemp(spec: HwlabRuntimeLaneSpec, sourceCommit: string): V02RenderTempResult { const token = runtimeLaneRenderToken(); return { result: g14HostScript(runtimeLaneControlPlaneRenderScript(spec, sourceCommit, token), 180_000), renderDir: runtimeLaneRenderDir(spec, sourceCommit, token), worktreeDir: runtimeLaneRenderWorktreeDir(spec, sourceCommit, token), token, }; } export function runV02RenderToTemp(sourceCommit: string): V02RenderTempResult { return runRuntimeLaneRenderToTemp(V02_LANE_SPEC, sourceCommit); } export function runtimeLaneRenderToken(): string { const random = Math.random().toString(16).slice(2, 10); return `${process.pid}-${Date.now().toString(36)}-${random}`.replace(/[^a-zA-Z0-9_.-]/gu, "-"); } export function v02RenderToken(): string { return runtimeLaneRenderToken(); } export function runtimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string { return `/tmp/hwlab-${spec.lane}-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`; } export function v02RenderDir(sourceCommit: string, token?: string): string { return runtimeLaneRenderDir(V02_LANE_SPEC, sourceCommit, token); } export function runtimeLaneRenderWorktreeDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string { return `/tmp/hwlab-${spec.lane}-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`; } export function v02RenderWorktreeDir(sourceCommit: string, token?: string): string { return runtimeLaneRenderWorktreeDir(V02_LANE_SPEC, sourceCommit, token); } export function cleanupRuntimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandJsonResult { return g14HostScript([ "set -eu", `render_dir=${shellQuote(renderDir)}`, "case \"$render_dir\" in", ` /tmp/hwlab-${spec.lane}-control-plane-*) rm -rf "$render_dir" ;;`, ` *) echo "refusing to cleanup unexpected ${spec.version} render dir: $render_dir" >&2; exit 44 ;;`, "esac", ].join("\n"), 60_000); } export function cleanupV02RenderDir(renderDir: string): CommandJsonResult { return cleanupRuntimeLaneRenderDir(V02_LANE_SPEC, renderDir); } export function legacyRuntimeLaneArgoApplications(spec: HwlabRuntimeLaneSpec): string[] { if (spec.lane === "v02") return []; return [`hwlab-g14-${spec.lane}`].filter((name) => name !== spec.app); } export function applyRuntimeLaneControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { const namespaceFile = `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`; const rbacFile = `${renderDir}/${spec.tektonDir}/rbac.yaml`; const pipelineFile = `${renderDir}/${spec.tektonDir}/pipeline.yaml`; const projectFile = `${renderDir}/argocd/project.yaml`; const applicationFile = `${renderDir}/argocd/${spec.argoApplicationFile}`; const resourceFiles = [namespaceFile, rbacFile, pipelineFile, projectFile, applicationFile]; const legacyApplications = legacyRuntimeLaneArgoApplications(spec); const serverSideArgs = [ "kubectl", "apply", "--server-side", "--force-conflicts", `--field-manager=${spec.controlPlaneFieldManager}`, ...(dryRun ? ["--dry-run=server"] : []), ...resourceFiles.flatMap((file) => ["-f", file]), ]; const command = ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, ...serverSideArgs]; const shellCommand = dryRun && spec.lane !== "v02" ? [ "set -eu", [ ...serverSideArgs.slice(0, 6), "-f", namespaceFile, "-f", pipelineFile, "-f", projectFile, "-f", applicationFile, ].map(shellQuote).join(" "), [ "kubectl", "apply", "--dry-run=client", ...resourceFiles.flatMap((file) => ["-f", file]), ].map(shellQuote).join(" "), ].join("\n") : `exec ${serverSideArgs.map(shellQuote).join(" ")}`; const cleanupCommands = legacyApplications.map((name) => [ "kubectl", "delete", "application", "-n", "argocd", name, "--ignore-not-found=true", ...(dryRun ? ["--dry-run=server", "-o", "name"] : []), ].map(shellQuote).join(" ")); const script = cleanupCommands.length > 0 ? ["set -eu", shellCommand.replace(/^exec /, ""), ...cleanupCommands].join("\n") : shellCommand; const visibleCommand = cleanupCommands.length > 0 ? ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, "sh", "--", script] : command; return runG14K3sRemoteAsync({ script, timeoutSeconds, label: `${spec.lane}-control-plane-apply`, token: runtimeLaneRenderToken(), command: visibleCommand, }); } export function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { return applyRuntimeLaneControlPlaneFiles(V02_LANE_SPEC, renderDir, dryRun, timeoutSeconds); } export interface V02RefreshMarker { ok: boolean; sourceCommit: string; refreshedAt: string; renderScriptHash: string; renderDir?: string | null; } export function v02ControlPlaneRefreshStateDir(): string { return rootPath(".state", "hwlab-g14", "v02-control-plane-refresh"); } export function v02ControlPlaneRefreshScriptHash(sourceCommit: string): string { return textHash(v02ControlPlaneRenderScript(sourceCommit, "contract-token")); } export function v02ControlPlaneRefreshMarkerPath(sourceCommit: string): string { return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.json`); } export function v02ControlPlaneRefreshLockDir(sourceCommit: string): string { return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.lock`); } export function readV02RefreshMarker(sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null { const path = v02ControlPlaneRefreshMarkerPath(sourceCommit); if (!existsSync(path)) return null; try { const marker = JSON.parse(readFileSync(path, "utf8")) as V02RefreshMarker; return v02ReusableRefreshMarker(marker, sourceCommit, scriptHash, nowMs); } catch { return null; } } export function v02ReusableRefreshMarker(marker: unknown, sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null { const candidate = record(marker) as V02RefreshMarker; const refreshedAtMs = timestampMs(candidate.refreshedAt); if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit || candidate.renderScriptHash !== scriptHash) return null; if (refreshedAtMs === null || nowMs - refreshedAtMs > V02_CONTROL_PLANE_REFRESH_TTL_MS) return null; return candidate; } export function writeV02RefreshMarker(sourceCommit: string, scriptHash: string, renderDir: string | null): V02RefreshMarker { const marker = { ok: true, sourceCommit, refreshedAt: new Date().toISOString(), renderScriptHash: scriptHash, renderDir, }; const dir = v02ControlPlaneRefreshStateDir(); mkdirSync(dir, { recursive: true }); writeFileSync(v02ControlPlaneRefreshMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8"); return marker; } export function acquireV02RefreshLock(sourceCommit: string): { acquired: boolean; lockDir: string; waitedMs: number } { const stateDir = v02ControlPlaneRefreshStateDir(); mkdirSync(stateDir, { recursive: true }); const lockDir = v02ControlPlaneRefreshLockDir(sourceCommit); const startedAtMs = Date.now(); for (;;) { try { mkdirSync(lockDir); writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8"); return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs }; } catch { const ageMs = (() => { try { return Date.now() - statSync(lockDir).mtimeMs; } catch { return 0; } })(); if (ageMs > V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS) { try { rmSync(lockDir, { recursive: true, force: true }); continue; } catch { return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs }; } } if (Date.now() - startedAtMs >= V02_CONTROL_PLANE_REFRESH_TTL_MS) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs }; const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 }); if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs }; } } } export function releaseV02RefreshLock(lockDir: string): void { try { rmSync(lockDir, { recursive: true, force: true }); } catch { // Best-effort cleanup; stale lock expiry handles interrupted workers. } } export function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record { const scriptHash = v02ControlPlaneRefreshScriptHash(sourceCommit); const existingMarker = readV02RefreshMarker(sourceCommit, scriptHash); if (existingMarker !== null) { return { ok: true, phase: "control-plane-refresh", mode: "reused-recent-refresh-marker", sourceCommit, renderDir: existingMarker.renderDir ?? null, marker: existingMarker, markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS, }; } const lock = acquireV02RefreshLock(sourceCommit); if (!lock.acquired) { return { ok: false, phase: "control-plane-refresh-lock", mode: "local-refresh-lock", sourceCommit, lockDir: lock.lockDir, waitedMs: lock.waitedMs, degradedReason: "control-plane-refresh-lock-timeout", }; } try { const markerAfterWait = readV02RefreshMarker(sourceCommit, scriptHash); if (markerAfterWait !== null) { return { ok: true, phase: "control-plane-refresh", mode: "waited-for-recent-refresh-marker", sourceCommit, renderDir: markerAfterWait.renderDir ?? null, marker: markerAfterWait, waitedMs: lock.waitedMs, markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS, }; } const render = runV02RenderToTemp(sourceCommit); if (!isCommandSuccess(render.result)) { return { ok: false, phase: "source-render", sourceCommit, renderDir: render.renderDir, render: compactCommandResult(render.result), degradedReason: "control-plane-render-failed", }; } const apply = applyV02ControlPlaneFiles(render.renderDir, false, timeoutSeconds); const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null; const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null; const ok = isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs)); const marker = ok ? writeV02RefreshMarker(sourceCommit, scriptHash, render.renderDir) : null; return { ok, phase: "control-plane-refresh", mode: "refreshed", sourceCommit, renderDir: render.renderDir, render: compactCommandResult(render.result), apply: compactCommandResult(apply), cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs), cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir), marker, waitedMs: lock.waitedMs, degradedReason: !isCommandSuccess(apply) ? "control-plane-apply-failed" : cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs) ? "obsolete-cronjob-cleanup-failed" : undefined, }; } finally { releaseV02RefreshLock(lock.lockDir); } } export function getV02ObsoleteCronJobs(): CommandJsonResult { return g14K3s([ "kubectl", "get", "cronjob", "-n", CI_NAMESPACE, V02_POLLER, V02_RECONCILER, "--ignore-not-found", "-o", "name", ], 60_000); } export function deleteV02ObsoleteCronJobs(dryRun: boolean): CommandJsonResult { return g14K3s([ "kubectl", "delete", "cronjob", "-n", CI_NAMESPACE, V02_POLLER, V02_RECONCILER, "--ignore-not-found=true", ...(dryRun ? ["--dry-run=server", "-o", "name"] : []), ], 60_000); }