Codex
410 lines
16 KiB
TypeScript
410 lines
16 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/hwlab-g14.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-g14.ts:3346-3736 for #903.
|
|
|
|
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand } from "../command";
|
|
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
|
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
|
|
import type { CommandJsonResult } from "./types";
|
|
import { g14HostScript } from "./images";
|
|
import { compactCommandResult, g14K3s, isCommandSuccess, record, runG14K3sRemoteAsync, shellQuote, shortSha, textHash } from "./remote";
|
|
import { runtimeLaneCicdRepoEnsureScript } from "./source";
|
|
import { CI_NAMESPACE, G14_PROVIDER, V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS, V02_CONTROL_PLANE_REFRESH_TTL_MS, V02_LANE_SPEC, V02_POLLER, V02_RECONCILER } from "./types";
|
|
import { timestampMs } from "./v02-status";
|
|
|
|
export function runtimeLaneControlPlaneRenderScript(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token = runtimeLaneRenderToken()): string {
|
|
const renderDir = runtimeLaneRenderDir(spec, sourceCommit, token);
|
|
const worktreeDir = runtimeLaneRenderWorktreeDir(spec, sourceCommit, token);
|
|
return [
|
|
"set -eu",
|
|
runtimeLaneCicdRepoEnsureScript(spec),
|
|
`source_commit=${shellQuote(sourceCommit)}`,
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
`worktree_dir=${shellQuote(worktreeDir)}`,
|
|
`render_lane=${shellQuote(spec.lane)}`,
|
|
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
|
|
"trap cleanup_render_worktree EXIT",
|
|
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = ${shellQuote(sourceCommit)}`,
|
|
"cleanup_render_worktree",
|
|
"rm -rf \"$render_dir\"",
|
|
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
|
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
|
|
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
|
|
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
|
|
"cd \"$worktree_dir\"",
|
|
"if [ ! -d node_modules/yaml ]; then",
|
|
" if [ -f package-lock.json ]; then",
|
|
" npm ci --ignore-scripts --no-audit --prefer-offline",
|
|
" elif [ -f bun.lock ] || [ -f bun.lockb ]; then",
|
|
" bun install --frozen-lockfile --ignore-scripts",
|
|
" else",
|
|
` echo "${spec.version} control-plane render cannot install dependencies: no lockfile found" >&2`,
|
|
" exit 42",
|
|
" fi",
|
|
"fi",
|
|
"if [ -f scripts/gitops-render.mjs ]; then",
|
|
" render_script=scripts/gitops-render.mjs",
|
|
"elif [ \"$render_lane\" = v02 ] && [ -f scripts/g14-gitops-render.mjs ]; then",
|
|
" render_script=scripts/g14-gitops-render.mjs",
|
|
"else",
|
|
` echo "${spec.version} control-plane render script missing: scripts/gitops-render.mjs" >&2`,
|
|
" exit 43",
|
|
"fi",
|
|
"if [ -f scripts/run-bun.mjs ]; then",
|
|
` node scripts/run-bun.mjs "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
|
|
"else",
|
|
` node "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
|
|
"fi",
|
|
].join("\n");
|
|
}
|
|
|
|
export function v02ControlPlaneRenderScript(sourceCommit: string, token = v02RenderToken()): string {
|
|
return runtimeLaneControlPlaneRenderScript(V02_LANE_SPEC, sourceCommit, token);
|
|
}
|
|
|
|
export interface V02RenderTempResult {
|
|
result: CommandJsonResult;
|
|
renderDir: string;
|
|
worktreeDir: string;
|
|
token: string;
|
|
}
|
|
|
|
export function runRuntimeLaneRenderToTemp(spec: HwlabRuntimeLaneSpec, sourceCommit: string): V02RenderTempResult {
|
|
const token = runtimeLaneRenderToken();
|
|
return {
|
|
result: g14HostScript(runtimeLaneControlPlaneRenderScript(spec, sourceCommit, token), 180_000),
|
|
renderDir: runtimeLaneRenderDir(spec, sourceCommit, token),
|
|
worktreeDir: runtimeLaneRenderWorktreeDir(spec, sourceCommit, token),
|
|
token,
|
|
};
|
|
}
|
|
|
|
export function runV02RenderToTemp(sourceCommit: string): V02RenderTempResult {
|
|
return runRuntimeLaneRenderToTemp(V02_LANE_SPEC, sourceCommit);
|
|
}
|
|
|
|
export function runtimeLaneRenderToken(): string {
|
|
const random = Math.random().toString(16).slice(2, 10);
|
|
return `${process.pid}-${Date.now().toString(36)}-${random}`.replace(/[^a-zA-Z0-9_.-]/gu, "-");
|
|
}
|
|
|
|
export function v02RenderToken(): string {
|
|
return runtimeLaneRenderToken();
|
|
}
|
|
|
|
export function runtimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string {
|
|
return `/tmp/hwlab-${spec.lane}-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
|
}
|
|
|
|
export function v02RenderDir(sourceCommit: string, token?: string): string {
|
|
return runtimeLaneRenderDir(V02_LANE_SPEC, sourceCommit, token);
|
|
}
|
|
|
|
export function runtimeLaneRenderWorktreeDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string {
|
|
return `/tmp/hwlab-${spec.lane}-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
|
}
|
|
|
|
export function v02RenderWorktreeDir(sourceCommit: string, token?: string): string {
|
|
return runtimeLaneRenderWorktreeDir(V02_LANE_SPEC, sourceCommit, token);
|
|
}
|
|
|
|
export function cleanupRuntimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandJsonResult {
|
|
return g14HostScript([
|
|
"set -eu",
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
"case \"$render_dir\" in",
|
|
` /tmp/hwlab-${spec.lane}-control-plane-*) rm -rf "$render_dir" ;;`,
|
|
` *) echo "refusing to cleanup unexpected ${spec.version} render dir: $render_dir" >&2; exit 44 ;;`,
|
|
"esac",
|
|
].join("\n"), 60_000);
|
|
}
|
|
|
|
export function cleanupV02RenderDir(renderDir: string): CommandJsonResult {
|
|
return cleanupRuntimeLaneRenderDir(V02_LANE_SPEC, renderDir);
|
|
}
|
|
|
|
export function legacyRuntimeLaneArgoApplications(spec: HwlabRuntimeLaneSpec): string[] {
|
|
if (spec.lane === "v02") return [];
|
|
return [`hwlab-g14-${spec.lane}`].filter((name) => name !== spec.app);
|
|
}
|
|
|
|
export function applyRuntimeLaneControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
|
const namespaceFile = `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`;
|
|
const rbacFile = `${renderDir}/${spec.tektonDir}/rbac.yaml`;
|
|
const pipelineFile = `${renderDir}/${spec.tektonDir}/pipeline.yaml`;
|
|
const projectFile = `${renderDir}/argocd/project.yaml`;
|
|
const applicationFile = `${renderDir}/argocd/${spec.argoApplicationFile}`;
|
|
const resourceFiles = [namespaceFile, rbacFile, pipelineFile, projectFile, applicationFile];
|
|
const legacyApplications = legacyRuntimeLaneArgoApplications(spec);
|
|
const serverSideArgs = [
|
|
"kubectl",
|
|
"apply",
|
|
"--server-side",
|
|
"--force-conflicts",
|
|
`--field-manager=${spec.controlPlaneFieldManager}`,
|
|
...(dryRun ? ["--dry-run=server"] : []),
|
|
...resourceFiles.flatMap((file) => ["-f", file]),
|
|
];
|
|
const command = ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, ...serverSideArgs];
|
|
const shellCommand = dryRun && spec.lane !== "v02"
|
|
? [
|
|
"set -eu",
|
|
[
|
|
...serverSideArgs.slice(0, 6),
|
|
"-f",
|
|
namespaceFile,
|
|
"-f",
|
|
pipelineFile,
|
|
"-f",
|
|
projectFile,
|
|
"-f",
|
|
applicationFile,
|
|
].map(shellQuote).join(" "),
|
|
[
|
|
"kubectl",
|
|
"apply",
|
|
"--dry-run=client",
|
|
...resourceFiles.flatMap((file) => ["-f", file]),
|
|
].map(shellQuote).join(" "),
|
|
].join("\n")
|
|
: `exec ${serverSideArgs.map(shellQuote).join(" ")}`;
|
|
const cleanupCommands = legacyApplications.map((name) => [
|
|
"kubectl",
|
|
"delete",
|
|
"application",
|
|
"-n",
|
|
"argocd",
|
|
name,
|
|
"--ignore-not-found=true",
|
|
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
|
].map(shellQuote).join(" "));
|
|
const script = cleanupCommands.length > 0
|
|
? ["set -eu", shellCommand.replace(/^exec /, ""), ...cleanupCommands].join("\n")
|
|
: shellCommand;
|
|
const visibleCommand = cleanupCommands.length > 0
|
|
? ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, "sh", "--", script]
|
|
: command;
|
|
return runG14K3sRemoteAsync({
|
|
script,
|
|
timeoutSeconds,
|
|
label: `${spec.lane}-control-plane-apply`,
|
|
token: runtimeLaneRenderToken(),
|
|
command: visibleCommand,
|
|
});
|
|
}
|
|
|
|
export function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
|
return applyRuntimeLaneControlPlaneFiles(V02_LANE_SPEC, renderDir, dryRun, timeoutSeconds);
|
|
}
|
|
|
|
export interface V02RefreshMarker {
|
|
ok: boolean;
|
|
sourceCommit: string;
|
|
refreshedAt: string;
|
|
renderScriptHash: string;
|
|
renderDir?: string | null;
|
|
}
|
|
|
|
export function v02ControlPlaneRefreshStateDir(): string {
|
|
return rootPath(".state", "hwlab-g14", "v02-control-plane-refresh");
|
|
}
|
|
|
|
export function v02ControlPlaneRefreshScriptHash(sourceCommit: string): string {
|
|
return textHash(v02ControlPlaneRenderScript(sourceCommit, "contract-token"));
|
|
}
|
|
|
|
export function v02ControlPlaneRefreshMarkerPath(sourceCommit: string): string {
|
|
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.json`);
|
|
}
|
|
|
|
export function v02ControlPlaneRefreshLockDir(sourceCommit: string): string {
|
|
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.lock`);
|
|
}
|
|
|
|
export function readV02RefreshMarker(sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
|
|
const path = v02ControlPlaneRefreshMarkerPath(sourceCommit);
|
|
if (!existsSync(path)) return null;
|
|
try {
|
|
const marker = JSON.parse(readFileSync(path, "utf8")) as V02RefreshMarker;
|
|
return v02ReusableRefreshMarker(marker, sourceCommit, scriptHash, nowMs);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function v02ReusableRefreshMarker(marker: unknown, sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
|
|
const candidate = record(marker) as V02RefreshMarker;
|
|
const refreshedAtMs = timestampMs(candidate.refreshedAt);
|
|
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit || candidate.renderScriptHash !== scriptHash) return null;
|
|
if (refreshedAtMs === null || nowMs - refreshedAtMs > V02_CONTROL_PLANE_REFRESH_TTL_MS) return null;
|
|
return candidate;
|
|
}
|
|
|
|
export function writeV02RefreshMarker(sourceCommit: string, scriptHash: string, renderDir: string | null): V02RefreshMarker {
|
|
const marker = {
|
|
ok: true,
|
|
sourceCommit,
|
|
refreshedAt: new Date().toISOString(),
|
|
renderScriptHash: scriptHash,
|
|
renderDir,
|
|
};
|
|
const dir = v02ControlPlaneRefreshStateDir();
|
|
mkdirSync(dir, { recursive: true });
|
|
writeFileSync(v02ControlPlaneRefreshMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
|
|
return marker;
|
|
}
|
|
|
|
export function acquireV02RefreshLock(sourceCommit: string): { acquired: boolean; lockDir: string; waitedMs: number } {
|
|
const stateDir = v02ControlPlaneRefreshStateDir();
|
|
mkdirSync(stateDir, { recursive: true });
|
|
const lockDir = v02ControlPlaneRefreshLockDir(sourceCommit);
|
|
const startedAtMs = Date.now();
|
|
for (;;) {
|
|
try {
|
|
mkdirSync(lockDir);
|
|
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
|
|
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
} catch {
|
|
const ageMs = (() => {
|
|
try {
|
|
return Date.now() - statSync(lockDir).mtimeMs;
|
|
} catch {
|
|
return 0;
|
|
}
|
|
})();
|
|
if (ageMs > V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS) {
|
|
try {
|
|
rmSync(lockDir, { recursive: true, force: true });
|
|
continue;
|
|
} catch {
|
|
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
}
|
|
}
|
|
if (Date.now() - startedAtMs >= V02_CONTROL_PLANE_REFRESH_TTL_MS) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
|
|
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
}
|
|
}
|
|
}
|
|
|
|
export function releaseV02RefreshLock(lockDir: string): void {
|
|
try {
|
|
rmSync(lockDir, { recursive: true, force: true });
|
|
} catch {
|
|
// Best-effort cleanup; stale lock expiry handles interrupted workers.
|
|
}
|
|
}
|
|
|
|
export function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record<string, unknown> {
|
|
const scriptHash = v02ControlPlaneRefreshScriptHash(sourceCommit);
|
|
const existingMarker = readV02RefreshMarker(sourceCommit, scriptHash);
|
|
if (existingMarker !== null) {
|
|
return {
|
|
ok: true,
|
|
phase: "control-plane-refresh",
|
|
mode: "reused-recent-refresh-marker",
|
|
sourceCommit,
|
|
renderDir: existingMarker.renderDir ?? null,
|
|
marker: existingMarker,
|
|
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
|
|
};
|
|
}
|
|
const lock = acquireV02RefreshLock(sourceCommit);
|
|
if (!lock.acquired) {
|
|
return {
|
|
ok: false,
|
|
phase: "control-plane-refresh-lock",
|
|
mode: "local-refresh-lock",
|
|
sourceCommit,
|
|
lockDir: lock.lockDir,
|
|
waitedMs: lock.waitedMs,
|
|
degradedReason: "control-plane-refresh-lock-timeout",
|
|
};
|
|
}
|
|
try {
|
|
const markerAfterWait = readV02RefreshMarker(sourceCommit, scriptHash);
|
|
if (markerAfterWait !== null) {
|
|
return {
|
|
ok: true,
|
|
phase: "control-plane-refresh",
|
|
mode: "waited-for-recent-refresh-marker",
|
|
sourceCommit,
|
|
renderDir: markerAfterWait.renderDir ?? null,
|
|
marker: markerAfterWait,
|
|
waitedMs: lock.waitedMs,
|
|
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
|
|
};
|
|
}
|
|
const render = runV02RenderToTemp(sourceCommit);
|
|
if (!isCommandSuccess(render.result)) {
|
|
return {
|
|
ok: false,
|
|
phase: "source-render",
|
|
sourceCommit,
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
degradedReason: "control-plane-render-failed",
|
|
};
|
|
}
|
|
const apply = applyV02ControlPlaneFiles(render.renderDir, false, timeoutSeconds);
|
|
const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null;
|
|
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
|
|
const ok = isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs));
|
|
const marker = ok ? writeV02RefreshMarker(sourceCommit, scriptHash, render.renderDir) : null;
|
|
return {
|
|
ok,
|
|
phase: "control-plane-refresh",
|
|
mode: "refreshed",
|
|
sourceCommit,
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
apply: compactCommandResult(apply),
|
|
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
|
|
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
|
marker,
|
|
waitedMs: lock.waitedMs,
|
|
degradedReason: !isCommandSuccess(apply)
|
|
? "control-plane-apply-failed"
|
|
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
|
|
? "obsolete-cronjob-cleanup-failed"
|
|
: undefined,
|
|
};
|
|
} finally {
|
|
releaseV02RefreshLock(lock.lockDir);
|
|
}
|
|
}
|
|
|
|
export function getV02ObsoleteCronJobs(): CommandJsonResult {
|
|
return g14K3s([
|
|
"kubectl",
|
|
"get",
|
|
"cronjob",
|
|
"-n",
|
|
CI_NAMESPACE,
|
|
V02_POLLER,
|
|
V02_RECONCILER,
|
|
"--ignore-not-found",
|
|
"-o",
|
|
"name",
|
|
], 60_000);
|
|
}
|
|
|
|
export function deleteV02ObsoleteCronJobs(dryRun: boolean): CommandJsonResult {
|
|
return g14K3s([
|
|
"kubectl",
|
|
"delete",
|
|
"cronjob",
|
|
"-n",
|
|
CI_NAMESPACE,
|
|
V02_POLLER,
|
|
V02_RECONCILER,
|
|
"--ignore-not-found=true",
|
|
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
|
], 60_000);
|
|
}
|