Codex
410 lines
17 KiB
TypeScript
410 lines
17 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. render module for scripts/src/hwlab-legacy-runtime.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-legacy-runtime.ts:3346-3736 for #903.
|
|
|
|
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand } from "../command";
|
|
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
|
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
|
|
import type { CommandJsonResult } from "./types";
|
|
import { legacyRuntimeHostScript } from "./images";
|
|
import { compactCommandResult, legacyRuntimeK3s, isCommandSuccess, record, runLegacyRuntimeK3sRemoteAsync, shellQuote, shortSha, textHash } from "./remote";
|
|
import { runtimeLaneCicdRepoEnsureScript } from "./source";
|
|
import { CI_NAMESPACE, LEGACY_RUNTIME_PROVIDER, DEFAULT_RUNTIME_LANE_CONTROL_PLANE_REFRESH_LOCK_STALE_MS, DEFAULT_RUNTIME_LANE_CONTROL_PLANE_REFRESH_TTL_MS, DEFAULT_RUNTIME_LANE_LANE_SPEC, DEFAULT_RUNTIME_LANE_POLLER, DEFAULT_RUNTIME_LANE_RECONCILER } from "./types";
|
|
import { timestampMs } from "./runtime-status";
|
|
|
|
export function runtimeLaneControlPlaneRenderScript(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token = runtimeLaneRenderToken()): string {
|
|
const renderDir = runtimeLaneRenderDir(spec, sourceCommit, token);
|
|
const worktreeDir = runtimeLaneRenderWorktreeDir(spec, sourceCommit, token);
|
|
return [
|
|
"set -eu",
|
|
runtimeLaneCicdRepoEnsureScript(spec),
|
|
`source_commit=${shellQuote(sourceCommit)}`,
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
`worktree_dir=${shellQuote(worktreeDir)}`,
|
|
`render_lane=${shellQuote(spec.lane)}`,
|
|
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
|
|
"trap cleanup_render_worktree EXIT",
|
|
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch})" = ${shellQuote(sourceCommit)}`,
|
|
"cleanup_render_worktree",
|
|
"rm -rf \"$render_dir\"",
|
|
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
|
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
|
|
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
|
|
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
|
|
"cd \"$worktree_dir\"",
|
|
"if [ ! -d node_modules/yaml ]; then",
|
|
" if [ -f package-lock.json ]; then",
|
|
" npm ci --ignore-scripts --no-audit --prefer-offline",
|
|
" elif [ -f bun.lock ] || [ -f bun.lockb ]; then",
|
|
" bun install --frozen-lockfile --ignore-scripts",
|
|
" else",
|
|
` echo "${spec.version} control-plane render cannot install dependencies: no lockfile found" >&2`,
|
|
" exit 42",
|
|
" fi",
|
|
"fi",
|
|
"if [ -f scripts/gitops-render.mjs ]; then",
|
|
" render_script=scripts/gitops-render.mjs",
|
|
"elif [ \"$render_lane\" = v02 ] && [ -f scripts/g14-gitops-render.mjs ]; then",
|
|
" render_script=scripts/g14-gitops-render.mjs",
|
|
"else",
|
|
` echo "${spec.version} control-plane render script missing: scripts/gitops-render.mjs" >&2`,
|
|
" exit 43",
|
|
"fi",
|
|
"if [ -f scripts/run-bun.mjs ]; then",
|
|
` node scripts/run-bun.mjs "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
|
|
"else",
|
|
` node "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
|
|
"fi",
|
|
].join("\n");
|
|
}
|
|
|
|
export function defaultRuntimeLaneControlPlaneRenderScript(sourceCommit: string, token = defaultRuntimeLaneRenderToken()): string {
|
|
return runtimeLaneControlPlaneRenderScript(DEFAULT_RUNTIME_LANE_LANE_SPEC, sourceCommit, token);
|
|
}
|
|
|
|
export interface DefaultRuntimeLaneRenderTempResult {
|
|
result: CommandJsonResult;
|
|
renderDir: string;
|
|
worktreeDir: string;
|
|
token: string;
|
|
}
|
|
|
|
export function runRuntimeLaneRenderToTemp(spec: HwlabRuntimeLaneSpec, sourceCommit: string): DefaultRuntimeLaneRenderTempResult {
|
|
const token = runtimeLaneRenderToken();
|
|
return {
|
|
result: legacyRuntimeHostScript(runtimeLaneControlPlaneRenderScript(spec, sourceCommit, token), 180_000),
|
|
renderDir: runtimeLaneRenderDir(spec, sourceCommit, token),
|
|
worktreeDir: runtimeLaneRenderWorktreeDir(spec, sourceCommit, token),
|
|
token,
|
|
};
|
|
}
|
|
|
|
export function runDefaultRuntimeLaneRenderToTemp(sourceCommit: string): DefaultRuntimeLaneRenderTempResult {
|
|
return runRuntimeLaneRenderToTemp(DEFAULT_RUNTIME_LANE_LANE_SPEC, sourceCommit);
|
|
}
|
|
|
|
export function runtimeLaneRenderToken(): string {
|
|
const random = Math.random().toString(16).slice(2, 10);
|
|
return `${process.pid}-${Date.now().toString(36)}-${random}`.replace(/[^a-zA-Z0-9_.-]/gu, "-");
|
|
}
|
|
|
|
export function defaultRuntimeLaneRenderToken(): string {
|
|
return runtimeLaneRenderToken();
|
|
}
|
|
|
|
export function runtimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string {
|
|
return `/tmp/hwlab-${spec.lane}-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
|
}
|
|
|
|
export function defaultRuntimeLaneRenderDir(sourceCommit: string, token?: string): string {
|
|
return runtimeLaneRenderDir(DEFAULT_RUNTIME_LANE_LANE_SPEC, sourceCommit, token);
|
|
}
|
|
|
|
export function runtimeLaneRenderWorktreeDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string {
|
|
return `/tmp/hwlab-${spec.lane}-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
|
}
|
|
|
|
export function defaultRuntimeLaneRenderWorktreeDir(sourceCommit: string, token?: string): string {
|
|
return runtimeLaneRenderWorktreeDir(DEFAULT_RUNTIME_LANE_LANE_SPEC, sourceCommit, token);
|
|
}
|
|
|
|
export function cleanupRuntimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandJsonResult {
|
|
return legacyRuntimeHostScript([
|
|
"set -eu",
|
|
`render_dir=${shellQuote(renderDir)}`,
|
|
"case \"$render_dir\" in",
|
|
` /tmp/hwlab-${spec.lane}-control-plane-*) rm -rf "$render_dir" ;;`,
|
|
` *) echo "refusing to cleanup unexpected ${spec.version} render dir: $render_dir" >&2; exit 44 ;;`,
|
|
"esac",
|
|
].join("\n"), 60_000);
|
|
}
|
|
|
|
export function cleanupDefaultRuntimeLaneRenderDir(renderDir: string): CommandJsonResult {
|
|
return cleanupRuntimeLaneRenderDir(DEFAULT_RUNTIME_LANE_LANE_SPEC, renderDir);
|
|
}
|
|
|
|
export function legacyRuntimeLaneArgoApplications(spec: HwlabRuntimeLaneSpec): string[] {
|
|
if (spec.lane === "v02") return [];
|
|
return [`hwlab-g14-${spec.lane}`].filter((name) => name !== spec.app);
|
|
}
|
|
|
|
export function applyRuntimeLaneControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
|
const namespaceFile = `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`;
|
|
const rbacFile = `${renderDir}/${spec.tektonDir}/rbac.yaml`;
|
|
const pipelineFile = `${renderDir}/${spec.tektonDir}/pipeline.yaml`;
|
|
const projectFile = `${renderDir}/argocd/project.yaml`;
|
|
const applicationFile = `${renderDir}/argocd/${spec.argoApplicationFile}`;
|
|
const resourceFiles = [namespaceFile, rbacFile, pipelineFile, projectFile, applicationFile];
|
|
const legacyApplications = legacyRuntimeLaneArgoApplications(spec);
|
|
const serverSideArgs = [
|
|
"kubectl",
|
|
"apply",
|
|
"--server-side",
|
|
"--force-conflicts",
|
|
`--field-manager=${spec.controlPlaneFieldManager}`,
|
|
...(dryRun ? ["--dry-run=server"] : []),
|
|
...resourceFiles.flatMap((file) => ["-f", file]),
|
|
];
|
|
const command = ["bun", "scripts/cli.ts", "ssh", `${LEGACY_RUNTIME_PROVIDER}:k3s`, ...serverSideArgs];
|
|
const shellCommand = dryRun && spec.lane !== "v02"
|
|
? [
|
|
"set -eu",
|
|
[
|
|
...serverSideArgs.slice(0, 6),
|
|
"-f",
|
|
namespaceFile,
|
|
"-f",
|
|
pipelineFile,
|
|
"-f",
|
|
projectFile,
|
|
"-f",
|
|
applicationFile,
|
|
].map(shellQuote).join(" "),
|
|
[
|
|
"kubectl",
|
|
"apply",
|
|
"--dry-run=client",
|
|
...resourceFiles.flatMap((file) => ["-f", file]),
|
|
].map(shellQuote).join(" "),
|
|
].join("\n")
|
|
: `exec ${serverSideArgs.map(shellQuote).join(" ")}`;
|
|
const cleanupCommands = legacyApplications.map((name) => [
|
|
"kubectl",
|
|
"delete",
|
|
"application",
|
|
"-n",
|
|
"argocd",
|
|
name,
|
|
"--ignore-not-found=true",
|
|
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
|
].map(shellQuote).join(" "));
|
|
const script = cleanupCommands.length > 0
|
|
? ["set -eu", shellCommand.replace(/^exec /, ""), ...cleanupCommands].join("\n")
|
|
: shellCommand;
|
|
const visibleCommand = cleanupCommands.length > 0
|
|
? ["bun", "scripts/cli.ts", "ssh", `${LEGACY_RUNTIME_PROVIDER}:k3s`, "sh", "--", script]
|
|
: command;
|
|
return runLegacyRuntimeK3sRemoteAsync({
|
|
script,
|
|
timeoutSeconds,
|
|
label: `${spec.lane}-control-plane-apply`,
|
|
token: runtimeLaneRenderToken(),
|
|
command: visibleCommand,
|
|
});
|
|
}
|
|
|
|
export function applyDefaultRuntimeLaneControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
|
return applyRuntimeLaneControlPlaneFiles(DEFAULT_RUNTIME_LANE_LANE_SPEC, renderDir, dryRun, timeoutSeconds);
|
|
}
|
|
|
|
export interface DefaultRuntimeLaneRefreshMarker {
|
|
ok: boolean;
|
|
sourceCommit: string;
|
|
refreshedAt: string;
|
|
renderScriptHash: string;
|
|
renderDir?: string | null;
|
|
}
|
|
|
|
export function defaultRuntimeLaneControlPlaneRefreshStateDir(): string {
|
|
return rootPath(".state", "hwlab-g14", "v02-control-plane-refresh");
|
|
}
|
|
|
|
export function defaultRuntimeLaneControlPlaneRefreshScriptHash(sourceCommit: string): string {
|
|
return textHash(defaultRuntimeLaneControlPlaneRenderScript(sourceCommit, "contract-token"));
|
|
}
|
|
|
|
export function defaultRuntimeLaneControlPlaneRefreshMarkerPath(sourceCommit: string): string {
|
|
return join(defaultRuntimeLaneControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.json`);
|
|
}
|
|
|
|
export function defaultRuntimeLaneControlPlaneRefreshLockDir(sourceCommit: string): string {
|
|
return join(defaultRuntimeLaneControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.lock`);
|
|
}
|
|
|
|
export function readDefaultRuntimeLaneRefreshMarker(sourceCommit: string, scriptHash: string, nowMs = Date.now()): DefaultRuntimeLaneRefreshMarker | null {
|
|
const path = defaultRuntimeLaneControlPlaneRefreshMarkerPath(sourceCommit);
|
|
if (!existsSync(path)) return null;
|
|
try {
|
|
const marker = JSON.parse(readFileSync(path, "utf8")) as DefaultRuntimeLaneRefreshMarker;
|
|
return defaultRuntimeLaneReusableRefreshMarker(marker, sourceCommit, scriptHash, nowMs);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function defaultRuntimeLaneReusableRefreshMarker(marker: unknown, sourceCommit: string, scriptHash: string, nowMs = Date.now()): DefaultRuntimeLaneRefreshMarker | null {
|
|
const candidate = record(marker) as DefaultRuntimeLaneRefreshMarker;
|
|
const refreshedAtMs = timestampMs(candidate.refreshedAt);
|
|
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit || candidate.renderScriptHash !== scriptHash) return null;
|
|
if (refreshedAtMs === null || nowMs - refreshedAtMs > DEFAULT_RUNTIME_LANE_CONTROL_PLANE_REFRESH_TTL_MS) return null;
|
|
return candidate;
|
|
}
|
|
|
|
export function writeDefaultRuntimeLaneRefreshMarker(sourceCommit: string, scriptHash: string, renderDir: string | null): DefaultRuntimeLaneRefreshMarker {
|
|
const marker = {
|
|
ok: true,
|
|
sourceCommit,
|
|
refreshedAt: new Date().toISOString(),
|
|
renderScriptHash: scriptHash,
|
|
renderDir,
|
|
};
|
|
const dir = defaultRuntimeLaneControlPlaneRefreshStateDir();
|
|
mkdirSync(dir, { recursive: true });
|
|
writeFileSync(defaultRuntimeLaneControlPlaneRefreshMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
|
|
return marker;
|
|
}
|
|
|
|
export function acquireDefaultRuntimeLaneRefreshLock(sourceCommit: string): { acquired: boolean; lockDir: string; waitedMs: number } {
|
|
const stateDir = defaultRuntimeLaneControlPlaneRefreshStateDir();
|
|
mkdirSync(stateDir, { recursive: true });
|
|
const lockDir = defaultRuntimeLaneControlPlaneRefreshLockDir(sourceCommit);
|
|
const startedAtMs = Date.now();
|
|
for (;;) {
|
|
try {
|
|
mkdirSync(lockDir);
|
|
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
|
|
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
} catch {
|
|
const ageMs = (() => {
|
|
try {
|
|
return Date.now() - statSync(lockDir).mtimeMs;
|
|
} catch {
|
|
return 0;
|
|
}
|
|
})();
|
|
if (ageMs > DEFAULT_RUNTIME_LANE_CONTROL_PLANE_REFRESH_LOCK_STALE_MS) {
|
|
try {
|
|
rmSync(lockDir, { recursive: true, force: true });
|
|
continue;
|
|
} catch {
|
|
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
}
|
|
}
|
|
if (Date.now() - startedAtMs >= DEFAULT_RUNTIME_LANE_CONTROL_PLANE_REFRESH_TTL_MS) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
|
|
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
|
}
|
|
}
|
|
}
|
|
|
|
export function releaseDefaultRuntimeLaneRefreshLock(lockDir: string): void {
|
|
try {
|
|
rmSync(lockDir, { recursive: true, force: true });
|
|
} catch {
|
|
// Best-effort cleanup; stale lock expiry handles interrupted workers.
|
|
}
|
|
}
|
|
|
|
export function refreshDefaultRuntimeLaneControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record<string, unknown> {
|
|
const scriptHash = defaultRuntimeLaneControlPlaneRefreshScriptHash(sourceCommit);
|
|
const existingMarker = readDefaultRuntimeLaneRefreshMarker(sourceCommit, scriptHash);
|
|
if (existingMarker !== null) {
|
|
return {
|
|
ok: true,
|
|
phase: "control-plane-refresh",
|
|
mode: "reused-recent-refresh-marker",
|
|
sourceCommit,
|
|
renderDir: existingMarker.renderDir ?? null,
|
|
marker: existingMarker,
|
|
markerTtlMs: DEFAULT_RUNTIME_LANE_CONTROL_PLANE_REFRESH_TTL_MS,
|
|
};
|
|
}
|
|
const lock = acquireDefaultRuntimeLaneRefreshLock(sourceCommit);
|
|
if (!lock.acquired) {
|
|
return {
|
|
ok: false,
|
|
phase: "control-plane-refresh-lock",
|
|
mode: "local-refresh-lock",
|
|
sourceCommit,
|
|
lockDir: lock.lockDir,
|
|
waitedMs: lock.waitedMs,
|
|
degradedReason: "control-plane-refresh-lock-timeout",
|
|
};
|
|
}
|
|
try {
|
|
const markerAfterWait = readDefaultRuntimeLaneRefreshMarker(sourceCommit, scriptHash);
|
|
if (markerAfterWait !== null) {
|
|
return {
|
|
ok: true,
|
|
phase: "control-plane-refresh",
|
|
mode: "waited-for-recent-refresh-marker",
|
|
sourceCommit,
|
|
renderDir: markerAfterWait.renderDir ?? null,
|
|
marker: markerAfterWait,
|
|
waitedMs: lock.waitedMs,
|
|
markerTtlMs: DEFAULT_RUNTIME_LANE_CONTROL_PLANE_REFRESH_TTL_MS,
|
|
};
|
|
}
|
|
const render = runDefaultRuntimeLaneRenderToTemp(sourceCommit);
|
|
if (!isCommandSuccess(render.result)) {
|
|
return {
|
|
ok: false,
|
|
phase: "source-render",
|
|
sourceCommit,
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
degradedReason: "control-plane-render-failed",
|
|
};
|
|
}
|
|
const apply = applyDefaultRuntimeLaneControlPlaneFiles(render.renderDir, false, timeoutSeconds);
|
|
const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteDefaultRuntimeLaneObsoleteCronJobs(false) : null;
|
|
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupDefaultRuntimeLaneRenderDir(render.renderDir) : null;
|
|
const ok = isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs));
|
|
const marker = ok ? writeDefaultRuntimeLaneRefreshMarker(sourceCommit, scriptHash, render.renderDir) : null;
|
|
return {
|
|
ok,
|
|
phase: "control-plane-refresh",
|
|
mode: "refreshed",
|
|
sourceCommit,
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
apply: compactCommandResult(apply),
|
|
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
|
|
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
|
marker,
|
|
waitedMs: lock.waitedMs,
|
|
degradedReason: !isCommandSuccess(apply)
|
|
? "control-plane-apply-failed"
|
|
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
|
|
? "obsolete-cronjob-cleanup-failed"
|
|
: undefined,
|
|
};
|
|
} finally {
|
|
releaseDefaultRuntimeLaneRefreshLock(lock.lockDir);
|
|
}
|
|
}
|
|
|
|
export function getDefaultRuntimeLaneObsoleteCronJobs(): CommandJsonResult {
|
|
return legacyRuntimeK3s([
|
|
"kubectl",
|
|
"get",
|
|
"cronjob",
|
|
"-n",
|
|
CI_NAMESPACE,
|
|
DEFAULT_RUNTIME_LANE_POLLER,
|
|
DEFAULT_RUNTIME_LANE_RECONCILER,
|
|
"--ignore-not-found",
|
|
"-o",
|
|
"name",
|
|
], 60_000);
|
|
}
|
|
|
|
export function deleteDefaultRuntimeLaneObsoleteCronJobs(dryRun: boolean): CommandJsonResult {
|
|
return legacyRuntimeK3s([
|
|
"kubectl",
|
|
"delete",
|
|
"cronjob",
|
|
"-n",
|
|
CI_NAMESPACE,
|
|
DEFAULT_RUNTIME_LANE_POLLER,
|
|
DEFAULT_RUNTIME_LANE_RECONCILER,
|
|
"--ignore-not-found=true",
|
|
...(dryRun ? ["--dry-run=server", "-o", "name"] : []),
|
|
], 60_000);
|
|
}
|