fix: harden hwlab v02 concurrent triggers
This commit is contained in:
Codex
@@ -1,4 +1,4 @@
|
||||
import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02TaskRunPerformanceSummary } from "./src/hwlab-g14";
|
||||
import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14";
|
||||
|
||||
function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
|
||||
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
|
||||
@@ -144,11 +144,43 @@ assertCondition(
|
||||
assertCondition(gitMirrorSummary.githubInSync === false, "git mirror status summary must expose GitHub GitOps drift", gitMirrorSummary);
|
||||
assertCondition(gitMirrorSummary.sourceInSync === true && gitMirrorSummary.gitopsInSync === false, "git mirror status must split source and gitops GitHub sync state", gitMirrorSummary);
|
||||
const renderScript = v02ControlPlaneRenderScript("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
|
||||
const renderScriptHash = v02ControlPlaneRefreshScriptHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
|
||||
assertCondition(
|
||||
renderScript.includes("git --git-dir=\"$cicd_repo\" worktree add --detach") && renderScript.includes("/tmp/hwlab-v02-control-plane-source-aaaaaaaaaaaa"),
|
||||
"v0.2 control-plane render must use a detached temp worktree from the CI/CD repo",
|
||||
renderScript.includes("git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"")
|
||||
&& renderScript.includes("git -C \"$worktree_dir\" checkout --detach \"$source_commit\"")
|
||||
&& renderScript.includes("/tmp/hwlab-v02-control-plane-source-aaaaaaaaaaaa-"),
|
||||
"v0.2 control-plane render must use an isolated temp clone from the CI/CD repo so same-commit concurrent triggers do not share worktree metadata",
|
||||
renderScript,
|
||||
);
|
||||
assertCondition(
|
||||
renderScript.includes("flock -w 120 9") && renderScript.includes("v0.2 CI/CD repo refresh failed status="),
|
||||
"v0.2 CI/CD repo refresh must serialize only the shared bare repo fetch and expose lock/fetch failures",
|
||||
renderScript,
|
||||
);
|
||||
const reusableRefreshMarker = v02ReusableRefreshMarker({
|
||||
ok: true,
|
||||
sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
refreshedAt: "2026-06-04T16:00:00.000Z",
|
||||
renderScriptHash,
|
||||
renderDir: "/tmp/hwlab-v02-control-plane-aaaaaaaaaaaa-contract",
|
||||
}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", renderScriptHash, Date.parse("2026-06-04T16:01:00.000Z"));
|
||||
const staleRefreshMarker = v02ReusableRefreshMarker({
|
||||
ok: true,
|
||||
sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
refreshedAt: "2026-06-04T16:00:00.000Z",
|
||||
renderScriptHash,
|
||||
}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", renderScriptHash, Date.parse("2026-06-04T16:10:01.000Z"));
|
||||
const wrongScriptMarker = v02ReusableRefreshMarker({
|
||||
ok: true,
|
||||
sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
refreshedAt: "2026-06-04T16:00:00.000Z",
|
||||
renderScriptHash: "old-script",
|
||||
}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", renderScriptHash, Date.parse("2026-06-04T16:01:00.000Z"));
|
||||
assertCondition(
|
||||
reusableRefreshMarker !== null && staleRefreshMarker === null && wrongScriptMarker === null,
|
||||
"v0.2 control-plane refresh marker must only reuse recent markers for the same render script contract",
|
||||
{ reusableRefreshMarker, staleRefreshMarker, wrongScriptMarker },
|
||||
);
|
||||
assertCondition(
|
||||
renderScript.includes("cicd_repo='/root/hwlab-v02-cicd.git'") && !renderScript.includes("git -C /root/hwlab-v02") && !renderScript.includes("git checkout v0.2"),
|
||||
"v0.2 control-plane render must not use the fixed workspace checkout or its clean status",
|
||||
@@ -205,6 +237,28 @@ assertCondition(
|
||||
latestOnlySuperseded,
|
||||
);
|
||||
|
||||
const latestOnlyStalePassed = v02LatestOnlyTargetValidation({
|
||||
targetMode: "pipeline-run",
|
||||
sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
pipelineRun: { exists: true, status: "True", pipelineRun: "hwlab-v02-ci-poll-aaaaaaaaaaaa" },
|
||||
commitAlignment: { staleReasons: ["latest-pipelinerun-not-current"], latestPipelineSourceCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" },
|
||||
targetValidation: {
|
||||
ok: true,
|
||||
state: "passed",
|
||||
failures: [],
|
||||
},
|
||||
});
|
||||
assertCondition(
|
||||
latestOnlyStalePassed.ok === true
|
||||
&& latestOnlyStalePassed.state === "superseded"
|
||||
&& latestOnlyStalePassed.latestOnlySuperseded === true
|
||||
&& latestOnlyStalePassed.originalState === "passed"
|
||||
&& Array.isArray(latestOnlyStalePassed.failures)
|
||||
&& latestOnlyStalePassed.failures.length === 0,
|
||||
"v0.2 latest-only target validation must mark stale successful historical runs as superseded instead of plain passed",
|
||||
latestOnlyStalePassed,
|
||||
);
|
||||
|
||||
const falseGreenPassed = v02FalseGreenGuard({
|
||||
sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
|
||||
pipelineRun: { exists: true, status: "True" },
|
||||
@@ -408,7 +462,8 @@ console.log(JSON.stringify({
|
||||
"trigger-current can decide whether v0.2 git mirror pre-sync is required",
|
||||
"git mirror status exposes source and gitops GitHub sync state plus controlled flush command",
|
||||
"v0.2 control-plane status help exposes targeted PipelineRun/source-commit inspection",
|
||||
"v0.2 control-plane render uses a detached temp worktree from a CI/CD dedicated bare repo",
|
||||
"v0.2 control-plane render uses an isolated temp clone from a CI/CD dedicated bare repo",
|
||||
"v0.2 control-plane refresh marker only reuses recent same-contract refreshes",
|
||||
"v0.2 status alignment reports stale-success without coupling CI to dirty workspace state",
|
||||
"v0.2 PipelineRun service matrix excludes hwlab-cli",
|
||||
"v0.2 false-green guard checks build TaskRuns, runtime artifact source commits, and reuse provenance",
|
||||
|
||||
+288
-70
@@ -1,4 +1,4 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createHash } from "node:crypto";
|
||||
import { repoRoot, rootPath, type Config } from "./config";
|
||||
@@ -64,6 +64,8 @@ const G14_BRIEF_INDEX_ISSUE = 7;
|
||||
const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000;
|
||||
const V02_BUILD_TASKRUN_WARNING_SECONDS = 120;
|
||||
const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180;
|
||||
const V02_CONTROL_PLANE_REFRESH_TTL_MS = 5 * 60 * 1000;
|
||||
const V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS = 5 * 60 * 1000;
|
||||
|
||||
export type G14MonitorLane = "g14" | "v02";
|
||||
|
||||
@@ -608,31 +610,58 @@ function v02CicdRepoEnsureScript(): string {
|
||||
return [
|
||||
`cicd_repo=${shellQuote(V02_CICD_REPO)}`,
|
||||
`cicd_url=${shellQuote(V02_GIT_URL)}`,
|
||||
"mkdir -p \"$(dirname \"$cicd_repo\")\"",
|
||||
"if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
|
||||
" :",
|
||||
"elif [ -e \"$cicd_repo\" ]; then",
|
||||
" echo \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2",
|
||||
" exit 41",
|
||||
"cicd_repo_lock=/tmp/hwlab-v02-cicd-repo.lock",
|
||||
"cicd_repo_ensure_fetch() {",
|
||||
" mkdir -p \"$(dirname \"$cicd_repo\")\" || return $?",
|
||||
" if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then",
|
||||
" :",
|
||||
" elif [ -e \"$cicd_repo\" ]; then",
|
||||
" echo \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2",
|
||||
" return 41",
|
||||
" else",
|
||||
" git clone --bare \"$cicd_url\" \"$cicd_repo\" || return $?",
|
||||
" fi",
|
||||
" git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\" || return $?",
|
||||
" git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' || return $?",
|
||||
" git --git-dir=\"$cicd_repo\" fetch origin '+refs/heads/v0.2:refs/remotes/origin/v0.2' --prune || return $?",
|
||||
"}",
|
||||
"cicd_repo_lock_mode=none",
|
||||
"if command -v flock >/dev/null 2>&1; then",
|
||||
" exec 9>\"$cicd_repo_lock\"",
|
||||
" flock -w 120 9 || { echo \"timed out waiting for v0.2 CI/CD repo lock: $cicd_repo_lock\" >&2; exit 42; }",
|
||||
" cicd_repo_lock_mode=flock",
|
||||
"else",
|
||||
" git clone --bare \"$cicd_url\" \"$cicd_repo\"",
|
||||
" cicd_repo_lock_dir=\"$cicd_repo_lock.d\"",
|
||||
" cicd_repo_lock_attempt=0",
|
||||
" while ! mkdir \"$cicd_repo_lock_dir\" 2>/dev/null; do",
|
||||
" cicd_repo_lock_attempt=$((cicd_repo_lock_attempt + 1))",
|
||||
" if [ \"$cicd_repo_lock_attempt\" -ge 120 ]; then echo \"timed out waiting for v0.2 CI/CD repo lock: $cicd_repo_lock_dir\" >&2; exit 42; fi",
|
||||
" sleep 1",
|
||||
" done",
|
||||
" cicd_repo_lock_mode=dir",
|
||||
"fi",
|
||||
"git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"",
|
||||
"git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'",
|
||||
"git --git-dir=\"$cicd_repo\" fetch origin '+refs/heads/v0.2:refs/remotes/origin/v0.2' --prune",
|
||||
"cicd_repo_status=0",
|
||||
"cicd_repo_ensure_fetch || cicd_repo_status=$?",
|
||||
"if [ \"$cicd_repo_lock_mode\" = flock ]; then flock -u 9 >/dev/null 2>&1 || true; fi",
|
||||
"if [ \"$cicd_repo_lock_mode\" = dir ]; then rmdir \"$cicd_repo_lock_dir\" >/dev/null 2>&1 || true; fi",
|
||||
"if [ \"$cicd_repo_status\" -ne 0 ]; then echo \"v0.2 CI/CD repo refresh failed status=$cicd_repo_status repo=$cicd_repo\" >&2; exit \"$cicd_repo_status\"; fi",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function getV02Head(): string | null {
|
||||
function resolveV02Head(): { sourceCommit: string | null; result: CommandJsonResult } {
|
||||
const result = g14HostScript([
|
||||
"set -eu",
|
||||
v02CicdRepoEnsureScript(),
|
||||
"git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2",
|
||||
].join("\n"), 180_000);
|
||||
if (!isCommandSuccess(result)) return null;
|
||||
if (!isCommandSuccess(result)) return { sourceCommit: null, result };
|
||||
const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim();
|
||||
const match = /[0-9a-f]{40}/iu.exec(output);
|
||||
return match?.[0] ?? null;
|
||||
return { sourceCommit: match?.[0] ?? null, result };
|
||||
}
|
||||
|
||||
function getV02Head(): string | null {
|
||||
return resolveV02Head().sourceCommit;
|
||||
}
|
||||
|
||||
function v02PipelineRunName(sourceCommit: string): string {
|
||||
@@ -1444,9 +1473,12 @@ export function v02LatestOnlyTargetValidation(input: {
|
||||
if (input.sourceCommit === null) return input.targetValidation;
|
||||
if (input.pipelineRun === null || input.pipelineRun.status !== "True") return input.targetValidation;
|
||||
const validation = record(input.targetValidation);
|
||||
if (validation.ok === true || validation.state === "passed") return input.targetValidation;
|
||||
const staleReasons = stringArray(input.commitAlignment.staleReasons);
|
||||
const sourceHeadAdvancedReasons = staleReasons.filter((reason) => reason === "origin-head-mismatch" || reason === "cicd-source-repo-stale");
|
||||
const sourceHeadAdvancedReasons = staleReasons.filter((reason) => (
|
||||
reason === "origin-head-mismatch"
|
||||
|| reason === "cicd-source-repo-stale"
|
||||
|| reason === "latest-pipelinerun-not-current"
|
||||
));
|
||||
if (sourceHeadAdvancedReasons.length === 0) return input.targetValidation;
|
||||
const failures = Array.isArray(validation.failures) ? validation.failures : [];
|
||||
return {
|
||||
@@ -1457,7 +1489,9 @@ export function v02LatestOnlyTargetValidation(input: {
|
||||
latestOnlySuperseded: true,
|
||||
latestOnlyReasons: sourceHeadAdvancedReasons,
|
||||
originalState: validation.state ?? null,
|
||||
summary: `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and was superseded by a newer v0.2 source head before GitOps/runtime writeback`,
|
||||
summary: validation.ok === true || validation.state === "passed"
|
||||
? `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and is superseded by the newer v0.2 source head`
|
||||
: `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and was superseded by a newer v0.2 source head before GitOps/runtime writeback`,
|
||||
supersededFailures: failures.slice(0, 10),
|
||||
failures: [],
|
||||
};
|
||||
@@ -1895,41 +1929,71 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string,
|
||||
};
|
||||
}
|
||||
|
||||
export function v02ControlPlaneRenderScript(sourceCommit: string): string {
|
||||
const renderDir = v02RenderDir(sourceCommit);
|
||||
const worktreeDir = v02RenderWorktreeDir(sourceCommit);
|
||||
export function v02ControlPlaneRenderScript(sourceCommit: string, token = v02RenderToken()): string {
|
||||
const renderDir = v02RenderDir(sourceCommit, token);
|
||||
const worktreeDir = v02RenderWorktreeDir(sourceCommit, token);
|
||||
return [
|
||||
"set -eu",
|
||||
v02CicdRepoEnsureScript(),
|
||||
`source_commit=${shellQuote(sourceCommit)}`,
|
||||
`render_dir=${shellQuote(renderDir)}`,
|
||||
`worktree_dir=${shellQuote(worktreeDir)}`,
|
||||
"cleanup_render_worktree() { git --git-dir=\"$cicd_repo\" worktree remove --force \"$worktree_dir\" >/dev/null 2>&1 || rm -rf \"$worktree_dir\"; }",
|
||||
"cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }",
|
||||
"trap cleanup_render_worktree EXIT",
|
||||
`test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/v0.2)" = ${shellQuote(sourceCommit)}`,
|
||||
"cleanup_render_worktree",
|
||||
"git --git-dir=\"$cicd_repo\" worktree prune >/dev/null 2>&1 || true",
|
||||
"rm -rf \"$render_dir\"",
|
||||
"mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"",
|
||||
`git --git-dir="$cicd_repo" worktree add --detach "$worktree_dir" ${shellQuote(sourceCommit)}`,
|
||||
"git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"",
|
||||
"git -C \"$worktree_dir\" checkout --detach \"$source_commit\"",
|
||||
"test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"",
|
||||
"cd \"$worktree_dir\"",
|
||||
`node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function runV02RenderToTemp(sourceCommit: string): CommandJsonResult {
|
||||
return g14HostScript(v02ControlPlaneRenderScript(sourceCommit), 180_000);
|
||||
interface V02RenderTempResult {
|
||||
result: CommandJsonResult;
|
||||
renderDir: string;
|
||||
worktreeDir: string;
|
||||
token: string;
|
||||
}
|
||||
|
||||
function v02RenderDir(sourceCommit: string): string {
|
||||
return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}`;
|
||||
function runV02RenderToTemp(sourceCommit: string): V02RenderTempResult {
|
||||
const token = v02RenderToken();
|
||||
return {
|
||||
result: g14HostScript(v02ControlPlaneRenderScript(sourceCommit, token), 180_000),
|
||||
renderDir: v02RenderDir(sourceCommit, token),
|
||||
worktreeDir: v02RenderWorktreeDir(sourceCommit, token),
|
||||
token,
|
||||
};
|
||||
}
|
||||
|
||||
function v02RenderWorktreeDir(sourceCommit: string): string {
|
||||
return `/tmp/hwlab-v02-control-plane-source-${shortSha(sourceCommit)}`;
|
||||
function v02RenderToken(): string {
|
||||
const random = Math.random().toString(16).slice(2, 10);
|
||||
return `${process.pid}-${Date.now().toString(36)}-${random}`.replace(/[^a-zA-Z0-9_.-]/gu, "-");
|
||||
}
|
||||
|
||||
function applyV02ControlPlaneFiles(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||||
const renderDir = v02RenderDir(sourceCommit);
|
||||
function v02RenderDir(sourceCommit: string, token?: string): string {
|
||||
return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
||||
}
|
||||
|
||||
function v02RenderWorktreeDir(sourceCommit: string, token?: string): string {
|
||||
return `/tmp/hwlab-v02-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`;
|
||||
}
|
||||
|
||||
function cleanupV02RenderDir(renderDir: string): CommandJsonResult {
|
||||
return g14HostScript([
|
||||
"set -eu",
|
||||
`render_dir=${shellQuote(renderDir)}`,
|
||||
"case \"$render_dir\" in",
|
||||
" /tmp/hwlab-v02-control-plane-*) rm -rf \"$render_dir\" ;;",
|
||||
" *) echo \"refusing to cleanup unexpected v0.2 render dir: $render_dir\" >&2; exit 44 ;;",
|
||||
"esac",
|
||||
].join("\n"), 60_000);
|
||||
}
|
||||
|
||||
function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||||
return g14K3s([
|
||||
"kubectl",
|
||||
"apply",
|
||||
@@ -1948,34 +2012,181 @@ function applyV02ControlPlaneFiles(sourceCommit: string, dryRun: boolean, timeou
|
||||
], timeoutSeconds * 1000);
|
||||
}
|
||||
|
||||
interface V02RefreshMarker {
|
||||
ok: boolean;
|
||||
sourceCommit: string;
|
||||
refreshedAt: string;
|
||||
renderScriptHash: string;
|
||||
renderDir?: string | null;
|
||||
}
|
||||
|
||||
function v02ControlPlaneRefreshStateDir(): string {
|
||||
return rootPath(".state", "hwlab-g14", "v02-control-plane-refresh");
|
||||
}
|
||||
|
||||
export function v02ControlPlaneRefreshScriptHash(sourceCommit: string): string {
|
||||
return textHash(v02ControlPlaneRenderScript(sourceCommit, "contract-token"));
|
||||
}
|
||||
|
||||
function v02ControlPlaneRefreshMarkerPath(sourceCommit: string): string {
|
||||
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.json`);
|
||||
}
|
||||
|
||||
function v02ControlPlaneRefreshLockDir(sourceCommit: string): string {
|
||||
return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.lock`);
|
||||
}
|
||||
|
||||
function readV02RefreshMarker(sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
|
||||
const path = v02ControlPlaneRefreshMarkerPath(sourceCommit);
|
||||
if (!existsSync(path)) return null;
|
||||
try {
|
||||
const marker = JSON.parse(readFileSync(path, "utf8")) as V02RefreshMarker;
|
||||
return v02ReusableRefreshMarker(marker, sourceCommit, scriptHash, nowMs);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function v02ReusableRefreshMarker(marker: unknown, sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null {
|
||||
const candidate = record(marker) as V02RefreshMarker;
|
||||
const refreshedAtMs = timestampMs(candidate.refreshedAt);
|
||||
if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit || candidate.renderScriptHash !== scriptHash) return null;
|
||||
if (refreshedAtMs === null || nowMs - refreshedAtMs > V02_CONTROL_PLANE_REFRESH_TTL_MS) return null;
|
||||
return candidate;
|
||||
}
|
||||
|
||||
function writeV02RefreshMarker(sourceCommit: string, scriptHash: string, renderDir: string | null): V02RefreshMarker {
|
||||
const marker = {
|
||||
ok: true,
|
||||
sourceCommit,
|
||||
refreshedAt: new Date().toISOString(),
|
||||
renderScriptHash: scriptHash,
|
||||
renderDir,
|
||||
};
|
||||
const dir = v02ControlPlaneRefreshStateDir();
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(v02ControlPlaneRefreshMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
|
||||
return marker;
|
||||
}
|
||||
|
||||
function acquireV02RefreshLock(sourceCommit: string): { acquired: boolean; lockDir: string; waitedMs: number } {
|
||||
const stateDir = v02ControlPlaneRefreshStateDir();
|
||||
mkdirSync(stateDir, { recursive: true });
|
||||
const lockDir = v02ControlPlaneRefreshLockDir(sourceCommit);
|
||||
const startedAtMs = Date.now();
|
||||
for (;;) {
|
||||
try {
|
||||
mkdirSync(lockDir);
|
||||
writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
|
||||
return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
} catch {
|
||||
const ageMs = (() => {
|
||||
try {
|
||||
return Date.now() - statSync(lockDir).mtimeMs;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
})();
|
||||
if (ageMs > V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS) {
|
||||
try {
|
||||
rmSync(lockDir, { recursive: true, force: true });
|
||||
continue;
|
||||
} catch {
|
||||
return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
}
|
||||
}
|
||||
if (Date.now() - startedAtMs >= V02_CONTROL_PLANE_REFRESH_TTL_MS) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
|
||||
if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function releaseV02RefreshLock(lockDir: string): void {
|
||||
try {
|
||||
rmSync(lockDir, { recursive: true, force: true });
|
||||
} catch {
|
||||
// Best-effort cleanup; stale lock expiry handles interrupted workers.
|
||||
}
|
||||
}
|
||||
|
||||
function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record<string, unknown> {
|
||||
const render = runV02RenderToTemp(sourceCommit);
|
||||
if (!isCommandSuccess(render)) {
|
||||
const scriptHash = v02ControlPlaneRefreshScriptHash(sourceCommit);
|
||||
const existingMarker = readV02RefreshMarker(sourceCommit, scriptHash);
|
||||
if (existingMarker !== null) {
|
||||
return {
|
||||
ok: false,
|
||||
phase: "source-render",
|
||||
ok: true,
|
||||
phase: "control-plane-refresh",
|
||||
mode: "reused-recent-refresh-marker",
|
||||
sourceCommit,
|
||||
renderDir: v02RenderDir(sourceCommit),
|
||||
render,
|
||||
degradedReason: "control-plane-render-failed",
|
||||
renderDir: existingMarker.renderDir ?? null,
|
||||
marker: existingMarker,
|
||||
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
|
||||
};
|
||||
}
|
||||
const apply = applyV02ControlPlaneFiles(sourceCommit, false, timeoutSeconds);
|
||||
const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null;
|
||||
return {
|
||||
ok: isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs)),
|
||||
phase: "control-plane-refresh",
|
||||
sourceCommit,
|
||||
renderDir: v02RenderDir(sourceCommit),
|
||||
render: commandData(render),
|
||||
apply: compactCommandResult(apply),
|
||||
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
|
||||
degradedReason: !isCommandSuccess(apply)
|
||||
? "control-plane-apply-failed"
|
||||
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
|
||||
? "obsolete-cronjob-cleanup-failed"
|
||||
: undefined,
|
||||
};
|
||||
const lock = acquireV02RefreshLock(sourceCommit);
|
||||
if (!lock.acquired) {
|
||||
return {
|
||||
ok: false,
|
||||
phase: "control-plane-refresh-lock",
|
||||
mode: "local-refresh-lock",
|
||||
sourceCommit,
|
||||
lockDir: lock.lockDir,
|
||||
waitedMs: lock.waitedMs,
|
||||
degradedReason: "control-plane-refresh-lock-timeout",
|
||||
};
|
||||
}
|
||||
try {
|
||||
const markerAfterWait = readV02RefreshMarker(sourceCommit, scriptHash);
|
||||
if (markerAfterWait !== null) {
|
||||
return {
|
||||
ok: true,
|
||||
phase: "control-plane-refresh",
|
||||
mode: "waited-for-recent-refresh-marker",
|
||||
sourceCommit,
|
||||
renderDir: markerAfterWait.renderDir ?? null,
|
||||
marker: markerAfterWait,
|
||||
waitedMs: lock.waitedMs,
|
||||
markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS,
|
||||
};
|
||||
}
|
||||
const render = runV02RenderToTemp(sourceCommit);
|
||||
if (!isCommandSuccess(render.result)) {
|
||||
return {
|
||||
ok: false,
|
||||
phase: "source-render",
|
||||
sourceCommit,
|
||||
renderDir: render.renderDir,
|
||||
render: compactCommandResult(render.result),
|
||||
degradedReason: "control-plane-render-failed",
|
||||
};
|
||||
}
|
||||
const apply = applyV02ControlPlaneFiles(render.renderDir, false, timeoutSeconds);
|
||||
const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null;
|
||||
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
|
||||
const ok = isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs));
|
||||
const marker = ok ? writeV02RefreshMarker(sourceCommit, scriptHash, render.renderDir) : null;
|
||||
return {
|
||||
ok,
|
||||
phase: "control-plane-refresh",
|
||||
mode: "refreshed",
|
||||
sourceCommit,
|
||||
renderDir: render.renderDir,
|
||||
render: compactCommandResult(render.result),
|
||||
apply: compactCommandResult(apply),
|
||||
cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs),
|
||||
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
||||
marker,
|
||||
waitedMs: lock.waitedMs,
|
||||
degradedReason: !isCommandSuccess(apply)
|
||||
? "control-plane-apply-failed"
|
||||
: cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs)
|
||||
? "obsolete-cronjob-cleanup-failed"
|
||||
: undefined,
|
||||
};
|
||||
} finally {
|
||||
releaseV02RefreshLock(lock.lockDir);
|
||||
}
|
||||
}
|
||||
|
||||
function getV02ObsoleteCronJobs(): CommandJsonResult {
|
||||
@@ -2297,33 +2508,37 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
|
||||
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
|
||||
if (options.action === "status" && options.pipelineRun !== undefined) return v02ControlPlaneStatus({ pipelineRun: options.pipelineRun, mode: "pipeline-run" });
|
||||
if (options.action === "status" && options.sourceCommit !== undefined) return v02ControlPlaneStatus({ sourceCommit: options.sourceCommit, mode: "source-commit" });
|
||||
const sourceCommit = getV02Head();
|
||||
const head = resolveV02Head();
|
||||
const sourceCommit = head.sourceCommit;
|
||||
if (sourceCommit === null) {
|
||||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE };
|
||||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE, headProbe: compactCommandResult(head.result) };
|
||||
}
|
||||
if (options.action === "runtime-migration") return runV02RuntimeMigration(options, sourceCommit);
|
||||
if (options.action === "status") return v02ControlPlaneStatus({ sourceCommit, mode: "latest-source-head" });
|
||||
if (options.action === "apply") {
|
||||
const render = runV02RenderToTemp(sourceCommit);
|
||||
if (!isCommandSuccess(render)) {
|
||||
if (!isCommandSuccess(render.result)) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `hwlab g14 control-plane ${options.action} --lane v02`,
|
||||
phase: "source-render",
|
||||
sourceCommit,
|
||||
render: compactCommandResult(render),
|
||||
renderDir: render.renderDir,
|
||||
render: compactCommandResult(render.result),
|
||||
};
|
||||
}
|
||||
const apply = applyV02ControlPlaneFiles(sourceCommit, options.dryRun, options.timeoutSeconds);
|
||||
const apply = applyV02ControlPlaneFiles(render.renderDir, options.dryRun, options.timeoutSeconds);
|
||||
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
|
||||
return {
|
||||
ok: isCommandSuccess(apply),
|
||||
command: "hwlab g14 control-plane apply --lane v02",
|
||||
lane: "v02",
|
||||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||||
sourceCommit,
|
||||
renderDir: v02RenderDir(sourceCommit),
|
||||
render: commandData(render),
|
||||
renderDir: render.renderDir,
|
||||
render: compactCommandResult(render.result),
|
||||
apply: compactCommandResult(apply),
|
||||
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
||||
cleanupObsoleteCronJobs: compactCommandResult(options.dryRun ? deleteV02ObsoleteCronJobs(true) : deleteV02ObsoleteCronJobs(false)),
|
||||
status: v02ControlPlaneStatus({ sourceCommit, mode: "latest-source-head" }),
|
||||
next: options.dryRun
|
||||
@@ -2575,8 +2790,7 @@ function deleteLegacyGitMirrorCronJob(dryRun: boolean): CommandJsonResult {
|
||||
], 60_000);
|
||||
}
|
||||
|
||||
function applyGitMirrorManifestFile(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||||
const renderDir = v02RenderDir(sourceCommit);
|
||||
function applyGitMirrorManifestFile(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult {
|
||||
return g14K3s([
|
||||
"kubectl",
|
||||
"apply",
|
||||
@@ -2981,21 +3195,23 @@ function runGitMirrorStatus(): Record<string, unknown> {
|
||||
}
|
||||
|
||||
function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown> {
|
||||
const sourceCommit = getV02Head();
|
||||
const head = resolveV02Head();
|
||||
const sourceCommit = head.sourceCommit;
|
||||
if (sourceCommit === null) {
|
||||
return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE };
|
||||
return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE, headProbe: compactCommandResult(head.result) };
|
||||
}
|
||||
const render = runV02RenderToTemp(sourceCommit);
|
||||
if (!isCommandSuccess(render)) {
|
||||
if (!isCommandSuccess(render.result)) {
|
||||
return {
|
||||
ok: false,
|
||||
command: "hwlab g14 git-mirror apply",
|
||||
phase: "source-render",
|
||||
sourceCommit,
|
||||
render: compactCommandResult(render),
|
||||
renderDir: render.renderDir,
|
||||
render: compactCommandResult(render.result),
|
||||
};
|
||||
}
|
||||
const apply = applyGitMirrorManifestFile(sourceCommit, options.dryRun, options.timeoutSeconds);
|
||||
const apply = applyGitMirrorManifestFile(render.renderDir, options.dryRun, options.timeoutSeconds);
|
||||
const cleanupLegacyCronJob = isCommandSuccess(apply)
|
||||
? deleteLegacyGitMirrorCronJob(options.dryRun)
|
||||
: {
|
||||
@@ -3006,15 +3222,17 @@ function runGitMirrorApply(options: G14GitMirrorOptions): Record<string, unknown
|
||||
stderr: "skipped because apply failed",
|
||||
parsed: null,
|
||||
};
|
||||
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null;
|
||||
return {
|
||||
ok: isCommandSuccess(apply) && isCommandSuccess(cleanupLegacyCronJob),
|
||||
command: "hwlab g14 git-mirror apply",
|
||||
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
||||
sourceCommit,
|
||||
manifest: `${v02RenderDir(sourceCommit)}/devops-infra/git-mirror.yaml`,
|
||||
render: commandData(render),
|
||||
manifest: `${render.renderDir}/devops-infra/git-mirror.yaml`,
|
||||
render: compactCommandResult(render.result),
|
||||
apply: compactCommandResult(apply),
|
||||
cleanupLegacyCronJob: compactCommandResult(cleanupLegacyCronJob),
|
||||
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
||||
status: runGitMirrorStatus(),
|
||||
next: options.dryRun
|
||||
? { apply: "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm" }
|
||||
|
||||
Reference in New Issue
Block a user