diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index 2ffc51d9..49c14b00 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -1,4 +1,4 @@ -import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02TaskRunPerformanceSummary } from "./src/hwlab-g14"; +import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14"; function assertCondition(condition: unknown, message: string, detail: unknown = {}): void { if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`); @@ -144,11 +144,43 @@ assertCondition( assertCondition(gitMirrorSummary.githubInSync === false, "git mirror status summary must expose GitHub GitOps drift", gitMirrorSummary); assertCondition(gitMirrorSummary.sourceInSync === true && gitMirrorSummary.gitopsInSync === false, "git mirror status must split source and gitops GitHub sync state", gitMirrorSummary); const renderScript = v02ControlPlaneRenderScript("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); +const renderScriptHash = v02ControlPlaneRefreshScriptHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); assertCondition( - renderScript.includes("git --git-dir=\"$cicd_repo\" worktree add --detach") && renderScript.includes("/tmp/hwlab-v02-control-plane-source-aaaaaaaaaaaa"), - "v0.2 control-plane render must use a detached temp worktree from the CI/CD repo", + renderScript.includes("git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"") + && renderScript.includes("git -C \"$worktree_dir\" checkout --detach \"$source_commit\"") + && renderScript.includes("/tmp/hwlab-v02-control-plane-source-aaaaaaaaaaaa-"), + "v0.2 control-plane render must use an isolated temp clone from the CI/CD repo so same-commit concurrent triggers do not share worktree metadata", renderScript, ); +assertCondition( + renderScript.includes("flock -w 120 9") && renderScript.includes("v0.2 CI/CD repo refresh failed status="), + "v0.2 CI/CD repo refresh must serialize only the shared bare repo fetch and expose lock/fetch failures", + renderScript, +); +const reusableRefreshMarker = v02ReusableRefreshMarker({ + ok: true, + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + refreshedAt: "2026-06-04T16:00:00.000Z", + renderScriptHash, + renderDir: "/tmp/hwlab-v02-control-plane-aaaaaaaaaaaa-contract", +}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", renderScriptHash, Date.parse("2026-06-04T16:01:00.000Z")); +const staleRefreshMarker = v02ReusableRefreshMarker({ + ok: true, + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + refreshedAt: "2026-06-04T16:00:00.000Z", + renderScriptHash, +}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", renderScriptHash, Date.parse("2026-06-04T16:10:01.000Z")); +const wrongScriptMarker = v02ReusableRefreshMarker({ + ok: true, + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + refreshedAt: "2026-06-04T16:00:00.000Z", + renderScriptHash: "old-script", +}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", renderScriptHash, Date.parse("2026-06-04T16:01:00.000Z")); +assertCondition( + reusableRefreshMarker !== null && staleRefreshMarker === null && wrongScriptMarker === null, + "v0.2 control-plane refresh marker must only reuse recent markers for the same render script contract", + { reusableRefreshMarker, staleRefreshMarker, wrongScriptMarker }, +); assertCondition( renderScript.includes("cicd_repo='/root/hwlab-v02-cicd.git'") && !renderScript.includes("git -C /root/hwlab-v02") && !renderScript.includes("git checkout v0.2"), "v0.2 control-plane render must not use the fixed workspace checkout or its clean status", @@ -205,6 +237,28 @@ assertCondition( latestOnlySuperseded, ); +const latestOnlyStalePassed = v02LatestOnlyTargetValidation({ + targetMode: "pipeline-run", + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + pipelineRun: { exists: true, status: "True", pipelineRun: "hwlab-v02-ci-poll-aaaaaaaaaaaa" }, + commitAlignment: { staleReasons: ["latest-pipelinerun-not-current"], latestPipelineSourceCommit: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, + targetValidation: { + ok: true, + state: "passed", + failures: [], + }, +}); +assertCondition( + latestOnlyStalePassed.ok === true + && latestOnlyStalePassed.state === "superseded" + && latestOnlyStalePassed.latestOnlySuperseded === true + && latestOnlyStalePassed.originalState === "passed" + && Array.isArray(latestOnlyStalePassed.failures) + && latestOnlyStalePassed.failures.length === 0, + "v0.2 latest-only target validation must mark stale successful historical runs as superseded instead of plain passed", + latestOnlyStalePassed, +); + const falseGreenPassed = v02FalseGreenGuard({ sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", pipelineRun: { exists: true, status: "True" }, @@ -408,7 +462,8 @@ console.log(JSON.stringify({ "trigger-current can decide whether v0.2 git mirror pre-sync is required", "git mirror status exposes source and gitops GitHub sync state plus controlled flush command", "v0.2 control-plane status help exposes targeted PipelineRun/source-commit inspection", - "v0.2 control-plane render uses a detached temp worktree from a CI/CD dedicated bare repo", + "v0.2 control-plane render uses an isolated temp clone from a CI/CD dedicated bare repo", + "v0.2 control-plane refresh marker only reuses recent same-contract refreshes", "v0.2 status alignment reports stale-success without coupling CI to dirty workspace state", "v0.2 PipelineRun service matrix excludes hwlab-cli", "v0.2 false-green guard checks build TaskRuns, runtime artifact source commits, and reuse provenance", diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 7af20bff..9151aa73 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { createHash } from "node:crypto"; import { repoRoot, rootPath, type Config } from "./config"; @@ -64,6 +64,8 @@ const G14_BRIEF_INDEX_ISSUE = 7; const BEIJING_OFFSET_MS = 8 * 60 * 60 * 1000; const V02_BUILD_TASKRUN_WARNING_SECONDS = 120; const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180; +const V02_CONTROL_PLANE_REFRESH_TTL_MS = 5 * 60 * 1000; +const V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS = 5 * 60 * 1000; export type G14MonitorLane = "g14" | "v02"; @@ -608,31 +610,58 @@ function v02CicdRepoEnsureScript(): string { return [ `cicd_repo=${shellQuote(V02_CICD_REPO)}`, `cicd_url=${shellQuote(V02_GIT_URL)}`, - "mkdir -p \"$(dirname \"$cicd_repo\")\"", - "if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then", - " :", - "elif [ -e \"$cicd_repo\" ]; then", - " echo \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2", - " exit 41", + "cicd_repo_lock=/tmp/hwlab-v02-cicd-repo.lock", + "cicd_repo_ensure_fetch() {", + " mkdir -p \"$(dirname \"$cicd_repo\")\" || return $?", + " if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then", + " :", + " elif [ -e \"$cicd_repo\" ]; then", + " echo \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2", + " return 41", + " else", + " git clone --bare \"$cicd_url\" \"$cicd_repo\" || return $?", + " fi", + " git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\" || return $?", + " git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' || return $?", + " git --git-dir=\"$cicd_repo\" fetch origin '+refs/heads/v0.2:refs/remotes/origin/v0.2' --prune || return $?", + "}", + "cicd_repo_lock_mode=none", + "if command -v flock >/dev/null 2>&1; then", + " exec 9>\"$cicd_repo_lock\"", + " flock -w 120 9 || { echo \"timed out waiting for v0.2 CI/CD repo lock: $cicd_repo_lock\" >&2; exit 42; }", + " cicd_repo_lock_mode=flock", "else", - " git clone --bare \"$cicd_url\" \"$cicd_repo\"", + " cicd_repo_lock_dir=\"$cicd_repo_lock.d\"", + " cicd_repo_lock_attempt=0", + " while ! mkdir \"$cicd_repo_lock_dir\" 2>/dev/null; do", + " cicd_repo_lock_attempt=$((cicd_repo_lock_attempt + 1))", + " if [ \"$cicd_repo_lock_attempt\" -ge 120 ]; then echo \"timed out waiting for v0.2 CI/CD repo lock: $cicd_repo_lock_dir\" >&2; exit 42; fi", + " sleep 1", + " done", + " cicd_repo_lock_mode=dir", "fi", - "git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\"", - "git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'", - "git --git-dir=\"$cicd_repo\" fetch origin '+refs/heads/v0.2:refs/remotes/origin/v0.2' --prune", + "cicd_repo_status=0", + "cicd_repo_ensure_fetch || cicd_repo_status=$?", + "if [ \"$cicd_repo_lock_mode\" = flock ]; then flock -u 9 >/dev/null 2>&1 || true; fi", + "if [ \"$cicd_repo_lock_mode\" = dir ]; then rmdir \"$cicd_repo_lock_dir\" >/dev/null 2>&1 || true; fi", + "if [ \"$cicd_repo_status\" -ne 0 ]; then echo \"v0.2 CI/CD repo refresh failed status=$cicd_repo_status repo=$cicd_repo\" >&2; exit \"$cicd_repo_status\"; fi", ].join("\n"); } -function getV02Head(): string | null { +function resolveV02Head(): { sourceCommit: string | null; result: CommandJsonResult } { const result = g14HostScript([ "set -eu", v02CicdRepoEnsureScript(), "git --git-dir=\"$cicd_repo\" rev-parse refs/remotes/origin/v0.2", ].join("\n"), 180_000); - if (!isCommandSuccess(result)) return null; + if (!isCommandSuccess(result)) return { sourceCommit: null, result }; const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim(); const match = /[0-9a-f]{40}/iu.exec(output); - return match?.[0] ?? null; + return { sourceCommit: match?.[0] ?? null, result }; +} + +function getV02Head(): string | null { + return resolveV02Head().sourceCommit; } function v02PipelineRunName(sourceCommit: string): string { @@ -1444,9 +1473,12 @@ export function v02LatestOnlyTargetValidation(input: { if (input.sourceCommit === null) return input.targetValidation; if (input.pipelineRun === null || input.pipelineRun.status !== "True") return input.targetValidation; const validation = record(input.targetValidation); - if (validation.ok === true || validation.state === "passed") return input.targetValidation; const staleReasons = stringArray(input.commitAlignment.staleReasons); - const sourceHeadAdvancedReasons = staleReasons.filter((reason) => reason === "origin-head-mismatch" || reason === "cicd-source-repo-stale"); + const sourceHeadAdvancedReasons = staleReasons.filter((reason) => ( + reason === "origin-head-mismatch" + || reason === "cicd-source-repo-stale" + || reason === "latest-pipelinerun-not-current" + )); if (sourceHeadAdvancedReasons.length === 0) return input.targetValidation; const failures = Array.isArray(validation.failures) ? validation.failures : []; return { @@ -1457,7 +1489,9 @@ export function v02LatestOnlyTargetValidation(input: { latestOnlySuperseded: true, latestOnlyReasons: sourceHeadAdvancedReasons, originalState: validation.state ?? null, - summary: `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and was superseded by a newer v0.2 source head before GitOps/runtime writeback`, + summary: validation.ok === true || validation.state === "passed" + ? `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and is superseded by the newer v0.2 source head` + : `target ${input.targetMode} completed for ${shortSha(input.sourceCommit)} and was superseded by a newer v0.2 source head before GitOps/runtime writeback`, supersededFailures: failures.slice(0, 10), failures: [], }; @@ -1895,41 +1929,71 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record/dev/null 2>&1 || rm -rf \"$worktree_dir\"; }", + "cleanup_render_worktree() { rm -rf \"$worktree_dir\"; }", "trap cleanup_render_worktree EXIT", `test "$(git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/v0.2)" = ${shellQuote(sourceCommit)}`, "cleanup_render_worktree", - "git --git-dir=\"$cicd_repo\" worktree prune >/dev/null 2>&1 || true", "rm -rf \"$render_dir\"", "mkdir -p \"$render_dir\" \"$(dirname \"$worktree_dir\")\"", - `git --git-dir="$cicd_repo" worktree add --detach "$worktree_dir" ${shellQuote(sourceCommit)}`, + "git clone --shared --no-checkout \"$cicd_repo\" \"$worktree_dir\"", + "git -C \"$worktree_dir\" checkout --detach \"$source_commit\"", + "test \"$(git -C \"$worktree_dir\" rev-parse HEAD)\" = \"$source_commit\"", "cd \"$worktree_dir\"", `node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, ].join("\n"); } -function runV02RenderToTemp(sourceCommit: string): CommandJsonResult { - return g14HostScript(v02ControlPlaneRenderScript(sourceCommit), 180_000); +interface V02RenderTempResult { + result: CommandJsonResult; + renderDir: string; + worktreeDir: string; + token: string; } -function v02RenderDir(sourceCommit: string): string { - return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}`; +function runV02RenderToTemp(sourceCommit: string): V02RenderTempResult { + const token = v02RenderToken(); + return { + result: g14HostScript(v02ControlPlaneRenderScript(sourceCommit, token), 180_000), + renderDir: v02RenderDir(sourceCommit, token), + worktreeDir: v02RenderWorktreeDir(sourceCommit, token), + token, + }; } -function v02RenderWorktreeDir(sourceCommit: string): string { - return `/tmp/hwlab-v02-control-plane-source-${shortSha(sourceCommit)}`; +function v02RenderToken(): string { + const random = Math.random().toString(16).slice(2, 10); + return `${process.pid}-${Date.now().toString(36)}-${random}`.replace(/[^a-zA-Z0-9_.-]/gu, "-"); } -function applyV02ControlPlaneFiles(sourceCommit: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { - const renderDir = v02RenderDir(sourceCommit); +function v02RenderDir(sourceCommit: string, token?: string): string { + return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`; +} + +function v02RenderWorktreeDir(sourceCommit: string, token?: string): string { + return `/tmp/hwlab-v02-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`; +} + +function cleanupV02RenderDir(renderDir: string): CommandJsonResult { + return g14HostScript([ + "set -eu", + `render_dir=${shellQuote(renderDir)}`, + "case \"$render_dir\" in", + " /tmp/hwlab-v02-control-plane-*) rm -rf \"$render_dir\" ;;", + " *) echo \"refusing to cleanup unexpected v0.2 render dir: $render_dir\" >&2; exit 44 ;;", + "esac", + ].join("\n"), 60_000); +} + +function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { return g14K3s([ "kubectl", "apply", @@ -1948,34 +2012,181 @@ function applyV02ControlPlaneFiles(sourceCommit: string, dryRun: boolean, timeou ], timeoutSeconds * 1000); } +interface V02RefreshMarker { + ok: boolean; + sourceCommit: string; + refreshedAt: string; + renderScriptHash: string; + renderDir?: string | null; +} + +function v02ControlPlaneRefreshStateDir(): string { + return rootPath(".state", "hwlab-g14", "v02-control-plane-refresh"); +} + +export function v02ControlPlaneRefreshScriptHash(sourceCommit: string): string { + return textHash(v02ControlPlaneRenderScript(sourceCommit, "contract-token")); +} + +function v02ControlPlaneRefreshMarkerPath(sourceCommit: string): string { + return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.json`); +} + +function v02ControlPlaneRefreshLockDir(sourceCommit: string): string { + return join(v02ControlPlaneRefreshStateDir(), `${shortSha(sourceCommit)}.lock`); +} + +function readV02RefreshMarker(sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null { + const path = v02ControlPlaneRefreshMarkerPath(sourceCommit); + if (!existsSync(path)) return null; + try { + const marker = JSON.parse(readFileSync(path, "utf8")) as V02RefreshMarker; + return v02ReusableRefreshMarker(marker, sourceCommit, scriptHash, nowMs); + } catch { + return null; + } +} + +export function v02ReusableRefreshMarker(marker: unknown, sourceCommit: string, scriptHash: string, nowMs = Date.now()): V02RefreshMarker | null { + const candidate = record(marker) as V02RefreshMarker; + const refreshedAtMs = timestampMs(candidate.refreshedAt); + if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit || candidate.renderScriptHash !== scriptHash) return null; + if (refreshedAtMs === null || nowMs - refreshedAtMs > V02_CONTROL_PLANE_REFRESH_TTL_MS) return null; + return candidate; +} + +function writeV02RefreshMarker(sourceCommit: string, scriptHash: string, renderDir: string | null): V02RefreshMarker { + const marker = { + ok: true, + sourceCommit, + refreshedAt: new Date().toISOString(), + renderScriptHash: scriptHash, + renderDir, + }; + const dir = v02ControlPlaneRefreshStateDir(); + mkdirSync(dir, { recursive: true }); + writeFileSync(v02ControlPlaneRefreshMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8"); + return marker; +} + +function acquireV02RefreshLock(sourceCommit: string): { acquired: boolean; lockDir: string; waitedMs: number } { + const stateDir = v02ControlPlaneRefreshStateDir(); + mkdirSync(stateDir, { recursive: true }); + const lockDir = v02ControlPlaneRefreshLockDir(sourceCommit); + const startedAtMs = Date.now(); + for (;;) { + try { + mkdirSync(lockDir); + writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8"); + return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs }; + } catch { + const ageMs = (() => { + try { + return Date.now() - statSync(lockDir).mtimeMs; + } catch { + return 0; + } + })(); + if (ageMs > V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS) { + try { + rmSync(lockDir, { recursive: true, force: true }); + continue; + } catch { + return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs }; + } + } + if (Date.now() - startedAtMs >= V02_CONTROL_PLANE_REFRESH_TTL_MS) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs }; + const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 }); + if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs }; + } + } +} + +function releaseV02RefreshLock(lockDir: string): void { + try { + rmSync(lockDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup; stale lock expiry handles interrupted workers. + } +} + function refreshV02ControlPlaneBeforeTrigger(sourceCommit: string, timeoutSeconds: number): Record { - const render = runV02RenderToTemp(sourceCommit); - if (!isCommandSuccess(render)) { + const scriptHash = v02ControlPlaneRefreshScriptHash(sourceCommit); + const existingMarker = readV02RefreshMarker(sourceCommit, scriptHash); + if (existingMarker !== null) { return { - ok: false, - phase: "source-render", + ok: true, + phase: "control-plane-refresh", + mode: "reused-recent-refresh-marker", sourceCommit, - renderDir: v02RenderDir(sourceCommit), - render, - degradedReason: "control-plane-render-failed", + renderDir: existingMarker.renderDir ?? null, + marker: existingMarker, + markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS, }; } - const apply = applyV02ControlPlaneFiles(sourceCommit, false, timeoutSeconds); - const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null; - return { - ok: isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs)), - phase: "control-plane-refresh", - sourceCommit, - renderDir: v02RenderDir(sourceCommit), - render: commandData(render), - apply: compactCommandResult(apply), - cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs), - degradedReason: !isCommandSuccess(apply) - ? "control-plane-apply-failed" - : cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs) - ? "obsolete-cronjob-cleanup-failed" - : undefined, - }; + const lock = acquireV02RefreshLock(sourceCommit); + if (!lock.acquired) { + return { + ok: false, + phase: "control-plane-refresh-lock", + mode: "local-refresh-lock", + sourceCommit, + lockDir: lock.lockDir, + waitedMs: lock.waitedMs, + degradedReason: "control-plane-refresh-lock-timeout", + }; + } + try { + const markerAfterWait = readV02RefreshMarker(sourceCommit, scriptHash); + if (markerAfterWait !== null) { + return { + ok: true, + phase: "control-plane-refresh", + mode: "waited-for-recent-refresh-marker", + sourceCommit, + renderDir: markerAfterWait.renderDir ?? null, + marker: markerAfterWait, + waitedMs: lock.waitedMs, + markerTtlMs: V02_CONTROL_PLANE_REFRESH_TTL_MS, + }; + } + const render = runV02RenderToTemp(sourceCommit); + if (!isCommandSuccess(render.result)) { + return { + ok: false, + phase: "source-render", + sourceCommit, + renderDir: render.renderDir, + render: compactCommandResult(render.result), + degradedReason: "control-plane-render-failed", + }; + } + const apply = applyV02ControlPlaneFiles(render.renderDir, false, timeoutSeconds); + const cleanupObsoleteCronJobs = isCommandSuccess(apply) ? deleteV02ObsoleteCronJobs(false) : null; + const cleanupRenderDir = isCommandSuccess(apply) ? cleanupV02RenderDir(render.renderDir) : null; + const ok = isCommandSuccess(apply) && (cleanupObsoleteCronJobs === null || isCommandSuccess(cleanupObsoleteCronJobs)); + const marker = ok ? writeV02RefreshMarker(sourceCommit, scriptHash, render.renderDir) : null; + return { + ok, + phase: "control-plane-refresh", + mode: "refreshed", + sourceCommit, + renderDir: render.renderDir, + render: compactCommandResult(render.result), + apply: compactCommandResult(apply), + cleanupObsoleteCronJobs: cleanupObsoleteCronJobs === null ? null : compactCommandResult(cleanupObsoleteCronJobs), + cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir), + marker, + waitedMs: lock.waitedMs, + degradedReason: !isCommandSuccess(apply) + ? "control-plane-apply-failed" + : cleanupObsoleteCronJobs !== null && !isCommandSuccess(cleanupObsoleteCronJobs) + ? "obsolete-cronjob-cleanup-failed" + : undefined, + }; + } finally { + releaseV02RefreshLock(lock.lockDir); + } } function getV02ObsoleteCronJobs(): CommandJsonResult { @@ -2297,33 +2508,37 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record { } function runGitMirrorApply(options: G14GitMirrorOptions): Record { - const sourceCommit = getV02Head(); + const head = resolveV02Head(); + const sourceCommit = head.sourceCommit; if (sourceCommit === null) { - return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE }; + return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE, headProbe: compactCommandResult(head.result) }; } const render = runV02RenderToTemp(sourceCommit); - if (!isCommandSuccess(render)) { + if (!isCommandSuccess(render.result)) { return { ok: false, command: "hwlab g14 git-mirror apply", phase: "source-render", sourceCommit, - render: compactCommandResult(render), + renderDir: render.renderDir, + render: compactCommandResult(render.result), }; } - const apply = applyGitMirrorManifestFile(sourceCommit, options.dryRun, options.timeoutSeconds); + const apply = applyGitMirrorManifestFile(render.renderDir, options.dryRun, options.timeoutSeconds); const cleanupLegacyCronJob = isCommandSuccess(apply) ? deleteLegacyGitMirrorCronJob(options.dryRun) : { @@ -3006,15 +3222,17 @@ function runGitMirrorApply(options: G14GitMirrorOptions): Record