From 570a2412f303c24047d55b263c0ec8075f2cb1c4 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 4 Jun 2026 16:33:50 +0000 Subject: [PATCH] fix: coalesce hwlab v02 mirror presync --- scripts/hwlab-g14-contract-test.ts | 25 +- scripts/src/hwlab-g14.ts | 363 ++++++++++++++++++++++++++--- 2 files changed, 352 insertions(+), 36 deletions(-) diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index 49c14b00..0a24bf56 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -1,4 +1,4 @@ -import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14"; +import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableGitMirrorPreSyncMarker, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14"; function assertCondition(condition: unknown, message: string, detail: unknown = {}): void { if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`); @@ -135,6 +135,28 @@ assertCondition( gitMirrorV02SyncRequirement("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", gitMirrorStatusRaw).required === true, "trigger-current must sync mirror before creating PipelineRun when local v0.2 is stale", ); +const reusableGitMirrorPreSyncMarker = v02ReusableGitMirrorPreSyncMarker({ + ok: true, + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + syncedAt: "2026-06-04T16:00:10.000Z", + localV02: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + githubV02: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", +}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Date.parse("2026-06-04T16:01:00.000Z"), Date.parse("2026-06-04T16:00:00.000Z")); +const preObservationGitMirrorMarker = v02ReusableGitMirrorPreSyncMarker({ + ok: true, + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + syncedAt: "2026-06-04T16:00:10.000Z", +}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Date.parse("2026-06-04T16:01:00.000Z"), Date.parse("2026-06-04T16:00:20.000Z")); +const staleGitMirrorPreSyncMarker = v02ReusableGitMirrorPreSyncMarker({ + ok: true, + sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + syncedAt: "2026-06-04T16:00:10.000Z", +}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Date.parse("2026-06-04T16:03:00.000Z"), Date.parse("2026-06-04T16:00:00.000Z")); +assertCondition( + reusableGitMirrorPreSyncMarker !== null && preObservationGitMirrorMarker === null && staleGitMirrorPreSyncMarker === null, + "v0.2 git mirror pre-sync marker must only reuse fresh same-commit markers written after the stale status observation", + { reusableGitMirrorPreSyncMarker, preObservationGitMirrorMarker, staleGitMirrorPreSyncMarker }, +); const gitMirrorSummary = gitMirrorStatusSummary(gitMirrorStatusRaw); assertCondition( gitMirrorSummary.flushNeeded === true && gitMirrorSummary.flushCommand === "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm", @@ -460,6 +482,7 @@ console.log(JSON.stringify({ "git mirror sync is a manual devops-infra Job, not a CronJob", "git mirror flush is a manual devops-infra Job, not a CronJob", "trigger-current can decide whether v0.2 git mirror pre-sync is required", + "v0.2 git mirror pre-sync marker only reuses fresh same-commit post-observation markers", "git mirror status exposes source and gitops GitHub sync state plus controlled flush command", "v0.2 control-plane status help exposes targeted PipelineRun/source-commit inspection", "v0.2 control-plane render uses an isolated temp clone from a CI/CD dedicated bare repo", diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 9151aa73..dea44923 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -66,6 +66,10 @@ const V02_BUILD_TASKRUN_WARNING_SECONDS = 120; const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180; const V02_CONTROL_PLANE_REFRESH_TTL_MS = 5 * 60 * 1000; const V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS = 5 * 60 * 1000; +const V02_GIT_MIRROR_PRESYNC_MIN_WAIT_MS = 30 * 1000; +const V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS = 120 * 1000; +const V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS = 5 * 60 * 1000; +const V02_GIT_MIRROR_PRESYNC_POLL_MS = 3 * 1000; export type G14MonitorLane = "g14" | "v02"; @@ -2585,7 +2589,16 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record): Record V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS) return null; + if (syncedAtMs < minSyncedAtMs) return null; + return candidate; +} + +function readV02GitMirrorPreSyncMarker(sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null { + const path = v02GitMirrorPreSyncMarkerPath(sourceCommit); + if (!existsSync(path)) return null; + try { + return v02ReusableGitMirrorPreSyncMarker(JSON.parse(readFileSync(path, "utf8")) as unknown, sourceCommit, nowMs, minSyncedAtMs); + } catch { + return null; + } +} + +function writeV02GitMirrorPreSyncMarker(sourceCommit: string, summary: Record): V02GitMirrorPreSyncMarker { + const marker = { + ok: true, + sourceCommit, + syncedAt: new Date().toISOString(), + localV02: stringOrNull(summary.localV02), + githubV02: stringOrNull(summary.githubV02), + reason: stringOrNull(summary.reason), + }; + const dir = v02GitMirrorPreSyncStateDir(); + mkdirSync(dir, { recursive: true }); + writeFileSync(v02GitMirrorPreSyncMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8"); + return marker; +} + +function acquireV02GitMirrorPreSyncLock(sourceCommit: string, waitMs: number): { acquired: boolean; lockDir: string; waitedMs: number } { + const stateDir = v02GitMirrorPreSyncStateDir(); + mkdirSync(stateDir, { recursive: true }); + const lockDir = v02GitMirrorPreSyncLockDir(sourceCommit); + const startedAtMs = Date.now(); + for (;;) { + try { + mkdirSync(lockDir); + writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8"); + return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs }; + } catch { + const ageMs = (() => { + try { + return Date.now() - statSync(lockDir).mtimeMs; + } catch { + return 0; + } + })(); + if (ageMs > V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS) { + try { + rmSync(lockDir, { recursive: true, force: true }); + continue; + } catch { + return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs }; + } + } + if (Date.now() - startedAtMs >= waitMs) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs }; + const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 }); + if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs }; + } + } +} + +function releaseV02GitMirrorPreSyncLock(lockDir: string): void { + try { + rmSync(lockDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup; stale lock expiry handles interrupted workers. + } +} + +function waitForV02GitMirrorPreSync(sourceCommit: string, waitMs: number): Record { + const startedAtMs = Date.now(); + const observations: Record[] = []; + let attempt = 0; + for (;;) { + attempt += 1; + const statusStartMs = Date.now(); + printProgressEvent("hwlab.v02.trigger.progress", { + stage: "git-mirror-pre-sync-wait", + status: "polling", + sourceCommit, + attempt, + elapsedMs: statusStartMs - startedAtMs, + }); + const status = runGitMirrorStatus(); + const summary = compactGitMirrorStatus(status, sourceCommit); + observations.push({ + attempt, + elapsedMs: Date.now() - startedAtMs, + statusElapsedMs: Date.now() - statusStartMs, + ok: summary.ok, + required: summary.required, + localV02: summary.localV02, + githubV02: summary.githubV02, + pendingFlush: summary.pendingFlush, + reason: summary.reason, + }); + printProgressEvent("hwlab.v02.trigger.progress", { + stage: "git-mirror-pre-sync-wait", + status: summary.required === false ? "succeeded" : "waiting", + sourceCommit, + attempt, + elapsedMs: Date.now() - startedAtMs, + statusElapsedMs: Date.now() - statusStartMs, + required: summary.required, + localV02: summary.localV02, + githubV02: summary.githubV02, + pendingFlush: summary.pendingFlush, + reason: summary.reason, + }); + if (summary.ok === true && summary.required === false) { + return { + ok: true, + sourceCommit, + attempts: attempt, + elapsedMs: Date.now() - startedAtMs, + final: summary, + observations: observations.slice(-6), + }; + } + if (Date.now() - startedAtMs >= waitMs) { + return { + ok: false, + sourceCommit, + attempts: attempt, + elapsedMs: Date.now() - startedAtMs, + final: summary, + observations: observations.slice(-6), + degradedReason: "git-mirror-local-v02-not-current-after-wait", + }; + } + const remainingMs = waitMs - (Date.now() - startedAtMs); + const sleepMs = Math.max(500, Math.min(V02_GIT_MIRROR_PRESYNC_POLL_MS, remainingMs)); + const wait = runCommand(["sleep", String(Math.ceil(sleepMs / 1000))], repoRoot, { timeoutMs: sleepMs + 2_000 }); + if (wait.exitCode !== 0 && wait.timedOut) { + return { + ok: false, + sourceCommit, + attempts: attempt, + elapsedMs: Date.now() - startedAtMs, + final: summary, + observations: observations.slice(-6), + degradedReason: "git-mirror-pre-sync-wait-sleep-timeout", + }; + } + } +} + function gitMirrorCacheProbeScript(): string { return [ "printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'", @@ -3042,6 +3239,7 @@ function gitMirrorCacheProbeScript(): string { } function preSyncV02GitMirror(sourceCommit: string, options: Pick): Record { + const waitMs = v02GitMirrorPreSyncWaitMs(options.timeoutSeconds); const statusStartMs = Date.now(); printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit }); const before = runGitMirrorStatus(); @@ -3074,40 +3272,135 @@ function preSyncV02GitMirror(sourceCommit: string, options: Pick {