fix: coalesce hwlab v02 mirror presync

2026-06-04 16:33:50 +00:00
parent 2450f8bc0c
commit 570a2412f3
2 changed files with 352 additions and 36 deletions
@@ -1,4 +1,4 @@
-import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14";
+import { gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parsePipelineTaskRunMetrics, rolloutRecordBody, semanticChangelogBullets, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02FalseGreenGuard, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableGitMirrorPreSyncMarker, v02ReusableRefreshMarker, v02TaskRunPerformanceSummary } from "./src/hwlab-g14";

 function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
  if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
@@ -135,6 +135,28 @@ assertCondition(
  gitMirrorV02SyncRequirement("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", gitMirrorStatusRaw).required === true,
  "trigger-current must sync mirror before creating PipelineRun when local v0.2 is stale",
 );
+const reusableGitMirrorPreSyncMarker = v02ReusableGitMirrorPreSyncMarker({
+  ok: true,
+  sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  syncedAt: "2026-06-04T16:00:10.000Z",
+  localV02: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  githubV02: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Date.parse("2026-06-04T16:01:00.000Z"), Date.parse("2026-06-04T16:00:00.000Z"));
+const preObservationGitMirrorMarker = v02ReusableGitMirrorPreSyncMarker({
+  ok: true,
+  sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  syncedAt: "2026-06-04T16:00:10.000Z",
+}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Date.parse("2026-06-04T16:01:00.000Z"), Date.parse("2026-06-04T16:00:20.000Z"));
+const staleGitMirrorPreSyncMarker = v02ReusableGitMirrorPreSyncMarker({
+  ok: true,
+  sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  syncedAt: "2026-06-04T16:00:10.000Z",
+}, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", Date.parse("2026-06-04T16:03:00.000Z"), Date.parse("2026-06-04T16:00:00.000Z"));
+assertCondition(
+  reusableGitMirrorPreSyncMarker !== null && preObservationGitMirrorMarker === null && staleGitMirrorPreSyncMarker === null,
+  "v0.2 git mirror pre-sync marker must only reuse fresh same-commit markers written after the stale status observation",
+  { reusableGitMirrorPreSyncMarker, preObservationGitMirrorMarker, staleGitMirrorPreSyncMarker },
+);
 const gitMirrorSummary = gitMirrorStatusSummary(gitMirrorStatusRaw);
 assertCondition(
  gitMirrorSummary.flushNeeded === true && gitMirrorSummary.flushCommand === "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm",
@@ -460,6 +482,7 @@ console.log(JSON.stringify({
    "git mirror sync is a manual devops-infra Job, not a CronJob",
    "git mirror flush is a manual devops-infra Job, not a CronJob",
    "trigger-current can decide whether v0.2 git mirror pre-sync is required",
+    "v0.2 git mirror pre-sync marker only reuses fresh same-commit post-observation markers",
    "git mirror status exposes source and gitops GitHub sync state plus controlled flush command",
    "v0.2 control-plane status help exposes targeted PipelineRun/source-commit inspection",
    "v0.2 control-plane render uses an isolated temp clone from a CI/CD dedicated bare repo",
@@ -66,6 +66,10 @@ const V02_BUILD_TASKRUN_WARNING_SECONDS = 120;
 const V02_BUILD_TASKRUN_CRITICAL_SECONDS = 180;
 const V02_CONTROL_PLANE_REFRESH_TTL_MS = 5 * 60 * 1000;
 const V02_CONTROL_PLANE_REFRESH_LOCK_STALE_MS = 5 * 60 * 1000;
+const V02_GIT_MIRROR_PRESYNC_MIN_WAIT_MS = 30 * 1000;
+const V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS = 120 * 1000;
+const V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS = 5 * 60 * 1000;
+const V02_GIT_MIRROR_PRESYNC_POLL_MS = 3 * 1000;

 export type G14MonitorLane = "g14" | "v02";

@@ -2585,7 +2589,16 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
  }
  printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit) });
  const controlPlaneRefresh = refreshV02ControlPlaneBeforeTrigger(sourceCommit, options.timeoutSeconds);
-  printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: controlPlaneRefresh.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun: v02PipelineRunName(sourceCommit), degradedReason: record(controlPlaneRefresh).degradedReason ?? null });
+  printProgressEvent("hwlab.v02.trigger.progress", {
+    stage: "control-plane-refresh",
+    status: controlPlaneRefresh.ok === true ? "succeeded" : "failed",
+    sourceCommit,
+    pipelineRun: v02PipelineRunName(sourceCommit),
+    mode: record(controlPlaneRefresh).mode ?? null,
+    waitedMs: record(controlPlaneRefresh).waitedMs ?? null,
+    renderDir: record(controlPlaneRefresh).renderDir ?? null,
+    degradedReason: record(controlPlaneRefresh).degradedReason ?? null,
+  });
  if (controlPlaneRefresh.ok !== true) {
    return {
      ok: false,
@@ -2616,9 +2629,13 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unk
    mode: record(gitMirrorPreSync).mode ?? null,
    required: nested(gitMirrorPreSync, ["before", "required"]) ?? null,
    localV02: nested(gitMirrorPreSync, ["before", "localV02"]) ?? null,
+    finalLocalV02: nested(gitMirrorPreSync, ["final", "localV02"]) ?? nested(gitMirrorPreSync, ["marker", "localV02"]) ?? nested(gitMirrorPreSync, ["after", "localV02"]) ?? null,
    pendingFlush: nested(gitMirrorPreSync, ["before", "pendingFlush"]) ?? null,
    reason: nested(gitMirrorPreSync, ["before", "reason"]) ?? null,
    syncElapsedMs: nested(gitMirrorPreSync, ["sync", "elapsedMs"]) ?? null,
+    waitElapsedMs: nested(gitMirrorPreSync, ["wait", "elapsedMs"]) ?? null,
+    waitAttempts: nested(gitMirrorPreSync, ["wait", "attempts"]) ?? null,
+    lockWaitedMs: record(gitMirrorPreSync).waitedMs ?? null,
    degradedReason: record(gitMirrorPreSync).degradedReason ?? null,
  });
  if (gitMirrorPreSync.ok !== true) {
@@ -3008,6 +3025,186 @@ function compactGitMirrorSync(sync: Record<string, unknown>): Record<string, unk
  };
 }

+interface V02GitMirrorPreSyncMarker {
+  ok: boolean;
+  sourceCommit: string;
+  syncedAt: string;
+  localV02?: string | null;
+  githubV02?: string | null;
+  reason?: string | null;
+}
+
+function v02GitMirrorPreSyncStateDir(): string {
+  return rootPath(".state", "hwlab-g14", "v02-git-mirror-presync");
+}
+
+function v02GitMirrorPreSyncMarkerPath(sourceCommit: string): string {
+  return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.json`);
+}
+
+function v02GitMirrorPreSyncLockDir(sourceCommit: string): string {
+  return join(v02GitMirrorPreSyncStateDir(), `${shortSha(sourceCommit)}.lock`);
+}
+
+function v02GitMirrorPreSyncWaitMs(timeoutSeconds: number): number {
+  const requestedMs = Math.max(0, Math.floor(timeoutSeconds * 1000));
+  if (requestedMs === 0) return V02_GIT_MIRROR_PRESYNC_MIN_WAIT_MS;
+  return Math.min(V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS, Math.max(V02_GIT_MIRROR_PRESYNC_MIN_WAIT_MS, requestedMs));
+}
+
+export function v02ReusableGitMirrorPreSyncMarker(marker: unknown, sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
+  const candidate = record(marker) as V02GitMirrorPreSyncMarker;
+  const syncedAtMs = timestampMs(candidate.syncedAt);
+  if (candidate.ok !== true || candidate.sourceCommit !== sourceCommit) return null;
+  if (syncedAtMs === null || nowMs - syncedAtMs > V02_GIT_MIRROR_PRESYNC_MAX_WAIT_MS) return null;
+  if (syncedAtMs < minSyncedAtMs) return null;
+  return candidate;
+}
+
+function readV02GitMirrorPreSyncMarker(sourceCommit: string, nowMs = Date.now(), minSyncedAtMs = 0): V02GitMirrorPreSyncMarker | null {
+  const path = v02GitMirrorPreSyncMarkerPath(sourceCommit);
+  if (!existsSync(path)) return null;
+  try {
+    return v02ReusableGitMirrorPreSyncMarker(JSON.parse(readFileSync(path, "utf8")) as unknown, sourceCommit, nowMs, minSyncedAtMs);
+  } catch {
+    return null;
+  }
+}
+
+function writeV02GitMirrorPreSyncMarker(sourceCommit: string, summary: Record<string, unknown>): V02GitMirrorPreSyncMarker {
+  const marker = {
+    ok: true,
+    sourceCommit,
+    syncedAt: new Date().toISOString(),
+    localV02: stringOrNull(summary.localV02),
+    githubV02: stringOrNull(summary.githubV02),
+    reason: stringOrNull(summary.reason),
+  };
+  const dir = v02GitMirrorPreSyncStateDir();
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(v02GitMirrorPreSyncMarkerPath(sourceCommit), `${JSON.stringify(marker, null, 2)}\n`, "utf8");
+  return marker;
+}
+
+function acquireV02GitMirrorPreSyncLock(sourceCommit: string, waitMs: number): { acquired: boolean; lockDir: string; waitedMs: number } {
+  const stateDir = v02GitMirrorPreSyncStateDir();
+  mkdirSync(stateDir, { recursive: true });
+  const lockDir = v02GitMirrorPreSyncLockDir(sourceCommit);
+  const startedAtMs = Date.now();
+  for (;;) {
+    try {
+      mkdirSync(lockDir);
+      writeFileSync(join(lockDir, "owner.json"), `${JSON.stringify({ pid: process.pid, sourceCommit, acquiredAt: new Date().toISOString() }, null, 2)}\n`, "utf8");
+      return { acquired: true, lockDir, waitedMs: Date.now() - startedAtMs };
+    } catch {
+      const ageMs = (() => {
+        try {
+          return Date.now() - statSync(lockDir).mtimeMs;
+        } catch {
+          return 0;
+        }
+      })();
+      if (ageMs > V02_GIT_MIRROR_PRESYNC_LOCK_STALE_MS) {
+        try {
+          rmSync(lockDir, { recursive: true, force: true });
+          continue;
+        } catch {
+          return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
+        }
+      }
+      if (Date.now() - startedAtMs >= waitMs) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
+      const wait = runCommand(["sleep", "1"], repoRoot, { timeoutMs: 2_000 });
+      if (wait.exitCode !== 0 && wait.timedOut) return { acquired: false, lockDir, waitedMs: Date.now() - startedAtMs };
+    }
+  }
+}
+
+function releaseV02GitMirrorPreSyncLock(lockDir: string): void {
+  try {
+    rmSync(lockDir, { recursive: true, force: true });
+  } catch {
+    // Best-effort cleanup; stale lock expiry handles interrupted workers.
+  }
+}
+
+function waitForV02GitMirrorPreSync(sourceCommit: string, waitMs: number): Record<string, unknown> {
+  const startedAtMs = Date.now();
+  const observations: Record<string, unknown>[] = [];
+  let attempt = 0;
+  for (;;) {
+    attempt += 1;
+    const statusStartMs = Date.now();
+    printProgressEvent("hwlab.v02.trigger.progress", {
+      stage: "git-mirror-pre-sync-wait",
+      status: "polling",
+      sourceCommit,
+      attempt,
+      elapsedMs: statusStartMs - startedAtMs,
+    });
+    const status = runGitMirrorStatus();
+    const summary = compactGitMirrorStatus(status, sourceCommit);
+    observations.push({
+      attempt,
+      elapsedMs: Date.now() - startedAtMs,
+      statusElapsedMs: Date.now() - statusStartMs,
+      ok: summary.ok,
+      required: summary.required,
+      localV02: summary.localV02,
+      githubV02: summary.githubV02,
+      pendingFlush: summary.pendingFlush,
+      reason: summary.reason,
+    });
+    printProgressEvent("hwlab.v02.trigger.progress", {
+      stage: "git-mirror-pre-sync-wait",
+      status: summary.required === false ? "succeeded" : "waiting",
+      sourceCommit,
+      attempt,
+      elapsedMs: Date.now() - startedAtMs,
+      statusElapsedMs: Date.now() - statusStartMs,
+      required: summary.required,
+      localV02: summary.localV02,
+      githubV02: summary.githubV02,
+      pendingFlush: summary.pendingFlush,
+      reason: summary.reason,
+    });
+    if (summary.ok === true && summary.required === false) {
+      return {
+        ok: true,
+        sourceCommit,
+        attempts: attempt,
+        elapsedMs: Date.now() - startedAtMs,
+        final: summary,
+        observations: observations.slice(-6),
+      };
+    }
+    if (Date.now() - startedAtMs >= waitMs) {
+      return {
+        ok: false,
+        sourceCommit,
+        attempts: attempt,
+        elapsedMs: Date.now() - startedAtMs,
+        final: summary,
+        observations: observations.slice(-6),
+        degradedReason: "git-mirror-local-v02-not-current-after-wait",
+      };
+    }
+    const remainingMs = waitMs - (Date.now() - startedAtMs);
+    const sleepMs = Math.max(500, Math.min(V02_GIT_MIRROR_PRESYNC_POLL_MS, remainingMs));
+    const wait = runCommand(["sleep", String(Math.ceil(sleepMs / 1000))], repoRoot, { timeoutMs: sleepMs + 2_000 });
+    if (wait.exitCode !== 0 && wait.timedOut) {
+      return {
+        ok: false,
+        sourceCommit,
+        attempts: attempt,
+        elapsedMs: Date.now() - startedAtMs,
+        final: summary,
+        observations: observations.slice(-6),
+        degradedReason: "git-mirror-pre-sync-wait-sleep-timeout",
+      };
+    }
+  }
+}
+
 function gitMirrorCacheProbeScript(): string {
  return [
    "printf 'lastSync='; cat /cache/HWLAB.last-sync.json 2>/dev/null || true; printf '\\n'",
@@ -3042,6 +3239,7 @@ function gitMirrorCacheProbeScript(): string {
 }

 function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlaneOptions, "dryRun" | "timeoutSeconds">): Record<string, unknown> {
+  const waitMs = v02GitMirrorPreSyncWaitMs(options.timeoutSeconds);
  const statusStartMs = Date.now();
  printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync-status", status: "started", sourceCommit });
  const before = runGitMirrorStatus();
@@ -3074,40 +3272,135 @@ function preSyncV02GitMirror(sourceCommit: string, options: Pick<G14ControlPlane
      before: beforeSummary,
    };
  }
-  printProgressEvent("hwlab.v02.trigger.progress", {
-    stage: "git-mirror-pre-sync-sync",
-    status: "started",
-    sourceCommit,
-    reason: beforeSummary.reason,
-    localV02: beforeSummary.localV02,
-    pendingFlush: beforeSummary.pendingFlush,
-  });
-  const sync = runGitMirrorSync({
-    action: "sync",
-    confirm: true,
-    dryRun: false,
-    timeoutSeconds: options.timeoutSeconds,
-  });
-  printProgressEvent("hwlab.v02.trigger.progress", {
-    stage: "git-mirror-pre-sync-sync",
-    status: sync.ok === true ? "succeeded" : "failed",
-    sourceCommit,
-    durationMs: sync.elapsedMs ?? null,
-    jobName: sync.jobName ?? null,
-  });
-  const syncStatus = record(sync.status);
-  const after = syncStatus.ok === true ? syncStatus : runGitMirrorStatus();
-  const afterSummary = compactGitMirrorStatus(after, sourceCommit);
-  const ok = sync.ok === true && afterSummary.required === false;
-  return {
-    ok,
-    mode: "auto-sync-before-trigger",
-    sourceCommit,
-    before: beforeSummary,
-    sync: compactGitMirrorSync(sync),
-    after: afterSummary,
-    degradedReason: ok ? undefined : "git-mirror-local-v02-not-current-after-sync",
-  };
+  const existingMarker = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
+  if (existingMarker !== null) {
+    return {
+      ok: true,
+      mode: "reused-recent-git-mirror-presync-marker",
+      sourceCommit,
+      before: beforeSummary,
+      marker: existingMarker,
+      waitMs,
+    };
+  }
+  const lock = acquireV02GitMirrorPreSyncLock(sourceCommit, waitMs);
+  if (!lock.acquired) {
+    const markerAfterLockTimeout = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
+    if (markerAfterLockTimeout !== null) {
+      return {
+        ok: true,
+        mode: "waited-for-recent-git-mirror-presync-marker",
+        sourceCommit,
+        before: beforeSummary,
+        marker: markerAfterLockTimeout,
+        waitedMs: lock.waitedMs,
+        waitMs,
+      };
+    }
+    return {
+      ok: false,
+      mode: "local-git-mirror-presync-lock",
+      sourceCommit,
+      before: beforeSummary,
+      lockDir: lock.lockDir,
+      waitedMs: lock.waitedMs,
+      waitMs,
+      degradedReason: "git-mirror-pre-sync-lock-timeout",
+    };
+  }
+  try {
+    const markerAfterWait = readV02GitMirrorPreSyncMarker(sourceCommit, Date.now(), statusStartMs);
+    if (markerAfterWait !== null) {
+      return {
+        ok: true,
+        mode: "waited-for-recent-git-mirror-presync-marker",
+        sourceCommit,
+        before: beforeSummary,
+        marker: markerAfterWait,
+        waitedMs: lock.waitedMs,
+        waitMs,
+      };
+    }
+    const recheckStartMs = Date.now();
+    printProgressEvent("hwlab.v02.trigger.progress", {
+      stage: "git-mirror-pre-sync-recheck",
+      status: "started",
+      sourceCommit,
+      waitedMs: lock.waitedMs,
+    });
+    const recheck = runGitMirrorStatus();
+    const recheckSummary = compactGitMirrorStatus(recheck, sourceCommit);
+    printProgressEvent("hwlab.v02.trigger.progress", {
+      stage: "git-mirror-pre-sync-recheck",
+      status: recheckSummary.ok === true ? "succeeded" : "failed",
+      sourceCommit,
+      durationMs: Date.now() - recheckStartMs,
+      required: recheckSummary.required,
+      localV02: recheckSummary.localV02,
+      githubV02: recheckSummary.githubV02,
+      pendingFlush: recheckSummary.pendingFlush,
+      reason: recheckSummary.reason,
+    });
+    if (recheckSummary.ok === true && recheckSummary.required === false) {
+      const marker = writeV02GitMirrorPreSyncMarker(sourceCommit, recheckSummary);
+      return {
+        ok: true,
+        mode: "became-current-after-lock-wait",
+        sourceCommit,
+        before: beforeSummary,
+        recheck: recheckSummary,
+        marker,
+        waitedMs: lock.waitedMs,
+        waitMs,
+      };
+    }
+    printProgressEvent("hwlab.v02.trigger.progress", {
+      stage: "git-mirror-pre-sync-sync",
+      status: "started",
+      sourceCommit,
+      reason: recheckSummary.reason,
+      localV02: recheckSummary.localV02,
+      pendingFlush: recheckSummary.pendingFlush,
+    });
+    const sync = runGitMirrorSync({
+      action: "sync",
+      confirm: true,
+      dryRun: false,
+      timeoutSeconds: options.timeoutSeconds,
+    });
+    printProgressEvent("hwlab.v02.trigger.progress", {
+      stage: "git-mirror-pre-sync-sync",
+      status: sync.ok === true ? "succeeded" : "failed",
+      sourceCommit,
+      durationMs: sync.elapsedMs ?? null,
+      jobName: sync.jobName ?? null,
+    });
+    const syncStatus = record(sync.status);
+    const after = syncStatus.ok === true ? syncStatus : runGitMirrorStatus();
+    const afterSummary = compactGitMirrorStatus(after, sourceCommit);
+    const wait = sync.ok === true && afterSummary.required === true ? waitForV02GitMirrorPreSync(sourceCommit, waitMs) : null;
+    const waitFinal = wait === null ? null : record(record(wait).final);
+    const finalSummary = wait !== null && wait.ok === true ? waitFinal : afterSummary;
+    const ok = sync.ok === true && finalSummary.required === false;
+    const marker = ok ? writeV02GitMirrorPreSyncMarker(sourceCommit, finalSummary) : null;
+    return {
+      ok,
+      mode: "auto-sync-before-trigger",
+      sourceCommit,
+      before: beforeSummary,
+      recheck: recheckSummary,
+      sync: compactGitMirrorSync(sync),
+      after: afterSummary,
+      wait,
+      final: finalSummary,
+      marker,
+      waitedMs: lock.waitedMs,
+      waitMs,
+      degradedReason: ok ? undefined : stringOrNull(record(wait).degradedReason) ?? "git-mirror-local-v02-not-current-after-sync",
+    };
+  } finally {
+    releaseV02GitMirrorPreSyncLock(lock.lockDir);
+  }
 }

 function runGitMirrorStatus(): Record<string, unknown> {