From 70fdaa4272cf244fa2e5c71c1589eb67e15ff2a5 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Jun 2026 18:06:35 +0000 Subject: [PATCH] fix(web-probe): skip sentinel source sync when mirror ready --- scripts/src/hwlab-node-web-sentinel-cicd.ts | 46 +++++++++++++++++---- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index aa6f29a2..94d26b11 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -648,13 +648,14 @@ function probeImageRegistry(state: SentinelCicdState, timeoutSeconds: number): R function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extract): RenderedCliResult { const startedAt = Date.now(); const command = "web-probe sentinel image build"; - const sourceMirrorSync = runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds); - const publish = sourceMirrorSync.ok === true + const sourceMirrorSync = state.sourceHead.ok === true ? sentinelSourceMirrorAlreadyPresentResult(state) : runSentinelSourceMirrorSyncJob(state, options.timeoutSeconds); + const sourceMirrorReady = sourceMirrorSync.ok === true || state.sourceHead.ok === true; + const publish = sourceMirrorReady ? runSentinelPublishJob(state, false, options.timeoutSeconds) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const registry = probeImageRegistry(state, options.timeoutSeconds); const registryReady = record(registry.probe).present === true; - const ok = state.configReady && state.sourceHead.ok && sourceMirrorSync.ok === true && publish.ok === true && registryReady; + const ok = state.configReady && state.sourceHead.ok && sourceMirrorReady && publish.ok === true && registryReady; const elapsedMs = Date.now() - startedAt; const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state); const result = { @@ -675,10 +676,11 @@ function runSentinelImageBuildConfirmed(state: SentinelCicdState, options: Extra ...sentinelCicdElapsedWarnings(elapsedMs, "sentinel image build confirm-wait", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds), + ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), ], blocker: ok ? null - : sourceMirrorSync.ok !== true + : !sourceMirrorReady ? { code: "sentinel-source-mirror-sync-failed", reason: "source mirror sync did not complete; investigate git mirror/proxy before image publish" } : publish.ok !== true ? { code: "sentinel-image-publish-failed", reason: "remote image publish job failed before registry validation" } @@ -699,10 +701,11 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext const cicdWaitWarningSeconds = controlPlaneWaitWarningSeconds(state); const deadline = startedAt + cicdWaitWarningSeconds * 1000; const remainingCicdWaitSeconds = () => remainingSeconds(deadline, Math.min(options.timeoutSeconds, cicdWaitWarningSeconds)); - const sourceMirrorSync = applyOnly ? null : runSentinelSourceMirrorSyncJob(state, remainingCicdWaitSeconds()); + const sourceMirrorSync = applyOnly ? null : state.sourceHead.ok === true ? sentinelSourceMirrorAlreadyPresentResult(state) : runSentinelSourceMirrorSyncJob(state, remainingCicdWaitSeconds()); + const sourceMirrorReady = applyOnly || record(sourceMirrorSync).ok === true || state.sourceHead.ok === true; const publish = applyOnly ? null - : record(sourceMirrorSync).ok === true + : sourceMirrorReady ? runSentinelPublishJob(state, true, remainingCicdWaitSeconds()) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const flush = !applyOnly && record(publish).ok === true @@ -717,7 +720,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext const targetValidationBlocked = false; const ok = state.configReady && state.sourceHead.ok - && (applyOnly || record(sourceMirrorSync).ok === true) + && sourceMirrorReady && (applyOnly || record(publish).ok === true) && (applyOnly || record(flush).ok === true) && record(runtimeSecretsApply).ok === true @@ -726,8 +729,8 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext && observedReady; const elapsedMs = Date.now() - startedAt; const blocker = ok ? null : { - code: record(sourceMirrorSync).ok === false ? "sentinel-source-mirror-sync-failed" : record(runtimeSecretsApply).ok === false ? "sentinel-runtime-secret-sync-failed" : "sentinel-control-plane-not-ready", - reason: record(sourceMirrorSync).ok === false + code: !sourceMirrorReady ? "sentinel-source-mirror-sync-failed" : record(runtimeSecretsApply).ok === false ? "sentinel-runtime-secret-sync-failed" : "sentinel-control-plane-not-ready", + reason: !sourceMirrorReady ? "source mirror sync did not complete; investigate git mirror/proxy before control-plane publish" : record(runtimeSecretsApply).ok === false ? "one or more YAML-declared runtime Secrets were not synced from sourceRef" @@ -781,6 +784,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds), ...asyncGitMirrorFlushWarnings(flush), + ...sourceMirrorAlreadyReadyWarnings(state, sourceMirrorSync), ...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds), ...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []), ...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []), @@ -1547,6 +1551,30 @@ function sentinelCicdElapsedWarnings(value: unknown, subject: string, budgetSeco return [`${subject} exceeded configured ${Math.round(budgetMs / 1000)}s CI/CD wait budget (${Math.round(elapsedMs / 1000)}s); optimize wait-stage latency before rerunning long confirm-wait operations.`]; } +function sourceMirrorAlreadyReadyWarnings(state: SentinelCicdState, sourceMirrorSync: unknown): string[] { + const sync = record(sourceMirrorSync); + if (sync.ok === true || state.sourceHead.ok !== true) return []; + return [`sentinel source mirror sync did not complete, but internal git mirror already contains ${short(state.sourceHead.commit)}; continuing publish from the YAML-declared read URL and treating the sync failure as a non-blocking egress warning.`]; +} + +function sentinelSourceMirrorAlreadyPresentResult(state: SentinelCicdState): Record { + return { + ok: true, + phase: "already-present", + jobName: null, + payload: { + ok: true, + status: "already-present", + sourceCommit: state.sourceHead.commit, + mirrorCommit: state.sourceHead.commit, + valuesRedacted: true, + }, + polls: 0, + elapsedMs: 0, + valuesRedacted: true, + }; +} + function targetValidationDeferredWarnings(state: SentinelCicdState, applyOnly: boolean, budgetSeconds: number): string[] { if (applyOnly) return []; const next = sentinelP5Next(state);