From acdedbc6bbe3faf7839b2f6fec2005db95013ce1 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 26 Jun 2026 14:50:50 +0000 Subject: [PATCH] fix(web-probe): defer sentinel git mirror flush --- scripts/src/hwlab-node-web-sentinel-cicd.ts | 55 +++++++++++++++++---- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts index 4e2f6cb3..4e1dfb0c 100644 --- a/scripts/src/hwlab-node-web-sentinel-cicd.ts +++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts @@ -686,12 +686,12 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext ? runSentinelPublishJob(state, true, remainingCicdWaitSeconds()) : sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started"); const flush = !applyOnly && record(publish).ok === true - ? runChildCli(["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"], remainingCicdWaitSeconds()) + ? startSentinelGitMirrorFlushAsync(state) : null; const runtimeSecretsApply = applySentinelRuntimeSecrets(state, remainingCicdWaitSeconds()); const publicExposureApply = applySentinelPublicExposure(state, remainingCicdWaitSeconds()); const argoApply = applySentinelArgoApplication(state, remainingCicdWaitSeconds()); - const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds()); + const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds(), undefined, false); const observedReady = sentinelObservedReady(observed); const targetValidation = null; const targetValidationBlocked = false; @@ -760,6 +760,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext ...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds), ...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds), + ...asyncGitMirrorFlushWarnings(flush), ...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds), ...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []), ...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []), @@ -795,7 +796,34 @@ function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "con return rendered(true, command, renderAsyncJobResult(result)); } -function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation): SentinelObservedStatus { +function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record { + const args = ["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"]; + const job = startJob( + `hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_git_mirror_flush`, + ["bun", "scripts/cli.ts", ...args], + `Flush HWLAB ${state.spec.lane} git mirror after web-probe sentinel ${state.sentinelId} GitOps publish for node ${state.spec.nodeId}`, + ); + return { + ok: true, + mode: "async-job", + job, + next: { + status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, + wait: ["bun", "scripts/cli.ts", ...args].join(" "), + gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`, + }, + valuesRedacted: true, + }; +} + +function asyncGitMirrorFlushWarnings(flush: unknown): string[] { + const item = record(flush); + if (item.mode !== "async-job") return []; + const next = record(item.next); + return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under 120s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`]; +} + +function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus { const registry = probeImageRegistry(state, timeoutSeconds); const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds); const effectiveExpectation = { @@ -805,29 +833,33 @@ function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: return { sourceMirror: probeSourceMirror(state, timeoutSeconds), registry, - gitMirror: runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds), + gitMirror: includeGitMirror + ? runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds) + : { ok: true, skipped: true, reason: "deferred-to-async-flush", valuesRedacted: true }, gitops, argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision), runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage), }; } -function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation): SentinelObservedStatus { +function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus { const startedAt = Date.now(); const timeoutMs = Math.max(1_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000)); - let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation); + let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror); while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) { runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 }); - observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation); + observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror); } return observed; } function sentinelObservedReady(value: Record | SentinelObservedStatus): boolean { const observed = record(value); + const gitMirror = record(observed.gitMirror); + const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true; return record(observed.sourceMirror).ok === true && record(record(observed.registry).probe).present === true - && record(observed.gitMirror).ok === true + && gitMirrorReady && record(observed.gitops).ok === true && record(observed.argo).ok === true && record(observed.runtime).ok === true; @@ -3543,7 +3575,11 @@ function renderControlPlaneResult(result: Record): string { "", Object.keys(publish).length === 0 ? "PUBLISH\n-" : table(["OK", "PHASE", "JOB", "DIGEST", "GITOPS"], [[publish.ok, publish.phase, publish.jobName, short(record(publish.payload).digestRef), short(record(publish.payload).gitopsCommit)]]), "", - Object.keys(flush).length === 0 ? "FLUSH\n-" : table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]), + Object.keys(flush).length === 0 + ? "FLUSH\n-" + : flush.mode === "async-job" + ? table(["OK", "MODE", "JOB", "STATUS"], [[flush.ok, flush.mode, record(flush.job).id, record(flush.next).status]]) + : table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]), "", Object.keys(runtimeSecretsApply).length === 0 ? "RUNTIME_SECRETS\n-" : table(["OK", "SECRETS", "KEYS", "SKIPPED"], [[runtimeSecretsApply.ok, runtimeSecretsApply.secretCount ?? "-", runtimeSecretsApply.keyCount ?? "-", runtimeSecretsApply.skippedKeyCount ?? "-"]]), "", @@ -3596,6 +3632,7 @@ function observedStatusRow(name: string, value: unknown): unknown[] | null { function observedDetail(name: string, item: Record): string { if (name === "source") return `${record(item.probe).mode ?? "mirror"} ${short(record(item.probe).commit)}/${short(record(item.probe).expectedCommit)}`; if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`; + if (name === "git-mirror" && item.skipped === true) return `${item.reason ?? "skipped"}`; if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`; if (name === "argo") return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}`; if (name === "runtime") {