mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(web-probe): defer sentinel git mirror flush

Browse Source
This commit is contained in:
Codex
2026-06-26 14:50:50 +00:00
parent 75a723d9f5
commit acdedbc6bb
1 changed files with 46 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-web-sentinel-cicd.ts
+46 -9
View File
@@ -686,12 +686,12 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
? runSentinelPublishJob(state, true, remainingCicdWaitSeconds())
: sentinelBlockedRemoteResult("source-mirror-sync-blocked", "sentinel source mirror sync failed; publish job was not started");
const flush = !applyOnly && record(publish).ok === true
? runChildCli(["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"], remainingCicdWaitSeconds())
? startSentinelGitMirrorFlushAsync(state)
: null;
const runtimeSecretsApply = applySentinelRuntimeSecrets(state, remainingCicdWaitSeconds());
const publicExposureApply = applySentinelPublicExposure(state, remainingCicdWaitSeconds());
const argoApply = applySentinelArgoApplication(state, remainingCicdWaitSeconds());
const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds());
const observed = waitForSentinelObservedStatus(state, remainingCicdWaitSeconds(), undefined, false);
const observedReady = sentinelObservedReady(observed);
const targetValidation = null;
const targetValidationBlocked = false;
@@ -760,6 +760,7 @@ function runSentinelControlPlaneConfirmed(state: SentinelCicdState, options: Ext
...sentinelCicdElapsedWarnings(record(sourceMirrorSync).elapsedMs, "sentinel source mirror sync", cicdWaitWarningSeconds),
...sentinelCicdElapsedWarnings(record(publish).elapsedMs, "sentinel publish", cicdWaitWarningSeconds),
...sentinelCicdElapsedWarnings(record(flush).result === undefined ? null : record(record(flush).result).durationMs, "sentinel git-mirror flush", cicdWaitWarningSeconds),
...asyncGitMirrorFlushWarnings(flush),
...targetValidationDeferredWarnings(state, applyOnly, cicdWaitWarningSeconds),
...(Array.isArray(record(targetValidation).warnings) ? record(targetValidation).warnings.map(text) : []),
...(targetValidationBlocked ? ["targetValidation is blocked; top-level STATUS only covers sentinel control-plane rollout. HWLAB business recovery remains pending; rerun quick verify after internal DB switch completes, without public fallback or a second execution path."] : []),
@@ -795,7 +796,34 @@ function renderAsyncSentinelJob(state: SentinelCicdState, domain: "image" | "con
return rendered(true, command, renderAsyncJobResult(result));
}
function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation): SentinelObservedStatus {
function startSentinelGitMirrorFlushAsync(state: SentinelCicdState): Record<string, unknown> {
const args = ["hwlab", "nodes", "git-mirror", "flush", "--node", state.spec.nodeId, "--lane", state.spec.lane, "--confirm", "--wait"];
const job = startJob(
`hwlab_nodes_${state.spec.lane}_web_probe_sentinel_${safeJobSegment(state.sentinelId)}_git_mirror_flush`,
["bun", "scripts/cli.ts", ...args],
`Flush HWLAB ${state.spec.lane} git mirror after web-probe sentinel ${state.sentinelId} GitOps publish for node ${state.spec.nodeId}`,
);
return {
ok: true,
mode: "async-job",
job,
next: {
status: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`,
wait: ["bun", "scripts/cli.ts", ...args].join(" "),
gitMirrorStatus: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${state.spec.nodeId} --lane ${state.spec.lane}`,
},
valuesRedacted: true,
};
}
function asyncGitMirrorFlushWarnings(flush: unknown): string[] {
const item = record(flush);
if (item.mode !== "async-job") return [];
const next = record(item.next);
return [`sentinel git-mirror flush is running asynchronously to keep control-plane confirm-wait under 120s; follow ${next.status ?? next.gitMirrorStatus ?? "the reported job status"} for GitHub mirror closeout.`];
}
function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
const registry = probeImageRegistry(state, timeoutSeconds);
const gitops = probeGitopsRuntimeManifest(state, timeoutSeconds);
const effectiveExpectation = {
@@ -805,29 +833,33 @@ function collectSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds:
return {
sourceMirror: probeSourceMirror(state, timeoutSeconds),
registry,
gitMirror: runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds),
gitMirror: includeGitMirror
? runChildCli(["hwlab", "nodes", "git-mirror", "status", "--node", state.spec.nodeId, "--lane", state.spec.lane], timeoutSeconds)
: { ok: true, skipped: true, reason: "deferred-to-async-flush", valuesRedacted: true },
gitops,
argo: probeArgoApplication(state, timeoutSeconds, effectiveExpectation.gitopsRevision),
runtime: probeRuntimeObjects(state, timeoutSeconds, effectiveExpectation.runtimeImage),
};
}
function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation): SentinelObservedStatus {
function waitForSentinelObservedStatus(state: SentinelCicdState, timeoutSeconds: number, expectation?: SentinelObservedExpectation, includeGitMirror = true): SentinelObservedStatus {
const startedAt = Date.now();
const timeoutMs = Math.max(1_000, Math.min(timeoutSeconds * 1000, controlPlaneWaitWarningSeconds(state) * 1000));
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation);
let observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
while (!sentinelObservedReady(observed) && Date.now() - startedAt < timeoutMs) {
runCommand(["sleep", "2"], repoRoot, { timeoutMs: 3_000 });
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation);
observed = collectSentinelObservedStatus(state, timeoutSeconds, expectation, includeGitMirror);
}
return observed;
}
function sentinelObservedReady(value: Record<string, unknown> | SentinelObservedStatus): boolean {
const observed = record(value);
const gitMirror = record(observed.gitMirror);
const gitMirrorReady = gitMirror.skipped === true || gitMirror.ok === true;
return record(observed.sourceMirror).ok === true
&& record(record(observed.registry).probe).present === true
&& record(observed.gitMirror).ok === true
&& gitMirrorReady
&& record(observed.gitops).ok === true
&& record(observed.argo).ok === true
&& record(observed.runtime).ok === true;
@@ -3543,7 +3575,11 @@ function renderControlPlaneResult(result: Record<string, unknown>): string {
"",
Object.keys(publish).length === 0 ? "PUBLISH\n-" : table(["OK", "PHASE", "JOB", "DIGEST", "GITOPS"], [[publish.ok, publish.phase, publish.jobName, short(record(publish.payload).digestRef), short(record(publish.payload).gitopsCommit)]]),
"",
Object.keys(flush).length === 0 ? "FLUSH\n-" : table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]),
Object.keys(flush).length === 0
? "FLUSH\n-"
: flush.mode === "async-job"
? table(["OK", "MODE", "JOB", "STATUS"], [[flush.ok, flush.mode, record(flush.job).id, record(flush.next).status]])
: table(["OK", "EXIT", "TIMED_OUT", "PREVIEW"], [[flush.ok, record(flush.result).exitCode, record(flush.result).timedOut, record(flush.result).stdoutPreview]]),
"",
Object.keys(runtimeSecretsApply).length === 0 ? "RUNTIME_SECRETS\n-" : table(["OK", "SECRETS", "KEYS", "SKIPPED"], [[runtimeSecretsApply.ok, runtimeSecretsApply.secretCount ?? "-", runtimeSecretsApply.keyCount ?? "-", runtimeSecretsApply.skippedKeyCount ?? "-"]]),
"",
@@ -3596,6 +3632,7 @@ function observedStatusRow(name: string, value: unknown): unknown[] | null {
function observedDetail(name: string, item: Record<string, unknown>): string {
if (name === "source") return `${record(item.probe).mode ?? "mirror"} ${short(record(item.probe).commit)}/${short(record(item.probe).expectedCommit)}`;
if (name === "registry") return `${record(item.probe).present === true ? "present" : "missing"} ${short(record(item.probe).digest)}`;
if (name === "git-mirror" && item.skipped === true) return `${item.reason ?? "skipped"}`;
if (name === "gitops") return `${short(item.revision)} image=${short(item.image)}`;
if (name === "argo") return `${item.syncStatus ?? "-"} ${item.healthStatus ?? "-"} ${short(item.revision)}/${short(item.expectedRevision)}`;
if (name === "runtime") {