mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: bound node runtime cicd wait closeout

Browse Source
This commit is contained in:
Codex
2026-07-02 04:38:46 +00:00
parent 1b24e994f8
commit ed32eb84b2
12 changed files with 880 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node/git-mirror.ts
+298 -33
View File
@@ -29,12 +29,12 @@ import { runDelegatedHwlabNodeCommand, type DelegatedNodeDomain } from "../hwlab
import type { RenderedCliResult } from "../output";
import { formatElapsedMs, isCommandSuccess, nodeRuntimePipelineRunName, nodeRuntimeRerunPipelineRunName, resolveNodeRuntimeLaneHead, shortValue } from "./cleanup";
import { nodeRuntimeApply } from "./control-actions";
import { nodeRuntimeApply, nodeRuntimeRefresh } from "./control-actions";
import { NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS, NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS } from "./entry";
import { parseNodeScopedDelegatedOptions } from "./plan";
import { compactNodeRuntimeTaskRunDiagnostic, nodeRuntimePipelineFailureSummary } from "./render";
import { compactNodeRuntimeTaskRunDiagnostic, nodeRuntimeControlPlaneStatus, nodeRuntimePipelineFailureSummary } from "./render";
import { compactRuntimeCommand } from "./runtime-common";
import { compactNodeRuntimeGitMirrorObservation, compactNodeRuntimeGitMirrorRun, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorRun, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput } from "./status";
import { compactNodeRuntimeGitMirrorObservation, compactNodeRuntimeGitMirrorRun, nodeRuntimeEnsureGitMirrorFlushed, nodeRuntimeEnsureGitMirrorSourceCurrent, nodeRuntimeExternalPostgresSecretRows, nodeRuntimeGitMirrorRun, nodeRuntimeGitMirrorStatus, nodeRuntimeOpportunisticGitMirrorFlush, nodeRuntimeOpportunisticGitMirrorSync, nodeScopedFullOutput, type NodeRuntimeGitMirrorRunOptions } from "./status";
import { record } from "./utils";
import { webObserveTable } from "./web-observe-render";
import { createNodeRuntimePipelineRun, getNodeRuntimePipelineRun, nodeRuntimePipelineRunManifest, printNodeRuntimeTriggerProgress, waitForNodeRuntimePipelineRunTerminal } from "./web-probe";
@@ -48,20 +48,25 @@ export function nodeRuntimeTriggerCurrentOutput(scoped: ReturnType<typeof parseN
export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const pipelineWaitSeconds = nodeRuntimeCicdWaitSeconds(scoped);
const triggerStartedAt = Date.now();
const triggerElapsedMs = () => Date.now() - triggerStartedAt;
const triggerDeadlineMs = triggerStartedAt + (nodeRuntimeCicdWaitSeconds(scoped) * 1000);
const remainingTriggerSeconds = () => Math.max(0, Math.floor((triggerDeadlineMs - Date.now()) / 1000));
const triggerGitMirrorOptions = nodeRuntimeTriggerGitMirrorOptions(triggerDeadlineMs);
const sourceSyncScoped = nodeRuntimeScopedForTriggerDeadline(scoped, triggerDeadlineMs);
const sourceSnapshotSync = scoped.dryRun
? null
: nodeRuntimeGitMirrorRun({
...scoped,
: sourceSyncScoped === null
? nodeRuntimeTriggerBudgetExhausted(scoped, "source-snapshot-sync", "-", "-")
: nodeRuntimeGitMirrorRun({
...sourceSyncScoped,
domain: "git-mirror",
action: "sync",
confirm: true,
dryRun: false,
wait: true,
discardStaleGitops: scoped.discardStaleGitops === true || scoped.rerun === true,
});
}, triggerGitMirrorOptions);
if (sourceSnapshotSync !== null && sourceSnapshotSync.ok !== true) {
return {
ok: false,
@@ -116,20 +121,24 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
}
if (!scoped.rerun && before.exists === true && (before.status === "True" || before.status === "Unknown")) {
const pipelineWait = before.status === "Unknown"
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, remainingTriggerSeconds(), {
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun, triggerGitMirrorOptions),
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun, triggerGitMirrorOptions),
})
: { ok: true, status: "already-succeeded", pipelineRun: before, polls: 0, elapsedMs: 0 };
const waitedPipelineRun = record(pipelineWait.pipelineRun);
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
const postFlush = waitedPipelineRun.status === "True"
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun, null, triggerGitMirrorOptions)
: null;
const closeout = waitedPipelineRun.status === "True" && (postFlush === null || postFlush.ok === true)
? waitForNodeRuntimeCicdCloseout(scoped, pipelineRun, sourceCommit, triggerDeadlineMs, { allowRefresh: true })
: null;
const pipelinePending = pipelineWait.status === "pending";
const ok = pipelineWait.ok === true && (postFlush === null || postFlush.ok === true);
const closeoutPending = closeout?.status === "pending";
const ok = pipelineWait.ok === true && (postFlush === null || postFlush.ok === true) && closeout?.ok === true;
const elapsedMs = triggerElapsedMs();
const triggerWarning = pipelinePending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
const triggerWarning = pipelinePending || closeoutPending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
if (triggerWarning !== null) printNodeRuntimeTriggerProgress(spec, { stage: "trigger-current", status: "warning", ...triggerWarning });
return {
ok,
@@ -137,15 +146,21 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
node: scoped.node,
lane: scoped.lane,
mode: "confirmed-trigger",
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
warning: pipelinePending ? pipelineWait.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : triggerWarning ?? undefined,
completion: pipelinePending || closeoutPending ? "pending" : ok ? "completed" : "failed",
warning: pipelinePending
? pipelineWait.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait)
: closeoutPending
? nodeRuntimeCloseoutWaitWarning(spec, pipelineRun, closeout)
: triggerWarning ?? undefined,
triggerElapsedMs: elapsedMs,
deadlineMs: triggerDeadlineMs,
sourceCommit,
pipelineRun,
before,
pipelineWait,
pipelineFailureSummary,
postFlush,
closeout,
sourceSnapshotSync: sourceSnapshotSync === null ? null : compactNodeRuntimeGitMirrorRun(sourceSnapshotSync),
skipped: true,
reason: before.status === "True" ? "existing-pipelinerun-succeeded" : "existing-pipelinerun-running",
@@ -153,7 +168,15 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
skipExplanation: "An existing PipelineRun for this HWLAB source commit was reused. If UniDesk node/lane YAML or other render inputs changed without a HWLAB source commit change, rerun the controlled publish with --rerun.",
rerunAvailable: true,
rerunCommand: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun`,
degradedReason: ok ? undefined : postFlush !== null && postFlush.ok !== true ? "node-runtime-git-mirror-post-flush-failed" : "node-runtime-existing-pipelinerun-not-succeeded",
degradedReason: ok
? undefined
: pipelinePending
? "node-runtime-existing-pipelinerun-not-succeeded"
: postFlush !== null && postFlush.ok !== true
? "node-runtime-git-mirror-post-flush-failed"
: closeoutPending
? "node-runtime-cicd-closeout-pending"
: "node-runtime-cicd-closeout-failed",
next: {
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}`,
rerun: `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm --rerun`,
@@ -161,7 +184,7 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
};
}
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun });
const gitMirror = nodeRuntimeEnsureGitMirrorSourceCurrent(scoped, sourceCommit, pipelineRun);
const gitMirror = nodeRuntimeEnsureGitMirrorSourceCurrent(scoped, sourceCommit, pipelineRun, triggerGitMirrorOptions);
if (gitMirror.ok !== true) {
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "failed", sourceCommit, pipelineRun, reason: gitMirror.degradedReason ?? null });
return {
@@ -180,7 +203,10 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
}
printNodeRuntimeTriggerProgress(spec, { stage: "git-mirror-pre-sync", status: "succeeded", sourceCommit, pipelineRun });
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun });
const refresh = nodeRuntimeApply({ ...scoped, action: "apply", dryRun: false });
const applyScoped = nodeRuntimeScopedForTriggerDeadline(scoped, triggerDeadlineMs, 1);
const refresh = applyScoped === null
? nodeRuntimeTriggerBudgetExhausted(scoped, "control-plane-apply", sourceCommit, pipelineRun)
: nodeRuntimeApply({ ...applyScoped, action: "apply", dryRun: false });
if (refresh.ok !== true) {
const diagnostics = record(refresh.diagnostics);
printNodeRuntimeTriggerProgress(spec, {
@@ -212,28 +238,32 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
}
printNodeRuntimeTriggerProgress(spec, { stage: "control-plane-refresh", status: "succeeded", sourceCommit, pipelineRun });
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun, rerun: scoped.rerun });
const create = createNodeRuntimePipelineRun(spec, sourceCommit, pipelineRun, scoped.timeoutSeconds, scoped.rerun);
const create = createNodeRuntimePipelineRun(spec, sourceCommit, pipelineRun, Math.max(1, remainingTriggerSeconds()), scoped.rerun);
const after = getNodeRuntimePipelineRun(spec, pipelineRun);
const createObserved = after.exists === true && (after.status === "Unknown" || after.status === "True");
const createOk = isCommandSuccess(create) || createObserved;
printNodeRuntimeTriggerProgress(spec, { stage: "create-pipelinerun", status: createOk ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined });
const pipelineWait = createOk
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, pipelineWaitSeconds, {
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun),
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun),
? waitForNodeRuntimePipelineRunTerminal(spec, pipelineRun, remainingTriggerSeconds(), {
opportunisticPostSync: () => nodeRuntimeOpportunisticGitMirrorSync(scoped, sourceCommit, pipelineRun, triggerGitMirrorOptions),
opportunisticPostFlush: () => nodeRuntimeOpportunisticGitMirrorFlush(scoped, sourceCommit, pipelineRun, triggerGitMirrorOptions),
})
: null;
const waitedPipelineRun = record(pipelineWait?.pipelineRun);
const pipelineFailureSummary = nodeRuntimePipelineFailureSummary(pipelineWait);
const postFlush = waitedPipelineRun.status === "True"
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun)
? nodeRuntimeEnsureGitMirrorFlushed(scoped, "post", sourceCommit, pipelineRun, null, triggerGitMirrorOptions)
: null;
const closeout = waitedPipelineRun.status === "True" && (postFlush === null || postFlush.ok === true)
? waitForNodeRuntimeCicdCloseout(scoped, pipelineRun, sourceCommit, triggerDeadlineMs, { allowRefresh: true })
: null;
const pipelineReady = pipelineWait !== null && pipelineWait.ok === true;
const postFlushOk = postFlush === null || postFlush.ok === true;
const pipelinePending = pipelineWait !== null && pipelineWait.status === "pending";
const ok = createOk && (pipelineReady || pipelinePending) && postFlushOk;
const closeoutPending = closeout?.status === "pending";
const ok = createOk && pipelineReady && postFlushOk && closeout?.ok === true;
const elapsedMs = triggerElapsedMs();
const triggerWarning = pipelinePending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
const triggerWarning = pipelinePending || closeoutPending ? null : nodeRuntimeTriggerElapsedWarning(spec, pipelineRun, elapsedMs);
if (triggerWarning !== null) printNodeRuntimeTriggerProgress(spec, { stage: "trigger-current", status: "warning", ...triggerWarning });
return {
ok,
@@ -241,9 +271,14 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
node: scoped.node,
lane: scoped.lane,
mode: "confirmed-trigger",
completion: pipelinePending ? "pending" : ok ? "completed" : "failed",
warning: pipelinePending ? pipelineWait?.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait) : triggerWarning ?? undefined,
completion: pipelinePending || closeoutPending ? "pending" : ok ? "completed" : "failed",
warning: pipelinePending
? pipelineWait?.warning ?? nodeRuntimeCicdWaitWarning(spec, pipelineRun, pipelineWait)
: closeoutPending
? nodeRuntimeCloseoutWaitWarning(spec, pipelineRun, closeout)
: triggerWarning ?? undefined,
triggerElapsedMs: elapsedMs,
deadlineMs: triggerDeadlineMs,
mutation: createOk,
sourceCommit,
pipelineRun,
@@ -258,14 +293,21 @@ export function nodeRuntimeTriggerCurrent(scoped: ReturnType<typeof parseNodeSco
pipelineWait,
pipelineFailureSummary,
postFlush,
closeout,
createObservedAfterTimeout: createObserved && !isCommandSuccess(create) ? true : undefined,
degradedReason: ok
? undefined
: !createOk
? "node-runtime-pipelinerun-create-failed"
: !pipelineReady
: pipelinePending
? "node-runtime-pipelinerun-not-succeeded"
: "node-runtime-git-mirror-post-flush-failed",
: !pipelineReady
? "node-runtime-pipelinerun-not-succeeded"
: !postFlushOk
? "node-runtime-git-mirror-post-flush-failed"
: closeoutPending
? "node-runtime-cicd-closeout-pending"
: "node-runtime-cicd-closeout-failed",
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun}` },
};
}
@@ -274,6 +316,174 @@ export function nodeRuntimeCicdWaitSeconds(scoped: ReturnType<typeof parseNodeSc
return Math.min(scoped.timeoutSeconds, NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS);
}
function nodeRuntimeTriggerGitMirrorOptions(deadlineMs: number): NodeRuntimeGitMirrorRunOptions {
return { deadlineMs, maxAttempts: 1 };
}
function nodeRuntimeScopedForTriggerDeadline(
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
deadlineMs: number,
minRemainingSeconds = 1,
): ReturnType<typeof parseNodeScopedDelegatedOptions> | null {
const remainingSeconds = Math.floor((deadlineMs - Date.now()) / 1000);
if (remainingSeconds < minRemainingSeconds) return null;
return { ...scoped, timeoutSeconds: Math.max(1, Math.min(scoped.timeoutSeconds, remainingSeconds)) };
}
function nodeRuntimeTriggerBudgetExhausted(
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
phase: string,
sourceCommit: string,
pipelineRun: string,
): Record<string, unknown> {
return {
ok: false,
command: `hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
phase,
sourceCommit,
pipelineRun,
status: "pending",
degradedReason: "node-runtime-cicd-budget-exhausted",
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun} --full` },
};
}
export function waitForNodeRuntimeCicdCloseout(
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
pipelineRun: string,
sourceCommit: string,
deadlineMs: number,
options: { allowRefresh?: boolean } = {},
): Record<string, unknown> {
const spec = scoped.spec;
const startedAt = Date.now();
let polls = 0;
let lastSummary: Record<string, unknown> | null = null;
let refresh: Record<string, unknown> | null = null;
printNodeRuntimeTriggerProgress(spec, { stage: "runtime-closeout", status: "started", pipelineRun, sourceCommit, remainingSeconds: Math.max(0, Math.floor((deadlineMs - Date.now()) / 1000)) });
while (Date.now() <= deadlineMs) {
const statusScoped = nodeRuntimeScopedForTriggerDeadline(scoped, deadlineMs, 5);
if (statusScoped === null) break;
polls += 1;
const status = nodeRuntimeControlPlaneStatus({
...statusScoped,
action: "status",
confirm: false,
dryRun: true,
wait: false,
originalArgs: ["status", "--node", scoped.node, "--lane", scoped.lane, "--source-commit", sourceCommit, "--pipeline-run", pipelineRun],
});
const summary = record(record(status).summary ?? status);
lastSummary = summary;
const pipeline = record(summary.pipelineRun);
const argo = record(summary.argo);
const runtime = record(summary.runtime);
const publicProbe = record(summary.publicProbe);
const gitMirror = record(summary.gitMirror);
const degradedReason = typeof summary.degradedReason === "string" ? summary.degradedReason : null;
printNodeRuntimeTriggerProgress(spec, {
stage: "runtime-closeout",
status: "poll",
pipelineRun,
sourceCommit,
polls,
elapsedMs: Date.now() - startedAt,
degradedReason,
pipelineStatus: pipeline.status ?? null,
argoSync: argo.syncStatus ?? null,
argoHealth: argo.health ?? null,
argoRevision: argo.syncRevision ?? null,
argoTargetRevision: argo.targetGitopsRevision ?? null,
runtimeReady: runtime.ready === true,
publicReady: publicProbe.ready === true,
gitMirrorPending: gitMirror.pendingFlush ?? null,
gitMirrorInSync: gitMirror.githubInSync ?? null,
});
if (summary.ok === true) {
printNodeRuntimeTriggerProgress(spec, { stage: "runtime-closeout", status: "succeeded", pipelineRun, sourceCommit, polls, elapsedMs: Date.now() - startedAt });
return {
ok: true,
status: "succeeded",
pipelineRun,
sourceCommit,
polls,
elapsedMs: Date.now() - startedAt,
summary,
refresh,
};
}
const refreshScoped = nodeRuntimeScopedForTriggerDeadline(scoped, deadlineMs, 5);
if (options.allowRefresh === true && refresh === null && refreshScoped !== null && (degradedReason === "argo-revision-not-observed" || degradedReason === "argo-target-revision-progressing" || degradedReason === "argo-health-progressing")) {
refresh = nodeRuntimeRefresh({ ...refreshScoped, action: "refresh", confirm: true, dryRun: false, wait: true });
printNodeRuntimeTriggerProgress(spec, {
stage: "runtime-closeout-refresh",
status: refresh.ok === true ? "succeeded" : "failed",
pipelineRun,
sourceCommit,
degradedReason: refresh.degradedReason ?? null,
});
}
if (degradedReason !== null && !nodeRuntimeCloseoutRetryableReason(degradedReason)) {
printNodeRuntimeTriggerProgress(spec, { stage: "runtime-closeout", status: "failed", pipelineRun, sourceCommit, polls, elapsedMs: Date.now() - startedAt, degradedReason });
return {
ok: false,
status: "failed",
pipelineRun,
sourceCommit,
polls,
elapsedMs: Date.now() - startedAt,
summary,
refresh,
degradedReason,
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun} --full` },
};
}
const remainingMs = deadlineMs - Date.now();
if (remainingMs <= 0) break;
sleepCloseout(Math.min(5_000, Math.max(1000, remainingMs)));
}
const elapsedMs = Date.now() - startedAt;
printNodeRuntimeTriggerProgress(spec, {
stage: "runtime-closeout",
status: "warning",
pipelineRun,
sourceCommit,
polls,
elapsedMs,
waitLimitSeconds: NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS,
degradedReason: lastSummary?.degradedReason ?? null,
});
return {
ok: false,
status: "pending",
pipelineRun,
sourceCommit,
polls,
elapsedMs,
summary: lastSummary,
refresh,
waitLimitSeconds: NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS,
degradedReason: "node-runtime-cicd-closeout-pending",
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${pipelineRun} --full` },
};
}
export function nodeRuntimeCloseoutRetryableReason(reason: string): boolean {
return reason === "argo-revision-not-observed"
|| reason === "argo-target-revision-progressing"
|| reason === "argo-health-progressing"
|| reason === "runtime-workloads-not-ready"
|| reason === "public-probe-not-ready"
|| reason === "git-mirror-pending-flush";
}
function sleepCloseout(ms: number): void {
const buffer = new SharedArrayBuffer(4);
Atomics.wait(new Int32Array(buffer), 0, 0, Math.max(0, ms));
}
export function nodeRuntimeCicdWaitWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, pipelineWait: unknown): Record<string, unknown> {
const wait = record(pipelineWait);
return {
@@ -287,6 +497,21 @@ export function nodeRuntimeCicdWaitWarning(spec: HwlabRuntimeLaneSpec, pipelineR
};
}
export function nodeRuntimeCloseoutWaitWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, closeout: unknown): Record<string, unknown> {
const wait = record(closeout);
const summary = record(wait.summary);
return {
code: "node-runtime-cicd-closeout-over-120s",
message: `PipelineRun ${pipelineRun} reached terminal state, but GitOps/Argo/runtime/public closeout did not converge inside the ${NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS}s one-command CICD budget.`,
waitedSeconds: NODE_RUNTIME_CICD_WAIT_WARNING_SECONDS,
elapsedMs: wait.elapsedMs ?? null,
polls: wait.polls ?? null,
degradedReason: summary.degradedReason ?? wait.degradedReason ?? null,
inspectCloseout: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
inspectGitMirror: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
};
}
export function nodeRuntimeTriggerElapsedWarning(spec: HwlabRuntimeLaneSpec, pipelineRun: string, elapsedMs: number): Record<string, unknown> | null {
if (elapsedMs < NODE_RUNTIME_TRIGGER_SEVERE_WARNING_MS) return null;
return {
@@ -304,6 +529,12 @@ export function withNodeRuntimeTriggerRendered(result: Record<string, unknown>,
const pipelineRunRecord = record(pipelineWait.pipelineRun ?? result.after ?? result.before);
const warning = record(result.warning ?? pipelineWait.warning);
const postFlush = record(result.postFlush);
const closeout = record(result.closeout);
const closeoutSummary = record(closeout.summary);
const closeoutArgo = record(closeoutSummary.argo);
const closeoutRuntime = record(closeoutSummary.runtime);
const closeoutPublic = record(closeoutSummary.publicProbe);
const closeoutGitMirror = record(closeoutSummary.gitMirror);
const gitMirror = record(result.gitMirror);
const gitMirrorSummary = record(postFlush.afterSummary ?? postFlush.beforeSummary ?? gitMirror.statusSummary ?? gitMirror.afterSummary ?? gitMirror.beforeSummary ?? gitMirror.summary);
const refresh = record(result.refresh);
@@ -311,12 +542,13 @@ export function withNodeRuntimeTriggerRendered(result: Record<string, unknown>,
const next = record(result.next);
const pipelineStatus = pipelineRunRecord.status ?? pipelineWait.status ?? "-";
const completion = result.completion ?? (result.ok === true ? "completed" : "failed");
const topStatus = result.ok === true ? "ok" : completion === "pending" ? "pending" : "failed";
const renderedText = [
"hwlab nodes control-plane trigger-current",
"",
webObserveTable(
["NODE", "LANE", "STATUS", "COMPLETION", "SOURCE", "PIPELINERUN"],
[[scoped.node, scoped.lane, result.ok === true ? "ok" : "failed", webObserveText(completion), shortValue(result.sourceCommit), webObserveText(result.pipelineRun)]],
[[scoped.node, scoped.lane, topStatus, webObserveText(completion), shortValue(result.sourceCommit), webObserveText(result.pipelineRun)]],
),
"",
webObserveTable(
@@ -327,6 +559,7 @@ export function withNodeRuntimeTriggerRendered(result: Record<string, unknown>,
["create", create.exitCode === 0 || result.mutation === true ? "ok" : create.exitCode === undefined ? "-" : "failed", result.createObservedAfterTimeout === true ? "observed-after-timeout" : `exit=${webObserveText(create.exitCode)}`],
["pipeline-wait", webObserveText(pipelineWait.status), `pipeline=${webObserveText(pipelineStatus)} polls=${webObserveText(pipelineWait.polls)} elapsed=${formatElapsedMs(pipelineWait.elapsedMs)}`],
["post-flush", postFlush.ok === true ? "ok" : postFlush.ok === false ? "failed" : "-", webObserveText(postFlush.mode ?? postFlush.degradedReason)],
["closeout", closeout.ok === true ? "ok" : closeout.status === "pending" ? "pending" : closeout.ok === false ? "failed" : "-", `reason=${webObserveText(closeoutSummary.degradedReason ?? closeout.degradedReason)} polls=${webObserveText(closeout.polls)} elapsed=${formatElapsedMs(closeout.elapsedMs)}`],
["total", result.triggerElapsedMs === undefined ? "-" : "elapsed", formatElapsedMs(result.triggerElapsedMs)],
],
),
@@ -343,11 +576,24 @@ export function withNodeRuntimeTriggerRendered(result: Record<string, unknown>,
: webObserveTable(
["CHECK", "COMMAND"],
[
["env-reuse", webObserveText(warning.inspectEnvReuse ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun} --full`)],
["closeout", webObserveText(warning.inspectCloseout ?? warning.inspectEnvReuse ?? `bun scripts/cli.ts hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane} --pipeline-run ${result.pipelineRun} --full`)],
["git-mirror", webObserveText(warning.inspectGitMirror ?? `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}`)],
],
),
"",
Object.keys(closeoutSummary).length === 0
? "CLOSEOUT\n-"
: webObserveTable(
["ARGO", "REV", "RUNTIME", "PUBLIC", "GIT_MIRROR"],
[[
`${webObserveText(closeoutArgo.syncStatus)}/${webObserveText(closeoutArgo.health)}`,
`${shortValue(closeoutArgo.syncRevision)}->${shortValue(closeoutArgo.targetGitopsRevision)}`,
`${webObserveText(closeoutRuntime.ready)} workloads=${webObserveText(closeoutRuntime.workloadReady)}`,
webObserveText(closeoutPublic.ready),
`pending=${webObserveText(closeoutGitMirror.pendingFlush)} inSync=${webObserveText(closeoutGitMirror.githubInSync)}`,
]],
),
"",
Object.keys(gitMirrorSummary).length === 0
? "GIT_MIRROR\n-"
: webObserveTable(
@@ -428,6 +674,9 @@ export function withNodeRuntimeControlPlaneStatusRendered(result: Record<string,
].join(" ");
const failedTaskRuns = Array.isArray(pipelineRunDiagnostics.failedTaskRuns) ? pipelineRunDiagnostics.failedTaskRuns.map(compactNodeRuntimeTaskRunDiagnostic).filter(Boolean) : [];
const pendingTaskRuns = Array.isArray(pipelineRunDiagnostics.pendingTaskRuns) ? pipelineRunDiagnostics.pendingTaskRuns.map(compactNodeRuntimeTaskRunDiagnostic).filter(Boolean) : [];
const pipelineNext = record(pipelineRunDiagnostics.next);
const pendingTaskRunCommand = typeof pipelineNext.pendingTaskRun === "string" ? pipelineNext.pendingTaskRun : null;
const pendingPodCommand = typeof pipelineNext.pendingPod === "string" ? pipelineNext.pendingPod : null;
const pipelineDetail = [
webObserveShort(webObserveText(pipelineRun.reason ?? pipelineRun.message), 50),
...(failedTaskRuns.length > 0 ? [`failed=${webObserveShort(failedTaskRuns.join(","), 80)}`] : []),
@@ -453,12 +702,18 @@ export function withNodeRuntimeControlPlaneStatusRendered(result: Record<string,
["STAGE", "STATUS", "DETAIL"],
[
["pipeline", pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", pipelineDetail],
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)}`],
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)} target=${shortValue(argo.targetGitopsRevision)}`],
["runtime", runtime.ready === true ? "ok" : "failed", runtimeDetail],
["public", publicProbe.ready === true ? "ok" : "failed", webObserveShort(webObserveText(diagnostic.kind ?? diagnostic.message), 80)],
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `snapshot=${webObserveText(gitMirror.sourceSnapshotReady)} pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
],
),
...(pendingTaskRunCommand === null && pendingPodCommand === null ? [] : [
"",
"DRILL_DOWN",
...(pendingTaskRunCommand === null ? [] : [` pending TaskRun: ${pendingTaskRunCommand}`]),
...(pendingPodCommand === null ? [] : [` pending Pod: ${pendingPodCommand}`]),
]),
"",
webObserveTable(
["LOCAL_SOURCE", "GITHUB_SOURCE", "SNAPSHOT", "LOCAL_GITOPS", "GITHUB_GITOPS"],
@@ -492,6 +747,10 @@ export function withNodeRuntimeControlPlaneStatusFullRendered(result: Record<str
const secrets = record(runtime.externalPostgresSecrets);
const next = record(summary.next);
const workloadReadiness = Array.isArray(runtime.workloadReadiness) ? runtime.workloadReadiness.map(record) : [];
const pipelineDiagnostics = record(record(summary.pipelineRun).diagnostics);
const pipelineNext = record(pipelineDiagnostics.next);
const pendingTaskRunCommand = typeof pipelineNext.pendingTaskRun === "string" ? pipelineNext.pendingTaskRun : null;
const pendingPodCommand = typeof pipelineNext.pendingPod === "string" ? pipelineNext.pendingPod : null;
const workloadRows = workloadReadiness.length === 0
? [["-", "-", "-", "-", "-"]]
: workloadReadiness.slice(0, 18).map((item) => [
@@ -531,12 +790,18 @@ export function withNodeRuntimeControlPlaneStatusFullRendered(result: Record<str
["STAGE", "STATUS", "DETAIL"],
[
["pipeline", pipelineRun.ready === true ? "ok" : pipelineRun.exists === true ? webObserveText(pipelineRun.status) : "missing", webObserveShort(webObserveText(pipelineRun.reason ?? pipelineRun.message), 80)],
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)}`],
["argo", argo.ready === true ? "ok" : "failed", `${webObserveText(argo.syncStatus)}/${webObserveText(argo.health)} rev=${shortValue(argo.syncRevision)} target=${shortValue(argo.targetGitopsRevision)}`],
["runtime", runtimeSummary.ready === true ? "ok" : "failed", `namespace=${webObserveText(runtimeSummary.namespace)} workloads=${webObserveText(runtimeSummary.workloadReady)} pg=${webObserveText(runtimeSummary.externalPostgresReady ?? runtimeSummary.localPostgresReady)}`],
["public", publicProbe.ready === true ? "ok" : "failed", webObserveShort(webObserveText(record(publicProbe.diagnostic).kind ?? record(publicProbe.diagnostic).message), 80)],
["git-mirror", gitMirror.ready === true ? "ok" : "failed", `pending=${webObserveText(gitMirror.pendingFlush)} inSync=${webObserveText(gitMirror.githubInSync)}`],
],
),
...(pendingTaskRunCommand === null && pendingPodCommand === null ? [] : [
"",
"PIPELINE_DRILL_DOWN",
...(pendingTaskRunCommand === null ? [] : [` pending TaskRun: ${pendingTaskRunCommand}`]),
...(pendingPodCommand === null ? [] : [` pending Pod: ${pendingPodCommand}`]),
]),
"",
"RUNTIME_WORKLOADS",
webObserveTable(["REF", "READY", "READY_REPLICAS", "CURRENT", "DESIRED"], workloadRows),
scripts/src/hwlab-node/render.ts
+154 -21
View File
@@ -169,33 +169,34 @@ export function nodeRuntimeGitMirrorGithubTransportSummary(mirror: NodeRuntimeGi
export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const spec = scoped.spec;
const probeTimeoutSeconds = Math.max(1, Math.min(60, scoped.timeoutSeconds));
const sourceCommitOverride = optionValue(scoped.originalArgs, "--source-commit");
const pipelineRunOverride = optionValue(scoped.originalArgs, "--pipeline-run");
const head = sourceCommitOverride === undefined ? resolveNodeRuntimeLaneHead(spec) : null;
const sourceCommit = sourceCommitOverride ?? head?.sourceCommit ?? null;
const pipelineRun = pipelineRunOverride ?? (sourceCommit === null ? null : nodeRuntimePipelineRunName(spec, sourceCommit));
const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], 60);
const namespace = runNodeK3sArgs(spec, ["kubectl", "get", "ns", spec.runtimeNamespace, "-o", "name"], probeTimeoutSeconds);
const namespaceExists = namespace.exitCode === 0;
const postgresObjects = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], 60)
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "statefulset,svc,pvc", "-o", "name"], probeTimeoutSeconds)
: null;
const localPostgresObjects = postgresObjects === null
? []
: postgresObjects.stdout.split(/\r?\n/u).map((line) => line.trim()).filter((line) => isLocalPostgresObject(line, spec));
const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], 60);
const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], 60);
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], 60);
const serviceAccount = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "serviceaccount", spec.serviceAccountName, "-o", "name"], probeTimeoutSeconds);
const pipeline = runNodeK3sArgs(spec, ["kubectl", "-n", "hwlab-ci", "get", "pipeline", spec.pipeline, "-o", "name"], probeTimeoutSeconds);
const argo = runNodeK3sArgs(spec, ["kubectl", "-n", "argocd", "get", "application", spec.app, "-o", "jsonpath={.spec.source.repoURL}{\"\\n\"}{.spec.source.targetRevision}{\"\\n\"}{.spec.source.path}{\"\\n\"}{.status.sync.revision}{\"\\n\"}{.status.sync.status}{\"\\n\"}{.status.health.status}{\"\\n\"}"], probeTimeoutSeconds);
const [repoURL = "", targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = argo.stdout.split(/\r?\n/u);
const pipelineRunProbe = pipelineRun === null ? null : getNodeRuntimePipelineRun(spec, pipelineRun);
const pipelineRunDiagnostics = pipelineRun !== null && pipelineRunProbe?.exists === true && pipelineRunProbe?.status !== "True"
? nodeRuntimePipelineRunDiagnostics(spec, pipelineRun)
: null;
const workloads = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], 60)
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset,svc,ingress,configmap", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "name"], probeTimeoutSeconds)
: null;
const workloadNames = workloads === null ? [] : workloads.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
const workloadReadinessProbe = namespaceExists
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], 60)
? runNodeK3sArgs(spec, ["kubectl", "-n", spec.runtimeNamespace, "get", "deploy,statefulset", "-l", `hwlab.pikastech.local/gitops-target=${spec.lane}`, "-o", "jsonpath={range .items[*]}{.kind}{\"/\"}{.metadata.name}{\"\\t\"}{.status.readyReplicas}{\"/\"}{.status.replicas}{\"/\"}{.spec.replicas}{\"\\n\"}{end}"], probeTimeoutSeconds)
: null;
const workloadReadiness = parseNodeRuntimeWorkloadReadiness(workloadReadinessProbe?.stdout ?? "");
const bridge = externalPostgresBridgeStatus(spec, namespaceExists);
@@ -228,7 +229,9 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
const pipelineRunReady = pipelineRunProbe !== null && pipelineRunProbe.status === "True";
const pipelineRunDegradedReason = typeof pipelineRunDiagnostics?.degradedReason === "string"
? pipelineRunDiagnostics.degradedReason
: "pipelinerun-not-succeeded";
: pipelineRunProbe === null || pipelineRunProbe.exists !== true
? "pipelinerun-not-found"
: "pipelinerun-not-succeeded";
const publicReady = publicProbes.ready === true;
const gitMirrorReady = gitMirror.ok === true && gitMirrorCompact.pendingFlush === false && gitMirrorCompact.githubInSync === true;
const gitMirrorDegradedReason = gitMirrorCompact.sourceSnapshotReady === false
@@ -236,6 +239,34 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
: gitMirrorCompact.pendingFlush === true
? "git-mirror-pending-flush"
: "git-mirror-not-in-sync";
const targetGitopsRevision = nodeRuntimeTargetGitopsRevision(gitMirrorCompact);
const argoDegradedReason = nodeRuntimeArgoDegradedReason({
argoCommandOk: argo.exitCode === 0,
repoURL,
expectedRepoURL: spec.argoRepoUrl,
targetRevision,
expectedTargetRevision: spec.gitopsBranch,
path,
expectedPath: spec.runtimePath,
syncRevision,
syncStatus,
health,
targetGitopsRevision,
runtimeReady,
publicReady,
});
const degradedReason = nodeRuntimeStatusDegradedReason({
controlPlaneReady,
pipelineRunReady,
pipelineRunDegradedReason,
gitMirrorReady,
gitMirrorDegradedReason,
argoReady,
argoDegradedReason,
runtimeReady,
runtimeDegradedReason,
publicReady,
});
const fullStatus = {
ok: controlPlaneReady && runtimeReady && argoReady && pipelineRunReady && publicReady && gitMirrorReady,
command: `hwlab nodes control-plane status --node ${scoped.node} --lane ${scoped.lane}`,
@@ -262,6 +293,7 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
targetRevision,
path,
syncRevision,
targetGitopsRevision,
syncStatus,
health,
result: compactRuntimeCommand(argo),
@@ -296,17 +328,7 @@ export function nodeRuntimeControlPlaneStatus(scoped: ReturnType<typeof parseNod
namespace: compactRuntimeCommand(namespace),
postgresObjects: postgresObjects === null ? null : compactRuntimeCommand(postgresObjects),
},
degradedReason: controlPlaneReady
? runtimeReady
? argoReady
? pipelineRunReady
? publicReady
? gitMirrorReady ? undefined : gitMirrorDegradedReason
: "public-probe-not-ready"
: pipelineRunDegradedReason
: "argo-not-synced-healthy"
: runtimeDegradedReason
: "control-plane-not-ready",
degradedReason,
next: {
plan: `bun scripts/cli.ts hwlab nodes control-plane plan --node ${scoped.node} --lane ${scoped.lane}`,
apply: `bun scripts/cli.ts hwlab nodes control-plane apply --node ${scoped.node} --lane ${scoped.lane} --confirm`,
@@ -343,6 +365,62 @@ export function nullableInteger(value: string): number | null {
return Number(value);
}
export function nodeRuntimeTargetGitopsRevision(gitMirrorCompact: Record<string, unknown>): string | null {
return typeof gitMirrorCompact.localGitops === "string" && /^[0-9a-f]{40}$/iu.test(gitMirrorCompact.localGitops)
? gitMirrorCompact.localGitops
: typeof gitMirrorCompact.githubGitops === "string" && /^[0-9a-f]{40}$/iu.test(gitMirrorCompact.githubGitops)
? gitMirrorCompact.githubGitops
: null;
}
export function nodeRuntimeArgoDegradedReason(input: {
argoCommandOk: boolean;
repoURL: string;
expectedRepoURL: string;
targetRevision: string;
expectedTargetRevision: string;
path: string;
expectedPath: string;
syncRevision: string;
syncStatus: string;
health: string;
targetGitopsRevision: string | null;
runtimeReady: boolean;
publicReady: boolean;
}): string | null {
if (!input.argoCommandOk) return "argo-application-not-readable";
if (input.repoURL !== input.expectedRepoURL || input.targetRevision !== input.expectedTargetRevision || input.path !== input.expectedPath) {
return "argo-application-spec-drift";
}
const argoAtTarget = input.targetGitopsRevision !== null && input.syncRevision === input.targetGitopsRevision;
if (argoAtTarget && input.syncStatus === "Synced" && input.health !== "Healthy") return "argo-health-progressing";
if (argoAtTarget && input.syncStatus !== "Synced" && input.runtimeReady && input.publicReady) return "argo-health-progressing";
if (argoAtTarget) return "argo-target-revision-progressing";
if (input.targetGitopsRevision !== null) return "argo-revision-not-observed";
return "argo-not-synced-healthy";
}
export function nodeRuntimeStatusDegradedReason(input: {
controlPlaneReady: boolean;
pipelineRunReady: boolean;
pipelineRunDegradedReason: string;
gitMirrorReady: boolean;
gitMirrorDegradedReason: string;
argoReady: boolean;
argoDegradedReason: string | null;
runtimeReady: boolean;
runtimeDegradedReason: string;
publicReady: boolean;
}): string | undefined {
if (!input.controlPlaneReady) return "control-plane-not-ready";
if (!input.pipelineRunReady) return input.pipelineRunDegradedReason;
if (!input.gitMirrorReady) return input.gitMirrorDegradedReason;
if (!input.argoReady) return input.argoDegradedReason ?? "argo-not-synced-healthy";
if (!input.runtimeReady) return input.runtimeDegradedReason;
if (!input.publicReady) return "public-probe-not-ready";
return undefined;
}
export function nodeRuntimePublicProbeStatus(spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
const web = publicHttpProbe("web", spec.publicWebUrl);
const apiHealth = publicHttpProbe("apiHealth", joinUrlPath(spec.publicApiUrl, "/health/live"));
@@ -479,6 +557,42 @@ export function compactNodeRuntimeTaskRunDiagnostic(value: unknown): string {
return [left, reason ? `(${webObserveShort(reason, 36)})` : ""].filter(Boolean).join("");
}
export function nodeRuntimePipelinePendingTaskRunSummaries(
spec: HwlabRuntimeLaneSpec,
pendingTaskRuns: Array<Record<string, unknown>>,
pods: Array<Record<string, unknown>>,
): Array<Record<string, unknown>> {
return pendingTaskRuns.slice(0, 16).map((taskRun) => {
const taskRunName = stringOrNull(taskRun.name);
const podName = stringOrNull(taskRun.podName);
const pod = pods.find((item) => item.name === podName || (taskRunName !== null && item.taskRun === taskRunName)) ?? {};
const containers = Array.isArray(pod.containers) ? pod.containers.map(record) : [];
const initContainers = Array.isArray(pod.initContainers) ? pod.initContainers.map(record) : [];
const waitingContainers = [...initContainers, ...containers].filter((container) => container.state === "waiting");
const runningContainers = [...initContainers, ...containers].filter((container) => container.state === "running");
return {
name: taskRunName,
taskRun: taskRunName,
pipelineTask: taskRun.pipelineTask ?? null,
taskRef: taskRun.taskRef ?? null,
status: taskRun.status ?? null,
reason: taskRun.reason ?? null,
message: diagnosticText(taskRun.message),
pod: podName,
podPhase: pod.phase ?? null,
scheduled: pod.scheduled ?? null,
scheduledReason: pod.scheduledReason ?? null,
scheduledMessage: diagnosticText(pod.scheduledMessage),
waitingContainers,
runningContainers,
taskRunCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["get", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName, "-o", "yaml"]),
taskRunDescribeCommand: taskRunName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "taskrun", "-n", HWLAB_CI_NAMESPACE, taskRunName]),
podDescribeCommand: podName === null ? null : nodeRuntimeK3sCommand(spec, ["describe", "pod", "-n", HWLAB_CI_NAMESPACE, podName]),
podLogsCommand: podName === null ? null : nodeRuntimePipelineLogsCommand(spec, podName, null),
};
});
}
export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, unknown>, scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
const pipelineRun = record(status.pipelineRun);
const pipelineRunDiagnostics = record(status.pipelineRunDiagnostics);
@@ -531,6 +645,8 @@ export function summarizeNodeRuntimeControlPlaneStatus(status: Record<string, un
application: argo.application ?? null,
ready: argo.ready === true,
syncRevision: argo.syncRevision ?? null,
targetGitopsRevision: argo.targetGitopsRevision ?? null,
revisionObserved: typeof argo.targetGitopsRevision === "string" && argo.syncRevision === argo.targetGitopsRevision,
syncStatus: argo.syncStatus ?? null,
health: argo.health ?? null,
},
@@ -610,9 +726,17 @@ export function nodeRuntimeStatusNextAction(status: Record<string, unknown>, sco
if (reason === "argo-not-synced-healthy") {
return `bun scripts/cli.ts hwlab nodes control-plane refresh --node ${scoped.node} --lane ${scoped.lane} --confirm`;
}
if (reason === "argo-revision-not-observed" || reason === "argo-target-revision-progressing" || reason === "argo-health-progressing") {
return `${nodeRuntimeStatusCommand(scoped)} --full`;
}
if (reason === "pipelinerun-not-succeeded") {
return `bun scripts/cli.ts hwlab nodes control-plane trigger-current --node ${scoped.node} --lane ${scoped.lane} --confirm`;
}
if (reason === "node-runtime-ci-taskrun-pending") {
const next = record(record(status.pipelineRunDiagnostics).next);
const pendingTaskRun = typeof next.pendingTaskRun === "string" ? next.pendingTaskRun : null;
return pendingTaskRun ?? `${nodeRuntimeStatusCommand(scoped)} --full`;
}
if (reason === "node-runtime-ci-step-publish-failed") {
return `bun scripts/cli.ts platform-infra sub2api status --target ${scoped.node}`;
}
@@ -659,6 +783,7 @@ export function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pi
const pendingTaskRuns = taskRuns.filter((item) => item.status !== "True" && item.status !== "False");
const failedTaskRuns = taskRuns.filter((item) => item.status === "False");
const failedTaskRunSummaries = nodeRuntimePipelineFailedTaskRunSummaries(spec, failedTaskRuns, pods);
const pendingTaskRunSummaries = nodeRuntimePipelinePendingTaskRunSummaries(spec, pendingTaskRuns, pods);
const stepPublishFailures = failedTaskRunSummaries.filter((item) => item.container === "step-publish" || item.step === "publish" || item.step === "step-publish");
const unscheduledPods = pods.filter((item) => item.scheduled === false);
const schedulingMessages = unscheduledPods
@@ -690,7 +815,8 @@ export function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pi
failedTaskRuns: failedTaskRunSummaries,
stepPublishFailures,
failureSummary,
pendingTaskRuns,
pendingTaskRuns: pendingTaskRunSummaries,
pendingTaskRunCount: pendingTaskRunSummaries.length,
unscheduledPods,
schedulingMessages,
degradedReason: tooManyPods
@@ -723,7 +849,14 @@ export function nodeRuntimePipelineRunDiagnostics(spec: HwlabRuntimeLaneSpec, pi
failedTaskRun: failedTaskRunSummaries[0]?.taskRunCommand ?? null,
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
}
: undefined,
: pendingTaskRunSummaries.length > 0
? {
pendingTaskRun: pendingTaskRunSummaries[0]?.taskRunDescribeCommand ?? pendingTaskRunSummaries[0]?.taskRunCommand ?? null,
pendingPod: pendingTaskRunSummaries[0]?.podDescribeCommand ?? null,
pendingPodLogs: pendingTaskRunSummaries[0]?.podLogsCommand ?? null,
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
}
: undefined,
};
}
scripts/src/hwlab-node/status.ts
+112 -21
View File
@@ -38,6 +38,12 @@ import { webObserveTable } from "./web-observe-render";
import { nodeRuntimeGitMirrorTarget, printNodeRuntimeTriggerProgress, sleepSync } from "./web-probe";
import { webObserveShort, webObserveText } from "./web-probe-observe";
type NodeScopedDelegatedOptions = ReturnType<typeof parseNodeScopedDelegatedOptions>;
export type NodeRuntimeGitMirrorRunOptions = {
maxAttempts?: number;
deadlineMs?: number;
};
export function nodeRuntimeExternalPostgresSecretRows(secrets: Record<string, unknown>): unknown[][] {
if (secrets.required === false) return [["not-required", "-", "-", "-", "false"]];
if (secrets.required === true && secrets.ready !== true && secrets.degradedReason !== undefined) {
@@ -240,12 +246,12 @@ export function nodeRuntimeGitMirrorStatus(scoped: ReturnType<typeof parseNodeSc
};
}
export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
export function nodeRuntimeGitMirrorRun(scoped: NodeScopedDelegatedOptions, options: NodeRuntimeGitMirrorRunOptions = {}): Record<string, unknown> {
if (scoped.action !== "sync" && scoped.action !== "flush") return nodeRuntimeUnsupportedAction(scoped);
if (!scoped.confirm && !scoped.dryRun) throw new Error(`git-mirror ${scoped.action} requires --dry-run or --confirm`);
const spec = scoped.spec;
const mirror = nodeRuntimeGitMirrorTarget(spec);
const retryMaxAttempts = !scoped.dryRun && (scoped.action === "sync" || scoped.action === "flush") ? 5 : 1;
const retryMaxAttempts = options.maxAttempts ?? (!scoped.dryRun && (scoped.action === "sync" || scoped.action === "flush") ? 5 : 1);
const attempts: Record<string, unknown>[] = [];
let finalAttempt: {
attempt: number;
@@ -259,13 +265,15 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
} | null = null;
for (let attempt = 1; attempt <= retryMaxAttempts; attempt += 1) {
const attemptScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs, 5);
if (attemptScoped === null) break;
const retryLabel = `${attempt}/${retryMaxAttempts}`;
const jobName = nodeRuntimeGitMirrorJobName(mirror, scoped.action);
const manifest = nodeRuntimeGitMirrorJobManifest(mirror, scoped.action, jobName, {
discardStaleGitops: scoped.discardStaleGitops === true,
});
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
const waitTimeoutSeconds = Math.max(5, Math.min(45, Math.max(5, scoped.timeoutSeconds - 10)));
const waitTimeoutSeconds = Math.max(1, Math.min(45, Math.max(1, attemptScoped.timeoutSeconds - 10)));
const script = [
"set -eu",
`namespace=${shellQuote(mirror.namespace)}`,
@@ -291,7 +299,7 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
"kubectl logs -n \"$namespace\" \"job/$job\" --tail=200 || true",
].join("\n"),
].join("\n");
const result = runNodeK3sScript(spec, script, scoped.timeoutSeconds);
const result = runNodeK3sScript(spec, script, attemptScoped.timeoutSeconds);
const partialSuccess = nodeRuntimeGitMirrorFlushPartialSuccess(scoped, mirror, result);
const actionSucceeded = isCommandSuccess(result);
const retryableFailure = !actionSucceeded
@@ -317,6 +325,7 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
finalAttempt = { attempt, retryLabel, jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded };
if (actionSucceeded || scoped.dryRun || !retryable || exhausted) break;
const backoffMs = nodeRuntimeGitMirrorRetryDelayMs(attempt);
if (options.deadlineMs !== undefined && Date.now() + backoffMs >= options.deadlineMs) break;
printNodeRuntimeTriggerProgress(spec, {
stage: `git-mirror-${scoped.action}-retry`,
status: "waiting",
@@ -328,11 +337,14 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
sleepSync(backoffMs);
}
if (finalAttempt === null) throw new Error("git-mirror run produced no attempts");
if (finalAttempt === null) {
return nodeRuntimeGitMirrorBudgetExhausted(scoped);
}
const { jobName, manifest, result, partialSuccess, retryableFailure, actionSucceeded } = finalAttempt;
const status = scoped.dryRun || !actionSucceeded
const statusScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs, 5);
const status = scoped.dryRun || !actionSucceeded || statusScoped === null
? undefined
: nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
: nodeRuntimeGitMirrorStatus({ ...statusScoped, action: "status", dryRun: true, confirm: false });
const retryExhausted = retryableFailure?.retryable === true && attempts.length >= retryMaxAttempts;
const stopped = !actionSucceeded && (retryExhausted || retryableFailure?.stopped === true || retryableFailure === null);
return {
@@ -348,6 +360,7 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
manifest: scoped.dryRun ? manifest : undefined,
result: compactRuntimeCommand(result),
status,
statusSkippedReason: statusScoped === null && !scoped.dryRun && actionSucceeded ? "node-runtime-cicd-budget-exhausted" : undefined,
retry: retryMaxAttempts > 1 ? {
policy: "exponential-backoff",
maxAttempts: retryMaxAttempts,
@@ -370,6 +383,27 @@ export function nodeRuntimeGitMirrorRun(scoped: ReturnType<typeof parseNodeScope
};
}
function nodeRuntimeScopedWithinDeadline(scoped: NodeScopedDelegatedOptions, deadlineMs?: number, minRemainingSeconds = 5): NodeScopedDelegatedOptions | null {
if (deadlineMs === undefined) return scoped;
const remainingSeconds = Math.floor((deadlineMs - Date.now()) / 1000);
if (remainingSeconds < minRemainingSeconds) return null;
return { ...scoped, timeoutSeconds: Math.max(1, Math.min(scoped.timeoutSeconds, remainingSeconds)) };
}
function nodeRuntimeGitMirrorBudgetExhausted(scoped: NodeScopedDelegatedOptions): Record<string, unknown> {
return {
ok: false,
command: `hwlab nodes git-mirror ${scoped.action} --node ${scoped.node} --lane ${scoped.lane}`,
node: scoped.node,
lane: scoped.lane,
mode: `confirmed-${scoped.action}`,
mutation: false,
status: "pending",
degradedReason: "node-runtime-cicd-budget-exhausted",
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
export function nodeRuntimeGitMirrorRetryableFailure(
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
mirror: NodeRuntimeGitMirrorTargetSpec,
@@ -480,9 +514,20 @@ export function nodeRuntimeGitMirrorHostWriteUrl(spec: HwlabRuntimeLaneSpec, mir
return `http://${value}/${mirror.sourceRepository}.git`;
}
export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>, sourceCommit: string, pipelineRun: string): Record<string, unknown> {
export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: NodeScopedDelegatedOptions, sourceCommit: string, pipelineRun: string, options: NodeRuntimeGitMirrorRunOptions = {}): Record<string, unknown> {
const full = nodeScopedFullOutput(scoped);
const before = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
const beforeScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
if (beforeScoped === null) {
return {
ok: false,
mode: "budget-exhausted-before-status",
sourceCommit,
status: "pending",
degradedReason: "node-runtime-cicd-budget-exhausted",
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
const before = nodeRuntimeGitMirrorStatus({ ...beforeScoped, action: "status", dryRun: true, confirm: false });
const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
const beforeGithubTransport = record(before.githubTransport);
if (beforeGithubTransport.required === true && beforeGithubTransport.ready === false) {
@@ -501,7 +546,7 @@ export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeo
};
}
if (before.ok === true && beforeSummary.localSource === sourceCommit && beforeSummary.githubSource === sourceCommit) {
const flush = nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, before);
const flush = nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, before, options);
return {
ok: flush.ok === true,
mode: "already-current",
@@ -512,19 +557,32 @@ export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeo
degradedReason: flush.ok === true ? undefined : "node-runtime-git-mirror-pre-flush-failed",
};
}
const syncScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
if (syncScoped === null) {
return {
ok: false,
mode: "budget-exhausted-before-sync",
sourceCommit,
beforeSummary,
before: full ? before : undefined,
status: "pending",
degradedReason: "node-runtime-cicd-budget-exhausted",
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
const sync = nodeRuntimeGitMirrorRun({
...scoped,
...syncScoped,
domain: "git-mirror",
action: "sync",
confirm: true,
dryRun: false,
wait: true,
discardStaleGitops: scoped.discardStaleGitops === true || scoped.rerun === true,
});
}, options);
const after = record(sync.status);
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
const sourceOk = sync.ok === true && afterSummary.localSource === sourceCommit && afterSummary.githubSource === sourceCommit;
const flush = sourceOk ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, after) : null;
const flush = sourceOk ? nodeRuntimeEnsureGitMirrorFlushed(scoped, "pre", sourceCommit, pipelineRun, after, options) : null;
const ok = sourceOk && (flush === null || flush.ok === true);
return {
ok,
@@ -545,15 +603,28 @@ export function nodeRuntimeEnsureGitMirrorSourceCurrent(scoped: ReturnType<typeo
}
export function nodeRuntimeEnsureGitMirrorFlushed(
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
scoped: NodeScopedDelegatedOptions,
phase: "pre" | "post" | "parallel",
sourceCommit: string,
pipelineRun: string | null,
statusInput: Record<string, unknown> | null = null,
options: NodeRuntimeGitMirrorRunOptions = {},
): Record<string, unknown> {
const stage = `git-mirror-${phase}-flush`;
const full = nodeScopedFullOutput(scoped);
const before = statusInput ?? nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
const beforeScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
if (beforeScoped === null) {
return {
ok: false,
phase,
mode: "budget-exhausted-before-status",
executed: false,
status: "pending",
degradedReason: "node-runtime-cicd-budget-exhausted",
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
const before = statusInput ?? nodeRuntimeGitMirrorStatus({ ...beforeScoped, action: "status", dryRun: true, confirm: false });
const beforeSummary = compactNodeRuntimeGitMirrorStatus(before);
if (before.ok !== true) {
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "failed", sourceCommit, pipelineRun, reason: "git-mirror-status-failed", ...beforeSummary });
@@ -583,7 +654,21 @@ export function nodeRuntimeEnsureGitMirrorFlushed(
};
}
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun, flushNeeded: true, ...beforeSummary });
const flush = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "flush", confirm: true, dryRun: false, wait: true });
const flushScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
if (flushScoped === null) {
return {
ok: false,
phase,
mode: "budget-exhausted-before-flush",
executed: false,
before: full ? before : undefined,
beforeSummary,
status: "pending",
degradedReason: "node-runtime-cicd-budget-exhausted",
next: { status: `bun scripts/cli.ts hwlab nodes git-mirror status --node ${scoped.node} --lane ${scoped.lane}` },
};
}
const flush = nodeRuntimeGitMirrorRun({ ...flushScoped, domain: "git-mirror", action: "flush", confirm: true, dryRun: false, wait: true }, options);
const after = record(flush.status);
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
const ok = flush.ok === true && Object.keys(after).length > 0 && !nodeRuntimeGitMirrorNeedsFlush(after);
@@ -613,14 +698,17 @@ export function nodeRuntimeEnsureGitMirrorFlushed(
}
export function nodeRuntimeOpportunisticGitMirrorSync(
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
scoped: NodeScopedDelegatedOptions,
sourceCommit: string,
pipelineRun: string,
options: NodeRuntimeGitMirrorRunOptions = {},
): Record<string, unknown> | null {
const stage = "git-mirror-parallel-sync";
const syncScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
if (syncScoped === null) return null;
const full = nodeScopedFullOutput(scoped);
printNodeRuntimeTriggerProgress(scoped.spec, { stage, status: "started", sourceCommit, pipelineRun });
const sync = nodeRuntimeGitMirrorRun({ ...scoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true });
const sync = nodeRuntimeGitMirrorRun({ ...syncScoped, domain: "git-mirror", action: "sync", confirm: true, dryRun: false, wait: true }, options);
const after = record(sync.status);
const afterSummary = Object.keys(after).length > 0 ? compactNodeRuntimeGitMirrorStatus(after) : {};
const ok = sync.ok === true;
@@ -649,13 +737,16 @@ export function nodeRuntimeOpportunisticGitMirrorSync(
}
export function nodeRuntimeOpportunisticGitMirrorFlush(
scoped: ReturnType<typeof parseNodeScopedDelegatedOptions>,
scoped: NodeScopedDelegatedOptions,
sourceCommit: string,
pipelineRun: string,
options: NodeRuntimeGitMirrorRunOptions = {},
): Record<string, unknown> | null {
const status = nodeRuntimeGitMirrorStatus({ ...scoped, action: "status", dryRun: true, confirm: false });
const statusScoped = nodeRuntimeScopedWithinDeadline(scoped, options.deadlineMs);
if (statusScoped === null) return null;
const status = nodeRuntimeGitMirrorStatus({ ...statusScoped, action: "status", dryRun: true, confirm: false });
if (status.ok !== true || !nodeRuntimeGitMirrorNeedsFlush(status)) return null;
return nodeRuntimeEnsureGitMirrorFlushed(scoped, "parallel", sourceCommit, pipelineRun, status);
return nodeRuntimeEnsureGitMirrorFlushed(scoped, "parallel", sourceCommit, pipelineRun, status, options);
}
export function nodeRuntimeGitMirrorNeedsFlush(status: Record<string, unknown>): boolean {
scripts/src/hwlab-node/web-probe.ts
+18 -3
View File
@@ -348,8 +348,9 @@ export function waitForNodeRuntimePipelineRunTerminal(
options: { opportunisticPostFlush?: () => Record<string, unknown> | null; opportunisticPostSync?: () => Record<string, unknown> | null } = {},
): Record<string, unknown> {
const severeWarningThresholdMs = 120_000;
const effectiveTimeoutSeconds = Math.max(0, timeoutSeconds);
const startedAt = Date.now();
const deadline = startedAt + timeoutSeconds * 1000;
const deadline = startedAt + effectiveTimeoutSeconds * 1000;
let polls = 0;
let last: Record<string, unknown> = { exists: false, name: pipelineRun };
let lastOpportunisticPostFlushAt = 0;
@@ -357,7 +358,21 @@ export function waitForNodeRuntimePipelineRunTerminal(
let opportunisticPostSyncAttempted = false;
const opportunisticPostFlushes: Record<string, unknown>[] = [];
const opportunisticPostSyncs: Record<string, unknown>[] = [];
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds });
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "started", pipelineRun, timeoutSeconds: effectiveTimeoutSeconds });
if (effectiveTimeoutSeconds <= 0) {
const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs: 0 });
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs: 0, waitLimitSeconds: effectiveTimeoutSeconds });
return {
ok: false,
status: "pending",
pipelineRun: last,
polls,
elapsedMs: 0,
waitLimitSeconds: effectiveTimeoutSeconds,
warning,
next: { status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun}` },
};
}
while (Date.now() <= deadline) {
polls += 1;
last = getNodeRuntimePipelineRun(spec, pipelineRun);
@@ -445,7 +460,7 @@ export function waitForNodeRuntimePipelineRunTerminal(
const warning = nodeRuntimeCicdWaitWarning(spec, pipelineRun, { polls, elapsedMs });
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "warning", pipelineRun, polls, elapsedMs, waitLimitSeconds: timeoutSeconds });
return {
ok: true,
ok: false,
status: "pending",
pipelineRun: last,
polls,