mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(hwlab): warn on slow node pipelinerun waits

Browse Source
This commit is contained in:
Codex
2026-06-24 13:28:09 +00:00
parent 93ec87cdd1
commit f490a9906f
1 changed files with 23 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-impl.ts
+23 -1
View File
@@ -6173,11 +6173,13 @@ function waitForNodeRuntimePipelineRunTerminal(
timeoutSeconds: number, timeoutSeconds: number,
options: { opportunisticPostFlush?: () => Record<string, unknown> | null; opportunisticPostSync?: () => Record<string, unknown> | null } = {}, options: { opportunisticPostFlush?: () => Record<string, unknown> | null; opportunisticPostSync?: () => Record<string, unknown> | null } = {},
): Record<string, unknown> { ): Record<string, unknown> {
const severeWarningThresholdMs = 120_000;
const startedAt = Date.now(); const startedAt = Date.now();
const deadline = startedAt + timeoutSeconds * 1000; const deadline = startedAt + timeoutSeconds * 1000;
let polls = 0; let polls = 0;
let last: Record<string, unknown> = { exists: false, name: pipelineRun }; let last: Record<string, unknown> = { exists: false, name: pipelineRun };
let lastOpportunisticPostFlushAt = 0; let lastOpportunisticPostFlushAt = 0;
let severeWarningEmitted = false;
let opportunisticPostSyncAttempted = false; let opportunisticPostSyncAttempted = false;
const opportunisticPostFlushes: Record<string, unknown>[] = []; const opportunisticPostFlushes: Record<string, unknown>[] = [];
const opportunisticPostSyncs: Record<string, unknown>[] = []; const opportunisticPostSyncs: Record<string, unknown>[] = [];
@@ -6187,7 +6189,27 @@ function waitForNodeRuntimePipelineRunTerminal(
last = getNodeRuntimePipelineRun(spec, pipelineRun); last = getNodeRuntimePipelineRun(spec, pipelineRun);
const status = typeof last.status === "string" ? last.status : null; const status = typeof last.status === "string" ? last.status : null;
const reason = typeof last.reason === "string" ? last.reason : null; const reason = typeof last.reason === "string" ? last.reason : null;
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt }); const elapsedMs = Date.now() - startedAt;
printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs });
if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") {
severeWarningEmitted = true;
printNodeRuntimeTriggerProgress(spec, {
stage: "pipelinerun-wait",
status: "warning",
warning: "pipelinerun-wait-severe-timeout",
severity: "warning",
pipelineRun,
pipelineStatus: status,
reason,
polls,
elapsedMs,
thresholdMs: severeWarningThresholdMs,
message: "PipelineRun has been non-terminal for more than 120s; investigate Tekton taskRuns/pods instead of treating this as normal wait time.",
next: {
status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
},
});
}
if (status === "True" || status === "False") { if (status === "True" || status === "False") {
const ok = status === "True"; const ok = status === "True";
const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun); const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);