diff --git a/scripts/src/hwlab-node-impl.ts b/scripts/src/hwlab-node-impl.ts index 59a0f28f..dc9dad77 100644 --- a/scripts/src/hwlab-node-impl.ts +++ b/scripts/src/hwlab-node-impl.ts @@ -6173,11 +6173,13 @@ function waitForNodeRuntimePipelineRunTerminal( timeoutSeconds: number, options: { opportunisticPostFlush?: () => Record | null; opportunisticPostSync?: () => Record | null } = {}, ): Record { + const severeWarningThresholdMs = 120_000; const startedAt = Date.now(); const deadline = startedAt + timeoutSeconds * 1000; let polls = 0; let last: Record = { exists: false, name: pipelineRun }; let lastOpportunisticPostFlushAt = 0; + let severeWarningEmitted = false; let opportunisticPostSyncAttempted = false; const opportunisticPostFlushes: Record[] = []; const opportunisticPostSyncs: Record[] = []; @@ -6187,7 +6189,27 @@ function waitForNodeRuntimePipelineRunTerminal( last = getNodeRuntimePipelineRun(spec, pipelineRun); const status = typeof last.status === "string" ? last.status : null; const reason = typeof last.reason === "string" ? last.reason : null; - printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt }); + const elapsedMs = Date.now() - startedAt; + printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs }); + if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") { + severeWarningEmitted = true; + printNodeRuntimeTriggerProgress(spec, { + stage: "pipelinerun-wait", + status: "warning", + warning: "pipelinerun-wait-severe-timeout", + severity: "warning", + pipelineRun, + pipelineStatus: status, + reason, + polls, + elapsedMs, + thresholdMs: severeWarningThresholdMs, + message: "PipelineRun has been non-terminal for more than 120s; investigate Tekton taskRuns/pods instead of treating this as normal wait time.", + next: { + status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`, + }, + }); + } if (status === "True" || status === "False") { const ok = status === "True"; const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);