fix(hwlab): warn on slow node pipelinerun waits

scripts/src/hwlab-node-impl.ts

@@ -6173,11 +6173,13 @@ function waitForNodeRuntimePipelineRunTerminal(
   timeoutSeconds: number,
   options: { opportunisticPostFlush?: () => Record<string, unknown> | null; opportunisticPostSync?: () => Record<string, unknown> | null } = {},
 ): Record<string, unknown> {
+  const severeWarningThresholdMs = 120_000;
   const startedAt = Date.now();
   const deadline = startedAt + timeoutSeconds * 1000;
   let polls = 0;
   let last: Record<string, unknown> = { exists: false, name: pipelineRun };
   let lastOpportunisticPostFlushAt = 0;
+  let severeWarningEmitted = false;
   let opportunisticPostSyncAttempted = false;
   const opportunisticPostFlushes: Record<string, unknown>[] = [];
   const opportunisticPostSyncs: Record<string, unknown>[] = [];
@@ -6187,7 +6189,27 @@ function waitForNodeRuntimePipelineRunTerminal(
     last = getNodeRuntimePipelineRun(spec, pipelineRun);
     const status = typeof last.status === "string" ? last.status : null;
     const reason = typeof last.reason === "string" ? last.reason : null;
-    printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs: Date.now() - startedAt });
+    const elapsedMs = Date.now() - startedAt;
+    printNodeRuntimeTriggerProgress(spec, { stage: "pipelinerun-wait", status: "poll", pipelineRun, pipelineStatus: status, reason, polls, elapsedMs });
+    if (!severeWarningEmitted && elapsedMs >= severeWarningThresholdMs && status !== "True" && status !== "False") {
+      severeWarningEmitted = true;
+      printNodeRuntimeTriggerProgress(spec, {
+        stage: "pipelinerun-wait",
+        status: "warning",
+        warning: "pipelinerun-wait-severe-timeout",
+        severity: "warning",
+        pipelineRun,
+        pipelineStatus: status,
+        reason,
+        polls,
+        elapsedMs,
+        thresholdMs: severeWarningThresholdMs,
+        message: "PipelineRun has been non-terminal for more than 120s; investigate Tekton taskRuns/pods instead of treating this as normal wait time.",
+        next: {
+          status: `bun scripts/cli.ts hwlab nodes control-plane status --node ${spec.nodeId} --lane ${spec.lane} --pipeline-run ${pipelineRun} --full`,
+        },
+      });
+    }
     if (status === "True" || status === "False") {
       const ok = status === "True";
       const diagnostics = ok ? null : nodeRuntimePipelineRunDiagnostics(spec, pipelineRun);