fix: handle stale web observe runners (#875)

Co-authored-by: Codex <codex@noreply.local>

scripts/src/hwlab-node-web-observe-render.ts

@@ -33,6 +33,8 @@ function renderWebObserveStatusTable(value: Record<string, unknown>): string {
   const observer = record(value.observer);
   const manifest = record(observer?.manifest);
   const heartbeat = record(observer?.heartbeat);
+  const diagnostics = record(observer?.diagnostics) ?? record(value.diagnostics);
+  const commands = record(observer?.commands);
   const tails = record(observer?.tails);
   const result = record(value.result);
   const samples = Array.isArray(tails?.samples) ? tails.samples.map(record).filter((item): item is Record<string, unknown> => item !== null).slice(-5) : [];
@@ -55,20 +57,38 @@ function renderWebObserveStatusTable(value: Record<string, unknown>): string {
   const manifestNetwork = record(manifest?.network);
   const manifestBrowser = record(manifestNetwork?.browser);
   const manifestProxy = record(manifestNetwork?.proxy);
+  const heartbeatAge = diagnostics?.heartbeatAgeSeconds ?? heartbeat?.ageSeconds;
+  const heartbeatStale = diagnostics?.heartbeatStale ?? heartbeat?.stale;
+  const heartbeatLiveness = diagnostics?.effectiveLiveness ?? heartbeat?.effectiveLiveness;
   const lines = [
     `hwlab nodes web-probe observe status (${webObserveText(value.status)})`,
     "",
-    webObserveTable(["ID", "NODE", "LANE", "ALIVE", "PID", "SAMPLE", "UPDATED", "TARGET"], [[
+    webObserveTable(["ID", "NODE", "LANE", "LIVENESS", "ALIVE", "PID", "SAMPLE", "CMD_SEQ", "HB_AGE_S", "STALE", "UPDATED", "TARGET"], [[
       value.id,
       value.node,
       value.lane,
+      heartbeatLiveness,
       observer?.processAlive,
       observer?.pid,
       heartbeat?.sampleSeq,
+      heartbeat?.commandSeq,
+      heartbeatAge,
+      heartbeatStale,
       webObserveShort(webObserveText(heartbeat?.updatedAt ?? heartbeat?.lastSampleAt), 24),
       webObserveShort(`${webObserveText(manifest?.baseUrl)}${webObserveText(manifest?.targetPath) === "-" ? "" : webObserveText(manifest?.targetPath)}`, 52),
     ]]),
     "",
+    "Runner diagnostics:",
+    webObserveTable(["HEARTBEAT_STALE_AFTER_S", "COMMAND_BACKLOG", "PENDING", "PROCESSING", "ABANDONED", "FAILED", "OLDEST_PENDING_S"], [[
+      diagnostics?.heartbeatStaleAfterSeconds,
+      diagnostics?.commandBacklog,
+      commands?.pendingCount,
+      commands?.processingCount,
+      commands?.abandonedCount,
+      commands?.failedCount,
+      diagnostics?.oldestPendingAgeSeconds,
+    ]]),
+    "",
     ...(value.ok === false ? [
       "Blocked detail:",
       webObserveTable(["REASON", "EXIT", "TIMEOUT", "STDOUT", "STDERR"], [[
@@ -153,6 +173,8 @@ function renderWebObserveStatusTable(value: Record<string, unknown>): string {
     const command = webObserveText(next?.[key]);
     if (command !== "-") lines.push(`  ${command}`);
   }
+  const forceStop = webObserveText(next?.forceStop);
+  if (forceStop !== "-") lines.push(`  ${forceStop}`);
   lines.push("", "Disclosure:");
   lines.push("  default view is a bounded table; use observe collect/analyze for artifact-level drill-down.");
   return lines.join("\n");