fix: improve workbench triad diagnostics

2026-07-01 16:15:50 +00:00
parent 113b6809d1
commit 624736f336
4 changed files with 385 additions and 33 deletions
@@ -0,0 +1,158 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readFile, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+import { test } from "bun:test";
+
+import { nodeWebObserveAnalyzerSource } from "../hwlab-node-web-observe-analyzer-source";
+
+const alertThresholds = {
+  sameOriginApiSlowMs: 60000,
+  partialApiSlowMs: 60000,
+  longLivedStreamOpenSlowMs: 60000,
+  visibleLoadingSlowMs: 60000,
+  turnTimingSampleSlackSeconds: 60,
+  turnElapsedSevereTimeoutSeconds: 3600,
+  domEvaluateTimeoutRedCount: 99,
+  domEvaluateTimeoutRedWindowMs: 60000,
+  screenshotTimeoutRedCount: 99,
+  pageErrorRedCount: 99,
+  browserProcessSampleIntervalMs: 1000,
+  browserTotalRssRedMb: 999999,
+  browserProcessRssRedMb: 999999,
+  browserRssGrowthRedMb: 999999,
+  browserRssGrowthWindowMs: 60000,
+  playwrightResponsivenessRedMs: 60000,
+  playwrightResponsivenessTimeoutRedCount: 99,
+  cdpMetricsTimeoutRedCount: 99,
+  uncommandedStateChangeCommandWindowMs: 1000,
+  scrollJumpCommandWindowMs: 1000,
+  scrollJumpFromY: 999999,
+  scrollJumpToY: 999999,
+  sessionRailFallbackRatio: 0.5,
+};
+
+const browserFreezePolicy = {
+  enabled: true,
+  blockerWindowMs: 60000,
+  memory: {
+    totalRssBlockerMb: 999999,
+    processRssBlockerMb: 999999,
+    growthBlockerMb: 999999,
+  },
+  responsiveness: {
+    latencyBlockerMs: 60000,
+    eventBlockerCount: 99,
+  },
+  cdp: {
+    metricsTimeoutBlockerCount: 99,
+  },
+  kill: {
+    enabled: false,
+    gracefulSignal: "SIGTERM",
+    forceSignal: "SIGKILL",
+    graceMs: 1000,
+    pollIntervalMs: 100,
+    exitCode: 124,
+  },
+};
+
+test("observe analyzer classifies stale completed session rail when a newer turn is running", async () => {
+  const stateDir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-analyzer-"));
+  const analyzerPath = join(stateDir, "analyze.mjs");
+  const samplesPath = join(stateDir, "samples.jsonl");
+  await writeFile(analyzerPath, nodeWebObserveAnalyzerSource(), { mode: 0o700 });
+  await writeFile(samplesPath, [
+    JSON.stringify({
+      seq: 1,
+      ts: "2026-07-01T15:21:24.767Z",
+      path: "/workbench/sessions/ses_triage",
+      url: "https://hwlab.example.test/workbench/sessions/ses_triage",
+      pageRole: "control",
+      pageId: "control-test",
+      routeSessionId: "ses_triage",
+      activeSessionId: "ses_triage",
+      sessionRail: {
+        items: [{
+          index: 0,
+          active: true,
+          status: "completed",
+          dataStatus: "completed",
+          running: false,
+          dataRunning: "false",
+          sessionId: "ses_triage",
+          sessionIdPrefix: "ses_triage",
+        }],
+      },
+      turns: [
+        {
+          role: "agent",
+          status: "completed",
+          traceId: "trc_previous_completed",
+          messageId: "msg_previous_completed_agent",
+          finalResponsePresent: true,
+          finalResponseTextBytes: 12,
+        },
+        {
+          role: "agent",
+          status: "running",
+          traceId: "trc_running_new_turn",
+          messageId: "msg_running_new_turn_agent",
+          finalResponsePresent: false,
+          finalResponseTextBytes: 0,
+        },
+      ],
+    }),
+    JSON.stringify({
+      seq: 2,
+      ts: "2026-07-01T15:22:49.918Z",
+      path: "/workbench/sessions/ses_triage",
+      url: "https://hwlab.example.test/workbench/sessions/ses_triage",
+      pageRole: "control",
+      pageId: "control-test",
+      routeSessionId: "ses_triage",
+      activeSessionId: "ses_triage",
+      sessionRail: {
+        items: [{
+          index: 0,
+          active: true,
+          status: "canceled",
+          dataStatus: "canceled",
+          running: false,
+          dataRunning: "false",
+          sessionId: "ses_triage",
+          sessionIdPrefix: "ses_triage",
+        }],
+      },
+      turns: [{
+        role: "agent",
+        status: "canceled",
+        traceId: "trc_canceled_terminal",
+        messageId: "msg_canceled_terminal_agent",
+        finalResponsePresent: true,
+        finalResponseTextBytes: 17,
+      }],
+    }),
+  ].join("\n") + "\n");
+
+  const result = spawnSync("bun", [analyzerPath, stateDir], {
+    cwd: join(import.meta.dir, "../../.."),
+    env: {
+      ...process.env,
+      UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES: "0",
+      UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON: JSON.stringify(alertThresholds),
+      UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON: JSON.stringify(browserFreezePolicy),
+    },
+    encoding: "utf8",
+  });
+  assert.equal(result.status, 0, result.stderr || result.stdout);
+
+  const report = JSON.parse(await readFile(join(stateDir, "analysis", "report.json"), "utf8"));
+  const finding = report.findings.find((item: Record<string, unknown>) => item.id === "workbench-turn-state-triad-inconsistent");
+  assert.equal(finding?.rootCause, "workbench_session_rail_status_stale_after_new_running_turn");
+  assert.equal(finding?.dominantMismatchKind, "rail-card-status-mismatch");
+  assert.match(String(finding?.summary), /previous completed terminal status/u);
+  assert.equal(report.sampleMetrics.workbenchTurnStateTriad.summary.invalidRowCount, 1);
+  assert.equal(report.sampleMetrics.workbenchTurnStateTriad.summary.cardFinalResponseMismatchCount, 0);
+}, 20_000);