fix: detect workbench in-place projection stalls

2026-06-27 08:08:12 +00:00
parent 35cb49f319
commit 2a3d0a4657
2 changed files with 419 additions and 3 deletions
@@ -2223,6 +2223,218 @@ function compactApiDomLagForOutput(report) {
  };
 }

+function detectWorkbenchInPlaceProjectionLag(samples, network) {
+  const terminalTraceMissing = detectWorkbenchTerminalTraceMissing(samples);
+  const terminalApiDomLag = detectWorkbenchTerminalApiDomLag(samples, network);
+  return {
+    terminalTraceMissing,
+    terminalApiDomLag,
+    valuesRedacted: true
+  };
+}
+
+function detectWorkbenchTerminalTraceMissing(samples) {
+  const rows = [];
+  for (const sample of Array.isArray(samples) ? samples : []) {
+    if (!isWorkbenchPathSample(sample)) continue;
+    const terminalTraceIds = workbenchTerminalTraceIdsFromDom(sample);
+    for (const traceId of terminalTraceIds) {
+      const traceRows = workbenchTraceRowsForTrace(sample, traceId);
+      if (traceRows.length > 0) continue;
+      rows.push({
+        ...ref(sample),
+        traceId,
+        messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0,
+        turnCount: Array.isArray(sample.turns) ? sample.turns.length : 0,
+        traceRowCount: 0,
+        finalMessageVisible: workbenchFinalMessageVisible(sample, traceId),
+        terminalTurnVisible: workbenchTerminalTurnVisible(sample, traceId),
+        detail: "terminal turn/message was visible for this trace, but the same in-place Workbench sample still had zero trace rows",
+        valuesRedacted: true
+      });
+    }
+  }
+  return rows;
+}
+
+function detectWorkbenchTerminalApiDomLag(samples, network) {
+  const windowMs = 30_000;
+  const budgetMs = Number.isFinite(Number(alertThresholds.sameOriginApiSlowMs)) ? Number(alertThresholds.sameOriginApiSlowMs) : 10_000;
+  const sampleRows = (Array.isArray(samples) ? samples : [])
+    .filter(isWorkbenchPathSample)
+    .map((sample) => ({ sample, tsMs: Date.parse(sample?.ts || ""), pageKey: samplePageKey(sample) }))
+    .filter((item) => Number.isFinite(item.tsMs))
+    .sort((a, b) => a.tsMs - b.tsMs);
+  const rowsByPage = new Map();
+  for (const row of sampleRows) {
+    const rows = rowsByPage.get(row.pageKey) || [];
+    rows.push(row);
+    rowsByPage.set(row.pageKey, rows);
+  }
+  const terminalEvents = (Array.isArray(network) ? network : [])
+    .map(compactWorkbenchTerminalApiEvent)
+    .filter((item) => item !== null);
+  const overBudget = [];
+  for (const event of terminalEvents) {
+    const pageSamples = rowsByPage.get(event.pageKey) || [];
+    const firstAfter = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event);
+    const resolved = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event, (row) => workbenchSampleHasTerminalProjection(row.sample, event));
+    const deltaMs = resolved ? Math.max(0, Math.round(resolved.tsMs - event.tsMs)) : null;
+    const unresolved = !resolved;
+    const exceedsBudget = unresolved || (Number.isFinite(deltaMs) && deltaMs > budgetMs);
+    if (!exceedsBudget) continue;
+    overBudget.push({
+      ts: event.ts,
+      pageRole: event.pageRole,
+      pageId: event.pageId,
+      routeKind: event.routeKind,
+      method: event.method,
+      path: event.path,
+      traceIds: event.traceIds.slice(0, 6),
+      sessionIds: event.sessionIds.slice(0, 4),
+      terminalEvidenceCount: event.terminalEvidenceCount,
+      traceEventLikeCount: event.traceEventLikeCount,
+      finalTextFieldCount: event.finalTextFieldCount,
+      budgetMs,
+      windowMs,
+      resolvedDeltaMs: deltaMs,
+      unresolvedWithinWindow: unresolved,
+      firstAfterSample: compactWorkbenchProjectionSample(firstAfter?.sample, event),
+      resolvedSample: compactWorkbenchProjectionSample(resolved?.sample, event),
+      valuesRedacted: true
+    });
+  }
+  return {
+    summary: {
+      terminalEventCount: terminalEvents.length,
+      overBudgetCount: overBudget.length,
+      budgetMs,
+      windowMs,
+      valuesRedacted: true
+    },
+    overBudget,
+    terminalEvents: terminalEvents.slice(-20),
+    valuesRedacted: true
+  };
+}
+
+function compactWorkbenchTerminalApiEvent(item) {
+  if (!item || item.type !== "response" || item.observerInitiated === true) return null;
+  const summary = objectValue(item.bodySummary);
+  if (!summary || Number(summary.terminalEvidenceCount ?? 0) <= 0) return null;
+  const parsed = parseApiDomLagUrl(item.url);
+  if (!String(parsed.path || "").startsWith("/v1/workbench/") && parsed.path !== "/v1/agent/chat" && parsed.path !== "/v1/agent/chat/steer") return null;
+  const tsMs = Date.parse(item.ts || "");
+  if (!Number.isFinite(tsMs)) return null;
+  return {
+    ts: item.ts,
+    tsMs,
+    pageRole: item.pageRole ?? null,
+    pageId: item.pageId ?? null,
+    pageKey: String(item.pageRole || "control") + ":" + String(item.pageId || "default"),
+    method: String(item.method || "GET").toUpperCase(),
+    path: parsed.path,
+    routeKind: summary.pathKind ?? apiDomLagRouteKind(parsed.path),
+    traceIds: uniqueSorted([...(Array.isArray(summary.traceIds) ? summary.traceIds : []), parsed.traceId].filter(Boolean)).slice(0, 12),
+    sessionIds: uniqueSorted([...(Array.isArray(summary.sessionIds) ? summary.sessionIds : []), parsed.sessionId].filter(Boolean)).slice(0, 12),
+    terminalEvidenceCount: Number(summary.terminalEvidenceCount ?? 0),
+    terminalStatusCount: Number(summary.terminalStatusCount ?? 0),
+    terminalTextCount: Number(summary.terminalTextCount ?? 0),
+    traceEventLikeCount: Number(summary.traceEventLikeCount ?? 0),
+    finalTextFieldCount: Number(summary.finalTextFieldCount ?? 0),
+    valuesRedacted: true
+  };
+}
+
+function firstWorkbenchSampleAfter(rows, startMs, endMs, event, predicate = null) {
+  for (const row of rows || []) {
+    if (row.tsMs < startMs) continue;
+    if (row.tsMs > endMs) break;
+    if (!workbenchSampleMatchesTerminalEvent(row.sample, event)) continue;
+    if (typeof predicate === "function" && !predicate(row)) continue;
+    return row;
+  }
+  return null;
+}
+
+function workbenchSampleMatchesTerminalEvent(sample, event) {
+  if (!sample || !event) return false;
+  if (event.sessionIds.length > 0) {
+    const sessionIds = new Set([sample.routeSessionId, sample.activeSessionId].filter(Boolean).map(String));
+    if (!event.sessionIds.some((id) => sessionIds.has(id))) return false;
+  }
+  if (event.traceIds.length > 0) {
+    const traces = sampleTraceIds(sample);
+    if (traces.size > 0 && !event.traceIds.some((id) => traces.has(id))) return false;
+  }
+  return true;
+}
+
+function workbenchSampleHasTerminalProjection(sample, event) {
+  const traceIds = event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample));
+  if (traceIds.length === 0) return false;
+  return traceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId) && workbenchTraceRowsForTrace(sample, traceId).length > 0);
+}
+
+function compactWorkbenchProjectionSample(sample, event = null) {
+  if (!sample) return null;
+  const eventTraceIds = event?.traceIds && event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample));
+  const visibleTraceIds = eventTraceIds.slice(0, 6);
+  return {
+    ...ref(sample),
+    messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0,
+    turnCount: Array.isArray(sample.turns) ? sample.turns.length : 0,
+    traceRowCount: Array.isArray(sample.traceRows) ? sample.traceRows.length : 0,
+    traceIds: visibleTraceIds,
+    finalMessageVisible: visibleTraceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId)),
+    terminalTurnVisible: visibleTraceIds.some((traceId) => workbenchTerminalTurnVisible(sample, traceId)),
+    valuesRedacted: true
+  };
+}
+
+function isWorkbenchPathSample(sample) {
+  return /\/workbench(?:\/|$)/u.test(String(sample?.path || sample?.url || ""));
+}
+
+function workbenchTerminalTraceIdsFromDom(sample) {
+  const ids = new Set();
+  for (const groupName of ["turns", "messages"]) {
+    const group = Array.isArray(sample?.[groupName]) ? sample[groupName] : [];
+    for (const item of group) {
+      const traceId = stringOrNull(item?.traceId);
+      if (!traceId) continue;
+      if (workbenchDomItemIsTerminal(item)) ids.add(traceId);
+    }
+  }
+  return Array.from(ids).sort();
+}
+
+function workbenchTraceRowsForTrace(sample, traceId) {
+  const rows = Array.isArray(sample?.traceRows) ? sample.traceRows : [];
+  return rows.filter((item) => !traceId || item?.traceId === traceId);
+}
+
+function workbenchFinalMessageVisible(sample, traceId) {
+  const messages = Array.isArray(sample?.messages) ? sample.messages : [];
+  return messages.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemLooksFinal(item));
+}
+
+function workbenchTerminalTurnVisible(sample, traceId) {
+  const turns = Array.isArray(sample?.turns) ? sample.turns : [];
+  return turns.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemIsTerminal(item));
+}
+
+function workbenchDomItemIsTerminal(item) {
+  const status = String(item?.status || "").toLowerCase();
+  if (/^(completed|complete|succeeded|success|failed|failure|error|canceled|cancelled|done)$/u.test(status)) return true;
+  return isTerminalTraceText([item?.status, item?.textPreview, item?.text, item?.durationText, item?.activityText].filter(Boolean).join(" "));
+}
+
+function workbenchDomItemLooksFinal(item) {
+  const text = [item?.status, item?.textPreview, item?.text].filter(Boolean).join(" ");
+  return workbenchDomItemIsTerminal(item) && isFinalResultText(text);
+}
+
 function promptCommandHasAuthoritativeSubmitSideEffect(control, promptRound) {
  const commandId = stringOrNull(promptRound?.promptCommandId);
  if (!commandId) return false;
@@ -2341,6 +2553,31 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
  const traceTerminal = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.some((row) => isTerminalTraceText((row.status || "") + " " + (row.textPreview || ""))));
  const traceSeen = samples.some((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0);
  if (traceSeen && !traceTerminal) findings.push({ id: "trace-without-terminal", severity: "amber", summary: "trace rows were visible but no terminal status was sampled", firstTraceSample: ref(samples.find((item) => Array.isArray(item.traceRows) && item.traceRows.length > 0)) });
+  const workbenchInPlaceProjectionLag = detectWorkbenchInPlaceProjectionLag(samples, network);
+  if (workbenchInPlaceProjectionLag.terminalTraceMissing.length > 0) findings.push({
+    id: "workbench-terminal-trace-not-hydrated-in-place",
+    severity: "red",
+    summary: "Workbench rendered a terminal turn/message in-place while the same trace still had no visible run-record trace rows",
+    count: workbenchInPlaceProjectionLag.terminalTraceMissing.length,
+    samples: workbenchInPlaceProjectionLag.terminalTraceMissing.slice(0, 20),
+    rootCause: "workbench_trace_projection_not_hydrated_in_place",
+    rootCauseStatus: "confirmed-from-dom-samples",
+    rootCauseConfidence: "high",
+    valuesRedacted: true
+  });
+  if (workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.length > 0) findings.push({
+    id: "workbench-terminal-api-dom-not-refreshed-in-place",
+    severity: "red",
+    summary: "Workbench REST returned terminal/final trace evidence but the same in-place page did not render terminal message plus trace rows within the configured budget",
+    count: workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.length,
+    budgetMs: workbenchInPlaceProjectionLag.terminalApiDomLag.summary.budgetMs,
+    windowMs: workbenchInPlaceProjectionLag.terminalApiDomLag.summary.windowMs,
+    samples: workbenchInPlaceProjectionLag.terminalApiDomLag.overBudget.slice(0, 20),
+    rootCause: "workbench_rest_terminal_projection_dom_lag",
+    rootCauseStatus: "confirmed-from-network-body-summary-and-dom-samples",
+    rootCauseConfidence: "high",
+    valuesRedacted: true
+  });
  const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false && !promptCommandHasAuthoritativeSubmitSideEffect(control, item)) : [];
  if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat or /v1/agent/chat/steer POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
  const promptSteerRounds = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.steerUsed === true) : [];
@@ -3556,6 +3793,7 @@ function prioritizeFindings(findings) {
    if (id.startsWith("project-management-") || id.startsWith("mdtodo-") || id === "workbench-launch-button-unavailable") return 0;
    if (id === "page-performance-slow-same-origin-api") return 0;
    if (id === "session-rail-title-fallback-majority") return 0.5;
+    if (id.startsWith("workbench-terminal-")) return 0.6;
    if (id.startsWith("code-agent-card-")) return 0.8;
    if (id.startsWith("round-completion-")) return 0.9;
    if (id.startsWith("turn-timing-total-elapsed")) return 1;