feat: add workbench triad sentinel check

2026-07-01 07:21:56 +00:00
parent b735b74b84
commit 098534e999
5 changed files with 276 additions and 5 deletions
@@ -176,6 +176,7 @@ console.log(JSON.stringify({
    turnColumns: recentWindow.sampleMetrics.turnColumns.slice(-12).map((item) => ({ label: item.label, source: item.source, pageRole: item.pageRole ?? null, pageId: item.pageId ?? null, pageEpoch: item.pageEpoch ?? null, promptIndex: item.promptIndex, lastPromptIndex: item.lastPromptIndex, firstSeq: item.firstSeq, lastSeq: item.lastSeq, traceId: item.traceId, messageId: item.messageId })),
    loading: compactLoadingMetricsForOutput(recentWindow.sampleMetrics.loading),
    sessionRailTitles: compactSessionRailTitleMetricsForOutput(recentWindow.sampleMetrics.sessionRailTitles),
+    workbenchTurnStateTriad: compactWorkbenchTurnStateTriadForOutput(recentWindow.sampleMetrics.workbenchTurnStateTriad),
  },
  pageProvenance: recentWindow.pageProvenance.summary,
  pagePerformance: recentWindow.pagePerformance.summary,
@@ -943,6 +944,11 @@ function compactSessionRail(value) {
    testId: item?.testId ?? null,
    role: item?.role ?? null,
    active: item?.active === true,
+    status: item?.status ?? null,
+    dataStatus: item?.dataStatus ?? null,
+    running: item?.running === true,
+    dataRunning: item?.dataRunning ?? null,
+    ariaBusy: item?.ariaBusy ?? null,
    sessionIdPrefix: item?.sessionIdPrefix ?? (item?.sessionId ? String(item.sessionId).slice(0, 12) : null),
    fallbackTitle: item?.fallbackTitle === true,
    titlePreview: limitText(String(item?.titlePreview || item?.titleText || ""), 180),
@@ -964,6 +970,7 @@ function compactSessionRail(value) {
    visibleCount: Number.isFinite(visibleCount) ? visibleCount : items.length,
    fallbackTitleCount: Number.isFinite(fallbackTitleCount) ? fallbackTitleCount : fallbackItems.length,
    fallbackTitleRatio: Number.isFinite(Number(value.fallbackTitleRatio)) ? Number(value.fallbackTitleRatio) : (items.length > 0 ? Number((fallbackItems.length / items.length).toFixed(4)) : 0),
+    activeItem: items.find((item) => item.active) || null,
    items,
    fallbackItems,
    valuesRedacted: true
@@ -1028,6 +1035,8 @@ function compactDomItem(item) {
  if (!item || typeof item !== "object") return item;
  const rawText = String(item.text ?? item.textPreview ?? "");
  const preview = String(item.textPreview ?? limitText(rawText, 240));
+  const hasBodyTextPresent = Object.prototype.hasOwnProperty.call(item, "bodyTextPresent");
+  const hasFinalResponsePresent = Object.prototype.hasOwnProperty.call(item, "finalResponsePresent");
  return {
    index: item.index ?? null,
    tag: item.tag ?? null,
@@ -1047,6 +1056,18 @@ function compactDomItem(item) {
    eventKind: item.eventKind ?? null,
    durationText: item.durationText ?? null,
    activityText: item.activityText ?? null,
+    bodyTextSource: item.bodyTextSource ?? null,
+    bodyTextCandidateCount: Number.isFinite(Number(item.bodyTextCandidateCount)) ? Number(item.bodyTextCandidateCount) : null,
+    bodyTextPresent: hasBodyTextPresent ? item.bodyTextPresent === true : null,
+    bodyTextPreview: item.bodyTextPreview ? limitText(item.bodyTextPreview, 600) : undefined,
+    bodyTextHash: item.bodyTextHash ?? null,
+    bodyTextBytes: Number.isFinite(Number(item.bodyTextBytes)) ? Number(item.bodyTextBytes) : null,
+    finalResponsePresent: hasFinalResponsePresent ? item.finalResponsePresent === true : null,
+    finalResponseTextSource: item.finalResponseTextSource ?? null,
+    finalResponseCandidateCount: Number.isFinite(Number(item.finalResponseCandidateCount)) ? Number(item.finalResponseCandidateCount) : null,
+    finalResponseTextPreview: item.finalResponseTextPreview ? limitText(item.finalResponseTextPreview, 600) : undefined,
+    finalResponseTextHash: item.finalResponseTextHash ?? null,
+    finalResponseTextBytes: Number.isFinite(Number(item.finalResponseTextBytes)) ? Number(item.finalResponseTextBytes) : null,
    className: item.className ?? null,
    diagnosticCode: item.diagnosticCode ?? null,
    source: item.source ?? null,
@@ -1107,6 +1128,17 @@ function compactSessionRailTitleMetricsForOutput(value) {
  };
 }

+function compactWorkbenchTurnStateTriadForOutput(value) {
+  if (!value || typeof value !== "object") return null;
+  return {
+    summary: value.summary ?? null,
+    invalidFullTriads: Array.isArray(value.invalidFullTriads) ? value.invalidFullTriads.slice(0, 8) : [],
+    cardFinalResponseMismatches: Array.isArray(value.cardFinalResponseMismatches) ? value.cardFinalResponseMismatches.slice(0, 8) : [],
+    collectorMissingRows: Array.isArray(value.collectorMissingRows) ? value.collectorMissingRows.slice(0, 8) : [],
+    valuesRedacted: true
+  };
+}
+
 function compactSamplePageProvenance(value) {
  if (!value || typeof value !== "object") return null;
  return {
@@ -2807,6 +2839,163 @@ function workbenchDomItemLooksFinal(item) {
  return workbenchDomItemIsTerminal(item) && isFinalResultText(text);
 }

+function buildWorkbenchTurnStateTriadMetrics(samples, timeline = []) {
+  const rows = [];
+  const sourceSamples = Array.isArray(samples) ? samples : [];
+  for (let index = 0; index < sourceSamples.length; index += 1) {
+    const sample = sourceSamples[index];
+    if (!isWorkbenchPathSample(sample)) continue;
+    const turns = Array.isArray(sample?.turns) ? sample.turns.filter((turn) => String(turn?.role || turn?.dataRole || "agent") === "agent") : [];
+    if (turns.length === 0) continue;
+    const rail = activeSessionRailItemForSample(sample);
+    for (const turn of turns.slice(-6)) {
+      const row = workbenchTurnStateTriadRow(sample, turn, rail, timeline[index]?.promptIndex ?? 0);
+      if (row) rows.push(row);
+    }
+  }
+  const invalidFullTriads = rows.filter((row) => row.invalid === true && row.fullTriad === true);
+  const cardFinalResponseMismatches = rows.filter((row) => row.cardFinalResponseMismatch === true || row.legacyCardFinalResponseMismatch === true);
+  const collectorMissingRows = rows.filter((row) => Array.isArray(row.collectorMissing) && row.collectorMissing.length > 0);
+  const invalidRowKeys = new Set([...invalidFullTriads, ...cardFinalResponseMismatches].map((row) => row.rowKey));
+  const collectorMissingFields = uniqueSorted(collectorMissingRows.flatMap((row) => row.collectorMissing || []));
+  return {
+    summary: {
+      sampleCount: sourceSamples.length,
+      rowCount: rows.length,
+      fullTriadRowCount: rows.filter((row) => row.fullTriad === true).length,
+      invalidRowCount: invalidRowKeys.size,
+      invalidFullTriadCount: invalidFullTriads.length,
+      cardFinalResponseMismatchCount: cardFinalResponseMismatches.length,
+      collectorMissingRowCount: collectorMissingRows.length,
+      collectorMissingFields,
+      valuesRedacted: true
+    },
+    invalidFullTriads: invalidFullTriads.slice(0, 120),
+    cardFinalResponseMismatches: cardFinalResponseMismatches.slice(0, 120),
+    collectorMissingRows: collectorMissingRows.slice(0, 120),
+    timeline: rows.slice(-300),
+    valuesRedacted: true
+  };
+}
+
+function workbenchTurnStateTriadRow(sample, turn, rail, promptIndex) {
+  const collectorMissing = [];
+  const railRawStatus = firstString(rail?.status, rail?.dataStatus);
+  const railRunning = rail?.running === true || String(rail?.dataRunning || rail?.ariaBusy || "").toLowerCase() === "true";
+  const railStatus = normalizeWorkbenchTriadStatus(railRawStatus, railRunning);
+  if (!rail) collectorMissing.push("sessionRail.activeItem");
+  if (!railRawStatus && railRunning !== true) collectorMissing.push("sessionRail.status");
+  const cardRawStatus = firstString(turn?.status);
+  const cardStatus = normalizeWorkbenchTriadStatus(cardRawStatus, false) || (workbenchDomItemIsTerminal(turn) ? "completed" : null);
+  if (!cardRawStatus && !cardStatus) collectorMissing.push("turn.status");
+  const finalPresence = finalResponsePresenceFromTurn(turn);
+  if (finalPresence.known !== true) collectorMissing.push("turn.finalResponseTextBytes");
+  const fullTriad = Boolean(railStatus && cardStatus && finalPresence.known === true);
+  const finalResponsePresent = finalPresence.known === true ? finalPresence.present : null;
+  const tupleAllowed = fullTriad
+    ? (railStatus === "completed" && cardStatus === "completed" && finalResponsePresent === true)
+      || (railStatus === "running" && cardStatus === "running" && finalResponsePresent === false)
+    : null;
+  const cardFinalResponseMismatch = cardStatus === "completed" && finalPresence.known === true && finalResponsePresent !== true;
+  const legacyCardFinalResponseMismatch = cardStatus === "completed" && finalPresence.known !== true && !workbenchDomItemLooksFinal(turn);
+  const railCardMismatch = Boolean(railStatus && cardStatus && railStatus !== cardStatus);
+  const invalid = tupleAllowed === false || cardFinalResponseMismatch || legacyCardFinalResponseMismatch || railCardMismatch;
+  const traceId = firstString(turn?.traceId, turn?.turnId);
+  const messageId = firstString(turn?.messageId);
+  const rowKey = [
+    sample?.seq ?? "-",
+    sample?.pageRole ?? "control",
+    sample?.pageId ?? "default",
+    sample?.routeSessionId ?? sample?.activeSessionId ?? rail?.sessionIdPrefix ?? "-",
+    traceId ?? messageId ?? turn?.index ?? "-"
+  ].join("|");
+  return {
+    ...ref(sample),
+    rowKey,
+    promptIndex,
+    sessionIdPrefix: sessionPrefixForSample(sample, rail),
+    traceId,
+    messageId,
+    turnId: firstString(turn?.turnId),
+    turnIndex: turn?.index ?? null,
+    railStatus,
+    railStatusRaw: railRawStatus ?? null,
+    railRunning: railRunning === true,
+    railSource: rail ? "sessionRail.activeItem" : null,
+    cardStatus,
+    cardStatusRaw: cardRawStatus ?? null,
+    finalResponsePresent,
+    finalResponseTextBytes: turn?.finalResponseTextBytes !== null && turn?.finalResponseTextBytes !== undefined && Number.isFinite(Number(turn.finalResponseTextBytes)) ? Number(turn.finalResponseTextBytes) : null,
+    finalResponseSource: turn?.finalResponseTextSource ?? null,
+    finalResponseTextHash: turn?.finalResponseTextHash ?? null,
+    finalResponseTextPreview: turn?.finalResponseTextPreview ? limitText(turn.finalResponseTextPreview, 160) : null,
+    fullTriad,
+    tupleAllowed,
+    invalid,
+    cardFinalResponseMismatch,
+    legacyCardFinalResponseMismatch,
+    railCardMismatch,
+    collectorMissing,
+    nearestCommandId: sample?.commandId ?? null,
+    relatedChecks: ["WBC-001", "WBC-003", "WBC-011", "WBC-022"],
+    valuesRedacted: true
+  };
+}
+
+function activeSessionRailItemForSample(sample) {
+  const rail = sample?.sessionRail && typeof sample.sessionRail === "object" ? sample.sessionRail : null;
+  if (!rail) return null;
+  const items = Array.isArray(rail.items) ? rail.items : [];
+  if (rail.activeItem) return rail.activeItem;
+  const routeSessionId = String(sample?.routeSessionId || "");
+  const activeSessionId = String(sample?.activeSessionId || "");
+  return items.find((item) => item?.active === true)
+    || items.find((item) => sessionRailItemMatchesSession(item, activeSessionId))
+    || items.find((item) => sessionRailItemMatchesSession(item, routeSessionId))
+    || null;
+}
+
+function sessionRailItemMatchesSession(item, sessionId) {
+  const id = String(sessionId || "");
+  const prefix = String(item?.sessionIdPrefix || item?.sessionId || "");
+  return Boolean(id && prefix && (id === prefix || id.startsWith(prefix) || prefix.startsWith(id)));
+}
+
+function sessionPrefixForSample(sample, rail) {
+  const id = firstString(sample?.activeSessionId, sample?.routeSessionId, rail?.sessionIdPrefix, rail?.sessionId);
+  return id ? String(id).slice(0, 12) : null;
+}
+
+function finalResponsePresenceFromTurn(turn) {
+  if (!turn || typeof turn !== "object") return { known: false, present: null };
+  if (turn.finalResponseTextBytes !== null && turn.finalResponseTextBytes !== undefined && Number.isFinite(Number(turn.finalResponseTextBytes))) {
+    const bytes = Number(turn.finalResponseTextBytes);
+    return { known: true, present: bytes > 0 };
+  }
+  if (Object.prototype.hasOwnProperty.call(turn, "finalResponsePresent") && turn.finalResponsePresent !== null && turn.finalResponsePresent !== undefined) {
+    return { known: true, present: turn.finalResponsePresent === true };
+  }
+  return { known: false, present: null };
+}
+
+function normalizeWorkbenchTriadStatus(status, running = false) {
+  const value = String(status || "").trim().toLowerCase().replace(/_/gu, "-");
+  if (running === true) return "running";
+  if (!value) return null;
+  if (/^(completed|complete|succeeded|success|finished|done|terminal|sealed)$/u.test(value)) return "completed";
+  if (/^(pending|running|active|busy|admitted|dispatching|executing|streaming|processing|queued|in-progress|creating)$/u.test(value)) return "running";
+  return null;
+}
+
+function firstString(...values) {
+  for (const value of values) {
+    if (typeof value !== "string") continue;
+    const text = value.trim();
+    if (text) return text;
+  }
+  return null;
+}
+
 function promptCommandHasAuthoritativeSubmitSideEffect(control, promptRound) {
  const commandId = stringOrNull(promptRound?.promptCommandId);
  if (!commandId) return false;
@@ -2958,6 +3147,33 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
    rootCauseConfidence: "high",
    valuesRedacted: true
  });
+  const turnStateTriad = sampleMetrics?.workbenchTurnStateTriad || {};
+  const turnStateTriadSummary = turnStateTriad.summary || {};
+  const turnStateTriadRows = [
+    ...(Array.isArray(turnStateTriad.invalidFullTriads) ? turnStateTriad.invalidFullTriads : []),
+    ...(Array.isArray(turnStateTriad.cardFinalResponseMismatches) ? turnStateTriad.cardFinalResponseMismatches : [])
+  ];
+  if (Number(turnStateTriadSummary.invalidRowCount ?? 0) > 0) findings.push({
+    id: "workbench-turn-state-triad-inconsistent",
+    severity: "red",
+    summary: "Workbench session rail status, turn card status, and Final Response body presence diverged from the allowed running/running/absent or completed/completed/present tuples",
+    count: turnStateTriadSummary.invalidRowCount,
+    fullTriadCount: turnStateTriadSummary.fullTriadRowCount,
+    invalidFullTriadCount: turnStateTriadSummary.invalidFullTriadCount,
+    cardFinalResponseMismatchCount: turnStateTriadSummary.cardFinalResponseMismatchCount,
+    legacyCollectorMissingCount: turnStateTriadSummary.collectorMissingRowCount,
+    collectorMissingFields: Array.isArray(turnStateTriadSummary.collectorMissingFields) ? turnStateTriadSummary.collectorMissingFields : [],
+    allowedTuples: [
+      { railStatus: "completed", cardStatus: "completed", finalResponsePresent: true },
+      { railStatus: "running", cardStatus: "running", finalResponsePresent: false }
+    ],
+    samples: turnStateTriadRows.slice(0, 20),
+    collectorMissingSamples: Array.isArray(turnStateTriad.collectorMissingRows) ? turnStateTriad.collectorMissingRows.slice(0, 10) : [],
+    rootCause: "workbench_projection_state_triad_not_sealed",
+    rootCauseStatus: "confirmed-from-dom-samples",
+    rootCauseConfidence: "high",
+    valuesRedacted: true
+  });
  const promptFailures = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.chatPostOk === false && !promptCommandHasAuthoritativeSubmitSideEffect(control, item)) : [];
  if (promptFailures.length > 0) findings.push({ id: "prompt-chat-submit-failed", severity: "red", summary: "sendPrompt command had no successful /v1/agent/chat or /v1/agent/chat/steer POST response in the sampling window", count: promptFailures.length, rounds: promptFailures.slice(0, 10) });
  const promptSteerRounds = Array.isArray(promptNetwork?.rounds) ? promptNetwork.rounds.filter((item) => item.steerUsed === true) : [];
@@ -4875,6 +5091,7 @@ function prioritizeFindings(findings) {
    if (id.startsWith("project-management-") || id.startsWith("mdtodo-") || id === "workbench-launch-button-unavailable") return 0;
    if (id === "page-performance-slow-same-origin-api") return 0;
    if (id === "session-rail-title-fallback-majority") return 0.5;
+    if (id === "workbench-turn-state-triad-inconsistent") return 0.55;
    if (id.startsWith("workbench-terminal-")) return 0.6;
    if (id.startsWith("code-agent-card-")) return 0.8;
    if (id.startsWith("round-completion-")) return 0.9;
@@ -4975,6 +5192,7 @@ function buildSampleMetrics(samples, control) {
  const diagnostics = timeline.filter((item) => item.diagnosticSeen).length;
  const loading = buildLoadingMetrics(samples, timeline);
  const sessionRailTitles = buildSessionRailTitleMetrics(samples, timeline);
+  const workbenchTurnStateTriad = buildWorkbenchTurnStateTriadMetrics(samples, timeline);
  const reportTurnTimingRows = boundedTurnTimingRowsForReport(turnTiming.rows);
  const reportTimeline = boundedRowsForReport(timeline);
  const rounds = buildRoundMetricSummaries(timeline, promptCommands, {
@@ -5009,6 +5227,11 @@ function buildSampleMetrics(samples, control) {
      sessionRailFallbackMaxRatio: sessionRailTitles.summary.maxFallbackRatio,
      sessionRailFallbackMaxVisibleCount: sessionRailTitles.summary.maxVisibleCount,
      sessionRailFallbackMaxCount: sessionRailTitles.summary.maxFallbackTitleCount,
+      workbenchTurnStateTriadRows: workbenchTurnStateTriad.summary.rowCount,
+      workbenchTurnStateTriadInvalidRows: workbenchTurnStateTriad.summary.invalidRowCount,
+      workbenchTurnStateTriadFullInvalidRows: workbenchTurnStateTriad.summary.invalidFullTriadCount,
+      workbenchTurnStateCardFinalResponseMismatchRows: workbenchTurnStateTriad.summary.cardFinalResponseMismatchCount,
+      workbenchTurnStateCollectorMissingRows: workbenchTurnStateTriad.summary.collectorMissingRowCount,
      promptSegments: Math.max(0, promptTimes.length),
      rounds: rounds.length,
      turnColumns: turnTiming.columns.length,
@@ -5048,6 +5271,7 @@ function buildSampleMetrics(samples, control) {
    },
    loading,
    sessionRailTitles,
+    workbenchTurnStateTriad,
    codeAgentCardTiming,
    traceOrder,
    rounds,