fix: detect workbench recovery authority fanout

scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts

@@ -152,6 +152,33 @@ export function nodeWebObserveAnalyzerFindingsSource(): string {
     rootCauseConfidence: "high",
     valuesRedacted: true
   });
+  const recoveryAuthority = workbenchInPlaceProjectionLag.recoveryAuthority || {};
+  const recoveryAuthoritySummary = recoveryAuthority.summary || {};
+  const forbiddenRecoveryCount = Number(recoveryAuthoritySummary.forbiddenAutomaticRecoveryCount ?? 0);
+  const legacyFanoutWindowCount = Number(recoveryAuthoritySummary.legacyFanoutWindowCount ?? 0);
+  if (forbiddenRecoveryCount > 0 || legacyFanoutWindowCount > 0) findings.push({
+    id: "workbench-automatic-recovery-fanout-authority",
+    severity: "red",
+    summary: "Workbench automatic recovery used direct trace/session/turn fan-out instead of sync replay authority",
+    count: forbiddenRecoveryCount + legacyFanoutWindowCount,
+    forbiddenAutomaticRecoveryCount: forbiddenRecoveryCount,
+    legacyFanoutWindowCount,
+    syncReplayEvidenceCount: Number(recoveryAuthoritySummary.syncReplayEvidenceCount ?? 0),
+    syncReplayAppliedCount: Number(recoveryAuthoritySummary.syncReplayAppliedCount ?? 0),
+    recoveryAuthority: {
+      summary: recoveryAuthoritySummary,
+      forbiddenAutomaticRecovery: Array.isArray(recoveryAuthority.forbiddenAutomaticRecovery) ? recoveryAuthority.forbiddenAutomaticRecovery.slice(0, 12) : [],
+      legacyFanoutWindows: Array.isArray(recoveryAuthority.legacyFanoutWindows) ? recoveryAuthority.legacyFanoutWindows.slice(0, 12) : [],
+      syncEvidence: Array.isArray(recoveryAuthority.syncEvidence) ? recoveryAuthority.syncEvidence.slice(-12) : [],
+      observabilityGaps: Array.isArray(recoveryAuthority.observabilityGaps) ? recoveryAuthority.observabilityGaps.slice(0, 8) : [],
+      valuesRedacted: true
+    },
+    sourceOfTruth: "Workbench sync replay / entity version / terminal seal authority; explicit detail/history reads are allowed but automatic recovery must not hydrate direct REST fan-out",
+    rootCause: "workbench_automatic_recovery_legacy_fanout",
+    rootCauseStatus: "confirmed-from-web-performance-runtime-diagnostics-and-network-artifacts",
+    rootCauseConfidence: "high",
+    valuesRedacted: true
+  });
   const turnStateTriad = sampleMetrics?.workbenchTurnStateTriad || {};
   const turnStateTriadSummary = turnStateTriad.summary || {};
   const turnStateTriadRows = [

scripts/src/hwlab-node-web-observe-analyzer-request-runtime-source.ts

@@ -701,7 +701,7 @@ function extractWebPerformanceRuntimeDiagnostics(network, promptTimes) {
     for (const event of events) {
       if (!event || typeof event !== "object") continue;
       const diagnosticCode = limitText(event.diagnosticCode || event.code || "", 120);
-      const reason = limitText(event.reason || "", 120);
+      const reason = limitText(event.rootCause || event.reason || "", 120);
       const eventType = limitText(event.eventType || event.type || event.kind || "", 120);
       if (!diagnosticCode && !reason && !eventType) continue;
       rows.push({
@@ -721,6 +721,17 @@ function extractWebPerformanceRuntimeDiagnostics(network, promptTimes) {
         diagnosticCode,
         reason,
         module: limitText(event.module || "", 120),
+        recoveryAction: limitText(event.recoveryAction || "", 160),
+        contractVersion: limitText(event.contractVersion || "", 80),
+        realtimeAuthority: limitText(event.realtimeAuthority || "", 120),
+        sinceOutboxSeq: numberOrNull(event.sinceOutboxSeq),
+        syncCursorOutboxSeq: numberOrNull(event.syncCursorOutboxSeq),
+        entityFamilyCount: numberOrNull(event.entityFamilyCount),
+        entityFamilies: limitText(event.entityFamilies || "", 160),
+        maxEntityVersion: numberOrNull(event.maxEntityVersion),
+        projectionRevision: limitText(event.projectionRevision || "", 120),
+        terminalSeal: typeof event.terminalSeal === "boolean" ? event.terminalSeal : null,
+        detailProjection: typeof event.detailProjection === "boolean" ? event.detailProjection : null,
         traceId: event.traceId ?? null,
         sessionIdHash: event.sessionIdHash ?? null,
         eventIdHash: event.eventIdHash ?? null,
@@ -823,6 +834,17 @@ function groupWebPerformanceRuntimeDiagnostics(events) {
       ts: item.ts || null,
       traceId: item.traceId || null,
       eventCount: item.eventCount ?? null,
+      recoveryAction: item.recoveryAction ?? null,
+      sinceOutboxSeq: item.sinceOutboxSeq ?? null,
+      syncCursorOutboxSeq: item.syncCursorOutboxSeq ?? null,
+      contractVersion: item.contractVersion ?? null,
+      realtimeAuthority: item.realtimeAuthority ?? null,
+      entityFamilyCount: item.entityFamilyCount ?? null,
+      entityFamilies: item.entityFamilies ?? null,
+      maxEntityVersion: item.maxEntityVersion ?? null,
+      projectionRevision: item.projectionRevision ?? null,
+      terminalSeal: item.terminalSeal ?? null,
+      detailProjection: item.detailProjection ?? null,
       deliveredCount: item.deliveredCount ?? null,
       chunkCount: item.chunkCount ?? null,
       flushDurationMs: item.flushDurationMs ?? null,

scripts/src/hwlab-node-web-observe-analyzer-source.ts

@@ -469,6 +469,7 @@ function prioritizeFindings(findings) {
     if (id.startsWith("frontend-long") || id.startsWith("frontend-event-loop-gap") || id === "frontend-cpu-profile-hotspots") return 0;
     if (id === "page-performance-slow-same-origin-api") return 0;
     if (id === "session-rail-title-fallback-majority") return 0.5;
+    if (id === "workbench-automatic-recovery-fanout-authority") return 0.52;
     if (id === "workbench-turn-state-triad-inconsistent") return 0.55;
     if (id.startsWith("workbench-terminal-")) return 0.6;
     if (id.startsWith("code-agent-card-")) return 0.8;

scripts/src/hwlab-node-web-observe-analyzer-workbench-triad-source.ts

@@ -6,9 +6,11 @@ export function nodeWebObserveAnalyzerWorkbenchTriadSource(): string {
   const canarySessionIds = sessionInvariantCanarySessionIds(control);
   const terminalTraceMissing = detectWorkbenchTerminalTraceMissing(samples, canarySessionIds);
   const terminalApiDomLag = detectWorkbenchTerminalApiDomLag(samples, network, canarySessionIds);
+  const recoveryAuthority = detectWorkbenchRecoveryAuthority(network, control);
   return {
     terminalTraceMissing,
     terminalApiDomLag,
+    recoveryAuthority,
     valuesRedacted: true
   };
 }
@@ -260,6 +262,268 @@ function isWorkbenchPathSample(sample) {
   return /\/workbench(?:\/|$)/u.test(String(sample?.path || sample?.url || ""));
 }
 
+function detectWorkbenchRecoveryAuthority(network, control = []) {
+  void control;
+  const diagnostics = workbenchRecoveryDiagnosticsFromNetwork(network);
+  const endpointReads = workbenchRecoveryEndpointReads(network);
+  const syncEvidence = workbenchSyncEvidenceFromNetwork(network, diagnostics);
+  const forbiddenAutomaticRecovery = diagnostics
+    .filter(isForbiddenWorkbenchRecoveryDiagnostic)
+    .map(compactWorkbenchRecoveryDiagnostic);
+  const legacyFanoutWindows = workbenchLegacyRecoveryFanoutWindows(endpointReads, diagnostics, syncEvidence);
+  const observabilityGaps = [];
+  if (diagnostics.length > 0 && syncEvidence.length === 0) {
+    observabilityGaps.push({
+      id: "workbench-sync-replay-evidence-missing",
+      severity: "amber",
+      summary: "Workbench recovery diagnostics were captured but no sync replay diagnostic or /v1/workbench/sync network row was present in the artifact window",
+      diagnosticCount: diagnostics.length,
+      valuesRedacted: true
+    });
+  }
+  return {
+    summary: {
+      diagnosticCount: diagnostics.length,
+      forbiddenAutomaticRecoveryCount: forbiddenAutomaticRecovery.length,
+      legacyFanoutWindowCount: legacyFanoutWindows.length,
+      syncReplayEvidenceCount: syncEvidence.length,
+      syncReplayAppliedCount: syncEvidence.filter((item) => item.code === "workbench_sync_replay_applied").length,
+      syncNetworkReadCount: endpointReads.filter((item) => item.kind === "workbench-sync-replay").length,
+      oldEndpointReadCount: endpointReads.filter((item) => item.legacyRead === true).length,
+      observabilityGapCount: observabilityGaps.length,
+      valuesRedacted: true
+    },
+    forbiddenAutomaticRecovery: forbiddenAutomaticRecovery.slice(0, 20),
+    legacyFanoutWindows: legacyFanoutWindows.slice(0, 20),
+    syncEvidence: syncEvidence.slice(-20),
+    diagnostics: diagnostics.slice(-20).map(compactWorkbenchRecoveryDiagnostic),
+    observabilityGaps,
+    valuesRedacted: true
+  };
+}
+
+function workbenchRecoveryDiagnosticsFromNetwork(network) {
+  const rows = [];
+  for (const item of Array.isArray(network) ? network : []) {
+    const payload = item?.webPerformancePayload && typeof item.webPerformancePayload === "object" ? item.webPerformancePayload : null;
+    if (!payload || payload.parseStatus !== "parsed" || payload.schemaVersion !== "hwlab-web-performance-v2") continue;
+    const events = Array.isArray(payload.events) ? payload.events : [];
+    for (const event of events) {
+      if (!event || typeof event !== "object") continue;
+      const eventType = firstString(event.eventType, event.type, event.kind);
+      const diagnosticCode = firstString(event.diagnosticCode, event.code, event.errorName);
+      const module = firstString(event.module);
+      const recoveryAction = firstString(event.recoveryAction);
+      const rootCause = firstString(event.rootCause);
+      const source = firstString(event.source);
+      const looksRelevant = eventType === "runtime_diagnostic"
+        || Boolean(diagnosticCode && /workbench|terminal|recovery|sync|hydrate|fanout/iu.test(diagnosticCode))
+        || Boolean(module && /workbench/iu.test(module))
+        || Boolean(recoveryAction);
+      if (!looksRelevant) continue;
+      rows.push({
+        ts: item.ts ?? event.ts ?? null,
+        tsMs: Date.parse(item.ts ?? event.ts ?? ""),
+        pageRole: item.pageRole ?? null,
+        pageId: item.pageId ?? null,
+        urlPath: urlPath(item.url),
+        eventType,
+        module,
+        diagnosticCode,
+        reason: rootCause ?? firstString(event.reason),
+        recoveryAction,
+        source,
+        contractVersion: firstString(event.contractVersion),
+        realtimeAuthority: firstString(event.realtimeAuthority),
+        eventCount: numberOrNull(event.eventCount),
+        sinceOutboxSeq: numberOrNull(event.sinceOutboxSeq),
+        syncCursorOutboxSeq: numberOrNull(event.syncCursorOutboxSeq),
+        entityFamilyCount: numberOrNull(event.entityFamilyCount),
+        entityFamilies: firstString(event.entityFamilies),
+        maxEntityVersion: numberOrNull(event.maxEntityVersion),
+        projectionRevision: firstString(event.projectionRevision),
+        terminalSeal: typeof event.terminalSeal === "boolean" ? event.terminalSeal : null,
+        detailProjection: typeof event.detailProjection === "boolean" ? event.detailProjection : null,
+        valuesRedacted: true
+      });
+    }
+  }
+  return rows.sort((left, right) => String(left.ts || "").localeCompare(String(right.ts || "")));
+}
+
+function workbenchSyncEvidenceFromNetwork(network, diagnostics) {
+  const rows = [];
+  for (const item of Array.isArray(network) ? network : []) {
+    if (item?.observerInitiated === true) continue;
+    const route = workbenchRecoveryNetworkRoute(item);
+    const summary = objectValue(item?.bodySummary);
+    if (route.kind !== "workbench-sync-replay" && String(summary?.pathKind || "") !== "workbench-sync-replay") continue;
+    rows.push({
+      ts: item.ts ?? null,
+      tsMs: Date.parse(item.ts || ""),
+      source: "network",
+      method: String(item.method || "GET").toUpperCase(),
+      status: item.status ?? null,
+      path: route.path,
+      code: "workbench_sync_network_read",
+      sinceOutboxSeq: numberOrNull(route.since ?? summary?.sinceOutboxSeq),
+      syncCursorOutboxSeq: numberOrNull(summary?.cursorOutboxSeq ?? summary?.syncCursorOutboxSeq),
+      eventCount: numberOrNull(summary?.eventCount ?? summary?.traceEventLikeCount),
+      contractVersion: firstString(summary?.contractVersion),
+      realtimeAuthority: firstString(summary?.realtimeAuthority),
+      entityFamilyCount: numberOrNull(summary?.entityFamilyCount),
+      maxEntityVersion: numberOrNull(summary?.maxEntityVersion),
+      terminalSeal: typeof summary?.terminalSeal === "boolean" ? summary.terminalSeal : null,
+      valuesRedacted: true
+    });
+  }
+  for (const item of Array.isArray(diagnostics) ? diagnostics : []) {
+    if (item.diagnosticCode !== "workbench_sync_replay_applied" && item.module !== "workbench-sync-replay") continue;
+    rows.push({
+      ts: item.ts ?? null,
+      tsMs: item.tsMs,
+      source: "web-performance-runtime-diagnostic",
+      code: item.diagnosticCode || "workbench_sync_replay_applied",
+      reason: item.reason ?? null,
+      eventCount: item.eventCount,
+      sinceOutboxSeq: item.sinceOutboxSeq,
+      syncCursorOutboxSeq: item.syncCursorOutboxSeq,
+      contractVersion: item.contractVersion,
+      realtimeAuthority: item.realtimeAuthority,
+      entityFamilyCount: item.entityFamilyCount,
+      entityFamilies: item.entityFamilies,
+      maxEntityVersion: item.maxEntityVersion,
+      projectionRevision: item.projectionRevision,
+      terminalSeal: item.terminalSeal,
+      detailProjection: item.detailProjection,
+      valuesRedacted: true
+    });
+  }
+  return rows.sort((left, right) => String(left.ts || "").localeCompare(String(right.ts || "")));
+}
+
+function workbenchRecoveryEndpointReads(network) {
+  const rows = [];
+  for (const item of Array.isArray(network) ? network : []) {
+    if (item?.observerInitiated === true) continue;
+    if (item?.type !== "request" && item?.type !== "response" && item?.type !== "requestfailed") continue;
+    const route = workbenchRecoveryNetworkRoute(item);
+    if (!route.kind) continue;
+    rows.push({
+      ts: item.ts ?? null,
+      tsMs: Date.parse(item.ts || ""),
+      type: item.type ?? null,
+      method: String(item.method || "GET").toUpperCase(),
+      status: item.status ?? null,
+      kind: route.kind,
+      legacyRead: route.legacyRead,
+      path: route.path,
+      traceId: route.traceId,
+      sessionIdPrefix: route.sessionId ? String(route.sessionId).slice(0, 12) : null,
+      valuesRedacted: true
+    });
+  }
+  return rows.sort((left, right) => Number(left.tsMs || 0) - Number(right.tsMs || 0));
+}
+
+function workbenchRecoveryNetworkRoute(item) {
+  const rawUrl = item?.url || item?.requestUrl || "";
+  const parsed = parseApiDomLagUrl(rawUrl);
+  const rawPath = String(parsed.rawPath || parsed.path || "");
+  const path = String(parsed.path || rawPath);
+  const sessionId = workbenchRecoveryQueryParam(rawUrl, "sessionId") || workbenchRecoveryQueryParam(rawUrl, "includeSessionId") || parsed.sessionId || null;
+  const traceId = workbenchRecoveryQueryParam(rawUrl, "traceId") || parsed.traceId || null;
+  const since = workbenchRecoveryQueryParam(rawUrl, "since");
+  if (rawPath === "/v1/workbench/sync" || path === "/v1/workbench/sync") return { kind: "workbench-sync-replay", legacyRead: false, path: "/v1/workbench/sync", traceId, sessionId, since, valuesRedacted: true };
+  let match = rawPath.match(/^\/v1\/workbench\/turns\/([^/]+)$/u);
+  if (match) return { kind: "workbench-turn", legacyRead: true, path: "/v1/workbench/turns/:id", traceId: match[1], sessionId, since, valuesRedacted: true };
+  match = rawPath.match(/^\/v1\/workbench\/sessions\/([^/]+)\/messages$/u);
+  if (match) return { kind: "workbench-session-messages", legacyRead: true, path: "/v1/workbench/sessions/:id/messages", traceId, sessionId: match[1], since, valuesRedacted: true };
+  match = rawPath.match(/^\/v1\/workbench\/traces\/([^/]+)\/events$/u);
+  if (match) return { kind: "workbench-trace-events", legacyRead: true, path: "/v1/workbench/traces/:id/events", traceId: match[1], sessionId, since, valuesRedacted: true };
+  return { kind: null, legacyRead: false, path, traceId: null, sessionId: null, since, valuesRedacted: true };
+}
+
+function workbenchRecoveryQueryParam(value, key) {
+  try {
+    const parsed = new URL(String(value || "http://invalid.local/"), "http://invalid.local/");
+    return parsed.searchParams.get(key) || null;
+  } catch {
+    return null;
+  }
+}
+
+function workbenchLegacyRecoveryFanoutWindows(endpointReads, diagnostics, syncEvidence) {
+  const rows = [];
+  const recoveryDiagnostics = (Array.isArray(diagnostics) ? diagnostics : []).filter(workbenchRecoveryDiagnosticLooksAutomatic);
+  for (const diagnostic of recoveryDiagnostics) {
+    const tsMs = Number(diagnostic.tsMs);
+    if (!Number.isFinite(tsMs)) continue;
+    const nearby = (Array.isArray(endpointReads) ? endpointReads : [])
+      .filter((item) => item.legacyRead === true && Number.isFinite(Number(item.tsMs)) && Math.abs(Number(item.tsMs) - tsMs) <= 15_000);
+    const kinds = uniqueSorted(nearby.map((item) => item.kind).filter(Boolean));
+    if (kinds.length < 2) continue;
+    const syncNearby = (Array.isArray(syncEvidence) ? syncEvidence : []).some((item) => Number.isFinite(Number(item.tsMs)) && Math.abs(Number(item.tsMs) - tsMs) <= 20_000);
+    rows.push({
+      ts: diagnostic.ts,
+      diagnostic: compactWorkbenchRecoveryDiagnostic(diagnostic),
+      oldEndpointKinds: kinds,
+      oldEndpointReadCount: nearby.length,
+      syncEvidenceNearby: syncNearby,
+      endpointTail: nearby.slice(-8).map((item) => ({
+        ts: item.ts,
+        type: item.type,
+        kind: item.kind,
+        method: item.method,
+        status: item.status,
+        path: item.path,
+        valuesRedacted: true
+      })),
+      valuesRedacted: true
+    });
+  }
+  return rows;
+}
+
+function isForbiddenWorkbenchRecoveryDiagnostic(item) {
+  const action = [item?.recoveryAction, item?.diagnosticCode, item?.source].filter(Boolean).join(" ").toLowerCase();
+  if (!action) return false;
+  if (/hydrate[-_]trace[-_]events|refresh[-_]session[-_]messages|refresh[-_]turn[-_]status/u.test(action)) return true;
+  if (item?.recoveryAction && /active[-_]rest[-_]gap|trace[-_]hydration|terminal[-_]rest|session[-_]detail/u.test(action)) return true;
+  return /legacy[-_]?fanout|automatic[-_]?recovery[-_]?fanout|direct[-_]?hydrate/u.test(action);
+}
+
+function workbenchRecoveryDiagnosticLooksAutomatic(item) {
+  const text = [item?.module, item?.diagnosticCode, item?.reason, item?.recoveryAction, item?.source].filter(Boolean).join(" ").toLowerCase();
+  return /recovery|sync[-_]?replay|stream[-_]?transport|sse[-_]?gap|hydrate[-_]?trace|active[-_]?rest[-_]?gap|refresh[-_]?session/u.test(text);
+}
+
+function compactWorkbenchRecoveryDiagnostic(item) {
+  if (!item) return null;
+  return {
+    ts: item.ts ?? null,
+    pageRole: item.pageRole ?? null,
+    pageId: item.pageId ?? null,
+    module: item.module ?? null,
+    diagnosticCode: item.diagnosticCode ?? null,
+    reason: item.reason ?? null,
+    recoveryAction: item.recoveryAction ?? null,
+    source: item.source ?? null,
+    eventCount: item.eventCount ?? null,
+    sinceOutboxSeq: item.sinceOutboxSeq ?? null,
+    syncCursorOutboxSeq: item.syncCursorOutboxSeq ?? null,
+    contractVersion: item.contractVersion ?? null,
+    realtimeAuthority: item.realtimeAuthority ?? null,
+    entityFamilyCount: item.entityFamilyCount ?? null,
+    entityFamilies: item.entityFamilies ?? null,
+    maxEntityVersion: item.maxEntityVersion ?? null,
+    projectionRevision: item.projectionRevision ?? null,
+    terminalSeal: item.terminalSeal ?? null,
+    detailProjection: item.detailProjection ?? null,
+    valuesRedacted: true
+  };
+}
+
 function workbenchTerminalTraceIdsFromDom(sample) {
   const ids = new Set();
   for (const groupName of ["turns", "messages"]) {

scripts/src/hwlab-node-web-observe-collect.ts

@@ -585,6 +585,31 @@ function compactFinding(item){
     rootCause:item.rootCause||null,
     rootCauseStatus:item.rootCauseStatus||null,
     rootCauseConfidence:item.rootCauseConfidence||null,
+    sourceOfTruth:item.sourceOfTruth?short(item.sourceOfTruth,180):null,
+    valuesRedacted:true
+  };
+}
+function recoveryAuthorityFromFindings(){
+  const finding=findingById('workbench-automatic-recovery-fanout-authority');
+  const authority=finding?.recoveryAuthority&&typeof finding.recoveryAuthority==='object'?finding.recoveryAuthority:null;
+  const summary=authority?.summary&&typeof authority.summary==='object'?authority.summary:null;
+  const syncEvidence=Array.isArray(authority?.syncEvidence)?authority.syncEvidence.slice(-6):[];
+  const forbidden=Array.isArray(authority?.forbiddenAutomaticRecovery)?authority.forbiddenAutomaticRecovery.slice(0,6):[];
+  const fanout=Array.isArray(authority?.legacyFanoutWindows)?authority.legacyFanoutWindows.slice(0,4):[];
+  const gaps=Array.isArray(authority?.observabilityGaps)?authority.observabilityGaps.slice(0,4):[];
+  return {
+    findingPresent:Boolean(finding),
+    forbiddenAutomaticRecoveryCount:Number(summary?.forbiddenAutomaticRecoveryCount??0),
+    legacyFanoutWindowCount:Number(summary?.legacyFanoutWindowCount??0),
+    syncReplayEvidenceCount:Number(summary?.syncReplayEvidenceCount??0),
+    syncReplayAppliedCount:Number(summary?.syncReplayAppliedCount??0),
+    syncNetworkReadCount:Number(summary?.syncNetworkReadCount??0),
+    oldEndpointReadCount:Number(summary?.oldEndpointReadCount??0),
+    observabilityGapCount:Number(summary?.observabilityGapCount??0),
+    latestSyncEvidence:syncEvidence.slice(-1)[0]||null,
+    forbiddenAutomaticRecovery:forbidden,
+    legacyFanoutWindows:fanout,
+    observabilityGaps:gaps,
     valuesRedacted:true
   };
 }
@@ -646,6 +671,7 @@ function workbenchTriadView(rows){
   const traceId=requestedTraceId||requestedRow?.traceId||rows.slice().reverse().find((row)=>row.traceId)?.traceId||traceIdsFromSamples(samples).slice(-1)[0]||null;
   const row=rows.find((item)=>item.traceId===traceId)||requestedRow||rows.slice(-1)[0]||null;
   const findings=[
+    compactFinding(findingById('workbench-automatic-recovery-fanout-authority')),
     compactFinding(findingById('workbench-terminal-api-dom-not-refreshed-in-place')),
     compactFinding(findingById('workbench-terminal-trace-not-hydrated-in-place')),
     compactFinding(findingById('workbench-turn-state-triad-inconsistent')),
@@ -656,6 +682,7 @@ function workbenchTriadView(rows){
   const networkEvidence=networkEvidenceForTrace(traceId);
   const dom=traceDomSnapshot(traceId);
   const freeze=browserFreezeEvidence();
+  const recoveryAuthority=recoveryAuthorityFromFindings();
   const statusGap={
     traceId,
     turnSummaryStatus:row?.status??null,
@@ -669,7 +696,7 @@ function workbenchTriadView(rows){
     valuesRedacted:true
   };
   const compactRow=row?{round:row.round??null,commandId:row.commandId??null,sessionId:row.sessionId??null,traceId:row.traceId??null,status:row.status??null,elapsedSeconds:row.elapsedSeconds??null,recentUpdateSeconds:row.recentUpdateSeconds??null,marks:row.marks??null,firstSeq:row.firstSeq??null,lastSeq:row.lastSeq??null,lastTs:row.lastTs??null,finalResponse:row.finalResponse??null,valuesRedacted:true}:null;
-  return {traceId,row:compactRow,dom,networkEvidence,freeze,findings, statusGap,renderedText:renderWorkbenchTriad({traceId,row:compactRow,dom,networkEvidence,freeze,findings,statusGap}),valuesRedacted:true};
+  return {traceId,row:compactRow,dom,networkEvidence,freeze,recoveryAuthority,findings, statusGap,renderedText:renderWorkbenchTriad({traceId,row:compactRow,dom,networkEvidence,freeze,recoveryAuthority,findings,statusGap}),valuesRedacted:true};
 }
 function renderWorkbenchTriad(input){
   const lines=['Workbench triad offline diagnosis '+(manifest.jobId||'-'),'======================================================='];
@@ -680,6 +707,9 @@ function renderWorkbenchTriad(input){
   lines.push('api evidence responses='+String(input.networkEvidence.responseCount)+' terminalResponses='+String(input.networkEvidence.terminalResponseCount)+' failed='+String(input.networkEvidence.failedCount)+' turnTerminal='+String(turnSummary.terminalEvidenceCount??'-')+' turnFinalBytes='+String(turnSummary.finalTextByteCount??'-')+' traceEvents='+String(traceSummary.traceEventLikeCount??'-')+' traceTerminal='+String(traceSummary.terminalEvidenceCount??'-'));
   if(input.networkEvidence.latestTurnResponse) lines.push('latest turn API '+String(input.networkEvidence.latestTurnResponse.ts||'-')+' status='+String(input.networkEvidence.latestTurnResponse.status??'-')+' sample='+String(input.networkEvidence.latestTurnResponse.sampleSeq??'-')+' path='+String(input.networkEvidence.latestTurnResponse.urlPath||'-'));
   if(input.networkEvidence.latestTraceEventsResponse) lines.push('latest trace events API '+String(input.networkEvidence.latestTraceEventsResponse.ts||'-')+' status='+String(input.networkEvidence.latestTraceEventsResponse.status??'-')+' sample='+String(input.networkEvidence.latestTraceEventsResponse.sampleSeq??'-')+' path='+String(input.networkEvidence.latestTraceEventsResponse.urlPath||'-'));
+  const recovery=input.recoveryAuthority||{};
+  lines.push('recovery authority forbidden='+String(recovery.forbiddenAutomaticRecoveryCount??0)+' legacyFanout='+String(recovery.legacyFanoutWindowCount??0)+' syncEvidence='+String(recovery.syncReplayEvidenceCount??0)+' syncApplied='+String(recovery.syncReplayAppliedCount??0)+' syncNetwork='+String(recovery.syncNetworkReadCount??0)+' oldReads='+String(recovery.oldEndpointReadCount??0)+' gaps='+String(recovery.observabilityGapCount??0));
+  if(recovery.latestSyncEvidence) lines.push('latest sync replay '+String(recovery.latestSyncEvidence.ts||'-')+' source='+String(recovery.latestSyncEvidence.source||'-')+' events='+String(recovery.latestSyncEvidence.eventCount??'-')+' since='+String(recovery.latestSyncEvidence.sinceOutboxSeq??'-')+' cursor='+String(recovery.latestSyncEvidence.syncCursorOutboxSeq??'-')+' authority='+String(recovery.latestSyncEvidence.realtimeAuthority||'-')+' terminalSeal='+String(recovery.latestSyncEvidence.terminalSeal??'-'));
   if(input.freeze.latestBlocker) lines.push('browser freeze '+String(input.freeze.latestBlocker.ts||'-')+' sample='+String(input.freeze.latestBlocker.sampleSeq??'-')+' rootCause='+String(input.freeze.latestBlocker.rootCause||'-')+' status='+String(input.freeze.latestBlocker.rootCauseStatus||'-')+' fallbackAllowed='+String(input.freeze.latestBlocker.fallbackAllowed===true));
   lines.push('','Findings');
   if(input.findings.length===0) lines.push('-');
@@ -983,7 +1013,7 @@ if(view==='timeline'){
 }
 if(view==='workbench-triad'){
   const triad=workbenchTriadView(rows);
-  console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,artifactFileCount:files.length,skippedFileCount:skippedFiles.length,skippedFiles:skippedFiles.slice(0,8),traceId:triad.traceId,dom:triad.dom,networkEvidence:triad.networkEvidence,freeze:triad.freeze,findings:triad.findings,statusGap:triad.statusGap,renderedText:triad.renderedText,sourceFiles:['samples.jsonl','network.jsonl','browser-process.jsonl','analysis/report.json'],valuesRedacted:true}));
+  console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,artifactFileCount:files.length,skippedFileCount:skippedFiles.length,skippedFiles:skippedFiles.slice(0,8),traceId:triad.traceId,dom:triad.dom,networkEvidence:triad.networkEvidence,freeze:triad.freeze,recoveryAuthority:triad.recoveryAuthority,findings:triad.findings,statusGap:triad.statusGap,renderedText:triad.renderedText,sourceFiles:['samples.jsonl','network.jsonl','browser-process.jsonl','analysis/report.json'],valuesRedacted:true}));
   process.exit(0);
 }
 const frame=renderTraceFrame(selectSample(rows),rows);

scripts/src/hwlab-node/web-observe-analyzer-triad.test.ts

@@ -65,6 +65,27 @@ const browserFreezePolicy = {
   },
 };
 
+async function runObserveAnalyzer(files: Record<string, string>): Promise<Record<string, any>> {
+  const stateDir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-analyzer-"));
+  const analyzerPath = join(stateDir, "analyze.mjs");
+  await writeFile(analyzerPath, nodeWebObserveAnalyzerSource(), { mode: 0o700 });
+  for (const [name, content] of Object.entries(files)) {
+    await writeFile(join(stateDir, name), content.endsWith("\n") ? content : content + "\n");
+  }
+  const result = spawnSync("bun", [analyzerPath, stateDir], {
+    cwd: join(import.meta.dir, "../../.."),
+    env: {
+      ...process.env,
+      UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES: "0",
+      UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON: JSON.stringify(alertThresholds),
+      UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON: JSON.stringify(browserFreezePolicy),
+    },
+    encoding: "utf8",
+  });
+  assert.equal(result.status, 0, result.stderr || result.stdout);
+  return JSON.parse(await readFile(join(stateDir, "analysis", "report.json"), "utf8"));
+}
+
 test("observe analyzer classifies stale completed session rail when a newer turn is running", async () => {
   const stateDir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-analyzer-"));
   const analyzerPath = join(stateDir, "analyze.mjs");
@@ -163,3 +184,105 @@ test("observe analyzer classifies stale completed session rail when a newer turn
   assert.equal(report.sampleMetrics.workbenchTurnStateTriad.summary.invalidRowCount, 1);
   assert.equal(report.sampleMetrics.workbenchTurnStateTriad.summary.cardFinalResponseMismatchCount, 0);
 }, 20_000);
+
+test("observe analyzer flags automatic recovery legacy fanout authority", async () => {
+  const samples = [
+    JSON.stringify({
+      seq: 1,
+      ts: "2026-07-01T15:21:25.000Z",
+      path: "/workbench/sessions/ses_auto",
+      url: "https://hwlab.example.test/workbench/sessions/ses_auto",
+      pageRole: "control",
+      pageId: "control-test",
+      routeSessionId: "ses_auto",
+      activeSessionId: "ses_auto",
+      turns: []
+    })
+  ].join("\n") + "\n";
+  const network = [
+    JSON.stringify({
+      ts: "2026-07-01T15:21:26.000Z",
+      type: "response",
+      method: "POST",
+      status: 202,
+      url: "https://hwlab.example.test/v1/web-performance",
+      webPerformancePayload: {
+        parseStatus: "parsed",
+        schemaVersion: "hwlab-web-performance-v2",
+        events: [{
+          kind: "workbench_ui_event",
+          eventType: "runtime_diagnostic",
+          module: "workbench-stream-transport",
+          diagnosticCode: "workbench_recovery_action",
+          recoveryAction: "hydrate-trace-events",
+          rootCause: "sse-gap",
+          eventCount: 0
+        }]
+      }
+    }),
+    JSON.stringify({ ts: "2026-07-01T15:21:27.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/turns/trc_auto", bodySummary: { pathKind: "workbench-turn", terminalEvidenceCount: 1, traceIds: ["trc_auto"] } }),
+    JSON.stringify({ ts: "2026-07-01T15:21:28.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/sessions/ses_auto/messages", bodySummary: { pathKind: "workbench-session-messages", terminalEvidenceCount: 1, sessionIds: ["ses_auto"] } }),
+    JSON.stringify({ ts: "2026-07-01T15:21:29.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/traces/trc_auto/events", bodySummary: { pathKind: "workbench-trace-events", traceEventLikeCount: 3, traceIds: ["trc_auto"] } }),
+    JSON.stringify({ ts: "2026-07-01T15:21:30.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/sync?sessionId=ses_auto&traceId=trc_auto&since=7", bodySummary: { pathKind: "workbench-sync-replay", eventCount: 1, cursorOutboxSeq: 12, realtimeAuthority: "workbench-realtime-authority-v2", terminalSeal: true } })
+  ].join("\n") + "\n";
+
+  const report = await runObserveAnalyzer({ "samples.jsonl": samples, "network.jsonl": network });
+  const finding = report.findings.find((item: Record<string, unknown>) => item.id === "workbench-automatic-recovery-fanout-authority");
+
+  assert.equal(finding?.rootCause, "workbench_automatic_recovery_legacy_fanout");
+  assert.equal(finding?.recoveryAuthority.summary.forbiddenAutomaticRecoveryCount, 1);
+  assert.equal(finding?.recoveryAuthority.summary.legacyFanoutWindowCount, 1);
+  assert.equal(finding?.recoveryAuthority.summary.syncReplayEvidenceCount, 1);
+  assert.equal(report.runtimeAlerts.webPerformanceRuntimeDiagnostics[0].recoveryAction, "hydrate-trace-events");
+}, 20_000);
+
+test("observe analyzer allows explicit detail trace reads when sync authority evidence exists", async () => {
+  const samples = JSON.stringify({
+    seq: 1,
+    ts: "2026-07-01T15:31:25.000Z",
+    path: "/workbench/sessions/ses_detail",
+    url: "https://hwlab.example.test/workbench/sessions/ses_detail",
+    pageRole: "control",
+    pageId: "control-test",
+    routeSessionId: "ses_detail",
+    activeSessionId: "ses_detail",
+    turns: []
+  }) + "\n";
+  const network = [
+    JSON.stringify({ ts: "2026-07-01T15:31:26.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/traces/trc_detail/events", bodySummary: { pathKind: "workbench-trace-events", traceEventLikeCount: 2, traceIds: ["trc_detail"], authority: "trace-detail-only", detailProjection: true } }),
+    JSON.stringify({
+      ts: "2026-07-01T15:31:27.000Z",
+      type: "response",
+      method: "POST",
+      status: 202,
+      url: "https://hwlab.example.test/v1/web-performance",
+      webPerformancePayload: {
+        parseStatus: "parsed",
+        schemaVersion: "hwlab-web-performance-v2",
+        events: [{
+          kind: "workbench_ui_event",
+          eventType: "runtime_diagnostic",
+          module: "workbench-sync-replay",
+          diagnosticCode: "workbench_sync_replay_applied",
+          rootCause: "sse-gap",
+          eventCount: 1,
+          sinceOutboxSeq: 7,
+          syncCursorOutboxSeq: 12,
+          realtimeAuthority: "workbench-realtime-authority-v2",
+          entityFamilyCount: 1,
+          entityFamilies: "turn",
+          maxEntityVersion: 3,
+          projectionRevision: "rev_detail",
+          terminalSeal: true,
+          detailProjection: false
+        }]
+      }
+    })
+  ].join("\n") + "\n";
+
+  const report = await runObserveAnalyzer({ "samples.jsonl": samples, "network.jsonl": network });
+  const finding = report.findings.find((item: Record<string, unknown>) => item.id === "workbench-automatic-recovery-fanout-authority");
+
+  assert.equal(finding, undefined);
+  assert.equal(report.runtimeAlerts.webPerformanceRuntimeDiagnostics[0].syncCursorOutboxSeq, 12);
+}, 20_000);

scripts/src/platform-infra-observability/apply-status-scripts.ts

@@ -253,7 +253,7 @@ export function compactSpanList(value: unknown, limit: number): unknown[] {
     return {
       name: span.name ?? null,
       service: span.service ?? null,
-      attributes: compactRecord(attrs, ["failureKind", "terminalStatus", "status", "eventType", "idleMs", "waitingFor", "lastEventLabel", "http.route", "http.status_code", "http.response.status_code", "workbench.session_id", "workbench.trace_id", "workbench.turn_id", "workbench.read_model.route", "workbench.read_model.count", "workbench.read_model.family", "workbench.read_model.status", "workbench.read_model.reason", "hwlab.http.stage", "hwlab.http.phase", "hwlab.http.phase.outcome", "hwlab.live_builds.service_id", "hwlab.live_builds.service_kind", "hwlab.live_builds.external", "hwlab.live_builds.deploy_manifest_status", "hwlab.live_builds.artifact_catalog_status", "providerProfile", "model", "modelId", "providerModel", "code_agent.stage", "agent.chat.session_id", "agent.chat.provider_profile", "agent.chat.provider_profile_source", "agent.chat.model", "agent.chat.model_id", "agent.chat.provider_model", "defaultProviderProfile", "adapter", "adapterEnabled", "agentRunManagerHost", "agentRunRunnerNamespace", "agentRunSourceCommitPresent"]),
+      attributes: compactRecord(attrs, ["failureKind", "terminalStatus", "status", "eventType", "idleMs", "waitingFor", "lastEventLabel", "http.route", "http.status_code", "http.response.status_code", "workbench.session_id", "workbench.trace_id", "workbench.turn_id", "workbench.read_model.route", "workbench.read_model.count", "workbench.read_model.family", "workbench.read_model.status", "workbench.read_model.reason", "workbench.sync.contract_version", "workbench.sync.since_outbox_seq", "workbench.sync.cursor_outbox_seq", "workbench.sync.event_count", "workbench.sync.entity_family_count", "workbench.sync.entity_families", "workbench.sync.max_entity_version", "workbench.realtime.authority", "workbench.projection.revision", "workbench.terminal.seal", "workbench.detail.projection", "hwlab.http.stage", "hwlab.http.phase", "hwlab.http.phase.outcome", "hwlab.live_builds.service_id", "hwlab.live_builds.service_kind", "hwlab.live_builds.external", "hwlab.live_builds.deploy_manifest_status", "hwlab.live_builds.artifact_catalog_status", "providerProfile", "model", "modelId", "providerModel", "code_agent.stage", "agent.chat.session_id", "agent.chat.provider_profile", "agent.chat.provider_profile_source", "agent.chat.model", "agent.chat.model_id", "agent.chat.provider_model", "defaultProviderProfile", "adapter", "adapterEnabled", "agentRunManagerHost", "agentRunRunnerNamespace", "agentRunSourceCommitPresent"]),
     };
   });
 }

scripts/src/platform-infra-observability/manifest.ts

@@ -120,6 +120,17 @@ export function spanDetail(span: Record<string, unknown>, maxLength: number): st
     attrPart(attrs, "workbench.ui.resource_response_transfer_ms"),
     attrPart(attrs, "workbench.ui.resource_next_hop_protocol"),
     attrPart(attrs, "workbench.ui.resource_server_timing"),
+    attrPart(attrs, "workbench.sync.contract_version"),
+    attrPart(attrs, "workbench.sync.since_outbox_seq"),
+    attrPart(attrs, "workbench.sync.cursor_outbox_seq"),
+    attrPart(attrs, "workbench.sync.event_count"),
+    attrPart(attrs, "workbench.sync.entity_family_count"),
+    attrPart(attrs, "workbench.sync.entity_families"),
+    attrPart(attrs, "workbench.sync.max_entity_version"),
+    attrPart(attrs, "workbench.realtime.authority"),
+    attrPart(attrs, "workbench.projection.revision"),
+    attrPart(attrs, "workbench.terminal.seal"),
+    attrPart(attrs, "workbench.detail.projection"),
     attrPart(attrs, "hwlab.http.stage"),
     attrPart(attrs, "hwlab.http.phase"),
     attrPart(attrs, "hwlab.http.phase.outcome"),

scripts/src/platform-infra-observability/render.ts

@@ -233,6 +233,17 @@ const knownTraceAttributeKeys = new Set([
   "opencode.provider.sse.done_lines",
   "opencode.provider.sse.json_errors",
   "opencode.provider.sse.reasoning_only_choices_dropped",
+  "workbench.sync.contract_version",
+  "workbench.sync.since_outbox_seq",
+  "workbench.sync.cursor_outbox_seq",
+  "workbench.sync.event_count",
+  "workbench.sync.entity_family_count",
+  "workbench.sync.entity_families",
+  "workbench.sync.max_entity_version",
+  "workbench.realtime.authority",
+  "workbench.projection.revision",
+  "workbench.terminal.seal",
+  "workbench.detail.projection",
 ]);
 
 function isKnownTraceAttributeKey(value: string): boolean {
@@ -432,7 +443,7 @@ export function renderTraceTable(input: {
     formatTable(["NAME", "TRACE", "SESSION", "TURN", "STATUS", "EVENT", "PROJECTED", "SOURCE", "MESSAGE"], projectionRows.length > 0 ? projectionRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
     "",
     "Projection sync cursors:",
-    formatTable(["SERVICE", "TRACE", "SESSION", "AFTER", "END", "EVENTS", "RAW", "MAX", "LAST", "TERM"], projectionSyncRows.length > 0 ? projectionSyncRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
+    formatTable(["SERVICE", "TRACE", "SESSION", "AFTER", "END", "EVENTS", "RAW", "MAX", "LAST", "TERM", "AUTH"], projectionSyncRows.length > 0 ? projectionSyncRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]),
     "",
     "Span name counts:",
     formatTable(["NAME", "COUNT"], countRows.length > 0 ? countRows : [["-", "-"]]),
@@ -599,19 +610,26 @@ function traceProjectionRows(spans: Record<string, unknown>[]): string[][] {
 }
 
 function traceProjectionSyncRows(spans: Record<string, unknown>[]): string[][] {
-  const rows = spans.filter((span) => textValue(span.name) === "projection_sync").map((span) => {
+  const rows = spans.filter((span) => {
+    const name = textValue(span.name);
+    const attrs = asPlainRecord(span.attributes);
+    return name === "projection_sync"
+      || name === "workbench.sync.replay"
+      || spanColumnAttr(attrs, ["workbench.sync.contract_version", "workbench.sync.cursor_outbox_seq", "workbench.sync.event_count", "workbench.realtime.authority"]) !== "-";
+  }).map((span) => {
     const attrs = asPlainRecord(span.attributes);
     return [
       shortenEnd(textValue(span.service), 18),
-      shortenMiddle(spanColumnAttr(attrs, ["traceId"]), 18),
-      shortenMiddle(spanColumnAttr(attrs, ["sessionId"]), 18),
-      shortenEnd(spanColumnAttr(attrs, ["afterSeq"]), 8),
-      shortenEnd(spanColumnAttr(attrs, ["endSeq"]), 8),
-      shortenEnd(spanColumnAttr(attrs, ["eventCount"]), 8),
+      shortenMiddle(spanColumnAttr(attrs, ["traceId", "workbench.trace_id"]), 18),
+      shortenMiddle(spanColumnAttr(attrs, ["sessionId", "workbench.session_id"]), 18),
+      shortenEnd(spanColumnAttr(attrs, ["afterSeq", "workbench.sync.since_outbox_seq"]), 8),
+      shortenEnd(spanColumnAttr(attrs, ["endSeq", "workbench.sync.cursor_outbox_seq"]), 8),
+      shortenEnd(spanColumnAttr(attrs, ["eventCount", "workbench.sync.event_count"]), 8),
       shortenEnd(spanColumnAttr(attrs, ["rawEventCount"]), 8),
-      shortenEnd(spanColumnAttr(attrs, ["maxSeq"]), 8),
+      shortenEnd(spanColumnAttr(attrs, ["maxSeq", "workbench.sync.max_entity_version"]), 8),
       shortenEnd(spanColumnAttr(attrs, ["traceLastSeq"]), 8),
-      shortenEnd(spanColumnAttr(attrs, ["terminalFromEvents", "terminalFromRawEvents", "terminalStatus"]), 12),
+      shortenEnd(spanColumnAttr(attrs, ["terminalFromEvents", "terminalFromRawEvents", "terminalStatus", "workbench.terminal.seal"]), 12),
+      shortenEnd(spanColumnAttr(attrs, ["workbench.realtime.authority", "workbench.sync.contract_version", "workbench.detail.projection"]), 18),
     ];
   });
   return dedupeRows(rows).slice(0, 16);