fix: detect workbench recovery authority fanout

2026-07-08 19:32:56 +02:00
parent bd4722c9a8
commit 2b0778f21f
9 changed files with 509 additions and 13 deletions
@@ -6,9 +6,11 @@ export function nodeWebObserveAnalyzerWorkbenchTriadSource(): string {
  const canarySessionIds = sessionInvariantCanarySessionIds(control);
  const terminalTraceMissing = detectWorkbenchTerminalTraceMissing(samples, canarySessionIds);
  const terminalApiDomLag = detectWorkbenchTerminalApiDomLag(samples, network, canarySessionIds);
+  const recoveryAuthority = detectWorkbenchRecoveryAuthority(network, control);
  return {
    terminalTraceMissing,
    terminalApiDomLag,
+    recoveryAuthority,
    valuesRedacted: true
  };
 }
@@ -260,6 +262,268 @@ function isWorkbenchPathSample(sample) {
  return /\/workbench(?:\/|$)/u.test(String(sample?.path || sample?.url || ""));
 }

+function detectWorkbenchRecoveryAuthority(network, control = []) {
+  void control;
+  const diagnostics = workbenchRecoveryDiagnosticsFromNetwork(network);
+  const endpointReads = workbenchRecoveryEndpointReads(network);
+  const syncEvidence = workbenchSyncEvidenceFromNetwork(network, diagnostics);
+  const forbiddenAutomaticRecovery = diagnostics
+    .filter(isForbiddenWorkbenchRecoveryDiagnostic)
+    .map(compactWorkbenchRecoveryDiagnostic);
+  const legacyFanoutWindows = workbenchLegacyRecoveryFanoutWindows(endpointReads, diagnostics, syncEvidence);
+  const observabilityGaps = [];
+  if (diagnostics.length > 0 && syncEvidence.length === 0) {
+    observabilityGaps.push({
+      id: "workbench-sync-replay-evidence-missing",
+      severity: "amber",
+      summary: "Workbench recovery diagnostics were captured but no sync replay diagnostic or /v1/workbench/sync network row was present in the artifact window",
+      diagnosticCount: diagnostics.length,
+      valuesRedacted: true
+    });
+  }
+  return {
+    summary: {
+      diagnosticCount: diagnostics.length,
+      forbiddenAutomaticRecoveryCount: forbiddenAutomaticRecovery.length,
+      legacyFanoutWindowCount: legacyFanoutWindows.length,
+      syncReplayEvidenceCount: syncEvidence.length,
+      syncReplayAppliedCount: syncEvidence.filter((item) => item.code === "workbench_sync_replay_applied").length,
+      syncNetworkReadCount: endpointReads.filter((item) => item.kind === "workbench-sync-replay").length,
+      oldEndpointReadCount: endpointReads.filter((item) => item.legacyRead === true).length,
+      observabilityGapCount: observabilityGaps.length,
+      valuesRedacted: true
+    },
+    forbiddenAutomaticRecovery: forbiddenAutomaticRecovery.slice(0, 20),
+    legacyFanoutWindows: legacyFanoutWindows.slice(0, 20),
+    syncEvidence: syncEvidence.slice(-20),
+    diagnostics: diagnostics.slice(-20).map(compactWorkbenchRecoveryDiagnostic),
+    observabilityGaps,
+    valuesRedacted: true
+  };
+}
+
+function workbenchRecoveryDiagnosticsFromNetwork(network) {
+  const rows = [];
+  for (const item of Array.isArray(network) ? network : []) {
+    const payload = item?.webPerformancePayload && typeof item.webPerformancePayload === "object" ? item.webPerformancePayload : null;
+    if (!payload || payload.parseStatus !== "parsed" || payload.schemaVersion !== "hwlab-web-performance-v2") continue;
+    const events = Array.isArray(payload.events) ? payload.events : [];
+    for (const event of events) {
+      if (!event || typeof event !== "object") continue;
+      const eventType = firstString(event.eventType, event.type, event.kind);
+      const diagnosticCode = firstString(event.diagnosticCode, event.code, event.errorName);
+      const module = firstString(event.module);
+      const recoveryAction = firstString(event.recoveryAction);
+      const rootCause = firstString(event.rootCause);
+      const source = firstString(event.source);
+      const looksRelevant = eventType === "runtime_diagnostic"
+        || Boolean(diagnosticCode && /workbench|terminal|recovery|sync|hydrate|fanout/iu.test(diagnosticCode))
+        || Boolean(module && /workbench/iu.test(module))
+        || Boolean(recoveryAction);
+      if (!looksRelevant) continue;
+      rows.push({
+        ts: item.ts ?? event.ts ?? null,
+        tsMs: Date.parse(item.ts ?? event.ts ?? ""),
+        pageRole: item.pageRole ?? null,
+        pageId: item.pageId ?? null,
+        urlPath: urlPath(item.url),
+        eventType,
+        module,
+        diagnosticCode,
+        reason: rootCause ?? firstString(event.reason),
+        recoveryAction,
+        source,
+        contractVersion: firstString(event.contractVersion),
+        realtimeAuthority: firstString(event.realtimeAuthority),
+        eventCount: numberOrNull(event.eventCount),
+        sinceOutboxSeq: numberOrNull(event.sinceOutboxSeq),
+        syncCursorOutboxSeq: numberOrNull(event.syncCursorOutboxSeq),
+        entityFamilyCount: numberOrNull(event.entityFamilyCount),
+        entityFamilies: firstString(event.entityFamilies),
+        maxEntityVersion: numberOrNull(event.maxEntityVersion),
+        projectionRevision: firstString(event.projectionRevision),
+        terminalSeal: typeof event.terminalSeal === "boolean" ? event.terminalSeal : null,
+        detailProjection: typeof event.detailProjection === "boolean" ? event.detailProjection : null,
+        valuesRedacted: true
+      });
+    }
+  }
+  return rows.sort((left, right) => String(left.ts || "").localeCompare(String(right.ts || "")));
+}
+
+function workbenchSyncEvidenceFromNetwork(network, diagnostics) {
+  const rows = [];
+  for (const item of Array.isArray(network) ? network : []) {
+    if (item?.observerInitiated === true) continue;
+    const route = workbenchRecoveryNetworkRoute(item);
+    const summary = objectValue(item?.bodySummary);
+    if (route.kind !== "workbench-sync-replay" && String(summary?.pathKind || "") !== "workbench-sync-replay") continue;
+    rows.push({
+      ts: item.ts ?? null,
+      tsMs: Date.parse(item.ts || ""),
+      source: "network",
+      method: String(item.method || "GET").toUpperCase(),
+      status: item.status ?? null,
+      path: route.path,
+      code: "workbench_sync_network_read",
+      sinceOutboxSeq: numberOrNull(route.since ?? summary?.sinceOutboxSeq),
+      syncCursorOutboxSeq: numberOrNull(summary?.cursorOutboxSeq ?? summary?.syncCursorOutboxSeq),
+      eventCount: numberOrNull(summary?.eventCount ?? summary?.traceEventLikeCount),
+      contractVersion: firstString(summary?.contractVersion),
+      realtimeAuthority: firstString(summary?.realtimeAuthority),
+      entityFamilyCount: numberOrNull(summary?.entityFamilyCount),
+      maxEntityVersion: numberOrNull(summary?.maxEntityVersion),
+      terminalSeal: typeof summary?.terminalSeal === "boolean" ? summary.terminalSeal : null,
+      valuesRedacted: true
+    });
+  }
+  for (const item of Array.isArray(diagnostics) ? diagnostics : []) {
+    if (item.diagnosticCode !== "workbench_sync_replay_applied" && item.module !== "workbench-sync-replay") continue;
+    rows.push({
+      ts: item.ts ?? null,
+      tsMs: item.tsMs,
+      source: "web-performance-runtime-diagnostic",
+      code: item.diagnosticCode || "workbench_sync_replay_applied",
+      reason: item.reason ?? null,
+      eventCount: item.eventCount,
+      sinceOutboxSeq: item.sinceOutboxSeq,
+      syncCursorOutboxSeq: item.syncCursorOutboxSeq,
+      contractVersion: item.contractVersion,
+      realtimeAuthority: item.realtimeAuthority,
+      entityFamilyCount: item.entityFamilyCount,
+      entityFamilies: item.entityFamilies,
+      maxEntityVersion: item.maxEntityVersion,
+      projectionRevision: item.projectionRevision,
+      terminalSeal: item.terminalSeal,
+      detailProjection: item.detailProjection,
+      valuesRedacted: true
+    });
+  }
+  return rows.sort((left, right) => String(left.ts || "").localeCompare(String(right.ts || "")));
+}
+
+function workbenchRecoveryEndpointReads(network) {
+  const rows = [];
+  for (const item of Array.isArray(network) ? network : []) {
+    if (item?.observerInitiated === true) continue;
+    if (item?.type !== "request" && item?.type !== "response" && item?.type !== "requestfailed") continue;
+    const route = workbenchRecoveryNetworkRoute(item);
+    if (!route.kind) continue;
+    rows.push({
+      ts: item.ts ?? null,
+      tsMs: Date.parse(item.ts || ""),
+      type: item.type ?? null,
+      method: String(item.method || "GET").toUpperCase(),
+      status: item.status ?? null,
+      kind: route.kind,
+      legacyRead: route.legacyRead,
+      path: route.path,
+      traceId: route.traceId,
+      sessionIdPrefix: route.sessionId ? String(route.sessionId).slice(0, 12) : null,
+      valuesRedacted: true
+    });
+  }
+  return rows.sort((left, right) => Number(left.tsMs || 0) - Number(right.tsMs || 0));
+}
+
+function workbenchRecoveryNetworkRoute(item) {
+  const rawUrl = item?.url || item?.requestUrl || "";
+  const parsed = parseApiDomLagUrl(rawUrl);
+  const rawPath = String(parsed.rawPath || parsed.path || "");
+  const path = String(parsed.path || rawPath);
+  const sessionId = workbenchRecoveryQueryParam(rawUrl, "sessionId") || workbenchRecoveryQueryParam(rawUrl, "includeSessionId") || parsed.sessionId || null;
+  const traceId = workbenchRecoveryQueryParam(rawUrl, "traceId") || parsed.traceId || null;
+  const since = workbenchRecoveryQueryParam(rawUrl, "since");
+  if (rawPath === "/v1/workbench/sync" || path === "/v1/workbench/sync") return { kind: "workbench-sync-replay", legacyRead: false, path: "/v1/workbench/sync", traceId, sessionId, since, valuesRedacted: true };
+  let match = rawPath.match(/^\/v1\/workbench\/turns\/([^/]+)$/u);
+  if (match) return { kind: "workbench-turn", legacyRead: true, path: "/v1/workbench/turns/:id", traceId: match[1], sessionId, since, valuesRedacted: true };
+  match = rawPath.match(/^\/v1\/workbench\/sessions\/([^/]+)\/messages$/u);
+  if (match) return { kind: "workbench-session-messages", legacyRead: true, path: "/v1/workbench/sessions/:id/messages", traceId, sessionId: match[1], since, valuesRedacted: true };
+  match = rawPath.match(/^\/v1\/workbench\/traces\/([^/]+)\/events$/u);
+  if (match) return { kind: "workbench-trace-events", legacyRead: true, path: "/v1/workbench/traces/:id/events", traceId: match[1], sessionId, since, valuesRedacted: true };
+  return { kind: null, legacyRead: false, path, traceId: null, sessionId: null, since, valuesRedacted: true };
+}
+
+function workbenchRecoveryQueryParam(value, key) {
+  try {
+    const parsed = new URL(String(value || "http://invalid.local/"), "http://invalid.local/");
+    return parsed.searchParams.get(key) || null;
+  } catch {
+    return null;
+  }
+}
+
+function workbenchLegacyRecoveryFanoutWindows(endpointReads, diagnostics, syncEvidence) {
+  const rows = [];
+  const recoveryDiagnostics = (Array.isArray(diagnostics) ? diagnostics : []).filter(workbenchRecoveryDiagnosticLooksAutomatic);
+  for (const diagnostic of recoveryDiagnostics) {
+    const tsMs = Number(diagnostic.tsMs);
+    if (!Number.isFinite(tsMs)) continue;
+    const nearby = (Array.isArray(endpointReads) ? endpointReads : [])
+      .filter((item) => item.legacyRead === true && Number.isFinite(Number(item.tsMs)) && Math.abs(Number(item.tsMs) - tsMs) <= 15_000);
+    const kinds = uniqueSorted(nearby.map((item) => item.kind).filter(Boolean));
+    if (kinds.length < 2) continue;
+    const syncNearby = (Array.isArray(syncEvidence) ? syncEvidence : []).some((item) => Number.isFinite(Number(item.tsMs)) && Math.abs(Number(item.tsMs) - tsMs) <= 20_000);
+    rows.push({
+      ts: diagnostic.ts,
+      diagnostic: compactWorkbenchRecoveryDiagnostic(diagnostic),
+      oldEndpointKinds: kinds,
+      oldEndpointReadCount: nearby.length,
+      syncEvidenceNearby: syncNearby,
+      endpointTail: nearby.slice(-8).map((item) => ({
+        ts: item.ts,
+        type: item.type,
+        kind: item.kind,
+        method: item.method,
+        status: item.status,
+        path: item.path,
+        valuesRedacted: true
+      })),
+      valuesRedacted: true
+    });
+  }
+  return rows;
+}
+
+function isForbiddenWorkbenchRecoveryDiagnostic(item) {
+  const action = [item?.recoveryAction, item?.diagnosticCode, item?.source].filter(Boolean).join(" ").toLowerCase();
+  if (!action) return false;
+  if (/hydrate[-_]trace[-_]events|refresh[-_]session[-_]messages|refresh[-_]turn[-_]status/u.test(action)) return true;
+  if (item?.recoveryAction && /active[-_]rest[-_]gap|trace[-_]hydration|terminal[-_]rest|session[-_]detail/u.test(action)) return true;
+  return /legacy[-_]?fanout|automatic[-_]?recovery[-_]?fanout|direct[-_]?hydrate/u.test(action);
+}
+
+function workbenchRecoveryDiagnosticLooksAutomatic(item) {
+  const text = [item?.module, item?.diagnosticCode, item?.reason, item?.recoveryAction, item?.source].filter(Boolean).join(" ").toLowerCase();
+  return /recovery|sync[-_]?replay|stream[-_]?transport|sse[-_]?gap|hydrate[-_]?trace|active[-_]?rest[-_]?gap|refresh[-_]?session/u.test(text);
+}
+
+function compactWorkbenchRecoveryDiagnostic(item) {
+  if (!item) return null;
+  return {
+    ts: item.ts ?? null,
+    pageRole: item.pageRole ?? null,
+    pageId: item.pageId ?? null,
+    module: item.module ?? null,
+    diagnosticCode: item.diagnosticCode ?? null,
+    reason: item.reason ?? null,
+    recoveryAction: item.recoveryAction ?? null,
+    source: item.source ?? null,
+    eventCount: item.eventCount ?? null,
+    sinceOutboxSeq: item.sinceOutboxSeq ?? null,
+    syncCursorOutboxSeq: item.syncCursorOutboxSeq ?? null,
+    contractVersion: item.contractVersion ?? null,
+    realtimeAuthority: item.realtimeAuthority ?? null,
+    entityFamilyCount: item.entityFamilyCount ?? null,
+    entityFamilies: item.entityFamilies ?? null,
+    maxEntityVersion: item.maxEntityVersion ?? null,
+    projectionRevision: item.projectionRevision ?? null,
+    terminalSeal: item.terminalSeal ?? null,
+    detailProjection: item.detailProjection ?? null,
+    valuesRedacted: true
+  };
+}
+
 function workbenchTerminalTraceIdsFromDom(sample) {
  const ids = new Set();
  for (const groupName of ["turns", "messages"]) {