From 2b0778f21f46a61e451ff0ee084ba116d7b113a1 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 8 Jul 2026 19:32:56 +0200 Subject: [PATCH] fix: detect workbench recovery authority fanout --- ...de-web-observe-analyzer-findings-source.ts | 27 ++ ...observe-analyzer-request-runtime-source.ts | 24 +- .../hwlab-node-web-observe-analyzer-source.ts | 1 + ...observe-analyzer-workbench-triad-source.ts | 264 ++++++++++++++++++ scripts/src/hwlab-node-web-observe-collect.ts | 34 ++- .../web-observe-analyzer-triad.test.ts | 123 ++++++++ .../apply-status-scripts.ts | 2 +- .../platform-infra-observability/manifest.ts | 11 + .../platform-infra-observability/render.ts | 36 ++- 9 files changed, 509 insertions(+), 13 deletions(-) diff --git a/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts index 60c44bbd..2dba9d12 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-findings-source.ts @@ -152,6 +152,33 @@ export function nodeWebObserveAnalyzerFindingsSource(): string { rootCauseConfidence: "high", valuesRedacted: true }); + const recoveryAuthority = workbenchInPlaceProjectionLag.recoveryAuthority || {}; + const recoveryAuthoritySummary = recoveryAuthority.summary || {}; + const forbiddenRecoveryCount = Number(recoveryAuthoritySummary.forbiddenAutomaticRecoveryCount ?? 0); + const legacyFanoutWindowCount = Number(recoveryAuthoritySummary.legacyFanoutWindowCount ?? 0); + if (forbiddenRecoveryCount > 0 || legacyFanoutWindowCount > 0) findings.push({ + id: "workbench-automatic-recovery-fanout-authority", + severity: "red", + summary: "Workbench automatic recovery used direct trace/session/turn fan-out instead of sync replay authority", + count: forbiddenRecoveryCount + legacyFanoutWindowCount, + forbiddenAutomaticRecoveryCount: forbiddenRecoveryCount, + legacyFanoutWindowCount, + syncReplayEvidenceCount: Number(recoveryAuthoritySummary.syncReplayEvidenceCount ?? 0), + syncReplayAppliedCount: Number(recoveryAuthoritySummary.syncReplayAppliedCount ?? 0), + recoveryAuthority: { + summary: recoveryAuthoritySummary, + forbiddenAutomaticRecovery: Array.isArray(recoveryAuthority.forbiddenAutomaticRecovery) ? recoveryAuthority.forbiddenAutomaticRecovery.slice(0, 12) : [], + legacyFanoutWindows: Array.isArray(recoveryAuthority.legacyFanoutWindows) ? recoveryAuthority.legacyFanoutWindows.slice(0, 12) : [], + syncEvidence: Array.isArray(recoveryAuthority.syncEvidence) ? recoveryAuthority.syncEvidence.slice(-12) : [], + observabilityGaps: Array.isArray(recoveryAuthority.observabilityGaps) ? recoveryAuthority.observabilityGaps.slice(0, 8) : [], + valuesRedacted: true + }, + sourceOfTruth: "Workbench sync replay / entity version / terminal seal authority; explicit detail/history reads are allowed but automatic recovery must not hydrate direct REST fan-out", + rootCause: "workbench_automatic_recovery_legacy_fanout", + rootCauseStatus: "confirmed-from-web-performance-runtime-diagnostics-and-network-artifacts", + rootCauseConfidence: "high", + valuesRedacted: true + }); const turnStateTriad = sampleMetrics?.workbenchTurnStateTriad || {}; const turnStateTriadSummary = turnStateTriad.summary || {}; const turnStateTriadRows = [ diff --git a/scripts/src/hwlab-node-web-observe-analyzer-request-runtime-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-request-runtime-source.ts index efb28c48..4e739c00 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-request-runtime-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-request-runtime-source.ts @@ -701,7 +701,7 @@ function extractWebPerformanceRuntimeDiagnostics(network, promptTimes) { for (const event of events) { if (!event || typeof event !== "object") continue; const diagnosticCode = limitText(event.diagnosticCode || event.code || "", 120); - const reason = limitText(event.reason || "", 120); + const reason = limitText(event.rootCause || event.reason || "", 120); const eventType = limitText(event.eventType || event.type || event.kind || "", 120); if (!diagnosticCode && !reason && !eventType) continue; rows.push({ @@ -721,6 +721,17 @@ function extractWebPerformanceRuntimeDiagnostics(network, promptTimes) { diagnosticCode, reason, module: limitText(event.module || "", 120), + recoveryAction: limitText(event.recoveryAction || "", 160), + contractVersion: limitText(event.contractVersion || "", 80), + realtimeAuthority: limitText(event.realtimeAuthority || "", 120), + sinceOutboxSeq: numberOrNull(event.sinceOutboxSeq), + syncCursorOutboxSeq: numberOrNull(event.syncCursorOutboxSeq), + entityFamilyCount: numberOrNull(event.entityFamilyCount), + entityFamilies: limitText(event.entityFamilies || "", 160), + maxEntityVersion: numberOrNull(event.maxEntityVersion), + projectionRevision: limitText(event.projectionRevision || "", 120), + terminalSeal: typeof event.terminalSeal === "boolean" ? event.terminalSeal : null, + detailProjection: typeof event.detailProjection === "boolean" ? event.detailProjection : null, traceId: event.traceId ?? null, sessionIdHash: event.sessionIdHash ?? null, eventIdHash: event.eventIdHash ?? null, @@ -823,6 +834,17 @@ function groupWebPerformanceRuntimeDiagnostics(events) { ts: item.ts || null, traceId: item.traceId || null, eventCount: item.eventCount ?? null, + recoveryAction: item.recoveryAction ?? null, + sinceOutboxSeq: item.sinceOutboxSeq ?? null, + syncCursorOutboxSeq: item.syncCursorOutboxSeq ?? null, + contractVersion: item.contractVersion ?? null, + realtimeAuthority: item.realtimeAuthority ?? null, + entityFamilyCount: item.entityFamilyCount ?? null, + entityFamilies: item.entityFamilies ?? null, + maxEntityVersion: item.maxEntityVersion ?? null, + projectionRevision: item.projectionRevision ?? null, + terminalSeal: item.terminalSeal ?? null, + detailProjection: item.detailProjection ?? null, deliveredCount: item.deliveredCount ?? null, chunkCount: item.chunkCount ?? null, flushDurationMs: item.flushDurationMs ?? null, diff --git a/scripts/src/hwlab-node-web-observe-analyzer-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-source.ts index 5339f43c..2135a58e 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-source.ts @@ -469,6 +469,7 @@ function prioritizeFindings(findings) { if (id.startsWith("frontend-long") || id.startsWith("frontend-event-loop-gap") || id === "frontend-cpu-profile-hotspots") return 0; if (id === "page-performance-slow-same-origin-api") return 0; if (id === "session-rail-title-fallback-majority") return 0.5; + if (id === "workbench-automatic-recovery-fanout-authority") return 0.52; if (id === "workbench-turn-state-triad-inconsistent") return 0.55; if (id.startsWith("workbench-terminal-")) return 0.6; if (id.startsWith("code-agent-card-")) return 0.8; diff --git a/scripts/src/hwlab-node-web-observe-analyzer-workbench-triad-source.ts b/scripts/src/hwlab-node-web-observe-analyzer-workbench-triad-source.ts index c0386d6a..76edb282 100644 --- a/scripts/src/hwlab-node-web-observe-analyzer-workbench-triad-source.ts +++ b/scripts/src/hwlab-node-web-observe-analyzer-workbench-triad-source.ts @@ -6,9 +6,11 @@ export function nodeWebObserveAnalyzerWorkbenchTriadSource(): string { const canarySessionIds = sessionInvariantCanarySessionIds(control); const terminalTraceMissing = detectWorkbenchTerminalTraceMissing(samples, canarySessionIds); const terminalApiDomLag = detectWorkbenchTerminalApiDomLag(samples, network, canarySessionIds); + const recoveryAuthority = detectWorkbenchRecoveryAuthority(network, control); return { terminalTraceMissing, terminalApiDomLag, + recoveryAuthority, valuesRedacted: true }; } @@ -260,6 +262,268 @@ function isWorkbenchPathSample(sample) { return /\/workbench(?:\/|$)/u.test(String(sample?.path || sample?.url || "")); } +function detectWorkbenchRecoveryAuthority(network, control = []) { + void control; + const diagnostics = workbenchRecoveryDiagnosticsFromNetwork(network); + const endpointReads = workbenchRecoveryEndpointReads(network); + const syncEvidence = workbenchSyncEvidenceFromNetwork(network, diagnostics); + const forbiddenAutomaticRecovery = diagnostics + .filter(isForbiddenWorkbenchRecoveryDiagnostic) + .map(compactWorkbenchRecoveryDiagnostic); + const legacyFanoutWindows = workbenchLegacyRecoveryFanoutWindows(endpointReads, diagnostics, syncEvidence); + const observabilityGaps = []; + if (diagnostics.length > 0 && syncEvidence.length === 0) { + observabilityGaps.push({ + id: "workbench-sync-replay-evidence-missing", + severity: "amber", + summary: "Workbench recovery diagnostics were captured but no sync replay diagnostic or /v1/workbench/sync network row was present in the artifact window", + diagnosticCount: diagnostics.length, + valuesRedacted: true + }); + } + return { + summary: { + diagnosticCount: diagnostics.length, + forbiddenAutomaticRecoveryCount: forbiddenAutomaticRecovery.length, + legacyFanoutWindowCount: legacyFanoutWindows.length, + syncReplayEvidenceCount: syncEvidence.length, + syncReplayAppliedCount: syncEvidence.filter((item) => item.code === "workbench_sync_replay_applied").length, + syncNetworkReadCount: endpointReads.filter((item) => item.kind === "workbench-sync-replay").length, + oldEndpointReadCount: endpointReads.filter((item) => item.legacyRead === true).length, + observabilityGapCount: observabilityGaps.length, + valuesRedacted: true + }, + forbiddenAutomaticRecovery: forbiddenAutomaticRecovery.slice(0, 20), + legacyFanoutWindows: legacyFanoutWindows.slice(0, 20), + syncEvidence: syncEvidence.slice(-20), + diagnostics: diagnostics.slice(-20).map(compactWorkbenchRecoveryDiagnostic), + observabilityGaps, + valuesRedacted: true + }; +} + +function workbenchRecoveryDiagnosticsFromNetwork(network) { + const rows = []; + for (const item of Array.isArray(network) ? network : []) { + const payload = item?.webPerformancePayload && typeof item.webPerformancePayload === "object" ? item.webPerformancePayload : null; + if (!payload || payload.parseStatus !== "parsed" || payload.schemaVersion !== "hwlab-web-performance-v2") continue; + const events = Array.isArray(payload.events) ? payload.events : []; + for (const event of events) { + if (!event || typeof event !== "object") continue; + const eventType = firstString(event.eventType, event.type, event.kind); + const diagnosticCode = firstString(event.diagnosticCode, event.code, event.errorName); + const module = firstString(event.module); + const recoveryAction = firstString(event.recoveryAction); + const rootCause = firstString(event.rootCause); + const source = firstString(event.source); + const looksRelevant = eventType === "runtime_diagnostic" + || Boolean(diagnosticCode && /workbench|terminal|recovery|sync|hydrate|fanout/iu.test(diagnosticCode)) + || Boolean(module && /workbench/iu.test(module)) + || Boolean(recoveryAction); + if (!looksRelevant) continue; + rows.push({ + ts: item.ts ?? event.ts ?? null, + tsMs: Date.parse(item.ts ?? event.ts ?? ""), + pageRole: item.pageRole ?? null, + pageId: item.pageId ?? null, + urlPath: urlPath(item.url), + eventType, + module, + diagnosticCode, + reason: rootCause ?? firstString(event.reason), + recoveryAction, + source, + contractVersion: firstString(event.contractVersion), + realtimeAuthority: firstString(event.realtimeAuthority), + eventCount: numberOrNull(event.eventCount), + sinceOutboxSeq: numberOrNull(event.sinceOutboxSeq), + syncCursorOutboxSeq: numberOrNull(event.syncCursorOutboxSeq), + entityFamilyCount: numberOrNull(event.entityFamilyCount), + entityFamilies: firstString(event.entityFamilies), + maxEntityVersion: numberOrNull(event.maxEntityVersion), + projectionRevision: firstString(event.projectionRevision), + terminalSeal: typeof event.terminalSeal === "boolean" ? event.terminalSeal : null, + detailProjection: typeof event.detailProjection === "boolean" ? event.detailProjection : null, + valuesRedacted: true + }); + } + } + return rows.sort((left, right) => String(left.ts || "").localeCompare(String(right.ts || ""))); +} + +function workbenchSyncEvidenceFromNetwork(network, diagnostics) { + const rows = []; + for (const item of Array.isArray(network) ? network : []) { + if (item?.observerInitiated === true) continue; + const route = workbenchRecoveryNetworkRoute(item); + const summary = objectValue(item?.bodySummary); + if (route.kind !== "workbench-sync-replay" && String(summary?.pathKind || "") !== "workbench-sync-replay") continue; + rows.push({ + ts: item.ts ?? null, + tsMs: Date.parse(item.ts || ""), + source: "network", + method: String(item.method || "GET").toUpperCase(), + status: item.status ?? null, + path: route.path, + code: "workbench_sync_network_read", + sinceOutboxSeq: numberOrNull(route.since ?? summary?.sinceOutboxSeq), + syncCursorOutboxSeq: numberOrNull(summary?.cursorOutboxSeq ?? summary?.syncCursorOutboxSeq), + eventCount: numberOrNull(summary?.eventCount ?? summary?.traceEventLikeCount), + contractVersion: firstString(summary?.contractVersion), + realtimeAuthority: firstString(summary?.realtimeAuthority), + entityFamilyCount: numberOrNull(summary?.entityFamilyCount), + maxEntityVersion: numberOrNull(summary?.maxEntityVersion), + terminalSeal: typeof summary?.terminalSeal === "boolean" ? summary.terminalSeal : null, + valuesRedacted: true + }); + } + for (const item of Array.isArray(diagnostics) ? diagnostics : []) { + if (item.diagnosticCode !== "workbench_sync_replay_applied" && item.module !== "workbench-sync-replay") continue; + rows.push({ + ts: item.ts ?? null, + tsMs: item.tsMs, + source: "web-performance-runtime-diagnostic", + code: item.diagnosticCode || "workbench_sync_replay_applied", + reason: item.reason ?? null, + eventCount: item.eventCount, + sinceOutboxSeq: item.sinceOutboxSeq, + syncCursorOutboxSeq: item.syncCursorOutboxSeq, + contractVersion: item.contractVersion, + realtimeAuthority: item.realtimeAuthority, + entityFamilyCount: item.entityFamilyCount, + entityFamilies: item.entityFamilies, + maxEntityVersion: item.maxEntityVersion, + projectionRevision: item.projectionRevision, + terminalSeal: item.terminalSeal, + detailProjection: item.detailProjection, + valuesRedacted: true + }); + } + return rows.sort((left, right) => String(left.ts || "").localeCompare(String(right.ts || ""))); +} + +function workbenchRecoveryEndpointReads(network) { + const rows = []; + for (const item of Array.isArray(network) ? network : []) { + if (item?.observerInitiated === true) continue; + if (item?.type !== "request" && item?.type !== "response" && item?.type !== "requestfailed") continue; + const route = workbenchRecoveryNetworkRoute(item); + if (!route.kind) continue; + rows.push({ + ts: item.ts ?? null, + tsMs: Date.parse(item.ts || ""), + type: item.type ?? null, + method: String(item.method || "GET").toUpperCase(), + status: item.status ?? null, + kind: route.kind, + legacyRead: route.legacyRead, + path: route.path, + traceId: route.traceId, + sessionIdPrefix: route.sessionId ? String(route.sessionId).slice(0, 12) : null, + valuesRedacted: true + }); + } + return rows.sort((left, right) => Number(left.tsMs || 0) - Number(right.tsMs || 0)); +} + +function workbenchRecoveryNetworkRoute(item) { + const rawUrl = item?.url || item?.requestUrl || ""; + const parsed = parseApiDomLagUrl(rawUrl); + const rawPath = String(parsed.rawPath || parsed.path || ""); + const path = String(parsed.path || rawPath); + const sessionId = workbenchRecoveryQueryParam(rawUrl, "sessionId") || workbenchRecoveryQueryParam(rawUrl, "includeSessionId") || parsed.sessionId || null; + const traceId = workbenchRecoveryQueryParam(rawUrl, "traceId") || parsed.traceId || null; + const since = workbenchRecoveryQueryParam(rawUrl, "since"); + if (rawPath === "/v1/workbench/sync" || path === "/v1/workbench/sync") return { kind: "workbench-sync-replay", legacyRead: false, path: "/v1/workbench/sync", traceId, sessionId, since, valuesRedacted: true }; + let match = rawPath.match(/^\/v1\/workbench\/turns\/([^/]+)$/u); + if (match) return { kind: "workbench-turn", legacyRead: true, path: "/v1/workbench/turns/:id", traceId: match[1], sessionId, since, valuesRedacted: true }; + match = rawPath.match(/^\/v1\/workbench\/sessions\/([^/]+)\/messages$/u); + if (match) return { kind: "workbench-session-messages", legacyRead: true, path: "/v1/workbench/sessions/:id/messages", traceId, sessionId: match[1], since, valuesRedacted: true }; + match = rawPath.match(/^\/v1\/workbench\/traces\/([^/]+)\/events$/u); + if (match) return { kind: "workbench-trace-events", legacyRead: true, path: "/v1/workbench/traces/:id/events", traceId: match[1], sessionId, since, valuesRedacted: true }; + return { kind: null, legacyRead: false, path, traceId: null, sessionId: null, since, valuesRedacted: true }; +} + +function workbenchRecoveryQueryParam(value, key) { + try { + const parsed = new URL(String(value || "http://invalid.local/"), "http://invalid.local/"); + return parsed.searchParams.get(key) || null; + } catch { + return null; + } +} + +function workbenchLegacyRecoveryFanoutWindows(endpointReads, diagnostics, syncEvidence) { + const rows = []; + const recoveryDiagnostics = (Array.isArray(diagnostics) ? diagnostics : []).filter(workbenchRecoveryDiagnosticLooksAutomatic); + for (const diagnostic of recoveryDiagnostics) { + const tsMs = Number(diagnostic.tsMs); + if (!Number.isFinite(tsMs)) continue; + const nearby = (Array.isArray(endpointReads) ? endpointReads : []) + .filter((item) => item.legacyRead === true && Number.isFinite(Number(item.tsMs)) && Math.abs(Number(item.tsMs) - tsMs) <= 15_000); + const kinds = uniqueSorted(nearby.map((item) => item.kind).filter(Boolean)); + if (kinds.length < 2) continue; + const syncNearby = (Array.isArray(syncEvidence) ? syncEvidence : []).some((item) => Number.isFinite(Number(item.tsMs)) && Math.abs(Number(item.tsMs) - tsMs) <= 20_000); + rows.push({ + ts: diagnostic.ts, + diagnostic: compactWorkbenchRecoveryDiagnostic(diagnostic), + oldEndpointKinds: kinds, + oldEndpointReadCount: nearby.length, + syncEvidenceNearby: syncNearby, + endpointTail: nearby.slice(-8).map((item) => ({ + ts: item.ts, + type: item.type, + kind: item.kind, + method: item.method, + status: item.status, + path: item.path, + valuesRedacted: true + })), + valuesRedacted: true + }); + } + return rows; +} + +function isForbiddenWorkbenchRecoveryDiagnostic(item) { + const action = [item?.recoveryAction, item?.diagnosticCode, item?.source].filter(Boolean).join(" ").toLowerCase(); + if (!action) return false; + if (/hydrate[-_]trace[-_]events|refresh[-_]session[-_]messages|refresh[-_]turn[-_]status/u.test(action)) return true; + if (item?.recoveryAction && /active[-_]rest[-_]gap|trace[-_]hydration|terminal[-_]rest|session[-_]detail/u.test(action)) return true; + return /legacy[-_]?fanout|automatic[-_]?recovery[-_]?fanout|direct[-_]?hydrate/u.test(action); +} + +function workbenchRecoveryDiagnosticLooksAutomatic(item) { + const text = [item?.module, item?.diagnosticCode, item?.reason, item?.recoveryAction, item?.source].filter(Boolean).join(" ").toLowerCase(); + return /recovery|sync[-_]?replay|stream[-_]?transport|sse[-_]?gap|hydrate[-_]?trace|active[-_]?rest[-_]?gap|refresh[-_]?session/u.test(text); +} + +function compactWorkbenchRecoveryDiagnostic(item) { + if (!item) return null; + return { + ts: item.ts ?? null, + pageRole: item.pageRole ?? null, + pageId: item.pageId ?? null, + module: item.module ?? null, + diagnosticCode: item.diagnosticCode ?? null, + reason: item.reason ?? null, + recoveryAction: item.recoveryAction ?? null, + source: item.source ?? null, + eventCount: item.eventCount ?? null, + sinceOutboxSeq: item.sinceOutboxSeq ?? null, + syncCursorOutboxSeq: item.syncCursorOutboxSeq ?? null, + contractVersion: item.contractVersion ?? null, + realtimeAuthority: item.realtimeAuthority ?? null, + entityFamilyCount: item.entityFamilyCount ?? null, + entityFamilies: item.entityFamilies ?? null, + maxEntityVersion: item.maxEntityVersion ?? null, + projectionRevision: item.projectionRevision ?? null, + terminalSeal: item.terminalSeal ?? null, + detailProjection: item.detailProjection ?? null, + valuesRedacted: true + }; +} + function workbenchTerminalTraceIdsFromDom(sample) { const ids = new Set(); for (const groupName of ["turns", "messages"]) { diff --git a/scripts/src/hwlab-node-web-observe-collect.ts b/scripts/src/hwlab-node-web-observe-collect.ts index 2aab0dd5..7a6a17f9 100644 --- a/scripts/src/hwlab-node-web-observe-collect.ts +++ b/scripts/src/hwlab-node-web-observe-collect.ts @@ -585,6 +585,31 @@ function compactFinding(item){ rootCause:item.rootCause||null, rootCauseStatus:item.rootCauseStatus||null, rootCauseConfidence:item.rootCauseConfidence||null, + sourceOfTruth:item.sourceOfTruth?short(item.sourceOfTruth,180):null, + valuesRedacted:true + }; +} +function recoveryAuthorityFromFindings(){ + const finding=findingById('workbench-automatic-recovery-fanout-authority'); + const authority=finding?.recoveryAuthority&&typeof finding.recoveryAuthority==='object'?finding.recoveryAuthority:null; + const summary=authority?.summary&&typeof authority.summary==='object'?authority.summary:null; + const syncEvidence=Array.isArray(authority?.syncEvidence)?authority.syncEvidence.slice(-6):[]; + const forbidden=Array.isArray(authority?.forbiddenAutomaticRecovery)?authority.forbiddenAutomaticRecovery.slice(0,6):[]; + const fanout=Array.isArray(authority?.legacyFanoutWindows)?authority.legacyFanoutWindows.slice(0,4):[]; + const gaps=Array.isArray(authority?.observabilityGaps)?authority.observabilityGaps.slice(0,4):[]; + return { + findingPresent:Boolean(finding), + forbiddenAutomaticRecoveryCount:Number(summary?.forbiddenAutomaticRecoveryCount??0), + legacyFanoutWindowCount:Number(summary?.legacyFanoutWindowCount??0), + syncReplayEvidenceCount:Number(summary?.syncReplayEvidenceCount??0), + syncReplayAppliedCount:Number(summary?.syncReplayAppliedCount??0), + syncNetworkReadCount:Number(summary?.syncNetworkReadCount??0), + oldEndpointReadCount:Number(summary?.oldEndpointReadCount??0), + observabilityGapCount:Number(summary?.observabilityGapCount??0), + latestSyncEvidence:syncEvidence.slice(-1)[0]||null, + forbiddenAutomaticRecovery:forbidden, + legacyFanoutWindows:fanout, + observabilityGaps:gaps, valuesRedacted:true }; } @@ -646,6 +671,7 @@ function workbenchTriadView(rows){ const traceId=requestedTraceId||requestedRow?.traceId||rows.slice().reverse().find((row)=>row.traceId)?.traceId||traceIdsFromSamples(samples).slice(-1)[0]||null; const row=rows.find((item)=>item.traceId===traceId)||requestedRow||rows.slice(-1)[0]||null; const findings=[ + compactFinding(findingById('workbench-automatic-recovery-fanout-authority')), compactFinding(findingById('workbench-terminal-api-dom-not-refreshed-in-place')), compactFinding(findingById('workbench-terminal-trace-not-hydrated-in-place')), compactFinding(findingById('workbench-turn-state-triad-inconsistent')), @@ -656,6 +682,7 @@ function workbenchTriadView(rows){ const networkEvidence=networkEvidenceForTrace(traceId); const dom=traceDomSnapshot(traceId); const freeze=browserFreezeEvidence(); + const recoveryAuthority=recoveryAuthorityFromFindings(); const statusGap={ traceId, turnSummaryStatus:row?.status??null, @@ -669,7 +696,7 @@ function workbenchTriadView(rows){ valuesRedacted:true }; const compactRow=row?{round:row.round??null,commandId:row.commandId??null,sessionId:row.sessionId??null,traceId:row.traceId??null,status:row.status??null,elapsedSeconds:row.elapsedSeconds??null,recentUpdateSeconds:row.recentUpdateSeconds??null,marks:row.marks??null,firstSeq:row.firstSeq??null,lastSeq:row.lastSeq??null,lastTs:row.lastTs??null,finalResponse:row.finalResponse??null,valuesRedacted:true}:null; - return {traceId,row:compactRow,dom,networkEvidence,freeze,findings, statusGap,renderedText:renderWorkbenchTriad({traceId,row:compactRow,dom,networkEvidence,freeze,findings,statusGap}),valuesRedacted:true}; + return {traceId,row:compactRow,dom,networkEvidence,freeze,recoveryAuthority,findings, statusGap,renderedText:renderWorkbenchTriad({traceId,row:compactRow,dom,networkEvidence,freeze,recoveryAuthority,findings,statusGap}),valuesRedacted:true}; } function renderWorkbenchTriad(input){ const lines=['Workbench triad offline diagnosis '+(manifest.jobId||'-'),'=======================================================']; @@ -680,6 +707,9 @@ function renderWorkbenchTriad(input){ lines.push('api evidence responses='+String(input.networkEvidence.responseCount)+' terminalResponses='+String(input.networkEvidence.terminalResponseCount)+' failed='+String(input.networkEvidence.failedCount)+' turnTerminal='+String(turnSummary.terminalEvidenceCount??'-')+' turnFinalBytes='+String(turnSummary.finalTextByteCount??'-')+' traceEvents='+String(traceSummary.traceEventLikeCount??'-')+' traceTerminal='+String(traceSummary.terminalEvidenceCount??'-')); if(input.networkEvidence.latestTurnResponse) lines.push('latest turn API '+String(input.networkEvidence.latestTurnResponse.ts||'-')+' status='+String(input.networkEvidence.latestTurnResponse.status??'-')+' sample='+String(input.networkEvidence.latestTurnResponse.sampleSeq??'-')+' path='+String(input.networkEvidence.latestTurnResponse.urlPath||'-')); if(input.networkEvidence.latestTraceEventsResponse) lines.push('latest trace events API '+String(input.networkEvidence.latestTraceEventsResponse.ts||'-')+' status='+String(input.networkEvidence.latestTraceEventsResponse.status??'-')+' sample='+String(input.networkEvidence.latestTraceEventsResponse.sampleSeq??'-')+' path='+String(input.networkEvidence.latestTraceEventsResponse.urlPath||'-')); + const recovery=input.recoveryAuthority||{}; + lines.push('recovery authority forbidden='+String(recovery.forbiddenAutomaticRecoveryCount??0)+' legacyFanout='+String(recovery.legacyFanoutWindowCount??0)+' syncEvidence='+String(recovery.syncReplayEvidenceCount??0)+' syncApplied='+String(recovery.syncReplayAppliedCount??0)+' syncNetwork='+String(recovery.syncNetworkReadCount??0)+' oldReads='+String(recovery.oldEndpointReadCount??0)+' gaps='+String(recovery.observabilityGapCount??0)); + if(recovery.latestSyncEvidence) lines.push('latest sync replay '+String(recovery.latestSyncEvidence.ts||'-')+' source='+String(recovery.latestSyncEvidence.source||'-')+' events='+String(recovery.latestSyncEvidence.eventCount??'-')+' since='+String(recovery.latestSyncEvidence.sinceOutboxSeq??'-')+' cursor='+String(recovery.latestSyncEvidence.syncCursorOutboxSeq??'-')+' authority='+String(recovery.latestSyncEvidence.realtimeAuthority||'-')+' terminalSeal='+String(recovery.latestSyncEvidence.terminalSeal??'-')); if(input.freeze.latestBlocker) lines.push('browser freeze '+String(input.freeze.latestBlocker.ts||'-')+' sample='+String(input.freeze.latestBlocker.sampleSeq??'-')+' rootCause='+String(input.freeze.latestBlocker.rootCause||'-')+' status='+String(input.freeze.latestBlocker.rootCauseStatus||'-')+' fallbackAllowed='+String(input.freeze.latestBlocker.fallbackAllowed===true)); lines.push('','Findings'); if(input.findings.length===0) lines.push('-'); @@ -983,7 +1013,7 @@ if(view==='timeline'){ } if(view==='workbench-triad'){ const triad=workbenchTriadView(rows); - console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,artifactFileCount:files.length,skippedFileCount:skippedFiles.length,skippedFiles:skippedFiles.slice(0,8),traceId:triad.traceId,dom:triad.dom,networkEvidence:triad.networkEvidence,freeze:triad.freeze,findings:triad.findings,statusGap:triad.statusGap,renderedText:triad.renderedText,sourceFiles:['samples.jsonl','network.jsonl','browser-process.jsonl','analysis/report.json'],valuesRedacted:true})); + console.log(JSON.stringify({ok:true,command:'web-probe-observe collect',view,stateDir:dir,artifactFileCount:files.length,skippedFileCount:skippedFiles.length,skippedFiles:skippedFiles.slice(0,8),traceId:triad.traceId,dom:triad.dom,networkEvidence:triad.networkEvidence,freeze:triad.freeze,recoveryAuthority:triad.recoveryAuthority,findings:triad.findings,statusGap:triad.statusGap,renderedText:triad.renderedText,sourceFiles:['samples.jsonl','network.jsonl','browser-process.jsonl','analysis/report.json'],valuesRedacted:true})); process.exit(0); } const frame=renderTraceFrame(selectSample(rows),rows); diff --git a/scripts/src/hwlab-node/web-observe-analyzer-triad.test.ts b/scripts/src/hwlab-node/web-observe-analyzer-triad.test.ts index 756847e2..89882ada 100644 --- a/scripts/src/hwlab-node/web-observe-analyzer-triad.test.ts +++ b/scripts/src/hwlab-node/web-observe-analyzer-triad.test.ts @@ -65,6 +65,27 @@ const browserFreezePolicy = { }, }; +async function runObserveAnalyzer(files: Record): Promise> { + const stateDir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-analyzer-")); + const analyzerPath = join(stateDir, "analyze.mjs"); + await writeFile(analyzerPath, nodeWebObserveAnalyzerSource(), { mode: 0o700 }); + for (const [name, content] of Object.entries(files)) { + await writeFile(join(stateDir, name), content.endsWith("\n") ? content : content + "\n"); + } + const result = spawnSync("bun", [analyzerPath, stateDir], { + cwd: join(import.meta.dir, "../../.."), + env: { + ...process.env, + UNIDESK_WEB_OBSERVE_ANALYZE_TAIL_SAMPLES: "0", + UNIDESK_WEB_OBSERVE_ALERT_THRESHOLDS_JSON: JSON.stringify(alertThresholds), + UNIDESK_WEB_OBSERVE_BROWSER_FREEZE_POLICY_JSON: JSON.stringify(browserFreezePolicy), + }, + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr || result.stdout); + return JSON.parse(await readFile(join(stateDir, "analysis", "report.json"), "utf8")); +} + test("observe analyzer classifies stale completed session rail when a newer turn is running", async () => { const stateDir = await mkdtemp(join(tmpdir(), "unidesk-web-observe-analyzer-")); const analyzerPath = join(stateDir, "analyze.mjs"); @@ -163,3 +184,105 @@ test("observe analyzer classifies stale completed session rail when a newer turn assert.equal(report.sampleMetrics.workbenchTurnStateTriad.summary.invalidRowCount, 1); assert.equal(report.sampleMetrics.workbenchTurnStateTriad.summary.cardFinalResponseMismatchCount, 0); }, 20_000); + +test("observe analyzer flags automatic recovery legacy fanout authority", async () => { + const samples = [ + JSON.stringify({ + seq: 1, + ts: "2026-07-01T15:21:25.000Z", + path: "/workbench/sessions/ses_auto", + url: "https://hwlab.example.test/workbench/sessions/ses_auto", + pageRole: "control", + pageId: "control-test", + routeSessionId: "ses_auto", + activeSessionId: "ses_auto", + turns: [] + }) + ].join("\n") + "\n"; + const network = [ + JSON.stringify({ + ts: "2026-07-01T15:21:26.000Z", + type: "response", + method: "POST", + status: 202, + url: "https://hwlab.example.test/v1/web-performance", + webPerformancePayload: { + parseStatus: "parsed", + schemaVersion: "hwlab-web-performance-v2", + events: [{ + kind: "workbench_ui_event", + eventType: "runtime_diagnostic", + module: "workbench-stream-transport", + diagnosticCode: "workbench_recovery_action", + recoveryAction: "hydrate-trace-events", + rootCause: "sse-gap", + eventCount: 0 + }] + } + }), + JSON.stringify({ ts: "2026-07-01T15:21:27.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/turns/trc_auto", bodySummary: { pathKind: "workbench-turn", terminalEvidenceCount: 1, traceIds: ["trc_auto"] } }), + JSON.stringify({ ts: "2026-07-01T15:21:28.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/sessions/ses_auto/messages", bodySummary: { pathKind: "workbench-session-messages", terminalEvidenceCount: 1, sessionIds: ["ses_auto"] } }), + JSON.stringify({ ts: "2026-07-01T15:21:29.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/traces/trc_auto/events", bodySummary: { pathKind: "workbench-trace-events", traceEventLikeCount: 3, traceIds: ["trc_auto"] } }), + JSON.stringify({ ts: "2026-07-01T15:21:30.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/sync?sessionId=ses_auto&traceId=trc_auto&since=7", bodySummary: { pathKind: "workbench-sync-replay", eventCount: 1, cursorOutboxSeq: 12, realtimeAuthority: "workbench-realtime-authority-v2", terminalSeal: true } }) + ].join("\n") + "\n"; + + const report = await runObserveAnalyzer({ "samples.jsonl": samples, "network.jsonl": network }); + const finding = report.findings.find((item: Record) => item.id === "workbench-automatic-recovery-fanout-authority"); + + assert.equal(finding?.rootCause, "workbench_automatic_recovery_legacy_fanout"); + assert.equal(finding?.recoveryAuthority.summary.forbiddenAutomaticRecoveryCount, 1); + assert.equal(finding?.recoveryAuthority.summary.legacyFanoutWindowCount, 1); + assert.equal(finding?.recoveryAuthority.summary.syncReplayEvidenceCount, 1); + assert.equal(report.runtimeAlerts.webPerformanceRuntimeDiagnostics[0].recoveryAction, "hydrate-trace-events"); +}, 20_000); + +test("observe analyzer allows explicit detail trace reads when sync authority evidence exists", async () => { + const samples = JSON.stringify({ + seq: 1, + ts: "2026-07-01T15:31:25.000Z", + path: "/workbench/sessions/ses_detail", + url: "https://hwlab.example.test/workbench/sessions/ses_detail", + pageRole: "control", + pageId: "control-test", + routeSessionId: "ses_detail", + activeSessionId: "ses_detail", + turns: [] + }) + "\n"; + const network = [ + JSON.stringify({ ts: "2026-07-01T15:31:26.000Z", type: "response", method: "GET", status: 200, url: "https://hwlab.example.test/v1/workbench/traces/trc_detail/events", bodySummary: { pathKind: "workbench-trace-events", traceEventLikeCount: 2, traceIds: ["trc_detail"], authority: "trace-detail-only", detailProjection: true } }), + JSON.stringify({ + ts: "2026-07-01T15:31:27.000Z", + type: "response", + method: "POST", + status: 202, + url: "https://hwlab.example.test/v1/web-performance", + webPerformancePayload: { + parseStatus: "parsed", + schemaVersion: "hwlab-web-performance-v2", + events: [{ + kind: "workbench_ui_event", + eventType: "runtime_diagnostic", + module: "workbench-sync-replay", + diagnosticCode: "workbench_sync_replay_applied", + rootCause: "sse-gap", + eventCount: 1, + sinceOutboxSeq: 7, + syncCursorOutboxSeq: 12, + realtimeAuthority: "workbench-realtime-authority-v2", + entityFamilyCount: 1, + entityFamilies: "turn", + maxEntityVersion: 3, + projectionRevision: "rev_detail", + terminalSeal: true, + detailProjection: false + }] + } + }) + ].join("\n") + "\n"; + + const report = await runObserveAnalyzer({ "samples.jsonl": samples, "network.jsonl": network }); + const finding = report.findings.find((item: Record) => item.id === "workbench-automatic-recovery-fanout-authority"); + + assert.equal(finding, undefined); + assert.equal(report.runtimeAlerts.webPerformanceRuntimeDiagnostics[0].syncCursorOutboxSeq, 12); +}, 20_000); diff --git a/scripts/src/platform-infra-observability/apply-status-scripts.ts b/scripts/src/platform-infra-observability/apply-status-scripts.ts index 60e26529..bbe00157 100644 --- a/scripts/src/platform-infra-observability/apply-status-scripts.ts +++ b/scripts/src/platform-infra-observability/apply-status-scripts.ts @@ -253,7 +253,7 @@ export function compactSpanList(value: unknown, limit: number): unknown[] { return { name: span.name ?? null, service: span.service ?? null, - attributes: compactRecord(attrs, ["failureKind", "terminalStatus", "status", "eventType", "idleMs", "waitingFor", "lastEventLabel", "http.route", "http.status_code", "http.response.status_code", "workbench.session_id", "workbench.trace_id", "workbench.turn_id", "workbench.read_model.route", "workbench.read_model.count", "workbench.read_model.family", "workbench.read_model.status", "workbench.read_model.reason", "hwlab.http.stage", "hwlab.http.phase", "hwlab.http.phase.outcome", "hwlab.live_builds.service_id", "hwlab.live_builds.service_kind", "hwlab.live_builds.external", "hwlab.live_builds.deploy_manifest_status", "hwlab.live_builds.artifact_catalog_status", "providerProfile", "model", "modelId", "providerModel", "code_agent.stage", "agent.chat.session_id", "agent.chat.provider_profile", "agent.chat.provider_profile_source", "agent.chat.model", "agent.chat.model_id", "agent.chat.provider_model", "defaultProviderProfile", "adapter", "adapterEnabled", "agentRunManagerHost", "agentRunRunnerNamespace", "agentRunSourceCommitPresent"]), + attributes: compactRecord(attrs, ["failureKind", "terminalStatus", "status", "eventType", "idleMs", "waitingFor", "lastEventLabel", "http.route", "http.status_code", "http.response.status_code", "workbench.session_id", "workbench.trace_id", "workbench.turn_id", "workbench.read_model.route", "workbench.read_model.count", "workbench.read_model.family", "workbench.read_model.status", "workbench.read_model.reason", "workbench.sync.contract_version", "workbench.sync.since_outbox_seq", "workbench.sync.cursor_outbox_seq", "workbench.sync.event_count", "workbench.sync.entity_family_count", "workbench.sync.entity_families", "workbench.sync.max_entity_version", "workbench.realtime.authority", "workbench.projection.revision", "workbench.terminal.seal", "workbench.detail.projection", "hwlab.http.stage", "hwlab.http.phase", "hwlab.http.phase.outcome", "hwlab.live_builds.service_id", "hwlab.live_builds.service_kind", "hwlab.live_builds.external", "hwlab.live_builds.deploy_manifest_status", "hwlab.live_builds.artifact_catalog_status", "providerProfile", "model", "modelId", "providerModel", "code_agent.stage", "agent.chat.session_id", "agent.chat.provider_profile", "agent.chat.provider_profile_source", "agent.chat.model", "agent.chat.model_id", "agent.chat.provider_model", "defaultProviderProfile", "adapter", "adapterEnabled", "agentRunManagerHost", "agentRunRunnerNamespace", "agentRunSourceCommitPresent"]), }; }); } diff --git a/scripts/src/platform-infra-observability/manifest.ts b/scripts/src/platform-infra-observability/manifest.ts index 6c1fd035..131926ac 100644 --- a/scripts/src/platform-infra-observability/manifest.ts +++ b/scripts/src/platform-infra-observability/manifest.ts @@ -120,6 +120,17 @@ export function spanDetail(span: Record, maxLength: number): st attrPart(attrs, "workbench.ui.resource_response_transfer_ms"), attrPart(attrs, "workbench.ui.resource_next_hop_protocol"), attrPart(attrs, "workbench.ui.resource_server_timing"), + attrPart(attrs, "workbench.sync.contract_version"), + attrPart(attrs, "workbench.sync.since_outbox_seq"), + attrPart(attrs, "workbench.sync.cursor_outbox_seq"), + attrPart(attrs, "workbench.sync.event_count"), + attrPart(attrs, "workbench.sync.entity_family_count"), + attrPart(attrs, "workbench.sync.entity_families"), + attrPart(attrs, "workbench.sync.max_entity_version"), + attrPart(attrs, "workbench.realtime.authority"), + attrPart(attrs, "workbench.projection.revision"), + attrPart(attrs, "workbench.terminal.seal"), + attrPart(attrs, "workbench.detail.projection"), attrPart(attrs, "hwlab.http.stage"), attrPart(attrs, "hwlab.http.phase"), attrPart(attrs, "hwlab.http.phase.outcome"), diff --git a/scripts/src/platform-infra-observability/render.ts b/scripts/src/platform-infra-observability/render.ts index 2a2a31c9..b5aec921 100644 --- a/scripts/src/platform-infra-observability/render.ts +++ b/scripts/src/platform-infra-observability/render.ts @@ -233,6 +233,17 @@ const knownTraceAttributeKeys = new Set([ "opencode.provider.sse.done_lines", "opencode.provider.sse.json_errors", "opencode.provider.sse.reasoning_only_choices_dropped", + "workbench.sync.contract_version", + "workbench.sync.since_outbox_seq", + "workbench.sync.cursor_outbox_seq", + "workbench.sync.event_count", + "workbench.sync.entity_family_count", + "workbench.sync.entity_families", + "workbench.sync.max_entity_version", + "workbench.realtime.authority", + "workbench.projection.revision", + "workbench.terminal.seal", + "workbench.detail.projection", ]); function isKnownTraceAttributeKey(value: string): boolean { @@ -432,7 +443,7 @@ export function renderTraceTable(input: { formatTable(["NAME", "TRACE", "SESSION", "TURN", "STATUS", "EVENT", "PROJECTED", "SOURCE", "MESSAGE"], projectionRows.length > 0 ? projectionRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Projection sync cursors:", - formatTable(["SERVICE", "TRACE", "SESSION", "AFTER", "END", "EVENTS", "RAW", "MAX", "LAST", "TERM"], projectionSyncRows.length > 0 ? projectionSyncRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]), + formatTable(["SERVICE", "TRACE", "SESSION", "AFTER", "END", "EVENTS", "RAW", "MAX", "LAST", "TERM", "AUTH"], projectionSyncRows.length > 0 ? projectionSyncRows : [["-", "-", "-", "-", "-", "-", "-", "-", "-", "-", "-"]]), "", "Span name counts:", formatTable(["NAME", "COUNT"], countRows.length > 0 ? countRows : [["-", "-"]]), @@ -599,19 +610,26 @@ function traceProjectionRows(spans: Record[]): string[][] { } function traceProjectionSyncRows(spans: Record[]): string[][] { - const rows = spans.filter((span) => textValue(span.name) === "projection_sync").map((span) => { + const rows = spans.filter((span) => { + const name = textValue(span.name); + const attrs = asPlainRecord(span.attributes); + return name === "projection_sync" + || name === "workbench.sync.replay" + || spanColumnAttr(attrs, ["workbench.sync.contract_version", "workbench.sync.cursor_outbox_seq", "workbench.sync.event_count", "workbench.realtime.authority"]) !== "-"; + }).map((span) => { const attrs = asPlainRecord(span.attributes); return [ shortenEnd(textValue(span.service), 18), - shortenMiddle(spanColumnAttr(attrs, ["traceId"]), 18), - shortenMiddle(spanColumnAttr(attrs, ["sessionId"]), 18), - shortenEnd(spanColumnAttr(attrs, ["afterSeq"]), 8), - shortenEnd(spanColumnAttr(attrs, ["endSeq"]), 8), - shortenEnd(spanColumnAttr(attrs, ["eventCount"]), 8), + shortenMiddle(spanColumnAttr(attrs, ["traceId", "workbench.trace_id"]), 18), + shortenMiddle(spanColumnAttr(attrs, ["sessionId", "workbench.session_id"]), 18), + shortenEnd(spanColumnAttr(attrs, ["afterSeq", "workbench.sync.since_outbox_seq"]), 8), + shortenEnd(spanColumnAttr(attrs, ["endSeq", "workbench.sync.cursor_outbox_seq"]), 8), + shortenEnd(spanColumnAttr(attrs, ["eventCount", "workbench.sync.event_count"]), 8), shortenEnd(spanColumnAttr(attrs, ["rawEventCount"]), 8), - shortenEnd(spanColumnAttr(attrs, ["maxSeq"]), 8), + shortenEnd(spanColumnAttr(attrs, ["maxSeq", "workbench.sync.max_entity_version"]), 8), shortenEnd(spanColumnAttr(attrs, ["traceLastSeq"]), 8), - shortenEnd(spanColumnAttr(attrs, ["terminalFromEvents", "terminalFromRawEvents", "terminalStatus"]), 12), + shortenEnd(spanColumnAttr(attrs, ["terminalFromEvents", "terminalFromRawEvents", "terminalStatus", "workbench.terminal.seal"]), 12), + shortenEnd(spanColumnAttr(attrs, ["workbench.realtime.authority", "workbench.sync.contract_version", "workbench.detail.projection"]), 18), ]; }); return dedupeRows(rows).slice(0, 16);