fix: improve workbench triad analysis drilldown

This commit is contained in:
Codex
2026-07-01 12:33:35 +00:00
parent ff2697358e
commit 1fa7df5aea
4 changed files with 385 additions and 3 deletions
@@ -1132,6 +1132,7 @@ function compactWorkbenchTurnStateTriadForOutput(value) {
if (!value || typeof value !== "object") return null;
return {
summary: value.summary ?? null,
drilldown: value.drilldown ?? null,
invalidFullTriads: Array.isArray(value.invalidFullTriads) ? value.invalidFullTriads.slice(0, 8) : [],
cardFinalResponseMismatches: Array.isArray(value.cardFinalResponseMismatches) ? value.cardFinalResponseMismatches.slice(0, 8) : [],
collectorMissingRows: Array.isArray(value.collectorMissingRows) ? value.collectorMissingRows.slice(0, 8) : [],
@@ -2858,6 +2859,8 @@ function buildWorkbenchTurnStateTriadMetrics(samples, timeline = []) {
const collectorMissingRows = rows.filter((row) => Array.isArray(row.collectorMissing) && row.collectorMissing.length > 0);
const invalidRowKeys = new Set([...invalidFullTriads, ...cardFinalResponseMismatches].map((row) => row.rowKey));
const collectorMissingFields = uniqueSorted(collectorMissingRows.flatMap((row) => row.collectorMissing || []));
const offendingRows = Array.from(new Map([...invalidFullTriads, ...cardFinalResponseMismatches].map((row) => [row.rowKey, row])).values());
const drilldown = buildWorkbenchTurnStateTriadDrilldown(offendingRows);
return {
summary: {
sampleCount: sourceSamples.length,
@@ -2866,10 +2869,14 @@ function buildWorkbenchTurnStateTriadMetrics(samples, timeline = []) {
invalidRowCount: invalidRowKeys.size,
invalidFullTriadCount: invalidFullTriads.length,
cardFinalResponseMismatchCount: cardFinalResponseMismatches.length,
invalidGroupCount: drilldown.summary.groupCount,
invalidTraceIdCount: drilldown.summary.traceIds.length,
invalidMaxObservedSpanMs: drilldown.summary.maxObservedSpanMs,
collectorMissingRowCount: collectorMissingRows.length,
collectorMissingFields,
valuesRedacted: true
},
drilldown,
invalidFullTriads: invalidFullTriads.slice(0, 120),
cardFinalResponseMismatches: cardFinalResponseMismatches.slice(0, 120),
collectorMissingRows: collectorMissingRows.slice(0, 120),
@@ -2878,6 +2885,210 @@ function buildWorkbenchTurnStateTriadMetrics(samples, timeline = []) {
};
}
function buildWorkbenchTurnStateTriadDrilldown(rows) {
const uniqueRows = Array.from(new Map((Array.isArray(rows) ? rows : [])
.filter(Boolean)
.map((row) => [row?.rowKey || [row?.seq, row?.traceId, row?.messageId, row?.pageRole].join("|"), row])).values());
const groupsByKey = new Map();
let firstAt = null;
let lastAt = null;
for (const row of uniqueRows) {
const traceId = firstString(row?.traceId);
const mismatchKind = workbenchTriadMismatchKind(row);
const tuple = workbenchTriadTuple(row);
const key = [
traceId || row?.messageId || row?.sessionIdPrefix || "-",
row?.sessionIdPrefix || "-",
row?.pageRole || "-",
mismatchKind,
tuple,
].join("|");
let group = groupsByKey.get(key);
if (!group) {
group = {
keyHash: sha256(key),
mismatchKind,
statusTuple: tuple,
traceId: traceId || null,
messageId: row?.messageId ?? null,
sessionIdPrefix: row?.sessionIdPrefix ?? null,
pageRole: row?.pageRole ?? null,
pageId: row?.pageId ?? null,
count: 0,
firstSeq: row?.seq ?? null,
lastSeq: row?.seq ?? null,
firstAt: row?.ts ?? null,
lastAt: row?.ts ?? null,
promptIndexes: new Set(),
commandIds: new Set(),
rawRailStatuses: new Set(),
rawCardStatuses: new Set(),
finalResponseTextBytes: new Set(),
examples: [],
valuesRedacted: true,
};
groupsByKey.set(key, group);
}
group.count += 1;
group.firstSeq = minNumberOrValue(group.firstSeq, row?.seq);
group.lastSeq = maxNumberOrValue(group.lastSeq, row?.seq);
group.firstAt = minIso(group.firstAt, row?.ts ?? null);
group.lastAt = maxIso(group.lastAt, row?.ts ?? null);
firstAt = minIso(firstAt, row?.ts ?? null);
lastAt = maxIso(lastAt, row?.ts ?? null);
if (row?.promptIndex !== null && row?.promptIndex !== undefined) group.promptIndexes.add(String(row.promptIndex));
if (row?.nearestCommandId) group.commandIds.add(String(row.nearestCommandId));
if (row?.railStatusRaw) group.rawRailStatuses.add(String(row.railStatusRaw));
if (row?.cardStatusRaw) group.rawCardStatuses.add(String(row.cardStatusRaw));
if (row?.finalResponseTextBytes !== null && row?.finalResponseTextBytes !== undefined) group.finalResponseTextBytes.add(String(row.finalResponseTextBytes));
if (group.examples.length < 3) {
group.examples.push({
seq: row?.seq ?? null,
ts: row?.ts ?? null,
traceId,
railStatus: row?.railStatus ?? null,
railStatusRaw: row?.railStatusRaw ?? null,
cardStatus: row?.cardStatus ?? null,
cardStatusRaw: row?.cardStatusRaw ?? null,
finalResponsePresent: row?.finalResponsePresent ?? null,
finalResponseTextBytes: row?.finalResponseTextBytes ?? null,
nearestCommandId: row?.nearestCommandId ?? null,
valuesRedacted: true,
});
}
}
const groups = Array.from(groupsByKey.values()).map((group) => {
const observedSpanMs = isoSpanMs(group.firstAt, group.lastAt);
return {
keyHash: group.keyHash,
mismatchKind: group.mismatchKind,
statusTuple: group.statusTuple,
traceId: group.traceId,
messageId: group.messageId,
sessionIdPrefix: group.sessionIdPrefix,
pageRole: group.pageRole,
pageId: group.pageId,
count: group.count,
firstSeq: group.firstSeq,
lastSeq: group.lastSeq,
firstAt: group.firstAt,
lastAt: group.lastAt,
observedSpanMs,
promptIndexes: Array.from(group.promptIndexes).slice(0, 8),
commandIds: Array.from(group.commandIds).slice(0, 8),
rawRailStatuses: Array.from(group.rawRailStatuses).slice(0, 8),
rawCardStatuses: Array.from(group.rawCardStatuses).slice(0, 8),
finalResponseTextBytes: Array.from(group.finalResponseTextBytes).slice(0, 8),
examples: group.examples,
valuesRedacted: true,
};
}).sort((left, right) => Number(right.observedSpanMs ?? 0) - Number(left.observedSpanMs ?? 0) || Number(right.count ?? 0) - Number(left.count ?? 0));
const traceIds = uniqueSorted(uniqueRows.map((row) => row?.traceId).filter(Boolean)).slice(0, 12);
const sessionPrefixes = uniqueSorted(uniqueRows.map((row) => row?.sessionIdPrefix).filter(Boolean)).slice(0, 12);
const mismatchKinds = uniqueSorted(uniqueRows.map(workbenchTriadMismatchKind).filter(Boolean));
return {
summary: {
invalidUniqueRows: uniqueRows.length,
groupCount: groups.length,
traceIds,
sessionPrefixes,
mismatchKinds,
firstAt,
lastAt,
maxObservedSpanMs: groups.reduce((value, item) => Math.max(value, Number(item.observedSpanMs ?? 0)), 0),
valuesRedacted: true,
},
groups: groups.slice(0, 12),
otelDrilldown: buildWorkbenchTriadOtelDrilldown(traceIds.slice(0, 4)),
staticSourceHints: workbenchTriadStaticSourceHints(),
unitTestReproHints: workbenchTriadUnitTestReproHints(),
valuesRedacted: true,
};
}
function buildWorkbenchTriadOtelDrilldown(traceIds) {
const target = otelTargetForAnalyzer();
const ids = (Array.isArray(traceIds) ? traceIds : []).filter(Boolean).slice(0, 4);
return {
target,
commands: ids.flatMap((traceId) => [
"bun scripts/cli.ts platform-infra observability diagnose-code-agent --target " + target + " --business-trace-id " + traceId + " --full",
"bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id <otel-trace-id-from-diagnose> --grep session_ --limit 60 --full",
"bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id <otel-trace-id-from-diagnose> --grep turn_status_read --limit 40 --full",
"bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id <otel-trace-id-from-diagnose> --grep projection --limit 120 --full",
]).slice(0, 16),
valuesRedacted: true,
};
}
function otelTargetForAnalyzer() {
const explicit = firstString(process.env.UNIDESK_WEB_OBSERVE_OTEL_TARGET, process.env.UNIDESK_OBSERVABILITY_TARGET);
if (explicit) return explicit;
const parts = String(stateDir || "").split(/[\\\/]+/u);
const index = parts.lastIndexOf("web-observe");
if (index >= 0 && parts[index + 1]) return String(parts[index + 1]).toUpperCase();
return "JD01";
}
function workbenchTriadMismatchKind(row) {
if (!row || typeof row !== "object") return "unknown";
if (row.railCardMismatch === true) return "rail-card-status-mismatch";
if (row.cardFinalResponseMismatch === true) return "completed-card-final-response-absent";
if (row.legacyCardFinalResponseMismatch === true) return "completed-card-final-response-uncollected-or-absent";
if (row.tupleAllowed === false) return "tuple-not-allowed";
return "unknown";
}
function workbenchTriadTuple(row) {
return [
"rail=" + (row?.railStatus ?? "-"),
"card=" + (row?.cardStatus ?? "-"),
"final=" + String(row?.finalResponsePresent ?? "unknown"),
].join(",");
}
function minNumberOrValue(left, right) {
const l = Number(left);
const r = Number(right);
if (!Number.isFinite(l)) return right ?? left ?? null;
if (!Number.isFinite(r)) return left ?? right ?? null;
return Math.min(l, r);
}
function maxNumberOrValue(left, right) {
const l = Number(left);
const r = Number(right);
if (!Number.isFinite(l)) return right ?? left ?? null;
if (!Number.isFinite(r)) return left ?? right ?? null;
return Math.max(l, r);
}
function isoSpanMs(firstAt, lastAt) {
const start = Date.parse(firstAt || "");
const end = Date.parse(lastAt || "");
if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return 0;
return end - start;
}
function workbenchTriadStaticSourceHints() {
return [
{ repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-session.ts", reason: "session rail/list status and active session projection source" },
{ repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-server-state.ts", reason: "server snapshot merge path that can overwrite sealed turn state" },
{ repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.ts", reason: "turn card and Final Response projection/runtime merge path" },
{ repo: "pikasTech/HWLAB", path: "internal/cloud/server-workbench-http*.go", reason: "session list/detail/messages/turn REST read-model contract" },
{ repo: "pikasTech/HWLAB", path: "internal/cloud/workbench-projection-*.go", reason: "durable projection writer and terminal seal contract" },
];
}
function workbenchTriadUnitTestReproHints() {
return [
"backend projector: terminal event must produce a single sealed turn tuple consumed by session list, session detail, messages and turn-status APIs",
"backend read model: completed rail status must not coexist with running turn card or missing Final Response for the same trace",
"frontend server-state merge: stale running/empty snapshots must not overwrite a sealed completed+Final Response turn",
"frontend projection runtime: cross-page hydration must converge from the same durable projection without DOM-only repair",
];
}
function workbenchTurnStateTriadRow(sample, turn, rail, promptIndex) {
const collectorMissing = [];
const railRawStatus = firstString(rail?.status, rail?.dataStatus);
@@ -3168,7 +3379,10 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
{ railStatus: "running", cardStatus: "running", finalResponsePresent: false }
],
samples: turnStateTriadRows.slice(0, 20),
drilldown: turnStateTriad.drilldown ?? buildWorkbenchTurnStateTriadDrilldown(turnStateTriadRows),
collectorMissingSamples: Array.isArray(turnStateTriad.collectorMissingRows) ? turnStateTriad.collectorMissingRows.slice(0, 10) : [],
sourceOfTruth: "durable Workbench projection/read model; do not repair via DOM fallback or GET-side state mutation",
nextAction: "Use drilldown.otelDrilldown.commands for the listed traceIds, then inspect staticSourceHints and add unit tests from unitTestReproHints before changing UI rendering.",
rootCause: "workbench_projection_state_triad_not_sealed",
rootCauseStatus: "confirmed-from-dom-samples",
rootCauseConfidence: "high",
@@ -3301,7 +3515,21 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
const evaluatedCrossPageProjectionDiffs = timedCrossPageProjectionDiffs.filter((item) => !controlledNavigationHydrationCrossPageDiff(item, controlledNavigationWindows, sampleBySeq) && !crossPageDiffHasWorkbenchAppShellNotReady(item, sampleBySeq));
const persistentCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) > crossPageProjectionBudgetMs);
const transientCrossPageProjectionDiffs = evaluatedCrossPageProjectionDiffs.filter((item) => Number(item.observedSpanMs ?? 0) <= crossPageProjectionBudgetMs);
if (persistentCrossPageProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-divergence", severity: "red", summary: "control and observer pages saw different projection state for the same sampled session beyond the configured budget", count: persistentCrossPageProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: persistentCrossPageProjectionDiffs.slice(0, 20) });
if (persistentCrossPageProjectionDiffs.length > 0) findings.push({
id: "cross-page-projection-divergence",
severity: "red",
summary: "control and observer pages saw different projection state for the same sampled session beyond the configured budget",
count: persistentCrossPageProjectionDiffs.length,
budgetMs: crossPageProjectionBudgetMs,
samples: persistentCrossPageProjectionDiffs.slice(0, 20),
drilldown: buildCrossPageProjectionDrilldown(persistentCrossPageProjectionDiffs),
sourceOfTruth: "session list/detail/messages/turn APIs must read from the same durable projection snapshot across pages",
rootCause: "workbench_cross_page_projection_not_single_source",
rootCauseStatus: "confirmed-from-control-observer-dom-samples",
rootCauseConfidence: "high",
nextAction: "Use drilldown.traceIds with diagnose-code-agent and compare session_list_read/session_messages_read/turn_status_read rows before changing renderer fallback behavior.",
valuesRedacted: true
});
if (transientCrossPageProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-transient-divergence", severity: "info", summary: "control and observer pages briefly differed near a sampled transition; retained as transient evidence but not treated as persistent projection failure", count: transientCrossPageProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: transientCrossPageProjectionDiffs.slice(0, 20) });
if (controlledNavigationHydrationProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-controlled-navigation-hydration", severity: "info", summary: "control and observer pages differed while a non-blocking session-invariance navigation command still had an unhydrated blank page; retained as context but not treated as a red projection blocker", count: controlledNavigationHydrationProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: controlledNavigationHydrationProjectionDiffs.slice(0, 20) });
if (appShellNotReadyProjectionDiffs.length > 0) findings.push({ id: "cross-page-projection-app-shell-not-ready", severity: "info", summary: "cross-page projection differences were explained by a page whose Workbench app shell was not mounted; see workbench-app-shell-not-ready for the blocking root cause", count: appShellNotReadyProjectionDiffs.length, budgetMs: crossPageProjectionBudgetMs, samples: appShellNotReadyProjectionDiffs.slice(0, 20) });
@@ -3339,6 +3567,114 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
return findings;
}
function buildCrossPageProjectionDrilldown(rows) {
const sourceRows = Array.isArray(rows) ? rows.filter(Boolean) : [];
const traceIds = uniqueSorted(sourceRows.flatMap((row) => collectIdsFromUnknown(row, "trace")).filter(Boolean)).slice(0, 12);
const sessionIds = uniqueSorted(sourceRows.flatMap((row) => collectIdsFromUnknown(row, "session")).filter(Boolean)).slice(0, 12);
const diffKinds = uniqueSorted(sourceRows.map((row) => row?.diffKind).filter(Boolean));
const groupsByKey = new Map();
for (const row of sourceRows) {
const rowTraceIds = collectIdsFromUnknown(row, "trace").slice(0, 8);
const rowSessionIds = collectIdsFromUnknown(row, "session").slice(0, 8);
const key = [
row?.diffKind || "projection",
rowSessionIds[0] || row?.sessionIdPrefix || "-",
rowTraceIds[0] || "-",
].join("|");
let group = groupsByKey.get(key);
if (!group) {
group = {
keyHash: sha256(key),
diffKind: row?.diffKind ?? null,
traceIds: new Set(),
sessionIds: new Set(),
count: 0,
firstSeq: row?.firstSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq ?? null,
lastSeq: row?.lastSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq ?? null,
firstAt: row?.firstAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null,
lastAt: row?.lastAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null,
maxObservedSpanMs: 0,
examples: [],
};
groupsByKey.set(key, group);
}
group.count += 1;
for (const traceId of rowTraceIds) group.traceIds.add(traceId);
for (const sessionId of rowSessionIds) group.sessionIds.add(sessionId);
group.firstSeq = minNumberOrValue(group.firstSeq, row?.firstSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq);
group.lastSeq = maxNumberOrValue(group.lastSeq, row?.lastSeq ?? row?.controlSeq ?? row?.observerSeq ?? row?.seq);
group.firstAt = minIso(group.firstAt, row?.firstAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null);
group.lastAt = maxIso(group.lastAt, row?.lastAt ?? row?.controlTs ?? row?.observerTs ?? row?.ts ?? null);
group.maxObservedSpanMs = Math.max(group.maxObservedSpanMs, Number(row?.observedSpanMs ?? 0) || 0);
if (group.examples.length < 3) {
group.examples.push({
diffKind: row?.diffKind ?? null,
firstSeq: row?.firstSeq ?? null,
lastSeq: row?.lastSeq ?? null,
observedSpanMs: row?.observedSpanMs ?? null,
controlMessageCount: row?.controlMessageCount ?? null,
observerMessageCount: row?.observerMessageCount ?? null,
traceIds: rowTraceIds.slice(0, 6),
sessionIds: rowSessionIds.slice(0, 4),
valuesRedacted: true,
});
}
}
const groups = Array.from(groupsByKey.values()).map((group) => ({
keyHash: group.keyHash,
diffKind: group.diffKind,
traceIds: Array.from(group.traceIds).slice(0, 8),
sessionIds: Array.from(group.sessionIds).slice(0, 6),
count: group.count,
firstSeq: group.firstSeq,
lastSeq: group.lastSeq,
firstAt: group.firstAt,
lastAt: group.lastAt,
observedSpanMs: Math.max(group.maxObservedSpanMs, isoSpanMs(group.firstAt, group.lastAt)),
examples: group.examples,
valuesRedacted: true,
})).sort((left, right) => Number(right.observedSpanMs ?? 0) - Number(left.observedSpanMs ?? 0) || Number(right.count ?? 0) - Number(left.count ?? 0));
return {
summary: {
rowCount: sourceRows.length,
groupCount: groups.length,
diffKinds,
traceIds,
sessionIds,
maxObservedSpanMs: groups.reduce((value, item) => Math.max(value, Number(item.observedSpanMs ?? 0)), 0),
valuesRedacted: true,
},
traceIds,
sessionIds,
groups: groups.slice(0, 12),
otelDrilldown: buildWorkbenchTriadOtelDrilldown(traceIds.slice(0, 4)),
staticSourceHints: workbenchTriadStaticSourceHints(),
unitTestReproHints: [
"two independent Workbench pages must converge to identical session list/detail/messages/turn projection after the same durable events",
"late session-list or message refresh must not drop a trace that another page already read from durable projection",
"read-side APIs must expose enough OTel fields to compare projection version/cursor and trace ids across pages",
],
valuesRedacted: true,
};
}
function collectIdsFromUnknown(value, kind, output = new Set(), depth = 0) {
if (depth > 4 || value === null || value === undefined) return Array.from(output);
if (typeof value === "string") {
const pattern = kind === "session" ? /\bses_[A-Za-z0-9_-]+\b/gu : /\btrc_[A-Za-z0-9_-]+\b/gu;
for (const match of value.match(pattern) || []) output.add(match);
return Array.from(output);
}
if (Array.isArray(value)) {
for (const item of value.slice(0, 40)) collectIdsFromUnknown(item, kind, output, depth + 1);
return Array.from(output);
}
if (typeof value === "object") {
for (const item of Object.values(value).slice(0, 80)) collectIdsFromUnknown(item, kind, output, depth + 1);
}
return Array.from(output);
}
function buildFrontendFreezeFindings(errors, control) {
const findings = [];
const promptTimes = (control || [])