feat: add web probe performance hotspot analysis
This commit is contained in:
Codex
@@ -0,0 +1,721 @@
|
||||
// SPEC: PJ2026-01040111 long-running Workbench observation.
|
||||
// Responsibility: Analyzer Workbench terminal projection and three-state triad source fragment.
|
||||
|
||||
export function nodeWebObserveAnalyzerWorkbenchTriadSource(): string {
|
||||
return String.raw`function detectWorkbenchInPlaceProjectionLag(samples, network, control = []) {
|
||||
const canarySessionIds = sessionInvariantCanarySessionIds(control);
|
||||
const terminalTraceMissing = detectWorkbenchTerminalTraceMissing(samples, canarySessionIds);
|
||||
const terminalApiDomLag = detectWorkbenchTerminalApiDomLag(samples, network, canarySessionIds);
|
||||
return {
|
||||
terminalTraceMissing,
|
||||
terminalApiDomLag,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function detectWorkbenchTerminalTraceMissing(samples, canarySessionIds = new Set()) {
|
||||
const budgetMs = Number.isFinite(Number(alertThresholds.sameOriginApiSlowMs)) ? Number(alertThresholds.sameOriginApiSlowMs) : 10_000;
|
||||
const rows = [];
|
||||
const open = new Map();
|
||||
const sortedSamples = (Array.isArray(samples) ? samples : [])
|
||||
.filter(isWorkbenchPathSample)
|
||||
.filter((sample) => workbenchSampleMatchesCanarySessions(sample, canarySessionIds))
|
||||
.slice()
|
||||
.sort((a, b) => Date.parse(a?.ts || "") - Date.parse(b?.ts || ""));
|
||||
const closeSegment = (key, lastSample = null) => {
|
||||
const segment = open.get(key);
|
||||
if (!segment) return;
|
||||
open.delete(key);
|
||||
const firstMs = Date.parse(segment.first.ts || "");
|
||||
const lastMs = Date.parse((lastSample ?? segment.last).ts || "");
|
||||
const observedMs = Number.isFinite(firstMs) && Number.isFinite(lastMs) && lastMs >= firstMs ? Math.round(lastMs - firstMs) : 0;
|
||||
if (observedMs <= budgetMs) return;
|
||||
rows.push({
|
||||
...ref(segment.first),
|
||||
lastSeq: segment.last.seq ?? null,
|
||||
lastAt: segment.last.ts ?? null,
|
||||
traceId: segment.traceId,
|
||||
messageCount: Array.isArray(segment.last.messages) ? segment.last.messages.length : 0,
|
||||
turnCount: Array.isArray(segment.last.turns) ? segment.last.turns.length : 0,
|
||||
traceRowCount: 0,
|
||||
sampleCount: segment.sampleCount,
|
||||
observedMs,
|
||||
budgetMs,
|
||||
finalMessageVisible: workbenchFinalMessageVisible(segment.last, segment.traceId),
|
||||
terminalTurnVisible: workbenchTerminalTurnVisible(segment.last, segment.traceId),
|
||||
detail: "terminal turn/message stayed visible for this trace while the same in-place Workbench page had zero trace rows beyond the configured budget",
|
||||
valuesRedacted: true
|
||||
});
|
||||
};
|
||||
for (const sample of sortedSamples) {
|
||||
if (!isWorkbenchPathSample(sample)) continue;
|
||||
const tsMs = Date.parse(sample?.ts || "");
|
||||
if (!Number.isFinite(tsMs)) continue;
|
||||
const terminalTraceIds = workbenchTerminalTraceIdsFromDom(sample);
|
||||
const presentKeys = new Set();
|
||||
for (const traceId of terminalTraceIds) {
|
||||
const key = [samplePageKey(sample), sample?.routeSessionId ?? "", sample?.activeSessionId ?? "", traceId].join("|");
|
||||
presentKeys.add(key);
|
||||
const traceRows = workbenchTraceRowsForTrace(sample, traceId);
|
||||
if (traceRows.length > 0 || workbenchSampleHasTerminalProjection(sample, { traceIds: [traceId] })) {
|
||||
closeSegment(key, sample);
|
||||
continue;
|
||||
}
|
||||
const existing = open.get(key);
|
||||
if (existing) {
|
||||
existing.last = sample;
|
||||
existing.sampleCount += 1;
|
||||
} else {
|
||||
open.set(key, { first: sample, last: sample, traceId, sampleCount: 1 });
|
||||
}
|
||||
}
|
||||
for (const [key, segment] of Array.from(open.entries())) {
|
||||
if (samplePageKey(sample) === samplePageKey(segment.first) && !presentKeys.has(key)) closeSegment(key, sample);
|
||||
}
|
||||
}
|
||||
for (const key of Array.from(open.keys())) closeSegment(key);
|
||||
return rows;
|
||||
}
|
||||
|
||||
function detectWorkbenchTerminalApiDomLag(samples, network, canarySessionIds = new Set()) {
|
||||
const windowMs = 30_000;
|
||||
const budgetMs = Number.isFinite(Number(alertThresholds.sameOriginApiSlowMs)) ? Number(alertThresholds.sameOriginApiSlowMs) : 10_000;
|
||||
const sampleRows = (Array.isArray(samples) ? samples : [])
|
||||
.filter(isWorkbenchPathSample)
|
||||
.filter((sample) => workbenchSampleMatchesCanarySessions(sample, canarySessionIds))
|
||||
.map((sample) => ({ sample, tsMs: Date.parse(sample?.ts || ""), pageKey: samplePageKey(sample) }))
|
||||
.filter((item) => Number.isFinite(item.tsMs))
|
||||
.sort((a, b) => a.tsMs - b.tsMs);
|
||||
const rowsByPage = new Map();
|
||||
for (const row of sampleRows) {
|
||||
const rows = rowsByPage.get(row.pageKey) || [];
|
||||
rows.push(row);
|
||||
rowsByPage.set(row.pageKey, rows);
|
||||
}
|
||||
const terminalEvents = (Array.isArray(network) ? network : [])
|
||||
.map(compactWorkbenchTerminalApiEvent)
|
||||
.filter((item) => workbenchTerminalEventMatchesCanarySessions(item, canarySessionIds))
|
||||
.filter((item) => item !== null);
|
||||
const overBudget = [];
|
||||
for (const event of terminalEvents) {
|
||||
const pageSamples = rowsByPage.get(event.pageKey) || [];
|
||||
if (pageSamples.length === 0 || event.tsMs < pageSamples[0].tsMs) continue;
|
||||
const alreadyVisible = lastWorkbenchSampleAtOrBefore(pageSamples, event.tsMs, event, (row) => workbenchSampleHasTerminalProjection(row.sample, event));
|
||||
if (alreadyVisible) continue;
|
||||
const firstAfter = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event);
|
||||
const firstMiss = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + budgetMs, event, (row) => !workbenchSampleHasTerminalProjection(row.sample, event));
|
||||
const resolved = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event, (row) => workbenchSampleHasTerminalProjection(row.sample, event));
|
||||
const deltaMs = resolved ? Math.max(0, Math.round(resolved.tsMs - event.tsMs)) : null;
|
||||
const unresolved = !resolved;
|
||||
const exceedsBudget = unresolved || (Number.isFinite(deltaMs) && deltaMs > budgetMs);
|
||||
if (!firstMiss) continue;
|
||||
if (!exceedsBudget) continue;
|
||||
overBudget.push({
|
||||
ts: event.ts,
|
||||
pageRole: event.pageRole,
|
||||
pageId: event.pageId,
|
||||
routeKind: event.routeKind,
|
||||
method: event.method,
|
||||
path: event.path,
|
||||
traceIds: event.traceIds.slice(0, 6),
|
||||
sessionIds: event.sessionIds.slice(0, 4),
|
||||
terminalEvidenceCount: event.terminalEvidenceCount,
|
||||
traceEventLikeCount: event.traceEventLikeCount,
|
||||
finalTextFieldCount: event.finalTextFieldCount,
|
||||
budgetMs,
|
||||
windowMs,
|
||||
resolvedDeltaMs: deltaMs,
|
||||
unresolvedWithinWindow: unresolved,
|
||||
firstAfterSample: compactWorkbenchProjectionSample(firstAfter?.sample, event),
|
||||
firstMissSample: compactWorkbenchProjectionSample(firstMiss?.sample, event),
|
||||
resolvedSample: compactWorkbenchProjectionSample(resolved?.sample, event),
|
||||
valuesRedacted: true
|
||||
});
|
||||
}
|
||||
return {
|
||||
summary: {
|
||||
terminalEventCount: terminalEvents.length,
|
||||
overBudgetCount: overBudget.length,
|
||||
budgetMs,
|
||||
windowMs,
|
||||
valuesRedacted: true
|
||||
},
|
||||
overBudget,
|
||||
terminalEvents: terminalEvents.slice(-20),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function workbenchSampleMatchesCanarySessions(sample, canarySessionIds) {
|
||||
if (!(canarySessionIds instanceof Set) || canarySessionIds.size === 0) return true;
|
||||
const sessionIds = [sample?.routeSessionId, sample?.activeSessionId].filter(Boolean).map(String);
|
||||
return sessionIds.some((id) => canarySessionIds.has(id));
|
||||
}
|
||||
|
||||
function workbenchTerminalEventMatchesCanarySessions(event, canarySessionIds) {
|
||||
if (!event) return false;
|
||||
if (!(canarySessionIds instanceof Set) || canarySessionIds.size === 0) return true;
|
||||
const eventSessionIds = Array.isArray(event.sessionIds) ? event.sessionIds.map(String) : [];
|
||||
return eventSessionIds.some((id) => canarySessionIds.has(id));
|
||||
}
|
||||
|
||||
function compactWorkbenchTerminalApiEvent(item) {
|
||||
if (!item || item.type !== "response" || item.observerInitiated === true) return null;
|
||||
const summary = objectValue(item.bodySummary);
|
||||
if (!summary || Number(summary.terminalEvidenceCount ?? 0) <= 0) return null;
|
||||
const parsed = parseApiDomLagUrl(item.url);
|
||||
if (!String(parsed.path || "").startsWith("/v1/workbench/") && parsed.path !== "/v1/agent/chat" && parsed.path !== "/v1/agent/chat/steer") return null;
|
||||
const routeKind = summary.pathKind ?? apiDomLagRouteKind(parsed.path);
|
||||
if (!isReliableWorkbenchTerminalApiEvent(summary, routeKind)) return null;
|
||||
const terminalTraceIds = uniqueSorted(Array.isArray(summary.terminalTraceIds) ? summary.terminalTraceIds : []).slice(0, 12);
|
||||
if (terminalTraceIds.length === 0) return null;
|
||||
const tsMs = Date.parse(item.ts || "");
|
||||
if (!Number.isFinite(tsMs)) return null;
|
||||
return {
|
||||
ts: item.ts,
|
||||
tsMs,
|
||||
pageRole: item.pageRole ?? null,
|
||||
pageId: item.pageId ?? null,
|
||||
pageKey: String(item.pageRole || "control") + ":" + String(item.pageId || "default"),
|
||||
method: String(item.method || "GET").toUpperCase(),
|
||||
path: parsed.path,
|
||||
routeKind,
|
||||
traceIds: terminalTraceIds,
|
||||
observedTraceIds: uniqueSorted([...(Array.isArray(summary.traceIds) ? summary.traceIds : []), parsed.traceId].filter(Boolean)).slice(0, 12),
|
||||
sessionIds: uniqueSorted([...(Array.isArray(summary.sessionIds) ? summary.sessionIds : []), parsed.sessionId].filter(Boolean)).slice(0, 12),
|
||||
terminalEvidenceCount: Number(summary.terminalEvidenceCount ?? 0),
|
||||
terminalStatusCount: Number(summary.terminalStatusCount ?? 0),
|
||||
terminalTextCount: Number(summary.terminalTextCount ?? 0),
|
||||
traceEventLikeCount: Number(summary.traceEventLikeCount ?? 0),
|
||||
finalTextFieldCount: Number(summary.finalTextFieldCount ?? 0),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function isReliableWorkbenchTerminalApiEvent(summary, routeKind) {
|
||||
if (!summary || routeKind !== "workbench-turn") return false;
|
||||
if (Number(summary.terminalEvidenceCount ?? 0) <= 0) return false;
|
||||
return Number(summary.runningStatusCount ?? 0) <= 0;
|
||||
}
|
||||
|
||||
function lastWorkbenchSampleAtOrBefore(rows, tsMs, event, predicate = null) {
|
||||
let result = null;
|
||||
for (const row of rows || []) {
|
||||
if (row.tsMs > tsMs) break;
|
||||
if (!workbenchSampleMatchesTerminalEvent(row.sample, event)) continue;
|
||||
if (typeof predicate === "function" && !predicate(row)) continue;
|
||||
result = row;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function firstWorkbenchSampleAfter(rows, startMs, endMs, event, predicate = null) {
|
||||
for (const row of rows || []) {
|
||||
if (row.tsMs < startMs) continue;
|
||||
if (row.tsMs > endMs) break;
|
||||
if (!workbenchSampleMatchesTerminalEvent(row.sample, event)) continue;
|
||||
if (typeof predicate === "function" && !predicate(row)) continue;
|
||||
return row;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function workbenchSampleMatchesTerminalEvent(sample, event) {
|
||||
if (!sample || !event) return false;
|
||||
if (event.sessionIds.length > 0) {
|
||||
const sessionIds = new Set([sample.routeSessionId, sample.activeSessionId].filter(Boolean).map(String));
|
||||
if (!event.sessionIds.some((id) => sessionIds.has(id))) return false;
|
||||
}
|
||||
if (event.traceIds.length > 0) {
|
||||
const traces = sampleTraceIds(sample);
|
||||
if (traces.size === 0) return false;
|
||||
if (!event.traceIds.some((id) => traces.has(id))) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function workbenchSampleHasTerminalProjection(sample, event) {
|
||||
const traceIds = event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample));
|
||||
if (traceIds.length === 0) return false;
|
||||
return traceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId) || workbenchTerminalTurnVisible(sample, traceId));
|
||||
}
|
||||
|
||||
function compactWorkbenchProjectionSample(sample, event = null) {
|
||||
if (!sample) return null;
|
||||
const eventTraceIds = event?.traceIds && event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample));
|
||||
const visibleTraceIds = eventTraceIds.slice(0, 6);
|
||||
return {
|
||||
...ref(sample),
|
||||
messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0,
|
||||
turnCount: Array.isArray(sample.turns) ? sample.turns.length : 0,
|
||||
traceRowCount: Array.isArray(sample.traceRows) ? sample.traceRows.length : 0,
|
||||
traceIds: visibleTraceIds,
|
||||
finalMessageVisible: visibleTraceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId)),
|
||||
terminalTurnVisible: visibleTraceIds.some((traceId) => workbenchTerminalTurnVisible(sample, traceId)),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function isWorkbenchPathSample(sample) {
|
||||
return /\/workbench(?:\/|$)/u.test(String(sample?.path || sample?.url || ""));
|
||||
}
|
||||
|
||||
function workbenchTerminalTraceIdsFromDom(sample) {
|
||||
const ids = new Set();
|
||||
for (const groupName of ["turns", "messages"]) {
|
||||
const group = Array.isArray(sample?.[groupName]) ? sample[groupName] : [];
|
||||
for (const item of group) {
|
||||
const traceId = stringOrNull(item?.traceId);
|
||||
if (!traceId) continue;
|
||||
if (workbenchDomItemIsTerminal(item)) ids.add(traceId);
|
||||
}
|
||||
}
|
||||
return Array.from(ids).sort();
|
||||
}
|
||||
|
||||
function workbenchTraceRowsForTrace(sample, traceId) {
|
||||
const rows = Array.isArray(sample?.traceRows) ? sample.traceRows : [];
|
||||
return rows.filter((item) => !traceId || item?.traceId === traceId);
|
||||
}
|
||||
|
||||
function workbenchFinalMessageVisible(sample, traceId) {
|
||||
const messages = Array.isArray(sample?.messages) ? sample.messages : [];
|
||||
return messages.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemLooksFinal(item));
|
||||
}
|
||||
|
||||
function workbenchTerminalTurnVisible(sample, traceId) {
|
||||
const turns = Array.isArray(sample?.turns) ? sample.turns : [];
|
||||
return turns.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemIsTerminal(item));
|
||||
}
|
||||
|
||||
function workbenchDomItemIsTerminal(item) {
|
||||
const status = String(item?.status || "").toLowerCase();
|
||||
if (/^(completed|complete|succeeded|success|failed|failure|error|canceled|cancelled|done)$/u.test(status)) return true;
|
||||
return isTerminalTraceText([item?.status, item?.textPreview, item?.text, item?.durationText, item?.activityText].filter(Boolean).join(" "));
|
||||
}
|
||||
|
||||
function workbenchDomItemLooksFinal(item) {
|
||||
const text = [item?.status, item?.textPreview, item?.text].filter(Boolean).join(" ");
|
||||
return workbenchDomItemIsTerminal(item) && isFinalResultText(text);
|
||||
}
|
||||
|
||||
function buildWorkbenchTurnStateTriadMetrics(samples, timeline = []) {
|
||||
const rows = [];
|
||||
const sourceSamples = Array.isArray(samples) ? samples : [];
|
||||
for (let index = 0; index < sourceSamples.length; index += 1) {
|
||||
const sample = sourceSamples[index];
|
||||
if (!isWorkbenchPathSample(sample)) continue;
|
||||
const turns = Array.isArray(sample?.turns) ? sample.turns.filter((turn) => String(turn?.role || turn?.dataRole || "agent") === "agent") : [];
|
||||
if (turns.length === 0) continue;
|
||||
const rail = activeSessionRailItemForSample(sample);
|
||||
for (const turn of turns.slice(-6)) {
|
||||
const row = workbenchTurnStateTriadRow(sample, turn, rail, timeline[index]?.promptIndex ?? 0);
|
||||
if (row) rows.push(row);
|
||||
}
|
||||
}
|
||||
const invalidFullTriads = rows.filter((row) => row.invalid === true && row.fullTriad === true);
|
||||
const cardFinalResponseMismatches = rows.filter((row) => row.cardFinalResponseMismatch === true || row.legacyCardFinalResponseMismatch === true);
|
||||
const collectorMissingRows = rows.filter((row) => Array.isArray(row.collectorMissing) && row.collectorMissing.length > 0);
|
||||
const invalidRowKeys = new Set([...invalidFullTriads, ...cardFinalResponseMismatches].map((row) => row.rowKey));
|
||||
const collectorMissingFields = uniqueSorted(collectorMissingRows.flatMap((row) => row.collectorMissing || []));
|
||||
const offendingRows = Array.from(new Map([...invalidFullTriads, ...cardFinalResponseMismatches].map((row) => [row.rowKey, row])).values());
|
||||
const drilldown = buildWorkbenchTurnStateTriadDrilldown(offendingRows);
|
||||
return {
|
||||
summary: {
|
||||
sampleCount: sourceSamples.length,
|
||||
rowCount: rows.length,
|
||||
fullTriadRowCount: rows.filter((row) => row.fullTriad === true).length,
|
||||
invalidRowCount: invalidRowKeys.size,
|
||||
invalidFullTriadCount: invalidFullTriads.length,
|
||||
cardFinalResponseMismatchCount: cardFinalResponseMismatches.length,
|
||||
invalidGroupCount: drilldown.summary.groupCount,
|
||||
invalidTraceIdCount: drilldown.summary.traceIds.length,
|
||||
invalidMaxObservedSpanMs: drilldown.summary.maxObservedSpanMs,
|
||||
collectorMissingRowCount: collectorMissingRows.length,
|
||||
collectorMissingFields,
|
||||
valuesRedacted: true
|
||||
},
|
||||
drilldown,
|
||||
invalidFullTriads: invalidFullTriads.slice(0, 120),
|
||||
cardFinalResponseMismatches: cardFinalResponseMismatches.slice(0, 120),
|
||||
collectorMissingRows: collectorMissingRows.slice(0, 120),
|
||||
timeline: rows.slice(-300),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function buildWorkbenchTurnStateTriadDrilldown(rows) {
|
||||
const uniqueRows = Array.from(new Map((Array.isArray(rows) ? rows : [])
|
||||
.filter(Boolean)
|
||||
.map((row) => [row?.rowKey || [row?.seq, row?.traceId, row?.messageId, row?.pageRole].join("|"), row])).values());
|
||||
const groupsByKey = new Map();
|
||||
let firstAt = null;
|
||||
let lastAt = null;
|
||||
for (const row of uniqueRows) {
|
||||
const traceId = firstString(row?.traceId);
|
||||
const mismatchKind = workbenchTriadMismatchKind(row);
|
||||
const tuple = workbenchTriadTuple(row);
|
||||
const key = [
|
||||
traceId || row?.messageId || row?.sessionIdPrefix || "-",
|
||||
row?.sessionIdPrefix || "-",
|
||||
row?.pageRole || "-",
|
||||
mismatchKind,
|
||||
tuple,
|
||||
].join("|");
|
||||
let group = groupsByKey.get(key);
|
||||
if (!group) {
|
||||
group = {
|
||||
keyHash: sha256(key),
|
||||
mismatchKind,
|
||||
statusTuple: tuple,
|
||||
traceId: traceId || null,
|
||||
messageId: row?.messageId ?? null,
|
||||
sessionIdPrefix: row?.sessionIdPrefix ?? null,
|
||||
pageRole: row?.pageRole ?? null,
|
||||
pageId: row?.pageId ?? null,
|
||||
count: 0,
|
||||
firstSeq: row?.seq ?? null,
|
||||
lastSeq: row?.seq ?? null,
|
||||
firstAt: row?.ts ?? null,
|
||||
lastAt: row?.ts ?? null,
|
||||
promptIndexes: new Set(),
|
||||
commandIds: new Set(),
|
||||
rawRailStatuses: new Set(),
|
||||
rawCardStatuses: new Set(),
|
||||
finalResponseTextBytes: new Set(),
|
||||
examples: [],
|
||||
valuesRedacted: true,
|
||||
};
|
||||
groupsByKey.set(key, group);
|
||||
}
|
||||
group.count += 1;
|
||||
group.firstSeq = minNumberOrValue(group.firstSeq, row?.seq);
|
||||
group.lastSeq = maxNumberOrValue(group.lastSeq, row?.seq);
|
||||
group.firstAt = minIso(group.firstAt, row?.ts ?? null);
|
||||
group.lastAt = maxIso(group.lastAt, row?.ts ?? null);
|
||||
firstAt = minIso(firstAt, row?.ts ?? null);
|
||||
lastAt = maxIso(lastAt, row?.ts ?? null);
|
||||
if (row?.promptIndex !== null && row?.promptIndex !== undefined) group.promptIndexes.add(String(row.promptIndex));
|
||||
if (row?.nearestCommandId) group.commandIds.add(String(row.nearestCommandId));
|
||||
if (row?.railStatusRaw) group.rawRailStatuses.add(String(row.railStatusRaw));
|
||||
if (row?.cardStatusRaw) group.rawCardStatuses.add(String(row.cardStatusRaw));
|
||||
if (row?.finalResponseTextBytes !== null && row?.finalResponseTextBytes !== undefined) group.finalResponseTextBytes.add(String(row.finalResponseTextBytes));
|
||||
if (group.examples.length < 3) {
|
||||
group.examples.push({
|
||||
seq: row?.seq ?? null,
|
||||
ts: row?.ts ?? null,
|
||||
traceId,
|
||||
railStatus: row?.railStatus ?? null,
|
||||
railStatusRaw: row?.railStatusRaw ?? null,
|
||||
cardStatus: row?.cardStatus ?? null,
|
||||
cardStatusRaw: row?.cardStatusRaw ?? null,
|
||||
finalResponsePresent: row?.finalResponsePresent ?? null,
|
||||
finalResponseTextBytes: row?.finalResponseTextBytes ?? null,
|
||||
nearestCommandId: row?.nearestCommandId ?? null,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
const groups = Array.from(groupsByKey.values()).map((group) => {
|
||||
const observedSpanMs = isoSpanMs(group.firstAt, group.lastAt);
|
||||
return {
|
||||
keyHash: group.keyHash,
|
||||
mismatchKind: group.mismatchKind,
|
||||
statusTuple: group.statusTuple,
|
||||
traceId: group.traceId,
|
||||
messageId: group.messageId,
|
||||
sessionIdPrefix: group.sessionIdPrefix,
|
||||
pageRole: group.pageRole,
|
||||
pageId: group.pageId,
|
||||
count: group.count,
|
||||
firstSeq: group.firstSeq,
|
||||
lastSeq: group.lastSeq,
|
||||
firstAt: group.firstAt,
|
||||
lastAt: group.lastAt,
|
||||
observedSpanMs,
|
||||
promptIndexes: Array.from(group.promptIndexes).slice(0, 8),
|
||||
commandIds: Array.from(group.commandIds).slice(0, 8),
|
||||
rawRailStatuses: Array.from(group.rawRailStatuses).slice(0, 8),
|
||||
rawCardStatuses: Array.from(group.rawCardStatuses).slice(0, 8),
|
||||
finalResponseTextBytes: Array.from(group.finalResponseTextBytes).slice(0, 8),
|
||||
examples: group.examples,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}).sort((left, right) => Number(right.observedSpanMs ?? 0) - Number(left.observedSpanMs ?? 0) || Number(right.count ?? 0) - Number(left.count ?? 0));
|
||||
const traceIds = uniqueSorted(uniqueRows.map((row) => row?.traceId).filter(Boolean)).slice(0, 12);
|
||||
const sessionPrefixes = uniqueSorted(uniqueRows.map((row) => row?.sessionIdPrefix).filter(Boolean)).slice(0, 12);
|
||||
const mismatchKinds = uniqueSorted(uniqueRows.map(workbenchTriadMismatchKind).filter(Boolean));
|
||||
return {
|
||||
summary: {
|
||||
invalidUniqueRows: uniqueRows.length,
|
||||
groupCount: groups.length,
|
||||
traceIds,
|
||||
sessionPrefixes,
|
||||
mismatchKinds,
|
||||
firstAt,
|
||||
lastAt,
|
||||
maxObservedSpanMs: groups.reduce((value, item) => Math.max(value, Number(item.observedSpanMs ?? 0)), 0),
|
||||
valuesRedacted: true,
|
||||
},
|
||||
groups: groups.slice(0, 12),
|
||||
otelDrilldown: buildWorkbenchTriadOtelDrilldown(traceIds.slice(0, 4)),
|
||||
staticSourceHints: workbenchTriadStaticSourceHints(),
|
||||
unitTestReproHints: workbenchTriadUnitTestReproHints(),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function buildWorkbenchTriadOtelDrilldown(traceIds) {
|
||||
const target = otelTargetForAnalyzer();
|
||||
const ids = (Array.isArray(traceIds) ? traceIds : []).filter(Boolean).slice(0, 4);
|
||||
return {
|
||||
target,
|
||||
commands: ids.flatMap((traceId) => [
|
||||
"bun scripts/cli.ts platform-infra observability diagnose-code-agent --target " + target + " --business-trace-id " + traceId + " --full",
|
||||
"bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id <otel-trace-id-from-diagnose> --grep session_ --limit 60 --full",
|
||||
"bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id <otel-trace-id-from-diagnose> --grep turn_status_read --limit 40 --full",
|
||||
"bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id <otel-trace-id-from-diagnose> --grep projection --limit 120 --full",
|
||||
]).slice(0, 16),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function otelTargetForAnalyzer() {
|
||||
const explicit = firstString(process.env.UNIDESK_WEB_OBSERVE_OTEL_TARGET, process.env.UNIDESK_OBSERVABILITY_TARGET);
|
||||
if (explicit) return explicit;
|
||||
const parts = String(stateDir || "").split(/[\\\/]+/u);
|
||||
const index = parts.lastIndexOf("web-observe");
|
||||
if (index >= 0 && parts[index + 1]) return String(parts[index + 1]).toUpperCase();
|
||||
return "JD01";
|
||||
}
|
||||
|
||||
function workbenchTriadMismatchKind(row) {
|
||||
if (!row || typeof row !== "object") return "unknown";
|
||||
if (row.railCardMismatch === true) return "rail-card-status-mismatch";
|
||||
if (row.cardFinalResponseMismatch === true) return "completed-card-final-response-absent";
|
||||
if (row.legacyCardFinalResponseMismatch === true) return "completed-card-final-response-uncollected-or-absent";
|
||||
if (row.tupleAllowed === false) return "tuple-not-allowed";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function workbenchTriadRootCauseFromDrilldown(drilldown, summary = {}) {
|
||||
const groups = Array.isArray(drilldown?.groups) ? drilldown.groups : [];
|
||||
const hasStaleCompletedRail = groups.some((group) => group?.mismatchKind === "rail-card-status-mismatch" && String(group?.statusTuple || "").includes("rail=completed,card=running,final=false"));
|
||||
if (hasStaleCompletedRail) return {
|
||||
rootCause: "workbench_session_rail_status_stale_after_new_running_turn",
|
||||
rootCauseStatus: "confirmed-from-dom-samples",
|
||||
rootCauseConfidence: "high",
|
||||
dominantMismatchKind: "rail-card-status-mismatch",
|
||||
summary: "Workbench session rail kept the previous completed terminal status while a newer turn card was running and Final Response was absent",
|
||||
nextAction: "Inspect HWLAB frontend session status authority/reducer, especially workbench-server-state sessionStatusAuthorityFromMessages and SessionRail sessionToSessionTab status input; session rail must derive from the latest active turn/message authority rather than the previous sealed terminal message.",
|
||||
sourceOfTruth: "latest durable Workbench turn/message projection for the active session",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
const finalMismatchCount = Number(summary?.cardFinalResponseMismatchCount ?? 0);
|
||||
const hasFinalMismatch = finalMismatchCount > 0 || groups.some((group) => /final=false/u.test(String(group?.statusTuple || "")) && group?.mismatchKind !== "rail-card-status-mismatch");
|
||||
if (hasFinalMismatch) return {
|
||||
rootCause: "workbench_terminal_final_response_not_sealed",
|
||||
rootCauseStatus: "confirmed-from-dom-samples",
|
||||
rootCauseConfidence: "high",
|
||||
dominantMismatchKind: "completed-card-final-response-absent",
|
||||
summary: "Workbench terminal turn card did not expose a structured Final Response body",
|
||||
nextAction: "Inspect HWLAB terminal message/finalResponse projection contract before changing renderer fallback behavior.",
|
||||
sourceOfTruth: "durable Workbench terminal message projection",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
const mismatchKinds = Array.isArray(drilldown?.summary?.mismatchKinds) ? drilldown.summary.mismatchKinds : [];
|
||||
return {
|
||||
rootCause: "workbench_projection_state_triad_not_sealed",
|
||||
rootCauseStatus: "confirmed-from-dom-samples",
|
||||
rootCauseConfidence: "high",
|
||||
dominantMismatchKind: mismatchKinds[0] ?? "unknown",
|
||||
summary: "Workbench session rail status, turn card status, and Final Response body presence diverged from the allowed state tuples",
|
||||
nextAction: "Use drilldown.otelDrilldown.commands for the listed traceIds, then inspect staticSourceHints and add unit tests from unitTestReproHints before changing UI rendering.",
|
||||
sourceOfTruth: "durable Workbench projection/read model",
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function workbenchTriadTuple(row) {
|
||||
return [
|
||||
"rail=" + (row?.railStatus ?? "-"),
|
||||
"card=" + (row?.cardStatus ?? "-"),
|
||||
"final=" + String(row?.finalResponsePresent ?? "unknown"),
|
||||
].join(",");
|
||||
}
|
||||
|
||||
function minNumberOrValue(left, right) {
|
||||
const l = Number(left);
|
||||
const r = Number(right);
|
||||
if (!Number.isFinite(l)) return right ?? left ?? null;
|
||||
if (!Number.isFinite(r)) return left ?? right ?? null;
|
||||
return Math.min(l, r);
|
||||
}
|
||||
|
||||
function maxNumberOrValue(left, right) {
|
||||
const l = Number(left);
|
||||
const r = Number(right);
|
||||
if (!Number.isFinite(l)) return right ?? left ?? null;
|
||||
if (!Number.isFinite(r)) return left ?? right ?? null;
|
||||
return Math.max(l, r);
|
||||
}
|
||||
|
||||
function isoSpanMs(firstAt, lastAt) {
|
||||
const start = Date.parse(firstAt || "");
|
||||
const end = Date.parse(lastAt || "");
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return 0;
|
||||
return end - start;
|
||||
}
|
||||
|
||||
function workbenchTriadStaticSourceHints() {
|
||||
return [
|
||||
{ repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-session.ts", reason: "session rail/list status and active session projection source" },
|
||||
{ repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-server-state.ts", reason: "server snapshot merge path that can overwrite sealed turn state" },
|
||||
{ repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.ts", reason: "turn card and Final Response projection/runtime merge path" },
|
||||
{ repo: "pikasTech/HWLAB", path: "internal/cloud/server-workbench-http*.go", reason: "session list/detail/messages/turn REST read-model contract" },
|
||||
{ repo: "pikasTech/HWLAB", path: "internal/cloud/workbench-projection-*.go", reason: "durable projection writer and terminal seal contract" },
|
||||
];
|
||||
}
|
||||
|
||||
function workbenchTriadUnitTestReproHints() {
|
||||
return [
|
||||
"frontend reducer: when a second trace is running in the same session, session rail status must not stay on the previous completed terminal trace",
|
||||
"backend projector: terminal event must produce a single sealed turn tuple consumed by session list, session detail, messages and turn-status APIs",
|
||||
"backend read model: completed rail status must not coexist with running turn card or missing Final Response for the same trace",
|
||||
"frontend server-state merge: stale running/empty snapshots must not overwrite a sealed completed+Final Response turn",
|
||||
"frontend projection runtime: cross-page hydration must converge from the same durable projection without DOM-only repair",
|
||||
];
|
||||
}
|
||||
|
||||
function workbenchTurnStateTriadRow(sample, turn, rail, promptIndex) {
|
||||
const collectorMissing = [];
|
||||
const railRawStatus = firstString(rail?.status, rail?.dataStatus);
|
||||
const railRunning = rail?.running === true || String(rail?.dataRunning || rail?.ariaBusy || "").toLowerCase() === "true";
|
||||
const railStatus = normalizeWorkbenchTriadStatus(railRawStatus, railRunning);
|
||||
if (!rail) collectorMissing.push("sessionRail.activeItem");
|
||||
if (!railRawStatus && railRunning !== true) collectorMissing.push("sessionRail.status");
|
||||
const cardRawStatus = firstString(turn?.status);
|
||||
const cardStatus = normalizeWorkbenchTriadStatus(cardRawStatus, false) || (workbenchDomItemIsTerminal(turn) ? "completed" : null);
|
||||
if (!cardRawStatus && !cardStatus) collectorMissing.push("turn.status");
|
||||
const finalPresence = finalResponsePresenceFromTurn(turn);
|
||||
if (finalPresence.known !== true) collectorMissing.push("turn.finalResponseTextBytes");
|
||||
const fullTriad = Boolean(railStatus && cardStatus && finalPresence.known === true);
|
||||
const finalResponsePresent = finalPresence.known === true ? finalPresence.present : null;
|
||||
const tupleAllowed = fullTriad
|
||||
? (railStatus === "completed" && cardStatus === "completed" && finalResponsePresent === true)
|
||||
|| (railStatus === "running" && cardStatus === "running" && finalResponsePresent === false)
|
||||
: null;
|
||||
const cardFinalResponseMismatch = cardStatus === "completed" && finalPresence.known === true && finalResponsePresent !== true;
|
||||
const legacyCardFinalResponseMismatch = cardStatus === "completed" && finalPresence.known !== true && !workbenchDomItemLooksFinal(turn);
|
||||
const railCardMismatch = Boolean(railStatus && cardStatus && railStatus !== cardStatus);
|
||||
const invalid = tupleAllowed === false || cardFinalResponseMismatch || legacyCardFinalResponseMismatch || railCardMismatch;
|
||||
const traceId = firstString(turn?.traceId, turn?.turnId);
|
||||
const messageId = firstString(turn?.messageId);
|
||||
const rowKey = [
|
||||
sample?.seq ?? "-",
|
||||
sample?.pageRole ?? "control",
|
||||
sample?.pageId ?? "default",
|
||||
sample?.routeSessionId ?? sample?.activeSessionId ?? rail?.sessionIdPrefix ?? "-",
|
||||
traceId ?? messageId ?? turn?.index ?? "-"
|
||||
].join("|");
|
||||
return {
|
||||
...ref(sample),
|
||||
rowKey,
|
||||
promptIndex,
|
||||
sessionIdPrefix: sessionPrefixForSample(sample, rail),
|
||||
traceId,
|
||||
messageId,
|
||||
turnId: firstString(turn?.turnId),
|
||||
turnIndex: turn?.index ?? null,
|
||||
railStatus,
|
||||
railStatusRaw: railRawStatus ?? null,
|
||||
railRunning: railRunning === true,
|
||||
railSource: rail ? "sessionRail.activeItem" : null,
|
||||
cardStatus,
|
||||
cardStatusRaw: cardRawStatus ?? null,
|
||||
finalResponsePresent,
|
||||
finalResponseTextBytes: turn?.finalResponseTextBytes !== null && turn?.finalResponseTextBytes !== undefined && Number.isFinite(Number(turn.finalResponseTextBytes)) ? Number(turn.finalResponseTextBytes) : null,
|
||||
finalResponseSource: turn?.finalResponseTextSource ?? null,
|
||||
finalResponseTextHash: turn?.finalResponseTextHash ?? null,
|
||||
finalResponseTextPreview: turn?.finalResponseTextPreview ? limitText(turn.finalResponseTextPreview, 160) : null,
|
||||
fullTriad,
|
||||
tupleAllowed,
|
||||
invalid,
|
||||
cardFinalResponseMismatch,
|
||||
legacyCardFinalResponseMismatch,
|
||||
railCardMismatch,
|
||||
collectorMissing,
|
||||
nearestCommandId: sample?.commandId ?? null,
|
||||
relatedChecks: ["WBC-001", "WBC-003", "WBC-011", "WBC-022"],
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function activeSessionRailItemForSample(sample) {
|
||||
const rail = sample?.sessionRail && typeof sample.sessionRail === "object" ? sample.sessionRail : null;
|
||||
if (!rail) return null;
|
||||
const items = Array.isArray(rail.items) ? rail.items : [];
|
||||
if (rail.activeItem) return rail.activeItem;
|
||||
const routeSessionId = String(sample?.routeSessionId || "");
|
||||
const activeSessionId = String(sample?.activeSessionId || "");
|
||||
return items.find((item) => item?.active === true)
|
||||
|| items.find((item) => sessionRailItemMatchesSession(item, activeSessionId))
|
||||
|| items.find((item) => sessionRailItemMatchesSession(item, routeSessionId))
|
||||
|| null;
|
||||
}
|
||||
|
||||
function sessionRailItemMatchesSession(item, sessionId) {
|
||||
const id = String(sessionId || "");
|
||||
const prefix = String(item?.sessionIdPrefix || item?.sessionId || "");
|
||||
return Boolean(id && prefix && (id === prefix || id.startsWith(prefix) || prefix.startsWith(id)));
|
||||
}
|
||||
|
||||
function sessionPrefixForSample(sample, rail) {
|
||||
const id = firstString(sample?.activeSessionId, sample?.routeSessionId, rail?.sessionIdPrefix, rail?.sessionId);
|
||||
return id ? String(id).slice(0, 12) : null;
|
||||
}
|
||||
|
||||
function finalResponsePresenceFromTurn(turn) {
|
||||
if (!turn || typeof turn !== "object") return { known: false, present: null };
|
||||
if (turn.finalResponseTextBytes !== null && turn.finalResponseTextBytes !== undefined && Number.isFinite(Number(turn.finalResponseTextBytes))) {
|
||||
const bytes = Number(turn.finalResponseTextBytes);
|
||||
return { known: true, present: bytes > 0 };
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(turn, "finalResponsePresent") && turn.finalResponsePresent !== null && turn.finalResponsePresent !== undefined) {
|
||||
return { known: true, present: turn.finalResponsePresent === true };
|
||||
}
|
||||
return { known: false, present: null };
|
||||
}
|
||||
|
||||
function normalizeWorkbenchTriadStatus(status, running = false) {
|
||||
const value = String(status || "").trim().toLowerCase().replace(/_/gu, "-");
|
||||
if (running === true) return "running";
|
||||
if (!value) return null;
|
||||
if (/^(completed|complete|succeeded|success|finished|done|terminal|sealed)$/u.test(value)) return "completed";
|
||||
if (/^(failed|failure|error|blocked|timeout|canceled|cancelled|stale|thread-resume-failed|interrupted|expired|idle)$/u.test(value)) return "completed";
|
||||
if (/^(pending|running|active|busy|admitted|dispatching|executing|streaming|processing|queued|in-progress|creating)$/u.test(value)) return "running";
|
||||
return null;
|
||||
}
|
||||
|
||||
function firstString(...values) {
|
||||
for (const value of values) {
|
||||
if (typeof value !== "string") continue;
|
||||
const text = value.trim();
|
||||
if (text) return text;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function promptCommandHasAuthoritativeSubmitSideEffect(control, promptRound) {
|
||||
const commandId = stringOrNull(promptRound?.promptCommandId);
|
||||
if (!commandId) return false;
|
||||
const row = (control || []).find((item) => item?.type === "sendPrompt" && item?.phase === "completed" && item?.commandId === commandId);
|
||||
const detail = objectValue(row?.detail);
|
||||
const chatSubmit = objectValue(detail.chatSubmit);
|
||||
const sideEffect = objectValue(chatSubmit.sideEffect);
|
||||
return chatSubmit.sideEffectObserved === true
|
||||
|| sideEffect.submitted === true
|
||||
|| Number(sideEffect.messageCountDelta ?? 0) > 0;
|
||||
}
|
||||
`;
|
||||
}
|
||||
Reference in New Issue
Block a user