722 lines
34 KiB
TypeScript
722 lines
34 KiB
TypeScript
// SPEC: PJ2026-01040111 long-running Workbench observation.
|
|
// Responsibility: Analyzer Workbench terminal projection and three-state triad source fragment.
|
|
|
|
export function nodeWebObserveAnalyzerWorkbenchTriadSource(): string {
|
|
return String.raw`function detectWorkbenchInPlaceProjectionLag(samples, network, control = []) {
|
|
const canarySessionIds = sessionInvariantCanarySessionIds(control);
|
|
const terminalTraceMissing = detectWorkbenchTerminalTraceMissing(samples, canarySessionIds);
|
|
const terminalApiDomLag = detectWorkbenchTerminalApiDomLag(samples, network, canarySessionIds);
|
|
return {
|
|
terminalTraceMissing,
|
|
terminalApiDomLag,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function detectWorkbenchTerminalTraceMissing(samples, canarySessionIds = new Set()) {
|
|
const budgetMs = Number.isFinite(Number(alertThresholds.sameOriginApiSlowMs)) ? Number(alertThresholds.sameOriginApiSlowMs) : 10_000;
|
|
const rows = [];
|
|
const open = new Map();
|
|
const sortedSamples = (Array.isArray(samples) ? samples : [])
|
|
.filter(isWorkbenchPathSample)
|
|
.filter((sample) => workbenchSampleMatchesCanarySessions(sample, canarySessionIds))
|
|
.slice()
|
|
.sort((a, b) => Date.parse(a?.ts || "") - Date.parse(b?.ts || ""));
|
|
const closeSegment = (key, lastSample = null) => {
|
|
const segment = open.get(key);
|
|
if (!segment) return;
|
|
open.delete(key);
|
|
const firstMs = Date.parse(segment.first.ts || "");
|
|
const lastMs = Date.parse((lastSample ?? segment.last).ts || "");
|
|
const observedMs = Number.isFinite(firstMs) && Number.isFinite(lastMs) && lastMs >= firstMs ? Math.round(lastMs - firstMs) : 0;
|
|
if (observedMs <= budgetMs) return;
|
|
rows.push({
|
|
...ref(segment.first),
|
|
lastSeq: segment.last.seq ?? null,
|
|
lastAt: segment.last.ts ?? null,
|
|
traceId: segment.traceId,
|
|
messageCount: Array.isArray(segment.last.messages) ? segment.last.messages.length : 0,
|
|
turnCount: Array.isArray(segment.last.turns) ? segment.last.turns.length : 0,
|
|
traceRowCount: 0,
|
|
sampleCount: segment.sampleCount,
|
|
observedMs,
|
|
budgetMs,
|
|
finalMessageVisible: workbenchFinalMessageVisible(segment.last, segment.traceId),
|
|
terminalTurnVisible: workbenchTerminalTurnVisible(segment.last, segment.traceId),
|
|
detail: "terminal turn/message stayed visible for this trace while the same in-place Workbench page had zero trace rows beyond the configured budget",
|
|
valuesRedacted: true
|
|
});
|
|
};
|
|
for (const sample of sortedSamples) {
|
|
if (!isWorkbenchPathSample(sample)) continue;
|
|
const tsMs = Date.parse(sample?.ts || "");
|
|
if (!Number.isFinite(tsMs)) continue;
|
|
const terminalTraceIds = workbenchTerminalTraceIdsFromDom(sample);
|
|
const presentKeys = new Set();
|
|
for (const traceId of terminalTraceIds) {
|
|
const key = [samplePageKey(sample), sample?.routeSessionId ?? "", sample?.activeSessionId ?? "", traceId].join("|");
|
|
presentKeys.add(key);
|
|
const traceRows = workbenchTraceRowsForTrace(sample, traceId);
|
|
if (traceRows.length > 0 || workbenchSampleHasTerminalProjection(sample, { traceIds: [traceId] })) {
|
|
closeSegment(key, sample);
|
|
continue;
|
|
}
|
|
const existing = open.get(key);
|
|
if (existing) {
|
|
existing.last = sample;
|
|
existing.sampleCount += 1;
|
|
} else {
|
|
open.set(key, { first: sample, last: sample, traceId, sampleCount: 1 });
|
|
}
|
|
}
|
|
for (const [key, segment] of Array.from(open.entries())) {
|
|
if (samplePageKey(sample) === samplePageKey(segment.first) && !presentKeys.has(key)) closeSegment(key, sample);
|
|
}
|
|
}
|
|
for (const key of Array.from(open.keys())) closeSegment(key);
|
|
return rows;
|
|
}
|
|
|
|
function detectWorkbenchTerminalApiDomLag(samples, network, canarySessionIds = new Set()) {
|
|
const windowMs = 30_000;
|
|
const budgetMs = Number.isFinite(Number(alertThresholds.sameOriginApiSlowMs)) ? Number(alertThresholds.sameOriginApiSlowMs) : 10_000;
|
|
const sampleRows = (Array.isArray(samples) ? samples : [])
|
|
.filter(isWorkbenchPathSample)
|
|
.filter((sample) => workbenchSampleMatchesCanarySessions(sample, canarySessionIds))
|
|
.map((sample) => ({ sample, tsMs: Date.parse(sample?.ts || ""), pageKey: samplePageKey(sample) }))
|
|
.filter((item) => Number.isFinite(item.tsMs))
|
|
.sort((a, b) => a.tsMs - b.tsMs);
|
|
const rowsByPage = new Map();
|
|
for (const row of sampleRows) {
|
|
const rows = rowsByPage.get(row.pageKey) || [];
|
|
rows.push(row);
|
|
rowsByPage.set(row.pageKey, rows);
|
|
}
|
|
const terminalEvents = (Array.isArray(network) ? network : [])
|
|
.map(compactWorkbenchTerminalApiEvent)
|
|
.filter((item) => workbenchTerminalEventMatchesCanarySessions(item, canarySessionIds))
|
|
.filter((item) => item !== null);
|
|
const overBudget = [];
|
|
for (const event of terminalEvents) {
|
|
const pageSamples = rowsByPage.get(event.pageKey) || [];
|
|
if (pageSamples.length === 0 || event.tsMs < pageSamples[0].tsMs) continue;
|
|
const alreadyVisible = lastWorkbenchSampleAtOrBefore(pageSamples, event.tsMs, event, (row) => workbenchSampleHasTerminalProjection(row.sample, event));
|
|
if (alreadyVisible) continue;
|
|
const firstAfter = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event);
|
|
const firstMiss = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + budgetMs, event, (row) => !workbenchSampleHasTerminalProjection(row.sample, event));
|
|
const resolved = firstWorkbenchSampleAfter(pageSamples, event.tsMs, event.tsMs + windowMs, event, (row) => workbenchSampleHasTerminalProjection(row.sample, event));
|
|
const deltaMs = resolved ? Math.max(0, Math.round(resolved.tsMs - event.tsMs)) : null;
|
|
const unresolved = !resolved;
|
|
const exceedsBudget = unresolved || (Number.isFinite(deltaMs) && deltaMs > budgetMs);
|
|
if (!firstMiss) continue;
|
|
if (!exceedsBudget) continue;
|
|
overBudget.push({
|
|
ts: event.ts,
|
|
pageRole: event.pageRole,
|
|
pageId: event.pageId,
|
|
routeKind: event.routeKind,
|
|
method: event.method,
|
|
path: event.path,
|
|
traceIds: event.traceIds.slice(0, 6),
|
|
sessionIds: event.sessionIds.slice(0, 4),
|
|
terminalEvidenceCount: event.terminalEvidenceCount,
|
|
traceEventLikeCount: event.traceEventLikeCount,
|
|
finalTextFieldCount: event.finalTextFieldCount,
|
|
budgetMs,
|
|
windowMs,
|
|
resolvedDeltaMs: deltaMs,
|
|
unresolvedWithinWindow: unresolved,
|
|
firstAfterSample: compactWorkbenchProjectionSample(firstAfter?.sample, event),
|
|
firstMissSample: compactWorkbenchProjectionSample(firstMiss?.sample, event),
|
|
resolvedSample: compactWorkbenchProjectionSample(resolved?.sample, event),
|
|
valuesRedacted: true
|
|
});
|
|
}
|
|
return {
|
|
summary: {
|
|
terminalEventCount: terminalEvents.length,
|
|
overBudgetCount: overBudget.length,
|
|
budgetMs,
|
|
windowMs,
|
|
valuesRedacted: true
|
|
},
|
|
overBudget,
|
|
terminalEvents: terminalEvents.slice(-20),
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function workbenchSampleMatchesCanarySessions(sample, canarySessionIds) {
|
|
if (!(canarySessionIds instanceof Set) || canarySessionIds.size === 0) return true;
|
|
const sessionIds = [sample?.routeSessionId, sample?.activeSessionId].filter(Boolean).map(String);
|
|
return sessionIds.some((id) => canarySessionIds.has(id));
|
|
}
|
|
|
|
function workbenchTerminalEventMatchesCanarySessions(event, canarySessionIds) {
|
|
if (!event) return false;
|
|
if (!(canarySessionIds instanceof Set) || canarySessionIds.size === 0) return true;
|
|
const eventSessionIds = Array.isArray(event.sessionIds) ? event.sessionIds.map(String) : [];
|
|
return eventSessionIds.some((id) => canarySessionIds.has(id));
|
|
}
|
|
|
|
function compactWorkbenchTerminalApiEvent(item) {
|
|
if (!item || item.type !== "response" || item.observerInitiated === true) return null;
|
|
const summary = objectValue(item.bodySummary);
|
|
if (!summary || Number(summary.terminalEvidenceCount ?? 0) <= 0) return null;
|
|
const parsed = parseApiDomLagUrl(item.url);
|
|
if (!String(parsed.path || "").startsWith("/v1/workbench/") && parsed.path !== "/v1/agent/chat" && parsed.path !== "/v1/agent/chat/steer") return null;
|
|
const routeKind = summary.pathKind ?? apiDomLagRouteKind(parsed.path);
|
|
if (!isReliableWorkbenchTerminalApiEvent(summary, routeKind)) return null;
|
|
const terminalTraceIds = uniqueSorted(Array.isArray(summary.terminalTraceIds) ? summary.terminalTraceIds : []).slice(0, 12);
|
|
if (terminalTraceIds.length === 0) return null;
|
|
const tsMs = Date.parse(item.ts || "");
|
|
if (!Number.isFinite(tsMs)) return null;
|
|
return {
|
|
ts: item.ts,
|
|
tsMs,
|
|
pageRole: item.pageRole ?? null,
|
|
pageId: item.pageId ?? null,
|
|
pageKey: String(item.pageRole || "control") + ":" + String(item.pageId || "default"),
|
|
method: String(item.method || "GET").toUpperCase(),
|
|
path: parsed.path,
|
|
routeKind,
|
|
traceIds: terminalTraceIds,
|
|
observedTraceIds: uniqueSorted([...(Array.isArray(summary.traceIds) ? summary.traceIds : []), parsed.traceId].filter(Boolean)).slice(0, 12),
|
|
sessionIds: uniqueSorted([...(Array.isArray(summary.sessionIds) ? summary.sessionIds : []), parsed.sessionId].filter(Boolean)).slice(0, 12),
|
|
terminalEvidenceCount: Number(summary.terminalEvidenceCount ?? 0),
|
|
terminalStatusCount: Number(summary.terminalStatusCount ?? 0),
|
|
terminalTextCount: Number(summary.terminalTextCount ?? 0),
|
|
traceEventLikeCount: Number(summary.traceEventLikeCount ?? 0),
|
|
finalTextFieldCount: Number(summary.finalTextFieldCount ?? 0),
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function isReliableWorkbenchTerminalApiEvent(summary, routeKind) {
|
|
if (!summary || routeKind !== "workbench-turn") return false;
|
|
if (Number(summary.terminalEvidenceCount ?? 0) <= 0) return false;
|
|
return Number(summary.runningStatusCount ?? 0) <= 0;
|
|
}
|
|
|
|
function lastWorkbenchSampleAtOrBefore(rows, tsMs, event, predicate = null) {
|
|
let result = null;
|
|
for (const row of rows || []) {
|
|
if (row.tsMs > tsMs) break;
|
|
if (!workbenchSampleMatchesTerminalEvent(row.sample, event)) continue;
|
|
if (typeof predicate === "function" && !predicate(row)) continue;
|
|
result = row;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function firstWorkbenchSampleAfter(rows, startMs, endMs, event, predicate = null) {
|
|
for (const row of rows || []) {
|
|
if (row.tsMs < startMs) continue;
|
|
if (row.tsMs > endMs) break;
|
|
if (!workbenchSampleMatchesTerminalEvent(row.sample, event)) continue;
|
|
if (typeof predicate === "function" && !predicate(row)) continue;
|
|
return row;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function workbenchSampleMatchesTerminalEvent(sample, event) {
|
|
if (!sample || !event) return false;
|
|
if (event.sessionIds.length > 0) {
|
|
const sessionIds = new Set([sample.routeSessionId, sample.activeSessionId].filter(Boolean).map(String));
|
|
if (!event.sessionIds.some((id) => sessionIds.has(id))) return false;
|
|
}
|
|
if (event.traceIds.length > 0) {
|
|
const traces = sampleTraceIds(sample);
|
|
if (traces.size === 0) return false;
|
|
if (!event.traceIds.some((id) => traces.has(id))) return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
function workbenchSampleHasTerminalProjection(sample, event) {
|
|
const traceIds = event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample));
|
|
if (traceIds.length === 0) return false;
|
|
return traceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId) || workbenchTerminalTurnVisible(sample, traceId));
|
|
}
|
|
|
|
function compactWorkbenchProjectionSample(sample, event = null) {
|
|
if (!sample) return null;
|
|
const eventTraceIds = event?.traceIds && event.traceIds.length > 0 ? event.traceIds : Array.from(sampleTraceIds(sample));
|
|
const visibleTraceIds = eventTraceIds.slice(0, 6);
|
|
return {
|
|
...ref(sample),
|
|
messageCount: Array.isArray(sample.messages) ? sample.messages.length : 0,
|
|
turnCount: Array.isArray(sample.turns) ? sample.turns.length : 0,
|
|
traceRowCount: Array.isArray(sample.traceRows) ? sample.traceRows.length : 0,
|
|
traceIds: visibleTraceIds,
|
|
finalMessageVisible: visibleTraceIds.some((traceId) => workbenchFinalMessageVisible(sample, traceId)),
|
|
terminalTurnVisible: visibleTraceIds.some((traceId) => workbenchTerminalTurnVisible(sample, traceId)),
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function isWorkbenchPathSample(sample) {
|
|
return /\/workbench(?:\/|$)/u.test(String(sample?.path || sample?.url || ""));
|
|
}
|
|
|
|
function workbenchTerminalTraceIdsFromDom(sample) {
|
|
const ids = new Set();
|
|
for (const groupName of ["turns", "messages"]) {
|
|
const group = Array.isArray(sample?.[groupName]) ? sample[groupName] : [];
|
|
for (const item of group) {
|
|
const traceId = stringOrNull(item?.traceId);
|
|
if (!traceId) continue;
|
|
if (workbenchDomItemIsTerminal(item)) ids.add(traceId);
|
|
}
|
|
}
|
|
return Array.from(ids).sort();
|
|
}
|
|
|
|
function workbenchTraceRowsForTrace(sample, traceId) {
|
|
const rows = Array.isArray(sample?.traceRows) ? sample.traceRows : [];
|
|
return rows.filter((item) => !traceId || item?.traceId === traceId);
|
|
}
|
|
|
|
function workbenchFinalMessageVisible(sample, traceId) {
|
|
const messages = Array.isArray(sample?.messages) ? sample.messages : [];
|
|
return messages.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemLooksFinal(item));
|
|
}
|
|
|
|
function workbenchTerminalTurnVisible(sample, traceId) {
|
|
const turns = Array.isArray(sample?.turns) ? sample.turns : [];
|
|
return turns.some((item) => (!traceId || item?.traceId === traceId) && workbenchDomItemIsTerminal(item));
|
|
}
|
|
|
|
function workbenchDomItemIsTerminal(item) {
|
|
const status = String(item?.status || "").toLowerCase();
|
|
if (/^(completed|complete|succeeded|success|failed|failure|error|canceled|cancelled|done)$/u.test(status)) return true;
|
|
return isTerminalTraceText([item?.status, item?.textPreview, item?.text, item?.durationText, item?.activityText].filter(Boolean).join(" "));
|
|
}
|
|
|
|
function workbenchDomItemLooksFinal(item) {
|
|
const text = [item?.status, item?.textPreview, item?.text].filter(Boolean).join(" ");
|
|
return workbenchDomItemIsTerminal(item) && isFinalResultText(text);
|
|
}
|
|
|
|
function buildWorkbenchTurnStateTriadMetrics(samples, timeline = []) {
|
|
const rows = [];
|
|
const sourceSamples = Array.isArray(samples) ? samples : [];
|
|
for (let index = 0; index < sourceSamples.length; index += 1) {
|
|
const sample = sourceSamples[index];
|
|
if (!isWorkbenchPathSample(sample)) continue;
|
|
const turns = Array.isArray(sample?.turns) ? sample.turns.filter((turn) => String(turn?.role || turn?.dataRole || "agent") === "agent") : [];
|
|
if (turns.length === 0) continue;
|
|
const rail = activeSessionRailItemForSample(sample);
|
|
for (const turn of turns.slice(-6)) {
|
|
const row = workbenchTurnStateTriadRow(sample, turn, rail, timeline[index]?.promptIndex ?? 0);
|
|
if (row) rows.push(row);
|
|
}
|
|
}
|
|
const invalidFullTriads = rows.filter((row) => row.invalid === true && row.fullTriad === true);
|
|
const cardFinalResponseMismatches = rows.filter((row) => row.cardFinalResponseMismatch === true || row.legacyCardFinalResponseMismatch === true);
|
|
const collectorMissingRows = rows.filter((row) => Array.isArray(row.collectorMissing) && row.collectorMissing.length > 0);
|
|
const invalidRowKeys = new Set([...invalidFullTriads, ...cardFinalResponseMismatches].map((row) => row.rowKey));
|
|
const collectorMissingFields = uniqueSorted(collectorMissingRows.flatMap((row) => row.collectorMissing || []));
|
|
const offendingRows = Array.from(new Map([...invalidFullTriads, ...cardFinalResponseMismatches].map((row) => [row.rowKey, row])).values());
|
|
const drilldown = buildWorkbenchTurnStateTriadDrilldown(offendingRows);
|
|
return {
|
|
summary: {
|
|
sampleCount: sourceSamples.length,
|
|
rowCount: rows.length,
|
|
fullTriadRowCount: rows.filter((row) => row.fullTriad === true).length,
|
|
invalidRowCount: invalidRowKeys.size,
|
|
invalidFullTriadCount: invalidFullTriads.length,
|
|
cardFinalResponseMismatchCount: cardFinalResponseMismatches.length,
|
|
invalidGroupCount: drilldown.summary.groupCount,
|
|
invalidTraceIdCount: drilldown.summary.traceIds.length,
|
|
invalidMaxObservedSpanMs: drilldown.summary.maxObservedSpanMs,
|
|
collectorMissingRowCount: collectorMissingRows.length,
|
|
collectorMissingFields,
|
|
valuesRedacted: true
|
|
},
|
|
drilldown,
|
|
invalidFullTriads: invalidFullTriads.slice(0, 120),
|
|
cardFinalResponseMismatches: cardFinalResponseMismatches.slice(0, 120),
|
|
collectorMissingRows: collectorMissingRows.slice(0, 120),
|
|
timeline: rows.slice(-300),
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function buildWorkbenchTurnStateTriadDrilldown(rows) {
|
|
const uniqueRows = Array.from(new Map((Array.isArray(rows) ? rows : [])
|
|
.filter(Boolean)
|
|
.map((row) => [row?.rowKey || [row?.seq, row?.traceId, row?.messageId, row?.pageRole].join("|"), row])).values());
|
|
const groupsByKey = new Map();
|
|
let firstAt = null;
|
|
let lastAt = null;
|
|
for (const row of uniqueRows) {
|
|
const traceId = firstString(row?.traceId);
|
|
const mismatchKind = workbenchTriadMismatchKind(row);
|
|
const tuple = workbenchTriadTuple(row);
|
|
const key = [
|
|
traceId || row?.messageId || row?.sessionIdPrefix || "-",
|
|
row?.sessionIdPrefix || "-",
|
|
row?.pageRole || "-",
|
|
mismatchKind,
|
|
tuple,
|
|
].join("|");
|
|
let group = groupsByKey.get(key);
|
|
if (!group) {
|
|
group = {
|
|
keyHash: sha256(key),
|
|
mismatchKind,
|
|
statusTuple: tuple,
|
|
traceId: traceId || null,
|
|
messageId: row?.messageId ?? null,
|
|
sessionIdPrefix: row?.sessionIdPrefix ?? null,
|
|
pageRole: row?.pageRole ?? null,
|
|
pageId: row?.pageId ?? null,
|
|
count: 0,
|
|
firstSeq: row?.seq ?? null,
|
|
lastSeq: row?.seq ?? null,
|
|
firstAt: row?.ts ?? null,
|
|
lastAt: row?.ts ?? null,
|
|
promptIndexes: new Set(),
|
|
commandIds: new Set(),
|
|
rawRailStatuses: new Set(),
|
|
rawCardStatuses: new Set(),
|
|
finalResponseTextBytes: new Set(),
|
|
examples: [],
|
|
valuesRedacted: true,
|
|
};
|
|
groupsByKey.set(key, group);
|
|
}
|
|
group.count += 1;
|
|
group.firstSeq = minNumberOrValue(group.firstSeq, row?.seq);
|
|
group.lastSeq = maxNumberOrValue(group.lastSeq, row?.seq);
|
|
group.firstAt = minIso(group.firstAt, row?.ts ?? null);
|
|
group.lastAt = maxIso(group.lastAt, row?.ts ?? null);
|
|
firstAt = minIso(firstAt, row?.ts ?? null);
|
|
lastAt = maxIso(lastAt, row?.ts ?? null);
|
|
if (row?.promptIndex !== null && row?.promptIndex !== undefined) group.promptIndexes.add(String(row.promptIndex));
|
|
if (row?.nearestCommandId) group.commandIds.add(String(row.nearestCommandId));
|
|
if (row?.railStatusRaw) group.rawRailStatuses.add(String(row.railStatusRaw));
|
|
if (row?.cardStatusRaw) group.rawCardStatuses.add(String(row.cardStatusRaw));
|
|
if (row?.finalResponseTextBytes !== null && row?.finalResponseTextBytes !== undefined) group.finalResponseTextBytes.add(String(row.finalResponseTextBytes));
|
|
if (group.examples.length < 3) {
|
|
group.examples.push({
|
|
seq: row?.seq ?? null,
|
|
ts: row?.ts ?? null,
|
|
traceId,
|
|
railStatus: row?.railStatus ?? null,
|
|
railStatusRaw: row?.railStatusRaw ?? null,
|
|
cardStatus: row?.cardStatus ?? null,
|
|
cardStatusRaw: row?.cardStatusRaw ?? null,
|
|
finalResponsePresent: row?.finalResponsePresent ?? null,
|
|
finalResponseTextBytes: row?.finalResponseTextBytes ?? null,
|
|
nearestCommandId: row?.nearestCommandId ?? null,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
}
|
|
const groups = Array.from(groupsByKey.values()).map((group) => {
|
|
const observedSpanMs = isoSpanMs(group.firstAt, group.lastAt);
|
|
return {
|
|
keyHash: group.keyHash,
|
|
mismatchKind: group.mismatchKind,
|
|
statusTuple: group.statusTuple,
|
|
traceId: group.traceId,
|
|
messageId: group.messageId,
|
|
sessionIdPrefix: group.sessionIdPrefix,
|
|
pageRole: group.pageRole,
|
|
pageId: group.pageId,
|
|
count: group.count,
|
|
firstSeq: group.firstSeq,
|
|
lastSeq: group.lastSeq,
|
|
firstAt: group.firstAt,
|
|
lastAt: group.lastAt,
|
|
observedSpanMs,
|
|
promptIndexes: Array.from(group.promptIndexes).slice(0, 8),
|
|
commandIds: Array.from(group.commandIds).slice(0, 8),
|
|
rawRailStatuses: Array.from(group.rawRailStatuses).slice(0, 8),
|
|
rawCardStatuses: Array.from(group.rawCardStatuses).slice(0, 8),
|
|
finalResponseTextBytes: Array.from(group.finalResponseTextBytes).slice(0, 8),
|
|
examples: group.examples,
|
|
valuesRedacted: true,
|
|
};
|
|
}).sort((left, right) => Number(right.observedSpanMs ?? 0) - Number(left.observedSpanMs ?? 0) || Number(right.count ?? 0) - Number(left.count ?? 0));
|
|
const traceIds = uniqueSorted(uniqueRows.map((row) => row?.traceId).filter(Boolean)).slice(0, 12);
|
|
const sessionPrefixes = uniqueSorted(uniqueRows.map((row) => row?.sessionIdPrefix).filter(Boolean)).slice(0, 12);
|
|
const mismatchKinds = uniqueSorted(uniqueRows.map(workbenchTriadMismatchKind).filter(Boolean));
|
|
return {
|
|
summary: {
|
|
invalidUniqueRows: uniqueRows.length,
|
|
groupCount: groups.length,
|
|
traceIds,
|
|
sessionPrefixes,
|
|
mismatchKinds,
|
|
firstAt,
|
|
lastAt,
|
|
maxObservedSpanMs: groups.reduce((value, item) => Math.max(value, Number(item.observedSpanMs ?? 0)), 0),
|
|
valuesRedacted: true,
|
|
},
|
|
groups: groups.slice(0, 12),
|
|
otelDrilldown: buildWorkbenchTriadOtelDrilldown(traceIds.slice(0, 4)),
|
|
staticSourceHints: workbenchTriadStaticSourceHints(),
|
|
unitTestReproHints: workbenchTriadUnitTestReproHints(),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function buildWorkbenchTriadOtelDrilldown(traceIds) {
|
|
const target = otelTargetForAnalyzer();
|
|
const ids = (Array.isArray(traceIds) ? traceIds : []).filter(Boolean).slice(0, 4);
|
|
return {
|
|
target,
|
|
commands: ids.flatMap((traceId) => [
|
|
"bun scripts/cli.ts platform-infra observability diagnose-code-agent --target " + target + " --business-trace-id " + traceId + " --full",
|
|
"bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id <otel-trace-id-from-diagnose> --grep session_ --limit 60 --full",
|
|
"bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id <otel-trace-id-from-diagnose> --grep turn_status_read --limit 40 --full",
|
|
"bun scripts/cli.ts platform-infra observability trace --target " + target + " --trace-id <otel-trace-id-from-diagnose> --grep projection --limit 120 --full",
|
|
]).slice(0, 16),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function otelTargetForAnalyzer() {
|
|
const explicit = firstString(process.env.UNIDESK_WEB_OBSERVE_OTEL_TARGET, process.env.UNIDESK_OBSERVABILITY_TARGET);
|
|
if (explicit) return explicit;
|
|
const parts = String(stateDir || "").split(/[\\\/]+/u);
|
|
const index = parts.lastIndexOf("web-observe");
|
|
if (index >= 0 && parts[index + 1]) return String(parts[index + 1]).toUpperCase();
|
|
return "JD01";
|
|
}
|
|
|
|
function workbenchTriadMismatchKind(row) {
|
|
if (!row || typeof row !== "object") return "unknown";
|
|
if (row.railCardMismatch === true) return "rail-card-status-mismatch";
|
|
if (row.cardFinalResponseMismatch === true) return "completed-card-final-response-absent";
|
|
if (row.legacyCardFinalResponseMismatch === true) return "completed-card-final-response-uncollected-or-absent";
|
|
if (row.tupleAllowed === false) return "tuple-not-allowed";
|
|
return "unknown";
|
|
}
|
|
|
|
function workbenchTriadRootCauseFromDrilldown(drilldown, summary = {}) {
|
|
const groups = Array.isArray(drilldown?.groups) ? drilldown.groups : [];
|
|
const hasStaleCompletedRail = groups.some((group) => group?.mismatchKind === "rail-card-status-mismatch" && String(group?.statusTuple || "").includes("rail=completed,card=running,final=false"));
|
|
if (hasStaleCompletedRail) return {
|
|
rootCause: "workbench_session_rail_status_stale_after_new_running_turn",
|
|
rootCauseStatus: "confirmed-from-dom-samples",
|
|
rootCauseConfidence: "high",
|
|
dominantMismatchKind: "rail-card-status-mismatch",
|
|
summary: "Workbench session rail kept the previous completed terminal status while a newer turn card was running and Final Response was absent",
|
|
nextAction: "Inspect HWLAB frontend session status authority/reducer, especially workbench-server-state sessionStatusAuthorityFromMessages and SessionRail sessionToSessionTab status input; session rail must derive from the latest active turn/message authority rather than the previous sealed terminal message.",
|
|
sourceOfTruth: "latest durable Workbench turn/message projection for the active session",
|
|
valuesRedacted: true,
|
|
};
|
|
const finalMismatchCount = Number(summary?.cardFinalResponseMismatchCount ?? 0);
|
|
const hasFinalMismatch = finalMismatchCount > 0 || groups.some((group) => /final=false/u.test(String(group?.statusTuple || "")) && group?.mismatchKind !== "rail-card-status-mismatch");
|
|
if (hasFinalMismatch) return {
|
|
rootCause: "workbench_terminal_final_response_not_sealed",
|
|
rootCauseStatus: "confirmed-from-dom-samples",
|
|
rootCauseConfidence: "high",
|
|
dominantMismatchKind: "completed-card-final-response-absent",
|
|
summary: "Workbench terminal turn card did not expose a structured Final Response body",
|
|
nextAction: "Inspect HWLAB terminal message/finalResponse projection contract before changing renderer fallback behavior.",
|
|
sourceOfTruth: "durable Workbench terminal message projection",
|
|
valuesRedacted: true,
|
|
};
|
|
const mismatchKinds = Array.isArray(drilldown?.summary?.mismatchKinds) ? drilldown.summary.mismatchKinds : [];
|
|
return {
|
|
rootCause: "workbench_projection_state_triad_not_sealed",
|
|
rootCauseStatus: "confirmed-from-dom-samples",
|
|
rootCauseConfidence: "high",
|
|
dominantMismatchKind: mismatchKinds[0] ?? "unknown",
|
|
summary: "Workbench session rail status, turn card status, and Final Response body presence diverged from the allowed state tuples",
|
|
nextAction: "Use drilldown.otelDrilldown.commands for the listed traceIds, then inspect staticSourceHints and add unit tests from unitTestReproHints before changing UI rendering.",
|
|
sourceOfTruth: "durable Workbench projection/read model",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function workbenchTriadTuple(row) {
|
|
return [
|
|
"rail=" + (row?.railStatus ?? "-"),
|
|
"card=" + (row?.cardStatus ?? "-"),
|
|
"final=" + String(row?.finalResponsePresent ?? "unknown"),
|
|
].join(",");
|
|
}
|
|
|
|
function minNumberOrValue(left, right) {
|
|
const l = Number(left);
|
|
const r = Number(right);
|
|
if (!Number.isFinite(l)) return right ?? left ?? null;
|
|
if (!Number.isFinite(r)) return left ?? right ?? null;
|
|
return Math.min(l, r);
|
|
}
|
|
|
|
function maxNumberOrValue(left, right) {
|
|
const l = Number(left);
|
|
const r = Number(right);
|
|
if (!Number.isFinite(l)) return right ?? left ?? null;
|
|
if (!Number.isFinite(r)) return left ?? right ?? null;
|
|
return Math.max(l, r);
|
|
}
|
|
|
|
function isoSpanMs(firstAt, lastAt) {
|
|
const start = Date.parse(firstAt || "");
|
|
const end = Date.parse(lastAt || "");
|
|
if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return 0;
|
|
return end - start;
|
|
}
|
|
|
|
function workbenchTriadStaticSourceHints() {
|
|
return [
|
|
{ repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-session.ts", reason: "session rail/list status and active session projection source" },
|
|
{ repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-server-state.ts", reason: "server snapshot merge path that can overwrite sealed turn state" },
|
|
{ repo: "pikasTech/HWLAB", path: "web/hwlab-cloud-web/src/stores/workbench-message-projection-runtime.ts", reason: "turn card and Final Response projection/runtime merge path" },
|
|
{ repo: "pikasTech/HWLAB", path: "internal/cloud/server-workbench-http*.go", reason: "session list/detail/messages/turn REST read-model contract" },
|
|
{ repo: "pikasTech/HWLAB", path: "internal/cloud/workbench-projection-*.go", reason: "durable projection writer and terminal seal contract" },
|
|
];
|
|
}
|
|
|
|
function workbenchTriadUnitTestReproHints() {
|
|
return [
|
|
"frontend reducer: when a second trace is running in the same session, session rail status must not stay on the previous completed terminal trace",
|
|
"backend projector: terminal event must produce a single sealed turn tuple consumed by session list, session detail, messages and turn-status APIs",
|
|
"backend read model: completed rail status must not coexist with running turn card or missing Final Response for the same trace",
|
|
"frontend server-state merge: stale running/empty snapshots must not overwrite a sealed completed+Final Response turn",
|
|
"frontend projection runtime: cross-page hydration must converge from the same durable projection without DOM-only repair",
|
|
];
|
|
}
|
|
|
|
function workbenchTurnStateTriadRow(sample, turn, rail, promptIndex) {
|
|
const collectorMissing = [];
|
|
const railRawStatus = firstString(rail?.status, rail?.dataStatus);
|
|
const railRunning = rail?.running === true || String(rail?.dataRunning || rail?.ariaBusy || "").toLowerCase() === "true";
|
|
const railStatus = normalizeWorkbenchTriadStatus(railRawStatus, railRunning);
|
|
if (!rail) collectorMissing.push("sessionRail.activeItem");
|
|
if (!railRawStatus && railRunning !== true) collectorMissing.push("sessionRail.status");
|
|
const cardRawStatus = firstString(turn?.status);
|
|
const cardStatus = normalizeWorkbenchTriadStatus(cardRawStatus, false) || (workbenchDomItemIsTerminal(turn) ? "completed" : null);
|
|
if (!cardRawStatus && !cardStatus) collectorMissing.push("turn.status");
|
|
const finalPresence = finalResponsePresenceFromTurn(turn);
|
|
if (finalPresence.known !== true) collectorMissing.push("turn.finalResponseTextBytes");
|
|
const fullTriad = Boolean(railStatus && cardStatus && finalPresence.known === true);
|
|
const finalResponsePresent = finalPresence.known === true ? finalPresence.present : null;
|
|
const tupleAllowed = fullTriad
|
|
? (railStatus === "completed" && cardStatus === "completed" && finalResponsePresent === true)
|
|
|| (railStatus === "running" && cardStatus === "running" && finalResponsePresent === false)
|
|
: null;
|
|
const cardFinalResponseMismatch = cardStatus === "completed" && finalPresence.known === true && finalResponsePresent !== true;
|
|
const legacyCardFinalResponseMismatch = cardStatus === "completed" && finalPresence.known !== true && !workbenchDomItemLooksFinal(turn);
|
|
const railCardMismatch = Boolean(railStatus && cardStatus && railStatus !== cardStatus);
|
|
const invalid = tupleAllowed === false || cardFinalResponseMismatch || legacyCardFinalResponseMismatch || railCardMismatch;
|
|
const traceId = firstString(turn?.traceId, turn?.turnId);
|
|
const messageId = firstString(turn?.messageId);
|
|
const rowKey = [
|
|
sample?.seq ?? "-",
|
|
sample?.pageRole ?? "control",
|
|
sample?.pageId ?? "default",
|
|
sample?.routeSessionId ?? sample?.activeSessionId ?? rail?.sessionIdPrefix ?? "-",
|
|
traceId ?? messageId ?? turn?.index ?? "-"
|
|
].join("|");
|
|
return {
|
|
...ref(sample),
|
|
rowKey,
|
|
promptIndex,
|
|
sessionIdPrefix: sessionPrefixForSample(sample, rail),
|
|
traceId,
|
|
messageId,
|
|
turnId: firstString(turn?.turnId),
|
|
turnIndex: turn?.index ?? null,
|
|
railStatus,
|
|
railStatusRaw: railRawStatus ?? null,
|
|
railRunning: railRunning === true,
|
|
railSource: rail ? "sessionRail.activeItem" : null,
|
|
cardStatus,
|
|
cardStatusRaw: cardRawStatus ?? null,
|
|
finalResponsePresent,
|
|
finalResponseTextBytes: turn?.finalResponseTextBytes !== null && turn?.finalResponseTextBytes !== undefined && Number.isFinite(Number(turn.finalResponseTextBytes)) ? Number(turn.finalResponseTextBytes) : null,
|
|
finalResponseSource: turn?.finalResponseTextSource ?? null,
|
|
finalResponseTextHash: turn?.finalResponseTextHash ?? null,
|
|
finalResponseTextPreview: turn?.finalResponseTextPreview ? limitText(turn.finalResponseTextPreview, 160) : null,
|
|
fullTriad,
|
|
tupleAllowed,
|
|
invalid,
|
|
cardFinalResponseMismatch,
|
|
legacyCardFinalResponseMismatch,
|
|
railCardMismatch,
|
|
collectorMissing,
|
|
nearestCommandId: sample?.commandId ?? null,
|
|
relatedChecks: ["WBC-001", "WBC-003", "WBC-011", "WBC-022"],
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function activeSessionRailItemForSample(sample) {
|
|
const rail = sample?.sessionRail && typeof sample.sessionRail === "object" ? sample.sessionRail : null;
|
|
if (!rail) return null;
|
|
const items = Array.isArray(rail.items) ? rail.items : [];
|
|
if (rail.activeItem) return rail.activeItem;
|
|
const routeSessionId = String(sample?.routeSessionId || "");
|
|
const activeSessionId = String(sample?.activeSessionId || "");
|
|
return items.find((item) => item?.active === true)
|
|
|| items.find((item) => sessionRailItemMatchesSession(item, activeSessionId))
|
|
|| items.find((item) => sessionRailItemMatchesSession(item, routeSessionId))
|
|
|| null;
|
|
}
|
|
|
|
function sessionRailItemMatchesSession(item, sessionId) {
|
|
const id = String(sessionId || "");
|
|
const prefix = String(item?.sessionIdPrefix || item?.sessionId || "");
|
|
return Boolean(id && prefix && (id === prefix || id.startsWith(prefix) || prefix.startsWith(id)));
|
|
}
|
|
|
|
function sessionPrefixForSample(sample, rail) {
|
|
const id = firstString(sample?.activeSessionId, sample?.routeSessionId, rail?.sessionIdPrefix, rail?.sessionId);
|
|
return id ? String(id).slice(0, 12) : null;
|
|
}
|
|
|
|
function finalResponsePresenceFromTurn(turn) {
|
|
if (!turn || typeof turn !== "object") return { known: false, present: null };
|
|
if (turn.finalResponseTextBytes !== null && turn.finalResponseTextBytes !== undefined && Number.isFinite(Number(turn.finalResponseTextBytes))) {
|
|
const bytes = Number(turn.finalResponseTextBytes);
|
|
return { known: true, present: bytes > 0 };
|
|
}
|
|
if (Object.prototype.hasOwnProperty.call(turn, "finalResponsePresent") && turn.finalResponsePresent !== null && turn.finalResponsePresent !== undefined) {
|
|
return { known: true, present: turn.finalResponsePresent === true };
|
|
}
|
|
return { known: false, present: null };
|
|
}
|
|
|
|
function normalizeWorkbenchTriadStatus(status, running = false) {
|
|
const value = String(status || "").trim().toLowerCase().replace(/_/gu, "-");
|
|
if (running === true) return "running";
|
|
if (!value) return null;
|
|
if (/^(completed|complete|succeeded|success|finished|done|terminal|sealed)$/u.test(value)) return "completed";
|
|
if (/^(failed|failure|error|blocked|timeout|canceled|cancelled|stale|thread-resume-failed|interrupted|expired|idle)$/u.test(value)) return "completed";
|
|
if (/^(pending|running|active|busy|admitted|dispatching|executing|streaming|processing|queued|in-progress|creating)$/u.test(value)) return "running";
|
|
return null;
|
|
}
|
|
|
|
function firstString(...values) {
|
|
for (const value of values) {
|
|
if (typeof value !== "string") continue;
|
|
const text = value.trim();
|
|
if (text) return text;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function promptCommandHasAuthoritativeSubmitSideEffect(control, promptRound) {
|
|
const commandId = stringOrNull(promptRound?.promptCommandId);
|
|
if (!commandId) return false;
|
|
const row = (control || []).find((item) => item?.type === "sendPrompt" && item?.phase === "completed" && item?.commandId === commandId);
|
|
const detail = objectValue(row?.detail);
|
|
const chatSubmit = objectValue(detail.chatSubmit);
|
|
const sideEffect = objectValue(chatSubmit.sideEffect);
|
|
return chatSubmit.sideEffectObserved === true
|
|
|| sideEffect.submitted === true
|
|
|| Number(sideEffect.messageCountDelta ?? 0) > 0;
|
|
}
|
|
`;
|
|
}
|