feat(monitor): surface workbench sentinel root causes
This commit is contained in:
Codex
@@ -1810,6 +1810,115 @@ function groupApiDomLagCandidates(candidates) {
|
||||
.sort((a, b) => Number(b.maxDomChangeDeltaMs ?? -1) - Number(a.maxDomChangeDeltaMs ?? -1) || b.count - a.count || String(a.path).localeCompare(String(b.path)));
|
||||
}
|
||||
|
||||
function detectTraceEventsPageReadIssues(network) {
|
||||
const events = (Array.isArray(network) ? network : [])
|
||||
.filter((item) => item?.observerInitiated !== true && (item?.type === "response" || item?.type === "requestfailed"))
|
||||
.map(compactTraceEventsPageReadEvent)
|
||||
.filter((item) => item !== null);
|
||||
const http404 = events.filter((item) => item.type === "response" && Number(item.status) === 404);
|
||||
const httpErrors = events.filter((item) => item.type === "response" && Number(item.status) >= 400 && Number(item.status) !== 404);
|
||||
const requestFailed = events.filter((item) => item.type === "requestfailed");
|
||||
return {
|
||||
events,
|
||||
http404,
|
||||
httpErrors,
|
||||
requestFailed,
|
||||
summary: traceEventsPageReadIssueSummary(events),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function compactTraceEventsPageReadEvent(item) {
|
||||
const parsed = parseTraceEventsPageReadUrl(item?.url);
|
||||
if (!parsed.match) return null;
|
||||
const failureText = item?.failureKind ?? item?.failure ?? item?.errorText ?? null;
|
||||
return {
|
||||
ts: item?.ts ?? null,
|
||||
pageRole: item?.pageRole ?? null,
|
||||
pageId: item?.pageId ?? null,
|
||||
commandId: item?.commandId ?? null,
|
||||
method: String(item?.method || "GET").toUpperCase(),
|
||||
type: item?.type ?? null,
|
||||
status: Number.isFinite(Number(item?.status)) ? Number(item.status) : null,
|
||||
path: "/v1/workbench/traces/:traceId/events",
|
||||
rawPath: parsed.rawPath,
|
||||
traceId: parsed.traceId,
|
||||
afterProjectedSeq: parsed.afterProjectedSeq,
|
||||
sinceSeq: parsed.sinceSeq,
|
||||
limit: parsed.limit,
|
||||
tail: parsed.tail,
|
||||
queryKeys: parsed.queryKeys,
|
||||
failureKind: failureText ? limitText(String(failureText), 120) : null,
|
||||
urlHash: item?.url ? sha256(item.url) : null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function parseTraceEventsPageReadUrl(value) {
|
||||
const fallback = {
|
||||
match: false,
|
||||
rawPath: urlPath(value),
|
||||
traceId: firstIdInText(String(value || ""), /\btrc_[A-Za-z0-9_-]+\b/u),
|
||||
afterProjectedSeq: null,
|
||||
sinceSeq: null,
|
||||
limit: null,
|
||||
tail: null,
|
||||
queryKeys: [],
|
||||
};
|
||||
try {
|
||||
const parsed = new URL(String(value || ""), "http://invalid.local/");
|
||||
const rawPath = parsed.pathname || "-";
|
||||
const match = rawPath.match(/^\/v1\/workbench\/traces\/([^/]+)\/events$/u);
|
||||
return {
|
||||
match: Boolean(match),
|
||||
rawPath,
|
||||
traceId: match ? decodeURIComponent(match[1]) : fallback.traceId,
|
||||
afterProjectedSeq: numericSearchParam(parsed.searchParams, "afterProjectedSeq"),
|
||||
sinceSeq: numericSearchParam(parsed.searchParams, "sinceSeq") ?? numericSearchParam(parsed.searchParams, "afterSeq"),
|
||||
limit: numericSearchParam(parsed.searchParams, "limit"),
|
||||
tail: numericSearchParam(parsed.searchParams, "tail"),
|
||||
queryKeys: Array.from(parsed.searchParams.keys()).sort().slice(0, 12),
|
||||
};
|
||||
} catch {
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function numericSearchParam(searchParams, key) {
|
||||
const raw = searchParams?.get?.(key);
|
||||
if (raw === null || raw === undefined || raw === "") return null;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function traceEventsPageReadIssueSummary(events) {
|
||||
const items = Array.isArray(events) ? events : [];
|
||||
const statuses = uniqueSorted(items.map((item) => item.status).filter((item) => item !== null && item !== undefined).map(String));
|
||||
const traceIds = uniqueSorted(items.map((item) => item.traceId).filter(Boolean).map(String)).slice(0, 8);
|
||||
const afterProjectedSeqs = uniqueSorted(items.map((item) => item.afterProjectedSeq).filter((item) => item !== null && item !== undefined).map(String)).slice(0, 8);
|
||||
const sinceSeqs = uniqueSorted(items.map((item) => item.sinceSeq).filter((item) => item !== null && item !== undefined).map(String)).slice(0, 8);
|
||||
const failureKinds = uniqueSorted(items.map((item) => item.failureKind).filter(Boolean).map(String)).slice(0, 6);
|
||||
return {
|
||||
eventCount: items.length,
|
||||
responseErrorCount: items.filter((item) => item.type === "response" && Number(item.status) >= 400).length,
|
||||
http404Count: items.filter((item) => item.type === "response" && Number(item.status) === 404).length,
|
||||
requestFailedCount: items.filter((item) => item.type === "requestfailed").length,
|
||||
statuses,
|
||||
traceIds,
|
||||
afterProjectedSeqs,
|
||||
sinceSeqs,
|
||||
failureKinds,
|
||||
firstAt: items.reduce((value, item) => minIso(value, item.ts ?? null), null),
|
||||
lastAt: items.reduce((value, item) => maxIso(value, item.ts ?? null), null),
|
||||
rootCauseVisibility: "browser network rows identify trace-events page-read path; OTel trace_events_read should confirm backend paging fields",
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function uniqueSorted(values) {
|
||||
return Array.from(new Set((values || []).filter((item) => item !== null && item !== undefined).map(String))).sort();
|
||||
}
|
||||
|
||||
function compactApiDomLagForOutput(report) {
|
||||
if (!report || typeof report !== "object") return null;
|
||||
return {
|
||||
@@ -1954,7 +2063,70 @@ function buildFindings(samples, control, network, errors, sampleMetrics, promptN
|
||||
if (Number(loadingSummary.longestContinuousSeconds ?? 0) > visibleLoadingSlowSeconds) findings.push({ id: "page-loading-visible-over-budget", severity: "red", summary: "visible 加载中 stayed on screen longer than configured YAML budget; fix real loading latency instead of revealing incomplete content early", count: loadingSummary.overBudgetSegmentCount ?? loadingSummary.overFiveSecondSegmentCount ?? 1, longestContinuousSeconds: loadingSummary.longestContinuousSeconds, budgetSeconds: visibleLoadingSlowSeconds, segments: sampleMetrics.loading.segments.slice(0, 20), owners: sampleMetrics.loading.owners.slice(0, 20) });
|
||||
if (Number(loadingSummary.maxSimultaneousCount ?? 0) > 1) findings.push({ id: "page-loading-concurrent", severity: "info", summary: "multiple 加载中 indicators were visible in the same sampled DOM point", count: loadingSummary.concurrentLoadingSampleCount ?? 0, maxSimultaneousCount: loadingSummary.maxSimultaneousCount, owners: sampleMetrics.loading.owners.slice(0, 20) });
|
||||
const sessionRailTitleSummary = sampleMetrics?.sessionRailTitles?.summary || {};
|
||||
if (Number(sessionRailTitleSummary.overThresholdSampleCount ?? sessionRailTitleSummary.majorityFallbackSampleCount ?? 0) > 0) findings.push({ id: "session-rail-title-fallback-over-threshold", severity: "red", summary: "visible session list rows exceeded configured YAML fallback-title ratio", count: sessionRailTitleSummary.overThresholdSampleCount ?? sessionRailTitleSummary.majorityFallbackSampleCount, thresholdRatio: alertThresholds.sessionRailFallbackRatio, maxFallbackRatio: sessionRailTitleSummary.maxFallbackRatio, maxFallbackTitleCount: sessionRailTitleSummary.maxFallbackTitleCount, samples: sampleMetrics.sessionRailTitles.samples.slice(0, 20), examples: sampleMetrics.sessionRailTitles.examples.slice(0, 20) });
|
||||
if (Number(sessionRailTitleSummary.overThresholdSampleCount ?? sessionRailTitleSummary.majorityFallbackSampleCount ?? 0) > 0) findings.push({
|
||||
id: "session-rail-title-fallback-root-cause",
|
||||
severity: "red",
|
||||
summary: "INV-02 root cause visible: session rail is rendering fallback Session ses_* titles, so list projection/read model or rail binding is missing stable title/preview data before DOM render",
|
||||
rootCause: "session-list-title-projection-missing-or-not-bound",
|
||||
rootCauseStatus: "confirmed-from-dom-session-rail",
|
||||
rootCauseConfidence: "high",
|
||||
nextAction: "Check OTel session_list_read fallbackTitleCount/fallbackTitleRatio for the same run; fix session list projection/read model title/firstUserMessagePreview before changing DOM fallback text.",
|
||||
count: sessionRailTitleSummary.overThresholdSampleCount ?? sessionRailTitleSummary.majorityFallbackSampleCount,
|
||||
thresholdRatio: alertThresholds.sessionRailFallbackRatio,
|
||||
maxFallbackRatio: sessionRailTitleSummary.maxFallbackRatio,
|
||||
maxFallbackTitleCount: sessionRailTitleSummary.maxFallbackTitleCount,
|
||||
evidence: {
|
||||
thresholdRatio: alertThresholds.sessionRailFallbackRatio,
|
||||
maxFallbackRatio: sessionRailTitleSummary.maxFallbackRatio,
|
||||
maxFallbackTitleCount: sessionRailTitleSummary.maxFallbackTitleCount,
|
||||
overThresholdSampleCount: sessionRailTitleSummary.overThresholdSampleCount ?? null,
|
||||
majorityFallbackSampleCount: sessionRailTitleSummary.majorityFallbackSampleCount ?? null,
|
||||
valuesRedacted: true
|
||||
},
|
||||
samples: sampleMetrics.sessionRailTitles.samples.slice(0, 20),
|
||||
examples: sampleMetrics.sessionRailTitles.examples.slice(0, 20),
|
||||
valuesRedacted: true
|
||||
});
|
||||
const traceEventsPageReadIssues = detectTraceEventsPageReadIssues(network);
|
||||
if (traceEventsPageReadIssues.http404.length > 0) findings.push({
|
||||
id: "trace-events-page-read-404-root-cause",
|
||||
severity: "red",
|
||||
summary: "INV-07 root cause visible: /v1/workbench/traces/:traceId/events returned HTTP 404 for a trace event page read, so the failure is in the trace-events API paging/read-model contract before DOM rendering",
|
||||
rootCause: "trace-events-api-page-read-returned-404",
|
||||
rootCauseStatus: "confirmed-from-browser-network",
|
||||
rootCauseConfidence: "high",
|
||||
nextAction: "Use OTel trace_events_read for the same trace to compare sinceSeq/afterProjectedSeq, returnedEvents, range, totalEvents, hasMore and fullTraceLoaded; fix backend paging contract or add missing instrumentation before changing renderer behavior.",
|
||||
count: traceEventsPageReadIssues.http404.length,
|
||||
evidence: traceEventsPageReadIssues.summary,
|
||||
samples: traceEventsPageReadIssues.http404.slice(0, 20),
|
||||
valuesRedacted: true
|
||||
});
|
||||
if (traceEventsPageReadIssues.httpErrors.length > 0) findings.push({
|
||||
id: "trace-events-page-read-http-error-root-cause",
|
||||
severity: "red",
|
||||
summary: "trace events page read returned HTTP error status; monitor can localize this to the trace-events API instead of a generic Workbench render failure",
|
||||
rootCause: "trace-events-api-page-read-http-error",
|
||||
rootCauseStatus: "confirmed-from-browser-network",
|
||||
rootCauseConfidence: "high",
|
||||
nextAction: "Drill down by traceId and afterProjectedSeq, then compare with OTel trace_events_read paging fields.",
|
||||
count: traceEventsPageReadIssues.httpErrors.length,
|
||||
evidence: traceEventsPageReadIssues.summary,
|
||||
samples: traceEventsPageReadIssues.httpErrors.slice(0, 20),
|
||||
valuesRedacted: true
|
||||
});
|
||||
if (traceEventsPageReadIssues.requestFailed.length > 0) findings.push({
|
||||
id: "trace-events-page-read-requestfailed-root-cause",
|
||||
severity: "amber",
|
||||
summary: "trace events page read failed at browser/network level; monitor localized the failure path, but HTTP status is unavailable so OTel/API instrumentation is needed to confirm the backend root cause",
|
||||
rootCause: "trace-events-api-page-read-network-failed",
|
||||
rootCauseStatus: "network-signal-needs-otel-confirmation",
|
||||
rootCauseConfidence: "medium",
|
||||
nextAction: "Check whether this happened during observer refresh/navigation; if not, query OTel by /v1/workbench/traces/:traceId/events and add route span fields when status/afterProjectedSeq are missing.",
|
||||
count: traceEventsPageReadIssues.requestFailed.length,
|
||||
evidence: traceEventsPageReadIssues.summary,
|
||||
samples: traceEventsPageReadIssues.requestFailed.slice(0, 20),
|
||||
valuesRedacted: true
|
||||
});
|
||||
if ((runtimeAlerts?.summary?.httpErrorCount ?? 0) > 0) findings.push({ id: "runtime-http-errors", severity: "amber", summary: "natural page requests returned HTTP error status during observation", count: runtimeAlerts.summary.httpErrorCount, groups: runtimeAlerts.networkHttpErrorsByPath.slice(0, 12) });
|
||||
if ((runtimeAlerts?.summary?.significantRequestFailedCount ?? runtimeAlerts?.summary?.requestFailedCount ?? 0) > 0) findings.push({ id: "runtime-requestfailed", severity: "amber", summary: "browser requestfailed events were captured during observation", count: runtimeAlerts.summary.significantRequestFailedCount ?? runtimeAlerts.summary.requestFailedCount, groups: (runtimeAlerts.networkSignificantRequestFailedByPath ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 12) });
|
||||
if ((runtimeAlerts?.summary?.domDiagnosticSampleCount ?? 0) > 0) findings.push({ id: "runtime-dom-diagnostics", severity: "amber", summary: "diagnostic/error/warning-like text was visible in sampled DOM", count: runtimeAlerts.summary.domDiagnosticSampleCount, groupCount: runtimeAlerts.summary.domDiagnosticGroupCount ?? 0, groups: runtimeAlerts.domDiagnosticsByText.slice(0, 12), samples: runtimeAlerts.domDiagnostics.slice(0, 12) });
|
||||
|
||||
Reference in New Issue
Block a user