feat: add sentinel error detail deep links
This commit is contained in:
Codex
@@ -279,7 +279,29 @@ export function nodeWebObserveAnalyzerFindingsSource(): string {
|
||||
samples: traceEventsPageReadIssues.requestFailed.slice(0, 20),
|
||||
valuesRedacted: true
|
||||
});
|
||||
if ((runtimeAlerts?.summary?.httpErrorCount ?? 0) > 0) findings.push({ id: "runtime-http-errors", severity: "amber", summary: "natural page requests returned HTTP error status during observation", count: runtimeAlerts.summary.httpErrorCount, groups: runtimeAlerts.networkHttpErrorsByPath.slice(0, 12) });
|
||||
const pageResponseErrorGroups = Array.isArray(runtimeAlerts?.networkPageResponseErrorsByPath)
|
||||
? runtimeAlerts.networkPageResponseErrorsByPath
|
||||
: (Array.isArray(runtimeAlerts?.networkHttpErrorsByPath) ? runtimeAlerts.networkHttpErrorsByPath.filter((item) => Number(item?.status) >= 500) : []);
|
||||
const nonBlockingHttpErrorGroups = Array.isArray(runtimeAlerts?.networkNonBlockingHttpErrorsByPath)
|
||||
? runtimeAlerts.networkNonBlockingHttpErrorsByPath
|
||||
: (Array.isArray(runtimeAlerts?.networkHttpErrorsByPath) ? runtimeAlerts.networkHttpErrorsByPath.filter((item) => Number(item?.status) < 500) : []);
|
||||
if ((runtimeAlerts?.summary?.pageResponseErrorCount ?? pageResponseErrorGroups.reduce((sum, item) => sum + Number(item?.count ?? 0), 0)) > 0) findings.push({
|
||||
id: "page-response-error",
|
||||
severity: "red",
|
||||
summary: "页面响应错误：浏览器在运行页面捕获到 HTTP 5xx 响应，属于阻塞错误。",
|
||||
displayTitleZh: "页面响应错误",
|
||||
errorTitleZh: "页面响应错误",
|
||||
rootCause: "page-response-http-5xx",
|
||||
rootCauseStatus: "confirmed-from-browser-network-response",
|
||||
rootCauseConfidence: "high",
|
||||
nextAction: "按状态码和路径追溯后端日志/OTel；先修复 5xx 响应，不要把它归类为前端渲染或采样抖动。",
|
||||
blocking: true,
|
||||
count: runtimeAlerts?.summary?.pageResponseErrorCount ?? pageResponseErrorGroups.reduce((sum, item) => sum + Number(item?.count ?? 0), 0),
|
||||
evidenceSummary: pageResponseErrorGroups.slice(0, 4).map((item) => "HTTP " + (item.status ?? "-") + " " + (item.method || "-") + " " + (item.urlPath || "-") + " count=" + (item.count ?? "-")).join("; "),
|
||||
groups: pageResponseErrorGroups.slice(0, 12),
|
||||
valuesRedacted: true
|
||||
});
|
||||
if ((runtimeAlerts?.summary?.nonBlockingHttpErrorCount ?? nonBlockingHttpErrorGroups.reduce((sum, item) => sum + Number(item?.count ?? 0), 0)) > 0) findings.push({ id: "runtime-http-errors", severity: "amber", summary: "natural page requests returned non-5xx HTTP error status during observation", count: runtimeAlerts?.summary?.nonBlockingHttpErrorCount ?? nonBlockingHttpErrorGroups.reduce((sum, item) => sum + Number(item?.count ?? 0), 0), groups: nonBlockingHttpErrorGroups.slice(0, 12) });
|
||||
if ((runtimeAlerts?.summary?.significantRequestFailedCount ?? runtimeAlerts?.summary?.requestFailedCount ?? 0) > 0) findings.push({ id: "runtime-requestfailed", severity: "amber", summary: "browser requestfailed events were captured during observation", count: runtimeAlerts.summary.significantRequestFailedCount ?? runtimeAlerts.summary.requestFailedCount, groups: (runtimeAlerts.networkSignificantRequestFailedByPath ?? runtimeAlerts.networkRequestFailedByPath).slice(0, 12) });
|
||||
if ((runtimeAlerts?.summary?.domDiagnosticSampleCount ?? 0) > 0) findings.push({ id: "runtime-dom-diagnostics", severity: "amber", summary: "diagnostic/error/warning-like text was visible in sampled DOM", count: runtimeAlerts.summary.domDiagnosticSampleCount, groupCount: runtimeAlerts.summary.domDiagnosticGroupCount ?? 0, groups: runtimeAlerts.domDiagnosticsByText.slice(0, 12), samples: runtimeAlerts.domDiagnostics.slice(0, 12) });
|
||||
if ((runtimeAlerts?.summary?.executionErrorCount ?? 0) > 0) findings.push({ id: "runtime-execution-errors", severity: "red", summary: "Workbench rendered execution failure/error rows during observation", count: runtimeAlerts.summary.executionErrorCount, groups: runtimeAlerts.runtimeExecutionErrorsByCode.slice(0, 12) });
|
||||
|
||||
@@ -440,6 +440,8 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) {
|
||||
const httpErrors = naturalNetwork
|
||||
.filter((item) => item?.type === "response" && Number(item.status) >= 400)
|
||||
.map((item) => networkAlertEvent(item, promptTimes));
|
||||
const pageResponseErrors = httpErrors.filter((item) => Number(item.status) >= 500);
|
||||
const nonBlockingHttpErrors = httpErrors.filter((item) => Number(item.status) < 500);
|
||||
const requestFailed = naturalNetwork
|
||||
.filter((item) => item?.type === "requestfailed")
|
||||
.map((item) => networkAlertEvent(item, promptTimes));
|
||||
@@ -586,6 +588,8 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) {
|
||||
return {
|
||||
summary: {
|
||||
httpErrorCount: httpErrors.length,
|
||||
pageResponseErrorCount: pageResponseErrors.length,
|
||||
nonBlockingHttpErrorCount: nonBlockingHttpErrors.length,
|
||||
requestFailedCount: requestFailed.length,
|
||||
significantRequestFailedCount: significantRequestFailed.length,
|
||||
workbenchSessionListReadCount,
|
||||
@@ -606,6 +610,7 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) {
|
||||
significantConsoleAlertCount: significantConsoleAlerts.length,
|
||||
pageErrorCount: pageErrors.length,
|
||||
networkErrorGroupCount: groupNetworkAlerts(httpErrors).length,
|
||||
pageResponseErrorGroupCount: groupNetworkAlerts(pageResponseErrors).length,
|
||||
requestFailedGroupCount: groupNetworkAlerts(requestFailed).length,
|
||||
significantRequestFailedGroupCount: groupNetworkAlerts(significantRequestFailed).length,
|
||||
executionErrorGroupCount: groupExecutionErrors(executionErrors).length,
|
||||
@@ -614,6 +619,8 @@ function buildRuntimeAlerts(samples, control, network, consoleEvents, errors) {
|
||||
significantConsoleAlertGroupCount: groupConsoleAlerts(significantConsoleAlerts).length
|
||||
},
|
||||
networkHttpErrorsByPath: groupNetworkAlerts(httpErrors),
|
||||
networkPageResponseErrorsByPath: groupNetworkAlerts(pageResponseErrors),
|
||||
networkNonBlockingHttpErrorsByPath: groupNetworkAlerts(nonBlockingHttpErrors),
|
||||
networkRequestFailedByPath: groupNetworkAlerts(requestFailed),
|
||||
networkSignificantRequestFailedByPath: groupNetworkAlerts(significantRequestFailed),
|
||||
webPerformancePayloadStates,
|
||||
|
||||
@@ -115,6 +115,17 @@ export type WebProbeSentinelOptions =
|
||||
readonly full: boolean;
|
||||
readonly timeoutSeconds: number;
|
||||
}
|
||||
| {
|
||||
readonly kind: "error";
|
||||
readonly action: "get";
|
||||
readonly node: string;
|
||||
readonly lane: string;
|
||||
readonly sentinelId: string | null;
|
||||
readonly errorId: string;
|
||||
readonly raw: boolean;
|
||||
readonly full: boolean;
|
||||
readonly timeoutSeconds: number;
|
||||
}
|
||||
| {
|
||||
readonly kind: "dashboard";
|
||||
readonly action: WebProbeSentinelDashboardAction;
|
||||
|
||||
@@ -22,7 +22,7 @@ import { readWebProbeSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-c
|
||||
import { effectiveWebProbeSentinelPublicExposure, requireSentinelIdForRegistry, resolveWebProbeSentinel } from "./hwlab-node-web-sentinel-resolver";
|
||||
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
||||
import type { RenderedCliResult } from "./output";
|
||||
import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
|
||||
import { probeSentinelRuntimeHealthEndpoint, runSentinelDashboard, runSentinelErrorDetail, runSentinelMaintenance, runSentinelReport, runSentinelValidate } from "./hwlab-node-web-sentinel-p5";
|
||||
import { runChildCli, sentinelP5Next } from "./hwlab-node-web-sentinel-p5-observe";
|
||||
import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
|
||||
import {
|
||||
@@ -159,6 +159,7 @@ export function runWebProbeSentinelCommand(spec: HwlabRuntimeLaneSpec, options:
|
||||
if (options.kind === "maintenance") return runSentinelMaintenance(state, options);
|
||||
if (options.kind === "validate") return runSentinelValidate(state, options);
|
||||
if (options.kind === "dashboard") return runSentinelDashboard(state, options);
|
||||
if (options.kind === "error") return runSentinelErrorDetail(state, options);
|
||||
return runSentinelReport(state, options);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
// SPEC: PJ2026-01060508 Web哨兵 issue-1569 error detail deep links.
|
||||
// Responsibility: Stable monitor-web error ids, bounded error detail payloads, and server-side OTel enrichment.
|
||||
import { createHash } from "node:crypto";
|
||||
import type { Database } from "bun:sqlite";
|
||||
import type { WebProbeSentinelServiceConfig } from "./hwlab-node-web-sentinel-service";
|
||||
|
||||
export interface WebProbeSentinelErrorReaders {
|
||||
readonly findingsForRow: (row: Record<string, unknown>, limit: number) => readonly Record<string, unknown>[];
|
||||
readonly runSummary: (row: Record<string, unknown>) => Record<string, unknown>;
|
||||
readonly traceability: (row: Record<string, unknown>) => Record<string, unknown>;
|
||||
}
|
||||
|
||||
export function decorateRunFindingsWithErrorDetails(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>, findings: readonly Record<string, unknown>[]): Record<string, unknown>[] {
|
||||
return findings.map((finding, index) => {
|
||||
const detail = buildErrorDetail(config, row, finding, index, false);
|
||||
return {
|
||||
...finding,
|
||||
errorId: detail.identity.errorId,
|
||||
detail,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function dashboardErrorDetail(config: WebProbeSentinelServiceConfig, db: Database, errorId: string, readers: WebProbeSentinelErrorReaders): Promise<Record<string, unknown>> {
|
||||
if (!/^wbe_[a-f0-9]{20}$/u.test(errorId)) return { ok: false, error: "invalid-error-id", errorId, valuesRedacted: true };
|
||||
const rows = db.query("SELECT * FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY created_at DESC LIMIT ?")
|
||||
.all(config.sentinelId, config.node, config.lane, 200) as Record<string, unknown>[];
|
||||
for (const row of rows) {
|
||||
const findings = decorateRunFindingsWithErrorDetails(config, row, readers.findingsForRow(row, 200));
|
||||
const findingIndex = findings.findIndex((item) => stringOrNull(item.errorId) === errorId);
|
||||
if (findingIndex < 0) continue;
|
||||
const finding = findings[findingIndex] ?? {};
|
||||
const detail = buildErrorDetail(config, row, finding, findingIndex, true);
|
||||
const traceRefs = arrayRecords(detail.traceRefs);
|
||||
return {
|
||||
ok: true,
|
||||
errorId,
|
||||
run: readers.runSummary(row),
|
||||
finding,
|
||||
detail: {
|
||||
...detail,
|
||||
otel: await buildOtelSummary(traceRefs),
|
||||
},
|
||||
traceability: readers.traceability(row),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
error: "error-not-found",
|
||||
errorId,
|
||||
searchedRunCount: rows.length,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function buildErrorDetail(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>, finding: Record<string, unknown>, occurrenceIndex: number, includeEvidence: boolean): Record<string, unknown> {
|
||||
const identity = errorIdentity(config, row, finding, occurrenceIndex);
|
||||
const traceRefs = extractTraceRefs(finding);
|
||||
return {
|
||||
identity,
|
||||
sentinelEvidence: includeEvidence ? compactSentinelEvidence(finding) : compactSentinelEvidenceSummary(finding),
|
||||
traceRefs,
|
||||
otel: traceRefs.length === 0 ? { status: "trace-ref-missing", summaries: [], valuesRedacted: true } : { status: "detail-api-required", summaries: [], valuesRedacted: true },
|
||||
links: errorLinks(config, identity.errorId),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function errorIdentity(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>, finding: Record<string, unknown>, occurrenceIndex: number): Record<string, unknown> {
|
||||
const runId = stringOrNull(row.id) ?? stringOrNull(finding.runId) ?? stringOrNull(finding.latestRunId) ?? "run";
|
||||
const findingId = stringOrNull(finding.findingId) ?? stringOrNull(finding.finding_id) ?? stringOrNull(finding.id) ?? stringOrNull(finding.kind) ?? stringOrNull(finding.code) ?? "finding";
|
||||
const severity = stringOrNull(finding.severity) ?? stringOrNull(finding.level) ?? stringOrNull(finding.checkLevel) ?? "unknown";
|
||||
const sampleSeq = stringOrNull(finding.sampleSeq) ?? stringOrNull(record(finding.evidence).sampleSeq);
|
||||
const commandId = stringOrNull(finding.commandId) ?? stringOrNull(record(finding.firstCommandFailure).commandId);
|
||||
const groupHash = sha256(JSON.stringify({
|
||||
summary: stringOrNull(finding.summary),
|
||||
evidenceSummary: stringOrNull(finding.evidenceSummary),
|
||||
groups: arrayRecords(finding.groups).slice(0, 4),
|
||||
peaks: arrayRecords(finding.peaks).slice(0, 4),
|
||||
})).slice(0, 16);
|
||||
const stableInput = { node: config.node, lane: config.lane, sentinelId: config.sentinelId, runId, findingId, severity, sampleSeq, commandId, occurrenceIndex, groupHash };
|
||||
return {
|
||||
errorId: `wbe_${sha256(JSON.stringify(stableInput)).slice(0, 20)}`,
|
||||
node: config.node,
|
||||
lane: config.lane,
|
||||
sentinelId: config.sentinelId,
|
||||
runId,
|
||||
scenarioId: stringOrNull(row.scenario_id) ?? stringOrNull(finding.scenarioId),
|
||||
findingId,
|
||||
severity,
|
||||
occurrenceIndex,
|
||||
sampleSeq,
|
||||
commandId,
|
||||
groupHash,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function compactSentinelEvidenceSummary(finding: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
summary: stringOrNull(finding.summary),
|
||||
displayTitleZh: stringOrNull(finding.displayTitleZh) ?? stringOrNull(finding.errorTitleZh),
|
||||
evidenceSummary: stringOrNull(finding.evidenceSummary),
|
||||
rootCause: stringOrNull(finding.rootCause),
|
||||
rootCauseStatus: stringOrNull(finding.rootCauseStatus),
|
||||
rootCauseConfidence: stringOrNull(finding.rootCauseConfidence),
|
||||
nextAction: stringOrNull(finding.nextAction),
|
||||
blocking: finding.blocking === true,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function compactSentinelEvidence(finding: Record<string, unknown>): Record<string, unknown> {
|
||||
return {
|
||||
...compactSentinelEvidenceSummary(finding),
|
||||
firstCommandFailure: compactRecord(record(finding.firstCommandFailure), ["commandId", "type", "phase", "status", "resultOk", "failureKind", "failedReason", "message", "exitCode", "timedOut", "artifactBucket", "artifactRef", "failedAt", "completedAt", "abandonedAt"]),
|
||||
rootCauseSignals: compactRootCauseSignals(finding.rootCauseSignals),
|
||||
browserNetworkErrors: arrayRecords(finding.groups).slice(0, 12).map((item) => compactRecord(item, ["method", "urlPath", "status", "type", "count", "firstAt", "lastAt", "promptIndexes", "failureKinds"])),
|
||||
samples: arrayRecords(finding.samples).slice(0, 8),
|
||||
peaks: arrayRecords(finding.peaks).slice(0, 8),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function buildOtelSummary(traceRefs: readonly Record<string, unknown>[]): Promise<Record<string, unknown>> {
|
||||
if (traceRefs.length === 0) return { status: "trace-ref-missing", summaries: [], valuesRedacted: true };
|
||||
const summaries = [];
|
||||
for (const ref of traceRefs.slice(0, 4)) {
|
||||
const traceId = stringOrNull(ref.traceId);
|
||||
if (traceId === null) continue;
|
||||
if (ref.type !== "otel") {
|
||||
summaries.push({
|
||||
traceId,
|
||||
status: "business-trace-id-not-tempo-id",
|
||||
note: "This id is not a 32 hex OTel trace id; use the linked CLI drill-down to map business trace ids first.",
|
||||
command: `bun scripts/cli.ts platform-infra observability diagnose-code-agent --trace-id ${traceId}`,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
summaries.push(await queryTempoTrace(traceId));
|
||||
}
|
||||
return {
|
||||
status: summaries.some((item) => item.status === "queried") ? "queried" : "not-queried",
|
||||
summaries,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
async function queryTempoTrace(traceId: string): Promise<Record<string, unknown>> {
|
||||
const baseUrl = String(process.env.UNIDESK_WEB_PROBE_SENTINEL_TEMPO_BASE_URL || process.env.UNIDESK_OBSERVABILITY_TEMPO_BASE_URL || "http://tempo.platform-infra.svc.cluster.local:3200").replace(/\/$/u, "");
|
||||
const url = `${baseUrl}/api/traces/${encodeURIComponent(traceId)}`;
|
||||
try {
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(2500), headers: { accept: "application/json" } });
|
||||
const payload = await response.json().catch(() => ({}));
|
||||
if (!response.ok) return { traceId, status: "query-failed", httpStatus: response.status, backend: "tempo", valuesRedacted: true };
|
||||
const spans = tempoSpans(payload);
|
||||
return {
|
||||
traceId,
|
||||
status: "queried",
|
||||
httpStatus: response.status,
|
||||
backend: "tempo",
|
||||
spanCount: spans.length,
|
||||
errorSpanCount: spans.filter((span) => span.error).length,
|
||||
services: [...new Set(spans.map((span) => span.service).filter(Boolean))].slice(0, 12),
|
||||
keySpans: spans.filter((span) => span.error || span.httpStatus !== null).slice(0, 8),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
} catch (error) {
|
||||
return { traceId, status: "query-failed", backend: "tempo", error: error instanceof Error ? error.message.slice(0, 180) : String(error).slice(0, 180), valuesRedacted: true };
|
||||
}
|
||||
}
|
||||
|
||||
function tempoSpans(payload: unknown): Record<string, unknown>[] {
|
||||
const batches = arrayRecords(record(payload).batches ?? record(record(payload).data).batches);
|
||||
const spans: Record<string, unknown>[] = [];
|
||||
for (const batch of batches) {
|
||||
const service = stringOrNull(record(record(batch.resource).attributes?.find?.((item: unknown) => record(item).key === "service.name")).value?.stringValue);
|
||||
for (const scope of arrayRecords(batch.scopeSpans ?? batch.instrumentationLibrarySpans)) {
|
||||
for (const span of arrayRecords(scope.spans)) {
|
||||
const attrs = spanAttributes(span.attributes);
|
||||
const httpStatus = numberOrNull(attrs["http.response.status_code"]) ?? numberOrNull(attrs["http.status_code"]);
|
||||
spans.push({
|
||||
name: stringOrNull(span.name),
|
||||
service,
|
||||
statusCode: stringOrNull(record(span.status).code),
|
||||
statusMessage: stringOrNull(record(span.status).message),
|
||||
httpRoute: stringOrNull(attrs["http.route"]) ?? stringOrNull(attrs["url.path"]),
|
||||
httpStatus,
|
||||
error: stringOrNull(record(span.status).code) === "STATUS_CODE_ERROR" || Number(httpStatus) >= 500,
|
||||
valuesRedacted: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
return spans;
|
||||
}
|
||||
|
||||
function spanAttributes(value: unknown): Record<string, unknown> {
|
||||
const output: Record<string, unknown> = {};
|
||||
for (const item of arrayRecords(value)) {
|
||||
const key = stringOrNull(item.key);
|
||||
if (key === null) continue;
|
||||
const raw = record(item.value);
|
||||
output[key] = raw.stringValue ?? raw.intValue ?? raw.doubleValue ?? raw.boolValue ?? null;
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
function extractTraceRefs(value: unknown): Record<string, unknown>[] {
|
||||
const text = JSON.stringify(value ?? {}).slice(0, 24000);
|
||||
const refs = new Map<string, Record<string, unknown>>();
|
||||
for (const match of text.matchAll(/\b[0-9a-fA-F]{32}\b/gu)) refs.set(match[0].toLowerCase(), { traceId: match[0].toLowerCase(), type: "otel", source: "sentinel-finding", valuesRedacted: true });
|
||||
for (const match of text.matchAll(/\btrc_[A-Za-z0-9_-]{8,}\b/gu)) refs.set(match[0], { traceId: match[0], type: "business", source: "sentinel-finding", valuesRedacted: true });
|
||||
return Array.from(refs.values()).slice(0, 12);
|
||||
}
|
||||
|
||||
function errorLinks(config: WebProbeSentinelServiceConfig, errorId: string): Record<string, unknown> {
|
||||
const base = stringOrNull(config.publicExposure.publicBaseUrl) ?? stringOrNull(config.publicExposure.origin) ?? "";
|
||||
const path = `/monitor-web?error=${encodeURIComponent(errorId)}&panel=detail`;
|
||||
return {
|
||||
canonicalPath: path,
|
||||
canonicalUrl: base ? `${base.replace(/\/$/u, "")}${path}` : path,
|
||||
apiPath: `/api/errors/${encodeURIComponent(errorId)}`,
|
||||
cli: `bun scripts/cli.ts web-probe sentinel error get --node ${config.node} --lane ${config.lane} --sentinel ${config.sentinelId} --error-id ${errorId}`,
|
||||
valuesRedacted: true,
|
||||
};
|
||||
}
|
||||
|
||||
function compactRootCauseSignals(value: unknown): Record<string, unknown> | null {
|
||||
const source = record(value);
|
||||
if (Object.keys(source).length === 0) return null;
|
||||
return compactRecord(source, ["sessionListReadCount", "traceEventsReadCount", "webPerformanceBeaconFailureCount", "eventSourceFailureCount", "requestFailedCount", "httpErrorCount", "requestfailedTop", "httpStatusTop", "topHttpErrorPaths", "topRequestFailedPaths"]);
|
||||
}
|
||||
|
||||
function compactRecord(source: Record<string, unknown>, keys: readonly string[]): Record<string, unknown> | null {
|
||||
const output: Record<string, unknown> = {};
|
||||
for (const key of keys) {
|
||||
const value = source[key];
|
||||
if (value === null || value === undefined) continue;
|
||||
output[key] = Array.isArray(value) ? value.slice(0, 8) : typeof value === "string" ? value.slice(0, 500) : value;
|
||||
}
|
||||
return Object.keys(output).length === 0 ? null : { ...output, valuesRedacted: true };
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
function arrayRecords(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.map(record).filter((item) => Object.keys(item).length > 0) : [];
|
||||
}
|
||||
|
||||
function stringOrNull(value: unknown): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function numberOrNull(value: unknown): number | null {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function sha256(value: string): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
@@ -254,6 +254,15 @@ export function runSentinelReport(state: SentinelCicdState, options: Extract<Web
|
||||
return rendered(report.ok && body.ok !== false, command, renderedText);
|
||||
}
|
||||
|
||||
export function runSentinelErrorDetail(state: SentinelCicdState, options: Extract<WebProbeSentinelOptions, { kind: "error" }>): RenderedCliResult {
|
||||
const command = "web-probe sentinel error get";
|
||||
const response = callSentinelService(state, "GET", `/api/errors/${encodeURIComponent(options.errorId)}`, null, options.timeoutSeconds);
|
||||
const body = record(response.bodyJson);
|
||||
const rawPayload = Object.keys(body).length > 0 ? body : response;
|
||||
if (options.full || options.raw) return rendered(response.ok && body.ok !== false, command, JSON.stringify(rawPayload, null, 2));
|
||||
return rendered(response.ok && body.ok !== false, command, renderSentinelErrorDetailResult({ command, node: state.spec.nodeId, lane: state.spec.lane, sentinelId: state.sentinelId, response, body, valuesRedacted: true }));
|
||||
}
|
||||
|
||||
function compactSentinelReportRawPayload(
|
||||
state: SentinelCicdState,
|
||||
body: Record<string, unknown>,
|
||||
@@ -580,6 +589,8 @@ function compactSentinelReportFinding(value: Record<string, unknown>): Record<st
|
||||
summary: reportText(value.summary ?? value.message, 220),
|
||||
valuesRedacted: true,
|
||||
};
|
||||
const errorId = stringAtNullable(value, "errorId");
|
||||
if (errorId !== null) result.errorId = errorId;
|
||||
const rootCause = stringAtNullable(value, "rootCause");
|
||||
if (rootCause !== null) result.rootCause = rootCause;
|
||||
const rootCauseStatus = stringAtNullable(value, "rootCauseStatus");
|
||||
@@ -2089,3 +2100,49 @@ function renderReportResult(result: Record<string, unknown>): string {
|
||||
" report reads sentinel indexed analyze summaries/views only; it does not resample, rerun analyze, or read Workbench.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function renderSentinelErrorDetailResult(result: Record<string, unknown>): string {
|
||||
const body = record(result.body);
|
||||
const detail = record(body.detail);
|
||||
const identity = record(detail.identity);
|
||||
const evidence = record(detail.sentinelEvidence);
|
||||
const otel = record(detail.otel);
|
||||
const finding = record(body.finding);
|
||||
const links = record(detail.links);
|
||||
return [
|
||||
String(result.command),
|
||||
"",
|
||||
table(["NODE", "LANE", "STATUS", "ERROR_ID", "RUN", "FINDING"], [[
|
||||
result.node,
|
||||
result.lane,
|
||||
body.ok === false ? "blocked" : "ok",
|
||||
body.errorId ?? identity.errorId ?? "-",
|
||||
record(body.run).id ?? identity.runId ?? "-",
|
||||
identity.findingId ?? finding.id ?? "-",
|
||||
]]),
|
||||
"",
|
||||
table(["LEVEL", "BLOCKING", "TITLE", "SUMMARY"], [[
|
||||
identity.severity ?? finding.severity ?? "-",
|
||||
evidence.blocking === true ? "yes" : "no",
|
||||
evidence.displayTitleZh ?? finding.displayTitleZh ?? "-",
|
||||
short(evidence.summary ?? finding.summary ?? "-"),
|
||||
]]),
|
||||
"",
|
||||
table(["OTEL_STATUS", "TRACE_REFS", "API", "LINK"], [[
|
||||
otel.status ?? "-",
|
||||
Array.isArray(detail.traceRefs) ? detail.traceRefs.length : 0,
|
||||
links.apiPath ?? "-",
|
||||
links.canonicalPath ?? "-",
|
||||
]]),
|
||||
"",
|
||||
"EVIDENCE",
|
||||
` rootCause=${evidence.rootCause ?? "-"} status=${evidence.rootCauseStatus ?? "-"}`,
|
||||
` evidence=${evidence.evidenceSummary ?? "-"}`,
|
||||
"",
|
||||
"NEXT",
|
||||
` json: bun scripts/cli.ts web-probe sentinel error get --node ${result.node} --lane ${result.lane} --sentinel ${result.sentinelId} --error-id ${body.errorId ?? identity.errorId ?? "-"} --raw`,
|
||||
"",
|
||||
"DISCLOSURE",
|
||||
" error get reads one bounded sentinel error detail by stable errorId and performs backend-controlled OTel lookup when an OTel trace id is present.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import { webProbeSentinelConfigPlan, type WebProbeSentinelConfigPlan } from "./h
|
||||
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
||||
import { effectiveWebProbeSentinelPublicExposure, resolveWebProbeSentinel, readConfigRefTarget as readSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-resolver";
|
||||
import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
|
||||
import { dashboardErrorDetail, decorateRunFindingsWithErrorDetails } from "./hwlab-node-web-sentinel-error-detail";
|
||||
|
||||
const DASHBOARD_CONTRACT_VERSION = "draft-2026-06-27-p11-monitor-web-observability-dashboard";
|
||||
const DASHBOARD_MAX_TEXT_BYTES = 16_000;
|
||||
@@ -78,6 +79,7 @@ export interface WebProbeSentinelService {
|
||||
overview(): Record<string, unknown>;
|
||||
dashboardRuns(url: URL): Record<string, unknown>;
|
||||
runDetail(runId: string): Record<string, unknown>;
|
||||
errorDetail(errorId: string): Promise<Record<string, unknown>>;
|
||||
findings(url: URL): Record<string, unknown>;
|
||||
runViews(runId: string, view: string | null, url: URL): Record<string, unknown>;
|
||||
maintenance(): MaintenanceState;
|
||||
@@ -207,6 +209,9 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp
|
||||
runDetail(runId: string) {
|
||||
return dashboardRunDetail(config, db, runId);
|
||||
},
|
||||
errorDetail(errorId: string) {
|
||||
return dashboardErrorDetail(config, db, errorId, { findingsForRow: (row, limit) => visibleFindingsForRun(config, db, row, limit, readAnalysisArtifactSummaryForRun(config, row, limit)), runSummary: (row) => dashboardRunSummary(config, db, row), traceability: (row) => runTraceability(config, row) });
|
||||
},
|
||||
findings(url: URL) {
|
||||
return dashboardFindings(config, db, url);
|
||||
},
|
||||
@@ -321,6 +326,11 @@ async function sentinelFetch(service: WebProbeSentinelService, request: Request)
|
||||
if (request.method === "GET" && pathname === "/api/overview") return jsonResponse(service.overview());
|
||||
if (request.method === "GET" && pathname === "/api/runs") return jsonResponse(service.dashboardRuns(url));
|
||||
if (request.method === "GET" && pathname === "/api/findings") return jsonResponse(service.findings(url));
|
||||
const errorDetailMatch = /^\/api\/errors\/([^/]+)$/u.exec(pathname);
|
||||
if (request.method === "GET" && errorDetailMatch !== null) {
|
||||
const result = await service.errorDetail(decodeURIComponent(errorDetailMatch[1]));
|
||||
return jsonResponse(result, result.ok === false ? 404 : 200);
|
||||
}
|
||||
const runViewsMatch = /^\/api\/runs\/([^/]+)\/views$/u.exec(pathname);
|
||||
if (request.method === "GET" && runViewsMatch !== null) {
|
||||
const view = url.searchParams.get("view");
|
||||
@@ -941,7 +951,7 @@ function dashboardRunDetail(config: WebProbeSentinelServiceConfig, db: Database,
|
||||
if (row === null) return { ok: false, error: "run-not-found", runId, valuesRedacted: true };
|
||||
const stored = readMetadata(db, `run.report.${runId}`) ?? {};
|
||||
const artifactSummary = readAnalysisArtifactSummaryForRun(config, row, dashboardMaxPageSize(config));
|
||||
const findings = visibleFindingsForRun(config, db, row, dashboardMaxPageSize(config), artifactSummary);
|
||||
const findings = decorateRunFindingsWithErrorDetails(config, row, visibleFindingsForRun(config, db, row, dashboardMaxPageSize(config), artifactSummary));
|
||||
const views = record(stored.views);
|
||||
const reportJsonSha256 = stringOrNull(row.report_json_sha256) ?? stringOrNull(artifactSummary.reportJsonSha256);
|
||||
return {
|
||||
|
||||
@@ -54,9 +54,10 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
|
||||
&& sentinelActionRaw !== "validate"
|
||||
&& sentinelActionRaw !== "maintenance"
|
||||
&& sentinelActionRaw !== "dashboard"
|
||||
&& sentinelActionRaw !== "error"
|
||||
&& sentinelActionRaw !== "report"
|
||||
) {
|
||||
throw new Error("web-probe sentinel usage: sentinel plan|status|image|control-plane|publish-current|validate|maintenance|dashboard|report --node NODE --lane vNN [--dry-run|--confirm]");
|
||||
throw new Error("web-probe sentinel usage: sentinel plan|status|image|control-plane|publish-current|validate|maintenance|dashboard|error|report --node NODE --lane vNN [--dry-run|--confirm]");
|
||||
}
|
||||
assertKnownOptions(args, new Set([
|
||||
"--node",
|
||||
@@ -67,6 +68,7 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
|
||||
"--view",
|
||||
"--run",
|
||||
"--run-id",
|
||||
"--error-id",
|
||||
"--trace-id",
|
||||
"--sample-seq",
|
||||
"--sentinel",
|
||||
@@ -156,6 +158,12 @@ export function parseNodeWebProbeSentinelOptions(args: string[]): NodeWebProbeSe
|
||||
fullPage: args.includes("--full-page") && !args.includes("--no-full-page"),
|
||||
raw: args.includes("--raw"),
|
||||
};
|
||||
} else if (sentinelActionRaw === "error") {
|
||||
const errorAction = args[1];
|
||||
if (errorAction !== "get") throw new Error("web-probe sentinel error usage: error get --node NODE --lane vNN --sentinel ID --error-id wbe_...");
|
||||
const errorId = requiredOption(args, "--error-id");
|
||||
if (!/^wbe_[a-f0-9]{20}$/u.test(errorId)) throw new Error(`web-probe sentinel error get --error-id must look like wbe_<20 hex>, got ${errorId}`);
|
||||
sentinel = { kind: "error", action: "get", node, lane, sentinelId, errorId, raw: args.includes("--raw"), full: args.includes("--full"), timeoutSeconds };
|
||||
} else {
|
||||
const view = parseWebProbeSentinelReportView(optionValue(args, "--view") ?? "summary");
|
||||
const latest = args.includes("--latest");
|
||||
|
||||
Reference in New Issue
Block a user