// SPEC: PJ2026-01060508 Web哨兵 issue-1569 error detail deep links. // Responsibility: Stable monitor-web error ids, bounded error detail payloads, and server-side OTel enrichment. import { createHash } from "node:crypto"; import type { Database } from "bun:sqlite"; import type { WebProbeSentinelServiceConfig } from "./hwlab-node-web-sentinel-service"; export interface WebProbeSentinelErrorReaders { readonly findingsForRow: (row: Record, limit: number) => readonly Record[]; readonly runSummary: (row: Record) => Record; readonly traceability: (row: Record) => Record; } export function decorateRunFindingsWithErrorDetails(config: WebProbeSentinelServiceConfig, row: Record, findings: readonly Record[]): Record[] { return findings.map((finding, index) => { const detail = buildErrorDetail(config, row, finding, index, false); return { ...finding, errorId: detail.identity.errorId, detail, valuesRedacted: true, }; }); } export async function dashboardErrorDetail(config: WebProbeSentinelServiceConfig, db: Database, errorId: string, readers: WebProbeSentinelErrorReaders): Promise> { if (!/^wbe_[a-f0-9]{20}$/u.test(errorId)) return { ok: false, error: "invalid-error-id", errorId, valuesRedacted: true }; const rows = db.query("SELECT * FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY created_at DESC LIMIT ?") .all(config.sentinelId, config.node, config.lane, 200) as Record[]; for (const row of rows) { const findings = decorateRunFindingsWithErrorDetails(config, row, readers.findingsForRow(row, 200)); const findingIndex = findings.findIndex((item) => stringOrNull(item.errorId) === errorId); if (findingIndex < 0) continue; const finding = findings[findingIndex] ?? {}; const detail = buildErrorDetail(config, row, finding, findingIndex, true); const traceRefs = arrayRecords(detail.traceRefs); return { ok: true, errorId, run: readers.runSummary(row), finding, detail: { ...detail, otel: await buildOtelSummary(traceRefs), }, traceability: readers.traceability(row), valuesRedacted: true, }; } return { ok: false, error: "error-not-found", errorId, searchedRunCount: rows.length, valuesRedacted: true, }; } function buildErrorDetail(config: WebProbeSentinelServiceConfig, row: Record, finding: Record, occurrenceIndex: number, includeEvidence: boolean): Record { const identity = errorIdentity(config, row, finding, occurrenceIndex); const traceRefs = extractTraceRefs(finding); return { identity, sentinelEvidence: includeEvidence ? compactSentinelEvidence(finding) : compactSentinelEvidenceSummary(finding), traceRefs, otel: traceRefs.length === 0 ? { status: "trace-ref-missing", summaries: [], valuesRedacted: true } : { status: "detail-api-required", summaries: [], valuesRedacted: true }, links: errorLinks(config, identity.errorId), valuesRedacted: true, }; } function errorIdentity(config: WebProbeSentinelServiceConfig, row: Record, finding: Record, occurrenceIndex: number): Record { const runId = stringOrNull(row.id) ?? stringOrNull(finding.runId) ?? stringOrNull(finding.latestRunId) ?? "run"; const findingId = stringOrNull(finding.findingId) ?? stringOrNull(finding.finding_id) ?? stringOrNull(finding.id) ?? stringOrNull(finding.kind) ?? stringOrNull(finding.code) ?? "finding"; const severity = stringOrNull(finding.severity) ?? stringOrNull(finding.level) ?? stringOrNull(finding.checkLevel) ?? "unknown"; const sampleSeq = stringOrNull(finding.sampleSeq) ?? stringOrNull(record(finding.evidence).sampleSeq); const commandId = stringOrNull(finding.commandId) ?? stringOrNull(record(finding.firstCommandFailure).commandId); const groupHash = sha256(JSON.stringify({ summary: stringOrNull(finding.summary), evidenceSummary: stringOrNull(finding.evidenceSummary), groups: arrayRecords(finding.groups).slice(0, 4), peaks: arrayRecords(finding.peaks).slice(0, 4), })).slice(0, 16); const stableInput = { node: config.node, lane: config.lane, sentinelId: config.sentinelId, runId, findingId, severity, sampleSeq, commandId, occurrenceIndex, groupHash }; return { errorId: `wbe_${sha256(JSON.stringify(stableInput)).slice(0, 20)}`, node: config.node, lane: config.lane, sentinelId: config.sentinelId, runId, scenarioId: stringOrNull(row.scenario_id) ?? stringOrNull(finding.scenarioId), findingId, severity, occurrenceIndex, sampleSeq, commandId, groupHash, valuesRedacted: true, }; } function compactSentinelEvidenceSummary(finding: Record): Record { return { summary: stringOrNull(finding.summary), displayTitleZh: stringOrNull(finding.displayTitleZh) ?? stringOrNull(finding.errorTitleZh), evidenceSummary: stringOrNull(finding.evidenceSummary), rootCause: stringOrNull(finding.rootCause), rootCauseStatus: stringOrNull(finding.rootCauseStatus), rootCauseConfidence: stringOrNull(finding.rootCauseConfidence), nextAction: stringOrNull(finding.nextAction), blocking: finding.blocking === true, valuesRedacted: true, }; } function compactSentinelEvidence(finding: Record): Record { return { ...compactSentinelEvidenceSummary(finding), firstCommandFailure: compactRecord(record(finding.firstCommandFailure), ["commandId", "type", "phase", "status", "resultOk", "failureKind", "failedReason", "message", "exitCode", "timedOut", "artifactBucket", "artifactRef", "failedAt", "completedAt", "abandonedAt"]), rootCauseSignals: compactRootCauseSignals(finding.rootCauseSignals), browserNetworkErrors: arrayRecords(finding.groups).slice(0, 12).map((item) => compactRecord(item, ["method", "urlPath", "status", "type", "count", "firstAt", "lastAt", "promptIndexes", "failureKinds"])), samples: arrayRecords(finding.samples).slice(0, 8), peaks: arrayRecords(finding.peaks).slice(0, 8), valuesRedacted: true, }; } async function buildOtelSummary(traceRefs: readonly Record[]): Promise> { if (traceRefs.length === 0) return { status: "trace-ref-missing", summaries: [], valuesRedacted: true }; const summaries = []; for (const ref of traceRefs.slice(0, 4)) { const traceId = stringOrNull(ref.traceId); if (traceId === null) continue; if (ref.type !== "otel") { summaries.push({ traceId, status: "business-trace-id-not-tempo-id", note: "This id is not a 32 hex OTel trace id; use the linked CLI drill-down to map business trace ids first.", command: `bun scripts/cli.ts platform-infra observability diagnose-code-agent --trace-id ${traceId}`, valuesRedacted: true, }); continue; } summaries.push(await queryTempoTrace(traceId)); } return { status: summaries.some((item) => item.status === "queried") ? "queried" : "not-queried", summaries, valuesRedacted: true, }; } async function queryTempoTrace(traceId: string): Promise> { const baseUrl = String(process.env.UNIDESK_WEB_PROBE_SENTINEL_TEMPO_BASE_URL || process.env.UNIDESK_OBSERVABILITY_TEMPO_BASE_URL || "http://tempo.platform-infra.svc.cluster.local:3200").replace(/\/$/u, ""); const url = `${baseUrl}/api/traces/${encodeURIComponent(traceId)}`; try { const response = await fetch(url, { signal: AbortSignal.timeout(2500), headers: { accept: "application/json" } }); const payload = await response.json().catch(() => ({})); if (!response.ok) return { traceId, status: "query-failed", httpStatus: response.status, backend: "tempo", valuesRedacted: true }; const spans = tempoSpans(payload); return { traceId, status: "queried", httpStatus: response.status, backend: "tempo", spanCount: spans.length, errorSpanCount: spans.filter((span) => span.error).length, services: [...new Set(spans.map((span) => span.service).filter(Boolean))].slice(0, 12), keySpans: spans.filter((span) => span.error || span.httpStatus !== null).slice(0, 8), valuesRedacted: true, }; } catch (error) { return { traceId, status: "query-failed", backend: "tempo", error: error instanceof Error ? error.message.slice(0, 180) : String(error).slice(0, 180), valuesRedacted: true }; } } function tempoSpans(payload: unknown): Record[] { const batches = arrayRecords(record(payload).batches ?? record(record(payload).data).batches); const spans: Record[] = []; for (const batch of batches) { const service = stringOrNull(record(record(batch.resource).attributes?.find?.((item: unknown) => record(item).key === "service.name")).value?.stringValue); for (const scope of arrayRecords(batch.scopeSpans ?? batch.instrumentationLibrarySpans)) { for (const span of arrayRecords(scope.spans)) { const attrs = spanAttributes(span.attributes); const httpStatus = numberOrNull(attrs["http.response.status_code"]) ?? numberOrNull(attrs["http.status_code"]); spans.push({ name: stringOrNull(span.name), service, statusCode: stringOrNull(record(span.status).code), statusMessage: stringOrNull(record(span.status).message), httpRoute: stringOrNull(attrs["http.route"]) ?? stringOrNull(attrs["url.path"]), httpStatus, error: stringOrNull(record(span.status).code) === "STATUS_CODE_ERROR" || Number(httpStatus) >= 500, valuesRedacted: true, }); } } } return spans; } function spanAttributes(value: unknown): Record { const output: Record = {}; for (const item of arrayRecords(value)) { const key = stringOrNull(item.key); if (key === null) continue; const raw = record(item.value); output[key] = raw.stringValue ?? raw.intValue ?? raw.doubleValue ?? raw.boolValue ?? null; } return output; } function extractTraceRefs(value: unknown): Record[] { const text = JSON.stringify(value ?? {}).slice(0, 24000); const refs = new Map>(); for (const match of text.matchAll(/\b[0-9a-fA-F]{32}\b/gu)) refs.set(match[0].toLowerCase(), { traceId: match[0].toLowerCase(), type: "otel", source: "sentinel-finding", valuesRedacted: true }); for (const match of text.matchAll(/\btrc_[A-Za-z0-9_-]{8,}\b/gu)) refs.set(match[0], { traceId: match[0], type: "business", source: "sentinel-finding", valuesRedacted: true }); return Array.from(refs.values()).slice(0, 12); } function errorLinks(config: WebProbeSentinelServiceConfig, errorId: string): Record { const base = stringOrNull(config.publicExposure.publicBaseUrl) ?? stringOrNull(config.publicExposure.origin) ?? ""; const path = `/monitor-web?error=${encodeURIComponent(errorId)}&panel=detail`; return { canonicalPath: path, canonicalUrl: base ? `${base.replace(/\/$/u, "")}${path}` : path, apiPath: `/api/errors/${encodeURIComponent(errorId)}`, cli: `bun scripts/cli.ts web-probe sentinel error get --node ${config.node} --lane ${config.lane} --sentinel ${config.sentinelId} --error-id ${errorId}`, valuesRedacted: true, }; } function compactRootCauseSignals(value: unknown): Record | null { const source = record(value); if (Object.keys(source).length === 0) return null; return compactRecord(source, ["sessionListReadCount", "traceEventsReadCount", "webPerformanceBeaconFailureCount", "eventSourceFailureCount", "requestFailedCount", "httpErrorCount", "requestfailedTop", "httpStatusTop", "topHttpErrorPaths", "topRequestFailedPaths"]); } function compactRecord(source: Record, keys: readonly string[]): Record | null { const output: Record = {}; for (const key of keys) { const value = source[key]; if (value === null || value === undefined) continue; output[key] = Array.isArray(value) ? value.slice(0, 8) : typeof value === "string" ? value.slice(0, 500) : value; } return Object.keys(output).length === 0 ? null : { ...output, valuesRedacted: true }; } function record(value: unknown): Record { return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; } function arrayRecords(value: unknown): Record[] { return Array.isArray(value) ? value.map(record).filter((item) => Object.keys(item).length > 0) : []; } function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } function numberOrNull(value: unknown): number | null { const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } function sha256(value: string): string { return createHash("sha256").update(value).digest("hex"); }