// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p7-web-probe-sentinel-dashboard. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-desktop-view-density. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries. // SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel. // Responsibility: Persistent HTTP wrapper service for web-probe observe scheduling, index, health, metrics, maintenance, and dashboard. import { Buffer } from "node:buffer"; import { createHash, randomUUID } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { request as httpsRequest } from "node:https"; import { join } from "node:path"; import { Database } from "bun:sqlite"; import { renderWebProbeSentinelDashboardHtml, webProbeSentinelDashboardAssetResponse } from "./hwlab-node-web-sentinel-dashboard-assets"; import { webProbeSentinelConfigPlan, type WebProbeSentinelConfigPlan } from "./hwlab-node-web-sentinel-config"; import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import { effectiveWebProbeSentinelPublicExposure, resolveWebProbeSentinel, readConfigRefTarget as readSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-resolver"; import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel"; const DASHBOARD_CONTRACT_VERSION = "draft-2026-06-27-p11-monitor-web-observability-dashboard"; const DASHBOARD_MAX_TEXT_BYTES = 16_000; export interface WebProbeSentinelServiceConfig { readonly node: string; readonly lane: string; readonly sentinelId: string; readonly plan: WebProbeSentinelConfigPlan; readonly runtime: Record; readonly scenarios: readonly Record[]; readonly reportViews: Record; readonly publicExposure: Record; readonly cicd: Record; readonly stateRoot: string; readonly sqlitePath: string; readonly listenHost: string; readonly servicePort: number; readonly schedulerIntervalMs: number; readonly schedulerHeartbeatStaleSeconds: number; readonly maxConcurrentRuns: number; } export interface WebProbeSentinelServiceOptions { readonly spec: HwlabRuntimeLaneSpec; readonly sentinelId?: string | null; readonly stateRootOverride?: string; readonly portOverride?: number; readonly hostOverride?: string; readonly schedulerEnabled?: boolean; } interface MaintenanceState { readonly active: boolean; readonly reason: string | null; readonly releaseId: string | null; readonly startedAt: string | null; readonly stoppedAt: string | null; readonly quickVerifyPlannedAt: string | null; readonly quickVerifyPlannedRunId: string | null; } interface CommandPlanStep { readonly phase: string; readonly argv: readonly string[]; readonly stdinSource: "none" | "prompt-source"; } export interface WebProbeSentinelService { readonly config: WebProbeSentinelServiceConfig; readonly db: Database; readonly schedulerEnabled: boolean; startScheduler(): void; stopScheduler(): void; close(): void; health(): Record; status(): Record; runs(limit?: number): readonly Record[]; overview(): Record; dashboardRuns(url: URL): Record; runDetail(runId: string): Record; findings(url: URL): Record; runViews(runId: string, view: string | null, url: URL): Record; maintenance(): MaintenanceState; setMaintenance(active: boolean, input: Record): MaintenanceState; planScenarioRun(scenarioId: string, reason: string): Record; triggerQuickVerify(input: Record): Promise>; recordRun(input: Record): Record; report(view: string, runId: string | null): Record; metrics(): string; dashboardHtml(): string; fetch(request: Request): Promise; } export function loadWebProbeSentinelServiceConfig(spec: HwlabRuntimeLaneSpec, options: Omit = {}): WebProbeSentinelServiceConfig { const sentinel = resolveWebProbeSentinel(spec, options.sentinelId ?? null); const plan = webProbeSentinelConfigPlan(spec, "status", sentinel.id); const runtime = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.runtime, spec)); const scenarios = scenarioArrayTarget(readSentinelConfigRefTarget(sentinel.configRefs.scenarios, spec)); const reportViews = resolveReportViewsWithCheckCatalog(recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.reportViews, spec)), spec); const rawPublicExposure = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.publicExposure, spec)); const publicExposure = effectiveWebProbeSentinelPublicExposure(spec, sentinel.id, rawPublicExposure); const cicd = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.cicd, spec)); const stateRoot = options.stateRootOverride ?? stringAt(runtime, "stateRoot"); const yamlSqlitePath = stringAt(runtime, "sqlite.path"); return { node: spec.nodeId, lane: spec.lane, sentinelId: sentinel.id, plan, runtime, scenarios, reportViews, publicExposure, cicd, stateRoot, sqlitePath: options.stateRootOverride === undefined ? yamlSqlitePath : join(stateRoot, "index.sqlite"), listenHost: options.hostOverride ?? stringAt(runtime, "listenHost"), servicePort: options.portOverride ?? numberAt(runtime, "servicePort"), schedulerIntervalMs: numberAt(runtime, "scheduler.intervalMs"), schedulerHeartbeatStaleSeconds: numberAt(runtime, "scheduler.heartbeatStaleSeconds"), maxConcurrentRuns: numberAt(runtime, "scheduler.maxConcurrentRuns"), }; } export function createWebProbeSentinelService(options: WebProbeSentinelServiceOptions): WebProbeSentinelService { const config = loadWebProbeSentinelServiceConfig(options.spec, options); mkdirSync(config.stateRoot, { recursive: true }); const db = new Database(config.sqlitePath); initializeIndex(db, config); const restored = markInterruptedRuns(config, db, nowIso()); const schedulerEnabled = options.schedulerEnabled ?? true; let schedulerTimer: ReturnType | null = null; let schedulerHeartbeatAt = nowIso(); let schedulerLastError: string | null = null; writeMetadata(db, "service.boot", { at: schedulerHeartbeatAt, restoredInterruptedRuns: restored, valuesRedacted: true }); writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "boot" }); emitSchedulerHeartbeatSpan(config, "boot", schedulerHeartbeatAt, true); const service: WebProbeSentinelService = { config, db, schedulerEnabled, startScheduler() { if (!schedulerEnabled || schedulerTimer !== null) return; schedulerHeartbeatAt = nowIso(); writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "started" }); emitSchedulerHeartbeatSpan(config, "started", schedulerHeartbeatAt, true); schedulerTimer = setInterval(() => { try { schedulerHeartbeatAt = nowIso(); writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "tick" }); const summary = schedulerSummary(config, db); writeMetadata(db, "scheduler.summary", summary); emitSchedulerHeartbeatSpan(config, "tick", schedulerHeartbeatAt, true); emitCadenceExpectedSpan(config, summary); if (summary.rootCause === "planned-run-not-consumed-by-host-cadence") emitSchedulerGapSpan(config, summary); schedulerLastError = null; } catch (error) { schedulerLastError = error instanceof Error ? error.message : String(error); writeMetadata(db, "scheduler.error", { at: nowIso(), message: schedulerLastError }); emitSchedulerHeartbeatSpan(config, "tick-error", nowIso(), false, schedulerLastError); } }, config.schedulerIntervalMs); }, stopScheduler() { if (schedulerTimer !== null) clearInterval(schedulerTimer); schedulerTimer = null; writeMetadata(db, "scheduler.heartbeat", { at: nowIso(), loop: "stopped" }); }, close() { this.stopScheduler(); db.close(); }, health() { return serviceHealth(config, db, { schedulerEnabled, schedulerHeartbeatAt, schedulerTimerActive: schedulerTimer !== null, schedulerLastError, }); }, status() { return { ok: true, node: config.node, lane: config.lane, sentinelId: config.sentinelId, status: "observed", configReady: config.plan.ok, scheduler: schedulerSummary(config, db), maintenance: this.maintenance(), runs: runCounts(config, db), latestRuns: this.runs(8), valuesRedacted: true, }; }, runs(limit = 20) { return db.query("SELECT id, sentinel_id, scenario_id, status, node, lane, observer_id, state_dir, report_json_sha256, finding_count, artifact_count, maintenance, created_at, updated_at, interrupted_at FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY created_at DESC LIMIT ?") .all(config.sentinelId, config.node, config.lane, limit) as Record[]; }, overview() { return dashboardOverview(config, db, this.health(), this.maintenance()); }, dashboardRuns(url: URL) { return dashboardRunList(config, db, url); }, runDetail(runId: string) { return dashboardRunDetail(config, db, runId); }, findings(url: URL) { return dashboardFindings(config, db, url); }, runViews(runId: string, view: string | null, url: URL) { return dashboardRunViews(config, db, runId, view, url); }, maintenance() { return readMetadata(db, "maintenance") as MaintenanceState | null ?? emptyMaintenance(); }, setMaintenance(active: boolean, input: Record) { const current = this.maintenance(); const next: MaintenanceState = active ? { active: true, reason: stringOrNull(input.reason), releaseId: stringOrNull(input.releaseId), startedAt: nowIso(), stoppedAt: current.stoppedAt, quickVerifyPlannedAt: current.quickVerifyPlannedAt, quickVerifyPlannedRunId: current.quickVerifyPlannedRunId, } : { active: false, reason: current.reason, releaseId: current.releaseId, startedAt: current.startedAt, stoppedAt: nowIso(), quickVerifyPlannedAt: nowIso(), quickVerifyPlannedRunId: null, }; writeMetadata(db, "maintenance", next); if (!active) { const scenarioId = firstEnabledScenarioId(config); if (scenarioId !== null) { const planned = this.planScenarioRun(scenarioId, "maintenance-stop-quick-verify"); const withPlan = { ...next, quickVerifyPlannedRunId: stringOrNull(planned.runId) }; writeMetadata(db, "maintenance", withPlan); return withPlan; } } return next; }, planScenarioRun(scenarioId: string, reason: string) { const scenario = config.scenarios.find((item) => stringAt(item, "id") === scenarioId); if (scenario === undefined) throw new Error(`scenario not found: ${scenarioId}`); const runId = `sentinel-run-${Date.now()}-${randomUUID().slice(0, 8)}`; const commandPlan = buildObserveCommandPlan(config, scenario); const createdAt = nowIso(); db.query("INSERT INTO runs (id, sentinel_id, scenario_id, node, lane, status, maintenance, created_at, updated_at, command_plan_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") .run(runId, config.sentinelId, scenarioId, config.node, config.lane, "planned", this.maintenance().active ? 1 : 0, createdAt, createdAt, JSON.stringify({ reason, commandPlan, valuesRedacted: true })); return { ok: true, runId, scenarioId, status: "planned", commandPlanSha256: sha256Json(commandPlan), valuesRedacted: true }; }, async triggerQuickVerify(input: Record) { const result = await triggerQuickVerifyCronJob(config, input); writeMetadata(db, "manual-trigger.latest", result); emitManualTriggerSpan(config, result); return result; }, recordRun(input: Record) { const result = recordRunResult(config, db, input); emitRecordRunSpan(config, input, result); return result; }, report(view: string, runId: string | null) { return reportRunView(config, db, view, runId); }, metrics() { return renderMetrics(config, db, this.health(), this.maintenance()); }, dashboardHtml() { return renderWebProbeSentinelDashboardHtml(config); }, async fetch(request: Request) { return sentinelFetch(service, request); }, }; service.startScheduler(); return service; } function resolveReportViewsWithCheckCatalog(reportViews: Record, spec: HwlabRuntimeLaneSpec): Record { const ref = stringOrNull(reportViews.checkCatalogRef); if (ref === null) return reportViews; return { ...reportViews, checkCatalog: recordTarget(readSentinelConfigRefTarget(ref, spec)), }; } export function startWebProbeSentinelHttpService(service: WebProbeSentinelService): { readonly url: string; readonly stop: () => void } { const server = Bun.serve({ hostname: service.config.listenHost, port: service.config.servicePort, fetch: (request) => service.fetch(request), }); return { url: `http://${service.config.listenHost}:${server.port}`, stop: () => server.stop(true), }; } async function sentinelFetch(service: WebProbeSentinelService, request: Request): Promise { const url = new URL(request.url); const routeMismatch = sentinelRouteMismatch(service.config, url.pathname); if (routeMismatch !== null) return jsonResponse(routeMismatch, 409); const pathname = normalizedSentinelRequestPath(service, url.pathname); if (request.method === "GET" && pathname === "/api/health") { const health = service.health(); return jsonResponse(health, health.serving === false ? 503 : 200); } if (request.method === "GET" && pathname === "/api/status") return jsonResponse(service.status()); if (request.method === "GET" && pathname === "/api/overview") return jsonResponse(service.overview()); if (request.method === "GET" && pathname === "/api/runs") return jsonResponse(service.dashboardRuns(url)); if (request.method === "GET" && pathname === "/api/findings") return jsonResponse(service.findings(url)); const runViewsMatch = /^\/api\/runs\/([^/]+)\/views$/u.exec(pathname); if (request.method === "GET" && runViewsMatch !== null) { const view = url.searchParams.get("view"); const result = service.runViews(decodeURIComponent(runViewsMatch[1]), view, url); return jsonResponse(result, result.ok === false ? 404 : 200); } const runDetailMatch = /^\/api\/runs\/([^/]+)$/u.exec(pathname); if (request.method === "GET" && runDetailMatch !== null) { const result = service.runDetail(decodeURIComponent(runDetailMatch[1])); return jsonResponse(result, result.ok === false ? 404 : 200); } if (request.method === "GET" && pathname === "/api/maintenance") return jsonResponse({ ok: true, maintenance: service.maintenance(), valuesRedacted: true }); if (request.method === "POST" && (pathname === "/api/maintenance/start" || pathname === "/api/maintenance/stop")) { const body = await readJsonBody(request); const active = pathname.endsWith("/start"); return jsonResponse({ ok: true, maintenance: service.setMaintenance(active, body), valuesRedacted: true }); } if (request.method === "POST" && pathname === "/api/runs/plan") { const body = await readJsonBody(request); return jsonResponse(service.planScenarioRun(stringField(body, "scenarioId"), stringOrNull(body.reason) ?? "manual")); } if (request.method === "POST" && pathname === "/api/quick-verify/trigger") { const body = await readJsonBody(request); const result = await service.triggerQuickVerify(body); return jsonResponse(result, result.ok === false ? 409 : 200); } if (request.method === "POST" && pathname === "/api/runs/record") { const body = await readJsonBody(request); return jsonResponse(service.recordRun(body)); } if (request.method === "GET" && pathname === "/api/report") { const view = url.searchParams.get("view") ?? stringOrNull(service.config.reportViews.defaultView) ?? "summary"; const runId = url.searchParams.get("run") ?? url.searchParams.get("runId"); const report = service.report(view, runId); return jsonResponse(report, report.ok === false ? 404 : 200); } if (request.method === "GET" && pathname === "/metrics") { return new Response(service.metrics(), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } }); } if (request.method === "GET" && (pathname.includes("/dashboard/assets/") || pathname.includes("/monitor-web/assets/"))) { const asset = webProbeSentinelDashboardAssetResponse(normalizedDashboardAssetPath(pathname)); if (asset !== null) return asset; } if (request.method === "GET" && (pathname === "/" || pathname === "/dashboard" || pathname === "/monitor-web")) { return new Response(service.dashboardHtml(), { headers: { "content-type": "text/html; charset=utf-8" } }); } return jsonResponse({ ok: false, error: "not-found", path: url.pathname, valuesRedacted: true }, 404); } function normalizedSentinelRequestPath(service: WebProbeSentinelService, pathname: string): string { const publicPath = publicExposurePath(service.config.publicExposure); if (publicPath.length > 0 && (pathname === publicPath || pathname === `${publicPath}/`)) return "/"; if (publicPath.length > 0 && pathname.startsWith(`${publicPath}/`)) return pathname.slice(publicPath.length) || "/"; if (/^\/sentinels\/[^/]+\/?$/u.test(pathname)) return "/"; for (const marker of ["/api/", "/monitor-web/assets/", "/dashboard/assets/"]) { const index = pathname.indexOf(marker); if (index > 0) return pathname.slice(index); } for (const marker of ["/monitor-web", "/dashboard", "/metrics"]) { if (pathname.endsWith(marker)) return marker; } return pathname; } function sentinelRouteMismatch(config: WebProbeSentinelServiceConfig, pathname: string): Record | null { const match = /^\/sentinels\/([^/]+)/u.exec(pathname); if (match === null) return null; const routeSegment = decodeURIComponent(match[1]); const nodePrefix = `${config.node.toLowerCase()}-`; const routeSentinelId = routeSegment.startsWith(nodePrefix) ? routeSegment.slice(nodePrefix.length) : routeSegment; if (routeSegment === config.sentinelId || routeSentinelId === config.sentinelId) return null; return { ok: false, error: "sentinel-route-mismatch", code: "sentinel-route-mismatch", blocker: true, routeSegment, routeSentinelId, serviceSentinelId: config.sentinelId, node: config.node, lane: config.lane, message: "route sentinelId does not match the selected sentinel runner; refusing cross-sentinel fallback", valuesRedacted: true, }; } function publicExposurePath(publicExposure: Record): string { const publicBaseUrl = stringOrNull(publicExposure.publicBaseUrl); if (publicBaseUrl === null) return ""; try { const pathname = new URL(publicBaseUrl).pathname.replace(/\/+$/u, ""); return pathname === "/" ? "" : pathname; } catch { return ""; } } function normalizedDashboardAssetPath(pathname: string): string { for (const marker of ["/monitor-web/assets/", "/dashboard/assets/"]) { const index = pathname.indexOf(marker); if (index >= 0) return pathname.slice(index); } return pathname; } function initializeIndex(db: Database, config: WebProbeSentinelServiceConfig): void { db.exec(` PRAGMA journal_mode = WAL; CREATE TABLE IF NOT EXISTS metadata ( key TEXT PRIMARY KEY, value_json TEXT NOT NULL, updated_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS runs ( id TEXT PRIMARY KEY, sentinel_id TEXT NOT NULL DEFAULT '', scenario_id TEXT NOT NULL, node TEXT NOT NULL, lane TEXT NOT NULL, status TEXT NOT NULL, observer_id TEXT, state_dir TEXT, report_json_sha256 TEXT, finding_count INTEGER NOT NULL DEFAULT 0, artifact_count INTEGER NOT NULL DEFAULT 0, maintenance INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, interrupted_at TEXT, command_plan_json TEXT ); CREATE TABLE IF NOT EXISTS findings ( run_id TEXT NOT NULL, sentinel_id TEXT NOT NULL DEFAULT '', finding_id TEXT NOT NULL, severity TEXT NOT NULL, count INTEGER NOT NULL DEFAULT 1, summary TEXT NOT NULL, report_json_sha256 TEXT, created_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS idx_runs_sentinel_updated ON runs(sentinel_id, node, lane, updated_at DESC); CREATE INDEX IF NOT EXISTS idx_runs_sentinel_scenario ON runs(sentinel_id, scenario_id, updated_at DESC); CREATE INDEX IF NOT EXISTS idx_findings_sentinel_run ON findings(sentinel_id, run_id); CREATE INDEX IF NOT EXISTS idx_findings_sentinel_code ON findings(sentinel_id, finding_id, created_at DESC); `); ensureColumn(db, "runs", "sentinel_id", "TEXT"); ensureColumn(db, "findings", "sentinel_id", "TEXT"); db.query("UPDATE runs SET sentinel_id = ? WHERE sentinel_id IS NULL OR sentinel_id = ''").run(config.sentinelId); db.query("UPDATE findings SET sentinel_id = ? WHERE sentinel_id IS NULL OR sentinel_id = ''").run(config.sentinelId); } function ensureColumn(db: Database, table: "runs" | "findings", column: string, definition: string): void { const rows = db.query(`PRAGMA table_info(${table})`).all() as Record[]; if (rows.some((row) => stringOrNull(row.name) === column)) return; db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`); } function markInterruptedRuns(config: WebProbeSentinelServiceConfig, db: Database, at: string): number { const result = db.query("UPDATE runs SET status = 'interrupted', interrupted_at = ?, updated_at = ? WHERE sentinel_id = ? AND node = ? AND lane = ? AND status IN ('queued', 'running', 'analyzing')").run(at, at, config.sentinelId, config.node, config.lane); return Number(result.changes ?? 0); } function serviceHealth(config: WebProbeSentinelServiceConfig, db: Database, scheduler: Record): Record { const checks: Record> = {}; checks.config = { ok: config.plan.ok, status: config.plan.status, conflicts: config.plan.conflicts.length }; checks.pvc = checkWritable(config.stateRoot); checks.sqlite = checkSqlite(db); const heartbeatAt = stringOrNull(scheduler.schedulerHeartbeatAt) ?? stringOrNull(readMetadata(db, "scheduler.heartbeat")?.at); const heartbeatAgeSeconds = heartbeatAt === null ? null : Math.max(0, Math.round((Date.now() - Date.parse(heartbeatAt)) / 1000)); const planned = plannedRunBacklog(config, db); const schedulerServing = scheduler.schedulerLastError === null && heartbeatAgeSeconds !== null && heartbeatAgeSeconds <= config.schedulerHeartbeatStaleSeconds; checks.scheduler = { ok: schedulerServing && !planned.stale, enabled: scheduler.schedulerEnabled === true, active: scheduler.schedulerTimerActive === true, heartbeatAt, heartbeatAgeSeconds, staleAfterSeconds: config.schedulerHeartbeatStaleSeconds, lastError: scheduler.schedulerLastError, plannedRuns: planned.count, oldestPlannedRunId: planned.oldestRunId, oldestPlannedRunScenarioId: planned.oldestScenarioId, oldestPlannedRunCreatedAt: planned.oldestCreatedAt, oldestPlannedRunAgeSeconds: planned.oldestAgeSeconds, plannedRunStaleAfterSeconds: planned.staleAfterSeconds, plannedRunStale: planned.stale, rootCause: planned.stale ? "planned-run-not-consumed-by-host-cadence" : null, }; checks.analyzer = { ok: true, source: "existing observe analyze CLI command", command: `bun scripts/cli.ts web-probe observe analyze --node ${config.node} --lane ${config.lane} --state-dir `, }; const ok = Object.values(checks).every((check) => check.ok === true); const serving = checks.config.ok === true && checks.pvc.ok === true && checks.sqlite.ok === true && schedulerServing; return { ok, serving, status: ok ? "healthy" : serving ? "degraded" : "unavailable", node: config.node, lane: config.lane, sentinelId: config.sentinelId, checks, valuesRedacted: true }; } function checkWritable(stateRoot: string): Record { try { mkdirSync(stateRoot, { recursive: true }); const path = join(stateRoot, ".health-write-probe"); writeFileSync(path, `${nowIso()}\n`); return { ok: true, stateRoot, probeFile: path }; } catch (error) { return { ok: false, stateRoot, error: error instanceof Error ? error.message : String(error) }; } } function checkSqlite(db: Database): Record { try { db.query("SELECT 1 AS ok").get(); writeMetadata(db, "sqlite.health", { at: nowIso() }); return { ok: true }; } catch (error) { return { ok: false, error: error instanceof Error ? error.message : String(error) }; } } function buildObserveCommandPlan(config: WebProbeSentinelServiceConfig, scenario: Record): readonly CommandPlanStep[] { const targetPath = stringAt(scenario, "observeTargetPath"); const startArgv = [ "bun", "scripts/cli.ts", "web-probe", "observe", "start", "--node", config.node, "--lane", config.lane, "--target-path", targetPath, "--sample-interval-ms", String(numberAt(scenario, "sampleIntervalMs")), "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), "--max-run-seconds", String(numberAt(scenario, "maxRunSeconds")), "--command-timeout-seconds", "55", ]; const viewport = stringOrNull(scenario.viewport); if (viewport !== null) startArgv.push("--viewport", viewport); const start: CommandPlanStep = { phase: "observe-start", argv: startArgv, stdinSource: "none", }; const commands = arrayAt(scenario, "commandSequence").map((item) => { const type = stringAt(item, "type"); const argv = ["bun", "scripts/cli.ts", "web-probe", "observe", "command", "", "--type", type]; if (type === "selectProvider") argv.push("--provider", stringAt(item, "provider")); if (type === "sendPrompt") { const inlineText = inlinePromptText(item); if (inlineText === null) argv.push("--text-stdin"); else argv.push("--text", ""); } if (type === "loginAccount" || type === "listSessions" || type === "logout") { const accountId = stringOrNull(item.accountId); if (accountId !== null) argv.push("--account-id", accountId); } if (type === "switchSessions") { const fromAccountId = stringOrNull(item.fromAccountId); const toAccountId = stringOrNull(item.toAccountId); if (fromAccountId !== null) argv.push("--from-account-id", fromAccountId); if (toAccountId !== null) argv.push("--to-account-id", toAccountId); } appendObserveCommandArgs(argv, item, { skipText: type === "sendPrompt" }); return { phase: `observe-command-${type}`, argv, stdinSource: type === "sendPrompt" ? inlinePromptText(item) === null ? "prompt-source" : "inline-text-redacted" : "none" } satisfies CommandPlanStep; }); const analyze: CommandPlanStep = { phase: "observe-analyze", argv: ["bun", "scripts/cli.ts", "web-probe", "observe", "analyze", ""], stdinSource: "none", }; const stop: CommandPlanStep = { phase: "observe-stop", argv: ["bun", "scripts/cli.ts", "web-probe", "observe", "stop", "", "--node", config.node, "--lane", config.lane, "--force", "--wait-ms", "55000", "--command-timeout-seconds", "55"], stdinSource: "none", }; const gc: CommandPlanStep = { phase: "observe-gc", argv: ["bun", "scripts/cli.ts", "web-probe", "observe", "gc", "--node", config.node, "--lane", config.lane, "--confirm", "--command-timeout-seconds", "55"], stdinSource: "none", }; return [start, ...commands, analyze, stop, gc]; } function inlinePromptText(item: Record): string | null { return stringOrNull(item.text) ?? stringOrNull(item.prompt) ?? stringOrNull(item.value); } function appendObserveCommandArgs(argv: string[], item: Record, options: { readonly skipText?: boolean } = {}): void { const mappings: readonly (readonly [string, string])[] = [ ["path", "--path"], ["label", "--label"], ["sessionId", "--session-id"], ["provider", "--provider"], ["accountId", "--account-id"], ["fromAccountId", "--from-account-id"], ["toAccountId", "--to-account-id"], ["sourceId", "--source-id"], ["fileRef", "--file-ref"], ["filename", "--filename"], ["taskRef", "--task-ref"], ["taskId", "--task-id"], ["task", "--task"], ["field", "--field"], ["link", "--link"], ["title", "--title"], ["body", "--body"], ["status", "--status"], ["hwpodId", "--hwpod-id"], ["nodeId", "--node-id"], ["workspaceRoot", "--workspace-root"], ["root", "--root"], ]; for (const [key, flag] of mappings) { if (argv.includes(flag)) continue; const value = stringOrNull(item[key]); if (value !== null) argv.push(flag, value); } if (options.skipText !== true) { const text = stringOrNull(item.text) ?? stringOrNull(item.value); if (text !== null) argv.push("--text", text); } } function schedulerSummary(config: WebProbeSentinelServiceConfig, db: Database): Record { const planned = plannedRunBacklog(config, db); return { enabledScenarios: config.scenarios.filter((item) => boolAt(item, "enabled")).map((item) => stringAt(item, "id")), intervalMs: config.schedulerIntervalMs, maxConcurrentRuns: config.maxConcurrentRuns, activeRuns: countWhere(config, db, "status IN ('queued', 'running', 'analyzing')"), plannedRuns: planned.count, oldestPlannedRunId: planned.oldestRunId, oldestPlannedRunScenarioId: planned.oldestScenarioId, oldestPlannedRunCreatedAt: planned.oldestCreatedAt, oldestPlannedRunAgeSeconds: planned.oldestAgeSeconds, plannedRunStaleAfterSeconds: planned.staleAfterSeconds, plannedRunStale: planned.stale, rootCause: planned.stale ? "planned-run-not-consumed-by-host-cadence" : null, heartbeat: readMetadata(db, "scheduler.heartbeat"), valuesRedacted: true, }; } function emitSchedulerHeartbeatSpan(config: WebProbeSentinelServiceConfig, loop: string, at: string, ok: boolean, failureKind: string | null = null): void { emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.scheduler.heartbeat", { status: ok ? "ok" : "error", failureKind, namespace: stringOrNull(config.runtime.namespace), heartbeatAt: at, cadence: firstEnabledScenarioCadence(config), valuesRedacted: true, }, ok); } function emitCadenceExpectedSpan(config: WebProbeSentinelServiceConfig, summary: Record): void { emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.cadence.expected", { cadence: firstEnabledScenarioCadence(config), scenarioId: firstEnabledScenarioId(config), status: summary.rootCause == null ? "ok" : "stale", activeRunCount: summary.activeRuns ?? null, plannedRunCount: summary.plannedRuns ?? null, valuesRedacted: true, }, summary.rootCause == null); } function emitSchedulerGapSpan(config: WebProbeSentinelServiceConfig, summary: Record): void { emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.scheduler_gap.detected", { cadence: firstEnabledScenarioCadence(config), scenarioId: summary.oldestPlannedRunScenarioId ?? firstEnabledScenarioId(config), runId: summary.oldestPlannedRunId ?? null, status: "planned-run-stale", failureKind: summary.rootCause, valuesRedacted: true, }, false); } function emitRecordRunSpan(config: WebProbeSentinelServiceConfig, input: Record, result: Record): void { emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.record_run", { scenarioId: result.scenarioId ?? input.scenarioId ?? null, runId: result.runId ?? input.runId ?? null, observerId: input.observerId ?? null, status: result.status ?? input.status ?? null, failureKind: result.ok === true ? null : result.error ?? "record-run-failed", valuesRedacted: true, }, result.ok === true); } function emitManualTriggerSpan(config: WebProbeSentinelServiceConfig, result: Record): void { emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.manual_trigger", { scenarioId: result.scenarioId ?? null, status: result.status ?? null, jobName: result.jobName ?? null, cronJobName: result.cronJobName ?? null, failureKind: result.ok === true ? null : result.reason ?? "manual-trigger-failed", valuesRedacted: true, }, result.ok === true); } function sentinelOtelContext(config: WebProbeSentinelServiceConfig): { readonly node: string; readonly lane: string; readonly sentinelId: string; readonly namespace: string | null; readonly runtime: Record; readonly cicd: Record } { return { node: config.node, lane: config.lane, sentinelId: config.sentinelId, namespace: stringOrNull(config.runtime.namespace), runtime: config.runtime, cicd: config.cicd, }; } function firstEnabledScenarioCadence(config: WebProbeSentinelServiceConfig): string | null { const scenario = config.scenarios.find((item) => boolAt(item, "enabled")); return scenario === undefined ? null : stringOrNull(scenario.cadence); } function renderMetrics(config: WebProbeSentinelServiceConfig, db: Database, health: Record, maintenance: MaintenanceState): string { const counts = runCounts(config, db); const heartbeat = record(readMetadata(db, "scheduler.heartbeat")); const heartbeatAt = stringOrNull(heartbeat.at); const heartbeatAge = heartbeatAt === null ? -1 : Math.max(0, Math.round((Date.now() - Date.parse(heartbeatAt)) / 1000)); const labels = `node="${metricLabel(config.node)}",lane="${metricLabel(config.lane)}",sentinel="${metricLabel(config.sentinelId)}"`; const lines = [ "# HELP web_probe_sentinel_config_ready Config reference graph is ready.", "# TYPE web_probe_sentinel_config_ready gauge", `web_probe_sentinel_config_ready{${labels}} ${config.plan.ok ? 1 : 0}`, "# HELP web_probe_sentinel_health Healthy status of the sentinel service.", "# TYPE web_probe_sentinel_health gauge", `web_probe_sentinel_health{${labels}} ${health.ok === true ? 1 : 0}`, "# HELP web_probe_sentinel_runs_total Runs indexed by status.", "# TYPE web_probe_sentinel_runs_total gauge", ...Object.entries(counts).map(([status, count]) => `web_probe_sentinel_runs_total{${labels},status="${metricLabel(status)}"} ${count}`), "# HELP web_probe_sentinel_active_runs Active observe runs known to the sentinel index.", "# TYPE web_probe_sentinel_active_runs gauge", `web_probe_sentinel_active_runs{${labels}} ${countWhere(config, db, "status IN ('queued', 'running', 'analyzing')")}`, "# HELP web_probe_sentinel_recent_findings Findings indexed from recent reports.", "# TYPE web_probe_sentinel_recent_findings gauge", `web_probe_sentinel_recent_findings{${labels}} ${sumColumn(config, db, "runs", "finding_count")}`, "# HELP web_probe_sentinel_maintenance_active Maintenance window active flag.", "# TYPE web_probe_sentinel_maintenance_active gauge", `web_probe_sentinel_maintenance_active{${labels}} ${maintenance.active ? 1 : 0}`, "# HELP web_probe_sentinel_scheduler_heartbeat_age_seconds Scheduler heartbeat age.", "# TYPE web_probe_sentinel_scheduler_heartbeat_age_seconds gauge", `web_probe_sentinel_scheduler_heartbeat_age_seconds{${labels}} ${heartbeatAge}`, "# HELP web_probe_sentinel_planned_runs Planned runs waiting for host cadence execution.", "# TYPE web_probe_sentinel_planned_runs gauge", `web_probe_sentinel_planned_runs{${labels}} ${countWhere(config, db, "status = 'planned'")}`, "# HELP web_probe_sentinel_oldest_planned_run_age_seconds Oldest planned run age, or -1 when no planned run exists.", "# TYPE web_probe_sentinel_oldest_planned_run_age_seconds gauge", `web_probe_sentinel_oldest_planned_run_age_seconds{${labels}} ${plannedRunBacklog(config, db).oldestAgeSeconds ?? -1}`, ]; return `${lines.join("\n")}\n`; } function plannedRunBacklog(config: WebProbeSentinelServiceConfig, db: Database): { readonly count: number; readonly oldestRunId: string | null; readonly oldestScenarioId: string | null; readonly oldestCreatedAt: string | null; readonly oldestAgeSeconds: number | null; readonly staleAfterSeconds: number; readonly stale: boolean; } { const activeWhere = [ "status = 'planned'", "AND NOT EXISTS (", "SELECT 1 FROM runs newer", "WHERE newer.sentinel_id = runs.sentinel_id", "AND newer.node = runs.node", "AND newer.lane = runs.lane", "AND newer.scenario_id = runs.scenario_id", "AND newer.status <> 'planned'", "AND newer.created_at > runs.created_at", ")", ].join(" "); const count = countWhere(config, db, activeWhere); const oldest = db.query(`SELECT id, scenario_id, created_at FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? AND ${activeWhere} ORDER BY created_at ASC LIMIT 1`) .get(config.sentinelId, config.node, config.lane) as Record | null; const oldestCreatedAt = stringOrNull(oldest?.created_at); const oldestAgeSeconds = oldestCreatedAt === null ? null : ageSeconds(oldestCreatedAt); const staleAfterSeconds = Math.max(60, Math.round(config.schedulerIntervalMs / 1000)); return { count, oldestRunId: stringOrNull(oldest?.id), oldestScenarioId: stringOrNull(oldest?.scenario_id), oldestCreatedAt, oldestAgeSeconds, staleAfterSeconds, stale: oldestAgeSeconds !== null && oldestAgeSeconds >= staleAfterSeconds, }; } function writeMetadata(db: Database, key: string, value: unknown): void { db.query("INSERT INTO metadata (key, value_json, updated_at) VALUES (?, ?, ?) ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, updated_at = excluded.updated_at") .run(key, JSON.stringify(value), nowIso()); } function readMetadata(db: Database, key: string): Record | null { const row = db.query("SELECT value_json FROM metadata WHERE key = ?").get(key) as { value_json?: string } | null; if (typeof row?.value_json !== "string") return null; try { const parsed = JSON.parse(row.value_json) as unknown; return record(parsed); } catch { return null; } } function runCounts(config: WebProbeSentinelServiceConfig, db: Database): Record { const rows = db.query("SELECT status, COUNT(*) AS count FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? GROUP BY status") .all(config.sentinelId, config.node, config.lane) as { status: string; count: number }[]; return Object.fromEntries(rows.map((row) => [row.status, Number(row.count)])); } function dashboardOverview(config: WebProbeSentinelServiceConfig, db: Database, health: Record, maintenance: MaintenanceState): Record { const latestRow = db.query("SELECT * FROM runs ORDER BY updated_at DESC LIMIT 1").get() as Record | null; const latestRun = latestRow === null ? null : dashboardRunSummary(config, db, latestRow); const severityCounts = globalSeverityCounts(config, db); const latestUpdatedAt = latestRow === null ? null : stringOrNull(latestRow.updated_at); const latestRunAgeSeconds = latestUpdatedAt === null ? null : ageSeconds(latestUpdatedAt); const heartbeatAgeSeconds = numberOr(record(record(health.checks).scheduler).heartbeatAgeSeconds, -1); const expectedCadence = firstEnabledScenarioCadence(config); const expectedCadenceSeconds = durationStringSeconds(expectedCadence); const staleMultiple = expectedCadenceSeconds === null || latestRunAgeSeconds === null ? null : latestRunAgeSeconds / expectedCadenceSeconds; const freshnessWarningMultiple = numberAt(config.runtime, "scheduler.freshnessWarningMultiple"); const scheduler = schedulerSummary(config, db); return { ok: health.ok === true, contractVersion: DASHBOARD_CONTRACT_VERSION, status: dashboardOverallStatus(health, latestRun, severityCounts), node: config.node, lane: config.lane, sentinelId: config.sentinelId, publicOrigin: stringOrNull(config.publicExposure.publicBaseUrl), configReady: config.plan.ok, health, scheduler, maintenance, latestRun, runCounts: runCounts(config, db), severityCounts, freshness: { latestRunUpdatedAt: latestUpdatedAt, latestRunAgeSeconds, schedulerHeartbeatAgeSeconds: heartbeatAgeSeconds, latestAnalyzedReportAgeSeconds: latestRow === null || stringOrNull(latestRow.report_json_sha256) === null ? null : latestRunAgeSeconds, }, cadence: { expectedCadence, expectedCadenceSeconds, schedulerHeartbeatAgeSeconds: heartbeatAgeSeconds, latestRunAgeSeconds, latestAnalyzedReportAgeSeconds: latestRow === null || stringOrNull(latestRow.report_json_sha256) === null ? null : latestRunAgeSeconds, activeRuns: scheduler.activeRuns ?? null, plannedRuns: scheduler.plannedRuns ?? null, nextRun: null, staleMultiple, freshnessWarningMultiple, status: scheduler.rootCause === "planned-run-not-consumed-by-host-cadence" ? "blocker" : staleMultiple !== null && staleMultiple > freshnessWarningMultiple ? "warning" : "fresh", cronJob: { observed: false, status: "control-plane-status-required", reason: "runner API does not query Kubernetes CronJob objects; use web-probe sentinel control-plane status for CronJob counts, lastScheduleTime and latest Jobs.", valuesRedacted: true, }, valuesRedacted: true, }, observability: webProbeSentinelOtelSummary(sentinelOtelContext(config)), targetValidation: { scenarioId: stringOrNull(record(config.cicd.targetValidation).scenarioId), maxSeconds: numberOr(record(config.cicd.targetValidation).maxSeconds, 120), sourceRef: targetValidationSourceRef(config), }, traceability: { source: "sqlite-index+run-report-metadata", stateRoot: config.stateRoot, latestRun: latestRow === null ? null : runTraceability(config, latestRow), }, valuesRedacted: true, }; } function targetValidationSourceRef(config: WebProbeSentinelServiceConfig): string | null { const cicdRef = config.plan.refs.find((item) => item.key === "cicd")?.ref ?? null; return cicdRef === null ? null : `${cicdRef}.targetValidation`; } function dashboardRunList(config: WebProbeSentinelServiceConfig, db: Database, url: URL): Record { const filters = dashboardRunFilters(url); const page = dashboardPage(url, config); const sort = dashboardRunSort(url); const where = runWhereClause(config, filters); const sql = `SELECT * FROM runs ${where.sql} ORDER BY ${sort.sql} LIMIT ? OFFSET ?`; const rows = db.query(sql).all(...where.params, page.limit + 1, page.offset) as Record[]; const visibleRows = rows.slice(0, page.limit); const items = visibleRows.map((row) => dashboardRunSummary(config, db, row)); return { ok: true, contractVersion: DASHBOARD_CONTRACT_VERSION, node: config.node, lane: config.lane, sentinelId: config.sentinelId, filters, page: { limit: page.limit, cursor: String(page.offset), nextCursor: rows.length > page.limit ? String(page.offset + visibleRows.length) : null, hasMore: rows.length > page.limit, sort: sort.name, direction: sort.direction, }, items, runs: items, traceability: { source: "sqlite-index", stateRoot: config.stateRoot, }, valuesRedacted: true, }; } function dashboardRunDetail(config: WebProbeSentinelServiceConfig, db: Database, runId: string): Record { const row = readRunRow(config, db, runId); if (row === null) return { ok: false, error: "run-not-found", runId, valuesRedacted: true }; const stored = readMetadata(db, `run.report.${runId}`) ?? {}; const artifactSummary = readAnalysisArtifactSummaryForRun(config, row, dashboardMaxPageSize(config)); const findings = visibleFindingsForRun(config, db, row, dashboardMaxPageSize(config), artifactSummary); const views = record(stored.views); const reportJsonSha256 = stringOrNull(row.report_json_sha256) ?? stringOrNull(artifactSummary.reportJsonSha256); return { ok: true, contractVersion: DASHBOARD_CONTRACT_VERSION, sentinelId: config.sentinelId, run: dashboardRunSummary(config, db, row), summary: record(stored.summary), memory: dashboardRunMemoryDetail(config, row, stored), requestRate: dashboardRunRequestRateDetail(config, row, stored, artifactSummary), findings, viewsAvailable: Object.keys(views), artifacts: { artifactCount: numberOr(row.artifact_count, 0), reportJsonSha256, analysisReport: artifactSummary, screenshot: compactArtifactRef(stored.screenshot), publicOrigin: stringOrNull(stored.publicOrigin), }, commands: { summary: `bun scripts/cli.ts web-probe sentinel report --node ${config.node} --lane ${config.lane} --sentinel ${config.sentinelId} --run ${runId} --view summary`, turnSummary: `bun scripts/cli.ts web-probe sentinel report --node ${config.node} --lane ${config.lane} --sentinel ${config.sentinelId} --run ${runId} --view turn-summary`, traceFrame: `bun scripts/cli.ts web-probe sentinel report --node ${config.node} --lane ${config.lane} --sentinel ${config.sentinelId} --run ${runId} --view trace-frame`, }, redaction: record(config.reportViews.redaction), traceability: runTraceability(config, row), valuesRedacted: true, }; } function dashboardFindings(config: WebProbeSentinelServiceConfig, db: Database, url: URL): Record { const limit = dashboardPage(url, config).limit; const filters = dashboardFindingFilters(url); const where = findingWhereClause(config, filters); const queryLimit = Math.min(dashboardMaxPageSize(config), Math.max(limit + 1, limit * 4)); const rows = db.query(` SELECT f.finding_id, (SELECT f2.severity FROM findings f2 JOIN runs r2 ON r2.id = f2.run_id WHERE f2.finding_id = f.finding_id AND r2.scenario_id = r.scenario_id ORDER BY f2.created_at DESC LIMIT 1) AS severity, r.scenario_id, SUM(f.count) AS count, COUNT(DISTINCT f.run_id) AS run_count, MAX(f.created_at) AS latest_at, MAX(f.summary) AS summary FROM findings f JOIN runs r ON r.id = f.run_id ${where.sql} GROUP BY f.finding_id, r.scenario_id ORDER BY latest_at DESC LIMIT ? `).all(...where.params, queryLimit) as Record[]; const severityFilter = stringOrNull(filters.severity); const items = rows.map((row) => { const latestRun = latestRunForFinding(config, db, row); const latestDetail = latestRun === null ? null : storedFindingDetailForRow(db, row, stringOrNull(latestRun.id)); return enrichFindingWithCheck(config, { code: stringOrNull(row.finding_id), findingId: stringOrNull(row.finding_id), severity: stringOrNull(row.severity), scenarioId: stringOrNull(row.scenario_id), count: numberOr(row.count, 0), runCount: numberOr(row.run_count, 0), latestAt: stringOrNull(row.latest_at), latestRunId: latestRun === null ? null : stringOrNull(latestRun.id), latestReportJsonSha256: latestRun === null ? null : stringOrNull(latestRun.report_json_sha256), summary: stringOrNull(row.summary), rootCause: stringOrNull(latestDetail?.rootCause), rootCauseStatus: stringOrNull(latestDetail?.rootCauseStatus), rootCauseConfidence: stringOrNull(latestDetail?.rootCauseConfidence), nextAction: stringOrNull(latestDetail?.nextAction), evidenceSummary: stringOrNull(latestDetail?.evidenceSummary), timingSourceOfTruth: stringOrNull(latestDetail?.timingSourceOfTruth), timingStatus: stringOrNull(latestDetail?.timingStatus), timingAlert: latestDetail?.timingAlert === true, traceability: latestRun === null ? null : runTraceability(config, latestRun), valuesRedacted: true, }); }).filter((item) => severityFilter === null || stringOrNull(item.checkLevel) === severityFilter || stringOrNull(item.severity) === severityFilter) .sort(compareCheckFindingRows) .slice(0, limit); return { ok: true, contractVersion: DASHBOARD_CONTRACT_VERSION, node: config.node, lane: config.lane, sentinelId: config.sentinelId, filters, page: { limit, hasMore: rows.length === queryLimit, sort: "severity", direction: "desc", }, groups: items, findings: items, traceability: { source: "sqlite-index-findings", stateRoot: config.stateRoot, }, valuesRedacted: true, }; } function dashboardRunViews(config: WebProbeSentinelServiceConfig, db: Database, runId: string, view: string | null, url: URL): Record { const row = readRunRow(config, db, runId); if (row === null) return { ok: false, error: "run-not-found", runId, valuesRedacted: true }; const requestedViews = view === null ? stringArrayAt(config.reportViews, "views") : [view]; const maxBytes = Math.min(numberParam(url, "maxBytes", DASHBOARD_MAX_TEXT_BYTES), 64_000); const views = requestedViews.map((item) => dashboardReportView(config, db, runId, item, maxBytes)); return { ok: true, contractVersion: DASHBOARD_CONTRACT_VERSION, sentinelId: config.sentinelId, run: dashboardRunSummary(config, db, row), views, view: view ?? null, redaction: record(config.reportViews.redaction), traceability: runTraceability(config, row), valuesRedacted: true, }; } function dashboardReportView(config: WebProbeSentinelServiceConfig, db: Database, runId: string, view: string, maxBytes: number): Record { const report = reportRunView(config, db, view, runId); const renderedText = typeof report.renderedText === "string" ? report.renderedText : ""; const bounded = boundedText(renderedText, maxBytes); return { ok: report.ok !== false, view, error: stringOrNull(report.error), availableViews: Array.isArray(report.availableViews) ? report.availableViews : undefined, renderedText: bounded.text, renderedTextBytes: bounded.bytes, truncated: bounded.truncated, finalResponse: view === "trace-frame" ? finalResponseBlock(bounded.text) : null, summary: record(report.summary), findingCount: Array.isArray(report.findings) ? report.findings.length : 0, valuesRedacted: true, }; } function dashboardRunMemoryDetail(config: WebProbeSentinelServiceConfig, row: Record, stored: Record): Record { const options = dashboardDetailMemoryOptions(config); const storedMemory = compactDashboardMemory(record(record(record(stored.summary).analysis).browserProcess), options, "recorded-analysis-summary"); if (storedMemory.pageSeries.length > 0) return storedMemory; const fromArtifacts = readDashboardMemoryFromStateDir(config, row, options); if (fromArtifacts.pageSeries.length > 0) return fromArtifacts; return { ok: false, source: "unavailable", unit: "MB", metric: "page-heap-used", pageCount: 0, sampleCount: 0, maxMemoryMb: null, maxPageLabel: null, pageSeries: [], reason: fromArtifacts.reason ?? storedMemory.reason ?? "page-memory-samples-missing", valuesRedacted: true, }; } function dashboardRunRequestRateDetail(config: WebProbeSentinelServiceConfig, row: Record, stored: Record, artifactSummary: Record = {}): Record { const options = dashboardDetailRequestRateOptions(config); const storedAnalysis = record(record(stored.summary).analysis); const storedRequestRate = compactDashboardRequestRate(record(storedAnalysis.requestRate), options, "recorded-analysis-summary"); if (storedRequestRate.ok === true && hasDashboardRequestRateSeries(storedRequestRate)) return storedRequestRate; const storedRequestRateCurve = compactDashboardRequestRate(record(storedAnalysis.requestRateCurve), options, "recorded-analysis-request-rate-curve"); if (storedRequestRateCurve.ok === true && hasDashboardRequestRateSeries(storedRequestRateCurve)) return storedRequestRateCurve; const artifactRequestRate = compactDashboardRequestRate(record(artifactSummary.requestRate), options, "artifact-summary-request-rate"); if (artifactRequestRate.ok === true && hasDashboardRequestRateSeries(artifactRequestRate)) return artifactRequestRate; const fromArtifacts = readDashboardRequestRateFromAnalysisReport(config, row, options); if (fromArtifacts.ok === true) return fromArtifacts; if (artifactRequestRate.ok === true) return artifactRequestRate; if (storedRequestRate.ok === true) return storedRequestRate; if (storedRequestRateCurve.ok === true) return storedRequestRateCurve; return { ok: false, source: "unavailable", unit: "rpm", metric: "same-origin-api-request-rate", requestCount: 0, bucketCount: 0, pageCount: 0, apiPathCount: 0, totalSeries: [], pageSeries: [], apiPathSeries: [], peaks: [], reason: fromArtifacts.reason ?? storedRequestRate.reason ?? storedRequestRateCurve.reason ?? "request-rate-samples-missing", valuesRedacted: true, }; } function hasDashboardRequestRateSeries(value: Record): boolean { return arrayRecords(value.totalSeries).length > 0 || arrayRecords(value.pageSeries).length > 0 || arrayRecords(value.apiPathSeries).length > 0; } function dashboardDetailMemoryOptions(config: WebProbeSentinelServiceConfig): Record { const detailMemory = recordTarget(valueAtPath(config.reportViews, "detailMemory")); return { maxPages: Math.max(1, Math.floor(numberAt(detailMemory, "maxPages"))), maxSamplesPerPage: Math.max(1, Math.floor(numberAt(detailMemory, "maxSamplesPerPage"))), }; } function dashboardDetailRequestRateOptions(config: WebProbeSentinelServiceConfig): Record { const detailRequestRate = recordTarget(valueAtPath(config.reportViews, "detailRequestRate")); return { maxTotalBuckets: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxTotalBuckets"))), maxPageSeries: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxPageSeries"))), maxApiPathSeries: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxApiPathSeries"))), maxBucketsPerSeries: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxBucketsPerSeries"))), maxPeaks: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxPeaks"))), }; } function readDashboardMemoryFromStateDir(config: WebProbeSentinelServiceConfig, row: Record, options: Record): Record { const stateDir = stringOrNull(row.state_dir) ?? stringOrNull(row.stateDir); if (stateDir === null) return { ok: false, source: "artifact-browser-process-jsonl", reason: "state-dir-missing", pageSeries: [], valuesRedacted: true }; if (!isSafeDashboardStateDir(stateDir)) return { ok: false, source: "artifact-browser-process-jsonl", reason: "unsafe-state-dir", pageSeries: [], valuesRedacted: true }; const browserProcessPath = join(config.stateRoot, stateDir, "browser-process.jsonl"); if (!existsSync(browserProcessPath)) return { ok: false, source: "artifact-browser-process-jsonl", reason: "browser-process-jsonl-missing", pageSeries: [], valuesRedacted: true }; try { const rows = readFileSync(browserProcessPath, "utf8") .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean) .flatMap((line) => { const parsed = parseJsonObject(line); return parsed === null ? [] : [parsed]; }); return buildDashboardMemoryDetail(rows, options, "artifact-browser-process-jsonl"); } catch (error) { return { ok: false, source: "artifact-browser-process-jsonl", reason: "browser-process-jsonl-read-failed", error: error instanceof Error ? error.message : String(error), pageSeries: [], valuesRedacted: true, }; } } function compactDashboardMemory(value: Record, options: Record, source: string): Record { const pageSeries = arrayRecords(value.pageSeries); if (pageSeries.length > 0) return finalizeDashboardMemorySeries(pageSeries, options, source); return { ok: false, source, reason: "recorded-page-series-missing", pageSeries: [], valuesRedacted: true }; } function readDashboardRequestRateFromAnalysisReport(config: WebProbeSentinelServiceConfig, row: Record, options: Record): Record { const stateDir = stringOrNull(row.state_dir) ?? stringOrNull(row.stateDir); if (stateDir === null) return { ok: false, source: "analysis-report-json", reason: "state-dir-missing", totalSeries: [], pageSeries: [], apiPathSeries: [], valuesRedacted: true }; if (!isSafeDashboardStateDir(stateDir)) return { ok: false, source: "analysis-report-json", reason: "unsafe-state-dir", totalSeries: [], pageSeries: [], apiPathSeries: [], valuesRedacted: true }; const reportJsonPath = join(config.stateRoot, stateDir, "analysis", "report.json"); const jsonBuffer = readFileBuffer(reportJsonPath); if (jsonBuffer === null) return { ok: false, source: "analysis-report-json", reason: "analysis-report-json-missing", totalSeries: [], pageSeries: [], apiPathSeries: [], valuesRedacted: true }; try { const report = record(JSON.parse(jsonBuffer.toString("utf8")) as unknown); return compactDashboardRequestRate(record(report.requestRate), options, "analysis-report-json"); } catch (error) { return { ok: false, source: "analysis-report-json", reason: "analysis-report-json-invalid", error: error instanceof Error ? error.message : String(error), totalSeries: [], pageSeries: [], apiPathSeries: [], valuesRedacted: true, }; } } function compactDashboardRequestRate(value: Record, options: Record, source: string): Record { const summary = requestRateSummary(record(value.summary).requestCount === undefined ? value : record(value.summary)); const totalSeries = finalizeDashboardRequestRateSeries([{ key: "total", label: "全部 API 请求", count: summary.requestCount, peakRequestPerMinute: summary.totalPeakPerMinute, buckets: arrayRecords(value.buckets), }], numberOr(options.maxTotalBuckets, 1), 1, "total"); const pageSeries = finalizeDashboardRequestRateSeries(arrayRecords(value.pageCurves), numberOr(options.maxBucketsPerSeries, 1), numberOr(options.maxPageSeries, 1), "page"); const apiPathSeries = finalizeDashboardRequestRateSeries(arrayRecords(value.apiPathCurves), numberOr(options.maxBucketsPerSeries, 1), numberOr(options.maxApiPathSeries, 1), "apiPath"); const peaks = compactDashboardRequestRatePeaks(arrayRecords(value.peaks), numberOr(options.maxPeaks, 1)); const hasRequestRate = summary.requestCount !== null || totalSeries.length > 0 || pageSeries.length > 0 || apiPathSeries.length > 0 || peaks.length > 0; if (!hasRequestRate) { return { ok: false, source, reason: "request-rate-report-missing", totalSeries: [], pageSeries: [], apiPathSeries: [], peaks: [], valuesRedacted: true }; } return { ok: true, source, unit: "rpm", metric: "same-origin-api-request-rate", ...summary, totalSeries, pageSeries, apiPathSeries, peaks, valuesRedacted: true, }; } function requestRateSummary(value: Record): Record { return { bucketMs: numberOrNull(value.bucketMs), bucketSeconds: numberOrNull(value.bucketSeconds), requestCount: numberOrNull(value.requestCount), bucketCount: numberOrNull(value.bucketCount), pageCount: numberOrNull(value.pageCount), apiPathCount: numberOrNull(value.apiPathCount), firstAt: stringOrNull(value.firstAt), lastAt: stringOrNull(value.lastAt), totalRedPerMinute: numberOrNull(value.totalRedPerMinute), pageRedPerMinute: numberOrNull(value.pageRedPerMinute), apiPathRedPerMinute: numberOrNull(value.apiPathRedPerMinute), totalPeakPerMinute: numberOrNull(value.totalPeakPerMinute), totalPeakCount: numberOrNull(value.totalPeakCount), totalPeakAt: stringOrNull(value.totalPeakAt), pagePeakPerMinute: numberOrNull(value.pagePeakPerMinute), pagePeakKey: stringOrNull(value.pagePeakKey), pagePeakPath: stringOrNull(value.pagePeakPath), apiPathPeakPerMinute: numberOrNull(value.apiPathPeakPerMinute), apiPathPeakKey: stringOrNull(value.apiPathPeakKey), overThresholdPeakCount: numberOrNull(value.overThresholdPeakCount), }; } function finalizeDashboardRequestRateSeries(rawSeries: readonly Record[], maxBucketsPerSeries: number, maxSeries: number, scope: string): Record[] { return rawSeries .map((item) => { const points = arrayRecords(item.buckets) .map((bucket) => ({ ts: stringOrNull(bucket.startAt) ?? stringOrNull(bucket.t), startAt: stringOrNull(bucket.startAt) ?? stringOrNull(bucket.t), endAt: stringOrNull(bucket.endAt) ?? stringOrNull(bucket.e), bucketStartMs: numberOrNull(bucket.bucketStartMs), bucketEndMs: numberOrNull(bucket.bucketEndMs), count: numberOrNull(bucket.count) ?? numberOrNull(bucket.c), requestPerMinute: numberOrNull(bucket.requestPerMinute) ?? numberOrNull(bucket.rpm), valuesRedacted: true, })) .filter((point) => point.ts !== null && point.requestPerMinute !== null); const thinned = thinDashboardPoints(points, maxBucketsPerSeries); const peakRequestPerMinute = numberOrNull(item.peakRequestPerMinute) ?? maxNumber(thinned.map((point) => numberOrNull(point.requestPerMinute))) ?? numberOrNull(item.totalPeakPerMinute) ?? 0; const count = numberOrNull(item.count) ?? numberOrNull(item.requestCount) ?? thinned.reduce((sum, point) => sum + numberOr(point.count, 0), 0); return { key: requestRateSeriesKey(item, scope), scope, label: requestRateSeriesLabel(item, scope), method: stringOrNull(item.method), path: stringOrNull(item.path), pageRole: stringOrNull(item.pageRole), pageId: stringOrNull(item.pageId), pageEpoch: numberOrNull(item.pageEpoch), count, bucketCount: numberOrNull(item.bucketCount) ?? thinned.length, peakRequestPerMinute, peakAt: stringOrNull(record(item.peakBucket).startAt), points: thinned, valuesRedacted: true, }; }) .filter((item) => arrayRecords(item.points).length > 0) .sort((a, b) => numberOr(b.peakRequestPerMinute, 0) - numberOr(a.peakRequestPerMinute, 0) || numberOr(b.count, 0) - numberOr(a.count, 0)) .slice(0, maxSeries); } function compactDashboardRequestRatePeaks(rawPeaks: readonly Record[], maxPeaks: number): Record[] { return rawPeaks.slice(0, maxPeaks).map((item) => ({ scope: stringOrNull(item.scope), thresholdPerMinute: numberOrNull(item.thresholdPerMinute), overThreshold: item.overThreshold === true, bucketMs: numberOrNull(item.bucketMs), startAt: stringOrNull(item.startAt), endAt: stringOrNull(item.endAt), count: numberOrNull(item.count), requestPerMinute: numberOrNull(item.requestPerMinute), method: stringOrNull(item.method), path: stringOrNull(item.path), apiKey: stringOrNull(item.apiKey), pageKey: stringOrNull(item.pageKey), pageRole: stringOrNull(item.pageRole), pageId: stringOrNull(item.pageId), pageEpoch: numberOrNull(item.pageEpoch), valuesRedacted: true, })); } function requestRateSeriesKey(item: Record, scope: string): string { const fallback = [stringOrNull(item.method), stringOrNull(item.path)].filter((part) => part !== null).join(" "); return stringOrNull(item.key) ?? stringOrNull(item.apiKey) ?? stringOrNull(item.pageKey) ?? (fallback.length > 0 ? fallback : scope); } function requestRateSeriesLabel(item: Record, scope: string): string { if (scope === "total") return "全部 API 请求"; if (scope === "apiPath") return (stringOrNull(item.apiKey) ?? [stringOrNull(item.method), stringOrNull(item.path)].filter((part) => part !== null).join(" ")) || "API"; return stringOrNull(item.path) ?? stringOrNull(item.pageKey) ?? stringOrNull(item.pageRole) ?? "page"; } function buildDashboardMemoryDetail(rows: readonly Record[], options: Record, source: string): Record { const seriesByKey = new Map>(); let firstTsMs: number | null = null; for (const row of rows) { if (stringOrNull(row.type) !== "browser-process-sample") continue; const ts = stringOrNull(row.ts); const tsMs = ts === null ? null : Date.parse(ts); if (ts === null || tsMs === null || !Number.isFinite(tsMs)) continue; for (const page of arrayRecords(row.pages)) { const memory = dashboardPageMemoryPoint(page); if (memory === null) continue; if (firstTsMs === null) firstTsMs = tsMs; const pageRole = stringOrNull(page.pageRole) ?? "page"; const pageId = stringOrNull(page.pageId) ?? `${pageRole}-${stringOrNull(page.pageEpoch) ?? seriesByKey.size}`; const key = `${pageRole}:${pageId}`; const existing = seriesByKey.get(key); const label = `${pageRole} ${shortDashboardText(pageId, 10)}`; const points = arrayRecords(existing?.points); points.push({ ts, elapsedSeconds: Math.max(0, Math.round((tsMs - firstTsMs) / 1000)), elapsedMinutes: Math.round(Math.max(0, (tsMs - firstTsMs) / 60_000) * 100) / 100, ...memory, valuesRedacted: true, }); seriesByKey.set(key, { key, pageRole, pageId, label, url: safeDashboardUrl(stringOrNull(page.url)), points, valuesRedacted: true, }); } } return finalizeDashboardMemorySeries([...seriesByKey.values()], options, source); } function finalizeDashboardMemorySeries(rawSeries: readonly Record[], options: Record, source: string): Record { const maxPages = numberOr(options.maxPages, 1); const maxSamplesPerPage = numberOr(options.maxSamplesPerPage, 1); const series = rawSeries .map((item) => { const points = arrayRecords(item.points) .map((point) => ({ ts: stringOrNull(point.ts), elapsedSeconds: numberOrNull(point.elapsedSeconds), elapsedMinutes: numberOrNull(point.elapsedMinutes), memoryMb: numberOrNull(point.memoryMb), heapUsedMb: numberOrNull(point.heapUsedMb), jsHeapUsedMb: numberOrNull(point.jsHeapUsedMb), effectiveHeapUsedMb: numberOrNull(point.effectiveHeapUsedMb), effectiveJsHeapUsedMb: numberOrNull(point.effectiveJsHeapUsedMb), domNodes: numberOrNull(point.domNodes), valuesRedacted: true, })) .filter((point) => point.ts !== null && point.memoryMb !== null); const thinned = thinDashboardPoints(points, maxSamplesPerPage); const maxMemoryMb = maxNumber(thinned.map((point) => numberOrNull(point.memoryMb))); return { key: stringOrNull(item.key) ?? stringOrNull(item.pageId) ?? "page", pageRole: stringOrNull(item.pageRole), pageId: stringOrNull(item.pageId), label: stringOrNull(item.label) ?? stringOrNull(item.pageRole) ?? stringOrNull(item.pageId) ?? "page", url: safeDashboardUrl(stringOrNull(item.url)), sampleCount: thinned.length, maxMemoryMb, latestMemoryMb: thinned.length > 0 ? numberOrNull(thinned[thinned.length - 1].memoryMb) : null, points: thinned, valuesRedacted: true, }; }) .filter((item) => item.sampleCount > 0) .sort((a, b) => numberOr(b.maxMemoryMb, 0) - numberOr(a.maxMemoryMb, 0)) .slice(0, maxPages); const sampleCount = series.reduce((sum, item) => sum + numberOr(item.sampleCount, 0), 0); const maxSeries = series.reduce | null>((best, item) => best === null || numberOr(item.maxMemoryMb, 0) > numberOr(best.maxMemoryMb, 0) ? item : best, null); return { ok: series.length > 0, source, unit: "MB", metric: "page-heap-used", pageCount: series.length, sampleCount, maxMemoryMb: maxSeries === null ? null : numberOrNull(maxSeries.maxMemoryMb), maxPageLabel: maxSeries === null ? null : stringOrNull(maxSeries.label), pageSeries: series, valuesRedacted: true, }; } function dashboardPageMemoryPoint(page: Record): Record | null { const effectiveMemory = record(page.effectiveMemory); const heapUsage = record(page.heapUsage); const performance = record(page.performance); const metrics = record(performance.metrics); const heapUsedMb = numberOrNull(effectiveMemory.heapUsedMb) ?? bytesToMbNullable(heapUsage.usedSize); const jsHeapUsedMb = numberOrNull(effectiveMemory.jsHeapUsedMb) ?? bytesToMbNullable(metrics.JSHeapUsedSize); const memoryMb = heapUsedMb ?? jsHeapUsedMb; if (memoryMb === null) return null; return { memoryMb, heapUsedMb, jsHeapUsedMb, effectiveHeapUsedMb: numberOrNull(effectiveMemory.effectiveHeapUsedMb), effectiveJsHeapUsedMb: numberOrNull(effectiveMemory.effectiveJsHeapUsedMb), domNodes: numberOrNull(effectiveMemory.domNodes) ?? numberOrNull(record(page.domCounters).nodes) ?? numberOrNull(metrics.Nodes), }; } function thinDashboardPoints(points: readonly Record[], maxSamples: number): readonly Record[] { if (points.length <= maxSamples) return points; const selected: Record[] = []; const last = points.length - 1; const targetLast = Math.max(1, maxSamples - 1); for (let index = 0; index < maxSamples; index += 1) { selected.push(points[Math.round((index / targetLast) * last)]); } return selected; } function isSafeDashboardStateDir(value: string): boolean { if (value.startsWith("/") || value.includes("\0")) return false; const normalized = value.replace(/\\/gu, "/"); if (!normalized.startsWith(".state/")) return false; return normalized.split("/").every((part) => part !== ".."); } function bytesToMbNullable(value: unknown): number | null { const bytes = numberOrNull(value); return bytes === null ? null : Math.round((bytes / 1024 / 1024) * 100) / 100; } function maxNumber(values: readonly (number | null)[]): number | null { const finite = values.filter((value): value is number => value !== null && Number.isFinite(value)); return finite.length === 0 ? null : Math.max(...finite); } function safeDashboardUrl(value: string | null): string | null { if (value === null) return null; return value.slice(0, 180); } function shortDashboardText(value: string, max: number): string { return value.length <= max ? value : `${value.slice(0, max)}`; } function dashboardRunSummary(config: WebProbeSentinelServiceConfig, db: Database, row: Record): Record { const id = stringOrNull(row.id); const artifactSummary = readAnalysisArtifactSummaryForRun(config, row, 50); const findings = visibleFindingsForRun(config, db, row, 50, artifactSummary); const severityCounts = checkLevelCounts(config, findings); const maxSeverity = maxSeverityFromCounts(severityCounts); const findingTypeCount = visibleFindingTypeCount(row, findings, artifactSummary); const findingSampleCount = Object.values(severityCounts).reduce((sum, value) => sum + numberOr(value, 0), 0); const findingAlertSampleCount = alertSeveritySampleCount(severityCounts); const stored = id === null ? {} : record(readMetadata(db, `run.report.${id}`)); const scenarioId = stringOrNull(row.scenario_id); const timing = dashboardRunTiming(config, row, stored); const reportJsonSha256 = stringOrNull(row.report_json_sha256) ?? stringOrNull(artifactSummary.reportJsonSha256); return { id, runId: id, scenario_id: scenarioId, scenarioId, status: stringOrNull(row.status), node: stringOrNull(row.node) ?? config.node, lane: stringOrNull(row.lane) ?? config.lane, sentinelId: config.sentinelId, observer_id: stringOrNull(row.observer_id), observerId: stringOrNull(row.observer_id), state_dir: stringOrNull(row.state_dir), stateDir: stringOrNull(row.state_dir), report_json_sha256: reportJsonSha256, reportJsonSha256, finding_count: findingTypeCount, findingCount: findingTypeCount, findingTypeCount, findingSampleCount, findingAlertSampleCount, artifact_count: numberOr(row.artifact_count, 0), artifactCount: numberOr(row.artifact_count, 0), maintenance: numberOr(row.maintenance, 0) === 1, created_at: stringOrNull(row.created_at), createdAt: stringOrNull(row.created_at), updated_at: stringOrNull(row.updated_at), updatedAt: stringOrNull(row.updated_at), interrupted_at: stringOrNull(row.interrupted_at), interruptedAt: stringOrNull(row.interrupted_at), startedAt: stringOrNull(timing.startedAt), finishedAt: stringOrNull(timing.finishedAt), durationMs: numberOrNull(timing.durationMs), runDurationMs: numberOrNull(timing.durationMs), durationMinutes: numberOrNull(timing.durationMinutes), runDurationMinutes: numberOrNull(timing.durationMinutes), durationSource: stringOrNull(timing.durationSource), scenarioCadence: stringOrNull(timing.scenarioCadence), scenarioCadenceMinutes: numberOrNull(timing.scenarioCadenceMinutes), scenarioMaxRunMinutes: numberOrNull(timing.scenarioMaxRunMinutes), schedulerIntervalMinutes: numberOrNull(timing.schedulerIntervalMinutes), timing, severityCounts, maxSeverity, traceability: runTraceability(config, row), valuesRedacted: true, }; } function dashboardRunTiming(config: WebProbeSentinelServiceConfig, row: Record, stored: Record): Record { const scenarioId = stringOrNull(row.scenario_id); const scenario = scenarioId === null ? null : config.scenarios.find((item) => stringOrNull(item.id) === scenarioId) ?? null; const summary = record(stored.summary); const summaryElapsedMs = numberOrNull(summary.elapsedMs); const startedAt = stringOrNull(row.created_at); const finishedAt = stringOrNull(row.interrupted_at) ?? stringOrNull(row.updated_at); const timestampDurationMs = durationMsBetween(startedAt, finishedAt); const durationMs = summaryElapsedMs ?? timestampDurationMs; const durationSource = summaryElapsedMs !== null ? "report-summary-elapsedMs" : timestampDurationMs !== null ? "sqlite-created-updated" : "unknown"; const scenarioCadence = scenario === null ? null : stringOrNull(scenario.cadence); const scenarioCadenceSeconds = durationStringSeconds(scenarioCadence); const scenarioMaxRunSeconds = scenario === null ? null : numberOrNull(scenario.maxRunSeconds); return { startedAt, finishedAt, durationMs, durationMinutes: minutesFromMs(durationMs), durationSource, scenarioCadence, scenarioCadenceSeconds, scenarioCadenceMinutes: minutesFromSeconds(scenarioCadenceSeconds), scenarioMaxRunSeconds, scenarioMaxRunMinutes: minutesFromSeconds(scenarioMaxRunSeconds), schedulerIntervalMs: config.schedulerIntervalMs, schedulerIntervalMinutes: minutesFromMs(config.schedulerIntervalMs), sourceOfTruth: "sqlite-run-timestamps+report-summary+yaml-scenario-runtime", valuesRedacted: true, }; } function alertSeveritySampleCount(counts: Record): number { return numberOr(counts.red, 0) + numberOr(counts.critical, 0) + numberOr(counts.error, 0) + numberOr(counts.warning, 0) + numberOr(counts.warn, 0) + numberOr(counts.amber, 0); } function dashboardOverallStatus(health: Record, latestRun: Record | null, severityCounts: Record): string { if (health.ok !== true) return "degraded"; const latestStatus = latestRun === null ? null : stringOrNull(latestRun.status); if (latestStatus !== null && /blocked|failed|error|timeout/iu.test(latestStatus)) return "blocked"; if (Object.values(severityCounts).some((count) => count > 0)) return "warning"; return latestRun === null ? "idle" : "healthy"; } function dashboardRunFilters(url: URL): Record { return { status: optionalSearchParam(url, "status"), scenario: optionalSearchParam(url, "scenario"), severity: optionalSearchParam(url, "severity"), maintenance: maintenanceFilter(url), observerId: optionalSearchParam(url, "observerId"), search: optionalSearchParam(url, "search"), from: validIsoParam(url, "from"), to: validIsoParam(url, "to"), }; } function dashboardFindingFilters(url: URL): Record { return { severity: optionalSearchParam(url, "severity"), code: optionalSearchParam(url, "code"), scenario: optionalSearchParam(url, "scenario"), search: optionalSearchParam(url, "search"), window: optionalSearchParam(url, "window"), since: windowSinceIso(optionalSearchParam(url, "window")) ?? validIsoParam(url, "from"), to: validIsoParam(url, "to"), }; } function runWhereClause(config: WebProbeSentinelServiceConfig, filters: Record): { readonly sql: string; readonly params: readonly (string | number)[] } { const clauses: string[] = ["sentinel_id = ?", "node = ?", "lane = ?"]; const params: (string | number)[] = [config.sentinelId, config.node, config.lane]; const status = stringOrNull(filters.status); if (status !== null) { clauses.push("status = ?"); params.push(status); } const scenario = stringOrNull(filters.scenario); if (scenario !== null) { clauses.push("scenario_id = ?"); params.push(scenario); } const observerId = stringOrNull(filters.observerId); if (observerId !== null) { clauses.push("observer_id = ?"); params.push(observerId); } const severity = stringOrNull(filters.severity); if (severity !== null) { clauses.push("EXISTS (SELECT 1 FROM findings f WHERE f.run_id = runs.id AND f.severity = ?)"); params.push(severity); } if (typeof filters.maintenance === "boolean") { clauses.push("maintenance = ?"); params.push(filters.maintenance ? 1 : 0); } const from = stringOrNull(filters.from); if (from !== null) { clauses.push("updated_at >= ?"); params.push(from); } const to = stringOrNull(filters.to); if (to !== null) { clauses.push("updated_at <= ?"); params.push(to); } const search = stringOrNull(filters.search); if (search !== null) { const pattern = `%${escapeSqlLike(search)}%`; clauses.push("(id LIKE ? ESCAPE '\\' OR scenario_id LIKE ? ESCAPE '\\' OR observer_id LIKE ? ESCAPE '\\' OR state_dir LIKE ? ESCAPE '\\' OR report_json_sha256 LIKE ? ESCAPE '\\')"); params.push(pattern, pattern, pattern, pattern, pattern); } return { sql: clauses.length === 0 ? "" : `WHERE ${clauses.join(" AND ")}`, params }; } function findingWhereClause(config: WebProbeSentinelServiceConfig, filters: Record): { readonly sql: string; readonly params: readonly (string | number)[] } { const clauses: string[] = ["r.sentinel_id = ?", "r.node = ?", "r.lane = ?", "f.sentinel_id = ?"]; const params: (string | number)[] = [config.sentinelId, config.node, config.lane, config.sentinelId]; const code = stringOrNull(filters.code); if (code !== null) { clauses.push("f.finding_id = ?"); params.push(code); } const scenario = stringOrNull(filters.scenario); if (scenario !== null) { clauses.push("r.scenario_id = ?"); params.push(scenario); } const since = stringOrNull(filters.since); if (since !== null) { clauses.push("f.created_at >= ?"); params.push(since); } const to = stringOrNull(filters.to); if (to !== null) { clauses.push("f.created_at <= ?"); params.push(to); } const search = stringOrNull(filters.search); if (search !== null) { const pattern = `%${escapeSqlLike(search)}%`; clauses.push("(f.finding_id LIKE ? ESCAPE '\\' OR f.summary LIKE ? ESCAPE '\\')"); params.push(pattern, pattern); } return { sql: clauses.length === 0 ? "" : `WHERE ${clauses.join(" AND ")}`, params }; } function dashboardRunSort(url: URL): { readonly name: string; readonly direction: "asc" | "desc"; readonly sql: string } { const requested = optionalSearchParam(url, "sort") ?? "updated"; const direction = optionalSearchParam(url, "direction") === "asc" ? "asc" : "desc"; const column = requested === "created" ? "created_at" : requested === "findings" ? "finding_count" : requested === "severity" ? severityRankSql("(SELECT severity FROM findings f WHERE f.run_id = runs.id ORDER BY " + severityRankSql("f.severity") + " DESC LIMIT 1)") : "updated_at"; return { name: requested, direction, sql: `${column} ${direction.toUpperCase()}, id ${direction.toUpperCase()}` }; } function dashboardPage(url: URL, config: WebProbeSentinelServiceConfig): { readonly limit: number; readonly offset: number } { const limit = Math.min(numberParam(url, "limit", dashboardPageSize(config)), dashboardMaxPageSize(config)); const rawCursor = url.searchParams.get("cursor"); const offset = rawCursor === null ? 0 : Math.max(0, Math.min(100_000, Number.isInteger(Number(rawCursor)) ? Number(rawCursor) : 0)); return { limit, offset }; } function dashboardPageSize(config: WebProbeSentinelServiceConfig): number { return Math.max(1, Math.min(dashboardMaxPageSize(config), numberOr(config.reportViews.pageSize, 20))); } function dashboardMaxPageSize(config: WebProbeSentinelServiceConfig): number { return Math.max(1, Math.min(200, numberOr(config.reportViews.maxPageSize, 100))); } function readRunRow(config: WebProbeSentinelServiceConfig, db: Database, runId: string): Record | null { return db.query("SELECT * FROM runs WHERE id = ? AND sentinel_id = ? AND node = ? AND lane = ?") .get(runId, config.sentinelId, config.node, config.lane) as Record | null; } function findingsForRun(config: WebProbeSentinelServiceConfig, db: Database, runId: string, limit: number): readonly Record[] { const rows = db.query("SELECT finding_id, severity, count, summary, report_json_sha256, created_at FROM findings WHERE run_id = ? AND sentinel_id = ? ORDER BY created_at DESC LIMIT ?") .all(runId, config.sentinelId, limit) as Record[]; return rows.map((row) => enrichFindingRowWithStoredDetail(config, db, runId, row)); } function visibleFindingsForRun( config: WebProbeSentinelServiceConfig, db: Database, row: Record, limit: number, artifactSummary: Record = readAnalysisArtifactSummaryForRun(config, row, limit), ): readonly Record[] { const runId = stringOrNull(row.id); const storedFindings = runId === null ? [] : findingsForRun(config, db, runId, limit); const artifactFindings = arrayRecords(artifactSummary.findings) .slice(0, limit) .map((item) => enrichFindingWithCheck(config, compactStoredFinding(item))); return mergeVisibleFindings(config, storedFindings, artifactFindings, limit); } function mergeVisibleFindings(config: WebProbeSentinelServiceConfig, primary: readonly Record[], artifact: readonly Record[], limit: number): readonly Record[] { const merged: Record[] = []; const seen = new Set(); for (const item of [...primary, ...artifact]) { const id = canonicalFindingIdentity(config, item); const key = id ?? `${JSON.stringify(item).slice(0, 160)}:${stringOrNull(item.severity) ?? stringOrNull(item.level) ?? ""}`; if (seen.has(key)) continue; seen.add(key); merged.push(item); if (merged.length >= limit) break; } return merged; } function canonicalFindingIdentity(config: WebProbeSentinelServiceConfig, item: Record): string | null { const check = record(item.check); const explicitCheckCode = stringOrNull(item.checkCode) ?? stringOrNull(check.code); if (explicitCheckCode !== null) return explicitCheckCode; const catalog = checkCatalogById(config); const explicitCheckId = stringOrNull(item.checkId) ?? stringOrNull(check.id); if (explicitCheckId !== null) { const entry = catalog.get(explicitCheckId); return stringOrNull(entry?.code) ?? stringOrNull(entry?.id) ?? explicitCheckId; } const findingId = findingIdentity(item); if (findingId === null) return null; const entry = catalog.get(findingId); return stringOrNull(entry?.code) ?? stringOrNull(entry?.id) ?? findingId; } function findingIdentity(item: Record): string | null { return stringOrNull(item.finding_id) ?? stringOrNull(item.findingId) ?? stringOrNull(item.id) ?? stringOrNull(item.kind) ?? stringOrNull(item.code); } function visibleFindingTypeCount(row: Record, findings: readonly Record[], artifactSummary: Record): number { return Math.max( numberOr(row.finding_count, 0), numberOr(artifactSummary.findingCount, 0), findings.length, ); } function readAnalysisArtifactSummaryForRun(config: WebProbeSentinelServiceConfig, row: Record, limit: number): Record { const stateDir = stringOrNull(row.state_dir) ?? stringOrNull(row.stateDir); if (stateDir === null) return { ok: false, reason: "state-dir-missing", findings: [], valuesRedacted: true }; if (!isSafeDashboardStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, findings: [], valuesRedacted: true }; const reportJsonPath = join(config.stateRoot, stateDir, "analysis", "report.json"); const reportMdPath = join(config.stateRoot, stateDir, "analysis", "report.md"); const jsonBuffer = readFileBuffer(reportJsonPath); const mdBuffer = readFileBuffer(reportMdPath); let parsed: unknown = null; if (jsonBuffer !== null) { try { parsed = JSON.parse(jsonBuffer.toString("utf8")) as unknown; } catch { parsed = null; } } const report = record(parsed); const reportFindings = arrayRecords(report.findings); const archiveFindings = arrayRecords(record(report.archiveSummary).redFindings); const findings = (reportFindings.length > 0 ? reportFindings : archiveFindings) .slice(0, limit) .map(compactStoredFinding); return { ok: parsed !== null, reason: parsed === null ? "analysis-report-json-missing-or-invalid" : null, stateDir, reportJsonPath, reportJsonSha256: sha256Buffer(jsonBuffer), reportMdSha256: sha256Buffer(mdBuffer), findingCount: numberOr(report.findingCount, findings.length), findings, counts: record(report.counts), valuesRedacted: true, }; } function readFileBuffer(path: string): Buffer | null { try { return readFileSync(path); } catch { return null; } } function sha256Buffer(value: Buffer | null): string | null { return value === null ? null : `sha256:${createHash("sha256").update(value).digest("hex")}`; } function enrichFindingRowWithStoredDetail(config: WebProbeSentinelServiceConfig, db: Database, runId: string, row: Record): Record { const detail = storedFindingDetailForRow(db, row, runId); return enrichFindingWithCheck(config, { ...row, code: stringOrNull(row.finding_id), findingId: stringOrNull(row.finding_id), rootCause: stringOrNull(detail?.rootCause), rootCauseStatus: stringOrNull(detail?.rootCauseStatus), rootCauseConfidence: stringOrNull(detail?.rootCauseConfidence), nextAction: stringOrNull(detail?.nextAction), evidenceSummary: stringOrNull(detail?.evidenceSummary), rootCauseSignals: record(detail?.rootCauseSignals), timingSourceOfTruth: stringOrNull(detail?.timingSourceOfTruth), timingStatus: stringOrNull(detail?.timingStatus), timingAlert: detail?.timingAlert === true, valuesRedacted: true, }); } function storedFindingDetailForRow(db: Database, row: Record, runId: string | null): Record | null { if (runId === null) return null; const stored = readMetadata(db, `run.report.${runId}`) ?? {}; const details = arrayRecords(stored.findings); if (details.length === 0) return null; const findingId = stringOrNull(row.finding_id) ?? stringOrNull(row.findingId) ?? stringOrNull(row.code); const severity = stringOrNull(row.severity); const sameIdDetails = details.filter((item) => { const itemId = stringOrNull(item.finding_id) ?? stringOrNull(item.findingId) ?? stringOrNull(item.id) ?? stringOrNull(item.code); return itemId === findingId; }); if (sameIdDetails.length === 0) return null; return sameIdDetails.find((item) => { const itemSeverity = normalizeCheckLevel(stringOrNull(item.severity) ?? stringOrNull(item.level)); return severity === null || itemSeverity === null || itemSeverity === severity; }) ?? sameIdDetails[0] ?? null; } function compactStoredFinding(value: unknown): Record { const item = record(value); const findingId = stringOrNull(item.id) ?? stringOrNull(item.kind) ?? stringOrNull(item.code) ?? "finding"; const severity = stringOrNull(item.severity) ?? stringOrNull(item.level) ?? "unknown"; const summary = stringOrNull(item.summary) ?? stringOrNull(item.message) ?? findingId; return { id: findingId, finding_id: findingId, findingId, code: findingId, severity, count: numberOr(item.count, numberOr(item.sampleCount, 1)), summary: summary.slice(0, 500), rootCause: stringOrNull(item.rootCause), rootCauseStatus: stringOrNull(item.rootCauseStatus), rootCauseConfidence: stringOrNull(item.rootCauseConfidence), nextAction: stringOrNull(item.nextAction), evidenceSummary: stringOrNull(item.evidenceSummary) ?? compactFindingEvidenceSummary(item.evidence), rootCauseSignals: compactFindingRootCauseSignals(item.rootCauseSignals), timingSourceOfTruth: stringOrNull(item.timingSourceOfTruth) ?? stringOrNull(item.expectedElapsedSource) ?? stringOrNull(item.evidenceKind), timingStatus: stringOrNull(item.timingStatus), timingAlert: item.timingAlert === true, valuesRedacted: true, }; } function compactFindingRootCauseSignals(value: unknown): Record | null { const item = record(value); const keys = [ "sessionListReadCount", "traceEventsReadCount", "webPerformanceBeaconFailureCount", "eventSourceFailureCount", "requestFailedCount", "httpErrorCount", "consoleAlertCount", "requestfailedTop", "httpStatusTop", ]; const compact: Record = {}; for (const key of keys) { const raw = item[key]; if (raw === null || raw === undefined) continue; compact[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw; } return Object.keys(compact).length === 0 ? null : { ...compact, valuesRedacted: true }; } function compactFindingEvidenceSummary(value: unknown): string | null { if (value === null || value === undefined) return null; if (typeof value === "string") return value.slice(0, 240); const item = record(value); const keys = [ "http404Count", "responseErrorCount", "requestFailedCount", "statuses", "afterProjectedSeqs", "sinceSeqs", "traceIds", "maxFallbackRatio", "maxFallbackTitleCount", "overThresholdSampleCount", "majorityFallbackSampleCount", ]; const compact: Record = {}; for (const key of keys) { const raw = item[key]; if (raw === null || raw === undefined) continue; compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw; } const text = Object.keys(compact).length > 0 ? JSON.stringify(compact) : JSON.stringify(item).slice(0, 240); return text.length > 240 ? `${text.slice(0, 239)}…` : text; } function enrichFindingWithCheck(config: WebProbeSentinelServiceConfig, value: Record): Record { const rawSeverity = stringOrNull(value.severity) ?? stringOrNull(value.level); const check = checkForFinding(config, value); const checkLevel = stringOrNull(check.level) ?? normalizeCheckLevel(rawSeverity) ?? "unknown"; return { ...value, rawSeverity, severity: checkLevel, level: checkLevel, check, checkId: stringOrNull(check.id), checkCode: stringOrNull(check.code), checkLevel, checkTitleZh: stringOrNull(check.titleZh), checkSummaryZh: stringOrNull(check.summaryZh), checkActionZh: stringOrNull(check.actionZh), checkRegistered: check.registered === true, blocking: value.blocking === true || check.blocking === true, valuesRedacted: true, }; } function checkForFinding(config: WebProbeSentinelServiceConfig, value: Record): Record { const findingId = stringOrNull(value.finding_id) ?? stringOrNull(value.findingId) ?? stringOrNull(value.id) ?? stringOrNull(value.kind) ?? stringOrNull(value.code) ?? "unknown-check"; const entry = checkCatalogById(config).get(findingId) ?? null; const rawLevel = normalizeCheckLevel(stringOrNull(value.severity) ?? stringOrNull(value.level)); if (entry === null) { return { id: findingId, code: null, level: rawLevel ?? "unknown", titleZh: "未登记监测项", summaryZh: "该监测项未在 YAML 中登记,配置校验需要补齐中文说明。", actionZh: "补齐 YAML 监测项说明后重新验证。", blocking: value.blocking === true, order: 10000, registered: false, valuesRedacted: true, }; } return { id: stringOrNull(entry.id) ?? findingId, code: stringOrNull(entry.code), level: normalizeCheckLevel(stringOrNull(entry.level)) ?? rawLevel ?? "unknown", titleZh: stringOrNull(entry.titleZh) ?? findingId, summaryZh: stringOrNull(entry.summaryZh) ?? "已记录监测项详情。", actionZh: stringOrNull(entry.actionZh) ?? "查看详情后处理。", blocking: entry.blocking === true, order: numberOr(entry.order, 10000), registered: true, valuesRedacted: true, }; } function checkCatalogById(config: WebProbeSentinelServiceConfig): Map> { const catalog = record(config.reportViews.checkCatalog); const items = arrayRecords(catalog.items); const result = new Map>(); for (const item of items) { const id = stringOrNull(item.id); if (id !== null) result.set(id, item); const aliases = Array.isArray(item.aliases) ? item.aliases : []; for (const alias of aliases) { if (typeof alias === "string" && alias.length > 0) result.set(alias, item); } } return result; } function checkLevelCounts(config: WebProbeSentinelServiceConfig, rows: readonly Record[]): Record { const counts: Record = {}; for (const row of rows) { const check = checkForFinding(config, row); const level = stringOrNull(check.level) ?? normalizeCheckLevel(stringOrNull(row.severity)) ?? "unknown"; counts[level] = (counts[level] ?? 0) + numberOr(row.count, 1); } return counts; } function compareCheckFindingRows(left: Record, right: Record): number { const severity = severityRank(stringOrNull(right.checkLevel) ?? stringOrNull(right.severity)) - severityRank(stringOrNull(left.checkLevel) ?? stringOrNull(left.severity)); if (severity !== 0) return severity; const order = numberOr(record(left.check).order, 10000) - numberOr(record(right.check).order, 10000); if (order !== 0) return order; const rightTime = Date.parse(stringOrNull(right.latestAt) ?? ""); const leftTime = Date.parse(stringOrNull(left.latestAt) ?? ""); return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0); } function normalizeCheckLevel(value: string | null): string | null { const normalized = (value ?? "").toLowerCase(); if (["critical", "fatal"].includes(normalized)) return "critical"; if (["red", "error", "failed", "blocked"].includes(normalized)) return "error"; if (["warning", "warn", "amber", "yellow"].includes(normalized)) return "warning"; if (["info", "notice"].includes(normalized)) return "info"; return null; } function globalSeverityCounts(config: WebProbeSentinelServiceConfig, db: Database): Record { const rows = db.query("SELECT finding_id, severity, count FROM findings WHERE sentinel_id = ?").all(config.sentinelId) as Record[]; return checkLevelCounts(config, rows); } function severityCountsForRun(config: WebProbeSentinelServiceConfig, db: Database, runId: string): Record { const rows = db.query("SELECT finding_id, severity, count FROM findings WHERE run_id = ? AND sentinel_id = ?").all(runId, config.sentinelId) as Record[]; return checkLevelCounts(config, rows); } function latestRunForFinding(config: WebProbeSentinelServiceConfig, db: Database, row: Record): Record | null { return db.query(` SELECT r.* FROM findings f JOIN runs r ON r.id = f.run_id WHERE f.finding_id = ? AND r.scenario_id = ? AND r.sentinel_id = ? AND r.node = ? AND r.lane = ? AND f.sentinel_id = ? ORDER BY f.created_at DESC LIMIT 1 `).get(stringOrNull(row.finding_id), stringOrNull(row.scenario_id), config.sentinelId, config.node, config.lane, config.sentinelId) as Record | null; } function runTraceability(config: WebProbeSentinelServiceConfig, row: Record): Record { return { source: "sqlite-index+run-report-metadata", node: stringOrNull(row.node) ?? config.node, lane: stringOrNull(row.lane) ?? config.lane, sentinelId: stringOrNull(row.sentinel_id) ?? config.sentinelId, runId: stringOrNull(row.id), observerId: stringOrNull(row.observer_id), stateDir: stringOrNull(row.state_dir), reportJsonSha256: stringOrNull(row.report_json_sha256), stateRoot: config.stateRoot, valuesRedacted: true, }; } function compactArtifactRef(value: unknown): Record | null { const source = record(value); if (Object.keys(source).length === 0) return null; return { path: stringOrNull(source.path) ?? stringOrNull(source.file) ?? stringOrNull(source.screenshotPath), sha256: stringOrNull(source.sha256) ?? stringOrNull(source.hash), byteCount: typeof source.byteCount === "number" ? source.byteCount : null, valuesRedacted: true, }; } function boundedText(value: string, maxBytes: number): { readonly text: string; readonly bytes: number; readonly truncated: boolean } { const bytes = Buffer.byteLength(value, "utf8"); if (bytes <= maxBytes) return { text: value, bytes, truncated: false }; return { text: Buffer.from(value, "utf8").subarray(0, maxBytes).toString("utf8"), bytes, truncated: true }; } function finalResponseBlock(text: string): Record { const index = text.toLowerCase().lastIndexOf("final response"); const body = index === -1 ? "" : text.slice(index + "final response".length).replace(/^[\s=\-:]+/u, "").trim(); const normalized = body.length === 0 ? "(空内容)" : body; return { label: "Final Response", empty: body.length === 0 || body === "(空内容)", text: normalized, byteCount: Buffer.byteLength(normalized, "utf8"), valuesRedacted: true, }; } function optionalSearchParam(url: URL, key: string): string | null { const raw = url.searchParams.get(key); if (raw === null) return null; const trimmed = raw.trim(); return trimmed.length === 0 ? null : trimmed.slice(0, 200); } function validIsoParam(url: URL, key: string): string | null { const value = optionalSearchParam(url, key); if (value === null) return null; return Number.isFinite(Date.parse(value)) ? new Date(value).toISOString() : null; } function maintenanceFilter(url: URL): boolean | null { const raw = optionalSearchParam(url, "maintenance"); if (raw === null) return null; if (/^(1|true|yes)$/iu.test(raw)) return true; if (/^(0|false|no)$/iu.test(raw)) return false; return null; } function windowSinceIso(value: string | null): string | null { if (value === null) return null; const match = /^(\d+)(m|h|d)$/iu.exec(value); if (match === null) return null; const amount = Number(match[1]); const unit = match[2].toLowerCase(); const ms = unit === "m" ? amount * 60_000 : unit === "h" ? amount * 3_600_000 : amount * 86_400_000; return new Date(Date.now() - ms).toISOString(); } function escapeSqlLike(value: string): string { return value.replace(/[\\%_]/gu, (char) => `\\${char}`); } function ageSeconds(iso: string): number | null { const parsed = Date.parse(iso); return Number.isFinite(parsed) ? Math.max(0, Math.round((Date.now() - parsed) / 1000)) : null; } function durationMsBetween(startIso: string | null, endIso: string | null): number | null { if (startIso === null || endIso === null) return null; const start = Date.parse(startIso); const end = Date.parse(endIso); if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null; return end - start; } function durationStringSeconds(value: string | null): number | null { if (value === null) return null; const match = /^(\d+(?:\.\d+)?)(ms|s|m|h|d)$/iu.exec(value.trim()); if (match === null) return null; const amount = Number(match[1]); if (!Number.isFinite(amount)) return null; const unit = match[2].toLowerCase(); const seconds = unit === "ms" ? amount / 1000 : unit === "s" ? amount : unit === "m" ? amount * 60 : unit === "h" ? amount * 3600 : amount * 86400; return Math.round(seconds * 1000) / 1000; } function minutesFromMs(value: unknown): number | null { const ms = numberOrNull(value); return ms === null ? null : roundMinutes(ms / 60_000); } function minutesFromSeconds(value: unknown): number | null { const seconds = numberOrNull(value); return seconds === null ? null : roundMinutes(seconds / 60); } function roundMinutes(value: number): number { return Math.round(value * 100) / 100; } function maxSeverityFromCounts(counts: Record): string | null { let best: string | null = null; for (const [severity, count] of Object.entries(counts)) { if (count <= 0) continue; if (best === null || severityRank(severity) > severityRank(best)) best = severity; } return best; } function severityRank(value: string | null): number { const normalized = (value ?? "").toLowerCase(); if (/critical|red|fatal/iu.test(normalized)) return 4; if (/error|failed|blocked/iu.test(normalized)) return 3; if (/warn|amber|yellow/iu.test(normalized)) return 2; if (/info|notice/iu.test(normalized)) return 1; return 0; } function severityRankSql(expression: string): string { return `CASE LOWER(COALESCE(${expression}, '')) WHEN 'critical' THEN 4 WHEN 'red' THEN 4 WHEN 'fatal' THEN 4 WHEN 'error' THEN 3 WHEN 'failed' THEN 3 WHEN 'blocked' THEN 3 WHEN 'warning' THEN 2 WHEN 'warn' THEN 2 WHEN 'amber' THEN 2 WHEN 'yellow' THEN 2 WHEN 'info' THEN 1 WHEN 'notice' THEN 1 ELSE 0 END`; } function countWhere(config: WebProbeSentinelServiceConfig, db: Database, where: string): number { const row = db.query(`SELECT COUNT(*) AS count FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? AND ${where}`) .get(config.sentinelId, config.node, config.lane) as { count?: number } | null; return Number(row?.count ?? 0); } function sumColumn(config: WebProbeSentinelServiceConfig, db: Database, table: "runs", column: "finding_count"): number { const row = db.query(`SELECT COALESCE(SUM(${column}), 0) AS total FROM ${table} WHERE sentinel_id = ? AND node = ? AND lane = ?`) .get(config.sentinelId, config.node, config.lane) as { total?: number } | null; return Number(row?.total ?? 0); } function firstEnabledScenarioId(config: WebProbeSentinelServiceConfig): string | null { const scenario = config.scenarios.find((item) => boolAt(item, "enabled")); return scenario === undefined ? null : stringAt(scenario, "id"); } async function triggerQuickVerifyCronJob(config: WebProbeSentinelServiceConfig, input: Record): Promise> { const namespace = stringAt(config.runtime, "namespace"); const cronJobName = sentinelCadenceCronJobName(config); const scenarioId = stringOrNull(input.scenarioId) ?? stringAtNullable(config.cicd, "targetValidation.scenarioId") ?? firstEnabledScenarioId(config); const reason = stringOrNull(input.reason) ?? "manual-web-ui"; const source = stringOrNull(input.source) ?? "monitor-web"; if (scenarioId === null) { return { ok: false, status: "blocked", reason: "scenario-missing", namespace, cronJobName, valuesRedacted: true }; } const activeJobs = await k8sApiJson(config, "GET", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs?labelSelector=${encodeURIComponent(`unidesk.ai/web-probe-sentinel-id=${config.sentinelId}`)}`, null); if (activeJobs.ok === true) { const active = k8sJobItems(activeJobs.bodyJson).filter((job) => numberAtNullable(job, "status.active") > 0); if (active.length > 0) { return { ok: true, status: "already-active", mutation: false, reason: "sentinel-quick-verify-job-already-active", namespace, cronJobName, jobName: stringAtNullable(active[0], "metadata.name"), activeJobCount: active.length, scenarioId, source, valuesRedacted: true, }; } } const cronJob = await k8sApiJson(config, "GET", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/cronjobs/${encodeURIComponent(cronJobName)}`, null); if (cronJob.ok !== true) { return { ok: false, status: "blocked", reason: "cadence-cronjob-read-failed", namespace, cronJobName, scenarioId, source, k8s: compactK8sApiResult(cronJob), valuesRedacted: true, }; } const now = nowIso(); const jobName = manualQuickVerifyJobName(config, now); const job = jobFromCronJobTemplate(config, record(cronJob.bodyJson), jobName, now, reason, source); const created = await k8sApiJson(config, "POST", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs`, job); if (created.ok !== true) { return { ok: false, status: "blocked", reason: "manual-job-create-failed", namespace, cronJobName, jobName, scenarioId, source, k8s: compactK8sApiResult(created), valuesRedacted: true, }; } return { ok: true, status: "triggered", mutation: true, namespace, cronJobName, jobName, scenarioId, source, createdAt: now, k8s: compactK8sApiResult(created), valuesRedacted: true, }; } function jobFromCronJobTemplate( config: WebProbeSentinelServiceConfig, cronJob: Record, jobName: string, triggeredAt: string, reason: string, source: string, ): Record { const namespace = stringAt(config.runtime, "namespace"); const jobTemplate = record(valueAtPath(cronJob, "spec.jobTemplate")); const jobSpec = cloneJson(record(jobTemplate.spec)); if (Object.keys(jobSpec).length === 0) throw new Error("cadence CronJob jobTemplate.spec missing"); const templateMetadata = record(valueAtPath(jobSpec, "template.metadata")); const templateLabels = labelRecord(templateMetadata.labels); const cronJobLabels = labelRecord(record(cronJob.metadata).labels); return { apiVersion: "batch/v1", kind: "Job", metadata: { name: jobName, namespace, labels: { ...cronJobLabels, ...templateLabels, "unidesk.ai/manual-trigger": "true", "unidesk.ai/trigger-source": safeKubernetesLabelValue(source), }, annotations: { "unidesk.ai/trigger-source": source, "unidesk.ai/trigger-reason": reason, "unidesk.ai/triggered-at": triggeredAt, "unidesk.ai/cronjob-name": stringAt(cronJob, "metadata.name"), }, }, spec: jobSpec, }; } async function k8sApiJson(config: WebProbeSentinelServiceConfig, method: "GET" | "POST", path: string, body: Record | null): Promise> { const host = stringAtNullable(config.runtime, "kubernetesApi.endpoint.host") ?? process.env.KUBERNETES_SERVICE_HOST; const port = numberAtNullable(config.runtime, "kubernetesApi.endpoint.port") || Number(process.env.KUBERNETES_SERVICE_PORT_HTTPS ?? process.env.KUBERNETES_SERVICE_PORT ?? 443); if (typeof host !== "string" || host.length === 0) return { ok: false, status: "blocked", reason: "kubernetes-service-env-missing", valuesRedacted: true }; const tokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"; const caPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; if (!existsSync(tokenPath)) return { ok: false, status: "blocked", reason: "serviceaccount-token-missing", valuesRedacted: true }; const bodyText = body === null ? "" : JSON.stringify(body); const token = readFileSync(tokenPath, "utf8").trim(); const ca = existsSync(caPath) ? readFileSync(caPath) : undefined; return await new Promise((resolve) => { const req = httpsRequest({ host, port, path, method, ca, timeout: 12_000, headers: { accept: "application/json", authorization: `Bearer ${token}`, ...(body === null ? {} : { "content-type": "application/json", "content-length": Buffer.byteLength(bodyText) }), }, }, (res) => { const chunks: Buffer[] = []; res.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk))); res.on("end", () => { const text = Buffer.concat(chunks).toString("utf8"); const bodyJson = parseJsonRecord(text); const httpStatus = Number(res.statusCode ?? 0); resolve({ ok: httpStatus >= 200 && httpStatus < 300, httpStatus, bodyJson, bodyTextPreview: bodyJson === null ? text.slice(0, 1000) : "", valuesRedacted: true, }); }); }); req.on("timeout", () => req.destroy(new Error("kubernetes-api-timeout"))); req.on("error", (error) => { resolve({ ok: false, httpStatus: null, error: String(error?.message || error).slice(0, 500), valuesRedacted: true }); }); if (bodyText.length > 0) req.write(bodyText); req.end(); }); } function compactK8sApiResult(result: Record): Record { const body = record(result.bodyJson); return { ok: result.ok === true, httpStatus: result.httpStatus ?? null, status: body.status ?? null, reason: body.reason ?? result.reason ?? null, message: typeof body.message === "string" ? body.message.slice(0, 300) : result.error ?? null, valuesRedacted: true, }; } function k8sJobItems(value: unknown): Record[] { const items = record(value).items; return Array.isArray(items) ? items.map(record) : []; } function sentinelCadenceCronJobName(config: WebProbeSentinelServiceConfig): string { return safeKubernetesSegment(`${stringAt(config.runtime, "deploymentName")}-quick-verify`, 52); } function manualQuickVerifyJobName(config: WebProbeSentinelServiceConfig, now: string): string { const stamp = Date.parse(now).toString(36).replace(/[^a-z0-9]/giu, "").toLowerCase(); const prefix = `wps-${config.node.toLowerCase()}-qv`; return safeKubernetesSegment(`${prefix}-${stamp}-${randomUUID().slice(0, 5)}`, 63); } function safeKubernetesSegment(value: string, maxLength: number): string { const cleaned = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/^-+|-+$/gu, "").replace(/-{2,}/gu, "-"); const clipped = cleaned.slice(0, maxLength).replace(/-+$/gu, ""); return clipped.length > 0 ? clipped : "web-probe-sentinel"; } function safeKubernetesLabelValue(value: string): string { return safeKubernetesSegment(value, 63); } function labelRecord(value: unknown): Record { const result: Record = {}; for (const [key, raw] of Object.entries(record(value))) { if (typeof raw === "string" && raw.length > 0) result[key] = raw; } return result; } function cloneJson(value: Record): Record { return JSON.parse(JSON.stringify(value)) as Record; } function parseJsonRecord(text: string): Record | null { try { return record(JSON.parse(text) as unknown); } catch { return null; } } async function readJsonBody(request: Request): Promise> { if ((request.headers.get("content-length") ?? "0") === "0") return {}; const value = await request.json().catch(() => ({})) as unknown; return record(value); } function jsonResponse(value: unknown, status = 200): Response { return new Response(JSON.stringify(value, null, 2), { status, headers: { "content-type": "application/json; charset=utf-8" } }); } function emptyMaintenance(): MaintenanceState { return { active: false, reason: null, releaseId: null, startedAt: null, stoppedAt: null, quickVerifyPlannedAt: null, quickVerifyPlannedRunId: null }; } function recordRunResult(config: WebProbeSentinelServiceConfig, db: Database, input: Record): Record { const now = nowIso(); const runId = stringOrNull(input.runId) ?? `sentinel-run-${Date.now()}-${randomUUID().slice(0, 8)}`; const scenarioId = stringOrNull(input.scenarioId) ?? firstEnabledScenarioId(config); if (scenarioId === null) return { ok: false, error: "scenario-missing", valuesRedacted: true }; const status = stringOrNull(input.status) ?? "analyzed"; const observerId = stringOrNull(input.observerId); const stateDir = stringOrNull(input.stateDir); const reportJsonSha256 = stringOrNull(input.reportJsonSha256); const findings = arrayRecords(input.findings).slice(0, 50); const findingCount = numberOr(input.findingCount, findings.length); const artifactCount = numberOr(input.artifactCount, 0); const createdAt = stringOrNull(input.createdAt) ?? now; db.query(` INSERT INTO runs (id, sentinel_id, scenario_id, node, lane, status, observer_id, state_dir, report_json_sha256, finding_count, artifact_count, maintenance, created_at, updated_at, command_plan_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(id) DO UPDATE SET sentinel_id = excluded.sentinel_id, status = excluded.status, observer_id = excluded.observer_id, state_dir = excluded.state_dir, report_json_sha256 = excluded.report_json_sha256, finding_count = excluded.finding_count, artifact_count = excluded.artifact_count, updated_at = excluded.updated_at `).run(runId, config.sentinelId, scenarioId, config.node, config.lane, status, observerId, stateDir, reportJsonSha256, findingCount, artifactCount, thisMaintenanceFlag(input), createdAt, now, JSON.stringify({ source: "recorded-analyze-summary", valuesRedacted: true })); const supersededPlannedRuns = db.query("UPDATE runs SET status = 'superseded', updated_at = ? WHERE sentinel_id = ? AND node = ? AND lane = ? AND status = 'planned' AND scenario_id = ? AND created_at < ?") .run(now, config.sentinelId, config.node, config.lane, scenarioId, createdAt); db.query("DELETE FROM findings WHERE run_id = ? AND sentinel_id = ?").run(runId, config.sentinelId); for (const item of findings) { const findingId = stringOrNull(item.id) ?? stringOrNull(item.kind) ?? stringOrNull(item.code) ?? "finding"; const check = checkForFinding(config, { ...item, id: findingId }); const severity = stringOrNull(check.level) ?? normalizeCheckLevel(stringOrNull(item.severity) ?? stringOrNull(item.level)) ?? "unknown"; const summary = stringOrNull(item.summary) ?? stringOrNull(item.message) ?? findingId; db.query("INSERT INTO findings (run_id, sentinel_id, finding_id, severity, count, summary, report_json_sha256, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)") .run(runId, config.sentinelId, findingId.slice(0, 160), severity.slice(0, 40), numberOr(item.count, 1), summary.slice(0, 500), reportJsonSha256, now); } writeMetadata(db, `run.report.${runId}`, { runId, sentinelId: config.sentinelId, scenarioId, observerId, stateDir, reportJsonSha256, summary: record(input.summary), views: record(input.views), findings: findings.map(compactStoredFinding), publicOrigin: stringOrNull(input.publicOrigin), screenshot: record(input.screenshot), artifactCount, findingCount, valuesRedacted: true, }); return { ok: true, runId, scenarioId, status, reportJsonSha256, findingCount, artifactCount, supersededPlannedRuns: Number(supersededPlannedRuns.changes ?? 0), valuesRedacted: true }; } function reportRunView(config: WebProbeSentinelServiceConfig, db: Database, view: string, runId: string | null): Record { if (!stringArrayAt(config.reportViews, "views").includes(view)) { return { ok: false, error: "unsupported-report-view", view, valuesRedacted: true }; } const row = runId === null ? db.query("SELECT * FROM runs WHERE report_json_sha256 IS NOT NULL AND sentinel_id = ? AND node = ? AND lane = ? ORDER BY updated_at DESC LIMIT 1").get(config.sentinelId, config.node, config.lane) as Record | null : readRunRow(config, db, runId); if (row === null) return { ok: false, error: "report-run-missing", runId, view, valuesRedacted: true }; const selectedRunId = stringOrNull(row.id); if (selectedRunId === null) return { ok: false, error: "report-run-id-missing", view, valuesRedacted: true }; const stored = readMetadata(db, `run.report.${selectedRunId}`) ?? {}; const artifactSummary = readAnalysisArtifactSummaryForRun(config, row, 50); const findings = visibleFindingsForRun(config, db, row, 50, artifactSummary); const views = record(stored.views); const storedView = record(views[view]); const renderedText = view === "findings" ? renderStoredFindings(row, findings, artifactSummary) : view === "summary" ? renderStoredSummary(config, row, stored, findings, artifactSummary) : typeof storedView.renderedText === "string" ? storedView.renderedText : null; if (renderedText === null) { return { ok: false, error: "report-view-not-indexed", runId: selectedRunId, view, availableViews: Object.keys(views), valuesRedacted: true }; } return { ok: true, view, run: dashboardRunSummary(config, db, row), summary: record(stored.summary), findings, artifactSummary, renderedText, valuesRedacted: true, }; } function formatStoredFindingLine(item: Record): string { const check = record(item.check); const code = stringOrNull(item.checkCode) ?? stringOrNull(check.code) ?? stringOrNull(item.finding_id) ?? stringOrNull(item.findingId) ?? stringOrNull(item.code) ?? "-"; const title = stringOrNull(item.checkTitleZh) ?? stringOrNull(check.titleZh) ?? stringOrNull(item.summary) ?? ""; const summary = stringOrNull(item.checkSummaryZh) ?? stringOrNull(check.summaryZh) ?? stringOrNull(item.summary) ?? ""; const rootCause = stringOrNull(item.rootCause); const status = stringOrNull(item.rootCauseStatus); const nextAction = stringOrNull(item.nextAction); const evidence = stringOrNull(item.evidenceSummary); const timing = formatStoredFindingTiming(item); return [ `${item.severity ?? "-"} ${code} ${title} count=${item.count ?? "-"} ${summary}`, timing, rootCause === null ? null : `rootCause=${rootCause}${status === null ? "" : ` status=${status}`}`, evidence === null ? null : `evidence=${evidence}`, nextAction === null ? null : `next=${nextAction}`, ].filter((part) => part !== null).join(" | "); } function formatStoredFindingTiming(item: Record): string | null { const source = stringOrNull(item.timingSourceOfTruth); const status = stringOrNull(item.timingStatus); if (source === null && status === null) return null; const pieces = [ status === null ? null : `status=${status}`, source === null ? null : `source=${source}`, item.timingAlert === true ? "alert=true" : null, ].filter((part): part is string => part !== null); return `timing=${pieces.join(" ")}`; } function renderStoredSummary( config: WebProbeSentinelServiceConfig, row: Record, stored: Record, findings: readonly Record[], artifactSummary: Record, ): string { const summary = record(stored.summary); const timing = dashboardRunTiming(config, row, stored); const reportSha = stringOrNull(row.report_json_sha256) ?? stringOrNull(artifactSummary.reportJsonSha256); const findingCount = visibleFindingTypeCount(row, findings, artifactSummary); return [ "Web Probe Sentinel Report", "=======================================================", `run=${stringOrNull(row.id) ?? "-"} scenario=${stringOrNull(row.scenario_id) ?? "-"} status=${stringOrNull(row.status) ?? "-"}`, `observer=${stringOrNull(row.observer_id) ?? "-"} stateDir=${stringOrNull(row.state_dir) ?? "-"}`, `report=${reportSha ?? "-"} artifacts=${String(row.artifact_count ?? 0)} findings=${String(findingCount)}`, `timing=${formatRunTimingSummary(timing)}`, `publicOrigin=${stringOrNull(stored.publicOrigin) ?? "-"}`, `analysisWindow=${formatStoredAnalysisWindow(summary.analysisWindow)}`, "", "Findings", findings.length === 0 ? "-" : findings.slice(0, 12).map(formatStoredFindingLine).join("\n"), ].join("\n"); } function renderStoredFindings(row: Record, findings: readonly Record[], artifactSummary: Record): string { const reportSha = stringOrNull(row.report_json_sha256) ?? stringOrNull(artifactSummary.reportJsonSha256); return [ "Web Probe Sentinel Findings", "=======================================================", `run=${stringOrNull(row.id) ?? "-"} report=${reportSha ?? "-"} findings=${String(visibleFindingTypeCount(row, findings, artifactSummary))}`, findings.length === 0 ? "-" : findings.map(formatStoredFindingLine).join("\n"), ].join("\n"); } function formatStoredAnalysisWindow(value: unknown): string { const window = record(value); const fields = [ ["start", stringOrNull(window.start)], ["end", stringOrNull(window.end)], ["samples", numberOr(window.sampleCount, numberOr(window.samples, -1)) >= 0 ? String(numberOr(window.sampleCount, numberOr(window.samples, 0))) : null], ["durationMs", numberOr(window.durationMs, -1) >= 0 ? String(numberOr(window.durationMs, 0)) : null], ].filter((entry): entry is [string, string] => entry[1] !== null); return fields.length === 0 ? "-" : fields.map(([key, item]) => `${key}=${item}`).join(" "); } function formatRunTimingSummary(value: Record): string { const fields = [ ["durationMinutes", numberOrNull(value.durationMinutes)], ["durationSource", stringOrNull(value.durationSource)], ["scenarioCadenceMinutes", numberOrNull(value.scenarioCadenceMinutes)], ["scenarioMaxRunMinutes", numberOrNull(value.scenarioMaxRunMinutes)], ["schedulerIntervalMinutes", numberOrNull(value.schedulerIntervalMinutes)], ].filter((entry): entry is [string, string | number] => entry[1] !== null); return fields.length === 0 ? "-" : fields.map(([key, item]) => `${key}=${item}`).join(" "); } function thisMaintenanceFlag(input: Record): number { return input.maintenance === true ? 1 : 0; } function numberOrNull(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) ? value : null; } function numberOr(value: unknown, fallback: number): number { return typeof value === "number" && Number.isFinite(value) ? value : fallback; } function arrayRecords(value: unknown): Record[] { return Array.isArray(value) ? value.map(record) : []; } function checkPath(value: unknown, path: string): unknown { const result = valueAtPath(value, path); if (result === undefined) throw new Error(`required field missing: ${path}`); return result; } function stringAt(value: unknown, path: string): string { const result = checkPath(value, path); if (typeof result !== "string" || result.length === 0) throw new Error(`${path} must be a non-empty string`); return result; } function stringAtNullable(value: unknown, path: string): string | null { const result = valueAtPath(value, path); return typeof result === "string" && result.length > 0 ? result : null; } function numberAt(value: unknown, path: string): number { const result = checkPath(value, path); if (typeof result !== "number" || !Number.isFinite(result)) throw new Error(`${path} must be a number`); return result; } function numberAtNullable(value: unknown, path: string): number { const result = valueAtPath(value, path); return typeof result === "number" && Number.isFinite(result) ? result : 0; } function boolAt(value: unknown, path: string): boolean { const result = checkPath(value, path); if (typeof result !== "boolean") throw new Error(`${path} must be a boolean`); return result; } function arrayAt(value: unknown, path: string): Record[] { const result = checkPath(value, path); if (!Array.isArray(result)) throw new Error(`${path} must be an array`); return result.filter(record); } function stringArrayAt(value: unknown, path: string): string[] { const result = checkPath(value, path); if (!Array.isArray(result)) throw new Error(`${path} must be an array`); return result.filter((item): item is string => typeof item === "string" && item.length > 0); } function stringField(value: Record, key: string): string { const found = value[key]; if (typeof found !== "string" || found.length === 0) throw new Error(`${key} must be a non-empty string`); return found; } function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } function numberParam(url: URL, key: string, defaultValue: number): number { const raw = url.searchParams.get(key); if (raw === null) return defaultValue; const parsed = Number(raw); return Number.isInteger(parsed) && parsed > 0 && parsed <= 200 ? parsed : defaultValue; } function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function recordTarget(value: unknown): Record { const result = record(value); if (Object.keys(result).length === 0) throw new Error("configRef target must be an object"); return result; } function arrayTarget(value: unknown): Record[] { if (!Array.isArray(value)) throw new Error("configRef target must be an array"); return value.map(recordTarget); } function scenarioArrayTarget(value: unknown): Record[] { if (Array.isArray(value)) return value.map(recordTarget); return [recordTarget(value)]; } function valueAtPath(value: unknown, path: string): unknown { let current: unknown = value; for (const segment of path.split(".")) { const match = /^(?:([A-Za-z0-9_-]+))?(?:\[(\d+)\])?$/u.exec(segment); if (match === null) return undefined; if (match[1] !== undefined) { const obj = record(current); current = obj[match[1]]; } if (match[2] !== undefined) { if (!Array.isArray(current)) return undefined; current = current[Number(match[2])]; } } return current; } function sha256Json(value: unknown): string { return `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`; } function nowIso(): string { return new Date().toISOString(); } function metricLabel(value: string): string { return value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"').replace(/\n/gu, "\\n"); }