Codex
2796 lines
127 KiB
TypeScript
2796 lines
127 KiB
TypeScript
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p7-web-probe-sentinel-dashboard.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-desktop-view-density.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries.
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
|
|
// Responsibility: Persistent HTTP wrapper service for web-probe observe scheduling, index, health, metrics, maintenance, and dashboard.
|
|
import { Buffer } from "node:buffer";
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
import { request as httpsRequest } from "node:https";
|
|
import { join } from "node:path";
|
|
import { Database } from "bun:sqlite";
|
|
import { renderWebProbeSentinelDashboardHtml, webProbeSentinelDashboardAssetResponse } from "./hwlab-node-web-sentinel-dashboard-assets";
|
|
import { webProbeSentinelConfigPlan, type WebProbeSentinelConfigPlan } from "./hwlab-node-web-sentinel-config";
|
|
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
|
import { effectiveWebProbeSentinelPublicExposure, resolveWebProbeSentinel, readConfigRefTarget as readSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-resolver";
|
|
import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
|
|
|
|
const DASHBOARD_CONTRACT_VERSION = "draft-2026-06-27-p11-monitor-web-observability-dashboard";
|
|
const DASHBOARD_MAX_TEXT_BYTES = 16_000;
|
|
|
|
export interface WebProbeSentinelServiceConfig {
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string;
|
|
readonly plan: WebProbeSentinelConfigPlan;
|
|
readonly runtime: Record<string, unknown>;
|
|
readonly scenarios: readonly Record<string, unknown>[];
|
|
readonly reportViews: Record<string, unknown>;
|
|
readonly publicExposure: Record<string, unknown>;
|
|
readonly cicd: Record<string, unknown>;
|
|
readonly stateRoot: string;
|
|
readonly sqlitePath: string;
|
|
readonly listenHost: string;
|
|
readonly servicePort: number;
|
|
readonly schedulerIntervalMs: number;
|
|
readonly schedulerHeartbeatStaleSeconds: number;
|
|
readonly maxConcurrentRuns: number;
|
|
}
|
|
|
|
export interface WebProbeSentinelServiceOptions {
|
|
readonly spec: HwlabRuntimeLaneSpec;
|
|
readonly sentinelId?: string | null;
|
|
readonly stateRootOverride?: string;
|
|
readonly portOverride?: number;
|
|
readonly hostOverride?: string;
|
|
readonly schedulerEnabled?: boolean;
|
|
}
|
|
|
|
interface MaintenanceState {
|
|
readonly active: boolean;
|
|
readonly reason: string | null;
|
|
readonly releaseId: string | null;
|
|
readonly startedAt: string | null;
|
|
readonly stoppedAt: string | null;
|
|
readonly quickVerifyPlannedAt: string | null;
|
|
readonly quickVerifyPlannedRunId: string | null;
|
|
}
|
|
|
|
interface CommandPlanStep {
|
|
readonly phase: string;
|
|
readonly argv: readonly string[];
|
|
readonly stdinSource: "none" | "prompt-source";
|
|
}
|
|
|
|
export interface WebProbeSentinelService {
|
|
readonly config: WebProbeSentinelServiceConfig;
|
|
readonly db: Database;
|
|
readonly schedulerEnabled: boolean;
|
|
startScheduler(): void;
|
|
stopScheduler(): void;
|
|
close(): void;
|
|
health(): Record<string, unknown>;
|
|
status(): Record<string, unknown>;
|
|
runs(limit?: number): readonly Record<string, unknown>[];
|
|
overview(): Record<string, unknown>;
|
|
dashboardRuns(url: URL): Record<string, unknown>;
|
|
runDetail(runId: string): Record<string, unknown>;
|
|
findings(url: URL): Record<string, unknown>;
|
|
runViews(runId: string, view: string | null, url: URL): Record<string, unknown>;
|
|
maintenance(): MaintenanceState;
|
|
setMaintenance(active: boolean, input: Record<string, unknown>): MaintenanceState;
|
|
planScenarioRun(scenarioId: string, reason: string): Record<string, unknown>;
|
|
triggerQuickVerify(input: Record<string, unknown>): Promise<Record<string, unknown>>;
|
|
recordRun(input: Record<string, unknown>): Record<string, unknown>;
|
|
report(view: string, runId: string | null): Record<string, unknown>;
|
|
metrics(): string;
|
|
dashboardHtml(): string;
|
|
fetch(request: Request): Promise<Response>;
|
|
}
|
|
|
|
export function loadWebProbeSentinelServiceConfig(spec: HwlabRuntimeLaneSpec, options: Omit<WebProbeSentinelServiceOptions, "spec"> = {}): WebProbeSentinelServiceConfig {
|
|
const sentinel = resolveWebProbeSentinel(spec, options.sentinelId ?? null);
|
|
const plan = webProbeSentinelConfigPlan(spec, "status", sentinel.id);
|
|
const runtime = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.runtime, spec));
|
|
const scenarios = scenarioArrayTarget(readSentinelConfigRefTarget(sentinel.configRefs.scenarios, spec));
|
|
const reportViews = resolveReportViewsWithCheckCatalog(recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.reportViews, spec)), spec);
|
|
const rawPublicExposure = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.publicExposure, spec));
|
|
const publicExposure = effectiveWebProbeSentinelPublicExposure(spec, sentinel.id, rawPublicExposure);
|
|
const cicd = recordTarget(readSentinelConfigRefTarget(sentinel.configRefs.cicd, spec));
|
|
const stateRoot = options.stateRootOverride ?? stringAt(runtime, "stateRoot");
|
|
const yamlSqlitePath = stringAt(runtime, "sqlite.path");
|
|
return {
|
|
node: spec.nodeId,
|
|
lane: spec.lane,
|
|
sentinelId: sentinel.id,
|
|
plan,
|
|
runtime,
|
|
scenarios,
|
|
reportViews,
|
|
publicExposure,
|
|
cicd,
|
|
stateRoot,
|
|
sqlitePath: options.stateRootOverride === undefined ? yamlSqlitePath : join(stateRoot, "index.sqlite"),
|
|
listenHost: options.hostOverride ?? stringAt(runtime, "listenHost"),
|
|
servicePort: options.portOverride ?? numberAt(runtime, "servicePort"),
|
|
schedulerIntervalMs: numberAt(runtime, "scheduler.intervalMs"),
|
|
schedulerHeartbeatStaleSeconds: numberAt(runtime, "scheduler.heartbeatStaleSeconds"),
|
|
maxConcurrentRuns: numberAt(runtime, "scheduler.maxConcurrentRuns"),
|
|
};
|
|
}
|
|
|
|
export function createWebProbeSentinelService(options: WebProbeSentinelServiceOptions): WebProbeSentinelService {
|
|
const config = loadWebProbeSentinelServiceConfig(options.spec, options);
|
|
mkdirSync(config.stateRoot, { recursive: true });
|
|
const db = new Database(config.sqlitePath);
|
|
initializeIndex(db, config);
|
|
const restored = markInterruptedRuns(config, db, nowIso());
|
|
const schedulerEnabled = options.schedulerEnabled ?? true;
|
|
let schedulerTimer: ReturnType<typeof setInterval> | null = null;
|
|
let schedulerHeartbeatAt = nowIso();
|
|
let schedulerLastError: string | null = null;
|
|
writeMetadata(db, "service.boot", { at: schedulerHeartbeatAt, restoredInterruptedRuns: restored, valuesRedacted: true });
|
|
writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "boot" });
|
|
emitSchedulerHeartbeatSpan(config, "boot", schedulerHeartbeatAt, true);
|
|
|
|
const service: WebProbeSentinelService = {
|
|
config,
|
|
db,
|
|
schedulerEnabled,
|
|
startScheduler() {
|
|
if (!schedulerEnabled || schedulerTimer !== null) return;
|
|
schedulerHeartbeatAt = nowIso();
|
|
writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "started" });
|
|
emitSchedulerHeartbeatSpan(config, "started", schedulerHeartbeatAt, true);
|
|
schedulerTimer = setInterval(() => {
|
|
try {
|
|
schedulerHeartbeatAt = nowIso();
|
|
writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "tick" });
|
|
const summary = schedulerSummary(config, db);
|
|
writeMetadata(db, "scheduler.summary", summary);
|
|
emitSchedulerHeartbeatSpan(config, "tick", schedulerHeartbeatAt, true);
|
|
emitCadenceExpectedSpan(config, summary);
|
|
if (summary.rootCause === "planned-run-not-consumed-by-host-cadence") emitSchedulerGapSpan(config, summary);
|
|
schedulerLastError = null;
|
|
} catch (error) {
|
|
schedulerLastError = error instanceof Error ? error.message : String(error);
|
|
writeMetadata(db, "scheduler.error", { at: nowIso(), message: schedulerLastError });
|
|
emitSchedulerHeartbeatSpan(config, "tick-error", nowIso(), false, schedulerLastError);
|
|
}
|
|
}, config.schedulerIntervalMs);
|
|
},
|
|
stopScheduler() {
|
|
if (schedulerTimer !== null) clearInterval(schedulerTimer);
|
|
schedulerTimer = null;
|
|
writeMetadata(db, "scheduler.heartbeat", { at: nowIso(), loop: "stopped" });
|
|
},
|
|
close() {
|
|
this.stopScheduler();
|
|
db.close();
|
|
},
|
|
health() {
|
|
return serviceHealth(config, db, {
|
|
schedulerEnabled,
|
|
schedulerHeartbeatAt,
|
|
schedulerTimerActive: schedulerTimer !== null,
|
|
schedulerLastError,
|
|
});
|
|
},
|
|
status() {
|
|
return {
|
|
ok: true,
|
|
node: config.node,
|
|
lane: config.lane,
|
|
sentinelId: config.sentinelId,
|
|
status: "observed",
|
|
configReady: config.plan.ok,
|
|
scheduler: schedulerSummary(config, db),
|
|
maintenance: this.maintenance(),
|
|
runs: runCounts(config, db),
|
|
latestRuns: this.runs(8),
|
|
valuesRedacted: true,
|
|
};
|
|
},
|
|
runs(limit = 20) {
|
|
return db.query("SELECT id, sentinel_id, scenario_id, status, node, lane, observer_id, state_dir, report_json_sha256, finding_count, artifact_count, maintenance, created_at, updated_at, interrupted_at FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? ORDER BY created_at DESC LIMIT ?")
|
|
.all(config.sentinelId, config.node, config.lane, limit) as Record<string, unknown>[];
|
|
},
|
|
overview() {
|
|
return dashboardOverview(config, db, this.health(), this.maintenance());
|
|
},
|
|
dashboardRuns(url: URL) {
|
|
return dashboardRunList(config, db, url);
|
|
},
|
|
runDetail(runId: string) {
|
|
return dashboardRunDetail(config, db, runId);
|
|
},
|
|
findings(url: URL) {
|
|
return dashboardFindings(config, db, url);
|
|
},
|
|
runViews(runId: string, view: string | null, url: URL) {
|
|
return dashboardRunViews(config, db, runId, view, url);
|
|
},
|
|
maintenance() {
|
|
return readMetadata(db, "maintenance") as MaintenanceState | null ?? emptyMaintenance();
|
|
},
|
|
setMaintenance(active: boolean, input: Record<string, unknown>) {
|
|
const current = this.maintenance();
|
|
const next: MaintenanceState = active
|
|
? {
|
|
active: true,
|
|
reason: stringOrNull(input.reason),
|
|
releaseId: stringOrNull(input.releaseId),
|
|
startedAt: nowIso(),
|
|
stoppedAt: current.stoppedAt,
|
|
quickVerifyPlannedAt: current.quickVerifyPlannedAt,
|
|
quickVerifyPlannedRunId: current.quickVerifyPlannedRunId,
|
|
}
|
|
: {
|
|
active: false,
|
|
reason: current.reason,
|
|
releaseId: current.releaseId,
|
|
startedAt: current.startedAt,
|
|
stoppedAt: nowIso(),
|
|
quickVerifyPlannedAt: nowIso(),
|
|
quickVerifyPlannedRunId: null,
|
|
};
|
|
writeMetadata(db, "maintenance", next);
|
|
if (!active) {
|
|
const scenarioId = firstEnabledScenarioId(config);
|
|
if (scenarioId !== null) {
|
|
const planned = this.planScenarioRun(scenarioId, "maintenance-stop-quick-verify");
|
|
const withPlan = { ...next, quickVerifyPlannedRunId: stringOrNull(planned.runId) };
|
|
writeMetadata(db, "maintenance", withPlan);
|
|
return withPlan;
|
|
}
|
|
}
|
|
return next;
|
|
},
|
|
planScenarioRun(scenarioId: string, reason: string) {
|
|
const scenario = config.scenarios.find((item) => stringAt(item, "id") === scenarioId);
|
|
if (scenario === undefined) throw new Error(`scenario not found: ${scenarioId}`);
|
|
const runId = `sentinel-run-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
const commandPlan = buildObserveCommandPlan(config, scenario);
|
|
const createdAt = nowIso();
|
|
db.query("INSERT INTO runs (id, sentinel_id, scenario_id, node, lane, status, maintenance, created_at, updated_at, command_plan_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")
|
|
.run(runId, config.sentinelId, scenarioId, config.node, config.lane, "planned", this.maintenance().active ? 1 : 0, createdAt, createdAt, JSON.stringify({ reason, commandPlan, valuesRedacted: true }));
|
|
return { ok: true, runId, scenarioId, status: "planned", commandPlanSha256: sha256Json(commandPlan), valuesRedacted: true };
|
|
},
|
|
async triggerQuickVerify(input: Record<string, unknown>) {
|
|
const result = await triggerQuickVerifyCronJob(config, input);
|
|
writeMetadata(db, "manual-trigger.latest", result);
|
|
emitManualTriggerSpan(config, result);
|
|
return result;
|
|
},
|
|
recordRun(input: Record<string, unknown>) {
|
|
const result = recordRunResult(config, db, input);
|
|
emitRecordRunSpan(config, input, result);
|
|
return result;
|
|
},
|
|
report(view: string, runId: string | null) {
|
|
return reportRunView(config, db, view, runId);
|
|
},
|
|
metrics() {
|
|
return renderMetrics(config, db, this.health(), this.maintenance());
|
|
},
|
|
dashboardHtml() {
|
|
return renderWebProbeSentinelDashboardHtml(config);
|
|
},
|
|
async fetch(request: Request) {
|
|
return sentinelFetch(service, request);
|
|
},
|
|
};
|
|
service.startScheduler();
|
|
return service;
|
|
}
|
|
|
|
function resolveReportViewsWithCheckCatalog(reportViews: Record<string, unknown>, spec: HwlabRuntimeLaneSpec): Record<string, unknown> {
|
|
const ref = stringOrNull(reportViews.checkCatalogRef);
|
|
if (ref === null) return reportViews;
|
|
return {
|
|
...reportViews,
|
|
checkCatalog: recordTarget(readSentinelConfigRefTarget(ref, spec)),
|
|
};
|
|
}
|
|
|
|
export function startWebProbeSentinelHttpService(service: WebProbeSentinelService): { readonly url: string; readonly stop: () => void } {
|
|
const server = Bun.serve({
|
|
hostname: service.config.listenHost,
|
|
port: service.config.servicePort,
|
|
fetch: (request) => service.fetch(request),
|
|
});
|
|
return {
|
|
url: `http://${service.config.listenHost}:${server.port}`,
|
|
stop: () => server.stop(true),
|
|
};
|
|
}
|
|
|
|
async function sentinelFetch(service: WebProbeSentinelService, request: Request): Promise<Response> {
|
|
const url = new URL(request.url);
|
|
const routeMismatch = sentinelRouteMismatch(service.config, url.pathname);
|
|
if (routeMismatch !== null) return jsonResponse(routeMismatch, 409);
|
|
const pathname = normalizedSentinelRequestPath(service, url.pathname);
|
|
if (request.method === "GET" && pathname === "/api/health") {
|
|
const health = service.health();
|
|
return jsonResponse(health, health.serving === false ? 503 : 200);
|
|
}
|
|
if (request.method === "GET" && pathname === "/api/status") return jsonResponse(service.status());
|
|
if (request.method === "GET" && pathname === "/api/overview") return jsonResponse(service.overview());
|
|
if (request.method === "GET" && pathname === "/api/runs") return jsonResponse(service.dashboardRuns(url));
|
|
if (request.method === "GET" && pathname === "/api/findings") return jsonResponse(service.findings(url));
|
|
const runViewsMatch = /^\/api\/runs\/([^/]+)\/views$/u.exec(pathname);
|
|
if (request.method === "GET" && runViewsMatch !== null) {
|
|
const view = url.searchParams.get("view");
|
|
const result = service.runViews(decodeURIComponent(runViewsMatch[1]), view, url);
|
|
return jsonResponse(result, result.ok === false ? 404 : 200);
|
|
}
|
|
const runDetailMatch = /^\/api\/runs\/([^/]+)$/u.exec(pathname);
|
|
if (request.method === "GET" && runDetailMatch !== null) {
|
|
const result = service.runDetail(decodeURIComponent(runDetailMatch[1]));
|
|
return jsonResponse(result, result.ok === false ? 404 : 200);
|
|
}
|
|
if (request.method === "GET" && pathname === "/api/maintenance") return jsonResponse({ ok: true, maintenance: service.maintenance(), valuesRedacted: true });
|
|
if (request.method === "POST" && (pathname === "/api/maintenance/start" || pathname === "/api/maintenance/stop")) {
|
|
const body = await readJsonBody(request);
|
|
const active = pathname.endsWith("/start");
|
|
return jsonResponse({ ok: true, maintenance: service.setMaintenance(active, body), valuesRedacted: true });
|
|
}
|
|
if (request.method === "POST" && pathname === "/api/runs/plan") {
|
|
const body = await readJsonBody(request);
|
|
return jsonResponse(service.planScenarioRun(stringField(body, "scenarioId"), stringOrNull(body.reason) ?? "manual"));
|
|
}
|
|
if (request.method === "POST" && pathname === "/api/quick-verify/trigger") {
|
|
const body = await readJsonBody(request);
|
|
const result = await service.triggerQuickVerify(body);
|
|
return jsonResponse(result, result.ok === false ? 409 : 200);
|
|
}
|
|
if (request.method === "POST" && pathname === "/api/runs/record") {
|
|
const body = await readJsonBody(request);
|
|
return jsonResponse(service.recordRun(body));
|
|
}
|
|
if (request.method === "GET" && pathname === "/api/report") {
|
|
const view = url.searchParams.get("view") ?? stringOrNull(service.config.reportViews.defaultView) ?? "summary";
|
|
const runId = url.searchParams.get("run") ?? url.searchParams.get("runId");
|
|
const report = service.report(view, runId);
|
|
return jsonResponse(report, report.ok === false ? 404 : 200);
|
|
}
|
|
if (request.method === "GET" && pathname === "/metrics") {
|
|
return new Response(service.metrics(), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } });
|
|
}
|
|
if (request.method === "GET" && (pathname.includes("/dashboard/assets/") || pathname.includes("/monitor-web/assets/"))) {
|
|
const asset = webProbeSentinelDashboardAssetResponse(normalizedDashboardAssetPath(pathname));
|
|
if (asset !== null) return asset;
|
|
}
|
|
if (request.method === "GET" && (pathname === "/" || pathname === "/dashboard" || pathname === "/monitor-web")) {
|
|
return new Response(service.dashboardHtml(), { headers: { "content-type": "text/html; charset=utf-8" } });
|
|
}
|
|
return jsonResponse({ ok: false, error: "not-found", path: url.pathname, valuesRedacted: true }, 404);
|
|
}
|
|
|
|
function normalizedSentinelRequestPath(service: WebProbeSentinelService, pathname: string): string {
|
|
const publicPath = publicExposurePath(service.config.publicExposure);
|
|
if (publicPath.length > 0 && (pathname === publicPath || pathname === `${publicPath}/`)) return "/";
|
|
if (publicPath.length > 0 && pathname.startsWith(`${publicPath}/`)) return pathname.slice(publicPath.length) || "/";
|
|
if (/^\/sentinels\/[^/]+\/?$/u.test(pathname)) return "/";
|
|
for (const marker of ["/api/", "/monitor-web/assets/", "/dashboard/assets/"]) {
|
|
const index = pathname.indexOf(marker);
|
|
if (index > 0) return pathname.slice(index);
|
|
}
|
|
for (const marker of ["/monitor-web", "/dashboard", "/metrics"]) {
|
|
if (pathname.endsWith(marker)) return marker;
|
|
}
|
|
return pathname;
|
|
}
|
|
|
|
function sentinelRouteMismatch(config: WebProbeSentinelServiceConfig, pathname: string): Record<string, unknown> | null {
|
|
const match = /^\/sentinels\/([^/]+)/u.exec(pathname);
|
|
if (match === null) return null;
|
|
const routeSegment = decodeURIComponent(match[1]);
|
|
const nodePrefix = `${config.node.toLowerCase()}-`;
|
|
const routeSentinelId = routeSegment.startsWith(nodePrefix) ? routeSegment.slice(nodePrefix.length) : routeSegment;
|
|
if (routeSegment === config.sentinelId || routeSentinelId === config.sentinelId) return null;
|
|
return {
|
|
ok: false,
|
|
error: "sentinel-route-mismatch",
|
|
code: "sentinel-route-mismatch",
|
|
blocker: true,
|
|
routeSegment,
|
|
routeSentinelId,
|
|
serviceSentinelId: config.sentinelId,
|
|
node: config.node,
|
|
lane: config.lane,
|
|
message: "route sentinelId does not match the selected sentinel runner; refusing cross-sentinel fallback",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function publicExposurePath(publicExposure: Record<string, unknown>): string {
|
|
const publicBaseUrl = stringOrNull(publicExposure.publicBaseUrl);
|
|
if (publicBaseUrl === null) return "";
|
|
try {
|
|
const pathname = new URL(publicBaseUrl).pathname.replace(/\/+$/u, "");
|
|
return pathname === "/" ? "" : pathname;
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
function normalizedDashboardAssetPath(pathname: string): string {
|
|
for (const marker of ["/monitor-web/assets/", "/dashboard/assets/"]) {
|
|
const index = pathname.indexOf(marker);
|
|
if (index >= 0) return pathname.slice(index);
|
|
}
|
|
return pathname;
|
|
}
|
|
|
|
function initializeIndex(db: Database, config: WebProbeSentinelServiceConfig): void {
|
|
db.exec(`
|
|
PRAGMA journal_mode = WAL;
|
|
CREATE TABLE IF NOT EXISTS metadata (
|
|
key TEXT PRIMARY KEY,
|
|
value_json TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS runs (
|
|
id TEXT PRIMARY KEY,
|
|
sentinel_id TEXT NOT NULL DEFAULT '',
|
|
scenario_id TEXT NOT NULL,
|
|
node TEXT NOT NULL,
|
|
lane TEXT NOT NULL,
|
|
status TEXT NOT NULL,
|
|
observer_id TEXT,
|
|
state_dir TEXT,
|
|
report_json_sha256 TEXT,
|
|
finding_count INTEGER NOT NULL DEFAULT 0,
|
|
artifact_count INTEGER NOT NULL DEFAULT 0,
|
|
maintenance INTEGER NOT NULL DEFAULT 0,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT NOT NULL,
|
|
interrupted_at TEXT,
|
|
command_plan_json TEXT
|
|
);
|
|
CREATE TABLE IF NOT EXISTS findings (
|
|
run_id TEXT NOT NULL,
|
|
sentinel_id TEXT NOT NULL DEFAULT '',
|
|
finding_id TEXT NOT NULL,
|
|
severity TEXT NOT NULL,
|
|
count INTEGER NOT NULL DEFAULT 1,
|
|
summary TEXT NOT NULL,
|
|
report_json_sha256 TEXT,
|
|
created_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_runs_sentinel_updated ON runs(sentinel_id, node, lane, updated_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_runs_sentinel_scenario ON runs(sentinel_id, scenario_id, updated_at DESC);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_sentinel_run ON findings(sentinel_id, run_id);
|
|
CREATE INDEX IF NOT EXISTS idx_findings_sentinel_code ON findings(sentinel_id, finding_id, created_at DESC);
|
|
`);
|
|
ensureColumn(db, "runs", "sentinel_id", "TEXT");
|
|
ensureColumn(db, "findings", "sentinel_id", "TEXT");
|
|
db.query("UPDATE runs SET sentinel_id = ? WHERE sentinel_id IS NULL OR sentinel_id = ''").run(config.sentinelId);
|
|
db.query("UPDATE findings SET sentinel_id = ? WHERE sentinel_id IS NULL OR sentinel_id = ''").run(config.sentinelId);
|
|
}
|
|
|
|
function ensureColumn(db: Database, table: "runs" | "findings", column: string, definition: string): void {
|
|
const rows = db.query(`PRAGMA table_info(${table})`).all() as Record<string, unknown>[];
|
|
if (rows.some((row) => stringOrNull(row.name) === column)) return;
|
|
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
}
|
|
|
|
function markInterruptedRuns(config: WebProbeSentinelServiceConfig, db: Database, at: string): number {
|
|
const result = db.query("UPDATE runs SET status = 'interrupted', interrupted_at = ?, updated_at = ? WHERE sentinel_id = ? AND node = ? AND lane = ? AND status IN ('queued', 'running', 'analyzing')").run(at, at, config.sentinelId, config.node, config.lane);
|
|
return Number(result.changes ?? 0);
|
|
}
|
|
|
|
function serviceHealth(config: WebProbeSentinelServiceConfig, db: Database, scheduler: Record<string, unknown>): Record<string, unknown> {
|
|
const checks: Record<string, Record<string, unknown>> = {};
|
|
checks.config = { ok: config.plan.ok, status: config.plan.status, conflicts: config.plan.conflicts.length };
|
|
checks.pvc = checkWritable(config.stateRoot);
|
|
checks.sqlite = checkSqlite(db);
|
|
const heartbeatAt = stringOrNull(scheduler.schedulerHeartbeatAt) ?? stringOrNull(readMetadata(db, "scheduler.heartbeat")?.at);
|
|
const heartbeatAgeSeconds = heartbeatAt === null ? null : Math.max(0, Math.round((Date.now() - Date.parse(heartbeatAt)) / 1000));
|
|
const planned = plannedRunBacklog(config, db);
|
|
const schedulerServing = scheduler.schedulerLastError === null && heartbeatAgeSeconds !== null && heartbeatAgeSeconds <= config.schedulerHeartbeatStaleSeconds;
|
|
checks.scheduler = {
|
|
ok: schedulerServing && !planned.stale,
|
|
enabled: scheduler.schedulerEnabled === true,
|
|
active: scheduler.schedulerTimerActive === true,
|
|
heartbeatAt,
|
|
heartbeatAgeSeconds,
|
|
staleAfterSeconds: config.schedulerHeartbeatStaleSeconds,
|
|
lastError: scheduler.schedulerLastError,
|
|
plannedRuns: planned.count,
|
|
oldestPlannedRunId: planned.oldestRunId,
|
|
oldestPlannedRunScenarioId: planned.oldestScenarioId,
|
|
oldestPlannedRunCreatedAt: planned.oldestCreatedAt,
|
|
oldestPlannedRunAgeSeconds: planned.oldestAgeSeconds,
|
|
plannedRunStaleAfterSeconds: planned.staleAfterSeconds,
|
|
plannedRunStale: planned.stale,
|
|
rootCause: planned.stale ? "planned-run-not-consumed-by-host-cadence" : null,
|
|
};
|
|
checks.analyzer = {
|
|
ok: true,
|
|
source: "existing observe analyze CLI command",
|
|
command: `bun scripts/cli.ts web-probe observe analyze --node ${config.node} --lane ${config.lane} --state-dir <stateDir>`,
|
|
};
|
|
const ok = Object.values(checks).every((check) => check.ok === true);
|
|
const serving = checks.config.ok === true && checks.pvc.ok === true && checks.sqlite.ok === true && schedulerServing;
|
|
return { ok, serving, status: ok ? "healthy" : serving ? "degraded" : "unavailable", node: config.node, lane: config.lane, sentinelId: config.sentinelId, checks, valuesRedacted: true };
|
|
}
|
|
|
|
function checkWritable(stateRoot: string): Record<string, unknown> {
|
|
try {
|
|
mkdirSync(stateRoot, { recursive: true });
|
|
const path = join(stateRoot, ".health-write-probe");
|
|
writeFileSync(path, `${nowIso()}\n`);
|
|
return { ok: true, stateRoot, probeFile: path };
|
|
} catch (error) {
|
|
return { ok: false, stateRoot, error: error instanceof Error ? error.message : String(error) };
|
|
}
|
|
}
|
|
|
|
function checkSqlite(db: Database): Record<string, unknown> {
|
|
try {
|
|
db.query("SELECT 1 AS ok").get();
|
|
writeMetadata(db, "sqlite.health", { at: nowIso() });
|
|
return { ok: true };
|
|
} catch (error) {
|
|
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
}
|
|
}
|
|
|
|
function buildObserveCommandPlan(config: WebProbeSentinelServiceConfig, scenario: Record<string, unknown>): readonly CommandPlanStep[] {
|
|
const targetPath = stringAt(scenario, "observeTargetPath");
|
|
const startArgv = [
|
|
"bun", "scripts/cli.ts", "web-probe", "observe", "start",
|
|
"--node", config.node,
|
|
"--lane", config.lane,
|
|
"--target-path", targetPath,
|
|
"--sample-interval-ms", String(numberAt(scenario, "sampleIntervalMs")),
|
|
"--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")),
|
|
"--max-run-seconds", String(numberAt(scenario, "maxRunSeconds")),
|
|
"--command-timeout-seconds", "55",
|
|
];
|
|
const viewport = stringOrNull(scenario.viewport);
|
|
if (viewport !== null) startArgv.push("--viewport", viewport);
|
|
const start: CommandPlanStep = {
|
|
phase: "observe-start",
|
|
argv: startArgv,
|
|
stdinSource: "none",
|
|
};
|
|
const commands = arrayAt(scenario, "commandSequence").map((item) => {
|
|
const type = stringAt(item, "type");
|
|
const argv = ["bun", "scripts/cli.ts", "web-probe", "observe", "command", "<observerId>", "--type", type];
|
|
if (type === "selectProvider") argv.push("--provider", stringAt(item, "provider"));
|
|
if (type === "sendPrompt") {
|
|
const inlineText = inlinePromptText(item);
|
|
if (inlineText === null) argv.push("--text-stdin");
|
|
else argv.push("--text", "<inline-command-text>");
|
|
}
|
|
if (type === "loginAccount" || type === "listSessions" || type === "logout") {
|
|
const accountId = stringOrNull(item.accountId);
|
|
if (accountId !== null) argv.push("--account-id", accountId);
|
|
}
|
|
if (type === "switchSessions") {
|
|
const fromAccountId = stringOrNull(item.fromAccountId);
|
|
const toAccountId = stringOrNull(item.toAccountId);
|
|
if (fromAccountId !== null) argv.push("--from-account-id", fromAccountId);
|
|
if (toAccountId !== null) argv.push("--to-account-id", toAccountId);
|
|
}
|
|
appendObserveCommandArgs(argv, item, { skipText: type === "sendPrompt" });
|
|
return { phase: `observe-command-${type}`, argv, stdinSource: type === "sendPrompt" ? inlinePromptText(item) === null ? "prompt-source" : "inline-text-redacted" : "none" } satisfies CommandPlanStep;
|
|
});
|
|
const analyze: CommandPlanStep = {
|
|
phase: "observe-analyze",
|
|
argv: ["bun", "scripts/cli.ts", "web-probe", "observe", "analyze", "<observerId>"],
|
|
stdinSource: "none",
|
|
};
|
|
const stop: CommandPlanStep = {
|
|
phase: "observe-stop",
|
|
argv: ["bun", "scripts/cli.ts", "web-probe", "observe", "stop", "<observerId>", "--node", config.node, "--lane", config.lane, "--force", "--wait-ms", "55000", "--command-timeout-seconds", "55"],
|
|
stdinSource: "none",
|
|
};
|
|
const gc: CommandPlanStep = {
|
|
phase: "observe-gc",
|
|
argv: ["bun", "scripts/cli.ts", "web-probe", "observe", "gc", "--node", config.node, "--lane", config.lane, "--confirm", "--command-timeout-seconds", "55"],
|
|
stdinSource: "none",
|
|
};
|
|
return [start, ...commands, analyze, stop, gc];
|
|
}
|
|
|
|
function inlinePromptText(item: Record<string, unknown>): string | null {
|
|
return stringOrNull(item.text) ?? stringOrNull(item.prompt) ?? stringOrNull(item.value);
|
|
}
|
|
|
|
function appendObserveCommandArgs(argv: string[], item: Record<string, unknown>, options: { readonly skipText?: boolean } = {}): void {
|
|
const mappings: readonly (readonly [string, string])[] = [
|
|
["path", "--path"],
|
|
["label", "--label"],
|
|
["sessionId", "--session-id"],
|
|
["provider", "--provider"],
|
|
["accountId", "--account-id"],
|
|
["fromAccountId", "--from-account-id"],
|
|
["toAccountId", "--to-account-id"],
|
|
["sourceId", "--source-id"],
|
|
["fileRef", "--file-ref"],
|
|
["filename", "--filename"],
|
|
["taskRef", "--task-ref"],
|
|
["taskId", "--task-id"],
|
|
["task", "--task"],
|
|
["field", "--field"],
|
|
["link", "--link"],
|
|
["title", "--title"],
|
|
["body", "--body"],
|
|
["status", "--status"],
|
|
["hwpodId", "--hwpod-id"],
|
|
["nodeId", "--node-id"],
|
|
["workspaceRoot", "--workspace-root"],
|
|
["root", "--root"],
|
|
];
|
|
for (const [key, flag] of mappings) {
|
|
if (argv.includes(flag)) continue;
|
|
const value = stringOrNull(item[key]);
|
|
if (value !== null) argv.push(flag, value);
|
|
}
|
|
if (options.skipText !== true) {
|
|
const text = stringOrNull(item.text) ?? stringOrNull(item.value);
|
|
if (text !== null) argv.push("--text", text);
|
|
}
|
|
}
|
|
|
|
function schedulerSummary(config: WebProbeSentinelServiceConfig, db: Database): Record<string, unknown> {
|
|
const planned = plannedRunBacklog(config, db);
|
|
return {
|
|
enabledScenarios: config.scenarios.filter((item) => boolAt(item, "enabled")).map((item) => stringAt(item, "id")),
|
|
intervalMs: config.schedulerIntervalMs,
|
|
maxConcurrentRuns: config.maxConcurrentRuns,
|
|
activeRuns: countWhere(config, db, "status IN ('queued', 'running', 'analyzing')"),
|
|
plannedRuns: planned.count,
|
|
oldestPlannedRunId: planned.oldestRunId,
|
|
oldestPlannedRunScenarioId: planned.oldestScenarioId,
|
|
oldestPlannedRunCreatedAt: planned.oldestCreatedAt,
|
|
oldestPlannedRunAgeSeconds: planned.oldestAgeSeconds,
|
|
plannedRunStaleAfterSeconds: planned.staleAfterSeconds,
|
|
plannedRunStale: planned.stale,
|
|
rootCause: planned.stale ? "planned-run-not-consumed-by-host-cadence" : null,
|
|
heartbeat: readMetadata(db, "scheduler.heartbeat"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function emitSchedulerHeartbeatSpan(config: WebProbeSentinelServiceConfig, loop: string, at: string, ok: boolean, failureKind: string | null = null): void {
|
|
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.scheduler.heartbeat", {
|
|
status: ok ? "ok" : "error",
|
|
failureKind,
|
|
namespace: stringOrNull(config.runtime.namespace),
|
|
heartbeatAt: at,
|
|
cadence: firstEnabledScenarioCadence(config),
|
|
valuesRedacted: true,
|
|
}, ok);
|
|
}
|
|
|
|
function emitCadenceExpectedSpan(config: WebProbeSentinelServiceConfig, summary: Record<string, unknown>): void {
|
|
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.cadence.expected", {
|
|
cadence: firstEnabledScenarioCadence(config),
|
|
scenarioId: firstEnabledScenarioId(config),
|
|
status: summary.rootCause == null ? "ok" : "stale",
|
|
activeRunCount: summary.activeRuns ?? null,
|
|
plannedRunCount: summary.plannedRuns ?? null,
|
|
valuesRedacted: true,
|
|
}, summary.rootCause == null);
|
|
}
|
|
|
|
function emitSchedulerGapSpan(config: WebProbeSentinelServiceConfig, summary: Record<string, unknown>): void {
|
|
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.scheduler_gap.detected", {
|
|
cadence: firstEnabledScenarioCadence(config),
|
|
scenarioId: summary.oldestPlannedRunScenarioId ?? firstEnabledScenarioId(config),
|
|
runId: summary.oldestPlannedRunId ?? null,
|
|
status: "planned-run-stale",
|
|
failureKind: summary.rootCause,
|
|
valuesRedacted: true,
|
|
}, false);
|
|
}
|
|
|
|
function emitRecordRunSpan(config: WebProbeSentinelServiceConfig, input: Record<string, unknown>, result: Record<string, unknown>): void {
|
|
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.record_run", {
|
|
scenarioId: result.scenarioId ?? input.scenarioId ?? null,
|
|
runId: result.runId ?? input.runId ?? null,
|
|
observerId: input.observerId ?? null,
|
|
status: result.status ?? input.status ?? null,
|
|
failureKind: result.ok === true ? null : result.error ?? "record-run-failed",
|
|
valuesRedacted: true,
|
|
}, result.ok === true);
|
|
}
|
|
|
|
function emitManualTriggerSpan(config: WebProbeSentinelServiceConfig, result: Record<string, unknown>): void {
|
|
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.manual_trigger", {
|
|
scenarioId: result.scenarioId ?? null,
|
|
status: result.status ?? null,
|
|
jobName: result.jobName ?? null,
|
|
cronJobName: result.cronJobName ?? null,
|
|
failureKind: result.ok === true ? null : result.reason ?? "manual-trigger-failed",
|
|
valuesRedacted: true,
|
|
}, result.ok === true);
|
|
}
|
|
|
|
function sentinelOtelContext(config: WebProbeSentinelServiceConfig): { readonly node: string; readonly lane: string; readonly sentinelId: string; readonly namespace: string | null; readonly runtime: Record<string, unknown>; readonly cicd: Record<string, unknown> } {
|
|
return {
|
|
node: config.node,
|
|
lane: config.lane,
|
|
sentinelId: config.sentinelId,
|
|
namespace: stringOrNull(config.runtime.namespace),
|
|
runtime: config.runtime,
|
|
cicd: config.cicd,
|
|
};
|
|
}
|
|
|
|
function firstEnabledScenarioCadence(config: WebProbeSentinelServiceConfig): string | null {
|
|
const scenario = config.scenarios.find((item) => boolAt(item, "enabled"));
|
|
return scenario === undefined ? null : stringOrNull(scenario.cadence);
|
|
}
|
|
|
|
function renderMetrics(config: WebProbeSentinelServiceConfig, db: Database, health: Record<string, unknown>, maintenance: MaintenanceState): string {
|
|
const counts = runCounts(config, db);
|
|
const heartbeat = record(readMetadata(db, "scheduler.heartbeat"));
|
|
const heartbeatAt = stringOrNull(heartbeat.at);
|
|
const heartbeatAge = heartbeatAt === null ? -1 : Math.max(0, Math.round((Date.now() - Date.parse(heartbeatAt)) / 1000));
|
|
const labels = `node="${metricLabel(config.node)}",lane="${metricLabel(config.lane)}",sentinel="${metricLabel(config.sentinelId)}"`;
|
|
const lines = [
|
|
"# HELP web_probe_sentinel_config_ready Config reference graph is ready.",
|
|
"# TYPE web_probe_sentinel_config_ready gauge",
|
|
`web_probe_sentinel_config_ready{${labels}} ${config.plan.ok ? 1 : 0}`,
|
|
"# HELP web_probe_sentinel_health Healthy status of the sentinel service.",
|
|
"# TYPE web_probe_sentinel_health gauge",
|
|
`web_probe_sentinel_health{${labels}} ${health.ok === true ? 1 : 0}`,
|
|
"# HELP web_probe_sentinel_runs_total Runs indexed by status.",
|
|
"# TYPE web_probe_sentinel_runs_total gauge",
|
|
...Object.entries(counts).map(([status, count]) => `web_probe_sentinel_runs_total{${labels},status="${metricLabel(status)}"} ${count}`),
|
|
"# HELP web_probe_sentinel_active_runs Active observe runs known to the sentinel index.",
|
|
"# TYPE web_probe_sentinel_active_runs gauge",
|
|
`web_probe_sentinel_active_runs{${labels}} ${countWhere(config, db, "status IN ('queued', 'running', 'analyzing')")}`,
|
|
"# HELP web_probe_sentinel_recent_findings Findings indexed from recent reports.",
|
|
"# TYPE web_probe_sentinel_recent_findings gauge",
|
|
`web_probe_sentinel_recent_findings{${labels}} ${sumColumn(config, db, "runs", "finding_count")}`,
|
|
"# HELP web_probe_sentinel_maintenance_active Maintenance window active flag.",
|
|
"# TYPE web_probe_sentinel_maintenance_active gauge",
|
|
`web_probe_sentinel_maintenance_active{${labels}} ${maintenance.active ? 1 : 0}`,
|
|
"# HELP web_probe_sentinel_scheduler_heartbeat_age_seconds Scheduler heartbeat age.",
|
|
"# TYPE web_probe_sentinel_scheduler_heartbeat_age_seconds gauge",
|
|
`web_probe_sentinel_scheduler_heartbeat_age_seconds{${labels}} ${heartbeatAge}`,
|
|
"# HELP web_probe_sentinel_planned_runs Planned runs waiting for host cadence execution.",
|
|
"# TYPE web_probe_sentinel_planned_runs gauge",
|
|
`web_probe_sentinel_planned_runs{${labels}} ${countWhere(config, db, "status = 'planned'")}`,
|
|
"# HELP web_probe_sentinel_oldest_planned_run_age_seconds Oldest planned run age, or -1 when no planned run exists.",
|
|
"# TYPE web_probe_sentinel_oldest_planned_run_age_seconds gauge",
|
|
`web_probe_sentinel_oldest_planned_run_age_seconds{${labels}} ${plannedRunBacklog(config, db).oldestAgeSeconds ?? -1}`,
|
|
];
|
|
return `${lines.join("\n")}\n`;
|
|
}
|
|
|
|
function plannedRunBacklog(config: WebProbeSentinelServiceConfig, db: Database): {
|
|
readonly count: number;
|
|
readonly oldestRunId: string | null;
|
|
readonly oldestScenarioId: string | null;
|
|
readonly oldestCreatedAt: string | null;
|
|
readonly oldestAgeSeconds: number | null;
|
|
readonly staleAfterSeconds: number;
|
|
readonly stale: boolean;
|
|
} {
|
|
const activeWhere = [
|
|
"status = 'planned'",
|
|
"AND NOT EXISTS (",
|
|
"SELECT 1 FROM runs newer",
|
|
"WHERE newer.sentinel_id = runs.sentinel_id",
|
|
"AND newer.node = runs.node",
|
|
"AND newer.lane = runs.lane",
|
|
"AND newer.scenario_id = runs.scenario_id",
|
|
"AND newer.status <> 'planned'",
|
|
"AND newer.created_at > runs.created_at",
|
|
")",
|
|
].join(" ");
|
|
const count = countWhere(config, db, activeWhere);
|
|
const oldest = db.query(`SELECT id, scenario_id, created_at FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? AND ${activeWhere} ORDER BY created_at ASC LIMIT 1`)
|
|
.get(config.sentinelId, config.node, config.lane) as Record<string, unknown> | null;
|
|
const oldestCreatedAt = stringOrNull(oldest?.created_at);
|
|
const oldestAgeSeconds = oldestCreatedAt === null ? null : ageSeconds(oldestCreatedAt);
|
|
const staleAfterSeconds = Math.max(60, Math.round(config.schedulerIntervalMs / 1000));
|
|
return {
|
|
count,
|
|
oldestRunId: stringOrNull(oldest?.id),
|
|
oldestScenarioId: stringOrNull(oldest?.scenario_id),
|
|
oldestCreatedAt,
|
|
oldestAgeSeconds,
|
|
staleAfterSeconds,
|
|
stale: oldestAgeSeconds !== null && oldestAgeSeconds >= staleAfterSeconds,
|
|
};
|
|
}
|
|
|
|
function writeMetadata(db: Database, key: string, value: unknown): void {
|
|
db.query("INSERT INTO metadata (key, value_json, updated_at) VALUES (?, ?, ?) ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, updated_at = excluded.updated_at")
|
|
.run(key, JSON.stringify(value), nowIso());
|
|
}
|
|
|
|
function readMetadata(db: Database, key: string): Record<string, unknown> | null {
|
|
const row = db.query("SELECT value_json FROM metadata WHERE key = ?").get(key) as { value_json?: string } | null;
|
|
if (typeof row?.value_json !== "string") return null;
|
|
try {
|
|
const parsed = JSON.parse(row.value_json) as unknown;
|
|
return record(parsed);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function runCounts(config: WebProbeSentinelServiceConfig, db: Database): Record<string, number> {
|
|
const rows = db.query("SELECT status, COUNT(*) AS count FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? GROUP BY status")
|
|
.all(config.sentinelId, config.node, config.lane) as { status: string; count: number }[];
|
|
return Object.fromEntries(rows.map((row) => [row.status, Number(row.count)]));
|
|
}
|
|
|
|
function dashboardOverview(config: WebProbeSentinelServiceConfig, db: Database, health: Record<string, unknown>, maintenance: MaintenanceState): Record<string, unknown> {
|
|
const latestRow = db.query("SELECT * FROM runs ORDER BY updated_at DESC LIMIT 1").get() as Record<string, unknown> | null;
|
|
const latestRun = latestRow === null ? null : dashboardRunSummary(config, db, latestRow);
|
|
const severityCounts = globalSeverityCounts(config, db);
|
|
const latestUpdatedAt = latestRow === null ? null : stringOrNull(latestRow.updated_at);
|
|
const latestRunAgeSeconds = latestUpdatedAt === null ? null : ageSeconds(latestUpdatedAt);
|
|
const heartbeatAgeSeconds = numberOr(record(record(health.checks).scheduler).heartbeatAgeSeconds, -1);
|
|
const expectedCadence = firstEnabledScenarioCadence(config);
|
|
const expectedCadenceSeconds = durationStringSeconds(expectedCadence);
|
|
const staleMultiple = expectedCadenceSeconds === null || latestRunAgeSeconds === null ? null : latestRunAgeSeconds / expectedCadenceSeconds;
|
|
const freshnessWarningMultiple = numberAt(config.runtime, "scheduler.freshnessWarningMultiple");
|
|
const scheduler = schedulerSummary(config, db);
|
|
return {
|
|
ok: health.ok === true,
|
|
contractVersion: DASHBOARD_CONTRACT_VERSION,
|
|
status: dashboardOverallStatus(health, latestRun, severityCounts),
|
|
node: config.node,
|
|
lane: config.lane,
|
|
sentinelId: config.sentinelId,
|
|
publicOrigin: stringOrNull(config.publicExposure.publicBaseUrl),
|
|
configReady: config.plan.ok,
|
|
health,
|
|
scheduler,
|
|
maintenance,
|
|
latestRun,
|
|
runCounts: runCounts(config, db),
|
|
severityCounts,
|
|
freshness: {
|
|
latestRunUpdatedAt: latestUpdatedAt,
|
|
latestRunAgeSeconds,
|
|
schedulerHeartbeatAgeSeconds: heartbeatAgeSeconds,
|
|
latestAnalyzedReportAgeSeconds: latestRow === null || stringOrNull(latestRow.report_json_sha256) === null ? null : latestRunAgeSeconds,
|
|
},
|
|
cadence: {
|
|
expectedCadence,
|
|
expectedCadenceSeconds,
|
|
schedulerHeartbeatAgeSeconds: heartbeatAgeSeconds,
|
|
latestRunAgeSeconds,
|
|
latestAnalyzedReportAgeSeconds: latestRow === null || stringOrNull(latestRow.report_json_sha256) === null ? null : latestRunAgeSeconds,
|
|
activeRuns: scheduler.activeRuns ?? null,
|
|
plannedRuns: scheduler.plannedRuns ?? null,
|
|
nextRun: null,
|
|
staleMultiple,
|
|
freshnessWarningMultiple,
|
|
status: scheduler.rootCause === "planned-run-not-consumed-by-host-cadence" ? "blocker" : staleMultiple !== null && staleMultiple > freshnessWarningMultiple ? "warning" : "fresh",
|
|
cronJob: {
|
|
observed: false,
|
|
status: "control-plane-status-required",
|
|
reason: "runner API does not query Kubernetes CronJob objects; use web-probe sentinel control-plane status for CronJob counts, lastScheduleTime and latest Jobs.",
|
|
valuesRedacted: true,
|
|
},
|
|
valuesRedacted: true,
|
|
},
|
|
observability: webProbeSentinelOtelSummary(sentinelOtelContext(config)),
|
|
targetValidation: {
|
|
scenarioId: stringOrNull(record(config.cicd.targetValidation).scenarioId),
|
|
maxSeconds: numberOr(record(config.cicd.targetValidation).maxSeconds, 120),
|
|
sourceRef: targetValidationSourceRef(config),
|
|
},
|
|
traceability: {
|
|
source: "sqlite-index+run-report-metadata",
|
|
stateRoot: config.stateRoot,
|
|
latestRun: latestRow === null ? null : runTraceability(config, latestRow),
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function targetValidationSourceRef(config: WebProbeSentinelServiceConfig): string | null {
|
|
const cicdRef = config.plan.refs.find((item) => item.key === "cicd")?.ref ?? null;
|
|
return cicdRef === null ? null : `${cicdRef}.targetValidation`;
|
|
}
|
|
|
|
function dashboardRunList(config: WebProbeSentinelServiceConfig, db: Database, url: URL): Record<string, unknown> {
|
|
const filters = dashboardRunFilters(url);
|
|
const page = dashboardPage(url, config);
|
|
const sort = dashboardRunSort(url);
|
|
const where = runWhereClause(config, filters);
|
|
const sql = `SELECT * FROM runs ${where.sql} ORDER BY ${sort.sql} LIMIT ? OFFSET ?`;
|
|
const rows = db.query(sql).all(...where.params, page.limit + 1, page.offset) as Record<string, unknown>[];
|
|
const visibleRows = rows.slice(0, page.limit);
|
|
const items = visibleRows.map((row) => dashboardRunSummary(config, db, row));
|
|
return {
|
|
ok: true,
|
|
contractVersion: DASHBOARD_CONTRACT_VERSION,
|
|
node: config.node,
|
|
lane: config.lane,
|
|
sentinelId: config.sentinelId,
|
|
filters,
|
|
page: {
|
|
limit: page.limit,
|
|
cursor: String(page.offset),
|
|
nextCursor: rows.length > page.limit ? String(page.offset + visibleRows.length) : null,
|
|
hasMore: rows.length > page.limit,
|
|
sort: sort.name,
|
|
direction: sort.direction,
|
|
},
|
|
items,
|
|
runs: items,
|
|
traceability: {
|
|
source: "sqlite-index",
|
|
stateRoot: config.stateRoot,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function dashboardRunDetail(config: WebProbeSentinelServiceConfig, db: Database, runId: string): Record<string, unknown> {
|
|
const row = readRunRow(config, db, runId);
|
|
if (row === null) return { ok: false, error: "run-not-found", runId, valuesRedacted: true };
|
|
const stored = readMetadata(db, `run.report.${runId}`) ?? {};
|
|
const artifactSummary = readAnalysisArtifactSummaryForRun(config, row, dashboardMaxPageSize(config));
|
|
const findings = visibleFindingsForRun(config, db, row, dashboardMaxPageSize(config), artifactSummary);
|
|
const views = record(stored.views);
|
|
const reportJsonSha256 = stringOrNull(row.report_json_sha256) ?? stringOrNull(artifactSummary.reportJsonSha256);
|
|
return {
|
|
ok: true,
|
|
contractVersion: DASHBOARD_CONTRACT_VERSION,
|
|
sentinelId: config.sentinelId,
|
|
run: dashboardRunSummary(config, db, row),
|
|
summary: record(stored.summary),
|
|
memory: dashboardRunMemoryDetail(config, row, stored),
|
|
requestRate: dashboardRunRequestRateDetail(config, row, stored, artifactSummary),
|
|
findings,
|
|
viewsAvailable: Object.keys(views),
|
|
artifacts: {
|
|
artifactCount: numberOr(row.artifact_count, 0),
|
|
reportJsonSha256,
|
|
analysisReport: artifactSummary,
|
|
screenshot: compactArtifactRef(stored.screenshot),
|
|
publicOrigin: stringOrNull(stored.publicOrigin),
|
|
},
|
|
commands: {
|
|
summary: `bun scripts/cli.ts web-probe sentinel report --node ${config.node} --lane ${config.lane} --sentinel ${config.sentinelId} --run ${runId} --view summary`,
|
|
turnSummary: `bun scripts/cli.ts web-probe sentinel report --node ${config.node} --lane ${config.lane} --sentinel ${config.sentinelId} --run ${runId} --view turn-summary`,
|
|
traceFrame: `bun scripts/cli.ts web-probe sentinel report --node ${config.node} --lane ${config.lane} --sentinel ${config.sentinelId} --run ${runId} --view trace-frame`,
|
|
},
|
|
redaction: record(config.reportViews.redaction),
|
|
traceability: runTraceability(config, row),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function dashboardFindings(config: WebProbeSentinelServiceConfig, db: Database, url: URL): Record<string, unknown> {
|
|
const limit = dashboardPage(url, config).limit;
|
|
const filters = dashboardFindingFilters(url);
|
|
const where = findingWhereClause(config, filters);
|
|
const queryLimit = Math.min(dashboardMaxPageSize(config), Math.max(limit + 1, limit * 4));
|
|
const rows = db.query(`
|
|
SELECT f.finding_id,
|
|
(SELECT f2.severity FROM findings f2 JOIN runs r2 ON r2.id = f2.run_id WHERE f2.finding_id = f.finding_id AND r2.scenario_id = r.scenario_id ORDER BY f2.created_at DESC LIMIT 1) AS severity,
|
|
r.scenario_id, SUM(f.count) AS count, COUNT(DISTINCT f.run_id) AS run_count,
|
|
MAX(f.created_at) AS latest_at, MAX(f.summary) AS summary
|
|
FROM findings f
|
|
JOIN runs r ON r.id = f.run_id
|
|
${where.sql}
|
|
GROUP BY f.finding_id, r.scenario_id
|
|
ORDER BY latest_at DESC
|
|
LIMIT ?
|
|
`).all(...where.params, queryLimit) as Record<string, unknown>[];
|
|
const severityFilter = stringOrNull(filters.severity);
|
|
const items = rows.map((row) => {
|
|
const latestRun = latestRunForFinding(config, db, row);
|
|
const latestDetail = latestRun === null ? null : storedFindingDetailForRow(db, row, stringOrNull(latestRun.id));
|
|
return enrichFindingWithCheck(config, {
|
|
code: stringOrNull(row.finding_id),
|
|
findingId: stringOrNull(row.finding_id),
|
|
severity: stringOrNull(row.severity),
|
|
scenarioId: stringOrNull(row.scenario_id),
|
|
count: numberOr(row.count, 0),
|
|
runCount: numberOr(row.run_count, 0),
|
|
latestAt: stringOrNull(row.latest_at),
|
|
latestRunId: latestRun === null ? null : stringOrNull(latestRun.id),
|
|
latestReportJsonSha256: latestRun === null ? null : stringOrNull(latestRun.report_json_sha256),
|
|
summary: stringOrNull(row.summary),
|
|
rootCause: stringOrNull(latestDetail?.rootCause),
|
|
rootCauseStatus: stringOrNull(latestDetail?.rootCauseStatus),
|
|
rootCauseConfidence: stringOrNull(latestDetail?.rootCauseConfidence),
|
|
nextAction: stringOrNull(latestDetail?.nextAction),
|
|
evidenceSummary: stringOrNull(latestDetail?.evidenceSummary),
|
|
timingSourceOfTruth: stringOrNull(latestDetail?.timingSourceOfTruth),
|
|
timingStatus: stringOrNull(latestDetail?.timingStatus),
|
|
timingAlert: latestDetail?.timingAlert === true,
|
|
traceability: latestRun === null ? null : runTraceability(config, latestRun),
|
|
valuesRedacted: true,
|
|
});
|
|
}).filter((item) => severityFilter === null || stringOrNull(item.checkLevel) === severityFilter || stringOrNull(item.severity) === severityFilter)
|
|
.sort(compareCheckFindingRows)
|
|
.slice(0, limit);
|
|
return {
|
|
ok: true,
|
|
contractVersion: DASHBOARD_CONTRACT_VERSION,
|
|
node: config.node,
|
|
lane: config.lane,
|
|
sentinelId: config.sentinelId,
|
|
filters,
|
|
page: {
|
|
limit,
|
|
hasMore: rows.length === queryLimit,
|
|
sort: "severity",
|
|
direction: "desc",
|
|
},
|
|
groups: items,
|
|
findings: items,
|
|
traceability: {
|
|
source: "sqlite-index-findings",
|
|
stateRoot: config.stateRoot,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function dashboardRunViews(config: WebProbeSentinelServiceConfig, db: Database, runId: string, view: string | null, url: URL): Record<string, unknown> {
|
|
const row = readRunRow(config, db, runId);
|
|
if (row === null) return { ok: false, error: "run-not-found", runId, valuesRedacted: true };
|
|
const requestedViews = view === null ? stringArrayAt(config.reportViews, "views") : [view];
|
|
const maxBytes = Math.min(numberParam(url, "maxBytes", DASHBOARD_MAX_TEXT_BYTES), 64_000);
|
|
const views = requestedViews.map((item) => dashboardReportView(config, db, runId, item, maxBytes));
|
|
return {
|
|
ok: true,
|
|
contractVersion: DASHBOARD_CONTRACT_VERSION,
|
|
sentinelId: config.sentinelId,
|
|
run: dashboardRunSummary(config, db, row),
|
|
views,
|
|
view: view ?? null,
|
|
redaction: record(config.reportViews.redaction),
|
|
traceability: runTraceability(config, row),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function dashboardReportView(config: WebProbeSentinelServiceConfig, db: Database, runId: string, view: string, maxBytes: number): Record<string, unknown> {
|
|
const report = reportRunView(config, db, view, runId);
|
|
const renderedText = typeof report.renderedText === "string" ? report.renderedText : "";
|
|
const bounded = boundedText(renderedText, maxBytes);
|
|
return {
|
|
ok: report.ok !== false,
|
|
view,
|
|
error: stringOrNull(report.error),
|
|
availableViews: Array.isArray(report.availableViews) ? report.availableViews : undefined,
|
|
renderedText: bounded.text,
|
|
renderedTextBytes: bounded.bytes,
|
|
truncated: bounded.truncated,
|
|
finalResponse: view === "trace-frame" ? finalResponseBlock(bounded.text) : null,
|
|
summary: record(report.summary),
|
|
findingCount: Array.isArray(report.findings) ? report.findings.length : 0,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function dashboardRunMemoryDetail(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>, stored: Record<string, unknown>): Record<string, unknown> {
|
|
const options = dashboardDetailMemoryOptions(config);
|
|
const storedMemory = compactDashboardMemory(record(record(record(stored.summary).analysis).browserProcess), options, "recorded-analysis-summary");
|
|
if (storedMemory.pageSeries.length > 0) return storedMemory;
|
|
const fromArtifacts = readDashboardMemoryFromStateDir(config, row, options);
|
|
if (fromArtifacts.pageSeries.length > 0) return fromArtifacts;
|
|
return {
|
|
ok: false,
|
|
source: "unavailable",
|
|
unit: "MB",
|
|
metric: "page-heap-used",
|
|
pageCount: 0,
|
|
sampleCount: 0,
|
|
maxMemoryMb: null,
|
|
maxPageLabel: null,
|
|
pageSeries: [],
|
|
reason: fromArtifacts.reason ?? storedMemory.reason ?? "page-memory-samples-missing",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function dashboardRunRequestRateDetail(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>, stored: Record<string, unknown>, artifactSummary: Record<string, unknown> = {}): Record<string, unknown> {
|
|
const options = dashboardDetailRequestRateOptions(config);
|
|
const storedAnalysis = record(record(stored.summary).analysis);
|
|
const storedRequestRate = compactDashboardRequestRate(record(storedAnalysis.requestRate), options, "recorded-analysis-summary");
|
|
if (storedRequestRate.ok === true && hasDashboardRequestRateSeries(storedRequestRate)) return storedRequestRate;
|
|
const storedRequestRateCurve = compactDashboardRequestRate(record(storedAnalysis.requestRateCurve), options, "recorded-analysis-request-rate-curve");
|
|
if (storedRequestRateCurve.ok === true && hasDashboardRequestRateSeries(storedRequestRateCurve)) return storedRequestRateCurve;
|
|
const artifactRequestRate = compactDashboardRequestRate(record(artifactSummary.requestRate), options, "artifact-summary-request-rate");
|
|
if (artifactRequestRate.ok === true && hasDashboardRequestRateSeries(artifactRequestRate)) return artifactRequestRate;
|
|
const fromArtifacts = readDashboardRequestRateFromAnalysisReport(config, row, options);
|
|
if (fromArtifacts.ok === true) return fromArtifacts;
|
|
if (artifactRequestRate.ok === true) return artifactRequestRate;
|
|
if (storedRequestRate.ok === true) return storedRequestRate;
|
|
if (storedRequestRateCurve.ok === true) return storedRequestRateCurve;
|
|
return {
|
|
ok: false,
|
|
source: "unavailable",
|
|
unit: "rpm",
|
|
metric: "same-origin-api-request-rate",
|
|
requestCount: 0,
|
|
bucketCount: 0,
|
|
pageCount: 0,
|
|
apiPathCount: 0,
|
|
totalSeries: [],
|
|
pageSeries: [],
|
|
apiPathSeries: [],
|
|
peaks: [],
|
|
reason: fromArtifacts.reason ?? storedRequestRate.reason ?? storedRequestRateCurve.reason ?? "request-rate-samples-missing",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function hasDashboardRequestRateSeries(value: Record<string, unknown>): boolean {
|
|
return arrayRecords(value.totalSeries).length > 0
|
|
|| arrayRecords(value.pageSeries).length > 0
|
|
|| arrayRecords(value.apiPathSeries).length > 0;
|
|
}
|
|
|
|
function dashboardDetailMemoryOptions(config: WebProbeSentinelServiceConfig): Record<string, number> {
|
|
const detailMemory = recordTarget(valueAtPath(config.reportViews, "detailMemory"));
|
|
return {
|
|
maxPages: Math.max(1, Math.floor(numberAt(detailMemory, "maxPages"))),
|
|
maxSamplesPerPage: Math.max(1, Math.floor(numberAt(detailMemory, "maxSamplesPerPage"))),
|
|
};
|
|
}
|
|
|
|
function dashboardDetailRequestRateOptions(config: WebProbeSentinelServiceConfig): Record<string, number> {
|
|
const detailRequestRate = recordTarget(valueAtPath(config.reportViews, "detailRequestRate"));
|
|
return {
|
|
maxTotalBuckets: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxTotalBuckets"))),
|
|
maxPageSeries: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxPageSeries"))),
|
|
maxApiPathSeries: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxApiPathSeries"))),
|
|
maxBucketsPerSeries: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxBucketsPerSeries"))),
|
|
maxPeaks: Math.max(1, Math.floor(numberAt(detailRequestRate, "maxPeaks"))),
|
|
};
|
|
}
|
|
|
|
function readDashboardMemoryFromStateDir(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>, options: Record<string, number>): Record<string, unknown> {
|
|
const stateDir = stringOrNull(row.state_dir) ?? stringOrNull(row.stateDir);
|
|
if (stateDir === null) return { ok: false, source: "artifact-browser-process-jsonl", reason: "state-dir-missing", pageSeries: [], valuesRedacted: true };
|
|
if (!isSafeDashboardStateDir(stateDir)) return { ok: false, source: "artifact-browser-process-jsonl", reason: "unsafe-state-dir", pageSeries: [], valuesRedacted: true };
|
|
const browserProcessPath = join(config.stateRoot, stateDir, "browser-process.jsonl");
|
|
if (!existsSync(browserProcessPath)) return { ok: false, source: "artifact-browser-process-jsonl", reason: "browser-process-jsonl-missing", pageSeries: [], valuesRedacted: true };
|
|
try {
|
|
const rows = readFileSync(browserProcessPath, "utf8")
|
|
.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.flatMap((line) => {
|
|
const parsed = parseJsonObject(line);
|
|
return parsed === null ? [] : [parsed];
|
|
});
|
|
return buildDashboardMemoryDetail(rows, options, "artifact-browser-process-jsonl");
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
source: "artifact-browser-process-jsonl",
|
|
reason: "browser-process-jsonl-read-failed",
|
|
error: error instanceof Error ? error.message : String(error),
|
|
pageSeries: [],
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
}
|
|
|
|
function compactDashboardMemory(value: Record<string, unknown>, options: Record<string, number>, source: string): Record<string, unknown> {
|
|
const pageSeries = arrayRecords(value.pageSeries);
|
|
if (pageSeries.length > 0) return finalizeDashboardMemorySeries(pageSeries, options, source);
|
|
return { ok: false, source, reason: "recorded-page-series-missing", pageSeries: [], valuesRedacted: true };
|
|
}
|
|
|
|
function readDashboardRequestRateFromAnalysisReport(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>, options: Record<string, number>): Record<string, unknown> {
|
|
const stateDir = stringOrNull(row.state_dir) ?? stringOrNull(row.stateDir);
|
|
if (stateDir === null) return { ok: false, source: "analysis-report-json", reason: "state-dir-missing", totalSeries: [], pageSeries: [], apiPathSeries: [], valuesRedacted: true };
|
|
if (!isSafeDashboardStateDir(stateDir)) return { ok: false, source: "analysis-report-json", reason: "unsafe-state-dir", totalSeries: [], pageSeries: [], apiPathSeries: [], valuesRedacted: true };
|
|
const reportJsonPath = join(config.stateRoot, stateDir, "analysis", "report.json");
|
|
const jsonBuffer = readFileBuffer(reportJsonPath);
|
|
if (jsonBuffer === null) return { ok: false, source: "analysis-report-json", reason: "analysis-report-json-missing", totalSeries: [], pageSeries: [], apiPathSeries: [], valuesRedacted: true };
|
|
try {
|
|
const report = record(JSON.parse(jsonBuffer.toString("utf8")) as unknown);
|
|
return compactDashboardRequestRate(record(report.requestRate), options, "analysis-report-json");
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
source: "analysis-report-json",
|
|
reason: "analysis-report-json-invalid",
|
|
error: error instanceof Error ? error.message : String(error),
|
|
totalSeries: [],
|
|
pageSeries: [],
|
|
apiPathSeries: [],
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
}
|
|
|
|
function compactDashboardRequestRate(value: Record<string, unknown>, options: Record<string, number>, source: string): Record<string, unknown> {
|
|
const summary = requestRateSummary(record(value.summary).requestCount === undefined ? value : record(value.summary));
|
|
const totalSeries = finalizeDashboardRequestRateSeries([{
|
|
key: "total",
|
|
label: "全部 API 请求",
|
|
count: summary.requestCount,
|
|
peakRequestPerMinute: summary.totalPeakPerMinute,
|
|
buckets: arrayRecords(value.buckets),
|
|
}], numberOr(options.maxTotalBuckets, 1), 1, "total");
|
|
const pageSeries = finalizeDashboardRequestRateSeries(arrayRecords(value.pageCurves), numberOr(options.maxBucketsPerSeries, 1), numberOr(options.maxPageSeries, 1), "page");
|
|
const apiPathSeries = finalizeDashboardRequestRateSeries(arrayRecords(value.apiPathCurves), numberOr(options.maxBucketsPerSeries, 1), numberOr(options.maxApiPathSeries, 1), "apiPath");
|
|
const peaks = compactDashboardRequestRatePeaks(arrayRecords(value.peaks), numberOr(options.maxPeaks, 1));
|
|
const hasRequestRate = summary.requestCount !== null || totalSeries.length > 0 || pageSeries.length > 0 || apiPathSeries.length > 0 || peaks.length > 0;
|
|
if (!hasRequestRate) {
|
|
return { ok: false, source, reason: "request-rate-report-missing", totalSeries: [], pageSeries: [], apiPathSeries: [], peaks: [], valuesRedacted: true };
|
|
}
|
|
return {
|
|
ok: true,
|
|
source,
|
|
unit: "rpm",
|
|
metric: "same-origin-api-request-rate",
|
|
...summary,
|
|
totalSeries,
|
|
pageSeries,
|
|
apiPathSeries,
|
|
peaks,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function requestRateSummary(value: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
bucketMs: numberOrNull(value.bucketMs),
|
|
bucketSeconds: numberOrNull(value.bucketSeconds),
|
|
requestCount: numberOrNull(value.requestCount),
|
|
bucketCount: numberOrNull(value.bucketCount),
|
|
pageCount: numberOrNull(value.pageCount),
|
|
apiPathCount: numberOrNull(value.apiPathCount),
|
|
firstAt: stringOrNull(value.firstAt),
|
|
lastAt: stringOrNull(value.lastAt),
|
|
totalRedPerMinute: numberOrNull(value.totalRedPerMinute),
|
|
pageRedPerMinute: numberOrNull(value.pageRedPerMinute),
|
|
apiPathRedPerMinute: numberOrNull(value.apiPathRedPerMinute),
|
|
totalPeakPerMinute: numberOrNull(value.totalPeakPerMinute),
|
|
totalPeakCount: numberOrNull(value.totalPeakCount),
|
|
totalPeakAt: stringOrNull(value.totalPeakAt),
|
|
pagePeakPerMinute: numberOrNull(value.pagePeakPerMinute),
|
|
pagePeakKey: stringOrNull(value.pagePeakKey),
|
|
pagePeakPath: stringOrNull(value.pagePeakPath),
|
|
apiPathPeakPerMinute: numberOrNull(value.apiPathPeakPerMinute),
|
|
apiPathPeakKey: stringOrNull(value.apiPathPeakKey),
|
|
overThresholdPeakCount: numberOrNull(value.overThresholdPeakCount),
|
|
};
|
|
}
|
|
|
|
function finalizeDashboardRequestRateSeries(rawSeries: readonly Record<string, unknown>[], maxBucketsPerSeries: number, maxSeries: number, scope: string): Record<string, unknown>[] {
|
|
return rawSeries
|
|
.map((item) => {
|
|
const points = arrayRecords(item.buckets)
|
|
.map((bucket) => ({
|
|
ts: stringOrNull(bucket.startAt) ?? stringOrNull(bucket.t),
|
|
startAt: stringOrNull(bucket.startAt) ?? stringOrNull(bucket.t),
|
|
endAt: stringOrNull(bucket.endAt) ?? stringOrNull(bucket.e),
|
|
bucketStartMs: numberOrNull(bucket.bucketStartMs),
|
|
bucketEndMs: numberOrNull(bucket.bucketEndMs),
|
|
count: numberOrNull(bucket.count) ?? numberOrNull(bucket.c),
|
|
requestPerMinute: numberOrNull(bucket.requestPerMinute) ?? numberOrNull(bucket.rpm),
|
|
valuesRedacted: true,
|
|
}))
|
|
.filter((point) => point.ts !== null && point.requestPerMinute !== null);
|
|
const thinned = thinDashboardPoints(points, maxBucketsPerSeries);
|
|
const peakRequestPerMinute = numberOrNull(item.peakRequestPerMinute) ?? maxNumber(thinned.map((point) => numberOrNull(point.requestPerMinute))) ?? numberOrNull(item.totalPeakPerMinute) ?? 0;
|
|
const count = numberOrNull(item.count) ?? numberOrNull(item.requestCount) ?? thinned.reduce((sum, point) => sum + numberOr(point.count, 0), 0);
|
|
return {
|
|
key: requestRateSeriesKey(item, scope),
|
|
scope,
|
|
label: requestRateSeriesLabel(item, scope),
|
|
method: stringOrNull(item.method),
|
|
path: stringOrNull(item.path),
|
|
pageRole: stringOrNull(item.pageRole),
|
|
pageId: stringOrNull(item.pageId),
|
|
pageEpoch: numberOrNull(item.pageEpoch),
|
|
count,
|
|
bucketCount: numberOrNull(item.bucketCount) ?? thinned.length,
|
|
peakRequestPerMinute,
|
|
peakAt: stringOrNull(record(item.peakBucket).startAt),
|
|
points: thinned,
|
|
valuesRedacted: true,
|
|
};
|
|
})
|
|
.filter((item) => arrayRecords(item.points).length > 0)
|
|
.sort((a, b) => numberOr(b.peakRequestPerMinute, 0) - numberOr(a.peakRequestPerMinute, 0) || numberOr(b.count, 0) - numberOr(a.count, 0))
|
|
.slice(0, maxSeries);
|
|
}
|
|
|
|
function compactDashboardRequestRatePeaks(rawPeaks: readonly Record<string, unknown>[], maxPeaks: number): Record<string, unknown>[] {
|
|
return rawPeaks.slice(0, maxPeaks).map((item) => ({
|
|
scope: stringOrNull(item.scope),
|
|
thresholdPerMinute: numberOrNull(item.thresholdPerMinute),
|
|
overThreshold: item.overThreshold === true,
|
|
bucketMs: numberOrNull(item.bucketMs),
|
|
startAt: stringOrNull(item.startAt),
|
|
endAt: stringOrNull(item.endAt),
|
|
count: numberOrNull(item.count),
|
|
requestPerMinute: numberOrNull(item.requestPerMinute),
|
|
method: stringOrNull(item.method),
|
|
path: stringOrNull(item.path),
|
|
apiKey: stringOrNull(item.apiKey),
|
|
pageKey: stringOrNull(item.pageKey),
|
|
pageRole: stringOrNull(item.pageRole),
|
|
pageId: stringOrNull(item.pageId),
|
|
pageEpoch: numberOrNull(item.pageEpoch),
|
|
valuesRedacted: true,
|
|
}));
|
|
}
|
|
|
|
function requestRateSeriesKey(item: Record<string, unknown>, scope: string): string {
|
|
const fallback = [stringOrNull(item.method), stringOrNull(item.path)].filter((part) => part !== null).join(" ");
|
|
return stringOrNull(item.key)
|
|
?? stringOrNull(item.apiKey)
|
|
?? stringOrNull(item.pageKey)
|
|
?? (fallback.length > 0 ? fallback : scope);
|
|
}
|
|
|
|
function requestRateSeriesLabel(item: Record<string, unknown>, scope: string): string {
|
|
if (scope === "total") return "全部 API 请求";
|
|
if (scope === "apiPath") return (stringOrNull(item.apiKey) ?? [stringOrNull(item.method), stringOrNull(item.path)].filter((part) => part !== null).join(" ")) || "API";
|
|
return stringOrNull(item.path) ?? stringOrNull(item.pageKey) ?? stringOrNull(item.pageRole) ?? "page";
|
|
}
|
|
|
|
function buildDashboardMemoryDetail(rows: readonly Record<string, unknown>[], options: Record<string, number>, source: string): Record<string, unknown> {
|
|
const seriesByKey = new Map<string, Record<string, unknown>>();
|
|
let firstTsMs: number | null = null;
|
|
for (const row of rows) {
|
|
if (stringOrNull(row.type) !== "browser-process-sample") continue;
|
|
const ts = stringOrNull(row.ts);
|
|
const tsMs = ts === null ? null : Date.parse(ts);
|
|
if (ts === null || tsMs === null || !Number.isFinite(tsMs)) continue;
|
|
for (const page of arrayRecords(row.pages)) {
|
|
const memory = dashboardPageMemoryPoint(page);
|
|
if (memory === null) continue;
|
|
if (firstTsMs === null) firstTsMs = tsMs;
|
|
const pageRole = stringOrNull(page.pageRole) ?? "page";
|
|
const pageId = stringOrNull(page.pageId) ?? `${pageRole}-${stringOrNull(page.pageEpoch) ?? seriesByKey.size}`;
|
|
const key = `${pageRole}:${pageId}`;
|
|
const existing = seriesByKey.get(key);
|
|
const label = `${pageRole} ${shortDashboardText(pageId, 10)}`;
|
|
const points = arrayRecords(existing?.points);
|
|
points.push({
|
|
ts,
|
|
elapsedSeconds: Math.max(0, Math.round((tsMs - firstTsMs) / 1000)),
|
|
elapsedMinutes: Math.round(Math.max(0, (tsMs - firstTsMs) / 60_000) * 100) / 100,
|
|
...memory,
|
|
valuesRedacted: true,
|
|
});
|
|
seriesByKey.set(key, {
|
|
key,
|
|
pageRole,
|
|
pageId,
|
|
label,
|
|
url: safeDashboardUrl(stringOrNull(page.url)),
|
|
points,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
}
|
|
return finalizeDashboardMemorySeries([...seriesByKey.values()], options, source);
|
|
}
|
|
|
|
function finalizeDashboardMemorySeries(rawSeries: readonly Record<string, unknown>[], options: Record<string, number>, source: string): Record<string, unknown> {
|
|
const maxPages = numberOr(options.maxPages, 1);
|
|
const maxSamplesPerPage = numberOr(options.maxSamplesPerPage, 1);
|
|
const series = rawSeries
|
|
.map((item) => {
|
|
const points = arrayRecords(item.points)
|
|
.map((point) => ({
|
|
ts: stringOrNull(point.ts),
|
|
elapsedSeconds: numberOrNull(point.elapsedSeconds),
|
|
elapsedMinutes: numberOrNull(point.elapsedMinutes),
|
|
memoryMb: numberOrNull(point.memoryMb),
|
|
heapUsedMb: numberOrNull(point.heapUsedMb),
|
|
jsHeapUsedMb: numberOrNull(point.jsHeapUsedMb),
|
|
effectiveHeapUsedMb: numberOrNull(point.effectiveHeapUsedMb),
|
|
effectiveJsHeapUsedMb: numberOrNull(point.effectiveJsHeapUsedMb),
|
|
domNodes: numberOrNull(point.domNodes),
|
|
valuesRedacted: true,
|
|
}))
|
|
.filter((point) => point.ts !== null && point.memoryMb !== null);
|
|
const thinned = thinDashboardPoints(points, maxSamplesPerPage);
|
|
const maxMemoryMb = maxNumber(thinned.map((point) => numberOrNull(point.memoryMb)));
|
|
return {
|
|
key: stringOrNull(item.key) ?? stringOrNull(item.pageId) ?? "page",
|
|
pageRole: stringOrNull(item.pageRole),
|
|
pageId: stringOrNull(item.pageId),
|
|
label: stringOrNull(item.label) ?? stringOrNull(item.pageRole) ?? stringOrNull(item.pageId) ?? "page",
|
|
url: safeDashboardUrl(stringOrNull(item.url)),
|
|
sampleCount: thinned.length,
|
|
maxMemoryMb,
|
|
latestMemoryMb: thinned.length > 0 ? numberOrNull(thinned[thinned.length - 1].memoryMb) : null,
|
|
points: thinned,
|
|
valuesRedacted: true,
|
|
};
|
|
})
|
|
.filter((item) => item.sampleCount > 0)
|
|
.sort((a, b) => numberOr(b.maxMemoryMb, 0) - numberOr(a.maxMemoryMb, 0))
|
|
.slice(0, maxPages);
|
|
const sampleCount = series.reduce((sum, item) => sum + numberOr(item.sampleCount, 0), 0);
|
|
const maxSeries = series.reduce<Record<string, unknown> | null>((best, item) => best === null || numberOr(item.maxMemoryMb, 0) > numberOr(best.maxMemoryMb, 0) ? item : best, null);
|
|
return {
|
|
ok: series.length > 0,
|
|
source,
|
|
unit: "MB",
|
|
metric: "page-heap-used",
|
|
pageCount: series.length,
|
|
sampleCount,
|
|
maxMemoryMb: maxSeries === null ? null : numberOrNull(maxSeries.maxMemoryMb),
|
|
maxPageLabel: maxSeries === null ? null : stringOrNull(maxSeries.label),
|
|
pageSeries: series,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function dashboardPageMemoryPoint(page: Record<string, unknown>): Record<string, unknown> | null {
|
|
const effectiveMemory = record(page.effectiveMemory);
|
|
const heapUsage = record(page.heapUsage);
|
|
const performance = record(page.performance);
|
|
const metrics = record(performance.metrics);
|
|
const heapUsedMb = numberOrNull(effectiveMemory.heapUsedMb) ?? bytesToMbNullable(heapUsage.usedSize);
|
|
const jsHeapUsedMb = numberOrNull(effectiveMemory.jsHeapUsedMb) ?? bytesToMbNullable(metrics.JSHeapUsedSize);
|
|
const memoryMb = heapUsedMb ?? jsHeapUsedMb;
|
|
if (memoryMb === null) return null;
|
|
return {
|
|
memoryMb,
|
|
heapUsedMb,
|
|
jsHeapUsedMb,
|
|
effectiveHeapUsedMb: numberOrNull(effectiveMemory.effectiveHeapUsedMb),
|
|
effectiveJsHeapUsedMb: numberOrNull(effectiveMemory.effectiveJsHeapUsedMb),
|
|
domNodes: numberOrNull(effectiveMemory.domNodes) ?? numberOrNull(record(page.domCounters).nodes) ?? numberOrNull(metrics.Nodes),
|
|
};
|
|
}
|
|
|
|
function thinDashboardPoints(points: readonly Record<string, unknown>[], maxSamples: number): readonly Record<string, unknown>[] {
|
|
if (points.length <= maxSamples) return points;
|
|
const selected: Record<string, unknown>[] = [];
|
|
const last = points.length - 1;
|
|
const targetLast = Math.max(1, maxSamples - 1);
|
|
for (let index = 0; index < maxSamples; index += 1) {
|
|
selected.push(points[Math.round((index / targetLast) * last)]);
|
|
}
|
|
return selected;
|
|
}
|
|
|
|
function isSafeDashboardStateDir(value: string): boolean {
|
|
if (value.startsWith("/") || value.includes("\0")) return false;
|
|
const normalized = value.replace(/\\/gu, "/");
|
|
if (!normalized.startsWith(".state/")) return false;
|
|
return normalized.split("/").every((part) => part !== "..");
|
|
}
|
|
|
|
function bytesToMbNullable(value: unknown): number | null {
|
|
const bytes = numberOrNull(value);
|
|
return bytes === null ? null : Math.round((bytes / 1024 / 1024) * 100) / 100;
|
|
}
|
|
|
|
function maxNumber(values: readonly (number | null)[]): number | null {
|
|
const finite = values.filter((value): value is number => value !== null && Number.isFinite(value));
|
|
return finite.length === 0 ? null : Math.max(...finite);
|
|
}
|
|
|
|
function safeDashboardUrl(value: string | null): string | null {
|
|
if (value === null) return null;
|
|
return value.slice(0, 180);
|
|
}
|
|
|
|
function shortDashboardText(value: string, max: number): string {
|
|
return value.length <= max ? value : `${value.slice(0, max)}`;
|
|
}
|
|
|
|
function dashboardRunSummary(config: WebProbeSentinelServiceConfig, db: Database, row: Record<string, unknown>): Record<string, unknown> {
|
|
const id = stringOrNull(row.id);
|
|
const artifactSummary = readAnalysisArtifactSummaryForRun(config, row, 50);
|
|
const findings = visibleFindingsForRun(config, db, row, 50, artifactSummary);
|
|
const severityCounts = checkLevelCounts(config, findings);
|
|
const maxSeverity = maxSeverityFromCounts(severityCounts);
|
|
const findingTypeCount = visibleFindingTypeCount(row, findings, artifactSummary);
|
|
const findingSampleCount = Object.values(severityCounts).reduce((sum, value) => sum + numberOr(value, 0), 0);
|
|
const findingAlertSampleCount = alertSeveritySampleCount(severityCounts);
|
|
const stored = id === null ? {} : record(readMetadata(db, `run.report.${id}`));
|
|
const scenarioId = stringOrNull(row.scenario_id);
|
|
const timing = dashboardRunTiming(config, row, stored);
|
|
const reportJsonSha256 = stringOrNull(row.report_json_sha256) ?? stringOrNull(artifactSummary.reportJsonSha256);
|
|
return {
|
|
id,
|
|
runId: id,
|
|
scenario_id: scenarioId,
|
|
scenarioId,
|
|
status: stringOrNull(row.status),
|
|
node: stringOrNull(row.node) ?? config.node,
|
|
lane: stringOrNull(row.lane) ?? config.lane,
|
|
sentinelId: config.sentinelId,
|
|
observer_id: stringOrNull(row.observer_id),
|
|
observerId: stringOrNull(row.observer_id),
|
|
state_dir: stringOrNull(row.state_dir),
|
|
stateDir: stringOrNull(row.state_dir),
|
|
report_json_sha256: reportJsonSha256,
|
|
reportJsonSha256,
|
|
finding_count: findingTypeCount,
|
|
findingCount: findingTypeCount,
|
|
findingTypeCount,
|
|
findingSampleCount,
|
|
findingAlertSampleCount,
|
|
artifact_count: numberOr(row.artifact_count, 0),
|
|
artifactCount: numberOr(row.artifact_count, 0),
|
|
maintenance: numberOr(row.maintenance, 0) === 1,
|
|
created_at: stringOrNull(row.created_at),
|
|
createdAt: stringOrNull(row.created_at),
|
|
updated_at: stringOrNull(row.updated_at),
|
|
updatedAt: stringOrNull(row.updated_at),
|
|
interrupted_at: stringOrNull(row.interrupted_at),
|
|
interruptedAt: stringOrNull(row.interrupted_at),
|
|
startedAt: stringOrNull(timing.startedAt),
|
|
finishedAt: stringOrNull(timing.finishedAt),
|
|
durationMs: numberOrNull(timing.durationMs),
|
|
runDurationMs: numberOrNull(timing.durationMs),
|
|
durationMinutes: numberOrNull(timing.durationMinutes),
|
|
runDurationMinutes: numberOrNull(timing.durationMinutes),
|
|
durationSource: stringOrNull(timing.durationSource),
|
|
scenarioCadence: stringOrNull(timing.scenarioCadence),
|
|
scenarioCadenceMinutes: numberOrNull(timing.scenarioCadenceMinutes),
|
|
scenarioMaxRunMinutes: numberOrNull(timing.scenarioMaxRunMinutes),
|
|
schedulerIntervalMinutes: numberOrNull(timing.schedulerIntervalMinutes),
|
|
timing,
|
|
severityCounts,
|
|
maxSeverity,
|
|
traceability: runTraceability(config, row),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function dashboardRunTiming(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>, stored: Record<string, unknown>): Record<string, unknown> {
|
|
const scenarioId = stringOrNull(row.scenario_id);
|
|
const scenario = scenarioId === null ? null : config.scenarios.find((item) => stringOrNull(item.id) === scenarioId) ?? null;
|
|
const summary = record(stored.summary);
|
|
const summaryElapsedMs = numberOrNull(summary.elapsedMs);
|
|
const startedAt = stringOrNull(row.created_at);
|
|
const finishedAt = stringOrNull(row.interrupted_at) ?? stringOrNull(row.updated_at);
|
|
const timestampDurationMs = durationMsBetween(startedAt, finishedAt);
|
|
const durationMs = summaryElapsedMs ?? timestampDurationMs;
|
|
const durationSource = summaryElapsedMs !== null
|
|
? "report-summary-elapsedMs"
|
|
: timestampDurationMs !== null
|
|
? "sqlite-created-updated"
|
|
: "unknown";
|
|
const scenarioCadence = scenario === null ? null : stringOrNull(scenario.cadence);
|
|
const scenarioCadenceSeconds = durationStringSeconds(scenarioCadence);
|
|
const scenarioMaxRunSeconds = scenario === null ? null : numberOrNull(scenario.maxRunSeconds);
|
|
return {
|
|
startedAt,
|
|
finishedAt,
|
|
durationMs,
|
|
durationMinutes: minutesFromMs(durationMs),
|
|
durationSource,
|
|
scenarioCadence,
|
|
scenarioCadenceSeconds,
|
|
scenarioCadenceMinutes: minutesFromSeconds(scenarioCadenceSeconds),
|
|
scenarioMaxRunSeconds,
|
|
scenarioMaxRunMinutes: minutesFromSeconds(scenarioMaxRunSeconds),
|
|
schedulerIntervalMs: config.schedulerIntervalMs,
|
|
schedulerIntervalMinutes: minutesFromMs(config.schedulerIntervalMs),
|
|
sourceOfTruth: "sqlite-run-timestamps+report-summary+yaml-scenario-runtime",
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function alertSeveritySampleCount(counts: Record<string, number>): number {
|
|
return numberOr(counts.red, 0)
|
|
+ numberOr(counts.critical, 0)
|
|
+ numberOr(counts.error, 0)
|
|
+ numberOr(counts.warning, 0)
|
|
+ numberOr(counts.warn, 0)
|
|
+ numberOr(counts.amber, 0);
|
|
}
|
|
|
|
function dashboardOverallStatus(health: Record<string, unknown>, latestRun: Record<string, unknown> | null, severityCounts: Record<string, number>): string {
|
|
if (health.ok !== true) return "degraded";
|
|
const latestStatus = latestRun === null ? null : stringOrNull(latestRun.status);
|
|
if (latestStatus !== null && /blocked|failed|error|timeout/iu.test(latestStatus)) return "blocked";
|
|
if (Object.values(severityCounts).some((count) => count > 0)) return "warning";
|
|
return latestRun === null ? "idle" : "healthy";
|
|
}
|
|
|
|
function dashboardRunFilters(url: URL): Record<string, unknown> {
|
|
return {
|
|
status: optionalSearchParam(url, "status"),
|
|
scenario: optionalSearchParam(url, "scenario"),
|
|
severity: optionalSearchParam(url, "severity"),
|
|
maintenance: maintenanceFilter(url),
|
|
observerId: optionalSearchParam(url, "observerId"),
|
|
search: optionalSearchParam(url, "search"),
|
|
from: validIsoParam(url, "from"),
|
|
to: validIsoParam(url, "to"),
|
|
};
|
|
}
|
|
|
|
function dashboardFindingFilters(url: URL): Record<string, unknown> {
|
|
return {
|
|
severity: optionalSearchParam(url, "severity"),
|
|
code: optionalSearchParam(url, "code"),
|
|
scenario: optionalSearchParam(url, "scenario"),
|
|
search: optionalSearchParam(url, "search"),
|
|
window: optionalSearchParam(url, "window"),
|
|
since: windowSinceIso(optionalSearchParam(url, "window")) ?? validIsoParam(url, "from"),
|
|
to: validIsoParam(url, "to"),
|
|
};
|
|
}
|
|
|
|
function runWhereClause(config: WebProbeSentinelServiceConfig, filters: Record<string, unknown>): { readonly sql: string; readonly params: readonly (string | number)[] } {
|
|
const clauses: string[] = ["sentinel_id = ?", "node = ?", "lane = ?"];
|
|
const params: (string | number)[] = [config.sentinelId, config.node, config.lane];
|
|
const status = stringOrNull(filters.status);
|
|
if (status !== null) {
|
|
clauses.push("status = ?");
|
|
params.push(status);
|
|
}
|
|
const scenario = stringOrNull(filters.scenario);
|
|
if (scenario !== null) {
|
|
clauses.push("scenario_id = ?");
|
|
params.push(scenario);
|
|
}
|
|
const observerId = stringOrNull(filters.observerId);
|
|
if (observerId !== null) {
|
|
clauses.push("observer_id = ?");
|
|
params.push(observerId);
|
|
}
|
|
const severity = stringOrNull(filters.severity);
|
|
if (severity !== null) {
|
|
clauses.push("EXISTS (SELECT 1 FROM findings f WHERE f.run_id = runs.id AND f.severity = ?)");
|
|
params.push(severity);
|
|
}
|
|
if (typeof filters.maintenance === "boolean") {
|
|
clauses.push("maintenance = ?");
|
|
params.push(filters.maintenance ? 1 : 0);
|
|
}
|
|
const from = stringOrNull(filters.from);
|
|
if (from !== null) {
|
|
clauses.push("updated_at >= ?");
|
|
params.push(from);
|
|
}
|
|
const to = stringOrNull(filters.to);
|
|
if (to !== null) {
|
|
clauses.push("updated_at <= ?");
|
|
params.push(to);
|
|
}
|
|
const search = stringOrNull(filters.search);
|
|
if (search !== null) {
|
|
const pattern = `%${escapeSqlLike(search)}%`;
|
|
clauses.push("(id LIKE ? ESCAPE '\\' OR scenario_id LIKE ? ESCAPE '\\' OR observer_id LIKE ? ESCAPE '\\' OR state_dir LIKE ? ESCAPE '\\' OR report_json_sha256 LIKE ? ESCAPE '\\')");
|
|
params.push(pattern, pattern, pattern, pattern, pattern);
|
|
}
|
|
return { sql: clauses.length === 0 ? "" : `WHERE ${clauses.join(" AND ")}`, params };
|
|
}
|
|
|
|
function findingWhereClause(config: WebProbeSentinelServiceConfig, filters: Record<string, unknown>): { readonly sql: string; readonly params: readonly (string | number)[] } {
|
|
const clauses: string[] = ["r.sentinel_id = ?", "r.node = ?", "r.lane = ?", "f.sentinel_id = ?"];
|
|
const params: (string | number)[] = [config.sentinelId, config.node, config.lane, config.sentinelId];
|
|
const code = stringOrNull(filters.code);
|
|
if (code !== null) {
|
|
clauses.push("f.finding_id = ?");
|
|
params.push(code);
|
|
}
|
|
const scenario = stringOrNull(filters.scenario);
|
|
if (scenario !== null) {
|
|
clauses.push("r.scenario_id = ?");
|
|
params.push(scenario);
|
|
}
|
|
const since = stringOrNull(filters.since);
|
|
if (since !== null) {
|
|
clauses.push("f.created_at >= ?");
|
|
params.push(since);
|
|
}
|
|
const to = stringOrNull(filters.to);
|
|
if (to !== null) {
|
|
clauses.push("f.created_at <= ?");
|
|
params.push(to);
|
|
}
|
|
const search = stringOrNull(filters.search);
|
|
if (search !== null) {
|
|
const pattern = `%${escapeSqlLike(search)}%`;
|
|
clauses.push("(f.finding_id LIKE ? ESCAPE '\\' OR f.summary LIKE ? ESCAPE '\\')");
|
|
params.push(pattern, pattern);
|
|
}
|
|
return { sql: clauses.length === 0 ? "" : `WHERE ${clauses.join(" AND ")}`, params };
|
|
}
|
|
|
|
function dashboardRunSort(url: URL): { readonly name: string; readonly direction: "asc" | "desc"; readonly sql: string } {
|
|
const requested = optionalSearchParam(url, "sort") ?? "updated";
|
|
const direction = optionalSearchParam(url, "direction") === "asc" ? "asc" : "desc";
|
|
const column = requested === "created" ? "created_at"
|
|
: requested === "findings" ? "finding_count"
|
|
: requested === "severity" ? severityRankSql("(SELECT severity FROM findings f WHERE f.run_id = runs.id ORDER BY " + severityRankSql("f.severity") + " DESC LIMIT 1)")
|
|
: "updated_at";
|
|
return { name: requested, direction, sql: `${column} ${direction.toUpperCase()}, id ${direction.toUpperCase()}` };
|
|
}
|
|
|
|
function dashboardPage(url: URL, config: WebProbeSentinelServiceConfig): { readonly limit: number; readonly offset: number } {
|
|
const limit = Math.min(numberParam(url, "limit", dashboardPageSize(config)), dashboardMaxPageSize(config));
|
|
const rawCursor = url.searchParams.get("cursor");
|
|
const offset = rawCursor === null ? 0 : Math.max(0, Math.min(100_000, Number.isInteger(Number(rawCursor)) ? Number(rawCursor) : 0));
|
|
return { limit, offset };
|
|
}
|
|
|
|
function dashboardPageSize(config: WebProbeSentinelServiceConfig): number {
|
|
return Math.max(1, Math.min(dashboardMaxPageSize(config), numberOr(config.reportViews.pageSize, 20)));
|
|
}
|
|
|
|
function dashboardMaxPageSize(config: WebProbeSentinelServiceConfig): number {
|
|
return Math.max(1, Math.min(200, numberOr(config.reportViews.maxPageSize, 100)));
|
|
}
|
|
|
|
function readRunRow(config: WebProbeSentinelServiceConfig, db: Database, runId: string): Record<string, unknown> | null {
|
|
return db.query("SELECT * FROM runs WHERE id = ? AND sentinel_id = ? AND node = ? AND lane = ?")
|
|
.get(runId, config.sentinelId, config.node, config.lane) as Record<string, unknown> | null;
|
|
}
|
|
|
|
function findingsForRun(config: WebProbeSentinelServiceConfig, db: Database, runId: string, limit: number): readonly Record<string, unknown>[] {
|
|
const rows = db.query("SELECT finding_id, severity, count, summary, report_json_sha256, created_at FROM findings WHERE run_id = ? AND sentinel_id = ? ORDER BY created_at DESC LIMIT ?")
|
|
.all(runId, config.sentinelId, limit) as Record<string, unknown>[];
|
|
return rows.map((row) => enrichFindingRowWithStoredDetail(config, db, runId, row));
|
|
}
|
|
|
|
function visibleFindingsForRun(
|
|
config: WebProbeSentinelServiceConfig,
|
|
db: Database,
|
|
row: Record<string, unknown>,
|
|
limit: number,
|
|
artifactSummary: Record<string, unknown> = readAnalysisArtifactSummaryForRun(config, row, limit),
|
|
): readonly Record<string, unknown>[] {
|
|
const runId = stringOrNull(row.id);
|
|
const storedFindings = runId === null ? [] : findingsForRun(config, db, runId, limit);
|
|
const artifactFindings = arrayRecords(artifactSummary.findings)
|
|
.slice(0, limit)
|
|
.map((item) => enrichFindingWithCheck(config, compactStoredFinding(item)));
|
|
return mergeVisibleFindings(config, storedFindings, artifactFindings, limit);
|
|
}
|
|
|
|
function mergeVisibleFindings(config: WebProbeSentinelServiceConfig, primary: readonly Record<string, unknown>[], artifact: readonly Record<string, unknown>[], limit: number): readonly Record<string, unknown>[] {
|
|
const merged: Record<string, unknown>[] = [];
|
|
const seen = new Set<string>();
|
|
for (const item of [...primary, ...artifact]) {
|
|
const id = canonicalFindingIdentity(config, item);
|
|
const key = id ?? `${JSON.stringify(item).slice(0, 160)}:${stringOrNull(item.severity) ?? stringOrNull(item.level) ?? ""}`;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
merged.push(item);
|
|
if (merged.length >= limit) break;
|
|
}
|
|
return merged;
|
|
}
|
|
|
|
function canonicalFindingIdentity(config: WebProbeSentinelServiceConfig, item: Record<string, unknown>): string | null {
|
|
const check = record(item.check);
|
|
const explicitCheckCode = stringOrNull(item.checkCode) ?? stringOrNull(check.code);
|
|
if (explicitCheckCode !== null) return explicitCheckCode;
|
|
const catalog = checkCatalogById(config);
|
|
const explicitCheckId = stringOrNull(item.checkId) ?? stringOrNull(check.id);
|
|
if (explicitCheckId !== null) {
|
|
const entry = catalog.get(explicitCheckId);
|
|
return stringOrNull(entry?.code) ?? stringOrNull(entry?.id) ?? explicitCheckId;
|
|
}
|
|
const findingId = findingIdentity(item);
|
|
if (findingId === null) return null;
|
|
const entry = catalog.get(findingId);
|
|
return stringOrNull(entry?.code) ?? stringOrNull(entry?.id) ?? findingId;
|
|
}
|
|
|
|
function findingIdentity(item: Record<string, unknown>): string | null {
|
|
return stringOrNull(item.finding_id)
|
|
?? stringOrNull(item.findingId)
|
|
?? stringOrNull(item.id)
|
|
?? stringOrNull(item.kind)
|
|
?? stringOrNull(item.code);
|
|
}
|
|
|
|
function visibleFindingTypeCount(row: Record<string, unknown>, findings: readonly Record<string, unknown>[], artifactSummary: Record<string, unknown>): number {
|
|
return Math.max(
|
|
numberOr(row.finding_count, 0),
|
|
numberOr(artifactSummary.findingCount, 0),
|
|
findings.length,
|
|
);
|
|
}
|
|
|
|
function readAnalysisArtifactSummaryForRun(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>, limit: number): Record<string, unknown> {
|
|
const stateDir = stringOrNull(row.state_dir) ?? stringOrNull(row.stateDir);
|
|
if (stateDir === null) return { ok: false, reason: "state-dir-missing", findings: [], valuesRedacted: true };
|
|
if (!isSafeDashboardStateDir(stateDir)) return { ok: false, reason: "unsafe-state-dir", stateDir, findings: [], valuesRedacted: true };
|
|
const reportJsonPath = join(config.stateRoot, stateDir, "analysis", "report.json");
|
|
const reportMdPath = join(config.stateRoot, stateDir, "analysis", "report.md");
|
|
const jsonBuffer = readFileBuffer(reportJsonPath);
|
|
const mdBuffer = readFileBuffer(reportMdPath);
|
|
let parsed: unknown = null;
|
|
if (jsonBuffer !== null) {
|
|
try {
|
|
parsed = JSON.parse(jsonBuffer.toString("utf8")) as unknown;
|
|
} catch {
|
|
parsed = null;
|
|
}
|
|
}
|
|
const report = record(parsed);
|
|
const reportFindings = arrayRecords(report.findings);
|
|
const archiveFindings = arrayRecords(record(report.archiveSummary).redFindings);
|
|
const findings = (reportFindings.length > 0 ? reportFindings : archiveFindings)
|
|
.slice(0, limit)
|
|
.map(compactStoredFinding);
|
|
return {
|
|
ok: parsed !== null,
|
|
reason: parsed === null ? "analysis-report-json-missing-or-invalid" : null,
|
|
stateDir,
|
|
reportJsonPath,
|
|
reportJsonSha256: sha256Buffer(jsonBuffer),
|
|
reportMdSha256: sha256Buffer(mdBuffer),
|
|
findingCount: numberOr(report.findingCount, findings.length),
|
|
findings,
|
|
counts: record(report.counts),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function readFileBuffer(path: string): Buffer | null {
|
|
try {
|
|
return readFileSync(path);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function sha256Buffer(value: Buffer | null): string | null {
|
|
return value === null ? null : `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
}
|
|
|
|
function enrichFindingRowWithStoredDetail(config: WebProbeSentinelServiceConfig, db: Database, runId: string, row: Record<string, unknown>): Record<string, unknown> {
|
|
const detail = storedFindingDetailForRow(db, row, runId);
|
|
return enrichFindingWithCheck(config, {
|
|
...row,
|
|
code: stringOrNull(row.finding_id),
|
|
findingId: stringOrNull(row.finding_id),
|
|
rootCause: stringOrNull(detail?.rootCause),
|
|
rootCauseStatus: stringOrNull(detail?.rootCauseStatus),
|
|
rootCauseConfidence: stringOrNull(detail?.rootCauseConfidence),
|
|
nextAction: stringOrNull(detail?.nextAction),
|
|
evidenceSummary: stringOrNull(detail?.evidenceSummary),
|
|
rootCauseSignals: record(detail?.rootCauseSignals),
|
|
timingSourceOfTruth: stringOrNull(detail?.timingSourceOfTruth),
|
|
timingStatus: stringOrNull(detail?.timingStatus),
|
|
timingAlert: detail?.timingAlert === true,
|
|
valuesRedacted: true,
|
|
});
|
|
}
|
|
|
|
function storedFindingDetailForRow(db: Database, row: Record<string, unknown>, runId: string | null): Record<string, unknown> | null {
|
|
if (runId === null) return null;
|
|
const stored = readMetadata(db, `run.report.${runId}`) ?? {};
|
|
const details = arrayRecords(stored.findings);
|
|
if (details.length === 0) return null;
|
|
const findingId = stringOrNull(row.finding_id) ?? stringOrNull(row.findingId) ?? stringOrNull(row.code);
|
|
const severity = stringOrNull(row.severity);
|
|
const sameIdDetails = details.filter((item) => {
|
|
const itemId = stringOrNull(item.finding_id) ?? stringOrNull(item.findingId) ?? stringOrNull(item.id) ?? stringOrNull(item.code);
|
|
return itemId === findingId;
|
|
});
|
|
if (sameIdDetails.length === 0) return null;
|
|
return sameIdDetails.find((item) => {
|
|
const itemSeverity = normalizeCheckLevel(stringOrNull(item.severity) ?? stringOrNull(item.level));
|
|
return severity === null || itemSeverity === null || itemSeverity === severity;
|
|
}) ?? sameIdDetails[0] ?? null;
|
|
}
|
|
|
|
function compactStoredFinding(value: unknown): Record<string, unknown> {
|
|
const item = record(value);
|
|
const findingId = stringOrNull(item.id) ?? stringOrNull(item.kind) ?? stringOrNull(item.code) ?? "finding";
|
|
const severity = stringOrNull(item.severity) ?? stringOrNull(item.level) ?? "unknown";
|
|
const summary = stringOrNull(item.summary) ?? stringOrNull(item.message) ?? findingId;
|
|
return {
|
|
id: findingId,
|
|
finding_id: findingId,
|
|
findingId,
|
|
code: findingId,
|
|
severity,
|
|
count: numberOr(item.count, numberOr(item.sampleCount, 1)),
|
|
summary: summary.slice(0, 500),
|
|
rootCause: stringOrNull(item.rootCause),
|
|
rootCauseStatus: stringOrNull(item.rootCauseStatus),
|
|
rootCauseConfidence: stringOrNull(item.rootCauseConfidence),
|
|
nextAction: stringOrNull(item.nextAction),
|
|
evidenceSummary: stringOrNull(item.evidenceSummary) ?? compactFindingEvidenceSummary(item.evidence),
|
|
rootCauseSignals: compactFindingRootCauseSignals(item.rootCauseSignals),
|
|
timingSourceOfTruth: stringOrNull(item.timingSourceOfTruth) ?? stringOrNull(item.expectedElapsedSource) ?? stringOrNull(item.evidenceKind),
|
|
timingStatus: stringOrNull(item.timingStatus),
|
|
timingAlert: item.timingAlert === true,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactFindingRootCauseSignals(value: unknown): Record<string, unknown> | null {
|
|
const item = record(value);
|
|
const keys = [
|
|
"sessionListReadCount",
|
|
"traceEventsReadCount",
|
|
"webPerformanceBeaconFailureCount",
|
|
"eventSourceFailureCount",
|
|
"requestFailedCount",
|
|
"httpErrorCount",
|
|
"consoleAlertCount",
|
|
"requestfailedTop",
|
|
"httpStatusTop",
|
|
];
|
|
const compact: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
const raw = item[key];
|
|
if (raw === null || raw === undefined) continue;
|
|
compact[key] = Array.isArray(raw) ? raw.slice(0, 8).map(record) : raw;
|
|
}
|
|
return Object.keys(compact).length === 0 ? null : { ...compact, valuesRedacted: true };
|
|
}
|
|
|
|
function compactFindingEvidenceSummary(value: unknown): string | null {
|
|
if (value === null || value === undefined) return null;
|
|
if (typeof value === "string") return value.slice(0, 240);
|
|
const item = record(value);
|
|
const keys = [
|
|
"http404Count",
|
|
"responseErrorCount",
|
|
"requestFailedCount",
|
|
"statuses",
|
|
"afterProjectedSeqs",
|
|
"sinceSeqs",
|
|
"traceIds",
|
|
"maxFallbackRatio",
|
|
"maxFallbackTitleCount",
|
|
"overThresholdSampleCount",
|
|
"majorityFallbackSampleCount",
|
|
];
|
|
const compact: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
const raw = item[key];
|
|
if (raw === null || raw === undefined) continue;
|
|
compact[key] = Array.isArray(raw) ? raw.slice(0, 6) : raw;
|
|
}
|
|
const text = Object.keys(compact).length > 0 ? JSON.stringify(compact) : JSON.stringify(item).slice(0, 240);
|
|
return text.length > 240 ? `${text.slice(0, 239)}…` : text;
|
|
}
|
|
|
|
function enrichFindingWithCheck(config: WebProbeSentinelServiceConfig, value: Record<string, unknown>): Record<string, unknown> {
|
|
const rawSeverity = stringOrNull(value.severity) ?? stringOrNull(value.level);
|
|
const check = checkForFinding(config, value);
|
|
const checkLevel = stringOrNull(check.level) ?? normalizeCheckLevel(rawSeverity) ?? "unknown";
|
|
return {
|
|
...value,
|
|
rawSeverity,
|
|
severity: checkLevel,
|
|
level: checkLevel,
|
|
check,
|
|
checkId: stringOrNull(check.id),
|
|
checkCode: stringOrNull(check.code),
|
|
checkLevel,
|
|
checkTitleZh: stringOrNull(check.titleZh),
|
|
checkSummaryZh: stringOrNull(check.summaryZh),
|
|
checkActionZh: stringOrNull(check.actionZh),
|
|
checkRegistered: check.registered === true,
|
|
blocking: value.blocking === true || check.blocking === true,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function checkForFinding(config: WebProbeSentinelServiceConfig, value: Record<string, unknown>): Record<string, unknown> {
|
|
const findingId = stringOrNull(value.finding_id) ?? stringOrNull(value.findingId) ?? stringOrNull(value.id) ?? stringOrNull(value.kind) ?? stringOrNull(value.code) ?? "unknown-check";
|
|
const entry = checkCatalogById(config).get(findingId) ?? null;
|
|
const rawLevel = normalizeCheckLevel(stringOrNull(value.severity) ?? stringOrNull(value.level));
|
|
if (entry === null) {
|
|
return {
|
|
id: findingId,
|
|
code: null,
|
|
level: rawLevel ?? "unknown",
|
|
titleZh: "未登记监测项",
|
|
summaryZh: "该监测项未在 YAML 中登记，配置校验需要补齐中文说明。",
|
|
actionZh: "补齐 YAML 监测项说明后重新验证。",
|
|
blocking: value.blocking === true,
|
|
order: 10000,
|
|
registered: false,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
return {
|
|
id: stringOrNull(entry.id) ?? findingId,
|
|
code: stringOrNull(entry.code),
|
|
level: normalizeCheckLevel(stringOrNull(entry.level)) ?? rawLevel ?? "unknown",
|
|
titleZh: stringOrNull(entry.titleZh) ?? findingId,
|
|
summaryZh: stringOrNull(entry.summaryZh) ?? "已记录监测项详情。",
|
|
actionZh: stringOrNull(entry.actionZh) ?? "查看详情后处理。",
|
|
blocking: entry.blocking === true,
|
|
order: numberOr(entry.order, 10000),
|
|
registered: true,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function checkCatalogById(config: WebProbeSentinelServiceConfig): Map<string, Record<string, unknown>> {
|
|
const catalog = record(config.reportViews.checkCatalog);
|
|
const items = arrayRecords(catalog.items);
|
|
const result = new Map<string, Record<string, unknown>>();
|
|
for (const item of items) {
|
|
const id = stringOrNull(item.id);
|
|
if (id !== null) result.set(id, item);
|
|
const aliases = Array.isArray(item.aliases) ? item.aliases : [];
|
|
for (const alias of aliases) {
|
|
if (typeof alias === "string" && alias.length > 0) result.set(alias, item);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function checkLevelCounts(config: WebProbeSentinelServiceConfig, rows: readonly Record<string, unknown>[]): Record<string, number> {
|
|
const counts: Record<string, number> = {};
|
|
for (const row of rows) {
|
|
const check = checkForFinding(config, row);
|
|
const level = stringOrNull(check.level) ?? normalizeCheckLevel(stringOrNull(row.severity)) ?? "unknown";
|
|
counts[level] = (counts[level] ?? 0) + numberOr(row.count, 1);
|
|
}
|
|
return counts;
|
|
}
|
|
|
|
function compareCheckFindingRows(left: Record<string, unknown>, right: Record<string, unknown>): number {
|
|
const severity = severityRank(stringOrNull(right.checkLevel) ?? stringOrNull(right.severity)) - severityRank(stringOrNull(left.checkLevel) ?? stringOrNull(left.severity));
|
|
if (severity !== 0) return severity;
|
|
const order = numberOr(record(left.check).order, 10000) - numberOr(record(right.check).order, 10000);
|
|
if (order !== 0) return order;
|
|
const rightTime = Date.parse(stringOrNull(right.latestAt) ?? "");
|
|
const leftTime = Date.parse(stringOrNull(left.latestAt) ?? "");
|
|
return (Number.isFinite(rightTime) ? rightTime : 0) - (Number.isFinite(leftTime) ? leftTime : 0);
|
|
}
|
|
|
|
function normalizeCheckLevel(value: string | null): string | null {
|
|
const normalized = (value ?? "").toLowerCase();
|
|
if (["critical", "fatal"].includes(normalized)) return "critical";
|
|
if (["red", "error", "failed", "blocked"].includes(normalized)) return "error";
|
|
if (["warning", "warn", "amber", "yellow"].includes(normalized)) return "warning";
|
|
if (["info", "notice"].includes(normalized)) return "info";
|
|
return null;
|
|
}
|
|
|
|
function globalSeverityCounts(config: WebProbeSentinelServiceConfig, db: Database): Record<string, number> {
|
|
const rows = db.query("SELECT finding_id, severity, count FROM findings WHERE sentinel_id = ?").all(config.sentinelId) as Record<string, unknown>[];
|
|
return checkLevelCounts(config, rows);
|
|
}
|
|
|
|
function severityCountsForRun(config: WebProbeSentinelServiceConfig, db: Database, runId: string): Record<string, number> {
|
|
const rows = db.query("SELECT finding_id, severity, count FROM findings WHERE run_id = ? AND sentinel_id = ?").all(runId, config.sentinelId) as Record<string, unknown>[];
|
|
return checkLevelCounts(config, rows);
|
|
}
|
|
|
|
function latestRunForFinding(config: WebProbeSentinelServiceConfig, db: Database, row: Record<string, unknown>): Record<string, unknown> | null {
|
|
return db.query(`
|
|
SELECT r.*
|
|
FROM findings f
|
|
JOIN runs r ON r.id = f.run_id
|
|
WHERE f.finding_id = ? AND r.scenario_id = ? AND r.sentinel_id = ? AND r.node = ? AND r.lane = ? AND f.sentinel_id = ?
|
|
ORDER BY f.created_at DESC
|
|
LIMIT 1
|
|
`).get(stringOrNull(row.finding_id), stringOrNull(row.scenario_id), config.sentinelId, config.node, config.lane, config.sentinelId) as Record<string, unknown> | null;
|
|
}
|
|
|
|
function runTraceability(config: WebProbeSentinelServiceConfig, row: Record<string, unknown>): Record<string, unknown> {
|
|
return {
|
|
source: "sqlite-index+run-report-metadata",
|
|
node: stringOrNull(row.node) ?? config.node,
|
|
lane: stringOrNull(row.lane) ?? config.lane,
|
|
sentinelId: stringOrNull(row.sentinel_id) ?? config.sentinelId,
|
|
runId: stringOrNull(row.id),
|
|
observerId: stringOrNull(row.observer_id),
|
|
stateDir: stringOrNull(row.state_dir),
|
|
reportJsonSha256: stringOrNull(row.report_json_sha256),
|
|
stateRoot: config.stateRoot,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function compactArtifactRef(value: unknown): Record<string, unknown> | null {
|
|
const source = record(value);
|
|
if (Object.keys(source).length === 0) return null;
|
|
return {
|
|
path: stringOrNull(source.path) ?? stringOrNull(source.file) ?? stringOrNull(source.screenshotPath),
|
|
sha256: stringOrNull(source.sha256) ?? stringOrNull(source.hash),
|
|
byteCount: typeof source.byteCount === "number" ? source.byteCount : null,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function boundedText(value: string, maxBytes: number): { readonly text: string; readonly bytes: number; readonly truncated: boolean } {
|
|
const bytes = Buffer.byteLength(value, "utf8");
|
|
if (bytes <= maxBytes) return { text: value, bytes, truncated: false };
|
|
return { text: Buffer.from(value, "utf8").subarray(0, maxBytes).toString("utf8"), bytes, truncated: true };
|
|
}
|
|
|
|
function finalResponseBlock(text: string): Record<string, unknown> {
|
|
const index = text.toLowerCase().lastIndexOf("final response");
|
|
const body = index === -1 ? "" : text.slice(index + "final response".length).replace(/^[\s=\-:]+/u, "").trim();
|
|
const normalized = body.length === 0 ? "(空内容)" : body;
|
|
return {
|
|
label: "Final Response",
|
|
empty: body.length === 0 || body === "(空内容)",
|
|
text: normalized,
|
|
byteCount: Buffer.byteLength(normalized, "utf8"),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function optionalSearchParam(url: URL, key: string): string | null {
|
|
const raw = url.searchParams.get(key);
|
|
if (raw === null) return null;
|
|
const trimmed = raw.trim();
|
|
return trimmed.length === 0 ? null : trimmed.slice(0, 200);
|
|
}
|
|
|
|
function validIsoParam(url: URL, key: string): string | null {
|
|
const value = optionalSearchParam(url, key);
|
|
if (value === null) return null;
|
|
return Number.isFinite(Date.parse(value)) ? new Date(value).toISOString() : null;
|
|
}
|
|
|
|
function maintenanceFilter(url: URL): boolean | null {
|
|
const raw = optionalSearchParam(url, "maintenance");
|
|
if (raw === null) return null;
|
|
if (/^(1|true|yes)$/iu.test(raw)) return true;
|
|
if (/^(0|false|no)$/iu.test(raw)) return false;
|
|
return null;
|
|
}
|
|
|
|
function windowSinceIso(value: string | null): string | null {
|
|
if (value === null) return null;
|
|
const match = /^(\d+)(m|h|d)$/iu.exec(value);
|
|
if (match === null) return null;
|
|
const amount = Number(match[1]);
|
|
const unit = match[2].toLowerCase();
|
|
const ms = unit === "m" ? amount * 60_000 : unit === "h" ? amount * 3_600_000 : amount * 86_400_000;
|
|
return new Date(Date.now() - ms).toISOString();
|
|
}
|
|
|
|
function escapeSqlLike(value: string): string {
|
|
return value.replace(/[\\%_]/gu, (char) => `\\${char}`);
|
|
}
|
|
|
|
function ageSeconds(iso: string): number | null {
|
|
const parsed = Date.parse(iso);
|
|
return Number.isFinite(parsed) ? Math.max(0, Math.round((Date.now() - parsed) / 1000)) : null;
|
|
}
|
|
|
|
function durationMsBetween(startIso: string | null, endIso: string | null): number | null {
|
|
if (startIso === null || endIso === null) return null;
|
|
const start = Date.parse(startIso);
|
|
const end = Date.parse(endIso);
|
|
if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null;
|
|
return end - start;
|
|
}
|
|
|
|
function durationStringSeconds(value: string | null): number | null {
|
|
if (value === null) return null;
|
|
const match = /^(\d+(?:\.\d+)?)(ms|s|m|h|d)$/iu.exec(value.trim());
|
|
if (match === null) return null;
|
|
const amount = Number(match[1]);
|
|
if (!Number.isFinite(amount)) return null;
|
|
const unit = match[2].toLowerCase();
|
|
const seconds = unit === "ms" ? amount / 1000
|
|
: unit === "s" ? amount
|
|
: unit === "m" ? amount * 60
|
|
: unit === "h" ? amount * 3600
|
|
: amount * 86400;
|
|
return Math.round(seconds * 1000) / 1000;
|
|
}
|
|
|
|
function minutesFromMs(value: unknown): number | null {
|
|
const ms = numberOrNull(value);
|
|
return ms === null ? null : roundMinutes(ms / 60_000);
|
|
}
|
|
|
|
function minutesFromSeconds(value: unknown): number | null {
|
|
const seconds = numberOrNull(value);
|
|
return seconds === null ? null : roundMinutes(seconds / 60);
|
|
}
|
|
|
|
function roundMinutes(value: number): number {
|
|
return Math.round(value * 100) / 100;
|
|
}
|
|
|
|
function maxSeverityFromCounts(counts: Record<string, number>): string | null {
|
|
let best: string | null = null;
|
|
for (const [severity, count] of Object.entries(counts)) {
|
|
if (count <= 0) continue;
|
|
if (best === null || severityRank(severity) > severityRank(best)) best = severity;
|
|
}
|
|
return best;
|
|
}
|
|
|
|
function severityRank(value: string | null): number {
|
|
const normalized = (value ?? "").toLowerCase();
|
|
if (/critical|red|fatal/iu.test(normalized)) return 4;
|
|
if (/error|failed|blocked/iu.test(normalized)) return 3;
|
|
if (/warn|amber|yellow/iu.test(normalized)) return 2;
|
|
if (/info|notice/iu.test(normalized)) return 1;
|
|
return 0;
|
|
}
|
|
|
|
function severityRankSql(expression: string): string {
|
|
return `CASE LOWER(COALESCE(${expression}, '')) WHEN 'critical' THEN 4 WHEN 'red' THEN 4 WHEN 'fatal' THEN 4 WHEN 'error' THEN 3 WHEN 'failed' THEN 3 WHEN 'blocked' THEN 3 WHEN 'warning' THEN 2 WHEN 'warn' THEN 2 WHEN 'amber' THEN 2 WHEN 'yellow' THEN 2 WHEN 'info' THEN 1 WHEN 'notice' THEN 1 ELSE 0 END`;
|
|
}
|
|
|
|
function countWhere(config: WebProbeSentinelServiceConfig, db: Database, where: string): number {
|
|
const row = db.query(`SELECT COUNT(*) AS count FROM runs WHERE sentinel_id = ? AND node = ? AND lane = ? AND ${where}`)
|
|
.get(config.sentinelId, config.node, config.lane) as { count?: number } | null;
|
|
return Number(row?.count ?? 0);
|
|
}
|
|
|
|
function sumColumn(config: WebProbeSentinelServiceConfig, db: Database, table: "runs", column: "finding_count"): number {
|
|
const row = db.query(`SELECT COALESCE(SUM(${column}), 0) AS total FROM ${table} WHERE sentinel_id = ? AND node = ? AND lane = ?`)
|
|
.get(config.sentinelId, config.node, config.lane) as { total?: number } | null;
|
|
return Number(row?.total ?? 0);
|
|
}
|
|
|
|
function firstEnabledScenarioId(config: WebProbeSentinelServiceConfig): string | null {
|
|
const scenario = config.scenarios.find((item) => boolAt(item, "enabled"));
|
|
return scenario === undefined ? null : stringAt(scenario, "id");
|
|
}
|
|
|
|
async function triggerQuickVerifyCronJob(config: WebProbeSentinelServiceConfig, input: Record<string, unknown>): Promise<Record<string, unknown>> {
|
|
const namespace = stringAt(config.runtime, "namespace");
|
|
const cronJobName = sentinelCadenceCronJobName(config);
|
|
const scenarioId = stringOrNull(input.scenarioId)
|
|
?? stringAtNullable(config.cicd, "targetValidation.scenarioId")
|
|
?? firstEnabledScenarioId(config);
|
|
const reason = stringOrNull(input.reason) ?? "manual-web-ui";
|
|
const source = stringOrNull(input.source) ?? "monitor-web";
|
|
if (scenarioId === null) {
|
|
return { ok: false, status: "blocked", reason: "scenario-missing", namespace, cronJobName, valuesRedacted: true };
|
|
}
|
|
|
|
const activeJobs = await k8sApiJson(config, "GET", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs?labelSelector=${encodeURIComponent(`unidesk.ai/web-probe-sentinel-id=${config.sentinelId}`)}`, null);
|
|
if (activeJobs.ok === true) {
|
|
const active = k8sJobItems(activeJobs.bodyJson).filter((job) => numberAtNullable(job, "status.active") > 0);
|
|
if (active.length > 0) {
|
|
return {
|
|
ok: true,
|
|
status: "already-active",
|
|
mutation: false,
|
|
reason: "sentinel-quick-verify-job-already-active",
|
|
namespace,
|
|
cronJobName,
|
|
jobName: stringAtNullable(active[0], "metadata.name"),
|
|
activeJobCount: active.length,
|
|
scenarioId,
|
|
source,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
}
|
|
|
|
const cronJob = await k8sApiJson(config, "GET", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/cronjobs/${encodeURIComponent(cronJobName)}`, null);
|
|
if (cronJob.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
status: "blocked",
|
|
reason: "cadence-cronjob-read-failed",
|
|
namespace,
|
|
cronJobName,
|
|
scenarioId,
|
|
source,
|
|
k8s: compactK8sApiResult(cronJob),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
const now = nowIso();
|
|
const jobName = manualQuickVerifyJobName(config, now);
|
|
const job = jobFromCronJobTemplate(config, record(cronJob.bodyJson), jobName, now, reason, source);
|
|
const created = await k8sApiJson(config, "POST", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs`, job);
|
|
if (created.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
status: "blocked",
|
|
reason: "manual-job-create-failed",
|
|
namespace,
|
|
cronJobName,
|
|
jobName,
|
|
scenarioId,
|
|
source,
|
|
k8s: compactK8sApiResult(created),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
return {
|
|
ok: true,
|
|
status: "triggered",
|
|
mutation: true,
|
|
namespace,
|
|
cronJobName,
|
|
jobName,
|
|
scenarioId,
|
|
source,
|
|
createdAt: now,
|
|
k8s: compactK8sApiResult(created),
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function jobFromCronJobTemplate(
|
|
config: WebProbeSentinelServiceConfig,
|
|
cronJob: Record<string, unknown>,
|
|
jobName: string,
|
|
triggeredAt: string,
|
|
reason: string,
|
|
source: string,
|
|
): Record<string, unknown> {
|
|
const namespace = stringAt(config.runtime, "namespace");
|
|
const jobTemplate = record(valueAtPath(cronJob, "spec.jobTemplate"));
|
|
const jobSpec = cloneJson(record(jobTemplate.spec));
|
|
if (Object.keys(jobSpec).length === 0) throw new Error("cadence CronJob jobTemplate.spec missing");
|
|
const templateMetadata = record(valueAtPath(jobSpec, "template.metadata"));
|
|
const templateLabels = labelRecord(templateMetadata.labels);
|
|
const cronJobLabels = labelRecord(record(cronJob.metadata).labels);
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name: jobName,
|
|
namespace,
|
|
labels: {
|
|
...cronJobLabels,
|
|
...templateLabels,
|
|
"unidesk.ai/manual-trigger": "true",
|
|
"unidesk.ai/trigger-source": safeKubernetesLabelValue(source),
|
|
},
|
|
annotations: {
|
|
"unidesk.ai/trigger-source": source,
|
|
"unidesk.ai/trigger-reason": reason,
|
|
"unidesk.ai/triggered-at": triggeredAt,
|
|
"unidesk.ai/cronjob-name": stringAt(cronJob, "metadata.name"),
|
|
},
|
|
},
|
|
spec: jobSpec,
|
|
};
|
|
}
|
|
|
|
async function k8sApiJson(config: WebProbeSentinelServiceConfig, method: "GET" | "POST", path: string, body: Record<string, unknown> | null): Promise<Record<string, unknown>> {
|
|
const host = stringAtNullable(config.runtime, "kubernetesApi.endpoint.host") ?? process.env.KUBERNETES_SERVICE_HOST;
|
|
const port = numberAtNullable(config.runtime, "kubernetesApi.endpoint.port") || Number(process.env.KUBERNETES_SERVICE_PORT_HTTPS ?? process.env.KUBERNETES_SERVICE_PORT ?? 443);
|
|
if (typeof host !== "string" || host.length === 0) return { ok: false, status: "blocked", reason: "kubernetes-service-env-missing", valuesRedacted: true };
|
|
const tokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token";
|
|
const caPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt";
|
|
if (!existsSync(tokenPath)) return { ok: false, status: "blocked", reason: "serviceaccount-token-missing", valuesRedacted: true };
|
|
const bodyText = body === null ? "" : JSON.stringify(body);
|
|
const token = readFileSync(tokenPath, "utf8").trim();
|
|
const ca = existsSync(caPath) ? readFileSync(caPath) : undefined;
|
|
return await new Promise((resolve) => {
|
|
const req = httpsRequest({
|
|
host,
|
|
port,
|
|
path,
|
|
method,
|
|
ca,
|
|
timeout: 12_000,
|
|
headers: {
|
|
accept: "application/json",
|
|
authorization: `Bearer ${token}`,
|
|
...(body === null ? {} : { "content-type": "application/json", "content-length": Buffer.byteLength(bodyText) }),
|
|
},
|
|
}, (res) => {
|
|
const chunks: Buffer[] = [];
|
|
res.on("data", (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
|
|
res.on("end", () => {
|
|
const text = Buffer.concat(chunks).toString("utf8");
|
|
const bodyJson = parseJsonRecord(text);
|
|
const httpStatus = Number(res.statusCode ?? 0);
|
|
resolve({
|
|
ok: httpStatus >= 200 && httpStatus < 300,
|
|
httpStatus,
|
|
bodyJson,
|
|
bodyTextPreview: bodyJson === null ? text.slice(0, 1000) : "",
|
|
valuesRedacted: true,
|
|
});
|
|
});
|
|
});
|
|
req.on("timeout", () => req.destroy(new Error("kubernetes-api-timeout")));
|
|
req.on("error", (error) => {
|
|
resolve({ ok: false, httpStatus: null, error: String(error?.message || error).slice(0, 500), valuesRedacted: true });
|
|
});
|
|
if (bodyText.length > 0) req.write(bodyText);
|
|
req.end();
|
|
});
|
|
}
|
|
|
|
function compactK8sApiResult(result: Record<string, unknown>): Record<string, unknown> {
|
|
const body = record(result.bodyJson);
|
|
return {
|
|
ok: result.ok === true,
|
|
httpStatus: result.httpStatus ?? null,
|
|
status: body.status ?? null,
|
|
reason: body.reason ?? result.reason ?? null,
|
|
message: typeof body.message === "string" ? body.message.slice(0, 300) : result.error ?? null,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function k8sJobItems(value: unknown): Record<string, unknown>[] {
|
|
const items = record(value).items;
|
|
return Array.isArray(items) ? items.map(record) : [];
|
|
}
|
|
|
|
function sentinelCadenceCronJobName(config: WebProbeSentinelServiceConfig): string {
|
|
return safeKubernetesSegment(`${stringAt(config.runtime, "deploymentName")}-quick-verify`, 52);
|
|
}
|
|
|
|
function manualQuickVerifyJobName(config: WebProbeSentinelServiceConfig, now: string): string {
|
|
const stamp = Date.parse(now).toString(36).replace(/[^a-z0-9]/giu, "").toLowerCase();
|
|
const prefix = `wps-${config.node.toLowerCase()}-qv`;
|
|
return safeKubernetesSegment(`${prefix}-${stamp}-${randomUUID().slice(0, 5)}`, 63);
|
|
}
|
|
|
|
function safeKubernetesSegment(value: string, maxLength: number): string {
|
|
const cleaned = value.toLowerCase().replace(/[^a-z0-9-]+/gu, "-").replace(/^-+|-+$/gu, "").replace(/-{2,}/gu, "-");
|
|
const clipped = cleaned.slice(0, maxLength).replace(/-+$/gu, "");
|
|
return clipped.length > 0 ? clipped : "web-probe-sentinel";
|
|
}
|
|
|
|
function safeKubernetesLabelValue(value: string): string {
|
|
return safeKubernetesSegment(value, 63);
|
|
}
|
|
|
|
function labelRecord(value: unknown): Record<string, string> {
|
|
const result: Record<string, string> = {};
|
|
for (const [key, raw] of Object.entries(record(value))) {
|
|
if (typeof raw === "string" && raw.length > 0) result[key] = raw;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
function cloneJson(value: Record<string, unknown>): Record<string, unknown> {
|
|
return JSON.parse(JSON.stringify(value)) as Record<string, unknown>;
|
|
}
|
|
|
|
function parseJsonRecord(text: string): Record<string, unknown> | null {
|
|
try {
|
|
return record(JSON.parse(text) as unknown);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function readJsonBody(request: Request): Promise<Record<string, unknown>> {
|
|
if ((request.headers.get("content-length") ?? "0") === "0") return {};
|
|
const value = await request.json().catch(() => ({})) as unknown;
|
|
return record(value);
|
|
}
|
|
|
|
function jsonResponse(value: unknown, status = 200): Response {
|
|
return new Response(JSON.stringify(value, null, 2), { status, headers: { "content-type": "application/json; charset=utf-8" } });
|
|
}
|
|
|
|
function emptyMaintenance(): MaintenanceState {
|
|
return { active: false, reason: null, releaseId: null, startedAt: null, stoppedAt: null, quickVerifyPlannedAt: null, quickVerifyPlannedRunId: null };
|
|
}
|
|
|
|
function recordRunResult(config: WebProbeSentinelServiceConfig, db: Database, input: Record<string, unknown>): Record<string, unknown> {
|
|
const now = nowIso();
|
|
const runId = stringOrNull(input.runId) ?? `sentinel-run-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
const scenarioId = stringOrNull(input.scenarioId) ?? firstEnabledScenarioId(config);
|
|
if (scenarioId === null) return { ok: false, error: "scenario-missing", valuesRedacted: true };
|
|
const status = stringOrNull(input.status) ?? "analyzed";
|
|
const observerId = stringOrNull(input.observerId);
|
|
const stateDir = stringOrNull(input.stateDir);
|
|
const reportJsonSha256 = stringOrNull(input.reportJsonSha256);
|
|
const findings = arrayRecords(input.findings).slice(0, 50);
|
|
const findingCount = numberOr(input.findingCount, findings.length);
|
|
const artifactCount = numberOr(input.artifactCount, 0);
|
|
const createdAt = stringOrNull(input.createdAt) ?? now;
|
|
db.query(`
|
|
INSERT INTO runs (id, sentinel_id, scenario_id, node, lane, status, observer_id, state_dir, report_json_sha256, finding_count, artifact_count, maintenance, created_at, updated_at, command_plan_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(id) DO UPDATE SET
|
|
sentinel_id = excluded.sentinel_id,
|
|
status = excluded.status,
|
|
observer_id = excluded.observer_id,
|
|
state_dir = excluded.state_dir,
|
|
report_json_sha256 = excluded.report_json_sha256,
|
|
finding_count = excluded.finding_count,
|
|
artifact_count = excluded.artifact_count,
|
|
updated_at = excluded.updated_at
|
|
`).run(runId, config.sentinelId, scenarioId, config.node, config.lane, status, observerId, stateDir, reportJsonSha256, findingCount, artifactCount, thisMaintenanceFlag(input), createdAt, now, JSON.stringify({ source: "recorded-analyze-summary", valuesRedacted: true }));
|
|
const supersededPlannedRuns = db.query("UPDATE runs SET status = 'superseded', updated_at = ? WHERE sentinel_id = ? AND node = ? AND lane = ? AND status = 'planned' AND scenario_id = ? AND created_at < ?")
|
|
.run(now, config.sentinelId, config.node, config.lane, scenarioId, createdAt);
|
|
db.query("DELETE FROM findings WHERE run_id = ? AND sentinel_id = ?").run(runId, config.sentinelId);
|
|
for (const item of findings) {
|
|
const findingId = stringOrNull(item.id) ?? stringOrNull(item.kind) ?? stringOrNull(item.code) ?? "finding";
|
|
const check = checkForFinding(config, { ...item, id: findingId });
|
|
const severity = stringOrNull(check.level) ?? normalizeCheckLevel(stringOrNull(item.severity) ?? stringOrNull(item.level)) ?? "unknown";
|
|
const summary = stringOrNull(item.summary) ?? stringOrNull(item.message) ?? findingId;
|
|
db.query("INSERT INTO findings (run_id, sentinel_id, finding_id, severity, count, summary, report_json_sha256, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)")
|
|
.run(runId, config.sentinelId, findingId.slice(0, 160), severity.slice(0, 40), numberOr(item.count, 1), summary.slice(0, 500), reportJsonSha256, now);
|
|
}
|
|
writeMetadata(db, `run.report.${runId}`, {
|
|
runId,
|
|
sentinelId: config.sentinelId,
|
|
scenarioId,
|
|
observerId,
|
|
stateDir,
|
|
reportJsonSha256,
|
|
summary: record(input.summary),
|
|
views: record(input.views),
|
|
findings: findings.map(compactStoredFinding),
|
|
publicOrigin: stringOrNull(input.publicOrigin),
|
|
screenshot: record(input.screenshot),
|
|
artifactCount,
|
|
findingCount,
|
|
valuesRedacted: true,
|
|
});
|
|
return { ok: true, runId, scenarioId, status, reportJsonSha256, findingCount, artifactCount, supersededPlannedRuns: Number(supersededPlannedRuns.changes ?? 0), valuesRedacted: true };
|
|
}
|
|
|
|
function reportRunView(config: WebProbeSentinelServiceConfig, db: Database, view: string, runId: string | null): Record<string, unknown> {
|
|
if (!stringArrayAt(config.reportViews, "views").includes(view)) {
|
|
return { ok: false, error: "unsupported-report-view", view, valuesRedacted: true };
|
|
}
|
|
const row = runId === null
|
|
? db.query("SELECT * FROM runs WHERE report_json_sha256 IS NOT NULL AND sentinel_id = ? AND node = ? AND lane = ? ORDER BY updated_at DESC LIMIT 1").get(config.sentinelId, config.node, config.lane) as Record<string, unknown> | null
|
|
: readRunRow(config, db, runId);
|
|
if (row === null) return { ok: false, error: "report-run-missing", runId, view, valuesRedacted: true };
|
|
const selectedRunId = stringOrNull(row.id);
|
|
if (selectedRunId === null) return { ok: false, error: "report-run-id-missing", view, valuesRedacted: true };
|
|
const stored = readMetadata(db, `run.report.${selectedRunId}`) ?? {};
|
|
const artifactSummary = readAnalysisArtifactSummaryForRun(config, row, 50);
|
|
const findings = visibleFindingsForRun(config, db, row, 50, artifactSummary);
|
|
const views = record(stored.views);
|
|
const storedView = record(views[view]);
|
|
const renderedText = view === "findings"
|
|
? renderStoredFindings(row, findings, artifactSummary)
|
|
: view === "summary"
|
|
? renderStoredSummary(config, row, stored, findings, artifactSummary)
|
|
: typeof storedView.renderedText === "string"
|
|
? storedView.renderedText
|
|
: null;
|
|
if (renderedText === null) {
|
|
return { ok: false, error: "report-view-not-indexed", runId: selectedRunId, view, availableViews: Object.keys(views), valuesRedacted: true };
|
|
}
|
|
return {
|
|
ok: true,
|
|
view,
|
|
run: dashboardRunSummary(config, db, row),
|
|
summary: record(stored.summary),
|
|
findings,
|
|
artifactSummary,
|
|
renderedText,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function formatStoredFindingLine(item: Record<string, unknown>): string {
|
|
const check = record(item.check);
|
|
const code = stringOrNull(item.checkCode) ?? stringOrNull(check.code) ?? stringOrNull(item.finding_id) ?? stringOrNull(item.findingId) ?? stringOrNull(item.code) ?? "-";
|
|
const title = stringOrNull(item.checkTitleZh) ?? stringOrNull(check.titleZh) ?? stringOrNull(item.summary) ?? "";
|
|
const summary = stringOrNull(item.checkSummaryZh) ?? stringOrNull(check.summaryZh) ?? stringOrNull(item.summary) ?? "";
|
|
const rootCause = stringOrNull(item.rootCause);
|
|
const status = stringOrNull(item.rootCauseStatus);
|
|
const nextAction = stringOrNull(item.nextAction);
|
|
const evidence = stringOrNull(item.evidenceSummary);
|
|
const timing = formatStoredFindingTiming(item);
|
|
return [
|
|
`${item.severity ?? "-"} ${code} ${title} count=${item.count ?? "-"} ${summary}`,
|
|
timing,
|
|
rootCause === null ? null : `rootCause=${rootCause}${status === null ? "" : ` status=${status}`}`,
|
|
evidence === null ? null : `evidence=${evidence}`,
|
|
nextAction === null ? null : `next=${nextAction}`,
|
|
].filter((part) => part !== null).join(" | ");
|
|
}
|
|
|
|
function formatStoredFindingTiming(item: Record<string, unknown>): string | null {
|
|
const source = stringOrNull(item.timingSourceOfTruth);
|
|
const status = stringOrNull(item.timingStatus);
|
|
if (source === null && status === null) return null;
|
|
const pieces = [
|
|
status === null ? null : `status=${status}`,
|
|
source === null ? null : `source=${source}`,
|
|
item.timingAlert === true ? "alert=true" : null,
|
|
].filter((part): part is string => part !== null);
|
|
return `timing=${pieces.join(" ")}`;
|
|
}
|
|
|
|
function renderStoredSummary(
|
|
config: WebProbeSentinelServiceConfig,
|
|
row: Record<string, unknown>,
|
|
stored: Record<string, unknown>,
|
|
findings: readonly Record<string, unknown>[],
|
|
artifactSummary: Record<string, unknown>,
|
|
): string {
|
|
const summary = record(stored.summary);
|
|
const timing = dashboardRunTiming(config, row, stored);
|
|
const reportSha = stringOrNull(row.report_json_sha256) ?? stringOrNull(artifactSummary.reportJsonSha256);
|
|
const findingCount = visibleFindingTypeCount(row, findings, artifactSummary);
|
|
return [
|
|
"Web Probe Sentinel Report",
|
|
"=======================================================",
|
|
`run=${stringOrNull(row.id) ?? "-"} scenario=${stringOrNull(row.scenario_id) ?? "-"} status=${stringOrNull(row.status) ?? "-"}`,
|
|
`observer=${stringOrNull(row.observer_id) ?? "-"} stateDir=${stringOrNull(row.state_dir) ?? "-"}`,
|
|
`report=${reportSha ?? "-"} artifacts=${String(row.artifact_count ?? 0)} findings=${String(findingCount)}`,
|
|
`timing=${formatRunTimingSummary(timing)}`,
|
|
`publicOrigin=${stringOrNull(stored.publicOrigin) ?? "-"}`,
|
|
`analysisWindow=${formatStoredAnalysisWindow(summary.analysisWindow)}`,
|
|
"",
|
|
"Findings",
|
|
findings.length === 0 ? "-" : findings.slice(0, 12).map(formatStoredFindingLine).join("\n"),
|
|
].join("\n");
|
|
}
|
|
|
|
function renderStoredFindings(row: Record<string, unknown>, findings: readonly Record<string, unknown>[], artifactSummary: Record<string, unknown>): string {
|
|
const reportSha = stringOrNull(row.report_json_sha256) ?? stringOrNull(artifactSummary.reportJsonSha256);
|
|
return [
|
|
"Web Probe Sentinel Findings",
|
|
"=======================================================",
|
|
`run=${stringOrNull(row.id) ?? "-"} report=${reportSha ?? "-"} findings=${String(visibleFindingTypeCount(row, findings, artifactSummary))}`,
|
|
findings.length === 0 ? "-" : findings.map(formatStoredFindingLine).join("\n"),
|
|
].join("\n");
|
|
}
|
|
|
|
function formatStoredAnalysisWindow(value: unknown): string {
|
|
const window = record(value);
|
|
const fields = [
|
|
["start", stringOrNull(window.start)],
|
|
["end", stringOrNull(window.end)],
|
|
["samples", numberOr(window.sampleCount, numberOr(window.samples, -1)) >= 0 ? String(numberOr(window.sampleCount, numberOr(window.samples, 0))) : null],
|
|
["durationMs", numberOr(window.durationMs, -1) >= 0 ? String(numberOr(window.durationMs, 0)) : null],
|
|
].filter((entry): entry is [string, string] => entry[1] !== null);
|
|
return fields.length === 0 ? "-" : fields.map(([key, item]) => `${key}=${item}`).join(" ");
|
|
}
|
|
|
|
function formatRunTimingSummary(value: Record<string, unknown>): string {
|
|
const fields = [
|
|
["durationMinutes", numberOrNull(value.durationMinutes)],
|
|
["durationSource", stringOrNull(value.durationSource)],
|
|
["scenarioCadenceMinutes", numberOrNull(value.scenarioCadenceMinutes)],
|
|
["scenarioMaxRunMinutes", numberOrNull(value.scenarioMaxRunMinutes)],
|
|
["schedulerIntervalMinutes", numberOrNull(value.schedulerIntervalMinutes)],
|
|
].filter((entry): entry is [string, string | number] => entry[1] !== null);
|
|
return fields.length === 0 ? "-" : fields.map(([key, item]) => `${key}=${item}`).join(" ");
|
|
}
|
|
|
|
function thisMaintenanceFlag(input: Record<string, unknown>): number {
|
|
return input.maintenance === true ? 1 : 0;
|
|
}
|
|
|
|
function numberOrNull(value: unknown): number | null {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
function numberOr(value: unknown, fallback: number): number {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
}
|
|
|
|
function arrayRecords(value: unknown): Record<string, unknown>[] {
|
|
return Array.isArray(value) ? value.map(record) : [];
|
|
}
|
|
|
|
function checkPath(value: unknown, path: string): unknown {
|
|
const result = valueAtPath(value, path);
|
|
if (result === undefined) throw new Error(`required field missing: ${path}`);
|
|
return result;
|
|
}
|
|
|
|
function stringAt(value: unknown, path: string): string {
|
|
const result = checkPath(value, path);
|
|
if (typeof result !== "string" || result.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
return result;
|
|
}
|
|
|
|
function stringAtNullable(value: unknown, path: string): string | null {
|
|
const result = valueAtPath(value, path);
|
|
return typeof result === "string" && result.length > 0 ? result : null;
|
|
}
|
|
|
|
function numberAt(value: unknown, path: string): number {
|
|
const result = checkPath(value, path);
|
|
if (typeof result !== "number" || !Number.isFinite(result)) throw new Error(`${path} must be a number`);
|
|
return result;
|
|
}
|
|
|
|
function numberAtNullable(value: unknown, path: string): number {
|
|
const result = valueAtPath(value, path);
|
|
return typeof result === "number" && Number.isFinite(result) ? result : 0;
|
|
}
|
|
|
|
function boolAt(value: unknown, path: string): boolean {
|
|
const result = checkPath(value, path);
|
|
if (typeof result !== "boolean") throw new Error(`${path} must be a boolean`);
|
|
return result;
|
|
}
|
|
|
|
function arrayAt(value: unknown, path: string): Record<string, unknown>[] {
|
|
const result = checkPath(value, path);
|
|
if (!Array.isArray(result)) throw new Error(`${path} must be an array`);
|
|
return result.filter(record);
|
|
}
|
|
|
|
function stringArrayAt(value: unknown, path: string): string[] {
|
|
const result = checkPath(value, path);
|
|
if (!Array.isArray(result)) throw new Error(`${path} must be an array`);
|
|
return result.filter((item): item is string => typeof item === "string" && item.length > 0);
|
|
}
|
|
|
|
function stringField(value: Record<string, unknown>, key: string): string {
|
|
const found = value[key];
|
|
if (typeof found !== "string" || found.length === 0) throw new Error(`${key} must be a non-empty string`);
|
|
return found;
|
|
}
|
|
|
|
function stringOrNull(value: unknown): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function numberParam(url: URL, key: string, defaultValue: number): number {
|
|
const raw = url.searchParams.get(key);
|
|
if (raw === null) return defaultValue;
|
|
const parsed = Number(raw);
|
|
return Number.isInteger(parsed) && parsed > 0 && parsed <= 200 ? parsed : defaultValue;
|
|
}
|
|
|
|
function record(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function recordTarget(value: unknown): Record<string, unknown> {
|
|
const result = record(value);
|
|
if (Object.keys(result).length === 0) throw new Error("configRef target must be an object");
|
|
return result;
|
|
}
|
|
|
|
function arrayTarget(value: unknown): Record<string, unknown>[] {
|
|
if (!Array.isArray(value)) throw new Error("configRef target must be an array");
|
|
return value.map(recordTarget);
|
|
}
|
|
|
|
function scenarioArrayTarget(value: unknown): Record<string, unknown>[] {
|
|
if (Array.isArray(value)) return value.map(recordTarget);
|
|
return [recordTarget(value)];
|
|
}
|
|
|
|
function valueAtPath(value: unknown, path: string): unknown {
|
|
let current: unknown = value;
|
|
for (const segment of path.split(".")) {
|
|
const match = /^(?:([A-Za-z0-9_-]+))?(?:\[(\d+)\])?$/u.exec(segment);
|
|
if (match === null) return undefined;
|
|
if (match[1] !== undefined) {
|
|
const obj = record(current);
|
|
current = obj[match[1]];
|
|
}
|
|
if (match[2] !== undefined) {
|
|
if (!Array.isArray(current)) return undefined;
|
|
current = current[Number(match[2])];
|
|
}
|
|
}
|
|
return current;
|
|
}
|
|
|
|
function sha256Json(value: unknown): string {
|
|
return `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`;
|
|
}
|
|
|
|
function nowIso(): string {
|
|
return new Date().toISOString();
|
|
}
|
|
|
|
function metricLabel(value: string): string {
|
|
return value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"').replace(/\n/gu, "\\n");
|
|
}
|