// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-25-p0-web-probe-sentinel. // Responsibility: Persistent HTTP wrapper service for web-probe observe scheduling, index, health, metrics, maintenance, and dashboard. import { createHash, randomUUID } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { Database } from "bun:sqlite"; import { rootPath } from "./config"; import { webProbeSentinelConfigPlan, type WebProbeSentinelConfigPlan } from "./hwlab-node-web-sentinel-config"; import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; export interface WebProbeSentinelServiceConfig { readonly node: string; readonly lane: string; readonly plan: WebProbeSentinelConfigPlan; readonly runtime: Record; readonly scenarios: readonly Record[]; readonly reportViews: Record; readonly stateRoot: string; readonly sqlitePath: string; readonly listenHost: string; readonly servicePort: number; readonly schedulerIntervalMs: number; readonly schedulerHeartbeatStaleSeconds: number; readonly maxConcurrentRuns: number; } export interface WebProbeSentinelServiceOptions { readonly spec: HwlabRuntimeLaneSpec; readonly stateRootOverride?: string; readonly portOverride?: number; readonly hostOverride?: string; readonly schedulerEnabled?: boolean; } interface MaintenanceState { readonly active: boolean; readonly reason: string | null; readonly releaseId: string | null; readonly startedAt: string | null; readonly stoppedAt: string | null; readonly quickVerifyPlannedAt: string | null; } interface CommandPlanStep { readonly phase: string; readonly argv: readonly string[]; readonly stdinSource: "none" | "prompt-source"; } export interface WebProbeSentinelService { readonly config: WebProbeSentinelServiceConfig; readonly db: Database; readonly schedulerEnabled: boolean; startScheduler(): void; stopScheduler(): void; close(): void; health(): Record; status(): Record; runs(limit?: number): readonly Record[]; maintenance(): MaintenanceState; setMaintenance(active: boolean, input: Record): MaintenanceState; planScenarioRun(scenarioId: string, reason: string): Record; metrics(): string; dashboardHtml(): string; fetch(request: Request): Promise; } export function loadWebProbeSentinelServiceConfig(spec: HwlabRuntimeLaneSpec, options: Omit = {}): WebProbeSentinelServiceConfig { const sentinel = spec.observability.webProbe?.sentinel; if (sentinel === undefined) throw new Error(`config/hwlab-node-lanes.yaml#lanes.${spec.lane}.targets.${spec.nodeId}.observability.webProbe.sentinel is missing`); const plan = webProbeSentinelConfigPlan(spec, "status"); const runtime = recordTarget(readConfigRefTarget(sentinel.configRefs.runtime)); const scenarios = arrayTarget(readConfigRefTarget(sentinel.configRefs.scenarios)); const reportViews = recordTarget(readConfigRefTarget(sentinel.configRefs.reportViews)); const stateRoot = options.stateRootOverride ?? stringAt(runtime, "stateRoot"); const yamlSqlitePath = stringAt(runtime, "sqlite.path"); return { node: spec.nodeId, lane: spec.lane, plan, runtime, scenarios, reportViews, stateRoot, sqlitePath: options.stateRootOverride === undefined ? yamlSqlitePath : join(stateRoot, "index.sqlite"), listenHost: options.hostOverride ?? stringAt(runtime, "listenHost"), servicePort: options.portOverride ?? numberAt(runtime, "servicePort"), schedulerIntervalMs: numberAt(runtime, "scheduler.intervalMs"), schedulerHeartbeatStaleSeconds: numberAt(runtime, "scheduler.heartbeatStaleSeconds"), maxConcurrentRuns: numberAt(runtime, "scheduler.maxConcurrentRuns"), }; } export function createWebProbeSentinelService(options: WebProbeSentinelServiceOptions): WebProbeSentinelService { const config = loadWebProbeSentinelServiceConfig(options.spec, options); mkdirSync(config.stateRoot, { recursive: true }); const db = new Database(config.sqlitePath); initializeIndex(db); const restored = markInterruptedRuns(db, nowIso()); const schedulerEnabled = options.schedulerEnabled ?? true; let schedulerTimer: ReturnType | null = null; let schedulerHeartbeatAt = nowIso(); let schedulerLastError: string | null = null; writeMetadata(db, "service.boot", { at: schedulerHeartbeatAt, restoredInterruptedRuns: restored, valuesRedacted: true }); writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "boot" }); const service: WebProbeSentinelService = { config, db, schedulerEnabled, startScheduler() { if (!schedulerEnabled || schedulerTimer !== null) return; schedulerHeartbeatAt = nowIso(); writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "started" }); schedulerTimer = setInterval(() => { try { schedulerHeartbeatAt = nowIso(); writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "tick" }); writeMetadata(db, "scheduler.summary", schedulerSummary(config, db)); schedulerLastError = null; } catch (error) { schedulerLastError = error instanceof Error ? error.message : String(error); writeMetadata(db, "scheduler.error", { at: nowIso(), message: schedulerLastError }); } }, config.schedulerIntervalMs); }, stopScheduler() { if (schedulerTimer !== null) clearInterval(schedulerTimer); schedulerTimer = null; writeMetadata(db, "scheduler.heartbeat", { at: nowIso(), loop: "stopped" }); }, close() { this.stopScheduler(); db.close(); }, health() { return serviceHealth(config, db, { schedulerEnabled, schedulerHeartbeatAt, schedulerTimerActive: schedulerTimer !== null, schedulerLastError, }); }, status() { return { ok: true, node: config.node, lane: config.lane, status: "observed", configReady: config.plan.ok, scheduler: schedulerSummary(config, db), maintenance: this.maintenance(), runs: runCounts(db), latestRuns: this.runs(8), valuesRedacted: true, }; }, runs(limit = 20) { return db.query("SELECT id, scenario_id, status, node, lane, observer_id, state_dir, report_json_sha256, finding_count, artifact_count, maintenance, created_at, updated_at, interrupted_at FROM runs ORDER BY created_at DESC LIMIT ?") .all(limit) as Record[]; }, maintenance() { return readMetadata(db, "maintenance") as MaintenanceState | null ?? emptyMaintenance(); }, setMaintenance(active: boolean, input: Record) { const current = this.maintenance(); const next: MaintenanceState = active ? { active: true, reason: stringOrNull(input.reason), releaseId: stringOrNull(input.releaseId), startedAt: nowIso(), stoppedAt: current.stoppedAt, quickVerifyPlannedAt: current.quickVerifyPlannedAt, } : { active: false, reason: current.reason, releaseId: current.releaseId, startedAt: current.startedAt, stoppedAt: nowIso(), quickVerifyPlannedAt: nowIso(), }; writeMetadata(db, "maintenance", next); if (!active) { const scenarioId = firstEnabledScenarioId(config); if (scenarioId !== null) this.planScenarioRun(scenarioId, "maintenance-stop-quick-verify"); } return next; }, planScenarioRun(scenarioId: string, reason: string) { const scenario = config.scenarios.find((item) => stringAt(item, "id") === scenarioId); if (scenario === undefined) throw new Error(`scenario not found: ${scenarioId}`); const runId = `sentinel-run-${Date.now()}-${randomUUID().slice(0, 8)}`; const commandPlan = buildObserveCommandPlan(config, scenario); const createdAt = nowIso(); db.query("INSERT INTO runs (id, scenario_id, node, lane, status, maintenance, created_at, updated_at, command_plan_json) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)") .run(runId, scenarioId, config.node, config.lane, "planned", this.maintenance().active ? 1 : 0, createdAt, createdAt, JSON.stringify({ reason, commandPlan, valuesRedacted: true })); return { ok: true, runId, scenarioId, status: "planned", commandPlanSha256: sha256Json(commandPlan), valuesRedacted: true }; }, metrics() { return renderMetrics(config, db, this.health(), this.maintenance()); }, dashboardHtml() { return renderDashboard(config, this.status()); }, async fetch(request: Request) { return sentinelFetch(service, request); }, }; service.startScheduler(); return service; } export function startWebProbeSentinelHttpService(service: WebProbeSentinelService): { readonly url: string; readonly stop: () => void } { const server = Bun.serve({ hostname: service.config.listenHost, port: service.config.servicePort, fetch: (request) => service.fetch(request), }); return { url: `http://${service.config.listenHost}:${server.port}`, stop: () => server.stop(true), }; } async function sentinelFetch(service: WebProbeSentinelService, request: Request): Promise { const url = new URL(request.url); if (request.method === "GET" && url.pathname === "/api/health") return jsonResponse(service.health(), service.health().ok === true ? 200 : 503); if (request.method === "GET" && url.pathname === "/api/status") return jsonResponse(service.status()); if (request.method === "GET" && url.pathname === "/api/runs") return jsonResponse({ ok: true, runs: service.runs(numberParam(url, "limit", 20)), valuesRedacted: true }); if (request.method === "GET" && url.pathname === "/api/maintenance") return jsonResponse({ ok: true, maintenance: service.maintenance(), valuesRedacted: true }); if (request.method === "POST" && (url.pathname === "/api/maintenance/start" || url.pathname === "/api/maintenance/stop")) { const body = await readJsonBody(request); const active = url.pathname.endsWith("/start"); return jsonResponse({ ok: true, maintenance: service.setMaintenance(active, body), valuesRedacted: true }); } if (request.method === "POST" && url.pathname === "/api/runs/plan") { const body = await readJsonBody(request); return jsonResponse(service.planScenarioRun(stringField(body, "scenarioId"), stringOrNull(body.reason) ?? "manual")); } if (request.method === "GET" && url.pathname === "/metrics") { return new Response(service.metrics(), { headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" } }); } if (request.method === "GET" && (url.pathname === "/" || url.pathname === "/dashboard")) { return new Response(service.dashboardHtml(), { headers: { "content-type": "text/html; charset=utf-8" } }); } return jsonResponse({ ok: false, error: "not-found", path: url.pathname, valuesRedacted: true }, 404); } function initializeIndex(db: Database): void { db.exec(` PRAGMA journal_mode = WAL; CREATE TABLE IF NOT EXISTS metadata ( key TEXT PRIMARY KEY, value_json TEXT NOT NULL, updated_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS runs ( id TEXT PRIMARY KEY, scenario_id TEXT NOT NULL, node TEXT NOT NULL, lane TEXT NOT NULL, status TEXT NOT NULL, observer_id TEXT, state_dir TEXT, report_json_sha256 TEXT, finding_count INTEGER NOT NULL DEFAULT 0, artifact_count INTEGER NOT NULL DEFAULT 0, maintenance INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, interrupted_at TEXT, command_plan_json TEXT ); CREATE TABLE IF NOT EXISTS findings ( run_id TEXT NOT NULL, finding_id TEXT NOT NULL, severity TEXT NOT NULL, count INTEGER NOT NULL DEFAULT 1, summary TEXT NOT NULL, report_json_sha256 TEXT, created_at TEXT NOT NULL ); `); } function markInterruptedRuns(db: Database, at: string): number { const result = db.query("UPDATE runs SET status = 'interrupted', interrupted_at = ?, updated_at = ? WHERE status IN ('queued', 'running', 'analyzing')").run(at, at); return Number(result.changes ?? 0); } function serviceHealth(config: WebProbeSentinelServiceConfig, db: Database, scheduler: Record): Record { const checks: Record> = {}; checks.config = { ok: config.plan.ok, status: config.plan.status, conflicts: config.plan.conflicts.length }; checks.pvc = checkWritable(config.stateRoot); checks.sqlite = checkSqlite(db); const heartbeatAt = stringOrNull(scheduler.schedulerHeartbeatAt) ?? stringOrNull(readMetadata(db, "scheduler.heartbeat")?.at); const heartbeatAgeSeconds = heartbeatAt === null ? null : Math.max(0, Math.round((Date.now() - Date.parse(heartbeatAt)) / 1000)); checks.scheduler = { ok: scheduler.schedulerLastError === null && heartbeatAgeSeconds !== null && heartbeatAgeSeconds <= config.schedulerHeartbeatStaleSeconds, enabled: scheduler.schedulerEnabled === true, active: scheduler.schedulerTimerActive === true, heartbeatAt, heartbeatAgeSeconds, staleAfterSeconds: config.schedulerHeartbeatStaleSeconds, lastError: scheduler.schedulerLastError, }; checks.analyzer = { ok: true, source: "existing observe analyze CLI command", command: `bun scripts/cli.ts hwlab nodes web-probe observe analyze --node ${config.node} --lane ${config.lane} --state-dir `, }; const ok = Object.values(checks).every((check) => check.ok === true); return { ok, status: ok ? "healthy" : "degraded", node: config.node, lane: config.lane, checks, valuesRedacted: true }; } function checkWritable(stateRoot: string): Record { try { mkdirSync(stateRoot, { recursive: true }); const path = join(stateRoot, ".health-write-probe"); writeFileSync(path, `${nowIso()}\n`); return { ok: true, stateRoot, probeFile: path }; } catch (error) { return { ok: false, stateRoot, error: error instanceof Error ? error.message : String(error) }; } } function checkSqlite(db: Database): Record { try { db.query("SELECT 1 AS ok").get(); writeMetadata(db, "sqlite.health", { at: nowIso() }); return { ok: true }; } catch (error) { return { ok: false, error: error instanceof Error ? error.message : String(error) }; } } function buildObserveCommandPlan(config: WebProbeSentinelServiceConfig, scenario: Record): readonly CommandPlanStep[] { const targetPath = stringAt(scenario, "observeTargetPath"); const start: CommandPlanStep = { phase: "observe-start", argv: [ "bun", "scripts/cli.ts", "hwlab", "nodes", "web-probe", "observe", "start", "--node", config.node, "--lane", config.lane, "--target-path", targetPath, "--sample-interval-ms", String(numberAt(scenario, "sampleIntervalMs")), "--screenshot-interval-ms", String(numberAt(scenario, "screenshotIntervalMs")), "--command-timeout-seconds", "55", ], stdinSource: "none", }; const commands = arrayAt(scenario, "commandSequence").map((item) => { const type = stringAt(item, "type"); const argv = ["bun", "scripts/cli.ts", "hwlab", "nodes", "web-probe", "observe", "command", "", "--type", type]; if (type === "selectProvider") argv.push("--provider", stringAt(item, "provider")); if (type === "sendPrompt") argv.push("--text-stdin"); return { phase: `observe-command-${type}`, argv, stdinSource: type === "sendPrompt" ? "prompt-source" : "none" } satisfies CommandPlanStep; }); const analyze: CommandPlanStep = { phase: "observe-analyze", argv: ["bun", "scripts/cli.ts", "hwlab", "nodes", "web-probe", "observe", "analyze", ""], stdinSource: "none", }; return [start, ...commands, analyze]; } function schedulerSummary(config: WebProbeSentinelServiceConfig, db: Database): Record { return { enabledScenarios: config.scenarios.filter((item) => boolAt(item, "enabled")).map((item) => stringAt(item, "id")), intervalMs: config.schedulerIntervalMs, maxConcurrentRuns: config.maxConcurrentRuns, activeRuns: countWhere(db, "status IN ('queued', 'running', 'analyzing')"), plannedRuns: countWhere(db, "status = 'planned'"), heartbeat: readMetadata(db, "scheduler.heartbeat"), valuesRedacted: true, }; } function renderMetrics(config: WebProbeSentinelServiceConfig, db: Database, health: Record, maintenance: MaintenanceState): string { const counts = runCounts(db); const heartbeat = record(readMetadata(db, "scheduler.heartbeat")); const heartbeatAt = stringOrNull(heartbeat.at); const heartbeatAge = heartbeatAt === null ? -1 : Math.max(0, Math.round((Date.now() - Date.parse(heartbeatAt)) / 1000)); const lines = [ "# HELP web_probe_sentinel_config_ready Config reference graph is ready.", "# TYPE web_probe_sentinel_config_ready gauge", `web_probe_sentinel_config_ready{node="${metricLabel(config.node)}",lane="${metricLabel(config.lane)}"} ${config.plan.ok ? 1 : 0}`, "# HELP web_probe_sentinel_health Healthy status of the sentinel service.", "# TYPE web_probe_sentinel_health gauge", `web_probe_sentinel_health{node="${metricLabel(config.node)}",lane="${metricLabel(config.lane)}"} ${health.ok === true ? 1 : 0}`, "# HELP web_probe_sentinel_runs_total Runs indexed by status.", "# TYPE web_probe_sentinel_runs_total gauge", ...Object.entries(counts).map(([status, count]) => `web_probe_sentinel_runs_total{status="${metricLabel(status)}"} ${count}`), "# HELP web_probe_sentinel_active_runs Active observe runs known to the sentinel index.", "# TYPE web_probe_sentinel_active_runs gauge", `web_probe_sentinel_active_runs ${countWhere(db, "status IN ('queued', 'running', 'analyzing')")}`, "# HELP web_probe_sentinel_recent_findings Findings indexed from recent reports.", "# TYPE web_probe_sentinel_recent_findings gauge", `web_probe_sentinel_recent_findings ${sumColumn(db, "runs", "finding_count")}`, "# HELP web_probe_sentinel_maintenance_active Maintenance window active flag.", "# TYPE web_probe_sentinel_maintenance_active gauge", `web_probe_sentinel_maintenance_active ${maintenance.active ? 1 : 0}`, "# HELP web_probe_sentinel_scheduler_heartbeat_age_seconds Scheduler heartbeat age.", "# TYPE web_probe_sentinel_scheduler_heartbeat_age_seconds gauge", `web_probe_sentinel_scheduler_heartbeat_age_seconds ${heartbeatAge}`, ]; return `${lines.join("\n")}\n`; } function renderDashboard(config: WebProbeSentinelServiceConfig, status: Record): string { const runs = Array.isArray(status.latestRuns) ? status.latestRuns.map(record) : []; const rows = runs.map((run) => `${escapeHtml(stringOrNull(run.id) ?? "-")}${escapeHtml(stringOrNull(run.scenario_id) ?? "-")}${escapeHtml(stringOrNull(run.status) ?? "-")}${escapeHtml(stringOrNull(run.report_json_sha256) ?? "-")}${escapeHtml(String(run.finding_count ?? 0))}`).join(""); return ` HWLAB Web Probe Sentinel

HWLAB Web Probe Sentinel

${escapeHtml(config.node)} / ${escapeHtml(config.lane)} configReady=${config.plan.ok ? "true" : "false"}

Dashboard is redacted: prompt text, assistant body, cookies, tokens, API keys, provider payload, and stdout/stderr are not displayed.

Latest Runs

${rows}
RunScenarioStatusReport SHAFindings
`; } function readConfigRefTarget(ref: string): unknown { const [file, path] = ref.split("#"); if (file === undefined || path === undefined) throw new Error(`invalid configRef: ${ref}`); const text = readFileSync(rootPath(file), "utf8"); return valueAtPath(Bun.YAML.parse(text) as unknown, path); } function writeMetadata(db: Database, key: string, value: unknown): void { db.query("INSERT INTO metadata (key, value_json, updated_at) VALUES (?, ?, ?) ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, updated_at = excluded.updated_at") .run(key, JSON.stringify(value), nowIso()); } function readMetadata(db: Database, key: string): Record | null { const row = db.query("SELECT value_json FROM metadata WHERE key = ?").get(key) as { value_json?: string } | null; if (typeof row?.value_json !== "string") return null; try { const parsed = JSON.parse(row.value_json) as unknown; return record(parsed); } catch { return null; } } function runCounts(db: Database): Record { const rows = db.query("SELECT status, COUNT(*) AS count FROM runs GROUP BY status").all() as { status: string; count: number }[]; return Object.fromEntries(rows.map((row) => [row.status, Number(row.count)])); } function countWhere(db: Database, where: string): number { const row = db.query(`SELECT COUNT(*) AS count FROM runs WHERE ${where}`).get() as { count?: number } | null; return Number(row?.count ?? 0); } function sumColumn(db: Database, table: string, column: string): number { const row = db.query(`SELECT COALESCE(SUM(${column}), 0) AS total FROM ${table}`).get() as { total?: number } | null; return Number(row?.total ?? 0); } function firstEnabledScenarioId(config: WebProbeSentinelServiceConfig): string | null { const scenario = config.scenarios.find((item) => boolAt(item, "enabled")); return scenario === undefined ? null : stringAt(scenario, "id"); } async function readJsonBody(request: Request): Promise> { if ((request.headers.get("content-length") ?? "0") === "0") return {}; const value = await request.json().catch(() => ({})) as unknown; return record(value); } function jsonResponse(value: unknown, status = 200): Response { return new Response(JSON.stringify(value, null, 2), { status, headers: { "content-type": "application/json; charset=utf-8" } }); } function emptyMaintenance(): MaintenanceState { return { active: false, reason: null, releaseId: null, startedAt: null, stoppedAt: null, quickVerifyPlannedAt: null }; } function checkPath(value: unknown, path: string): unknown { const result = valueAtPath(value, path); if (result === undefined) throw new Error(`required field missing: ${path}`); return result; } function stringAt(value: unknown, path: string): string { const result = checkPath(value, path); if (typeof result !== "string" || result.length === 0) throw new Error(`${path} must be a non-empty string`); return result; } function numberAt(value: unknown, path: string): number { const result = checkPath(value, path); if (typeof result !== "number" || !Number.isFinite(result)) throw new Error(`${path} must be a number`); return result; } function boolAt(value: unknown, path: string): boolean { const result = checkPath(value, path); if (typeof result !== "boolean") throw new Error(`${path} must be a boolean`); return result; } function arrayAt(value: unknown, path: string): Record[] { const result = checkPath(value, path); if (!Array.isArray(result)) throw new Error(`${path} must be an array`); return result.filter(record); } function stringField(value: Record, key: string): string { const found = value[key]; if (typeof found !== "string" || found.length === 0) throw new Error(`${key} must be a non-empty string`); return found; } function stringOrNull(value: unknown): string | null { return typeof value === "string" && value.length > 0 ? value : null; } function numberParam(url: URL, key: string, defaultValue: number): number { const raw = url.searchParams.get(key); if (raw === null) return defaultValue; const parsed = Number(raw); return Number.isInteger(parsed) && parsed > 0 && parsed <= 200 ? parsed : defaultValue; } function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function recordTarget(value: unknown): Record { const result = record(value); if (Object.keys(result).length === 0) throw new Error("configRef target must be an object"); return result; } function arrayTarget(value: unknown): Record[] { if (!Array.isArray(value)) throw new Error("configRef target must be an array"); return value.map(recordTarget); } function valueAtPath(value: unknown, path: string): unknown { let current: unknown = value; for (const segment of path.split(".")) { const match = /^(?:([A-Za-z0-9_-]+))?(?:\[(\d+)\])?$/u.exec(segment); if (match === null) return undefined; if (match[1] !== undefined) { const obj = record(current); current = obj[match[1]]; } if (match[2] !== undefined) { if (!Array.isArray(current)) return undefined; current = current[Number(match[2])]; } } return current; } function sha256Json(value: unknown): string { return `sha256:${createHash("sha256").update(JSON.stringify(value)).digest("hex")}`; } function nowIso(): string { return new Date().toISOString(); } function metricLabel(value: string): string { return value.replace(/\\/gu, "\\\\").replace(/"/gu, '\\"').replace(/\n/gu, "\\n"); } function escapeHtml(value: string): string { return value.replace(/&/gu, "&").replace(//gu, ">").replace(/"/gu, """); }