+
+
{{ dot.title }}
@@ -267,6 +314,18 @@ createApp({
+
+ {{ shortId(hoveredTrendDot.runId) }}
+ {{ hoveredTrendDot.absoluteTime }}
+ 状态 {{ hoveredTrendDot.status }}
+ 红色 {{ hoveredTrendDot.red }} / 警告 {{ hoveredTrendDot.warning }} / 总量 {{ hoveredTrendDot.total }}
+ report {{ hoveredTrendDot.reportSha }}
+
暂无运行数据
@@ -515,6 +574,13 @@ function formatDate(value) {
return date.toISOString().slice(5, 16).replace("T", " ");
}
+function formatAbsoluteDate(value) {
+ if (!value) return "-";
+ const date = new Date(value);
+ if (Number.isNaN(date.getTime())) return String(value);
+ return `${date.toISOString().slice(0, 19).replace("T", " ")} UTC`;
+}
+
function formatDuration(seconds) {
const value = Math.max(0, Number(seconds || 0));
if (value < 90) return `${Math.round(value)}s`;
@@ -528,6 +594,16 @@ function shortId(value) {
return text.length > 18 ? `${text.slice(0, 10)}...${text.slice(-6)}` : text || "-";
}
+function shortHash(value) {
+ const text = String(value || "");
+ if (text.length === 0) return "";
+ return text.length > 12 ? text.slice(0, 12) : text;
+}
+
+function clamp(value, min, max) {
+ return Math.max(min, Math.min(max, value));
+}
+
function rootCauseText(item) {
return item?.rootCause || item?.evidenceSummary || item?.summary || "尚未记录根因，等待下一次 OTel/报告归因。";
}
diff --git a/scripts/src/hwlab-node-web-sentinel-cicd.ts b/scripts/src/hwlab-node-web-sentinel-cicd.ts
index 63492199..b5b1223d 100644
--- a/scripts/src/hwlab-node-web-sentinel-cicd.ts
+++ b/scripts/src/hwlab-node-web-sentinel-cicd.ts
@@ -3,6 +3,7 @@
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p10-monitor-web-aggregation.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
+// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web.
// Responsibility: YAML-first CI/CD, image, GitOps and Argo command plan for the web-probe sentinel.
import { createHash, randomUUID } from "node:crypto";
import { existsSync, readFileSync } from "node:fs";
@@ -1999,6 +2000,24 @@ for (let attempt = 1; attempt <= maxNavigationAttempts; attempt += 1) {
await page.waitForTimeout(750 * attempt);
}
+await page.evaluate(() => {
+ const detailPane = document.querySelector(".workspace-grid .pane-detail");
+ if (detailPane instanceof HTMLElement) detailPane.scrollTop = Math.min(96, Math.max(0, detailPane.scrollHeight - detailPane.clientHeight));
+}).catch(() => {});
+await page.waitForTimeout(150);
+
+const trendHoverPoint = await page.evaluate(() => {
+ const target = document.querySelector(".trend-dot-hit .trend-dot-red") || document.querySelector(".trend-dot-hit .trend-dot-warning");
+ if (!(target instanceof SVGElement)) return null;
+ const rect = target.getBoundingClientRect();
+ if (rect.width <= 0 || rect.height <= 0) return null;
+ return { x: rect.left + rect.width / 2, y: rect.top + rect.height / 2 };
+}).catch(() => null);
+if (trendHoverPoint) {
+ await page.mouse.move(trendHoverPoint.x, trendHoverPoint.y);
+ await page.waitForTimeout(250);
+}
+
if (captureScreenshot && screenshotPath) {
await page.screenshot({ path: screenshotPath, fullPage, animations: "disabled" }).catch((error) => {
pageErrors.push({ message: "screenshot failed: " + String(error?.message || error).slice(0, 400) });
@@ -2012,9 +2031,12 @@ const dom = await page.evaluate(() => {
const shell = document.querySelector("[data-monitor-shell='true']");
const error = document.querySelector("#monitor-web-error");
const trend = document.querySelector("[data-monitor-trend-curve]");
+ const trendTooltip = document.querySelector("[data-monitor-trend-tooltip='true']");
const timeline = document.querySelector("[data-monitor-timeline='true']");
const workspace = document.querySelector("[data-monitor-independent-scroll='true']");
const panes = Array.from(document.querySelectorAll(".workspace-grid .pane"));
+ const detailPane = document.querySelector(".workspace-grid .pane-detail");
+ const detailHeader = document.querySelector("#monitor-web-root > div > section.workspace-grid > main > div.pane-header");
const doc = document.documentElement;
const body = document.body;
const viewport = { width: window.innerWidth, height: window.innerHeight };
@@ -2063,6 +2085,8 @@ const dom = await page.evaluate(() => {
runRows: document.querySelectorAll(".run-list .run-row").length,
findingItems: document.querySelectorAll(".finding-list .finding-card").length,
trendCurve: Boolean(trend),
+ trendDotCount: document.querySelectorAll(".trend-dot-hit").length,
+ trendTooltip: tooltipSummary(trendTooltip),
trendPanelText: text("#trend-heading"),
timelineItems: document.querySelectorAll(".timeline-list .timeline-item").length,
timelineVisible: Boolean(timeline),
@@ -2089,6 +2113,7 @@ const dom = await page.evaluate(() => {
const style = window.getComputedStyle(pane);
return style.overflowY === "auto" || style.overflowY === "scroll";
}),
+ stickyHeader: stickyHeaderSummary(detailPane, detailHeader),
},
layout: {
viewport,
@@ -2098,6 +2123,46 @@ const dom = await page.evaluate(() => {
overflow,
},
};
+
+ function tooltipSummary(element) {
+ const body = String(element?.textContent || "").replace(/\s+/g, " ").trim();
+ return {
+ visible: Boolean(element && body.length > 0),
+ text: body.slice(0, 240),
+ hasValues: /红色\s+\d+/u.test(body) && /警告\s+\d+/u.test(body) && /总量\s+\d+/u.test(body),
+ hasTime: /UTC/u.test(body) || /\d{4}-\d{2}-\d{2}/u.test(body),
+ };
+ }
+
+ function stickyHeaderSummary(pane, header) {
+ if (!(pane instanceof HTMLElement) || !(header instanceof HTMLElement)) {
+ return { present: false, coversScroll: false, backgroundOpaque: false, detailScrollTop: null };
+ }
+ const rect = header.getBoundingClientRect();
+ const style = window.getComputedStyle(header);
+ const sampleX = Math.round(rect.left + Math.min(32, Math.max(2, rect.width / 2)));
+ const sampleY = Math.round(rect.top + Math.min(12, Math.max(2, rect.height / 2)));
+ const topElement = document.elementFromPoint(sampleX, sampleY);
+ return {
+ present: true,
+ detailScrollTop: pane.scrollTop,
+ headerTop: Math.round(rect.top),
+ headerBottom: Math.round(rect.bottom),
+ zIndex: style.zIndex,
+ backgroundColor: style.backgroundColor,
+ coversScroll: Boolean(topElement && header.contains(topElement)),
+ backgroundOpaque: backgroundIsOpaque(style.backgroundColor),
+ topElementClass: String(topElement?.className || "").slice(0, 80),
+ };
+ }
+
+ function backgroundIsOpaque(value) {
+ const rgba = /rgba?\(([^)]+)\)/u.exec(value);
+ if (rgba === null) return value.length > 0 && value !== "transparent";
+ const parts = rgba[1].split(",").map((part) => part.trim());
+ if (parts.length < 4) return true;
+ return Number(parts[3]) >= 0.99;
+ }
});
const consoleErrors = consoleMessages.filter((item) => item.type === "error");
@@ -2109,8 +2174,12 @@ const ok = !navigationError
&& dom.ready === true
&& dom.errorVisible !== true
&& dom.trendCurve === true
+ && (dom.trendDotCount === 0 || (dom.trendTooltip?.visible === true && dom.trendTooltip?.hasValues === true && dom.trendTooltip?.hasTime === true))
&& dom.timelineVisible === true
&& dom.scrollModel?.independentScroll === true
+ && dom.scrollModel?.stickyHeader?.present === true
+ && dom.scrollModel?.stickyHeader?.coversScroll === true
+ && dom.scrollModel?.stickyHeader?.backgroundOpaque === true
&& dom.layout?.horizontalOverflow !== true
&& pageErrors.length === 0;
diff --git a/scripts/web-probe-sentinel-scheduler.ts b/scripts/web-probe-sentinel-scheduler.ts
new file mode 100644
index 00000000..0df6577c
--- /dev/null
+++ b/scripts/web-probe-sentinel-scheduler.ts
@@ -0,0 +1,554 @@
+#!/usr/bin/env bun
+// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web.
+// Responsibility: Host-side cadence scheduler for YAML-first web-probe sentinels; it triggers the existing validate quick-verify path when runs become stale.
+import { existsSync, mkdirSync, openSync, closeSync, statSync, unlinkSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { repoRoot, rootPath } from "./src/config";
+import { runCommand, runCommandObserved, type CommandResult } from "./src/command";
+import { hwlabDefaultRuntimeTarget, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane } from "./src/hwlab-node-lanes";
+import { readConfigRefTarget, resolveWebProbeSentinel, webProbeSentinelRegistryRows } from "./src/hwlab-node-web-sentinel-resolver";
+
+type SchedulerAction = "run" | "install-systemd" | "status-systemd";
+
+interface SchedulerOptions {
+ readonly action: SchedulerAction;
+ readonly node: string;
+ readonly lane: string;
+ readonly sentinelId: string | null;
+ readonly dryRun: boolean;
+ readonly force: boolean;
+ readonly confirm: boolean;
+ readonly staleMultiplier: number;
+ readonly timeoutSeconds: number | null;
+ readonly fetchTimeoutMs: number;
+}
+
+interface SentinelSchedule {
+ readonly sentinelId: string;
+ readonly enabled: boolean;
+ readonly publicBaseUrl: string;
+ readonly cadenceSeconds: number;
+ readonly timeoutSeconds: number;
+ readonly scenarioIds: readonly string[];
+}
+
+interface OverviewSnapshot {
+ readonly ok: boolean;
+ readonly latestRunId: string | null;
+ readonly latestRunAt: string | null;
+ readonly latestRunAgeSeconds: number | null;
+ readonly schedulerHeartbeatAt: string | null;
+ readonly schedulerHeartbeatAgeSeconds: number | null;
+ readonly error: string | null;
+}
+
+interface TriggerResult {
+ readonly attempted: boolean;
+ readonly exitCode: number | null;
+ readonly timedOut: boolean;
+ readonly durationMs: number | null;
+ readonly recorded: boolean;
+ readonly latestRunIdBefore: string | null;
+ readonly latestRunIdAfter: string | null;
+ readonly status: string;
+ readonly stdoutTail: string;
+ readonly stderrTail: string;
+}
+
+const DEFAULT_STALE_MULTIPLIER = 1;
+const DEFAULT_FETCH_TIMEOUT_MS = 15_000;
+const HOST_SCHEDULER_INTERVAL_SECONDS = 120;
+const STATE_DIR = rootPath(".state", "web-probe-sentinel-scheduler");
+
+await main().catch((error) => {
+ const message = error instanceof Error ? error.stack || error.message : String(error);
+ console.error(message);
+ process.exit(1);
+});
+
+async function main(): Promise {
+ const options = parseArgs(process.argv.slice(2));
+ if (options.action === "install-systemd") {
+ installSystemd(options);
+ return;
+ }
+ if (options.action === "status-systemd") {
+ statusSystemd(options);
+ return;
+ }
+ await runScheduler(options);
+}
+
+async function runScheduler(options: SchedulerOptions): Promise {
+ const spec = specFor(options);
+ const schedules = sentinelSchedules(spec, options);
+ const rows: Record[] = [];
+ let infraFailure = false;
+
+ for (const schedule of schedules) {
+ if (!schedule.enabled) {
+ rows.push(rowFor(schedule, null, false, "disabled", null));
+ continue;
+ }
+ const before = await readOverview(schedule, options.fetchTimeoutMs);
+ const latestAge = before.latestRunAgeSeconds;
+ const dueThresholdSeconds = Math.max(1, Math.round(schedule.cadenceSeconds * options.staleMultiplier));
+ const due = options.force || latestAge === null || latestAge >= dueThresholdSeconds;
+ let trigger: TriggerResult | null = null;
+ if (due && !options.dryRun) {
+ const lock = acquireLock(options, schedule.sentinelId, schedule.timeoutSeconds);
+ if (lock.acquired) {
+ try {
+ trigger = await triggerSentinel(options, schedule, before);
+ infraFailure = infraFailure || trigger.status === "infra-failed" || trigger.status === "timeout";
+ } finally {
+ releaseLock(lock.path);
+ }
+ } else {
+ trigger = {
+ attempted: false,
+ exitCode: null,
+ timedOut: false,
+ durationMs: null,
+ recorded: false,
+ latestRunIdBefore: before.latestRunId,
+ latestRunIdAfter: before.latestRunId,
+ status: `lock-held:${lock.reason}`,
+ stdoutTail: "",
+ stderrTail: "",
+ };
+ }
+ }
+ const status = due ? options.dryRun ? "due-dry-run" : trigger?.status ?? "due" : "fresh";
+ const row = rowFor(schedule, before, due, status, trigger);
+ rows.push(row);
+ appendEvent({ at: new Date().toISOString(), node: options.node, lane: options.lane, ...row, valuesRedacted: true });
+ }
+
+ printRows(rows);
+ if (infraFailure) process.exitCode = 2;
+}
+
+function specFor(options: SchedulerOptions) {
+ if (!isHwlabRuntimeLane(options.lane)) throw new Error(`unknown lane ${options.lane}`);
+ return hwlabRuntimeLaneSpecForNode(options.lane, options.node);
+}
+
+function sentinelSchedules(spec: ReturnType, options: SchedulerOptions): SentinelSchedule[] {
+ const registry = webProbeSentinelRegistryRows(spec);
+ const selectedRows = options.sentinelId === null
+ ? registry
+ : registry.filter((row) => row.id === options.sentinelId);
+ if (selectedRows.length === 0) {
+ const ids = registry.map((row) => row.id).join(", ");
+ throw new Error(`unknown sentinel ${options.sentinelId ?? "-"}; available: ${ids}`);
+ }
+ return selectedRows.map((row) => {
+ const sentinel = resolveWebProbeSentinel(spec, row.id);
+ const publicExposure = record(readConfigRefTarget(sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure);
+ const runtime = record(readConfigRefTarget(sentinel.configRefs.runtime), sentinel.configRefs.runtime);
+ const cicd = record(readConfigRefTarget(sentinel.configRefs.cicd), sentinel.configRefs.cicd);
+ const scenarios = scenarioRows(readConfigRefTarget(sentinel.configRefs.scenarios));
+ const enabledScenarios = scenarios.filter((scenario) => scenario.enabled !== false);
+ const scenarioCadences = enabledScenarios
+ .map((scenario) => typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null)
+ .filter((value): value is number => value !== null && value > 0);
+ const runtimeInterval = numberAt(runtime, "scheduler.intervalMs");
+ const yamlTimeout = numberAtNullable(cicd, "targetValidation.maxSeconds");
+ return {
+ sentinelId: sentinel.id,
+ enabled: row.enabled && sentinel.enabled && enabledScenarios.length > 0,
+ publicBaseUrl: stringAt(publicExposure, "publicBaseUrl").replace(/\/+$/u, ""),
+ cadenceSeconds: Math.min(...(scenarioCadences.length > 0 ? scenarioCadences : [Math.max(1, Math.round(runtimeInterval / 1000))])),
+ timeoutSeconds: options.timeoutSeconds ?? yamlTimeout ?? 300,
+ scenarioIds: enabledScenarios.map((scenario) => String(scenario.id || sentinel.id)),
+ };
+ });
+}
+
+async function triggerSentinel(options: SchedulerOptions, schedule: SentinelSchedule, before: OverviewSnapshot): Promise {
+ const command = [
+ "bun",
+ "scripts/cli.ts",
+ "web-probe",
+ "sentinel",
+ "validate",
+ "--node",
+ options.node,
+ "--lane",
+ options.lane,
+ "--sentinel",
+ schedule.sentinelId,
+ "--quick-verify",
+ "--confirm",
+ "--wait",
+ "--timeout-seconds",
+ String(schedule.timeoutSeconds),
+ ];
+ const result = await runCommandObserved(command, repoRoot, {
+ timeoutMs: Math.max(60, schedule.timeoutSeconds + 90) * 1000,
+ heartbeatMs: 30_000,
+ maxCaptureChars: 24_000,
+ env: { ...process.env, NO_COLOR: "1" },
+ });
+ const after = await readOverview(schedule, options.fetchTimeoutMs);
+ const recorded = after.ok && (
+ before.latestRunId === null
+ || after.latestRunId !== before.latestRunId
+ || (after.latestRunAt !== null && after.latestRunAt !== before.latestRunAt)
+ );
+ const status = result.timedOut
+ ? "timeout"
+ : recorded
+ ? result.exitCode === 0 ? "recorded" : "recorded-with-findings"
+ : result.exitCode === 0 ? "completed-no-new-run" : "infra-failed";
+ return {
+ attempted: true,
+ exitCode: result.exitCode,
+ timedOut: result.timedOut,
+ durationMs: result.durationMs ?? null,
+ recorded,
+ latestRunIdBefore: before.latestRunId,
+ latestRunIdAfter: after.latestRunId,
+ status,
+ stdoutTail: tail(result.stdout, 900),
+ stderrTail: tail(result.stderr, 900),
+ };
+}
+
+async function readOverview(schedule: SentinelSchedule, timeoutMs: number): Promise {
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
+ try {
+ const response = await fetch(`${schedule.publicBaseUrl}/api/overview`, { cache: "no-store", signal: controller.signal });
+ if (!response.ok) throw new Error(`HTTP ${response.status}`);
+ const overview = record(await response.json(), `${schedule.publicBaseUrl}/api/overview`);
+ const latestRun = isRecord(overview.latestRun) ? overview.latestRun : {};
+ const freshness = isRecord(overview.freshness) ? overview.freshness : {};
+ const latestRunAt = stringAtNullable(latestRun, "updatedAt") ?? stringAtNullable(latestRun, "createdAt");
+ return {
+ ok: true,
+ latestRunId: stringAtNullable(latestRun, "id"),
+ latestRunAt,
+ latestRunAgeSeconds: numberAtNullable(freshness, "latestRunAgeSeconds") ?? ageSeconds(latestRunAt),
+ schedulerHeartbeatAt: stringAtNullable(overview, "scheduler.heartbeatAt") ?? stringAtNullable(freshness, "schedulerHeartbeatAt"),
+ schedulerHeartbeatAgeSeconds: numberAtNullable(freshness, "schedulerHeartbeatAgeSeconds"),
+ error: null,
+ };
+ } catch (error) {
+ return {
+ ok: false,
+ latestRunId: null,
+ latestRunAt: null,
+ latestRunAgeSeconds: null,
+ schedulerHeartbeatAt: null,
+ schedulerHeartbeatAgeSeconds: null,
+ error: error instanceof Error ? error.message : String(error),
+ };
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
+function installSystemd(options: SchedulerOptions): void {
+ const unit = systemdUnitName(options);
+ const servicePath = `/etc/systemd/system/${unit}.service`;
+ const timerPath = `/etc/systemd/system/${unit}.timer`;
+ const service = `[Unit]
+Description=UniDesk web-probe sentinel host cadence scheduler for ${options.node}/${options.lane}
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=${repoRoot}
+ExecStart=/usr/bin/env bun ${join(repoRoot, "scripts", "web-probe-sentinel-scheduler.ts")} run --node ${options.node} --lane ${options.lane} --stale-multiplier ${options.staleMultiplier}
+`;
+ const timer = `[Unit]
+Description=Run UniDesk web-probe sentinel host cadence scheduler for ${options.node}/${options.lane}
+
+[Timer]
+OnBootSec=${HOST_SCHEDULER_INTERVAL_SECONDS}s
+OnUnitActiveSec=${HOST_SCHEDULER_INTERVAL_SECONDS}s
+AccuracySec=15s
+Persistent=true
+Unit=${unit}.service
+
+[Install]
+WantedBy=timers.target
+`;
+
+ if (!options.confirm || options.dryRun) {
+ console.log(JSON.stringify({ ok: true, mode: "dry-run", servicePath, timerPath, service, timer, valuesRedacted: true }, null, 2));
+ return;
+ }
+ writeFileSync(servicePath, service, "utf8");
+ writeFileSync(timerPath, timer, "utf8");
+ const commands = [
+ ["systemctl", "daemon-reload"],
+ ["systemctl", "enable", "--now", `${unit}.timer`],
+ ];
+ const results = commands.map((command) => runCommand(command, "/"));
+ printSystemdResult(unit, servicePath, timerPath, results);
+ if (results.some((result) => result.exitCode !== 0)) process.exitCode = 2;
+}
+
+function statusSystemd(options: SchedulerOptions): void {
+ const unit = systemdUnitName(options);
+ const results = [
+ runCommand(["systemctl", "is-enabled", `${unit}.timer`], "/"),
+ runCommand(["systemctl", "is-active", `${unit}.timer`], "/"),
+ runCommand(["systemctl", "show", `${unit}.timer`, "--property=NextElapseUSecRealtime", "--property=LastTriggerUSec"], "/"),
+ ];
+ printSystemdResult(unit, `/etc/systemd/system/${unit}.service`, `/etc/systemd/system/${unit}.timer`, results);
+ if (results[1]?.exitCode !== 0) process.exitCode = 2;
+}
+
+function printSystemdResult(unit: string, servicePath: string, timerPath: string, results: readonly CommandResult[]): void {
+ console.log(JSON.stringify({
+ ok: results.every((result) => result.exitCode === 0),
+ unit,
+ servicePath,
+ timerPath,
+ results: results.map(compactCommand),
+ valuesRedacted: true,
+ }, null, 2));
+}
+
+function rowFor(schedule: SentinelSchedule, overview: OverviewSnapshot | null, due: boolean, status: string, trigger: TriggerResult | null): Record {
+ return {
+ sentinelId: schedule.sentinelId,
+ enabled: schedule.enabled,
+ cadence: formatSeconds(schedule.cadenceSeconds),
+ latestAge: overview?.latestRunAgeSeconds === null || overview?.latestRunAgeSeconds === undefined ? "-" : formatSeconds(overview.latestRunAgeSeconds),
+ heartbeatAge: overview?.schedulerHeartbeatAgeSeconds === null || overview?.schedulerHeartbeatAgeSeconds === undefined ? "-" : formatSeconds(overview.schedulerHeartbeatAgeSeconds),
+ due,
+ status,
+ latestRunId: trigger?.latestRunIdAfter ?? overview?.latestRunId ?? null,
+ scenarios: schedule.scenarioIds.join(","),
+ overviewOk: overview?.ok ?? null,
+ overviewError: overview?.error ?? null,
+ trigger: trigger === null ? null : {
+ attempted: trigger.attempted,
+ exitCode: trigger.exitCode,
+ timedOut: trigger.timedOut,
+ durationMs: trigger.durationMs,
+ recorded: trigger.recorded,
+ latestRunIdBefore: trigger.latestRunIdBefore,
+ latestRunIdAfter: trigger.latestRunIdAfter,
+ },
+ valuesRedacted: true,
+ };
+}
+
+function printRows(rows: readonly Record[]): void {
+ const headers = ["SENTINEL", "CADENCE", "LATEST_AGE", "DUE", "STATUS", "LATEST_RUN"];
+ const body = rows.map((row) => [
+ String(row.sentinelId ?? ""),
+ String(row.cadence ?? ""),
+ String(row.latestAge ?? ""),
+ String(row.due ?? ""),
+ String(row.status ?? ""),
+ String(row.latestRunId ?? "-"),
+ ]);
+ const widths = headers.map((header, index) => Math.max(header.length, ...body.map((line) => line[index].length)));
+ console.log(headers.map((header, index) => header.padEnd(widths[index])).join(" "));
+ for (const line of body) console.log(line.map((value, index) => value.padEnd(widths[index])).join(" "));
+}
+
+function acquireLock(options: SchedulerOptions, sentinelId: string, timeoutSeconds: number): { acquired: true; path: string } | { acquired: false; path: string; reason: string } {
+ const lockDir = join(STATE_DIR, "locks");
+ mkdirSync(lockDir, { recursive: true });
+ const lockPath = join(lockDir, `${safeSegment(options.node)}-${safeSegment(options.lane)}-${safeSegment(sentinelId)}.lock`);
+ const maxLockAgeMs = Math.max(3_600_000, (timeoutSeconds + 300) * 1000);
+ if (existsSync(lockPath)) {
+ const ageMs = Date.now() - statSync(lockPath).mtimeMs;
+ if (ageMs > maxLockAgeMs) unlinkSync(lockPath);
+ }
+ try {
+ const fd = openSync(lockPath, "wx");
+ writeFileSync(fd, JSON.stringify({ pid: process.pid, at: new Date().toISOString(), sentinelId, valuesRedacted: true }));
+ closeSync(fd);
+ return { acquired: true, path: lockPath };
+ } catch (error) {
+ const reason = error instanceof Error ? error.message : String(error);
+ return { acquired: false, path: lockPath, reason };
+ }
+}
+
+function releaseLock(lockPath: string): void {
+ try {
+ unlinkSync(lockPath);
+ } catch {
+ // Best-effort cleanup; stale locks are aged out on the next tick.
+ }
+}
+
+function appendEvent(event: Record): void {
+ mkdirSync(STATE_DIR, { recursive: true });
+ const date = new Date().toISOString().slice(0, 10).replaceAll("-", "");
+ const path = join(STATE_DIR, `run-${date}.jsonl`);
+ writeFileSync(path, `${JSON.stringify(event)}\n`, { flag: "a" });
+}
+
+function parseArgs(argv: readonly string[]): SchedulerOptions {
+ const defaults = hwlabDefaultRuntimeTarget();
+ let action: SchedulerAction = "run";
+ let node = defaults.node;
+ let lane = defaults.lane;
+ let sentinelId: string | null = null;
+ let dryRun = false;
+ let force = false;
+ let confirm = false;
+ let staleMultiplier = DEFAULT_STALE_MULTIPLIER;
+ let timeoutSeconds: number | null = null;
+ let fetchTimeoutMs = DEFAULT_FETCH_TIMEOUT_MS;
+
+ const args = [...argv];
+ if (args[0] === "run" || args[0] === "install-systemd" || args[0] === "status-systemd") {
+ action = args.shift() as SchedulerAction;
+ }
+ while (args.length > 0) {
+ const arg = args.shift();
+ if (arg === undefined) break;
+ if (arg === "--node") node = requireValue(arg, args);
+ else if (arg === "--lane") lane = requireValue(arg, args);
+ else if (arg === "--sentinel") sentinelId = requireValue(arg, args);
+ else if (arg === "--dry-run") dryRun = true;
+ else if (arg === "--force") force = true;
+ else if (arg === "--confirm") confirm = true;
+ else if (arg === "--stale-multiplier") staleMultiplier = positiveNumber(requireValue(arg, args), arg);
+ else if (arg === "--timeout-seconds") timeoutSeconds = positiveInteger(requireValue(arg, args), arg);
+ else if (arg === "--fetch-timeout-ms") fetchTimeoutMs = positiveInteger(requireValue(arg, args), arg);
+ else if (arg === "-h" || arg === "--help") {
+ printUsage();
+ process.exit(0);
+ } else {
+ throw new Error(`unknown option ${arg}`);
+ }
+ }
+ return { action, node, lane, sentinelId, dryRun, force, confirm, staleMultiplier, timeoutSeconds, fetchTimeoutMs };
+}
+
+function printUsage(): void {
+ console.log(`Usage:
+ bun scripts/web-probe-sentinel-scheduler.ts run [--node D601] [--lane v03] [--sentinel ID] [--dry-run] [--force]
+ bun scripts/web-probe-sentinel-scheduler.ts install-systemd --node D601 --lane v03 --confirm
+ bun scripts/web-probe-sentinel-scheduler.ts status-systemd --node D601 --lane v03
+`);
+}
+
+function requireValue(flag: string, args: string[]): string {
+ const value = args.shift();
+ if (value === undefined || value.length === 0) throw new Error(`${flag} requires a value`);
+ return value;
+}
+
+function positiveInteger(value: string, flag: string): number {
+ const parsed = Number(value);
+ if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} must be a positive integer`);
+ return parsed;
+}
+
+function positiveNumber(value: string, flag: string): number {
+ const parsed = Number(value);
+ if (!Number.isFinite(parsed) || parsed <= 0) throw new Error(`${flag} must be a positive number`);
+ return parsed;
+}
+
+function scenarioRows(value: unknown): Record[] {
+ if (Array.isArray(value)) return value.map((item) => record(item, "scenario"));
+ if (!isRecord(value)) throw new Error("scenario configRef must point to a YAML object or array");
+ if (Array.isArray(value.scenarios)) return value.scenarios.map((item) => record(item, "scenario"));
+ if (isRecord(value.workflow)) return [value.workflow];
+ return [value];
+}
+
+function parseDurationSeconds(value: string): number | null {
+ const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim());
+ if (match === null) return null;
+ const amount = Number(match[1]);
+ const unit = match[2];
+ if (unit === "ms") return Math.max(1, Math.ceil(amount / 1000));
+ if (unit === "s") return amount;
+ if (unit === "m") return amount * 60;
+ if (unit === "h") return amount * 3600;
+ return null;
+}
+
+function formatSeconds(seconds: number): string {
+ if (seconds < 90) return `${Math.round(seconds)}s`;
+ if (seconds < 7200) return `${Math.round(seconds / 60)}m`;
+ if (seconds < 172800) return `${Math.round(seconds / 3600)}h`;
+ return `${Math.round(seconds / 86400)}d`;
+}
+
+function ageSeconds(value: string | null): number | null {
+ if (value === null) return null;
+ const parsed = Date.parse(value);
+ if (!Number.isFinite(parsed)) return null;
+ return Math.max(0, Math.round((Date.now() - parsed) / 1000));
+}
+
+function stringAt(value: unknown, path: string): string {
+ const found = valueAtPath(value, path);
+ if (typeof found !== "string" || found.length === 0) throw new Error(`${path} must be a non-empty string`);
+ return found;
+}
+
+function stringAtNullable(value: unknown, path: string): string | null {
+ const found = valueAtPath(value, path);
+ return typeof found === "string" && found.length > 0 ? found : null;
+}
+
+function numberAt(value: unknown, path: string): number {
+ const found = valueAtPath(value, path);
+ if (typeof found !== "number" || !Number.isFinite(found)) throw new Error(`${path} must be a number`);
+ return found;
+}
+
+function numberAtNullable(value: unknown, path: string): number | null {
+ const found = valueAtPath(value, path);
+ return typeof found === "number" && Number.isFinite(found) ? found : null;
+}
+
+function valueAtPath(value: unknown, path: string): unknown {
+ let current = value;
+ for (const segment of path.split(".")) {
+ if (!isRecord(current)) return undefined;
+ current = current[segment];
+ }
+ return current;
+}
+
+function record(value: unknown, label: string): Record {
+ if (!isRecord(value)) throw new Error(`${label} must be an object`);
+ return value;
+}
+
+function isRecord(value: unknown): value is Record {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function compactCommand(result: CommandResult): Record {
+ return {
+ command: result.command.join(" "),
+ exitCode: result.exitCode,
+ timedOut: result.timedOut,
+ durationMs: result.durationMs ?? null,
+ stdoutTail: tail(result.stdout, 900),
+ stderrTail: tail(result.stderr, 900),
+ };
+}
+
+function tail(value: string, maxChars: number): string {
+ return value.length <= maxChars ? value : value.slice(-maxChars);
+}
+
+function systemdUnitName(options: SchedulerOptions): string {
+ return `unidesk-web-probe-sentinel-scheduler-${safeSegment(options.node)}-${safeSegment(options.lane)}`;
+}
+
+function safeSegment(value: string): string {
+ return value.toLowerCase().replace(/[^a-z0-9._-]+/gu, "-").replace(/^-+|-+$/gu, "") || "default";
+}