Codex
685 lines
27 KiB
TypeScript
685 lines
27 KiB
TypeScript
#!/usr/bin/env bun
|
|
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p12-cadence-scheduler-monitor-web.
|
|
// Responsibility: Host-side cadence scheduler for YAML-first web-probe sentinels; it triggers the existing validate quick-verify path when runs become stale.
|
|
import { existsSync, mkdirSync, openSync, closeSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { repoRoot, rootPath } from "./src/config";
|
|
import { runCommand, runCommandObserved, type CommandResult } from "./src/command";
|
|
import { hwlabDefaultRuntimeTarget, hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane } from "./src/hwlab-node-lanes";
|
|
import { readConfigRefTarget, resolveWebProbeSentinel, webProbeSentinelRegistryRows } from "./src/hwlab-node-web-sentinel-resolver";
|
|
|
|
type SchedulerAction = "run" | "install-systemd" | "status-systemd";
|
|
|
|
interface SchedulerOptions {
|
|
readonly action: SchedulerAction;
|
|
readonly node: string;
|
|
readonly lane: string;
|
|
readonly sentinelId: string | null;
|
|
readonly dryRun: boolean;
|
|
readonly force: boolean;
|
|
readonly confirm: boolean;
|
|
readonly staleMultiplier: number;
|
|
readonly timeoutSeconds: number | null;
|
|
readonly fetchTimeoutMs: number;
|
|
}
|
|
|
|
interface SentinelSchedule {
|
|
readonly sentinelId: string;
|
|
readonly enabled: boolean;
|
|
readonly publicBaseUrl: string;
|
|
readonly cadenceSeconds: number;
|
|
readonly timeoutSeconds: number;
|
|
readonly scenarioIds: readonly string[];
|
|
}
|
|
|
|
interface OverviewSnapshot {
|
|
readonly ok: boolean;
|
|
readonly latestRunId: string | null;
|
|
readonly latestRunAt: string | null;
|
|
readonly latestRunAgeSeconds: number | null;
|
|
readonly schedulerHeartbeatAt: string | null;
|
|
readonly schedulerHeartbeatAgeSeconds: number | null;
|
|
readonly error: string | null;
|
|
}
|
|
|
|
interface TriggerResult {
|
|
readonly attempted: boolean;
|
|
readonly exitCode: number | null;
|
|
readonly timedOut: boolean;
|
|
readonly durationMs: number | null;
|
|
readonly recorded: boolean;
|
|
readonly latestRunIdBefore: string | null;
|
|
readonly latestRunIdAfter: string | null;
|
|
readonly status: string;
|
|
readonly stdoutTail: string;
|
|
readonly stderrTail: string;
|
|
}
|
|
|
|
interface SystemdUnitPlan {
|
|
readonly sentinelId: string;
|
|
readonly unit: string;
|
|
readonly servicePath: string;
|
|
readonly timerPath: string;
|
|
readonly service: string;
|
|
readonly timer: string;
|
|
}
|
|
|
|
const DEFAULT_STALE_MULTIPLIER = 1;
|
|
const DEFAULT_FETCH_TIMEOUT_MS = 15_000;
|
|
const HOST_SCHEDULER_INTERVAL_SECONDS = 120;
|
|
const STATE_DIR = rootPath(".state", "web-probe-sentinel-scheduler");
|
|
const BUN_EXECUTABLE = existsSync("/usr/bin/bun") ? "/usr/bin/bun" : process.execPath || "bun";
|
|
const SYSTEMD_PATH = "/root/.local/bin:/root/.bun/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
const SYSTEMD_NO_PROXY = noProxyValue();
|
|
|
|
await main().catch((error) => {
|
|
const message = error instanceof Error ? error.stack || error.message : String(error);
|
|
console.error(message);
|
|
process.exit(1);
|
|
});
|
|
|
|
async function main(): Promise<void> {
|
|
const options = parseArgs(process.argv.slice(2));
|
|
if (options.action === "install-systemd") {
|
|
installSystemd(options);
|
|
return;
|
|
}
|
|
if (options.action === "status-systemd") {
|
|
statusSystemd(options);
|
|
return;
|
|
}
|
|
await runScheduler(options);
|
|
}
|
|
|
|
async function runScheduler(options: SchedulerOptions): Promise<void> {
|
|
const spec = specFor(options);
|
|
const schedules = sentinelSchedules(spec, options);
|
|
const rows: Record<string, unknown>[] = [];
|
|
let infraFailure = false;
|
|
|
|
for (const schedule of schedules) {
|
|
if (!schedule.enabled) {
|
|
rows.push(rowFor(schedule, null, false, "disabled", null));
|
|
continue;
|
|
}
|
|
const before = await readOverview(schedule, options.fetchTimeoutMs);
|
|
const latestAge = before.latestRunAgeSeconds;
|
|
const dueThresholdSeconds = Math.max(1, Math.round(schedule.cadenceSeconds * options.staleMultiplier));
|
|
const due = options.force || latestAge === null || latestAge >= dueThresholdSeconds;
|
|
let trigger: TriggerResult | null = null;
|
|
if (due && !options.dryRun) {
|
|
const lock = acquireLock(options, schedule.sentinelId, schedule.timeoutSeconds);
|
|
if (lock.acquired) {
|
|
try {
|
|
trigger = await triggerSentinel(options, schedule, before);
|
|
infraFailure = infraFailure || trigger.status === "infra-failed" || trigger.status === "timeout";
|
|
} finally {
|
|
releaseLock(lock.path);
|
|
}
|
|
} else {
|
|
trigger = {
|
|
attempted: false,
|
|
exitCode: null,
|
|
timedOut: false,
|
|
durationMs: null,
|
|
recorded: false,
|
|
latestRunIdBefore: before.latestRunId,
|
|
latestRunIdAfter: before.latestRunId,
|
|
status: `lock-held:${lock.reason}`,
|
|
stdoutTail: "",
|
|
stderrTail: "",
|
|
};
|
|
}
|
|
}
|
|
const status = due ? options.dryRun ? "due-dry-run" : trigger?.status ?? "due" : "fresh";
|
|
const row = rowFor(schedule, before, due, status, trigger);
|
|
rows.push(row);
|
|
if (!options.dryRun) appendEvent({ at: new Date().toISOString(), node: options.node, lane: options.lane, ...row, valuesRedacted: true });
|
|
}
|
|
|
|
printRows(rows);
|
|
if (infraFailure) process.exitCode = 2;
|
|
}
|
|
|
|
function specFor(options: SchedulerOptions) {
|
|
if (!isHwlabRuntimeLane(options.lane)) throw new Error(`unknown lane ${options.lane}`);
|
|
return hwlabRuntimeLaneSpecForNode(options.lane, options.node);
|
|
}
|
|
|
|
function sentinelSchedules(spec: ReturnType<typeof hwlabRuntimeLaneSpecForNode>, options: SchedulerOptions): SentinelSchedule[] {
|
|
const registry = webProbeSentinelRegistryRows(spec);
|
|
const selectedRows = options.sentinelId === null
|
|
? registry
|
|
: registry.filter((row) => row.id === options.sentinelId);
|
|
if (selectedRows.length === 0) {
|
|
const ids = registry.map((row) => row.id).join(", ");
|
|
throw new Error(`unknown sentinel ${options.sentinelId ?? "-"}; available: ${ids}`);
|
|
}
|
|
return selectedRows.map((row) => {
|
|
const sentinel = resolveWebProbeSentinel(spec, row.id);
|
|
const publicExposure = record(readConfigRefTarget(sentinel.configRefs.publicExposure), sentinel.configRefs.publicExposure);
|
|
const runtime = record(readConfigRefTarget(sentinel.configRefs.runtime), sentinel.configRefs.runtime);
|
|
const cicd = record(readConfigRefTarget(sentinel.configRefs.cicd), sentinel.configRefs.cicd);
|
|
const scenarios = scenarioRows(readConfigRefTarget(sentinel.configRefs.scenarios));
|
|
const enabledScenarios = scenarios.filter((scenario) => scenario.enabled !== false);
|
|
const scenarioCadences = enabledScenarios
|
|
.map((scenario) => typeof scenario.cadence === "string" ? parseDurationSeconds(scenario.cadence) : null)
|
|
.filter((value): value is number => value !== null && value > 0);
|
|
const scenarioTimeouts = enabledScenarios
|
|
.map((scenario) => numberAtNullable(scenario, "maxRunSeconds"))
|
|
.filter((value): value is number => value !== null && value > 0);
|
|
const runtimeInterval = numberAt(runtime, "scheduler.intervalMs");
|
|
const yamlTimeout = numberAtNullable(cicd, "targetValidation.maxSeconds");
|
|
const schedulerTimeout = scenarioTimeouts.length > 0 ? Math.max(...scenarioTimeouts) : null;
|
|
return {
|
|
sentinelId: sentinel.id,
|
|
enabled: row.enabled && sentinel.enabled && enabledScenarios.length > 0,
|
|
publicBaseUrl: stringAt(publicExposure, "publicBaseUrl").replace(/\/+$/u, ""),
|
|
cadenceSeconds: Math.min(...(scenarioCadences.length > 0 ? scenarioCadences : [Math.max(1, Math.round(runtimeInterval / 1000))])),
|
|
timeoutSeconds: options.timeoutSeconds ?? schedulerTimeout ?? yamlTimeout ?? 300,
|
|
scenarioIds: enabledScenarios.map((scenario) => String(scenario.id || sentinel.id)),
|
|
};
|
|
});
|
|
}
|
|
|
|
async function triggerSentinel(options: SchedulerOptions, schedule: SentinelSchedule, before: OverviewSnapshot): Promise<TriggerResult> {
|
|
const command = [
|
|
BUN_EXECUTABLE,
|
|
"scripts/cli.ts",
|
|
"web-probe",
|
|
"sentinel",
|
|
"validate",
|
|
"--node",
|
|
options.node,
|
|
"--lane",
|
|
options.lane,
|
|
"--sentinel",
|
|
schedule.sentinelId,
|
|
"--quick-verify",
|
|
"--confirm",
|
|
"--wait",
|
|
"--timeout-seconds",
|
|
String(schedule.timeoutSeconds),
|
|
];
|
|
const hardTimeoutMs = schedulerHardTimeoutMs(schedule);
|
|
const result = await runCommandObserved(command, repoRoot, {
|
|
timeoutMs: hardTimeoutMs,
|
|
heartbeatMs: 30_000,
|
|
killAfterMs: 3_000,
|
|
maxCaptureChars: 24_000,
|
|
env: { ...process.env, NO_COLOR: "1" },
|
|
});
|
|
const after = await readOverview(schedule, options.fetchTimeoutMs);
|
|
const recorded = after.ok && (
|
|
before.latestRunId === null
|
|
|| after.latestRunId !== before.latestRunId
|
|
|| (after.latestRunAt !== null && after.latestRunAt !== before.latestRunAt)
|
|
);
|
|
const status = result.timedOut
|
|
? "timeout"
|
|
: recorded
|
|
? result.exitCode === 0 ? "recorded" : "recorded-with-findings"
|
|
: result.exitCode === 0 ? "completed-no-new-run" : "infra-failed";
|
|
return {
|
|
attempted: true,
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
durationMs: result.durationMs ?? null,
|
|
recorded,
|
|
latestRunIdBefore: before.latestRunId,
|
|
latestRunIdAfter: after.latestRunId,
|
|
status,
|
|
stdoutTail: tail(result.stdout, 900),
|
|
stderrTail: tail(result.timedOut ? `${result.stderr}\nscheduler hard timeout after ${Math.round(hardTimeoutMs / 1000)}s` : result.stderr, 900),
|
|
};
|
|
}
|
|
|
|
async function readOverview(schedule: SentinelSchedule, timeoutMs: number): Promise<OverviewSnapshot> {
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
const response = await fetch(`${schedule.publicBaseUrl}/api/overview`, { cache: "no-store", signal: controller.signal });
|
|
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
|
const overview = record(await response.json(), `${schedule.publicBaseUrl}/api/overview`);
|
|
const latestRun = isRecord(overview.latestRun) ? overview.latestRun : {};
|
|
const freshness = isRecord(overview.freshness) ? overview.freshness : {};
|
|
const latestRunAt = stringAtNullable(latestRun, "updatedAt") ?? stringAtNullable(latestRun, "createdAt");
|
|
return {
|
|
ok: true,
|
|
latestRunId: stringAtNullable(latestRun, "id"),
|
|
latestRunAt,
|
|
latestRunAgeSeconds: numberAtNullable(freshness, "latestRunAgeSeconds") ?? ageSeconds(latestRunAt),
|
|
schedulerHeartbeatAt: stringAtNullable(overview, "scheduler.heartbeatAt") ?? stringAtNullable(freshness, "schedulerHeartbeatAt"),
|
|
schedulerHeartbeatAgeSeconds: numberAtNullable(freshness, "schedulerHeartbeatAgeSeconds"),
|
|
error: null,
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
latestRunId: null,
|
|
latestRunAt: null,
|
|
latestRunAgeSeconds: null,
|
|
schedulerHeartbeatAt: null,
|
|
schedulerHeartbeatAgeSeconds: null,
|
|
error: error instanceof Error ? error.message : String(error),
|
|
};
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
function installSystemd(options: SchedulerOptions): void {
|
|
const plans = systemdUnitPlans(options);
|
|
const legacyAggregate = options.sentinelId === null ? legacyAggregateSystemdPaths(options) : null;
|
|
if (!options.confirm || options.dryRun) {
|
|
console.log(JSON.stringify({
|
|
ok: true,
|
|
mode: "dry-run",
|
|
unitCount: plans.length,
|
|
units: plans,
|
|
legacyAggregate,
|
|
valuesRedacted: true,
|
|
}, null, 2));
|
|
return;
|
|
}
|
|
|
|
for (const plan of plans) {
|
|
writeFileSync(plan.servicePath, plan.service, "utf8");
|
|
writeFileSync(plan.timerPath, plan.timer, "utf8");
|
|
}
|
|
|
|
const cleanupResults = cleanupLegacyAggregateTimer(legacyAggregate);
|
|
const results = [
|
|
...cleanupResults,
|
|
runCommand(["systemctl", "daemon-reload"], "/"),
|
|
...plans.map((plan) => runCommand(["systemctl", "enable", "--now", `${plan.unit}.timer`], "/")),
|
|
];
|
|
printSystemdResult({ units: plans.map(unitSummary), legacyAggregate, results });
|
|
if (results.some((result) => result.exitCode !== 0)) process.exitCode = 2;
|
|
}
|
|
|
|
function systemdUnitPlans(options: SchedulerOptions): SystemdUnitPlan[] {
|
|
const schedules = systemdSchedules(options);
|
|
return schedules.map((schedule) => systemdUnitPlan(options, schedule));
|
|
}
|
|
|
|
function systemdUnitPlan(options: SchedulerOptions, schedule: SentinelSchedule): SystemdUnitPlan {
|
|
const unit = systemdUnitName(options, schedule.sentinelId);
|
|
const servicePath = `/etc/systemd/system/${unit}.service`;
|
|
const timerPath = `/etc/systemd/system/${unit}.timer`;
|
|
const timeoutArg = options.timeoutSeconds === null ? "" : ` --timeout-seconds ${options.timeoutSeconds}`;
|
|
const serviceTimeoutSeconds = systemdServiceTimeoutSeconds(schedule, options);
|
|
const service = `[Unit]
|
|
Description=UniDesk web-probe sentinel host cadence scheduler for ${options.node}/${options.lane}/${schedule.sentinelId}
|
|
Wants=network-online.target
|
|
After=network-online.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
TimeoutStartSec=${serviceTimeoutSeconds}s
|
|
TimeoutStopSec=15s
|
|
KillMode=control-group
|
|
Environment=HOME=/root
|
|
Environment=PATH=${SYSTEMD_PATH}
|
|
Environment=NO_PROXY=${SYSTEMD_NO_PROXY}
|
|
Environment=no_proxy=${SYSTEMD_NO_PROXY}
|
|
WorkingDirectory=${repoRoot}
|
|
ExecStart=${BUN_EXECUTABLE} ${join(repoRoot, "scripts", "web-probe-sentinel-scheduler.ts")} run --node ${options.node} --lane ${options.lane} --sentinel ${schedule.sentinelId} --stale-multiplier ${options.staleMultiplier}${timeoutArg}
|
|
`;
|
|
const timer = `[Unit]
|
|
Description=Run UniDesk web-probe sentinel host cadence scheduler for ${options.node}/${options.lane}/${schedule.sentinelId}
|
|
|
|
[Timer]
|
|
OnBootSec=${HOST_SCHEDULER_INTERVAL_SECONDS}s
|
|
OnUnitActiveSec=${HOST_SCHEDULER_INTERVAL_SECONDS}s
|
|
AccuracySec=15s
|
|
Persistent=true
|
|
Unit=${unit}.service
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
`;
|
|
return { sentinelId: schedule.sentinelId, unit, servicePath, timerPath, service, timer };
|
|
}
|
|
|
|
function schedulerHardTimeoutMs(schedule: SentinelSchedule): number {
|
|
return Math.max(60, schedule.timeoutSeconds) * 1000;
|
|
}
|
|
|
|
function systemdServiceTimeoutSeconds(schedule: SentinelSchedule, options: SchedulerOptions): number {
|
|
return Math.max(60, schedule.timeoutSeconds) + Math.max(30, Math.ceil(options.fetchTimeoutMs / 1000) + 15);
|
|
}
|
|
|
|
function statusSystemd(options: SchedulerOptions): void {
|
|
const plans = systemdUnitPlans(options);
|
|
const results = plans.flatMap((plan) => [
|
|
runCommand(["systemctl", "is-enabled", `${plan.unit}.timer`], "/"),
|
|
runCommand(["systemctl", "is-active", `${plan.unit}.timer`], "/"),
|
|
runCommand(["systemctl", "show", `${plan.unit}.timer`, "--property=NextElapseUSecRealtime", "--property=LastTriggerUSec"], "/"),
|
|
]);
|
|
printSystemdResult({
|
|
units: plans.map(unitSummary),
|
|
legacyAggregate: options.sentinelId === null ? legacyAggregateSystemdPaths(options) : null,
|
|
results,
|
|
});
|
|
if (results.some((result) => result.exitCode !== 0)) process.exitCode = 2;
|
|
}
|
|
|
|
function systemdSchedules(options: SchedulerOptions): SentinelSchedule[] {
|
|
const schedules = sentinelSchedules(specFor(options), options);
|
|
const selected = options.sentinelId === null ? schedules.filter((schedule) => schedule.enabled) : schedules;
|
|
if (selected.length === 0) throw new Error(`no enabled sentinels found for ${options.node}/${options.lane}`);
|
|
return selected;
|
|
}
|
|
|
|
function legacyAggregateSystemdPaths(options: SchedulerOptions): { unit: string; servicePath: string; timerPath: string } {
|
|
const unit = systemdUnitName(options, null);
|
|
return {
|
|
unit,
|
|
servicePath: `/etc/systemd/system/${unit}.service`,
|
|
timerPath: `/etc/systemd/system/${unit}.timer`,
|
|
};
|
|
}
|
|
|
|
function cleanupLegacyAggregateTimer(legacyAggregate: { unit: string; servicePath: string; timerPath: string } | null): CommandResult[] {
|
|
if (legacyAggregate === null) return [];
|
|
if (!existsSync(legacyAggregate.servicePath) && !existsSync(legacyAggregate.timerPath)) return [];
|
|
const results = [runCommand(["systemctl", "disable", "--now", `${legacyAggregate.unit}.timer`], "/")];
|
|
for (const path of [legacyAggregate.servicePath, legacyAggregate.timerPath]) {
|
|
if (!existsSync(path)) continue;
|
|
try {
|
|
unlinkSync(path);
|
|
} catch (error) {
|
|
results.push(syntheticCommandResult(["unlink", path], error));
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function syntheticCommandResult(command: string[], error: unknown): CommandResult {
|
|
return {
|
|
command,
|
|
cwd: "/",
|
|
exitCode: 1,
|
|
stdout: "",
|
|
stderr: error instanceof Error ? error.message : String(error),
|
|
signal: null,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
|
|
function unitSummary(plan: SystemdUnitPlan): Record<string, unknown> {
|
|
return {
|
|
sentinelId: plan.sentinelId,
|
|
unit: plan.unit,
|
|
servicePath: plan.servicePath,
|
|
timerPath: plan.timerPath,
|
|
};
|
|
}
|
|
|
|
function printSystemdResult(payload: {
|
|
readonly units: readonly Record<string, unknown>[];
|
|
readonly legacyAggregate: { unit: string; servicePath: string; timerPath: string } | null;
|
|
readonly results: readonly CommandResult[];
|
|
}): void {
|
|
console.log(JSON.stringify({
|
|
ok: payload.results.every((result) => result.exitCode === 0),
|
|
unitCount: payload.units.length,
|
|
units: payload.units,
|
|
legacyAggregate: payload.legacyAggregate,
|
|
results: payload.results.map(compactCommand),
|
|
valuesRedacted: true,
|
|
}, null, 2));
|
|
}
|
|
|
|
function rowFor(schedule: SentinelSchedule, overview: OverviewSnapshot | null, due: boolean, status: string, trigger: TriggerResult | null): Record<string, unknown> {
|
|
return {
|
|
sentinelId: schedule.sentinelId,
|
|
enabled: schedule.enabled,
|
|
cadence: formatSeconds(schedule.cadenceSeconds),
|
|
latestAge: overview?.latestRunAgeSeconds === null || overview?.latestRunAgeSeconds === undefined ? "-" : formatSeconds(overview.latestRunAgeSeconds),
|
|
heartbeatAge: overview?.schedulerHeartbeatAgeSeconds === null || overview?.schedulerHeartbeatAgeSeconds === undefined ? "-" : formatSeconds(overview.schedulerHeartbeatAgeSeconds),
|
|
due,
|
|
status,
|
|
latestRunId: trigger?.latestRunIdAfter ?? overview?.latestRunId ?? null,
|
|
scenarios: schedule.scenarioIds.join(","),
|
|
overviewOk: overview?.ok ?? null,
|
|
overviewError: overview?.error ?? null,
|
|
trigger: trigger === null ? null : {
|
|
attempted: trigger.attempted,
|
|
exitCode: trigger.exitCode,
|
|
timedOut: trigger.timedOut,
|
|
durationMs: trigger.durationMs,
|
|
recorded: trigger.recorded,
|
|
latestRunIdBefore: trigger.latestRunIdBefore,
|
|
latestRunIdAfter: trigger.latestRunIdAfter,
|
|
stdoutTail: trigger.stdoutTail,
|
|
stderrTail: trigger.stderrTail,
|
|
},
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
function printRows(rows: readonly Record<string, unknown>[]): void {
|
|
const headers = ["SENTINEL", "CADENCE", "LATEST_AGE", "DUE", "STATUS", "LATEST_RUN"];
|
|
const body = rows.map((row) => [
|
|
String(row.sentinelId ?? ""),
|
|
String(row.cadence ?? ""),
|
|
String(row.latestAge ?? ""),
|
|
String(row.due ?? ""),
|
|
String(row.status ?? ""),
|
|
String(row.latestRunId ?? "-"),
|
|
]);
|
|
const widths = headers.map((header, index) => Math.max(header.length, ...body.map((line) => line[index].length)));
|
|
console.log(headers.map((header, index) => header.padEnd(widths[index])).join(" "));
|
|
for (const line of body) console.log(line.map((value, index) => value.padEnd(widths[index])).join(" "));
|
|
}
|
|
|
|
function acquireLock(options: SchedulerOptions, sentinelId: string, timeoutSeconds: number): { acquired: true; path: string } | { acquired: false; path: string; reason: string } {
|
|
const lockDir = join(STATE_DIR, "locks");
|
|
mkdirSync(lockDir, { recursive: true });
|
|
const lockPath = join(lockDir, `${safeSegment(options.node)}-${safeSegment(options.lane)}-${safeSegment(sentinelId)}.lock`);
|
|
const maxLockAgeMs = Math.max(3_600_000, (timeoutSeconds + 300) * 1000);
|
|
if (existsSync(lockPath)) {
|
|
const ageMs = Date.now() - statSync(lockPath).mtimeMs;
|
|
if (ageMs > maxLockAgeMs) unlinkSync(lockPath);
|
|
}
|
|
try {
|
|
const fd = openSync(lockPath, "wx");
|
|
writeFileSync(fd, JSON.stringify({ pid: process.pid, at: new Date().toISOString(), sentinelId, valuesRedacted: true }));
|
|
closeSync(fd);
|
|
return { acquired: true, path: lockPath };
|
|
} catch (error) {
|
|
const reason = error instanceof Error ? error.message : String(error);
|
|
return { acquired: false, path: lockPath, reason };
|
|
}
|
|
}
|
|
|
|
function releaseLock(lockPath: string): void {
|
|
try {
|
|
unlinkSync(lockPath);
|
|
} catch {
|
|
// Best-effort cleanup; stale locks are aged out on the next tick.
|
|
}
|
|
}
|
|
|
|
function appendEvent(event: Record<string, unknown>): void {
|
|
mkdirSync(STATE_DIR, { recursive: true });
|
|
const date = new Date().toISOString().slice(0, 10).replaceAll("-", "");
|
|
const path = join(STATE_DIR, `run-${date}.jsonl`);
|
|
writeFileSync(path, `${JSON.stringify(event)}\n`, { flag: "a" });
|
|
}
|
|
|
|
function parseArgs(argv: readonly string[]): SchedulerOptions {
|
|
const defaults = hwlabDefaultRuntimeTarget();
|
|
let action: SchedulerAction = "run";
|
|
let node = defaults.node;
|
|
let lane = defaults.lane;
|
|
let sentinelId: string | null = null;
|
|
let dryRun = false;
|
|
let force = false;
|
|
let confirm = false;
|
|
let staleMultiplier = DEFAULT_STALE_MULTIPLIER;
|
|
let timeoutSeconds: number | null = null;
|
|
let fetchTimeoutMs = DEFAULT_FETCH_TIMEOUT_MS;
|
|
|
|
const args = [...argv];
|
|
if (args[0] === "run" || args[0] === "install-systemd" || args[0] === "status-systemd") {
|
|
action = args.shift() as SchedulerAction;
|
|
}
|
|
while (args.length > 0) {
|
|
const arg = args.shift();
|
|
if (arg === undefined) break;
|
|
if (arg === "--node") node = requireValue(arg, args);
|
|
else if (arg === "--lane") lane = requireValue(arg, args);
|
|
else if (arg === "--sentinel") sentinelId = requireValue(arg, args);
|
|
else if (arg === "--dry-run") dryRun = true;
|
|
else if (arg === "--force") force = true;
|
|
else if (arg === "--confirm") confirm = true;
|
|
else if (arg === "--stale-multiplier") staleMultiplier = positiveNumber(requireValue(arg, args), arg);
|
|
else if (arg === "--timeout-seconds") timeoutSeconds = positiveInteger(requireValue(arg, args), arg);
|
|
else if (arg === "--fetch-timeout-ms") fetchTimeoutMs = positiveInteger(requireValue(arg, args), arg);
|
|
else if (arg === "-h" || arg === "--help") {
|
|
printUsage();
|
|
process.exit(0);
|
|
} else {
|
|
throw new Error(`unknown option ${arg}`);
|
|
}
|
|
}
|
|
return { action, node, lane, sentinelId, dryRun, force, confirm, staleMultiplier, timeoutSeconds, fetchTimeoutMs };
|
|
}
|
|
|
|
function printUsage(): void {
|
|
console.log(`Usage:
|
|
bun scripts/web-probe-sentinel-scheduler.ts run [--node D601] [--lane v03] [--sentinel ID] [--dry-run] [--force]
|
|
bun scripts/web-probe-sentinel-scheduler.ts install-systemd --node D601 --lane v03 [--sentinel ID] --confirm
|
|
bun scripts/web-probe-sentinel-scheduler.ts status-systemd --node D601 --lane v03 [--sentinel ID]
|
|
`);
|
|
}
|
|
|
|
function requireValue(flag: string, args: string[]): string {
|
|
const value = args.shift();
|
|
if (value === undefined || value.length === 0) throw new Error(`${flag} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
function positiveInteger(value: string, flag: string): number {
|
|
const parsed = Number(value);
|
|
if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} must be a positive integer`);
|
|
return parsed;
|
|
}
|
|
|
|
function positiveNumber(value: string, flag: string): number {
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed) || parsed <= 0) throw new Error(`${flag} must be a positive number`);
|
|
return parsed;
|
|
}
|
|
|
|
function scenarioRows(value: unknown): Record<string, unknown>[] {
|
|
if (Array.isArray(value)) return value.map((item) => record(item, "scenario"));
|
|
if (!isRecord(value)) throw new Error("scenario configRef must point to a YAML object or array");
|
|
if (Array.isArray(value.scenarios)) return value.scenarios.map((item) => record(item, "scenario"));
|
|
if (isRecord(value.workflow)) return [value.workflow];
|
|
return [value];
|
|
}
|
|
|
|
function parseDurationSeconds(value: string): number | null {
|
|
const match = /^(\d+)(ms|s|m|h)$/u.exec(value.trim());
|
|
if (match === null) return null;
|
|
const amount = Number(match[1]);
|
|
const unit = match[2];
|
|
if (unit === "ms") return Math.max(1, Math.ceil(amount / 1000));
|
|
if (unit === "s") return amount;
|
|
if (unit === "m") return amount * 60;
|
|
if (unit === "h") return amount * 3600;
|
|
return null;
|
|
}
|
|
|
|
function formatSeconds(seconds: number): string {
|
|
if (seconds < 90) return `${Math.round(seconds)}s`;
|
|
if (seconds < 7200) return `${Math.round(seconds / 60)}m`;
|
|
if (seconds < 172800) return `${Math.round(seconds / 3600)}h`;
|
|
return `${Math.round(seconds / 86400)}d`;
|
|
}
|
|
|
|
function ageSeconds(value: string | null): number | null {
|
|
if (value === null) return null;
|
|
const parsed = Date.parse(value);
|
|
if (!Number.isFinite(parsed)) return null;
|
|
return Math.max(0, Math.round((Date.now() - parsed) / 1000));
|
|
}
|
|
|
|
function stringAt(value: unknown, path: string): string {
|
|
const found = valueAtPath(value, path);
|
|
if (typeof found !== "string" || found.length === 0) throw new Error(`${path} must be a non-empty string`);
|
|
return found;
|
|
}
|
|
|
|
function stringAtNullable(value: unknown, path: string): string | null {
|
|
const found = valueAtPath(value, path);
|
|
return typeof found === "string" && found.length > 0 ? found : null;
|
|
}
|
|
|
|
function numberAt(value: unknown, path: string): number {
|
|
const found = valueAtPath(value, path);
|
|
if (typeof found !== "number" || !Number.isFinite(found)) throw new Error(`${path} must be a number`);
|
|
return found;
|
|
}
|
|
|
|
function numberAtNullable(value: unknown, path: string): number | null {
|
|
const found = valueAtPath(value, path);
|
|
return typeof found === "number" && Number.isFinite(found) ? found : null;
|
|
}
|
|
|
|
function valueAtPath(value: unknown, path: string): unknown {
|
|
let current = value;
|
|
for (const segment of path.split(".")) {
|
|
if (!isRecord(current)) return undefined;
|
|
current = current[segment];
|
|
}
|
|
return current;
|
|
}
|
|
|
|
function record(value: unknown, label: string): Record<string, unknown> {
|
|
if (!isRecord(value)) throw new Error(`${label} must be an object`);
|
|
return value;
|
|
}
|
|
|
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
}
|
|
|
|
function compactCommand(result: CommandResult): Record<string, unknown> {
|
|
return {
|
|
command: result.command.join(" "),
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
durationMs: result.durationMs ?? null,
|
|
stdoutTail: tail(result.stdout, 900),
|
|
stderrTail: tail(result.stderr, 900),
|
|
};
|
|
}
|
|
|
|
function tail(value: string, maxChars: number): string {
|
|
return value.length <= maxChars ? value : value.slice(-maxChars);
|
|
}
|
|
|
|
function systemdUnitName(options: SchedulerOptions, sentinelId: string | null = options.sentinelId): string {
|
|
const sentinel = sentinelId === null ? "" : `-${safeSegment(sentinelId)}`;
|
|
return `unidesk-web-probe-sentinel-scheduler-${safeSegment(options.node)}-${safeSegment(options.lane)}${sentinel}`;
|
|
}
|
|
|
|
function safeSegment(value: string): string {
|
|
return value.toLowerCase().replace(/[^a-z0-9._-]+/gu, "-").replace(/^-+|-+$/gu, "") || "default";
|
|
}
|
|
|
|
function noProxyValue(): string {
|
|
const raw = process.env.NO_PROXY || process.env.no_proxy || "";
|
|
const required = ["localhost", "127.0.0.1", "::1", "hyueapi.com", ".hyueapi.com"];
|
|
const values = raw.split(",").map((item) => item.trim()).filter(Boolean);
|
|
for (const item of required) {
|
|
if (!values.includes(item)) values.push(item);
|
|
}
|
|
return values.join(",");
|
|
}
|