mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sentinel): restore JD01 cadence cronjob visibility

Browse Source
This commit is contained in:
Codex
2026-07-01 06:43:30 +00:00
parent 622804f95d
commit c7ea34f253
12 changed files with 503 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/hwlab-node-web-sentinel-service.ts
+103 -4
View File
@@ -4,6 +4,7 @@
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-26-p9-multi-web-probe-sentinel.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-27-p11-monitor-web-observability-dashboard.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-06-28-p13-1206-multi-runner-boundaries.
// SPEC: PJ2026-01060508 Web哨兵 draft-2026-07-01-p15-cadence-otel.
// Responsibility: Persistent HTTP wrapper service for web-probe observe scheduling, index, health, metrics, maintenance, and dashboard.
import { Buffer } from "node:buffer";
import { createHash, randomUUID } from "node:crypto";
@@ -14,6 +15,7 @@ import { renderWebProbeSentinelDashboardHtml, webProbeSentinelDashboardAssetResp
import { webProbeSentinelConfigPlan, type WebProbeSentinelConfigPlan } from "./hwlab-node-web-sentinel-config";
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
import { effectiveWebProbeSentinelPublicExposure, resolveWebProbeSentinel, readConfigRefTarget as readSentinelConfigRefTarget } from "./hwlab-node-web-sentinel-resolver";
import { emitWebProbeSentinelSpan, webProbeSentinelOtelSummary } from "./hwlab-node-web-sentinel-otel";
const DASHBOARD_CONTRACT_VERSION = "draft-2026-06-27-p11-monitor-web-observability-dashboard";
const DASHBOARD_MAX_TEXT_BYTES = 16_000;
@@ -130,6 +132,7 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp
let schedulerLastError: string | null = null;
writeMetadata(db, "service.boot", { at: schedulerHeartbeatAt, restoredInterruptedRuns: restored, valuesRedacted: true });
writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "boot" });
emitSchedulerHeartbeatSpan(config, "boot", schedulerHeartbeatAt, true);
const service: WebProbeSentinelService = {
config,
@@ -139,15 +142,21 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp
if (!schedulerEnabled || schedulerTimer !== null) return;
schedulerHeartbeatAt = nowIso();
writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "started" });
emitSchedulerHeartbeatSpan(config, "started", schedulerHeartbeatAt, true);
schedulerTimer = setInterval(() => {
try {
schedulerHeartbeatAt = nowIso();
writeMetadata(db, "scheduler.heartbeat", { at: schedulerHeartbeatAt, loop: "tick" });
writeMetadata(db, "scheduler.summary", schedulerSummary(config, db));
const summary = schedulerSummary(config, db);
writeMetadata(db, "scheduler.summary", summary);
emitSchedulerHeartbeatSpan(config, "tick", schedulerHeartbeatAt, true);
emitCadenceExpectedSpan(config, summary);
if (summary.rootCause === "planned-run-not-consumed-by-host-cadence") emitSchedulerGapSpan(config, summary);
schedulerLastError = null;
} catch (error) {
schedulerLastError = error instanceof Error ? error.message : String(error);
writeMetadata(db, "scheduler.error", { at: nowIso(), message: schedulerLastError });
emitSchedulerHeartbeatSpan(config, "tick-error", nowIso(), false, schedulerLastError);
}
}, config.schedulerIntervalMs);
},
@@ -249,7 +258,9 @@ export function createWebProbeSentinelService(options: WebProbeSentinelServiceOp
return { ok: true, runId, scenarioId, status: "planned", commandPlanSha256: sha256Json(commandPlan), valuesRedacted: true };
},
recordRun(input: Record<string, unknown>) {
return recordRunResult(config, db, input);
const result = recordRunResult(config, db, input);
emitRecordRunSpan(config, input, result);
return result;
},
report(view: string, runId: string | null) {
return reportRunView(config, db, view, runId);
@@ -636,6 +647,66 @@ function schedulerSummary(config: WebProbeSentinelServiceConfig, db: Database):
};
}
function emitSchedulerHeartbeatSpan(config: WebProbeSentinelServiceConfig, loop: string, at: string, ok: boolean, failureKind: string | null = null): void {
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.scheduler.heartbeat", {
status: ok ? "ok" : "error",
failureKind,
namespace: stringOrNull(config.runtime.namespace),
heartbeatAt: at,
cadence: firstEnabledScenarioCadence(config),
valuesRedacted: true,
}, ok);
}
function emitCadenceExpectedSpan(config: WebProbeSentinelServiceConfig, summary: Record<string, unknown>): void {
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.cadence.expected", {
cadence: firstEnabledScenarioCadence(config),
scenarioId: firstEnabledScenarioId(config),
status: summary.rootCause == null ? "ok" : "stale",
activeRunCount: summary.activeRuns ?? null,
plannedRunCount: summary.plannedRuns ?? null,
valuesRedacted: true,
}, summary.rootCause == null);
}
function emitSchedulerGapSpan(config: WebProbeSentinelServiceConfig, summary: Record<string, unknown>): void {
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.scheduler_gap.detected", {
cadence: firstEnabledScenarioCadence(config),
scenarioId: summary.oldestPlannedRunScenarioId ?? firstEnabledScenarioId(config),
runId: summary.oldestPlannedRunId ?? null,
status: "planned-run-stale",
failureKind: summary.rootCause,
valuesRedacted: true,
}, false);
}
function emitRecordRunSpan(config: WebProbeSentinelServiceConfig, input: Record<string, unknown>, result: Record<string, unknown>): void {
emitWebProbeSentinelSpan(sentinelOtelContext(config), "web_probe_sentinel.record_run", {
scenarioId: result.scenarioId ?? input.scenarioId ?? null,
runId: result.runId ?? input.runId ?? null,
observerId: input.observerId ?? null,
status: result.status ?? input.status ?? null,
failureKind: result.ok === true ? null : result.error ?? "record-run-failed",
valuesRedacted: true,
}, result.ok === true);
}
function sentinelOtelContext(config: WebProbeSentinelServiceConfig): { readonly node: string; readonly lane: string; readonly sentinelId: string; readonly namespace: string | null; readonly runtime: Record<string, unknown>; readonly cicd: Record<string, unknown> } {
return {
node: config.node,
lane: config.lane,
sentinelId: config.sentinelId,
namespace: stringOrNull(config.runtime.namespace),
runtime: config.runtime,
cicd: config.cicd,
};
}
function firstEnabledScenarioCadence(config: WebProbeSentinelServiceConfig): string | null {
const scenario = config.scenarios.find((item) => boolAt(item, "enabled"));
return scenario === undefined ? null : stringOrNull(scenario.cadence);
}
function renderMetrics(config: WebProbeSentinelServiceConfig, db: Database, health: Record<string, unknown>, maintenance: MaintenanceState): string {
const counts = runCounts(config, db);
const heartbeat = record(readMetadata(db, "scheduler.heartbeat"));
@@ -740,6 +811,12 @@ function dashboardOverview(config: WebProbeSentinelServiceConfig, db: Database,
const severityCounts = globalSeverityCounts(config, db);
const latestUpdatedAt = latestRow === null ? null : stringOrNull(latestRow.updated_at);
const latestRunAgeSeconds = latestUpdatedAt === null ? null : ageSeconds(latestUpdatedAt);
const heartbeatAgeSeconds = numberOr(record(record(health.checks).scheduler).heartbeatAgeSeconds, -1);
const expectedCadence = firstEnabledScenarioCadence(config);
const expectedCadenceSeconds = durationStringSeconds(expectedCadence);
const staleMultiple = expectedCadenceSeconds === null || latestRunAgeSeconds === null ? null : latestRunAgeSeconds / expectedCadenceSeconds;
const freshnessWarningMultiple = numberAt(config.runtime, "scheduler.freshnessWarningMultiple");
const scheduler = schedulerSummary(config, db);
return {
ok: health.ok === true,
contractVersion: DASHBOARD_CONTRACT_VERSION,
@@ -750,7 +827,7 @@ function dashboardOverview(config: WebProbeSentinelServiceConfig, db: Database,
publicOrigin: stringOrNull(config.publicExposure.publicBaseUrl),
configReady: config.plan.ok,
health,
scheduler: schedulerSummary(config, db),
scheduler,
maintenance,
latestRun,
runCounts: runCounts(config, db),
@@ -758,8 +835,30 @@ function dashboardOverview(config: WebProbeSentinelServiceConfig, db: Database,
freshness: {
latestRunUpdatedAt: latestUpdatedAt,
latestRunAgeSeconds,
schedulerHeartbeatAgeSeconds: numberOr(record(record(health.checks).scheduler).heartbeatAgeSeconds, -1),
schedulerHeartbeatAgeSeconds: heartbeatAgeSeconds,
latestAnalyzedReportAgeSeconds: latestRow === null || stringOrNull(latestRow.report_json_sha256) === null ? null : latestRunAgeSeconds,
},
cadence: {
expectedCadence,
expectedCadenceSeconds,
schedulerHeartbeatAgeSeconds: heartbeatAgeSeconds,
latestRunAgeSeconds,
latestAnalyzedReportAgeSeconds: latestRow === null || stringOrNull(latestRow.report_json_sha256) === null ? null : latestRunAgeSeconds,
activeRuns: scheduler.activeRuns ?? null,
plannedRuns: scheduler.plannedRuns ?? null,
nextRun: null,
staleMultiple,
freshnessWarningMultiple,
status: scheduler.rootCause === "planned-run-not-consumed-by-host-cadence" ? "blocker" : staleMultiple !== null && staleMultiple > freshnessWarningMultiple ? "warning" : "fresh",
cronJob: {
observed: false,
status: "control-plane-status-required",
reason: "runner API does not query Kubernetes CronJob objects; use web-probe sentinel control-plane status for CronJob counts, lastScheduleTime and latest Jobs.",
valuesRedacted: true,
},
valuesRedacted: true,
},
observability: webProbeSentinelOtelSummary(sentinelOtelContext(config)),
targetValidation: {
scenarioId: stringOrNull(record(config.cicd.targetValidation).scenarioId),
maxSeconds: numberOr(record(config.cicd.targetValidation).maxSeconds, 120),