fix: expose d518 sentinel cadence root cause

scripts/src/hwlab-node-web-sentinel-service.ts

@@ -414,14 +414,23 @@ function serviceHealth(config: WebProbeSentinelServiceConfig, db: Database, sche
   checks.sqlite = checkSqlite(db);
   const heartbeatAt = stringOrNull(scheduler.schedulerHeartbeatAt) ?? stringOrNull(readMetadata(db, "scheduler.heartbeat")?.at);
   const heartbeatAgeSeconds = heartbeatAt === null ? null : Math.max(0, Math.round((Date.now() - Date.parse(heartbeatAt)) / 1000));
+  const planned = plannedRunBacklog(config, db);
   checks.scheduler = {
-    ok: scheduler.schedulerLastError === null && heartbeatAgeSeconds !== null && heartbeatAgeSeconds <= config.schedulerHeartbeatStaleSeconds,
+    ok: scheduler.schedulerLastError === null && heartbeatAgeSeconds !== null && heartbeatAgeSeconds <= config.schedulerHeartbeatStaleSeconds && !planned.stale,
     enabled: scheduler.schedulerEnabled === true,
     active: scheduler.schedulerTimerActive === true,
     heartbeatAt,
     heartbeatAgeSeconds,
     staleAfterSeconds: config.schedulerHeartbeatStaleSeconds,
     lastError: scheduler.schedulerLastError,
+    plannedRuns: planned.count,
+    oldestPlannedRunId: planned.oldestRunId,
+    oldestPlannedRunScenarioId: planned.oldestScenarioId,
+    oldestPlannedRunCreatedAt: planned.oldestCreatedAt,
+    oldestPlannedRunAgeSeconds: planned.oldestAgeSeconds,
+    plannedRunStaleAfterSeconds: planned.staleAfterSeconds,
+    plannedRunStale: planned.stale,
+    rootCause: planned.stale ? "planned-run-not-consumed-by-host-cadence" : null,
   };
   checks.analyzer = {
     ok: true,
@@ -534,12 +543,20 @@ function appendObserveCommandArgs(argv: string[], item: Record<string, unknown>,
 }
 
 function schedulerSummary(config: WebProbeSentinelServiceConfig, db: Database): Record<string, unknown> {
+  const planned = plannedRunBacklog(config, db);
   return {
     enabledScenarios: config.scenarios.filter((item) => boolAt(item, "enabled")).map((item) => stringAt(item, "id")),
     intervalMs: config.schedulerIntervalMs,
     maxConcurrentRuns: config.maxConcurrentRuns,
     activeRuns: countWhere(db, "status IN ('queued', 'running', 'analyzing')"),
-    plannedRuns: countWhere(db, "status = 'planned'"),
+    plannedRuns: planned.count,
+    oldestPlannedRunId: planned.oldestRunId,
+    oldestPlannedRunScenarioId: planned.oldestScenarioId,
+    oldestPlannedRunCreatedAt: planned.oldestCreatedAt,
+    oldestPlannedRunAgeSeconds: planned.oldestAgeSeconds,
+    plannedRunStaleAfterSeconds: planned.staleAfterSeconds,
+    plannedRunStale: planned.stale,
+    rootCause: planned.stale ? "planned-run-not-consumed-by-host-cadence" : null,
     heartbeat: readMetadata(db, "scheduler.heartbeat"),
     valuesRedacted: true,
   };
@@ -573,10 +590,42 @@ function renderMetrics(config: WebProbeSentinelServiceConfig, db: Database, heal
     "# HELP web_probe_sentinel_scheduler_heartbeat_age_seconds Scheduler heartbeat age.",
     "# TYPE web_probe_sentinel_scheduler_heartbeat_age_seconds gauge",
     `web_probe_sentinel_scheduler_heartbeat_age_seconds{${labels}} ${heartbeatAge}`,
+    "# HELP web_probe_sentinel_planned_runs Planned runs waiting for host cadence execution.",
+    "# TYPE web_probe_sentinel_planned_runs gauge",
+    `web_probe_sentinel_planned_runs{${labels}} ${countWhere(db, "status = 'planned'")}`,
+    "# HELP web_probe_sentinel_oldest_planned_run_age_seconds Oldest planned run age, or -1 when no planned run exists.",
+    "# TYPE web_probe_sentinel_oldest_planned_run_age_seconds gauge",
+    `web_probe_sentinel_oldest_planned_run_age_seconds{${labels}} ${plannedRunBacklog(config, db).oldestAgeSeconds ?? -1}`,
   ];
   return `${lines.join("\n")}\n`;
 }
 
+function plannedRunBacklog(config: WebProbeSentinelServiceConfig, db: Database): {
+  readonly count: number;
+  readonly oldestRunId: string | null;
+  readonly oldestScenarioId: string | null;
+  readonly oldestCreatedAt: string | null;
+  readonly oldestAgeSeconds: number | null;
+  readonly staleAfterSeconds: number;
+  readonly stale: boolean;
+} {
+  const count = countWhere(db, "status = 'planned'");
+  const oldest = db.query("SELECT id, scenario_id, created_at FROM runs WHERE status = 'planned' ORDER BY created_at ASC LIMIT 1")
+    .get() as Record<string, unknown> | null;
+  const oldestCreatedAt = stringOrNull(oldest?.created_at);
+  const oldestAgeSeconds = oldestCreatedAt === null ? null : ageSeconds(oldestCreatedAt);
+  const staleAfterSeconds = Math.max(60, Math.round(config.schedulerIntervalMs / 1000));
+  return {
+    count,
+    oldestRunId: stringOrNull(oldest?.id),
+    oldestScenarioId: stringOrNull(oldest?.scenario_id),
+    oldestCreatedAt,
+    oldestAgeSeconds,
+    staleAfterSeconds,
+    stale: oldestAgeSeconds !== null && oldestAgeSeconds >= staleAfterSeconds,
+  };
+}
+
 function readConfigRefTarget(ref: string): unknown {
   const [file, path] = ref.split("#");
   if (file === undefined || path === undefined) throw new Error(`invalid configRef: ${ref}`);
@@ -634,7 +683,7 @@ function dashboardOverview(config: WebProbeSentinelServiceConfig, db: Database,
     targetValidation: {
       scenarioId: stringOrNull(record(config.cicd.targetValidation).scenarioId),
       maxSeconds: numberOr(record(config.cicd.targetValidation).maxSeconds, 120),
-      sourceRef: "config/hwlab-web-probe-sentinel/cicd.d601-v03.yaml#sentinel.cicd.targetValidation",
+      sourceRef: targetValidationSourceRef(config),
     },
     traceability: {
       source: "sqlite-index+run-report-metadata",
@@ -645,6 +694,11 @@ function dashboardOverview(config: WebProbeSentinelServiceConfig, db: Database,
   };
 }
 
+function targetValidationSourceRef(config: WebProbeSentinelServiceConfig): string | null {
+  const cicdRef = config.plan.refs.find((item) => item.key === "cicd")?.ref ?? null;
+  return cicdRef === null ? null : `${cicdRef}.targetValidation`;
+}
+
 function dashboardRunList(config: WebProbeSentinelServiceConfig, db: Database, url: URL): Record<string, unknown> {
   const filters = dashboardRunFilters(url);
   const page = dashboardPage(url, config);