fix: clarify code queue split-brain liveness

src/components/microservices/code-queue-mgr/src-rs/main.rs

@@ -737,6 +737,14 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
         .filter(|task_id| heartbeat_expired_set.contains(*task_id))
         .cloned()
         .collect();
+    let mut heartbeat_risk_task_ids: Vec<String> = heartbeat_expired_task_ids
+        .iter()
+        .chain(heartbeat_missing_task_ids.iter())
+        .chain(stale_recovery_candidate_task_ids.iter())
+        .cloned()
+        .collect();
+    heartbeat_risk_task_ids.sort();
+    heartbeat_risk_task_ids.dedup();
     let mut trace_gap_task_ids: Vec<String> = active_heartbeats
         .iter()
         .filter(|(task_id, heartbeat)| {
@@ -754,6 +762,7 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
         .collect();
 
     let split_brain = !database_active_task_ids.is_empty() && !heartbeat_fresh_task_ids.is_empty();
+    let split_brain_live = split_brain && heartbeat_risk_task_ids.is_empty();
     let stale_active = !stale_recovery_candidate_task_ids.is_empty();
     let degraded = split_brain
         || stale_active
@@ -769,6 +778,33 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
     } else {
         "healthy"
     };
+    let effective_liveness = if !heartbeat_risk_task_ids.is_empty() {
+        "at-risk"
+    } else if split_brain_live {
+        "live"
+    } else if degraded {
+        "degraded"
+    } else {
+        "healthy"
+    };
+    let recommended_action = if !heartbeat_risk_task_ids.is_empty() {
+        "investigate-heartbeat-risk"
+    } else if split_brain_live {
+        "continue-supervision"
+    } else if degraded {
+        "observe-degraded"
+    } else {
+        "none"
+    };
+    let liveness_summary = if !heartbeat_risk_task_ids.is_empty() {
+        "Heartbeat is expired, missing, or stale-recovery eligible for at least one database-active task; investigate before assuming the task is still live."
+    } else if split_brain_live {
+        "PostgreSQL and the local control-plane view are split, but scheduler-owned heartbeat is fresh; treat the task as live and continue supervision."
+    } else if degraded {
+        "Execution diagnostics are degraded; inspect the listed reasons together with heartbeat and trace progress."
+    } else {
+        "Execution diagnostics are healthy."
+    };
     let mut reasons = Vec::new();
     if split_brain {
         reasons.push("postgres control-plane has database-active tasks while its local active slots are empty, but scheduler heartbeat is fresh");
@@ -795,6 +831,10 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
         "health": state,
         "degraded": degraded,
         "splitBrain": split_brain,
+        "splitBrainLive": split_brain_live,
+        "effectiveLiveness": effective_liveness,
+        "recommendedAction": recommended_action,
+        "livenessSummary": liveness_summary,
         "executionStateSource": "postgres-control-plane",
         "controlPlane": "master-code-queue-mgr",
         "databaseActiveTaskIds": database_active_task_ids,
@@ -812,6 +852,7 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
         "heartbeatExpiredTaskIds": heartbeat_expired_task_ids,
         "heartbeatMissingTaskIds": heartbeat_missing_task_ids,
         "staleRecoveryCandidateTaskIds": stale_recovery_candidate_task_ids,
+        "heartbeatRiskTaskIds": heartbeat_risk_task_ids,
         "traceGapTaskIds": trace_gap_task_ids,
         "traceGapNotStaleTaskIds": trace_gap_not_stale_task_ids,
         "schedulerHeartbeatStaleMs": SCHEDULER_HEARTBEAT_STALE_MS,

src/components/microservices/code-queue-mgr/src/index.ts

@@ -859,7 +859,13 @@ function controlPlaneExecutionDiagnostics(tasks: QueueTask[], now = nowIso()): J
   const staleRecoveryCandidateTaskIds = heartbeatExpiredTaskIds.slice();
   const traceGapTaskIds = heartbeatRows.filter((row) => traceGap(row.task, row.heartbeat, nowMs)).map((row) => row.task.id).sort();
   const traceGapNotStaleTaskIds = traceGapTaskIds.filter((taskId) => heartbeatFreshSet.has(taskId));
+  const heartbeatRiskTaskIds = Array.from(new Set([
+    ...heartbeatExpiredTaskIds,
+    ...heartbeatMissingTaskIds,
+    ...staleRecoveryCandidateTaskIds,
+  ])).sort();
   const splitBrain = databaseActiveTaskIds.length > 0 && heartbeatFreshTaskIds.length > 0;
+  const splitBrainLive = splitBrain && heartbeatRiskTaskIds.length === 0;
   const staleActive = staleRecoveryCandidateTaskIds.length > 0;
   const degraded = splitBrain
     || staleActive
@@ -867,6 +873,15 @@ function controlPlaneExecutionDiagnostics(tasks: QueueTask[], now = nowIso()): J
     || heartbeatExpiredTaskIds.length > 0
     || traceGapTaskIds.length > 0;
   const state = splitBrain ? "split-brain" : staleActive ? "stale-active" : degraded ? "degraded" : "healthy";
+  const effectiveLiveness = heartbeatRiskTaskIds.length > 0 ? "at-risk" : splitBrainLive ? "live" : degraded ? "degraded" : "healthy";
+  const recommendedAction = heartbeatRiskTaskIds.length > 0 ? "investigate-heartbeat-risk" : splitBrainLive ? "continue-supervision" : degraded ? "observe-degraded" : "none";
+  const livenessSummary = heartbeatRiskTaskIds.length > 0
+    ? "Heartbeat is expired, missing, or stale-recovery eligible for at least one database-active task; investigate before assuming the task is still live."
+    : splitBrainLive
+      ? "PostgreSQL and the local control-plane view are split, but scheduler-owned heartbeat is fresh; treat the task as live and continue supervision."
+      : degraded
+        ? "Execution diagnostics are degraded; inspect the listed reasons together with heartbeat and trace progress."
+        : "Execution diagnostics are healthy.";
   const reasons: string[] = [];
   if (splitBrain) reasons.push("postgres control-plane has database-active tasks while its local active slots are empty, but scheduler heartbeat is fresh");
   if (staleActive) reasons.push("owner heartbeat is expired and scheduler has no local active run for at least one database-active task");
@@ -877,6 +892,10 @@ function controlPlaneExecutionDiagnostics(tasks: QueueTask[], now = nowIso()): J
     health: state,
     degraded,
     splitBrain,
+    splitBrainLive,
+    effectiveLiveness,
+    recommendedAction,
+    livenessSummary,
     executionStateSource: "postgres-control-plane",
     controlPlane: "master-code-queue-mgr",
     databaseActiveTaskIds,
@@ -894,6 +913,7 @@ function controlPlaneExecutionDiagnostics(tasks: QueueTask[], now = nowIso()): J
     heartbeatExpiredTaskIds,
     heartbeatMissingTaskIds,
     staleRecoveryCandidateTaskIds,
+    heartbeatRiskTaskIds,
     traceGapTaskIds,
     traceGapNotStaleTaskIds,
     schedulerHeartbeatStaleMs,