fix(code-queue): add active-run liveness diagnostics

src/components/microservices/code-queue-mgr/src-rs/main.rs

@@ -782,6 +782,13 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
     if !trace_gap_not_stale_task_ids.is_empty() {
         reasons.push("trace progress is stale while scheduler heartbeat is fresh; this is a trace gap, not stale active");
     }
+    let database_active_task_count = database_active_task_ids.len();
+    let scheduler_orphaned_active_task_ids = database_active_task_ids.clone();
+    let scheduler_orphaned_active_task_count = scheduler_orphaned_active_task_ids.len();
+    let active_heartbeat_count = active_heartbeat_task_ids.len();
+    let last_scheduler_heartbeat_at = max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastLocalHeartbeatAt")).collect());
+    let last_observed_agent_event_at = max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastObservedAgentEventAt")).collect());
+    let last_persisted_trace_at = max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastPersistedTraceAt")).collect());
 
     json!({
         "state": state,
@@ -791,16 +798,16 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
         "executionStateSource": "postgres-control-plane",
         "controlPlane": "master-code-queue-mgr",
         "databaseActiveTaskIds": database_active_task_ids,
-        "databaseActiveTaskCount": database_active_task_ids.len(),
+        "databaseActiveTaskCount": database_active_task_count,
         "schedulerActiveTaskIds": [],
         "schedulerActiveTaskCount": 0,
         "schedulerActiveRunSlotCount": 0,
         "schedulerActiveQueueIds": [],
         "schedulerProcessingQueueIds": [],
-        "schedulerOrphanedActiveTaskIds": database_active_task_ids,
-        "schedulerOrphanedActiveTaskCount": database_active_task_ids.len(),
+        "schedulerOrphanedActiveTaskIds": scheduler_orphaned_active_task_ids,
+        "schedulerOrphanedActiveTaskCount": scheduler_orphaned_active_task_count,
         "activeHeartbeatTaskIds": active_heartbeat_task_ids,
-        "activeHeartbeatCount": active_heartbeat_task_ids.len(),
+        "activeHeartbeatCount": active_heartbeat_count,
         "heartbeatFreshTaskIds": heartbeat_fresh_task_ids,
         "heartbeatExpiredTaskIds": heartbeat_expired_task_ids,
         "heartbeatMissingTaskIds": heartbeat_missing_task_ids,
@@ -809,9 +816,9 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
         "traceGapNotStaleTaskIds": trace_gap_not_stale_task_ids,
         "schedulerHeartbeatStaleMs": SCHEDULER_HEARTBEAT_STALE_MS,
         "now": now,
-        "lastSchedulerHeartbeatAt": max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastLocalHeartbeatAt")).collect()),
-        "lastObservedAgentEventAt": max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastObservedAgentEventAt")).collect()),
-        "lastPersistedTraceAt": max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastPersistedTraceAt")).collect()),
+        "lastSchedulerHeartbeatAt": last_scheduler_heartbeat_at,
+        "lastObservedAgentEventAt": last_observed_agent_event_at,
+        "lastPersistedTraceAt": last_persisted_trace_at,
         "oaPublisher": Value::Null,
         "reasons": reasons,
         "guidance": [
@@ -1179,6 +1186,7 @@ fn queue_summary(state: &AppState) -> Result<Value, String> {
     let active_tasks: Vec<TaskMeta> = active_rows.iter().map(|row| row_to_task(row, true)).collect();
     let mut database_active_task_ids: Vec<String> = active_tasks.iter().map(|task| task.id.clone()).collect();
     database_active_task_ids.sort();
+    let database_active_task_count = database_active_task_ids.len();
     let execution_diagnostics = execution_diagnostics_from_tasks(&active_tasks, &now_iso());
     if !queues.iter().any(|queue| queue.get("id").and_then(Value::as_str) == Some(DEFAULT_QUEUE_ID)) {
         let now = now_iso();
@@ -1215,7 +1223,7 @@ fn queue_summary(state: &AppState) -> Result<Value, String> {
         "activeTaskIds": [],
         "activeTaskId": Value::Null,
         "databaseActiveTaskIds": database_active_task_ids,
-        "databaseActiveTaskCount": database_active_task_ids.len(),
+        "databaseActiveTaskCount": database_active_task_count,
         "executionStateSource": "postgres-control-plane",
         "executionDiagnostics": execution_diagnostics,
         "schedulerHeartbeatStaleMs": SCHEDULER_HEARTBEAT_STALE_MS,

src/components/microservices/code-queue/src/index.ts

@@ -3501,9 +3501,7 @@ function taskIsRecoverableOrphanedActive(task: QueueTask, staleMs = orphanedActi
   if (task.status !== "running" && task.status !== "judging") return false;
   if (activeTaskHasLocalRunSlotOrWaiter(task)) return false;
   const decision = staleRecoveryCandidate({ task, localActive: false, heartbeatStaleMs: staleMs, now: nowIso() });
-  if (decision.allowed) return true;
-  if (decision.reason === "owner-heartbeat-missing") return orphanedActiveTaskAgeMs(task) >= staleMs;
-  return false;
+  return decision.allowed;
 }
 
 function schedulerReconcileStatus(tasks: QueueTask[] = state.tasks): JsonValue {

src/components/microservices/code-queue/src/self-tests.ts

@@ -256,6 +256,23 @@ function runQueueOrderingSelfTest(): JsonValue {
   const orphanRunning = queueOrderTestTask("codex_4400_orphan_running", "running", "2026-05-11T13:00:00.000Z", "2026-05-11T13:00:00.000Z");
   orphanRunning.queueId = "queue_orphan_recovery";
   orphanRunning.activeTurnId = null;
+  orphanRunning.schedulerHeartbeat = {
+    taskId: orphanRunning.id,
+    queueId: orphanRunning.queueId,
+    attempt: 1,
+    activeTurnId: null,
+    codexThreadId: null,
+    owner: "self-test",
+    schedulerInstance: "self-test",
+    executionPlane: "scheduler-execution-plane",
+    agentPort: "codex",
+    status: "running",
+    lastLocalHeartbeatAt: "2026-05-11T13:00:00.000Z",
+    lastObservedAgentEventAt: null,
+    lastPersistedTraceAt: null,
+    outputMaxSeq: 0,
+    source: "scheduler",
+  };
   const queuedBehindOrphan = queueOrderTestTask("codex_4401_queued", "queued", "2026-05-11T13:01:00.000Z", "2026-05-11T13:01:00.000Z");
   queuedBehindOrphan.queueId = "queue_orphan_recovery";
   const originalMaxActiveQueues = ctx().config.maxActiveQueues;