mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(code-queue): add issue 3 gateway diagnostics

Browse Source
This commit is contained in:
Codex
2026-05-19 03:32:16 +00:00
parent c00c695197
commit 9d46ca2531
18 changed files with 1494 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/components/microservices/code-queue-mgr/src-rs/main.rs
+219 -6
View File
@@ -1,9 +1,9 @@
#![recursion_limit = "512"]
use chrono::{SecondsFormat, Utc};
use chrono::{DateTime, SecondsFormat, Utc};
use postgres::{Client, NoTls};
use serde_json::{json, Map, Value};
use std::collections::VecDeque;
use std::collections::{HashSet, VecDeque};
use std::env;
use std::error::Error;
use std::fs::{create_dir_all, OpenOptions};
@@ -19,6 +19,7 @@ const MAX_LIST_LIMIT: i64 = 200;
const MAX_OUTPUT_PAGE_LIMIT: usize = 500;
const MAX_PREVIEW_CHARS: usize = 6000;
const WORKDIR_MAX_CHARS: usize = 512;
const SCHEDULER_HEARTBEAT_STALE_MS: i64 = 5 * 60 * 1000;
#[derive(Clone)]
struct Config {
@@ -556,6 +557,10 @@ fn terminal_status(status: &str) -> bool {
matches!(status, "succeeded" | "failed" | "canceled")
}
fn active_status(status: &str) -> bool {
matches!(status, "running" | "judging")
}
fn queued_prompt_editable(task: &TaskMeta) -> bool {
task.status == "queued"
&& task.started_at.is_none()
@@ -636,6 +641,188 @@ fn output_max_seq(task: &TaskMeta) -> i64 {
task.last_output_seq.max(from_output)
}
fn task_scheduler_heartbeat(task: &TaskMeta) -> Option<Value> {
task.task_json
.as_ref()
.and_then(|value| value.get("schedulerHeartbeat"))
.filter(|value| value.is_object())
.cloned()
}
fn parse_iso_ms(value: Option<&str>) -> Option<i64> {
value
.and_then(|text| DateTime::parse_from_rfc3339(text).ok())
.map(|date| date.timestamp_millis())
}
fn heartbeat_string_field(heartbeat: &Value, key: &str) -> Option<String> {
heartbeat.get(key).and_then(Value::as_str).map(|value| value.to_string())
}
fn heartbeat_task_id(heartbeat: &Value) -> String {
heartbeat_string_field(heartbeat, "taskId").unwrap_or_default()
}
fn heartbeat_fresh(heartbeat: &Value, now_ms: i64) -> bool {
parse_iso_ms(heartbeat.get("lastLocalHeartbeatAt").and_then(Value::as_str))
.map(|at| now_ms.saturating_sub(at) < SCHEDULER_HEARTBEAT_STALE_MS)
.unwrap_or(false)
}
fn max_timestamp(values: Vec<Option<String>>) -> Value {
let mut best: Option<(String, i64)> = None;
for value in values.into_iter().flatten() {
let Some(ms) = parse_iso_ms(Some(&value)) else { continue };
if best.as_ref().map(|(_, best_ms)| ms >= *best_ms).unwrap_or(true) {
best = Some((value, ms));
}
}
best.map(|(value, _)| json!(value)).unwrap_or(Value::Null)
}
fn task_has_trace_gap(task: &TaskMeta, heartbeat: &Value, now_ms: i64) -> bool {
let trace_ms = parse_iso_ms(heartbeat.get("lastPersistedTraceAt").and_then(Value::as_str));
let output_seq = heartbeat.get("outputMaxSeq").and_then(Value::as_i64).unwrap_or_else(|| output_max_seq(task));
trace_ms
.map(|at| now_ms.saturating_sub(at) >= SCHEDULER_HEARTBEAT_STALE_MS && output_seq > 0)
.unwrap_or(false)
}
fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
let now_ms = parse_iso_ms(Some(now)).unwrap_or_else(|| Utc::now().timestamp_millis());
let mut database_active_task_ids: Vec<String> = tasks.iter().filter(|task| active_status(&task.status)).map(|task| task.id.clone()).collect();
database_active_task_ids.sort();
let mut active_heartbeats: Vec<(String, Value)> = tasks
.iter()
.filter_map(|task| task_scheduler_heartbeat(task).map(|heartbeat| (task.id.clone(), heartbeat)))
.collect();
active_heartbeats.sort_by(|left, right| left.0.cmp(&right.0));
let mut active_heartbeat_task_ids: Vec<String> = active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_task_id(heartbeat)).filter(|id| !id.is_empty()).collect();
active_heartbeat_task_ids.sort();
active_heartbeat_task_ids.dedup();
let mut heartbeat_fresh_task_ids: Vec<String> = active_heartbeats
.iter()
.filter(|(_, heartbeat)| heartbeat_fresh(heartbeat, now_ms))
.map(|(_, heartbeat)| heartbeat_task_id(heartbeat))
.filter(|id| !id.is_empty())
.collect();
heartbeat_fresh_task_ids.sort();
heartbeat_fresh_task_ids.dedup();
let mut heartbeat_expired_task_ids: Vec<String> = active_heartbeats
.iter()
.filter(|(task_id, heartbeat)| {
tasks.iter().any(|task| &task.id == task_id && active_status(&task.status))
&& !heartbeat_fresh(heartbeat, now_ms)
})
.map(|(_, heartbeat)| heartbeat_task_id(heartbeat))
.filter(|id| !id.is_empty())
.collect();
heartbeat_expired_task_ids.sort();
heartbeat_expired_task_ids.dedup();
let heartbeat_task_set: HashSet<String> = active_heartbeat_task_ids.iter().cloned().collect();
let heartbeat_fresh_set: HashSet<String> = heartbeat_fresh_task_ids.iter().cloned().collect();
let heartbeat_expired_set: HashSet<String> = heartbeat_expired_task_ids.iter().cloned().collect();
let heartbeat_missing_task_ids: Vec<String> = database_active_task_ids
.iter()
.filter(|task_id| !heartbeat_task_set.contains(*task_id))
.cloned()
.collect();
let stale_recovery_candidate_task_ids: Vec<String> = database_active_task_ids
.iter()
.filter(|task_id| heartbeat_expired_set.contains(*task_id))
.cloned()
.collect();
let mut trace_gap_task_ids: Vec<String> = active_heartbeats
.iter()
.filter(|(task_id, heartbeat)| {
tasks.iter().find(|task| &task.id == task_id).map(|task| task_has_trace_gap(task, heartbeat, now_ms)).unwrap_or(false)
})
.map(|(_, heartbeat)| heartbeat_task_id(heartbeat))
.filter(|id| !id.is_empty())
.collect();
trace_gap_task_ids.sort();
trace_gap_task_ids.dedup();
let trace_gap_not_stale_task_ids: Vec<String> = trace_gap_task_ids
.iter()
.filter(|task_id| heartbeat_fresh_set.contains(*task_id))
.cloned()
.collect();
let split_brain = !database_active_task_ids.is_empty() && !heartbeat_fresh_task_ids.is_empty();
let stale_active = !stale_recovery_candidate_task_ids.is_empty();
let degraded = split_brain
|| stale_active
|| !heartbeat_missing_task_ids.is_empty()
|| !heartbeat_expired_task_ids.is_empty()
|| !trace_gap_task_ids.is_empty();
let state = if split_brain {
"split-brain"
} else if stale_active {
"stale-active"
} else if degraded {
"degraded"
} else {
"healthy"
};
let mut reasons = Vec::new();
if split_brain {
reasons.push("postgres control-plane has database-active tasks while its local active slots are empty, but scheduler heartbeat is fresh");
}
if stale_active {
reasons.push("owner heartbeat is expired and scheduler has no local active run for at least one database-active task");
}
if !heartbeat_missing_task_ids.is_empty() {
reasons.push("database-active tasks are missing scheduler-owned heartbeat");
}
if !trace_gap_not_stale_task_ids.is_empty() {
reasons.push("trace progress is stale while scheduler heartbeat is fresh; this is a trace gap, not stale active");
}
json!({
"state": state,
"health": state,
"degraded": degraded,
"splitBrain": split_brain,
"executionStateSource": "postgres-control-plane",
"controlPlane": "master-code-queue-mgr",
"databaseActiveTaskIds": database_active_task_ids,
"databaseActiveTaskCount": database_active_task_ids.len(),
"schedulerActiveTaskIds": [],
"schedulerActiveTaskCount": 0,
"schedulerActiveRunSlotCount": 0,
"schedulerActiveQueueIds": [],
"schedulerProcessingQueueIds": [],
"schedulerOrphanedActiveTaskIds": database_active_task_ids,
"schedulerOrphanedActiveTaskCount": database_active_task_ids.len(),
"activeHeartbeatTaskIds": active_heartbeat_task_ids,
"activeHeartbeatCount": active_heartbeat_task_ids.len(),
"heartbeatFreshTaskIds": heartbeat_fresh_task_ids,
"heartbeatExpiredTaskIds": heartbeat_expired_task_ids,
"heartbeatMissingTaskIds": heartbeat_missing_task_ids,
"staleRecoveryCandidateTaskIds": stale_recovery_candidate_task_ids,
"traceGapTaskIds": trace_gap_task_ids,
"traceGapNotStaleTaskIds": trace_gap_not_stale_task_ids,
"schedulerHeartbeatStaleMs": SCHEDULER_HEARTBEAT_STALE_MS,
"now": now,
"lastSchedulerHeartbeatAt": max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastLocalHeartbeatAt")).collect()),
"lastObservedAgentEventAt": max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastObservedAgentEventAt")).collect()),
"lastPersistedTraceAt": max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastPersistedTraceAt")).collect()),
"oaPublisher": Value::Null,
"reasons": reasons,
"guidance": [
"Do not use master activeRunSlotCount=0 alone to declare a task stuck.",
"Check PostgreSQL task status, scheduler active runs/slots, scheduler heartbeat, and trace/OA publisher progress together.",
"Trace gap with a fresh scheduler heartbeat must not trigger stale retry.",
"Stale recovery is allowed only when the scheduler has no local active run and the owner heartbeat is expired."
]
})
}
fn last_assistant_message(task: &TaskMeta) -> Value {
let final_text = final_response(task);
if !final_text.trim().is_empty() {
@@ -724,6 +911,7 @@ fn task_list_response(_state: &AppState, task: &TaskMeta, lite: bool) -> Value {
"currentMode": task.current_mode,
"judgeFailCount": task.task_json.as_ref().and_then(|value| value.get("judgeFailCount")).and_then(Value::as_i64).unwrap_or(0),
"judgeFailRetryLimit": 3,
"schedulerHeartbeat": task_scheduler_heartbeat(task).unwrap_or(Value::Null),
"codexThreadId": task.codex_thread_id,
"activeTurnId": task.active_turn_id,
"lastError": task.last_error,
@@ -936,7 +1124,6 @@ fn queue_summary(state: &AppState) -> Result<Value, String> {
let mut counts = Map::new();
let mut total = 0_i64;
let mut unread_terminal_total = 0_i64;
let mut database_active_task_ids = Vec::new();
for row in rows {
let id = safe_queue_id(row.get::<_, String>("id").as_str());
let status_counts = json!({
@@ -956,9 +1143,6 @@ fn queue_summary(state: &AppState) -> Result<Value, String> {
let unread_terminal = row.get::<_, i64>("unread_terminal");
total += queue_total;
unread_terminal_total += unread_terminal;
if let Some(active) = row.get::<_, Option<String>>("active_task_id") {
database_active_task_ids.push(active);
}
queues.push(json!({
"id": id,
"name": row.get::<_, String>("name"),
@@ -972,6 +1156,30 @@ fn queue_summary(state: &AppState) -> Result<Value, String> {
"updatedAt": row.get::<_, Option<String>>("updated_at")
}));
}
let active_rows = client
.query(
"
SELECT id, queue_id, status, provider_id, execution_mode, model, cwd, prompt, base_prompt,
reference_task_ids, reference_injection, reasoning_effort, max_attempts, current_attempt,
current_mode, codex_thread_id, active_turn_id,
to_json(created_at)#>>'{}' AS created_at,
to_json(updated_at)#>>'{}' AS updated_at,
to_json(started_at)#>>'{}' AS started_at,
to_json(finished_at)#>>'{}' AS finished_at,
to_json(read_at)#>>'{}' AS read_at,
last_error, last_judge, output_count, event_count, attempt_count, last_output_seq,
task_json - 'output' - 'events' - 'promptHistory' AS task_json
FROM unidesk_code_queue_tasks
WHERE status IN ('running', 'judging')
ORDER BY updated_at DESC, id ASC
",
&[],
)
.map_err(|error| error.to_string())?;
let active_tasks: Vec<TaskMeta> = active_rows.iter().map(|row| row_to_task(row, true)).collect();
let mut database_active_task_ids: Vec<String> = active_tasks.iter().map(|task| task.id.clone()).collect();
database_active_task_ids.sort();
let execution_diagnostics = execution_diagnostics_from_tasks(&active_tasks, &now_iso());
if !queues.iter().any(|queue| queue.get("id").and_then(Value::as_str) == Some(DEFAULT_QUEUE_ID)) {
let now = now_iso();
queues.insert(0, json!({
@@ -1009,6 +1217,8 @@ fn queue_summary(state: &AppState) -> Result<Value, String> {
"databaseActiveTaskIds": database_active_task_ids,
"databaseActiveTaskCount": database_active_task_ids.len(),
"executionStateSource": "postgres-control-plane",
"executionDiagnostics": execution_diagnostics,
"schedulerHeartbeatStaleMs": SCHEDULER_HEARTBEAT_STALE_MS,
"orphanedActiveTaskIds": [],
"orphanedActiveTaskCount": 0,
"processing": false,
@@ -1793,6 +2003,7 @@ fn task_summary(task: &TaskMeta) -> Value {
"currentMode": task.current_mode,
"judgeFailCount": task.task_json.as_ref().and_then(|value| value.get("judgeFailCount")).and_then(Value::as_i64).unwrap_or(0),
"judgeFailRetryLimit": 3,
"schedulerHeartbeat": task_scheduler_heartbeat(task).unwrap_or(Value::Null),
"codexThreadId": task.codex_thread_id,
"activeTurnId": task.active_turn_id,
"createdAt": task.created_at,
@@ -1835,6 +2046,7 @@ fn trace_summary(task: &TaskMeta) -> Value {
"stepCount": task.task_json.as_ref().and_then(|value| value.get("stepCount")).and_then(Value::as_i64).unwrap_or(outputs.len() as i64),
"retainedStepCount": outputs.len(),
"outputMaxSeq": output_max_seq(task),
"schedulerHeartbeat": task_scheduler_heartbeat(task).unwrap_or(Value::Null),
"statsSource": "code-queue-mgr-rust-postgres",
"attempts": attempts_array(task),
"prompt": { "initialPreview": preview(&task.prompt, 1600), "basePreview": preview(&task.base_prompt, 1600), "chars": task.prompt.chars().count(), "lines": task.prompt.lines().count() },
@@ -2144,6 +2356,7 @@ fn health(state: &AppState) -> (Value, u16) {
"schemaReady": ready,
"schemaLastError": last_error,
"taskCount": task_count,
"queue": if ready { queue_summary(state).unwrap_or(Value::Null) } else { Value::Null },
"resourceBudget": {
"targetMemoryMb": 100,
"mgrPoolMax": state.config.mgr_pool_max,
src/components/microservices/code-queue-mgr/src/index.ts
+112
View File
@@ -166,6 +166,7 @@ interface QueueTask {
currentMode: RunMode | null;
codexThreadId: string | null;
activeTurnId: string | null;
schedulerHeartbeat?: JsonRecord | null;
finalResponse: string;
stepCount?: number;
llmStepCount?: number;
@@ -287,6 +288,7 @@ const workdirMaxLength = 512;
const defaultCodeModels = ["gpt-5.5", "gpt-5.4-mini", "gpt-5.4", "minimax-m2.7"];
const codeExecutionModes: CodeExecutionMode[] = ["default", "windows-native"];
const codexStatsTimeZone = "Asia/Shanghai";
const schedulerHeartbeatStaleMs = 5 * 60 * 1000;
const codexStatsDateFormatter = new Intl.DateTimeFormat("en-CA", {
timeZone: codexStatsTimeZone,
year: "numeric",
@@ -808,6 +810,108 @@ function outputMaxSeq(task: QueueTask): number {
return Math.max(0, numberField(task.outputMaxSeq, 0), ...outputSeq, ...promptSeq, ...attemptSeq);
}
function activeTask(task: QueueTask): boolean {
return task.status === "running" || task.status === "judging";
}
function schedulerHeartbeat(task: QueueTask): JsonRecord | null {
const heartbeat = asRecord(task.schedulerHeartbeat);
return heartbeat !== null && heartbeat.taskId === task.id ? heartbeat as JsonRecord : null;
}
function heartbeatTimestamp(heartbeat: JsonRecord | null, key: string): string | null {
const value = heartbeat?.[key];
return typeof value === "string" && timestampMs(value) !== null ? value : null;
}
function maxTimestamp(values: Array<string | null | undefined>): string | null {
let best: string | null = null;
let bestMs = -Infinity;
for (const value of values) {
const ms = timestampMs(value ?? null);
if (ms === null || ms < bestMs) continue;
best = value ?? null;
bestMs = ms;
}
return best;
}
function heartbeatFresh(heartbeat: JsonRecord | null, nowMs: number): boolean {
const atMs = timestampMs(heartbeatTimestamp(heartbeat, "lastLocalHeartbeatAt"));
return atMs !== null && nowMs - atMs < schedulerHeartbeatStaleMs;
}
function traceGap(task: QueueTask, heartbeat: JsonRecord | null, nowMs: number): boolean {
const traceAt = timestampMs(heartbeatTimestamp(heartbeat, "lastPersistedTraceAt"));
return traceAt !== null && nowMs - traceAt >= schedulerHeartbeatStaleMs && outputMaxSeq(task) > 0;
}
function controlPlaneExecutionDiagnostics(tasks: QueueTask[], now = nowIso()): JsonRecord {
const nowMs = timestampMs(now) ?? Date.now();
const databaseActiveTaskIds = tasks.filter(activeTask).map((task) => task.id).sort();
const heartbeatRows = tasks.map((task) => ({ task, heartbeat: schedulerHeartbeat(task) })).filter((row): row is { task: QueueTask; heartbeat: JsonRecord } => row.heartbeat !== null);
const activeHeartbeatTaskIds = heartbeatRows.map((row) => String(row.heartbeat.taskId ?? row.task.id)).sort();
const heartbeatFreshTaskIds = heartbeatRows.filter((row) => heartbeatFresh(row.heartbeat, nowMs)).map((row) => String(row.heartbeat.taskId ?? row.task.id)).sort();
const heartbeatFreshSet = new Set(heartbeatFreshTaskIds);
const heartbeatExpiredTaskIds = heartbeatRows.filter((row) => activeTask(row.task) && !heartbeatFreshSet.has(row.task.id)).map((row) => row.task.id).sort();
const activeHeartbeatSet = new Set(activeHeartbeatTaskIds);
const heartbeatMissingTaskIds = databaseActiveTaskIds.filter((taskId) => !activeHeartbeatSet.has(taskId));
const staleRecoveryCandidateTaskIds = heartbeatExpiredTaskIds.slice();
const traceGapTaskIds = heartbeatRows.filter((row) => traceGap(row.task, row.heartbeat, nowMs)).map((row) => row.task.id).sort();
const traceGapNotStaleTaskIds = traceGapTaskIds.filter((taskId) => heartbeatFreshSet.has(taskId));
const splitBrain = databaseActiveTaskIds.length > 0 && heartbeatFreshTaskIds.length > 0;
const staleActive = staleRecoveryCandidateTaskIds.length > 0;
const degraded = splitBrain
|| staleActive
|| heartbeatMissingTaskIds.length > 0
|| heartbeatExpiredTaskIds.length > 0
|| traceGapTaskIds.length > 0;
const state = splitBrain ? "split-brain" : staleActive ? "stale-active" : degraded ? "degraded" : "healthy";
const reasons: string[] = [];
if (splitBrain) reasons.push("postgres control-plane has database-active tasks while its local active slots are empty, but scheduler heartbeat is fresh");
if (staleActive) reasons.push("owner heartbeat is expired and scheduler has no local active run for at least one database-active task");
if (heartbeatMissingTaskIds.length > 0) reasons.push("database-active tasks are missing scheduler-owned heartbeat");
if (traceGapNotStaleTaskIds.length > 0) reasons.push("trace progress is stale while scheduler heartbeat is fresh; this is a trace gap, not stale active");
return {
state,
health: state,
degraded,
splitBrain,
executionStateSource: "postgres-control-plane",
controlPlane: "master-code-queue-mgr",
databaseActiveTaskIds,
databaseActiveTaskCount: databaseActiveTaskIds.length,
schedulerActiveTaskIds: [],
schedulerActiveTaskCount: 0,
schedulerActiveRunSlotCount: 0,
schedulerActiveQueueIds: [],
schedulerProcessingQueueIds: [],
schedulerOrphanedActiveTaskIds: databaseActiveTaskIds,
schedulerOrphanedActiveTaskCount: databaseActiveTaskIds.length,
activeHeartbeatTaskIds,
activeHeartbeatCount: activeHeartbeatTaskIds.length,
heartbeatFreshTaskIds,
heartbeatExpiredTaskIds,
heartbeatMissingTaskIds,
staleRecoveryCandidateTaskIds,
traceGapTaskIds,
traceGapNotStaleTaskIds,
schedulerHeartbeatStaleMs,
now,
lastSchedulerHeartbeatAt: maxTimestamp(heartbeatRows.map((row) => heartbeatTimestamp(row.heartbeat, "lastLocalHeartbeatAt"))),
lastObservedAgentEventAt: maxTimestamp(heartbeatRows.map((row) => heartbeatTimestamp(row.heartbeat, "lastObservedAgentEventAt"))),
lastPersistedTraceAt: maxTimestamp(heartbeatRows.map((row) => heartbeatTimestamp(row.heartbeat, "lastPersistedTraceAt"))),
oaPublisher: null,
reasons,
guidance: [
"Do not use master activeRunSlotCount=0 alone to declare a task stuck.",
"Check PostgreSQL task status, scheduler active runs/slots, scheduler heartbeat, and trace/OA publisher progress together.",
"Trace gap with a fresh scheduler heartbeat must not trigger stale retry.",
"Stale recovery is allowed only when the scheduler has no local active run and the owner heartbeat is expired.",
],
};
}
function normalizeOutput(value: unknown): LiveOutput[] {
if (!Array.isArray(value)) return [];
return value.flatMap((item): LiveOutput[] => {
@@ -876,6 +980,7 @@ function normalizeTask(value: unknown): QueueTask {
currentMode: record.currentMode === "initial" || record.currentMode === "retry" ? record.currentMode : null,
codexThreadId: typeof record.codexThreadId === "string" ? record.codexThreadId : null,
activeTurnId: typeof record.activeTurnId === "string" ? record.activeTurnId : null,
schedulerHeartbeat: asRecord(record.schedulerHeartbeat) as JsonRecord | null,
finalResponse: typeof record.finalResponse === "string" ? record.finalResponse : "",
lastError: typeof record.lastError === "string" ? record.lastError : null,
lastJudge: toJsonValue(record.lastJudge ?? null),
@@ -1153,6 +1258,7 @@ async function createTaskFromRequest(value: unknown): Promise<QueueTask> {
currentMode: null,
codexThreadId: null,
activeTurnId: null,
schedulerHeartbeat: null,
finalResponse: "",
stepCount: 0,
llmStepCount: 0,
@@ -1581,6 +1687,8 @@ async function queueSummary(tasks?: QueueTask[], queueRecords?: QueueRecord[]):
databaseActiveTaskIds,
databaseActiveTaskCount: databaseActiveTaskIds.length,
executionStateSource: "postgres-control-plane",
executionDiagnostics: controlPlaneExecutionDiagnostics(taskRows),
schedulerHeartbeatStaleMs,
orphanedActiveTaskIds: [],
orphanedActiveTaskCount: 0,
processing: false,
@@ -1817,6 +1925,7 @@ function taskListResponse(task: QueueTask, lite = true): JsonRecord {
currentMode: task.currentMode,
judgeFailCount: task.judgeFailCount,
judgeFailRetryLimit: 3,
schedulerHeartbeat: task.schedulerHeartbeat ?? null,
codexThreadId: task.codexThreadId,
activeTurnId: task.activeTurnId,
lastError: task.lastError,
@@ -2280,6 +2389,7 @@ function traceSummary(task: QueueTask): JsonRecord {
stepCount: numberField(task.stepCount ?? task.llmStepCount, steps.length),
retainedStepCount: steps.length,
outputMaxSeq: outputMaxSeq(task),
schedulerHeartbeat: task.schedulerHeartbeat ?? null,
statsSource: "code-queue-mgr-postgres",
attempts: task.attempts.map((attempt) => ({
index: attempt.index,
@@ -2766,6 +2876,7 @@ async function route(req: Request): Promise<Response> {
try {
if (url.pathname === "/" || url.pathname === "/health" || url.pathname === "/live") {
const taskCount = schemaReady ? Number((await traceSql<Array<{ count: string | number }>>`SELECT COUNT(*) AS count FROM unidesk_code_queue_tasks`)[0]?.count ?? 0) : null;
const queue = schemaReady ? await queueSummary() : null;
return jsonResponse({
ok: schemaReady,
service: "code-queue-mgr",
@@ -2780,6 +2891,7 @@ async function route(req: Request): Promise<Response> {
schemaReady,
schemaLastError,
taskCount,
queue,
resourceBudget: {
targetMemoryMb: 100,
mgrPoolMax: config.mgrDatabasePoolMax,