fix(code-queue): add active-run liveness diagnostics
This commit is contained in:
Codex
@@ -2169,6 +2169,81 @@ input:focus, select:focus, textarea:focus { border-color: var(--accent-2); }
|
||||
.codex-trace-status-chip.service {
|
||||
white-space: normal;
|
||||
}
|
||||
.codex-trace-status-chip.liveness.ok {
|
||||
border-color: rgba(78, 183, 168, 0.50);
|
||||
color: var(--accent-2);
|
||||
}
|
||||
.codex-trace-status-chip.liveness.warn {
|
||||
border-color: rgba(215, 161, 58, 0.55);
|
||||
color: #ffe0a2;
|
||||
}
|
||||
.codex-trace-status-chip.liveness.failed {
|
||||
border-color: rgba(255, 98, 98, 0.58);
|
||||
color: #ffb2b2;
|
||||
}
|
||||
.codex-liveness-panel .panel-body {
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
}
|
||||
.codex-liveness-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, minmax(130px, 1fr));
|
||||
gap: 8px;
|
||||
min-width: 0;
|
||||
}
|
||||
.codex-liveness-metric {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
min-width: 0;
|
||||
padding: 8px;
|
||||
border: 1px solid rgba(78, 183, 168, 0.22);
|
||||
background:
|
||||
linear-gradient(135deg, rgba(78, 183, 168, 0.08), rgba(255,255,255,0.015)),
|
||||
rgba(0,0,0,0.16);
|
||||
}
|
||||
.codex-liveness-metric.warn {
|
||||
border-color: rgba(215, 161, 58, 0.44);
|
||||
}
|
||||
.codex-liveness-metric.failed {
|
||||
border-color: rgba(255, 98, 98, 0.46);
|
||||
}
|
||||
.codex-liveness-metric.ok {
|
||||
border-color: rgba(78, 183, 168, 0.45);
|
||||
}
|
||||
.codex-liveness-metric span {
|
||||
color: var(--muted);
|
||||
font-size: 10px;
|
||||
letter-spacing: 0.12em;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
.codex-liveness-metric strong {
|
||||
min-width: 0;
|
||||
color: var(--text);
|
||||
font-size: 15px;
|
||||
font-weight: 700;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.codex-liveness-metric code {
|
||||
min-width: 0;
|
||||
color: var(--muted);
|
||||
font-size: 11px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.codex-liveness-reasons {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 5px;
|
||||
min-width: 0;
|
||||
}
|
||||
.codex-liveness-reasons span {
|
||||
max-width: 100%;
|
||||
padding: 4px 7px;
|
||||
border: 1px solid rgba(215, 161, 58, 0.28);
|
||||
color: #ffe0a2;
|
||||
background: rgba(215, 161, 58, 0.07);
|
||||
font-size: 11px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.codex-mark-all-read-btn {
|
||||
border-color: rgba(78, 183, 168, 0.40);
|
||||
color: #bdece4;
|
||||
@@ -5490,7 +5565,7 @@ input:focus, select:focus, textarea:focus { border-color: var(--accent-2); }
|
||||
}
|
||||
|
||||
@media (max-width: 1120px) {
|
||||
.metric-grid, .policy-grid, .security-board, .docker-metrics, .monitor-chart-grid, .monitor-summary-grid, .performance-metric-stack, .codex-load-test-grid, .baidu-doc-grid, .filebrowser-target-grid, .oa-flow-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.metric-grid, .policy-grid, .security-board, .docker-metrics, .monitor-chart-grid, .monitor-summary-grid, .performance-metric-stack, .codex-load-test-grid, .codex-liveness-grid, .baidu-doc-grid, .filebrowser-target-grid, .oa-flow-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.pipeline-oa-guarantees { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.dispatch-form, .schedule-form { grid-template-columns: 1fr 1fr; }
|
||||
.dispatch-actions { align-items: center; }
|
||||
@@ -5680,7 +5755,7 @@ input:focus, select:focus, textarea:focus { border-color: var(--accent-2); }
|
||||
padding: 4px 9px;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.metric-grid, .policy-grid, .security-board, .dispatch-form, .schedule-form, .schedule-card-grid, .docker-metrics, .monitor-chart-grid, .monitor-summary-grid, .gateway-record-grid, .met-detail-kv, .code-queue-metrics, .codex-stats-summary-grid, .codex-form-grid, .baidu-doc-grid { grid-template-columns: 1fr; }
|
||||
.metric-grid, .policy-grid, .security-board, .dispatch-form, .schedule-form, .schedule-card-grid, .docker-metrics, .monitor-chart-grid, .monitor-summary-grid, .gateway-record-grid, .met-detail-kv, .code-queue-metrics, .codex-stats-summary-grid, .codex-liveness-grid, .codex-form-grid, .baidu-doc-grid { grid-template-columns: 1fr; }
|
||||
.compact-row, .heartbeat-row, .log-row, .endpoint-list article, .volume-route, .findjob-hero, .pipeline-hero, .code-queue-hero, .claudeqq-login-card, .baidu-login-card, .baidu-pathbar { grid-template-columns: 1fr; align-items: start; }
|
||||
.codex-output-line { grid-template-columns: 1fr; }
|
||||
.codex-transcript { min-height: 360px; }
|
||||
|
||||
@@ -219,6 +219,34 @@ function activeTaskIds(queue: any): string[] {
|
||||
return Array.isArray(queue?.activeTaskIds) ? queue.activeTaskIds.map((id: any) => String(id || "")).filter(Boolean) : [String(queue?.activeTaskId || "")].filter(Boolean);
|
||||
}
|
||||
|
||||
function stringArray(value: any): string[] {
|
||||
return Array.isArray(value) ? value.map((item: any) => String(item || "")).filter(Boolean) : [];
|
||||
}
|
||||
|
||||
function compactIdList(value: any, limit = 3): string {
|
||||
const ids = stringArray(value);
|
||||
if (ids.length === 0) return "--";
|
||||
const visible = ids.slice(0, limit).join(" / ");
|
||||
return ids.length > limit ? `${visible} +${ids.length - limit}` : visible;
|
||||
}
|
||||
|
||||
function executionDiagnosticsFromQueue(queue: any, health: any): AnyRecord {
|
||||
return objectRecord(queue?.executionDiagnostics)
|
||||
|| objectRecord(health?.body?.queue?.executionDiagnostics)
|
||||
|| objectRecord(health?.queue?.executionDiagnostics)
|
||||
|| objectRecord(health?.body?.executionDiagnostics)
|
||||
|| objectRecord(health?.executionDiagnostics)
|
||||
|| {};
|
||||
}
|
||||
|
||||
function diagnosticsTone(state: any): string {
|
||||
const value = String(state || "unknown").toLowerCase();
|
||||
if (value === "healthy") return "ok";
|
||||
if (value === "split-brain" || value === "stale-active") return "failed";
|
||||
if (value === "degraded") return "warn";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
const allQueuesId = "__all__";
|
||||
const queueMobileMediaQuery = "(max-width: 760px)";
|
||||
const queueDesktopMediaQuery = "(min-width: 761px)";
|
||||
@@ -1536,6 +1564,48 @@ function CodexStatsIcon() {
|
||||
);
|
||||
}
|
||||
|
||||
function LivenessMetric({ label, value, hint, tone }: AnyRecord) {
|
||||
return h("div", { className: `codex-liveness-metric ${tone || ""}` },
|
||||
h("span", null, label),
|
||||
h("strong", null, value ?? "--"),
|
||||
hint ? h("code", null, hint) : null,
|
||||
);
|
||||
}
|
||||
|
||||
function CodeQueueLivenessPanel({ diagnostics, queue, onRaw }: AnyRecord) {
|
||||
const state = String(diagnostics?.state || diagnostics?.health || "unknown");
|
||||
const oaPublisher = objectRecord(diagnostics?.oaPublisher);
|
||||
const reasons = stringArray(diagnostics?.reasons).slice(0, 3);
|
||||
const tone = diagnosticsTone(state);
|
||||
return h(Panel, {
|
||||
title: "执行活性",
|
||||
eyebrow: `${String(diagnostics?.executionStateSource || queue?.executionStateSource || "unknown")} / ${String(diagnostics?.controlPlane || "code-queue")}`,
|
||||
summary: h("div", { className: "codex-trace-status" },
|
||||
h("span", { className: `codex-trace-status-chip liveness ${tone}` }, h("b", null, "状态"), state),
|
||||
h("span", { className: "codex-trace-status-chip" }, h("b", null, "DB active"), String(diagnostics?.databaseActiveTaskCount ?? queue?.databaseActiveTaskCount ?? 0)),
|
||||
h("span", { className: "codex-trace-status-chip" }, h("b", null, "scheduler slots"), String(diagnostics?.schedulerActiveRunSlotCount ?? queue?.activeRunSlotCount ?? 0)),
|
||||
h("span", { className: "codex-trace-status-chip" }, h("b", null, "heartbeat"), `${stringArray(diagnostics?.heartbeatFreshTaskIds).length} fresh / ${stringArray(diagnostics?.heartbeatExpiredTaskIds).length} expired`),
|
||||
oaPublisher ? h("span", { className: "codex-trace-status-chip" }, h("b", null, "OA"), `${Number(oaPublisher.pending || 0)} pending${oaPublisher.lastError ? " / error" : ""}`) : null,
|
||||
),
|
||||
actions: h(RawButton, { title: "Code Queue Execution Diagnostics", data: diagnostics, onOpen: onRaw, testId: "raw-code-queue-execution-diagnostics" }),
|
||||
className: "codex-liveness-panel",
|
||||
},
|
||||
h("div", { className: "codex-liveness-grid", "data-testid": "codex-liveness-diagnostics" },
|
||||
h(LivenessMetric, { tone, label: "健康状态", value: state, hint: diagnostics?.splitBrain ? "split-brain" : diagnostics?.degraded ? "degraded" : "ready" }),
|
||||
h(LivenessMetric, { label: "PostgreSQL active", value: String(diagnostics?.databaseActiveTaskCount ?? queue?.databaseActiveTaskCount ?? 0), hint: compactIdList(diagnostics?.databaseActiveTaskIds ?? queue?.databaseActiveTaskIds) }),
|
||||
h(LivenessMetric, { label: "Scheduler active", value: String(diagnostics?.schedulerActiveRunSlotCount ?? queue?.activeRunSlotCount ?? 0), hint: compactIdList(diagnostics?.schedulerActiveTaskIds ?? queue?.activeTaskIds) }),
|
||||
h(LivenessMetric, { label: "Fresh heartbeat", value: String(stringArray(diagnostics?.heartbeatFreshTaskIds).length), hint: compactIdList(diagnostics?.heartbeatFreshTaskIds) }),
|
||||
h(LivenessMetric, { tone: stringArray(diagnostics?.traceGapNotStaleTaskIds).length > 0 ? "warn" : "", label: "Trace gap", value: String(stringArray(diagnostics?.traceGapTaskIds).length), hint: compactIdList(diagnostics?.traceGapNotStaleTaskIds) }),
|
||||
h(LivenessMetric, { tone: stringArray(diagnostics?.staleRecoveryCandidateTaskIds).length > 0 ? "failed" : "", label: "Stale candidates", value: String(stringArray(diagnostics?.staleRecoveryCandidateTaskIds).length), hint: compactIdList(diagnostics?.staleRecoveryCandidateTaskIds) }),
|
||||
h(LivenessMetric, { label: "Last scheduler heartbeat", value: fmtRelativeAge(diagnostics?.lastSchedulerHeartbeatAt), hint: String(diagnostics?.lastSchedulerHeartbeatAt || "--") }),
|
||||
h(LivenessMetric, { label: "Last agent event", value: fmtRelativeAge(diagnostics?.lastObservedAgentEventAt), hint: String(diagnostics?.lastObservedAgentEventAt || "--") }),
|
||||
h(LivenessMetric, { label: "Last trace persist", value: fmtRelativeAge(diagnostics?.lastPersistedTraceAt), hint: String(diagnostics?.lastPersistedTraceAt || "--") }),
|
||||
h(LivenessMetric, { tone: oaPublisher?.lastError ? "warn" : "", label: "OA publisher", value: `${Number(oaPublisher?.pending || 0)} pending`, hint: oaPublisher?.lastError ? shortText(oaPublisher.lastError, 90) : "ok" }),
|
||||
),
|
||||
reasons.length > 0 ? h("div", { className: "codex-liveness-reasons" }, reasons.map((reason: string) => h("span", { key: reason }, reason))) : null,
|
||||
);
|
||||
}
|
||||
|
||||
function CodexStatsPanel({ stats, queueName: activeQueueName, onRaw }: AnyRecord) {
|
||||
const rows = taskStatisticsRows(stats);
|
||||
const totals = taskStatisticsTotals(stats);
|
||||
@@ -2112,6 +2182,7 @@ export function CodeQueuePage({ microservices, onRaw, apiBaseUrl = "/api", initi
|
||||
const tasks = applyLocalReadStateToRows(taskRows(tasksData));
|
||||
const loadedUnreadTerminalTasks = tasks.filter(taskIsUnreadTerminal);
|
||||
const queue = tasksData?.queue || health?.body?.queue || health?.queue || {};
|
||||
const executionDiagnostics = executionDiagnosticsFromQueue(queue, health);
|
||||
const statistics = taskStatistics(tasksData, queue);
|
||||
const pagination = taskPagination(tasksData);
|
||||
const queueRows = knownQueueRows(queue, queueId);
|
||||
@@ -3828,6 +3899,7 @@ export function CodeQueuePage({ microservices, onRaw, apiBaseUrl = "/api", initi
|
||||
h(UniDeskErrorBanner, { error, wide: true }),
|
||||
mergeDialog,
|
||||
h("div", { className: "codex-session-stage codex-session-stage-top" },
|
||||
h(CodeQueueLivenessPanel, { diagnostics: executionDiagnostics, queue, onRaw }),
|
||||
sessionPanel,
|
||||
),
|
||||
h("div", { className: "code-queue-layout" },
|
||||
|
||||
@@ -782,6 +782,13 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
|
||||
if !trace_gap_not_stale_task_ids.is_empty() {
|
||||
reasons.push("trace progress is stale while scheduler heartbeat is fresh; this is a trace gap, not stale active");
|
||||
}
|
||||
let database_active_task_count = database_active_task_ids.len();
|
||||
let scheduler_orphaned_active_task_ids = database_active_task_ids.clone();
|
||||
let scheduler_orphaned_active_task_count = scheduler_orphaned_active_task_ids.len();
|
||||
let active_heartbeat_count = active_heartbeat_task_ids.len();
|
||||
let last_scheduler_heartbeat_at = max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastLocalHeartbeatAt")).collect());
|
||||
let last_observed_agent_event_at = max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastObservedAgentEventAt")).collect());
|
||||
let last_persisted_trace_at = max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastPersistedTraceAt")).collect());
|
||||
|
||||
json!({
|
||||
"state": state,
|
||||
@@ -791,16 +798,16 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
|
||||
"executionStateSource": "postgres-control-plane",
|
||||
"controlPlane": "master-code-queue-mgr",
|
||||
"databaseActiveTaskIds": database_active_task_ids,
|
||||
"databaseActiveTaskCount": database_active_task_ids.len(),
|
||||
"databaseActiveTaskCount": database_active_task_count,
|
||||
"schedulerActiveTaskIds": [],
|
||||
"schedulerActiveTaskCount": 0,
|
||||
"schedulerActiveRunSlotCount": 0,
|
||||
"schedulerActiveQueueIds": [],
|
||||
"schedulerProcessingQueueIds": [],
|
||||
"schedulerOrphanedActiveTaskIds": database_active_task_ids,
|
||||
"schedulerOrphanedActiveTaskCount": database_active_task_ids.len(),
|
||||
"schedulerOrphanedActiveTaskIds": scheduler_orphaned_active_task_ids,
|
||||
"schedulerOrphanedActiveTaskCount": scheduler_orphaned_active_task_count,
|
||||
"activeHeartbeatTaskIds": active_heartbeat_task_ids,
|
||||
"activeHeartbeatCount": active_heartbeat_task_ids.len(),
|
||||
"activeHeartbeatCount": active_heartbeat_count,
|
||||
"heartbeatFreshTaskIds": heartbeat_fresh_task_ids,
|
||||
"heartbeatExpiredTaskIds": heartbeat_expired_task_ids,
|
||||
"heartbeatMissingTaskIds": heartbeat_missing_task_ids,
|
||||
@@ -809,9 +816,9 @@ fn execution_diagnostics_from_tasks(tasks: &[TaskMeta], now: &str) -> Value {
|
||||
"traceGapNotStaleTaskIds": trace_gap_not_stale_task_ids,
|
||||
"schedulerHeartbeatStaleMs": SCHEDULER_HEARTBEAT_STALE_MS,
|
||||
"now": now,
|
||||
"lastSchedulerHeartbeatAt": max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastLocalHeartbeatAt")).collect()),
|
||||
"lastObservedAgentEventAt": max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastObservedAgentEventAt")).collect()),
|
||||
"lastPersistedTraceAt": max_timestamp(active_heartbeats.iter().map(|(_, heartbeat)| heartbeat_string_field(heartbeat, "lastPersistedTraceAt")).collect()),
|
||||
"lastSchedulerHeartbeatAt": last_scheduler_heartbeat_at,
|
||||
"lastObservedAgentEventAt": last_observed_agent_event_at,
|
||||
"lastPersistedTraceAt": last_persisted_trace_at,
|
||||
"oaPublisher": Value::Null,
|
||||
"reasons": reasons,
|
||||
"guidance": [
|
||||
@@ -1179,6 +1186,7 @@ fn queue_summary(state: &AppState) -> Result<Value, String> {
|
||||
let active_tasks: Vec<TaskMeta> = active_rows.iter().map(|row| row_to_task(row, true)).collect();
|
||||
let mut database_active_task_ids: Vec<String> = active_tasks.iter().map(|task| task.id.clone()).collect();
|
||||
database_active_task_ids.sort();
|
||||
let database_active_task_count = database_active_task_ids.len();
|
||||
let execution_diagnostics = execution_diagnostics_from_tasks(&active_tasks, &now_iso());
|
||||
if !queues.iter().any(|queue| queue.get("id").and_then(Value::as_str) == Some(DEFAULT_QUEUE_ID)) {
|
||||
let now = now_iso();
|
||||
@@ -1215,7 +1223,7 @@ fn queue_summary(state: &AppState) -> Result<Value, String> {
|
||||
"activeTaskIds": [],
|
||||
"activeTaskId": Value::Null,
|
||||
"databaseActiveTaskIds": database_active_task_ids,
|
||||
"databaseActiveTaskCount": database_active_task_ids.len(),
|
||||
"databaseActiveTaskCount": database_active_task_count,
|
||||
"executionStateSource": "postgres-control-plane",
|
||||
"executionDiagnostics": execution_diagnostics,
|
||||
"schedulerHeartbeatStaleMs": SCHEDULER_HEARTBEAT_STALE_MS,
|
||||
|
||||
@@ -3501,9 +3501,7 @@ function taskIsRecoverableOrphanedActive(task: QueueTask, staleMs = orphanedActi
|
||||
if (task.status !== "running" && task.status !== "judging") return false;
|
||||
if (activeTaskHasLocalRunSlotOrWaiter(task)) return false;
|
||||
const decision = staleRecoveryCandidate({ task, localActive: false, heartbeatStaleMs: staleMs, now: nowIso() });
|
||||
if (decision.allowed) return true;
|
||||
if (decision.reason === "owner-heartbeat-missing") return orphanedActiveTaskAgeMs(task) >= staleMs;
|
||||
return false;
|
||||
return decision.allowed;
|
||||
}
|
||||
|
||||
function schedulerReconcileStatus(tasks: QueueTask[] = state.tasks): JsonValue {
|
||||
|
||||
@@ -256,6 +256,23 @@ function runQueueOrderingSelfTest(): JsonValue {
|
||||
const orphanRunning = queueOrderTestTask("codex_4400_orphan_running", "running", "2026-05-11T13:00:00.000Z", "2026-05-11T13:00:00.000Z");
|
||||
orphanRunning.queueId = "queue_orphan_recovery";
|
||||
orphanRunning.activeTurnId = null;
|
||||
orphanRunning.schedulerHeartbeat = {
|
||||
taskId: orphanRunning.id,
|
||||
queueId: orphanRunning.queueId,
|
||||
attempt: 1,
|
||||
activeTurnId: null,
|
||||
codexThreadId: null,
|
||||
owner: "self-test",
|
||||
schedulerInstance: "self-test",
|
||||
executionPlane: "scheduler-execution-plane",
|
||||
agentPort: "codex",
|
||||
status: "running",
|
||||
lastLocalHeartbeatAt: "2026-05-11T13:00:00.000Z",
|
||||
lastObservedAgentEventAt: null,
|
||||
lastPersistedTraceAt: null,
|
||||
outputMaxSeq: 0,
|
||||
source: "scheduler",
|
||||
};
|
||||
const queuedBehindOrphan = queueOrderTestTask("codex_4401_queued", "queued", "2026-05-11T13:01:00.000Z", "2026-05-11T13:01:00.000Z");
|
||||
queuedBehindOrphan.queueId = "queue_orphan_recovery";
|
||||
const originalMaxActiveQueues = ctx().config.maxActiveQueues;
|
||||
|
||||
Reference in New Issue
Block a user