fix(code-queue): add active-run liveness diagnostics

This commit is contained in:
Codex
2026-05-19 03:48:17 +00:00
parent f36ea37548
commit 2a45bfe180
13 changed files with 561 additions and 17 deletions
@@ -0,0 +1,24 @@
import { CODE_QUEUE_LIVENESS_CHECK_NAMES, runCodeQueueLivenessFixtureChecks } from "./src/code-queue-liveness-fixtures";
function optionValues(args: string[], name: string): string[] {
const values: string[] = [];
for (let index = 0; index < args.length; index += 1) {
if (args[index] !== name) continue;
const raw = args[index + 1];
if (raw === undefined || raw.startsWith("--")) throw new Error(`${name} requires a check name`);
values.push(...raw.split(",").map((item) => item.trim()).filter(Boolean));
index += 1;
}
return values;
}
function main(): void {
const only = optionValues(Bun.argv.slice(2), "--only");
const unknown = only.filter((name) => !CODE_QUEUE_LIVENESS_CHECK_NAMES.includes(name as never));
if (unknown.length > 0) throw new Error(`unknown Code Queue liveness check(s): ${unknown.join(", ")}`);
const result = runCodeQueueLivenessFixtureChecks(only);
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
if (!result.ok) process.exit(1);
}
if (import.meta.main) main();
+8
View File
@@ -237,6 +237,8 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
fileItem("src/components/microservices/code-queue-mgr/src/prompt-observation.ts"),
fileItem("scripts/src/deploy.ts"),
fileItem("scripts/code-queue-issue3-regression-test.ts"),
fileItem("scripts/code-queue-liveness-diagnostics-test.ts"),
fileItem("scripts/src/code-queue-liveness-fixtures.ts"),
fileItem("scripts/src/ci.ts"),
fileItem("scripts/src/e2e.ts"),
fileItem("scripts/code-queue-prompt-observation-test.ts"),
@@ -250,10 +252,16 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default
items.push(commandItem("typescript:scripts", ["bunx", "tsc", "-p", "scripts/tsconfig.json", "--noEmit", "--pretty", "false"], 120_000));
items.push(commandItem("code-queue:prompt-observation-contract", ["bun", "scripts/code-queue-prompt-observation-test.ts"], 30_000));
items.push(commandItem("code-queue:issue3-diagnostics-and-image-preflight", ["bun", "scripts/code-queue-issue3-regression-test.ts"], 30_000));
items.push(commandItem("code-queue:active-run-heartbeat-visible", ["bun", "scripts/code-queue-liveness-diagnostics-test.ts", "--only", "code-queue:active-run-heartbeat-visible"], 30_000));
items.push(commandItem("code-queue:trace-gap-not-stale", ["bun", "scripts/code-queue-liveness-diagnostics-test.ts", "--only", "code-queue:trace-gap-not-stale"], 30_000));
items.push(commandItem("code-queue:stale-active-owner-expired", ["bun", "scripts/code-queue-liveness-diagnostics-test.ts", "--only", "code-queue:stale-active-owner-expired"], 30_000));
items.push(commandItem("code-queue:control-plane-split-brain-diagnostics", ["bun", "scripts/code-queue-liveness-diagnostics-test.ts", "--only", "code-queue:control-plane-split-brain-diagnostics"], 30_000));
items.push(commandItem("code-queue:oa-publisher-degraded-visible", ["bun", "scripts/code-queue-liveness-diagnostics-test.ts", "--only", "code-queue:oa-publisher-degraded-visible"], 30_000));
} else {
items.push(skippedItem("typescript:scripts", "scripts TypeScript typecheck is opt-in", "--scripts-typecheck or --full"));
items.push(skippedItem("code-queue:prompt-observation-contract", "prompt observation contract is opt-in with script checks", "--scripts-typecheck or --full"));
items.push(skippedItem("code-queue:issue3-diagnostics-and-image-preflight", "Code Queue issue #3 regression fixtures are opt-in with script checks", "--scripts-typecheck or --full"));
items.push(skippedItem("code-queue:liveness-diagnostics-fixtures", "Code Queue liveness diagnostics fixtures are opt-in with script checks", "--scripts-typecheck or --full"));
}
if (options.logs) {
items.push(unifiedLogRotationItem());
+259
View File
@@ -0,0 +1,259 @@
import { buildExecutionDiagnostics, buildSchedulerHeartbeat, staleRecoveryCandidate, taskHasTraceGapButFreshHeartbeat } from "../../src/components/microservices/code-queue/src/execution-diagnostics";
import type { ActiveRun } from "../../src/components/microservices/code-queue/src/code-agent/common";
import type { CodeQueueExecutionDiagnostics, QueueTask, SchedulerActiveRunHeartbeat, TaskStatus } from "../../src/components/microservices/code-queue/src/types";
export const CODE_QUEUE_LIVENESS_CHECK_NAMES = [
"code-queue:active-run-heartbeat-visible",
"code-queue:trace-gap-not-stale",
"code-queue:stale-active-owner-expired",
"code-queue:control-plane-split-brain-diagnostics",
"code-queue:oa-publisher-degraded-visible",
] as const;
type CodeQueueLivenessCheckName = typeof CODE_QUEUE_LIVENESS_CHECK_NAMES[number];
interface FixtureCheck {
name: CodeQueueLivenessCheckName;
ok: boolean;
detail: Record<string, unknown>;
}
const now = "2026-05-19T00:10:00.000Z";
const freshAt = "2026-05-19T00:09:50.000Z";
const oldTraceAt = "2026-05-18T23:40:00.000Z";
const expiredAt = "2026-05-18T23:50:00.000Z";
function assertCondition(condition: unknown, message: string, detail: Record<string, unknown> = {}): void {
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
}
function fixtureTask(id: string, status: TaskStatus, heartbeat: SchedulerActiveRunHeartbeat | null = null): QueueTask {
return {
id,
queueId: "default",
queueEnteredAt: "2026-05-19T00:00:00.000Z",
prompt: `${id} prompt`,
basePrompt: `${id} prompt`,
referenceTaskIds: [],
referenceInjection: null,
providerId: "D601",
cwd: "/workspace",
model: "gpt-5.5",
reasoningEffort: null,
executionMode: "default",
maxAttempts: 99,
status,
createdAt: "2026-05-19T00:00:00.000Z",
updatedAt: "2026-05-19T00:00:00.000Z",
startedAt: status === "running" || status === "judging" ? "2026-05-19T00:00:00.000Z" : null,
finishedAt: null,
readAt: null,
currentAttempt: status === "queued" ? 0 : 1,
currentMode: status === "queued" ? null : "initial",
codexThreadId: heartbeat?.codexThreadId ?? null,
activeTurnId: heartbeat?.activeTurnId ?? null,
schedulerHeartbeat: heartbeat,
finalResponse: "",
outputMaxSeq: heartbeat?.outputMaxSeq ?? 0,
lastError: null,
lastJudge: null,
judgeFailCount: 0,
promptHistory: [],
output: [],
events: [],
attempts: [],
cancelRequested: false,
nextPrompt: null,
nextMode: null,
};
}
function heartbeat(taskId: string, at: string, overrides: Partial<SchedulerActiveRunHeartbeat> = {}): SchedulerActiveRunHeartbeat {
return {
taskId,
queueId: "default",
attempt: 1,
activeTurnId: "turn_fixture",
codexThreadId: "thread_fixture",
owner: "D601",
schedulerInstance: "code-queue-scheduler-fixture",
executionPlane: "scheduler-execution-plane",
agentPort: "codex",
status: "running",
lastLocalHeartbeatAt: at,
lastObservedAgentEventAt: at,
lastPersistedTraceAt: at,
outputMaxSeq: 10,
source: "scheduler",
...overrides,
};
}
function activeRun(taskId: string, queueId = "default"): ActiveRun {
return {
taskId,
queueId,
app: { stop: () => undefined },
port: "codex",
threadId: "thread_fixture",
turnId: "turn_fixture",
startedAt: "2026-05-19T00:00:00.000Z",
lastLocalHeartbeatAt: freshAt,
lastObservedAgentEventAt: freshAt,
lastPersistedTraceAt: freshAt,
};
}
function schedulerDiagnostics(tasks: QueueTask[], activeRuns: ActiveRun[] = [], oaPublisher: unknown = null): CodeQueueExecutionDiagnostics {
return buildExecutionDiagnostics({
now,
controlPlane: "D601-code-queue-scheduler",
executionStateSource: "scheduler-execution-plane",
tasks,
activeRuns,
activeRunSlotCount: activeRuns.length,
activeQueueIds: activeRuns.map((run) => run.queueId),
processingQueueIds: [],
orphanedActiveTaskIds: tasks.filter((task) => (task.status === "running" || task.status === "judging") && !activeRuns.some((run) => run.taskId === task.id)).map((task) => task.id),
oaPublisher: oaPublisher as never,
});
}
function checkActiveRunHeartbeatVisible(): FixtureCheck {
const task = fixtureTask("codex_fixture_active_1", "running");
const run = activeRun(task.id);
task.schedulerHeartbeat = buildSchedulerHeartbeat(task, run, {
now: freshAt,
owner: "D601",
schedulerInstance: "code-queue-scheduler-fixture",
agentPort: "codex",
lastObservedAgentEventAt: freshAt,
lastPersistedTraceAt: freshAt,
});
const diagnostics = schedulerDiagnostics([task], [run]);
assertCondition(diagnostics.activeHeartbeatTaskIds.includes(task.id), "active heartbeat task id must be visible", diagnostics as unknown as Record<string, unknown>);
assertCondition(diagnostics.schedulerActiveTaskIds.includes(task.id), "scheduler active task id must be visible", diagnostics as unknown as Record<string, unknown>);
assertCondition(diagnostics.lastSchedulerHeartbeatAt === freshAt, "last scheduler heartbeat must be surfaced", diagnostics as unknown as Record<string, unknown>);
return {
name: "code-queue:active-run-heartbeat-visible",
ok: true,
detail: {
state: diagnostics.state,
schedulerActiveTaskIds: diagnostics.schedulerActiveTaskIds,
activeHeartbeatTaskIds: diagnostics.activeHeartbeatTaskIds,
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt,
heartbeat: task.schedulerHeartbeat,
},
};
}
function checkTraceGapNotStale(): FixtureCheck {
const task = fixtureTask("codex_fixture_trace_gap_1", "running", heartbeat("codex_fixture_trace_gap_1", freshAt, {
lastPersistedTraceAt: oldTraceAt,
outputMaxSeq: 89112,
}));
const decision = staleRecoveryCandidate({ task, localActive: false, now });
const hasTraceGap = taskHasTraceGapButFreshHeartbeat(task, now);
const diagnostics = schedulerDiagnostics([task], []);
assertCondition(hasTraceGap, "trace gap with fresh owner heartbeat should be classified", { decision, diagnostics });
assertCondition(decision.allowed === false && decision.reason === "owner-heartbeat-fresh", "fresh heartbeat must block stale retry", { decision });
assertCondition(diagnostics.traceGapNotStaleTaskIds.includes(task.id), "diagnostics must expose trace gap as not stale", diagnostics as unknown as Record<string, unknown>);
assertCondition(!diagnostics.staleRecoveryCandidateTaskIds.includes(task.id), "trace gap must not enter stale recovery candidates", diagnostics as unknown as Record<string, unknown>);
return {
name: "code-queue:trace-gap-not-stale",
ok: true,
detail: {
decision,
traceGapNotStaleTaskIds: diagnostics.traceGapNotStaleTaskIds,
staleRecoveryCandidateTaskIds: diagnostics.staleRecoveryCandidateTaskIds,
},
};
}
function checkStaleActiveOwnerExpired(): FixtureCheck {
const task = fixtureTask("codex_fixture_stale_1", "running", heartbeat("codex_fixture_stale_1", expiredAt, {
lastObservedAgentEventAt: expiredAt,
lastPersistedTraceAt: expiredAt,
}));
const decision = staleRecoveryCandidate({ task, localActive: false, now });
const diagnostics = schedulerDiagnostics([task], []);
assertCondition(decision.allowed === true && decision.reason === "owner-heartbeat-expired", "expired owner heartbeat should be the stale recovery gate", { decision });
assertCondition(diagnostics.state === "stale-active", "diagnostics must mark stale active only after owner heartbeat expiry", diagnostics as unknown as Record<string, unknown>);
assertCondition(diagnostics.staleRecoveryCandidateTaskIds.includes(task.id), "expired owner heartbeat should create a stale candidate", diagnostics as unknown as Record<string, unknown>);
return {
name: "code-queue:stale-active-owner-expired",
ok: true,
detail: {
decision,
state: diagnostics.state,
staleRecoveryCandidateTaskIds: diagnostics.staleRecoveryCandidateTaskIds,
},
};
}
function checkControlPlaneSplitBrainDiagnostics(): FixtureCheck {
const task = fixtureTask("codex_fixture_split_1", "running", heartbeat("codex_fixture_split_1", freshAt));
const diagnostics = buildExecutionDiagnostics({
now,
controlPlane: "master-code-queue-mgr",
executionStateSource: "postgres-control-plane",
tasks: [task],
activeRuns: [],
activeRunSlotCount: 0,
oaPublisher: null,
});
assertCondition(diagnostics.state === "split-brain" && diagnostics.splitBrain === true, "master postgres-control-plane must report split-brain when DB active has fresh scheduler heartbeat", diagnostics as unknown as Record<string, unknown>);
assertCondition(diagnostics.schedulerActiveRunSlotCount === 0 && diagnostics.databaseActiveTaskCount === 1, "split-brain fixture should preserve the exact control-plane divergence", diagnostics as unknown as Record<string, unknown>);
return {
name: "code-queue:control-plane-split-brain-diagnostics",
ok: true,
detail: {
state: diagnostics.state,
splitBrain: diagnostics.splitBrain,
executionStateSource: diagnostics.executionStateSource,
databaseActiveTaskIds: diagnostics.databaseActiveTaskIds,
schedulerActiveRunSlotCount: diagnostics.schedulerActiveRunSlotCount,
heartbeatFreshTaskIds: diagnostics.heartbeatFreshTaskIds,
},
};
}
function checkOaPublisherDegradedVisible(): FixtureCheck {
const oaPublisher = { pending: 3, lastError: "fixture OA publish retry", lastPublishedAt: null };
const diagnostics = schedulerDiagnostics([], [], oaPublisher);
assertCondition(diagnostics.state === "degraded", "OA publisher pending/lastError must degrade diagnostics", diagnostics as unknown as Record<string, unknown>);
assertCondition(diagnostics.oaPublisher === oaPublisher, "OA publisher detail must remain visible", diagnostics as unknown as Record<string, unknown>);
return {
name: "code-queue:oa-publisher-degraded-visible",
ok: true,
detail: {
state: diagnostics.state,
degraded: diagnostics.degraded,
oaPublisher: diagnostics.oaPublisher as Record<string, unknown>,
},
};
}
export function runCodeQueueLivenessFixtureChecks(only: string[] = []): { ok: boolean; checks: FixtureCheck[]; failures: Array<{ name: string; error: string }> } {
const selected = new Set(only.filter((name) => name.trim().length > 0));
const runners: Array<[CodeQueueLivenessCheckName, () => FixtureCheck]> = [
["code-queue:active-run-heartbeat-visible", checkActiveRunHeartbeatVisible],
["code-queue:trace-gap-not-stale", checkTraceGapNotStale],
["code-queue:stale-active-owner-expired", checkStaleActiveOwnerExpired],
["code-queue:control-plane-split-brain-diagnostics", checkControlPlaneSplitBrainDiagnostics],
["code-queue:oa-publisher-degraded-visible", checkOaPublisherDegradedVisible],
];
const checks: FixtureCheck[] = [];
const failures: Array<{ name: string; error: string }> = [];
for (const [name, run] of runners) {
if (selected.size > 0 && !selected.has(name)) continue;
try {
checks.push(run());
} catch (error) {
failures.push({ name, error: error instanceof Error ? error.message : String(error) });
checks.push({ name, ok: false, detail: { error: error instanceof Error ? error.message : String(error) } });
}
}
if (checks.length === 0) throw new Error(`no Code Queue liveness fixture checks matched: ${Array.from(selected).join(", ")}`);
return { ok: failures.length === 0, checks, failures };
}
+58 -1
View File
@@ -197,6 +197,52 @@ function compactLastAssistant(value: unknown, full: boolean): Record<string, unk
};
}
function compactSchedulerHeartbeat(value: unknown): Record<string, unknown> | null {
const record = asRecord(value);
if (record === null) return null;
return {
taskId: record.taskId ?? null,
attempt: record.attempt ?? null,
owner: record.owner ?? null,
schedulerInstance: record.schedulerInstance ?? null,
agentPort: record.agentPort ?? null,
activeTurnId: record.activeTurnId ?? null,
codexThreadId: record.codexThreadId ?? null,
lastLocalHeartbeatAt: record.lastLocalHeartbeatAt ?? null,
lastObservedAgentEventAt: record.lastObservedAgentEventAt ?? null,
lastPersistedTraceAt: record.lastPersistedTraceAt ?? null,
outputMaxSeq: record.outputMaxSeq ?? null,
};
}
function compactExecutionDiagnostics(value: unknown): Record<string, unknown> | null {
const record = asRecord(value);
if (record === null) return null;
return {
state: record.state ?? record.health ?? null,
degraded: record.degraded ?? null,
splitBrain: record.splitBrain ?? null,
executionStateSource: record.executionStateSource ?? null,
controlPlane: record.controlPlane ?? null,
databaseActiveTaskCount: record.databaseActiveTaskCount ?? null,
databaseActiveTaskIds: record.databaseActiveTaskIds ?? [],
schedulerActiveRunSlotCount: record.schedulerActiveRunSlotCount ?? null,
schedulerActiveTaskIds: record.schedulerActiveTaskIds ?? [],
activeHeartbeatTaskIds: record.activeHeartbeatTaskIds ?? [],
heartbeatFreshTaskIds: record.heartbeatFreshTaskIds ?? [],
heartbeatExpiredTaskIds: record.heartbeatExpiredTaskIds ?? [],
heartbeatMissingTaskIds: record.heartbeatMissingTaskIds ?? [],
staleRecoveryCandidateTaskIds: record.staleRecoveryCandidateTaskIds ?? [],
traceGapTaskIds: record.traceGapTaskIds ?? [],
traceGapNotStaleTaskIds: record.traceGapNotStaleTaskIds ?? [],
lastSchedulerHeartbeatAt: record.lastSchedulerHeartbeatAt ?? null,
lastObservedAgentEventAt: record.lastObservedAgentEventAt ?? null,
lastPersistedTraceAt: record.lastPersistedTraceAt ?? null,
oaPublisher: record.oaPublisher ?? null,
reasons: record.reasons ?? [],
};
}
function compactToolSummary(value: unknown, full: boolean): Record<string, unknown> {
const record = asRecord(value) ?? {};
const items = asArray(record.items).map((item) => {
@@ -252,6 +298,7 @@ function compactSummary(summary: unknown, options: CodexTaskOptions, taskId: str
codexThreadId: record.codexThreadId ?? null,
activeTurnId: record.activeTurnId ?? null,
cancelRequested: record.cancelRequested ?? null,
schedulerHeartbeat: compactSchedulerHeartbeat(record.schedulerHeartbeat),
},
timing: record.timing ?? null,
createdAt: record.createdAt ?? null,
@@ -718,7 +765,12 @@ function requireMergeTargetQueueId(args: string[], command: string): string {
}
function codeQueues(): unknown {
return unwrapCodexResponse(coreInternalFetch(codeQueueProxyPath("/api/queues")));
const response = unwrapCodexResponse(coreInternalFetch(codeQueueProxyPath("/api/queues")));
return {
upstream: response.upstream,
queues: response.body.queues ?? [],
queue: compactQueueMutationSummary(response.body.queue),
};
}
function codexCreateQueue(queueId: string): unknown {
@@ -824,6 +876,7 @@ function compactTaskMutationResponse(task: unknown, options: CompactTaskMutation
maxAttempts: record.maxAttempts ?? null,
currentAttempt: record.currentAttempt ?? null,
cancelRequested: record.cancelRequested ?? null,
schedulerHeartbeat: compactSchedulerHeartbeat(record.schedulerHeartbeat),
createdAt: record.createdAt ?? null,
startedAt: record.startedAt ?? null,
updatedAt: record.updatedAt ?? null,
@@ -845,6 +898,10 @@ function compactQueueMutationSummary(value: unknown): Record<string, unknown> |
return {
activeQueueIds: record.activeQueueIds ?? null,
activeTaskIds: record.activeTaskIds ?? null,
databaseActiveTaskCount: record.databaseActiveTaskCount ?? null,
databaseActiveTaskIds: record.databaseActiveTaskIds ?? null,
schedulerHeartbeatStaleMs: record.schedulerHeartbeatStaleMs ?? null,
executionDiagnostics: compactExecutionDiagnostics(record.executionDiagnostics),
queuedTaskIds: record.queuedTaskIds ?? null,
counts: record.counts ?? null,
byQueue: Array.isArray(record.byQueue) ? record.byQueue : undefined,
+18
View File
@@ -3,6 +3,7 @@ import { connect } from "node:net";
import { join } from "node:path";
import { chromium, type Page } from "playwright";
import { createRouteRegistry, MODULES } from "../../src/components/frontend/src/navigation";
import { CODE_QUEUE_LIVENESS_CHECK_NAMES, runCodeQueueLivenessFixtureChecks } from "./code-queue-liveness-fixtures";
import { runCommand } from "./command";
import { type UniDeskConfig, repoRoot, rootPath } from "./config";
import { boundedJsonDetail } from "./preview";
@@ -166,11 +167,14 @@ const FRONTEND_CHECK_NAMES = [
"frontend:no-console-errors",
] as const;
const CODE_QUEUE_FIXTURE_CHECK_NAMES = [...CODE_QUEUE_LIVENESS_CHECK_NAMES] as const;
const ALL_E2E_CHECK_NAMES = [
...NETWORK_CHECK_NAMES,
...SERVICE_CHECK_NAMES,
...DATABASE_CHECK_NAMES,
...FRONTEND_CHECK_NAMES,
...CODE_QUEUE_FIXTURE_CHECK_NAMES,
] as const;
function uniqueText(values: string[]): string[] {
@@ -552,6 +556,14 @@ function addSelectedCheck(checks: E2ECheck[], options: E2ERunOptions, name: stri
addCheck(checks, name, passed, detail);
}
function codeQueueFixtureChecks(checks: E2ECheck[], options: E2ERunOptions): void {
const selected = CODE_QUEUE_FIXTURE_CHECK_NAMES.filter((name) => wantsCheck(options, name));
const result = runCodeQueueLivenessFixtureChecks(selected);
for (const check of result.checks) {
addSelectedCheck(checks, options, check.name, check.ok, check.detail);
}
}
function safeTestId(value: string): string {
return value.replace(/[^a-zA-Z0-9_-]/g, "_");
}
@@ -3420,8 +3432,14 @@ export async function runE2E(
const needDatabase = wantsPrefix(options, "database")
|| wantsCheck(options, "frontend:task-history-diagnostics");
const needFrontend = wantsPrefix(options, "frontend");
const needCodeQueueFixtures = wantsAnyCheck(options, [...CODE_QUEUE_FIXTURE_CHECK_NAMES]);
const executedSections: string[] = [];
if (needCodeQueueFixtures) {
executedSections.push("code-queue-fixtures");
codeQueueFixtureChecks(checks, options);
}
if (needNetwork) {
executedSections.push("network");
await exposureChecks(config, urls, checks, options);