feat: harden code queue runtime UX

- add queue merge controls and no-cache overview loading\n- improve runtime probes, judge retries, and task trace rendering\n- extend CLI/E2E/docs coverage for code queue and user services
This commit is contained in:
Codex
2026-05-14 02:20:48 +00:00
parent 79154d9b3f
commit b36d7f94d7
26 changed files with 1323 additions and 349 deletions
@@ -129,11 +129,22 @@ function remoteOpenCodeRunCommand(task: QueueTask, prompt: string): string {
`mkdir -p ${ctx().shellQuote(task.cwd)}`,
`cd ${ctx().shellQuote(task.cwd)}`,
envExports,
`exec ${shellJoin(openCodeRunArgs(task, prompt))}`,
`exec opencode ${shellJoin(openCodeRunArgs(task, prompt))}`,
].join("; ");
return `docker exec -i ${ctx().shellQuote(plan.containerName)} bash -lc ${ctx().shellQuote(inner)}`;
}
export function remoteOpenCodeRunCommandForTest(task: QueueTask, prompt: string): string {
return remoteOpenCodeRunCommand(task, prompt);
}
export function openCodeTransportClosedBeforeTerminal(exitOk: boolean, hasFinalResponse: boolean, stepFinished: boolean): boolean {
// Some OpenCode JSON streams can exit cleanly with visible assistant text but
// without a step_finish event. Treat exit=0 plus a current final response as
// terminal; otherwise the judge safety gate will retry a completed turn.
return !exitOk || (!hasFinalResponse && !stepFinished);
}
function openCodeSessionIdFromRecord(record: Record<string, unknown>, part: Record<string, unknown> | null): string | null {
const top = ctx().recordStringField(record, ["sessionID", "sessionId", "session_id"]);
if (top.length > 0) return top;
@@ -326,7 +337,7 @@ async function runOpenCodeTurnOnce(task: QueueTask, prompt: string): Promise<Cod
return {
threadId: task.codexThreadId,
turnId: null,
finalResponse: task.finalResponse,
finalResponse: "",
terminalStatus: "failed",
terminalError: message,
transportClosedBeforeTerminal: true,
@@ -350,7 +361,7 @@ async function runOpenCodeTurnOnce(task: QueueTask, prompt: string): Promise<Cod
try {
const exit = await app.closedPromise;
clearInterval(activityWatchdog);
const finalResponse = app.finalResponse || task.finalResponse;
const finalResponse = app.finalResponse.trim().length > 0 ? app.finalResponse : "";
const exitOk = exit.code === 0;
const hasFinal = finalResponse.trim().length > 0;
const status: TerminalStatus = exitOk && hasFinal ? "completed" : "failed";
@@ -372,7 +383,7 @@ async function runOpenCodeTurnOnce(task: QueueTask, prompt: string): Promise<Cod
finalResponse,
terminalStatus: status,
terminalError,
transportClosedBeforeTerminal: !exitOk || !app.stepFinished,
transportClosedBeforeTerminal: openCodeTransportClosedBeforeTerminal(exitOk, hasFinal, app.stepFinished),
appServerExit: exit,
events: app.events,
};
@@ -385,7 +396,7 @@ async function runOpenCodeTurnOnce(task: QueueTask, prompt: string): Promise<Cod
return {
threadId: app.sessionId ?? task.codexThreadId,
turnId: app.runId,
finalResponse: app.finalResponse || task.finalResponse,
finalResponse: app.finalResponse.trim().length > 0 ? app.finalResponse : "",
terminalStatus: "failed",
terminalError: message,
transportClosedBeforeTerminal: true,
@@ -26,7 +26,7 @@ export interface DevContainerContext {
remoteContainerStartScript: (plan: DevContainerPlan, forceRecreate: boolean) => string;
remoteHostWorkdirForTask: (task: QueueTask) => string;
remoteKeyInstallScript: (plan: DevContainerPlan, privateKeyBase64: string) => string;
runCodeQueueSsh: (providerId: string, script: string, timeoutMs: number, name: string) => DevContainerCommandLog;
runCodeQueueSsh: (providerId: string, script: string, timeoutMs: number, name: string) => Promise<DevContainerCommandLog>;
safePreview: (value: string, max?: number) => string;
shellQuote: (value: string) => string;
throwIfCommandFailed: (command: DevContainerCommandLog) => void;
@@ -51,29 +51,29 @@ async function startDevContainerPlan(plan: DevContainerPlan, options: { forceRec
verification?: Record<string, JsonValue>;
}> {
const commands: DevContainerCommandLog[] = [];
const run = (targetProviderId: string, script: string, timeoutMs: number, name: string): DevContainerCommandLog => {
const command = ctx().runCodeQueueSsh(targetProviderId, script, timeoutMs, name);
const run = async (targetProviderId: string, script: string, timeoutMs: number, name: string): Promise<DevContainerCommandLog> => {
const command = await ctx().runCodeQueueSsh(targetProviderId, script, timeoutMs, name);
commands.push(command);
ctx().throwIfCommandFailed(command);
return command;
};
run("main-server", ctx().masterKeySetupScript(plan), 45_000, "master-key-setup");
const keyRead = run("main-server", ctx().masterKeyReadScript(plan), 15_000, "master-key-read");
await run("main-server", ctx().masterKeySetupScript(plan), 45_000, "master-key-setup");
const keyRead = await run("main-server", ctx().masterKeyReadScript(plan), 15_000, "master-key-read");
const keyBase64 = keyRead.stdout.trim();
if (!/^[A-Za-z0-9+/=\r\n]+$/u.test(keyBase64) || keyBase64.length < 40) throw new Error("master key read returned invalid base64");
keyRead.stdout = "[redacted private tunnel key]\n";
run(plan.providerId, ctx().remoteKeyInstallScript(plan, keyBase64), 30_000, "remote-key-install");
run(plan.providerId, ctx().remoteCodexConfigInstallScript(plan), 30_000, "remote-codex-config-install");
run(plan.providerId, ctx().remoteContainerStartScript(plan, options.forceRecreate), 60_000, "remote-container-start");
run("main-server", ctx().masterProxyPrepareScript(plan), 30_000, "master-proxy-prepare");
run(plan.providerId, ctx().containerTunnelStartScript(plan), 45_000, "remote-tunnel-start");
run("main-server", ctx().masterProxyFinishScript(plan), 30_000, "master-proxy-finish");
if (options.prepareRuntime) run(plan.providerId, ctx().remoteCodexRuntimePrepareScript(plan), 180_000, "remote-codex-runtime-prepare");
await run(plan.providerId, ctx().remoteKeyInstallScript(plan, keyBase64), 30_000, "remote-key-install");
await run(plan.providerId, ctx().remoteCodexConfigInstallScript(plan), 30_000, "remote-codex-config-install");
await run(plan.providerId, ctx().remoteContainerStartScript(plan, options.forceRecreate), 60_000, "remote-container-start");
await run("main-server", ctx().masterProxyPrepareScript(plan), 30_000, "master-proxy-prepare");
await run(plan.providerId, ctx().containerTunnelStartScript(plan), 45_000, "remote-tunnel-start");
await run("main-server", ctx().masterProxyFinishScript(plan), 30_000, "master-proxy-finish");
if (options.prepareRuntime) await run(plan.providerId, ctx().remoteCodexRuntimePrepareScript(plan), 600_000, "remote-codex-runtime-prepare");
const verification: Record<string, JsonValue> = {};
if (options.verifyPing) {
const before = run("main-server", ctx().masterProxyEvidenceScript(plan), 15_000, "master-proxy-evidence-before-ping");
const ping = run(plan.providerId, ctx().devContainerPingScript(plan), 20_000, "remote-container-ping-google");
const after = run("main-server", ctx().masterProxyEvidenceScript(plan), 15_000, "master-proxy-evidence-after-ping");
const before = await run("main-server", ctx().masterProxyEvidenceScript(plan), 15_000, "master-proxy-evidence-before-ping");
const ping = await run(plan.providerId, ctx().devContainerPingScript(plan), 20_000, "remote-container-ping-google");
const after = await run("main-server", ctx().masterProxyEvidenceScript(plan), 15_000, "master-proxy-evidence-after-ping");
verification.pingGoogleOk = ping.exitCode === 0 && /0% packet loss|1 packets received|1 received/iu.test(ping.stdout);
verification.directPingEvidence = commands.find((command) => command.name === "remote-tunnel-start")?.stdout.includes("direct_ping=failed_expected") === true
? `direct ${plan.providerId} container ping failed before tunnel`
@@ -175,8 +175,8 @@ docker inspect "$CONTAINER" --format 'container={{.Name}} state={{.State.Status}
if docker inspect "$CONTAINER" >/dev/null 2>&1; then
docker exec "$CONTAINER" bash -lc 'export PATH=/tmp/unidesk-tools:$PATH; echo default=$(ip route show default | head -1); echo resolv=$(tr "\\n" " " </etc/resolv.conf); echo pwd=$(pwd); command -v codex || true; ip addr show ${plan.tunName} 2>/dev/null || true' || true
fi`;
commands.push(ctx().runCodeQueueSsh(providerId, statusScript, 15_000, "remote-container-status"));
commands.push(ctx().runCodeQueueSsh("main-server", ctx().masterProxyEvidenceScript(plan), 15_000, "master-proxy-status"));
commands.push(await ctx().runCodeQueueSsh(providerId, statusScript, 15_000, "remote-container-status"));
commands.push(await ctx().runCodeQueueSsh("main-server", ctx().masterProxyEvidenceScript(plan), 15_000, "master-proxy-status"));
return ctx().jsonResponse({
ok: commands.every((command) => command.exitCode === 0),
providerId,
@@ -123,6 +123,8 @@ import {
statsDaysFromUrl,
taskForMetaResponse,
taskForResponse,
taskListStepCount,
taskOutputMaxSeq as taskViewOutputMaxSeq,
taskStatisticsSummary,
taskSummaryResponse,
timestampMs,
@@ -148,10 +150,36 @@ const retryBackoffMaxMs = 10 * 60 * 1000;
const queueIdPattern = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u;
const queueNameMaxLength = 80;
const config = readConfig();
type CodeQueueEventType = "hello" | "task-updated" | "task-step" | "queue-updated";
interface CodeQueueEvent {
sequence: number;
type: CodeQueueEventType;
at: string;
reason: string;
taskId?: string;
queueId?: string;
status?: TaskStatus;
updatedAt?: string;
stepCount?: number;
outputMaxSeq?: number;
}
interface CodeQueueEventClient {
enqueue: (chunk: string) => void;
close: () => void;
}
const codeQueueEventClients = new Set<CodeQueueEventClient>();
const codeQueueEventEncoder = new TextEncoder();
const taskEventSnapshots = new Map<string, { stepCount: number; outputMaxSeq: number; status: TaskStatus; updatedAt: string }>();
let codeQueueEventSequence = 0;
const logger = createLogger("code-queue", config.logFile);
const state = emptyState();
let processing = false;
const processingQueues = new Set<string>();
const mergingQueues = new Set<string>();
const activeRuns = new Map<string, ActiveRun>();
const activeRunSlotReservations = new Set<string>();
const activeRunSlotWaiters: ActiveRunSlotWaiter[] = [];
@@ -582,6 +610,142 @@ function nowIso(): string {
return new Date().toISOString();
}
function sseChunk(event: CodeQueueEvent): string {
return [
`id: ${event.sequence}`,
`event: ${event.type}`,
`data: ${JSON.stringify(event)}`,
"",
"",
].join("\n");
}
function emitCodeQueueEvent(event: Omit<CodeQueueEvent, "sequence" | "at"> & { at?: string }): void {
const payload: CodeQueueEvent = {
...event,
sequence: ++codeQueueEventSequence,
at: event.at || nowIso(),
};
if (codeQueueEventClients.size === 0) return;
const chunk = sseChunk(payload);
for (const client of Array.from(codeQueueEventClients)) {
try {
client.enqueue(chunk);
} catch {
client.close();
}
}
}
function taskOutputMaxSeq(task: QueueTask): number {
return taskViewOutputMaxSeq(task);
}
function publishTaskEvent(task: QueueTask, reason: string, options: { onlyStepChange?: boolean; stepChanged?: boolean } = {}): void {
// Hot path: never build full transcripts or read output archives here.
const stepCount = taskListStepCount(task);
const outputMaxSeq = taskOutputMaxSeq(task);
const previous = taskEventSnapshots.get(task.id);
const stepChanged = options.stepChanged ?? (previous === undefined ? stepCount > 0 : stepCount > previous.stepCount);
if (options.onlyStepChange === true && !stepChanged) {
taskEventSnapshots.set(task.id, { stepCount, outputMaxSeq, status: task.status, updatedAt: task.updatedAt });
return;
}
const type: CodeQueueEventType = stepChanged ? "task-step" : "task-updated";
taskEventSnapshots.set(task.id, { stepCount, outputMaxSeq, status: task.status, updatedAt: task.updatedAt });
emitCodeQueueEvent({
type,
reason,
taskId: task.id,
queueId: queueIdOf(task),
status: task.status,
updatedAt: task.updatedAt,
stepCount,
outputMaxSeq,
});
}
function publishQueueEvent(reason: string, queueId = ""): void {
emitCodeQueueEvent({ type: "queue-updated", reason, queueId: queueId || undefined });
}
function outputCanChangeStepCount(output: LiveOutput): boolean {
return output.channel === "command" || output.channel === "diff" || output.channel === "tool";
}
function outputStartsTraceStep(output: LiveOutput): boolean {
if (output.channel === "diff" || output.channel === "tool") return true;
if (output.channel !== "command") return false;
const method = String(output.method || "");
return method === "item/started" || (method !== "item/commandExecution/outputDelta" && method !== "item/completed");
}
function recordTaskOutputMetrics(task: QueueTask, output: LiveOutput, op: "set" | "append"): boolean {
task.outputMaxSeq = Math.max(taskOutputMaxSeq(task), Number.isFinite(Number(output.seq)) ? Number(output.seq) : 0);
if (op === "append" || !outputStartsTraceStep(output)) return false;
const storedStepCount = Number(task.stepCount ?? task.llmStepCount);
const nextStepCount = Number.isFinite(storedStepCount) && storedStepCount >= 0
? Math.floor(storedStepCount) + 1
: task.output.reduce((count, item) => count + (outputStartsTraceStep(item) ? 1 : 0), 0);
task.stepCount = nextStepCount;
task.llmStepCount = nextStepCount;
return true;
}
function codeQueueEventsResponse(): Response {
let heartbeat: ReturnType<typeof setInterval> | null = null;
let client: CodeQueueEventClient | null = null;
const stream = new ReadableStream<Uint8Array>({
start(controller) {
client = {
enqueue(chunk: string) {
controller.enqueue(codeQueueEventEncoder.encode(chunk));
},
close() {
if (heartbeat !== null) {
clearInterval(heartbeat);
heartbeat = null;
}
if (client !== null) codeQueueEventClients.delete(client);
client = null;
try {
controller.close();
} catch {
// The browser may have already closed the EventSource connection.
}
},
};
codeQueueEventClients.add(client);
controller.enqueue(codeQueueEventEncoder.encode("retry: 1000\n\n"));
client.enqueue(sseChunk({
sequence: codeQueueEventSequence,
type: "hello",
at: nowIso(),
reason: "connected",
}));
heartbeat = setInterval(() => {
try {
controller.enqueue(codeQueueEventEncoder.encode(`: heartbeat ${nowIso()}\n\n`));
} catch {
client?.close();
}
}, 25_000);
heartbeat.unref?.();
},
cancel() {
client?.close();
},
});
return new Response(stream, {
headers: {
"content-type": "text/event-stream; charset=utf-8",
"cache-control": "no-store, no-transform",
"connection": "keep-alive",
"x-accel-buffering": "no",
},
});
}
function resolveReasoningEffort(model: string, explicit?: string | null): string | null {
const requested = explicit?.trim();
if (requested) return requested;
@@ -704,6 +868,15 @@ function pruneTaskHotState(task: QueueTask): void {
}
}
function taskRetainedTraceStepCount(task: QueueTask): number {
return task.output.reduce((count, output) => count + (outputStartsTraceStep(output) ? 1 : 0), 0);
}
function normalizeTaskMetric(value: unknown): number | null {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : null;
}
function normalizeTask(task: QueueTask): QueueTask {
task.queueId = safeQueueId(task.queueId);
task.queueEnteredAt = taskQueueEnteredAt(task);
@@ -726,6 +899,11 @@ function normalizeTask(task: QueueTask): QueueTask {
task.judgeFailCount = Number.isInteger(task.judgeFailCount) && task.judgeFailCount >= 0 ? task.judgeFailCount : judgeFailCountFromOutput(task);
const persistedPromptHistory = Array.isArray(task.promptHistory) ? task.promptHistory : [];
task.promptHistory = mergePromptHistory([...persistedPromptHistory, ...outputPromptHistory(task)]);
const storedStepCount = normalizeTaskMetric(task.stepCount) ?? normalizeTaskMetric(task.llmStepCount);
const stepCount = storedStepCount ?? taskRetainedTraceStepCount(task);
task.stepCount = stepCount;
task.llmStepCount = stepCount;
task.outputMaxSeq = Math.max(normalizeTaskMetric(task.outputMaxSeq) ?? 0, taskViewOutputMaxSeq(task));
pruneTaskHotState(task);
return task;
}
@@ -779,6 +957,7 @@ function markTaskDirty(taskId: string): void {
function persistTaskState(task: QueueTask): void {
markTaskDirty(task.id);
persistState(false);
publishTaskEvent(task, "persist");
}
function markQueueDirty(queueId: string): void {
@@ -815,14 +994,12 @@ function taskTimestamp(value: string | null): string | null {
}
function lastOutputSeq(task: QueueTask): number {
return task.output.at(-1)?.seq ?? 0;
return taskOutputMaxSeq(task);
}
function updateNextSeqFromTasks(): void {
let nextSeq = Math.max(1, Number.isFinite(state.nextSeq) ? Math.floor(state.nextSeq) : 1);
for (const task of state.tasks) {
for (const output of task.output) nextSeq = Math.max(nextSeq, Number(output.seq) + 1);
}
for (const task of state.tasks) nextSeq = Math.max(nextSeq, lastOutputSeq(task) + 1);
state.nextSeq = nextSeq;
}
@@ -1603,6 +1780,7 @@ function addEvent(task: QueueTask, event: CodexEventSummary): void {
task.events.push(event);
if (config.maxInMemoryEventRecords > 0 && task.events.length > config.maxInMemoryEventRecords) task.events.splice(0, task.events.length - config.maxInMemoryEventRecords);
markTaskDirty(task.id);
publishTaskEvent(task, "agent-event", { onlyStepChange: true });
}
function taskReferencesEqual(left: string[], right: string[]): boolean {
@@ -1789,6 +1967,12 @@ configureTaskOutput({
logger,
markTaskDirty,
nowIso,
onOutputAppended: (task, output, op) => {
const archiveOp = op === "append" ? "append" : "set";
const stepChanged = recordTaskOutputMetrics(task, output, archiveOp);
if (archiveOp === "append" && !outputCanChangeStepCount(output)) return;
publishTaskEvent(task, "output", { onlyStepChange: archiveOp === "append", stepChanged });
},
schedulePersistState,
});
@@ -2223,7 +2407,7 @@ async function runTask(task: QueueTask): Promise<void> {
}
const finishedAt = nowIso();
task.finalResponse = result.finalResponse || task.finalResponse;
task.attempts.push(attemptFromResult(task, mode, startedAt, finishedAt, result, attemptStartOutput?.seq ?? null, taskFullOutput(task).at(-1)?.seq ?? null, recoveryPrompt));
task.attempts.push(attemptFromResult(task, mode, startedAt, finishedAt, result, attemptStartOutput?.seq ?? null, taskOutputMaxSeq(task), recoveryPrompt));
task.status = "judging";
task.activeTurnId = null;
task.updatedAt = nowIso();
@@ -2510,6 +2694,9 @@ function queuedStatusReason(task: QueueTask, tasks: QueueTask[] = state.tasks):
if (!serviceReady) {
return queuedReason("service", "SERVICE", "Code Queue is still starting and has not enabled scheduling yet.");
}
if (mergingQueues.has(queueId)) {
return queuedReason("queue_merge", "MERGING", "Queue merge is rewriting queue membership; scheduling will resume immediately after it finishes.");
}
const memoryPressure = activeRunMemoryPressure();
if (memoryPressure !== null) {
@@ -2592,6 +2779,7 @@ function scheduleQueue(queueId?: string): void {
if (!serviceReady || shutdownRequested) return;
const ids = queueId === undefined ? runnableQueueIds() : [queueId];
for (const id of ids) {
if (mergingQueues.has(id)) continue;
if (processingQueues.has(id)) continue;
void processQueue(id).catch((error) => {
logger("error", "queue_loop_failed", { queueId: id, error: error instanceof Error ? error.stack ?? error.message : String(error) });
@@ -2705,6 +2893,8 @@ function requestErrorResponse(error: unknown): Response | null {
error.message === "request body must be an object"
|| error.message === "prompt is required"
|| error.message === "a task cannot reference itself while editing prompt"
|| error.message === "sourceQueueId is required"
|| error.message === "source queue must be different from target queue"
|| error.message.startsWith("referenceTaskIds supports at most ")
|| error.message.startsWith("queueId must match ")
|| error.message.startsWith("queue name must be ")
@@ -2738,6 +2928,8 @@ async function createTasks(req: Request): Promise<Response> {
for (const task of tasks) appendOutput(task, "user", `${task.prompt}\n`, "enqueue");
state.tasks.push(...tasks);
if (tasks.length > 0) armIdleNotification();
for (const task of tasks) publishTaskEvent(task, "enqueue");
for (const id of new Set(tasks.map(queueIdOf))) publishQueueEvent("enqueue", id);
persistState();
logger("info", "tasks_enqueued", { count: tasks.length, ids: tasks.map((task) => task.id), queueIds: Array.from(new Set(tasks.map(queueIdOf))), providerIds: Array.from(new Set(tasks.map((task) => task.providerId))) });
scheduleQueue();
@@ -2865,6 +3057,7 @@ async function markTaskRead(task: QueueTask): Promise<Response> {
task.readAt = nowIso();
markTaskDirty(task.id);
persistState(false);
publishTaskEvent(task, "read");
logger("info", "task_marked_read", { taskId: task.id, queueId: queueIdOf(task), status: task.status });
}
await flushDirtyTasksToDatabase(true);
@@ -2907,7 +3100,10 @@ async function markTaskReadById(taskId: string): Promise<Response> {
AND read_at IS NULL
`;
const hotTask = findTask(taskId);
if (hotTask !== null) hotTask.readAt = readAt;
if (hotTask !== null) {
hotTask.readAt = readAt;
publishTaskEvent(hotTask, "read");
}
logger("info", "task_marked_read", { taskId, queueId: safeQueueId(row.queue_id), status: row.status });
}
return compactJsonResponse({
@@ -2952,6 +3148,7 @@ async function markTerminalTasksRead(url: URL): Promise<Response> {
for (const task of state.tasks) {
if (!ids.has(task.id)) continue;
task.readAt = readAt;
publishTaskEvent(task, "read-all");
}
if (ids.size > 0) {
logger("info", "terminal_tasks_marked_read", { count: ids.size, queueId });
@@ -2964,6 +3161,7 @@ async function markTerminalTasksRead(url: URL): Promise<Response> {
if (!terminalTaskUnread(task)) continue;
task.readAt = readAt;
markTaskDirty(task.id);
publishTaskEvent(task, "read-all");
count += 1;
}
if (count > 0) {
@@ -2983,6 +3181,7 @@ async function createQueue(req: Request): Promise<Response> {
queue.updatedAt = nowIso();
markQueueDirty(queue.id);
persistState(false);
publishQueueEvent("queue-created", queue.id);
logger("info", "queue_created", { queueId, name: queue.name, existed: beforeCount === state.queues.length });
await flushDirtyTasksToDatabase(true);
const tasks = await loadAllTasksForRead();
@@ -3003,12 +3202,154 @@ async function updateQueue(queueIdValue: string, req: Request): Promise<Response
queue.updatedAt = nowIso();
markQueueDirty(queue.id);
persistState(false);
publishQueueEvent("queue-updated", queue.id);
logger("info", "queue_updated", { queueId, previousName, name: queue.name });
await flushDirtyTasksToDatabase(true);
const tasks = await loadAllTasksForRead();
return jsonResponse({ ok: true, queue, queues: perQueueSummaries(tasks), summary: queueSummary(false, tasks) });
}
function knownQueue(queueId: string): QueueRecord | null {
return state.queues.find((queue) => queue.id === queueId) ?? null;
}
function queueHasKnownTasks(queueId: string): boolean {
return state.tasks.some((task) => queueIdOf(task) === queueId);
}
function queueMergeBlocker(queueId: string): string | null {
if (processingQueues.has(queueId)) return "queue processor is currently active";
if (activeRuns.has(queueId)) return "queue has an active agent run";
if (activeRunSlotReservations.has(queueId)) return "queue is reserving an active run slot";
if (activeRunSlotWaiters.some((waiter) => waiter.queueId === queueId)) return "queue is waiting for an active run slot";
const activeTask = state.tasks.find((task) => queueIdOf(task) === queueId && (task.status === "running" || task.status === "judging"));
return activeTask === undefined ? null : `task ${activeTask.id} is ${activeTask.status}`;
}
function parseSourceQueueIds(record: Record<string, unknown>, targetQueueId: string): string[] {
const raw = record.sourceQueueIds ?? record.sources ?? record.sourceQueueId ?? record.fromQueueId ?? record.from ?? record.queueId ?? record.id;
const values = Array.isArray(raw)
? raw
: typeof raw === "string"
? raw.split(/[,\s]+/u)
: [];
const ids: string[] = [];
const seen = new Set<string>();
for (const value of values) {
const id = normalizeQueueId(value);
if (id === targetQueueId) throw new Error("source queue must be different from target queue");
if (seen.has(id)) continue;
seen.add(id);
ids.push(id);
}
if (ids.length === 0) throw new Error("sourceQueueId is required");
return ids;
}
async function mergeDatabaseQueueTasks(sourceQueueIds: string[], targetQueueId: string): Promise<string[]> {
if (!databaseReady || sourceQueueIds.length === 0) return [];
const rows = await sql<Array<{ id: string }>>`
UPDATE unidesk_code_queue_tasks
SET
queue_id = ${targetQueueId},
task_json = jsonb_set(
jsonb_set(
task_json,
'{queueId}',
to_jsonb(${targetQueueId}::text),
true
),
'{queueEnteredAt}',
to_jsonb(COALESCE(NULLIF(task_json->>'queueEnteredAt', ''), to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'))::text),
true
)
WHERE queue_id IN ${sql(sourceQueueIds)}
RETURNING id
`;
return rows.map((row) => row.id);
}
async function mergeQueues(targetQueueIdValue: string | null, req: Request): Promise<Response> {
const body = await readJson(req);
const record = typeof body === "object" && body !== null && !Array.isArray(body) ? body as Record<string, unknown> : {};
const targetQueueId = normalizeQueueId(targetQueueIdValue ?? record.targetQueueId ?? record.intoQueueId ?? record.into);
const sourceQueueIds = parseSourceQueueIds(record, targetQueueId);
const missingSources = sourceQueueIds.filter((id) => knownQueue(id) === null && !queueHasKnownTasks(id));
if (missingSources.length > 0) return jsonResponse({ ok: false, error: `source queue not found: ${missingSources.join(", ")}` }, 404);
const mergeQueueIds = Array.from(new Set([targetQueueId, ...sourceQueueIds]));
for (const id of mergeQueueIds) mergingQueues.add(id);
try {
for (const id of mergeQueueIds) {
const blocker = queueMergeBlocker(id);
if (blocker !== null) {
return jsonResponse({ ok: false, error: `cannot merge queue ${id}: ${blocker}` }, 409);
}
}
const mergedAt = nowIso();
const targetQueue = ensureQueue(targetQueueId);
const sourceQueues = sourceQueueIds.map((id) => ensureQueue(id));
targetQueue.updatedAt = mergedAt;
markQueueDirty(targetQueue.id);
for (const queue of sourceQueues) {
queue.updatedAt = mergedAt;
markQueueDirty(queue.id);
}
const sourceSet = new Set(sourceQueueIds);
const hotMovedTasks: QueueTask[] = [];
for (const task of state.tasks) {
const previousQueueId = queueIdOf(task);
if (!sourceSet.has(previousQueueId)) continue;
task.queueEnteredAt = taskQueueEnteredAt(task);
task.queueId = targetQueueId;
hotMovedTasks.push(task);
markTaskDirty(task.id);
publishTaskEvent(task, "queue-merged");
}
const databaseMovedTaskIds = await mergeDatabaseQueueTasks(sourceQueueIds, targetQueueId);
persistState(false);
publishQueueEvent("queue-merged", targetQueueId);
for (const sourceQueueId of sourceQueueIds) publishQueueEvent("queue-merged", sourceQueueId);
if (hotMovedTasks.some((task) => task.status === "queued" || task.status === "retry_wait")) armIdleNotification();
logger("info", "queues_merged", {
targetQueueId,
sourceQueueIds,
hotMovedTaskCount: hotMovedTasks.length,
databaseMovedTaskCount: databaseReady ? databaseMovedTaskIds.length : null,
});
for (const id of mergeQueueIds) mergingQueues.delete(id);
scheduleQueue(targetQueueId);
await flushDirtyTasksToDatabase(true);
const tasks = await loadAllTasksForRead();
const movedIdSet = new Set(databaseReady ? databaseMovedTaskIds : hotMovedTasks.map((task) => task.id));
const orderedMovedTaskIds = tasks
.filter((task) => movedIdSet.has(task.id))
.sort(compareTaskQueueOrder)
.map((task) => task.id);
const targetTaskOrder = tasks
.filter((task) => queueIdOf(task) === targetQueueId)
.sort(compareTaskQueueOrder)
.map((task) => ({ id: task.id, queueEnteredAt: taskQueueEnteredAt(task), createdAt: task.createdAt, status: task.status }));
return jsonResponse({
ok: true,
targetQueueId,
sourceQueueIds,
mergedTaskCount: databaseReady ? databaseMovedTaskIds.length : hotMovedTasks.length,
movedTaskIds: orderedMovedTaskIds.slice(0, 500),
targetTaskOrder: targetTaskOrder.slice(0, 500),
order: "merged tasks keep their original queueEnteredAt/createdAt ordering; source queue records are retained and never deleted",
targetQueue,
sourceQueues,
queues: perQueueSummaries(tasks),
summary: queueSummary(false, tasks),
}, 202);
} finally {
for (const id of mergeQueueIds) mergingQueues.delete(id);
}
}
async function moveTaskToQueue(task: QueueTask, req: Request): Promise<Response> {
if (task.status === "running" || task.status === "judging") {
return jsonResponse({ ok: false, error: `cannot move active task ${task.id} while status=${task.status}`, task: taskForResponse(task) }, 409);
@@ -3027,6 +3368,8 @@ async function moveTaskToQueue(task: QueueTask, req: Request): Promise<Response>
appendOutput(task, "system", `moved from queue=${previousQueueId} to queue=${queueId}\n`, "queue/move");
if (task.status === "queued" || task.status === "retry_wait") armIdleNotification();
persistState();
publishQueueEvent("task-moved", previousQueueId);
publishQueueEvent("task-moved", queueId);
logger("info", "task_moved_queue", { taskId: task.id, previousQueueId, queueId, status: task.status });
if (task.status === "queued" || task.status === "retry_wait") scheduleQueue(queueId);
await flushDirtyTasksToDatabase(true);
@@ -3039,6 +3382,7 @@ async function route(req: Request): Promise<Response> {
try {
if (url.pathname === "/" || url.pathname === "/health") return jsonResponse({ ok: true, service: "code-queue", queue: await queueSummaryForHealth(), startedAt: serviceStartedAt });
if (url.pathname === "/logs") return jsonResponse({ ok: true, logs: recentLogs.slice(-parseLimit(url)) });
if (url.pathname === "/api/events" && req.method === "GET") return codeQueueEventsResponse();
if (url.pathname === "/api/dev-ready" && req.method === "GET") return jsonResponse({ ok: true, devReady: collectDevReady() });
if (url.pathname === "/api/dev-containers" && req.method === "GET") {
const plan = buildDevContainerPlan(normalizeProviderId(config.devContainerDefaultProviderId) ?? "D601", {});
@@ -3091,6 +3435,9 @@ async function route(req: Request): Promise<Response> {
return jsonResponse({ ok: true, queues: perQueueSummaries(tasks), queue: queueSummary(false, tasks) });
}
if (url.pathname === "/api/queues" && req.method === "POST") return await createQueue(req);
if (url.pathname === "/api/queues/merge" && req.method === "POST") return await mergeQueues(null, req);
const queueMergeMatch = url.pathname.match(/^\/api\/queues\/([^/]+)\/merge$/u);
if (queueMergeMatch !== null && req.method === "POST") return await mergeQueues(decodeURIComponent(queueMergeMatch[1] ?? ""), req);
const queueMatch = url.pathname.match(/^\/api\/queues\/([^/]+)$/u);
if (queueMatch !== null && (req.method === "PATCH" || req.method === "PUT" || req.method === "POST")) return await updateQueue(decodeURIComponent(queueMatch[1] ?? ""), req);
if (url.pathname === "/api/tasks/read-all" && req.method === "POST") return await markTerminalTasksRead(url);
@@ -20,6 +20,23 @@ export const defaultJudgeProbeCases: JudgeProbeCase[] = [
],
events: [{ at: nowIso(), method: "turn/completed", status: "completed" }],
},
{
id: "completed_with_tool_evidence_but_no_current_final_response_should_retry",
prompt: "调查 /mnt/f/Work/ConStart/docs/MDTODO 的最新进展,只读,并在最终回复中总结。",
finalResponse: "",
expected: "retry",
expectedContinuePromptIncludes: [
"最终 assistant response",
"不能只执行工具",
],
terminalStatus: "completed",
outputs: [
{ channel: "user", text: "调查 /mnt/f/Work/ConStart/docs/MDTODO 的最新进展,只读,并在最终回复中总结。\n", method: "enqueue" },
{ channel: "tool", text: "opencode/tool: read /mnt/f/Work/ConStart/docs/MDTODO/20260419_频率判断.md status=completed; R1/R2 completed, R3 pending", method: "opencode/tool" },
{ channel: "system", text: "opencode completed status=completed exit=0 signal=null\n", method: "opencode/complete" },
],
events: [{ at: nowIso(), method: "opencode/step_finish", status: "completed" }],
},
{
id: "completed_but_plan_only",
prompt: "Create /root/unidesk/tmp/judge_probe.txt containing exactly judge-probe-ok, then summarize the file path.",
@@ -588,20 +588,63 @@ function originalRequirementOnlyFeedbackPrompt(task: QueueTask, reason: string):
return `未完成原始需求:${originalTask || "原始任务尚未完成"}。继续按照原始需求完成剩余任务。`;
}
function applyFallbackSafetyOverrides(task: QueueTask, result: CodexRunResult, judge: JudgeResult): JudgeResult {
// Non-LLM string/regex overrides are a last-resort fallback only. Never call
// this on a successful MiniMax judge result; MiniMax is the authoritative judge.
if (judge.source !== "fallback") return judge;
function missingFinalResponseFeedbackPrompt(task: QueueTask, reason: string): string {
return [
retryInstruction,
`上一次 attempt 未完成:${judgeReasonForPrompt(reason)}`,
"本轮必须在最终 assistant response 中给出可见交付结果;不能只执行工具、输出 reasoning 或依赖旧 finalResponse。",
"原始任务摘要/按需查询:",
compactRetryTaskContext(task),
].join("\n\n");
}
function applyJudgeSafetyOverrides(task: QueueTask, result: CodexRunResult, judge: JudgeResult): JudgeResult {
const currentFinalText = result.finalResponse || "";
if (judge.decision === "retry" && rateLimitFeedbackNeedsOriginalRequirementOnly(task, result)) {
const reason = judge.reason || "任务因限流/传输中断,原始需求尚未完成。";
if (result.terminalStatus === "interrupted" && explicitUserInterrupt(task, result)) {
return {
...judge,
decision: "fail",
confidence: Math.max(judge.confidence, 0.9),
reason: "Codex turn 被用户请求打断。",
raw: { previous: judge.raw ?? null, _safetyOverride: "explicit_user_interrupt" },
};
}
if (rateLimitFeedbackNeedsOriginalRequirementOnly(task, result)) {
const reason = judge.reason || result.terminalError || "任务因限流/传输中断,原始需求尚未完成。";
return {
...judge,
decision: "retry",
confidence: Math.max(judge.confidence, 0.9),
reason,
continuePrompt: originalRequirementOnlyFeedbackPrompt(task, reason),
raw: { previous: judge.raw ?? null, _safetyOverride: "original_requirement_only_feedback" },
};
}
if (result.terminalStatus === "interrupted") return judge;
if (result.terminalStatus === "failed" || result.terminalStatus === null || result.transportClosedBeforeTerminal) {
const reason = result.terminalError
? `当前 attempt 未完成:${result.terminalError}`
: "当前 attempt 未正常完成。";
return {
...judge,
decision: "retry",
confidence: Math.max(judge.confidence, 0.9),
reason,
continuePrompt: missingFinalResponseFeedbackPrompt(task, reason),
raw: { previous: judge.raw ?? null, _safetyOverride: "terminal_not_completed" },
};
}
if (result.finalResponse.trim().length === 0) {
const reason = "当前 attempt 没有生成新的最终 assistant response,不能复用旧 finalResponse 判定完成。";
return {
...judge,
decision: "retry",
confidence: Math.max(judge.confidence, 0.92),
reason,
continuePrompt: missingFinalResponseFeedbackPrompt(task, reason),
raw: { previous: judge.raw ?? null, _safetyOverride: "missing_current_final_response" },
};
}
if (asksToConfirmConcurrentFileInsteadOfDelivery(currentFinalText)) {
const reason = "最终回复停下来询问用户如何处理并发修改/无关文件,而不是自主完成自己的变更交付范围。";
return {
@@ -628,7 +671,7 @@ function applyFallbackSafetyOverrides(task: QueueTask, result: CodexRunResult, j
}
export async function judgeTask(task: QueueTask, result: CodexRunResult): Promise<JudgeResult> {
if (config().minimaxApiKey.length === 0) return applyFallbackSafetyOverrides(task, result, fallbackJudge(result));
if (config().minimaxApiKey.length === 0) return applyJudgeSafetyOverrides(task, result, fallbackJudge(result));
const judgePromptContent = judgePrompt(task, result);
try {
let lastParseError: string | null = null;
@@ -654,9 +697,7 @@ export async function judgeTask(task: QueueTask, result: CodexRunResult): Promis
source: "minimax",
raw: { ...(parsed as Record<string, JsonValue>), _parseSource: parsedResult.source, _repairAttempt: repairAttempt },
};
// No local hard-gate validation or safety override is applied to a
// successful MiniMax result. Fallback rules are only for MiniMax failure.
return judge;
return applyJudgeSafetyOverrides(task, result, judge);
} catch (error) {
lastParseError = error instanceof Error ? error.message : String(error);
if (repairAttempt >= config().judgeRepairAttempts) throw new Error(lastParseError);
@@ -674,7 +715,7 @@ export async function judgeTask(task: QueueTask, result: CodexRunResult): Promis
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
logger("warn", "judge_failed_fallback", { taskId: task.id, error: message });
return applyFallbackSafetyOverrides(task, result, fallbackJudge(result, message));
return applyJudgeSafetyOverrides(task, result, fallbackJudge(result, message));
}
}
@@ -1,6 +1,6 @@
// 重构前 index.ts 只读参考:commit 6a04144d3f5103014f75b637d7e6bc2f45bf007fblob 56e590c1a6b5ca7ad128bf2c992f60e46c355a58;可用 `git show 6a04144d3f5103014f75b637d7e6bc2f45bf007f:src/components/microservices/code-queue/src/index.ts` 查看。
import { spawnSync } from "node:child_process";
import { spawn } from "node:child_process";
import { existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { opencodeNpmPackage } from "./code-agent/common";
@@ -124,27 +124,66 @@ function buildDevContainerPlan(providerId: string, body: Record<string, unknown>
};
}
function runCodeQueueSsh(providerId: string, script: string, timeoutMs: number, name: string): DevContainerCommandLog {
function appendLimited(buffer: string, chunk: Buffer | string, maxBytes: number): { value: string; truncated: boolean } {
const text = typeof chunk === "string" ? chunk : chunk.toString("utf8");
if (buffer.length >= maxBytes) return { value: buffer, truncated: true };
const remaining = maxBytes - buffer.length;
if (text.length <= remaining) return { value: buffer + text, truncated: false };
return { value: buffer + text.slice(0, remaining), truncated: true };
}
function runCodeQueueSsh(providerId: string, script: string, timeoutMs: number, name: string): Promise<DevContainerCommandLog> {
const started = Date.now();
const result = spawnSync("bun", ["scripts/cli.ts", "ssh", providerId, "bash -s"], {
cwd: ctx().config.defaultWorkdir,
input: script,
encoding: "utf8",
timeout: timeoutMs,
maxBuffer: 8 * 1024 * 1024,
const maxBuffer = 8 * 1024 * 1024;
return new Promise((resolveLog) => {
let stdout = "";
let stderr = "";
let truncated = false;
let timedOut = false;
let spawnError: Error | null = null;
const child = spawn("bun", ["scripts/cli.ts", "ssh", providerId, "bash -s"], {
cwd: ctx().config.defaultWorkdir,
stdio: ["pipe", "pipe", "pipe"],
});
const timer = setTimeout(() => {
timedOut = true;
child.kill("SIGTERM");
setTimeout(() => child.kill("SIGKILL"), 2_000).unref?.();
}, timeoutMs);
timer.unref?.();
child.stdout.on("data", (chunk: Buffer) => {
const next = appendLimited(stdout, chunk, maxBuffer);
stdout = next.value;
truncated ||= next.truncated;
});
child.stderr.on("data", (chunk: Buffer) => {
const next = appendLimited(stderr, chunk, maxBuffer);
stderr = next.value;
truncated ||= next.truncated;
});
child.on("error", (error) => {
spawnError = error;
});
child.stdin.on("error", () => undefined);
child.on("close", (code, signal) => {
clearTimeout(timer);
const errorParts: string[] = [];
if (stderr.length > 0) errorParts.push(stderr);
if (timedOut) errorParts.push(`Error: command timed out after ${timeoutMs}ms`);
if (spawnError !== null) errorParts.push(`${spawnError.name}: ${spawnError.message}`);
if (truncated) errorParts.push(`output truncated at ${maxBuffer} bytes`);
resolveLog({
name,
providerId,
exitCode: timedOut && code === 0 ? null : code,
signal: signal as NodeJS.Signals | null,
durationMs: Date.now() - started,
stdout,
stderr: errorParts.join("\n").trim(),
});
});
child.stdin.end(script);
});
const stdout = typeof result.stdout === "string" ? result.stdout : String(result.stdout ?? "");
const stderr = typeof result.stderr === "string" ? result.stderr : String(result.stderr ?? "");
const errorText = result.error === undefined ? "" : `${result.error.name}: ${result.error.message}`;
return {
name,
providerId,
exitCode: result.status,
signal: result.signal,
durationMs: Date.now() - started,
stdout,
stderr: errorText.length > 0 ? `${stderr}\n${errorText}`.trim() : stderr,
};
}
function throwIfCommandFailed(command: DevContainerCommandLog): void {
@@ -246,6 +285,8 @@ IMAGE="$REQUESTED_IMAGE"
SSH_DIR="\${UNIDESK_HOST_ROOT_SSH_DIR:-$HOME/.ssh}"
SSH_MOUNT_ARGS=()
if [ -d "$SSH_DIR" ]; then SSH_MOUNT_ARGS=(-v "$SSH_DIR":/root/.ssh:ro); fi
HOST_PATH_MOUNT_ARGS=()
if [ -d /mnt ]; then HOST_PATH_MOUNT_ARGS+=(-v /mnt:/mnt); fi
if ! docker image inspect "$IMAGE" >/dev/null 2>&1 && docker image inspect "$CODEX_IMAGE" >/dev/null 2>&1; then
IMAGE="$CODEX_IMAGE"
fi
@@ -295,6 +336,7 @@ cid=$(docker run -d \\
-v "$KEY_DIR/codex-home":${shellQuote(plan.remoteCodexHome)} \\
-v "$KEY_DIR/opencode-xdg":${shellQuote(plan.remoteOpencodeXdgDir)} \\
"\${SSH_MOUNT_ARGS[@]}" \\
"\${HOST_PATH_MOUNT_ARGS[@]}" \\
-v "$WORKDIR":"$CONTAINER_WORKDIR" \\
-v "$WORKDIR":/root/unidesk \\
-w "$CONTAINER_WORKDIR" \\
@@ -341,14 +383,45 @@ KNOWN_HOSTS=/tmp/unidesk-dev-proxy-known_hosts
cp /run/unidesk-dev-proxy/known_hosts "$KNOWN_HOSTS"
chmod 600 "$KNOWN_HOSTS"
DEFAULT_LINE=$(ip route show default | head -1)
ORIG_GW=$(printf '%s\\n' "$DEFAULT_LINE" | sed -n 's/^default via \\([^ ]*\\) dev \\([^ ]*\\).*/\\1/p')
ORIG_DEV=$(printf '%s\\n' "$DEFAULT_LINE" | sed -n 's/^default via \\([^ ]*\\) dev \\([^ ]*\\).*/\\2/p')
if [ -z "$ORIG_GW" ] || [ -z "$ORIG_DEV" ]; then echo "cannot parse default route: $DEFAULT_LINE" >&2; exit 1; fi
DEFAULT_GW=$(printf '%s\\n' "$DEFAULT_LINE" | sed -n 's/^default via \\([^ ]*\\) dev \\([^ ]*\\).*/\\1/p')
DEFAULT_DEV=$(printf '%s\\n' "$DEFAULT_LINE" | sed -n 's/^default via \\([^ ]*\\) dev \\([^ ]*\\).*/\\2/p')
LINK_ROUTE=$(ip route show | awk '$2=="dev" && $3 !~ /^tun/ { print; exit }')
ORIG_DEV=$(printf '%s\\n' "$LINK_ROUTE" | awk '{print $3}')
if [ -n "$DEFAULT_GW" ] && [ -n "$DEFAULT_DEV" ] && [ "$DEFAULT_DEV" != "$TUN" ]; then
ORIG_GW="$DEFAULT_GW"
ORIG_DEV="$DEFAULT_DEV"
else
ORIG_IP=$(ip addr show "$ORIG_DEV" | sed -n 's/.*inet \\([0-9.]*\\)\\/.*/\\1/p' | head -1)
ORIG_GW=$(printf '%s\\n' "$ORIG_IP" | awk -F. 'NF==4 { printf "%s.%s.%s.1", $1, $2, $3 }')
fi
if [ -z "$ORIG_GW" ] || [ -z "$ORIG_DEV" ]; then echo "cannot detect docker bridge route: default=$DEFAULT_LINE link=$LINK_ROUTE" >&2; exit 1; fi
( ping -c 1 -W 3 google.com >/tmp/direct-ping.log 2>&1 && echo direct_ping=unexpected_ok ) || echo direct_ping=failed_expected
if command -v pkill >/dev/null 2>&1; then
pkill -f "ssh .* -w $TUN_ID:$TUN_ID .*$MASTER" >/dev/null 2>&1 || true
elif command -v busybox >/dev/null 2>&1; then
busybox ps | while read -r pid user command args; do
case "$command $args" in
*"ssh -f -N -w $TUN_ID:$TUN_ID"*|*"ssh -N -w $TUN_ID:$TUN_ID"*) kill "$pid" >/dev/null 2>&1 || true ;;
esac
done
fi
ip link delete "$TUN" >/dev/null 2>&1 || true
ip route replace $MASTER/32 via "$ORIG_GW" dev "$ORIG_DEV"
if command -v pkill >/dev/null 2>&1; then pkill -f "ssh .* -w $TUN_ID:$TUN_ID .*$MASTER" >/dev/null 2>&1 || true; fi
ssh -f -N -w $TUN_ID:$TUN_ID -o Tunnel=point-to-point -o ExitOnForwardFailure=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=2 -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$KNOWN_HOSTS" -i /run/unidesk-dev-proxy/id_ed25519 root@$MASTER
for i in 1 2 3 4 5; do ip link show "$TUN" >/dev/null 2>&1 && break; sleep 1; done
ip route replace default via "$ORIG_GW" dev "$ORIG_DEV"
SSH_LOG=/tmp/unidesk-dev-proxy-ssh.log
rm -f "$SSH_LOG"
if ! ssh -f -N -w $TUN_ID:$TUN_ID -o Tunnel=point-to-point -o ExitOnForwardFailure=yes -o ServerAliveInterval=15 -o ServerAliveCountMax=2 -o StrictHostKeyChecking=yes -o UserKnownHostsFile="$KNOWN_HOSTS" -i /run/unidesk-dev-proxy/id_ed25519 root@$MASTER 2>"$SSH_LOG"; then
cat "$SSH_LOG" >&2 2>/dev/null || true
exit 1
fi
for i in 1 2 3 4 5 6 7 8 9 10; do ip link show "$TUN" >/dev/null 2>&1 && break; sleep 1; done
if ! ip link show "$TUN" >/dev/null 2>&1; then
echo "ssh tunnel did not create $TUN via $ORIG_GW/$ORIG_DEV" >&2
cat "$SSH_LOG" >&2 2>/dev/null || true
if command -v busybox >/dev/null 2>&1; then busybox ps >&2 || true; fi
ip route show >&2 || true
exit 1
fi
ip addr replace $CLIENT_IP peer $SERVER_IP dev "$TUN"
ip link set "$TUN" up
ip route replace default via $SERVER_IP dev "$TUN"
@@ -5,7 +5,7 @@ import { codeAgentPortForModel, codeAgentPortInfo, codeModelPorts as codeModelPo
import { claudeQqNotificationOutboxStats, notificationTargetConfigured, notificationTargetLabel } from "./notifications";
import { executionProviderOptions } from "./provider-runtime";
import { taskFullOutput } from "./task-output";
import { buildCompactTaskTranscript, buildTaskTranscript, cachedPreviewTranscript, fullTranscript, prefixPreview, safePreview, statsDaysFromUrl, taskForCompactMetaResponse, taskForMetaResponse, taskLlmStepCount, taskStatisticsSummary, taskTiming, timestampMs } from "./task-view";
import { buildCompactTaskTranscript, buildTaskTranscript, cachedPreviewTranscript, fullTranscript, prefixPreview, safePreview, statsDaysFromUrl, taskForCompactMetaResponse, taskForMetaResponse, taskListStepCount, taskStatisticsSummary, taskTiming, timestampMs } from "./task-view";
import { userPromptForDisplay } from "./prompts";
import type { ActiveRun, ActiveRunSlotWaiter } from "./code-agent/common";
import type { JsonValue, QueueRecord, QueuedStatusReason, QueueTask, RuntimeConfig, TaskStatus, TranscriptLine } from "./types";
@@ -143,7 +143,7 @@ function outputChunkResponse(task: QueueTask, url: URL): Response {
function taskForListResponse(task: QueueTask, lite = false, queueTasks?: QueueTask[]): JsonValue {
const timing = taskTiming(task);
const displayPrompt = task.basePrompt || userPromptForDisplay(task.prompt);
const stepCount = taskLlmStepCount(task);
const stepCount = taskListStepCount(task);
if (lite) {
return {
id: task.id,
@@ -1,6 +1,7 @@
// 重构前 index.ts 只读参考:commit 6a04144d3f5103014f75b637d7e6bc2f45bf007fblob 56e590c1a6b5ca7ad128bf2c992f60e46c355a58;可用 `git show 6a04144d3f5103014f75b637d7e6bc2f45bf007f:src/components/microservices/code-queue/src/index.ts` 查看。
import { minimaxM27Model } from "./code-agent/common";
import { openCodeTransportClosedBeforeTerminal, remoteOpenCodeRunCommandForTest } from "./code-agent/opencode";
import { continuePromptSourceBudgetChars, miniMaxJudgeMessages, parsedContinuePromptForJudge, parseJudgeJson, queueRecoveryRetryPrompt, retryPrompt } from "./judge";
import { codeQueueEnvironmentHintTitle, injectCodeQueueEnvironmentHint, promptWithCodeQueueEnvironmentHint, userPromptForDisplay } from "./prompts";
import { buildTaskTranscript, safePreview, transcriptLineSummaryLines } from "./task-view";
@@ -201,12 +202,19 @@ function runQueueOrderingSelfTest(): JsonValue {
const queuedBehindRunning = queueOrderTestTask("codex_4101_queued", "queued", "2026-05-11T10:01:00.000Z", "2026-05-11T10:01:00.000Z");
const terminalAhead = queueOrderTestTask("codex_4200_done", "succeeded", "2026-05-11T11:00:00.000Z", "2026-05-11T11:00:00.000Z");
const queuedAfterTerminal = queueOrderTestTask("codex_4201_queued", "queued", "2026-05-11T11:01:00.000Z", "2026-05-11T11:01:00.000Z");
const mergeTargetEarly = queueOrderTestTask("codex_4300_target_early", "queued", "2026-05-11T12:00:00.000Z", "2026-05-11T12:00:00.000Z");
const mergeSourceMiddle = queueOrderTestTask("codex_4301_source_middle", "queued", "2026-05-11T12:01:00.000Z", "2026-05-11T12:01:00.000Z");
const mergeTargetLate = queueOrderTestTask("codex_4302_target_late", "queued", "2026-05-11T12:02:00.000Z", "2026-05-11T12:02:00.000Z");
mergeTargetEarly.queueId = "queue_merge_target";
mergeSourceMiddle.queueId = "queue_merge_target";
mergeTargetLate.queueId = "queue_merge_target";
const originalMaxActiveQueues = ctx().config.maxActiveQueues;
assertReferenceTest(ctx().queueHeadTask("queue_order_test", blockedByRetry)?.id === activeRetry.id, "retry_wait head must keep blocking a moved older-created task");
assertReferenceTest(ctx().nextRunnableTaskFrom("queue_order_test", blockedByRetry)?.id === activeRetry.id, "next runnable should be the retry_wait head");
assertReferenceTest(ctx().nextRunnableTaskFrom("queue_order_test", [queuedBehindRunning, runningHead]) === null, "running head must block queued tasks behind it");
assertReferenceTest(ctx().nextRunnableTaskFrom("queue_order_test", [queuedAfterTerminal, terminalAhead])?.id === queuedAfterTerminal.id, "terminal head should not block later queued task");
assertReferenceTest(ctx().queueHeadTask("queue_merge_target", [mergeTargetLate, mergeSourceMiddle, mergeTargetEarly])?.id === mergeTargetEarly.id, "merged queue should keep the earliest queueEnteredAt task first");
assertReferenceTest(ctx().queuedStatusReason(queuedBehindRunning, [runningHead, queuedBehindRunning])?.label === "PREV TASK", "queued task behind an active same-queue task should expose PREV TASK reason");
assertReferenceTest(ctx().availableQueueStartSlotsFor(8, 0) === Number.POSITIVE_INFINITY, "maxActiveQueues=0 should leave queue-to-queue concurrency unbounded");
assertReferenceTest(ctx().availableQueueStartSlotsFor(0, 1) === 1, "empty active run slots should leave one slot available");
@@ -245,6 +253,7 @@ function runQueueOrderingSelfTest(): JsonValue {
{ name: "retry_wait_head_blocks_moved_older_created_task", ok: true, head: activeRetry.id, moved: movedOlderCreated.id },
{ name: "running_head_blocks_later_queued_task", ok: true },
{ name: "terminal_task_does_not_block_queue", ok: true },
{ name: "queue_merge_preserves_chronological_task_order", ok: true },
{ name: "queued_reason_prev_task", ok: true },
{ name: "max_active_queues_zero_is_unbounded", ok: true },
{ name: "idle_processing_queue_does_not_consume_active_run_slot", ok: true },
@@ -335,6 +344,16 @@ function runTracePortSelfTest(): JsonValue {
assertReferenceTest(String(edited.bodyPreview || "").includes("+const after = true;"), "opencode edit should use metadata diff for line diff display");
assertReferenceTest(!transcript.some((line) => line.status === "opencode/step-start" || line.status === "opencode/step-finish"), "opencode step boundaries should stay out of trace");
assertReferenceTest(!transcript.some((line) => String(line.bodyPreview || "").includes("<think>hidden reasoning</think>")), "reasoning-only opencode assistant text should not duplicate reasoning");
const remoteTask = testTask("codex_5001_remote_opencode", "remote command prompt", "", [], "2026-05-12T00:01:00.000Z");
remoteTask.providerId = "D601";
remoteTask.cwd = "/home/ubuntu";
remoteTask.model = minimaxM27Model;
const remoteCommand = remoteOpenCodeRunCommandForTest(remoteTask, "hello 'world'");
assertReferenceTest(remoteCommand.includes("exec opencode"), "remote opencode command must exec the opencode binary, not the run subcommand");
assertReferenceTest(!remoteCommand.includes("exec 'run'") && !remoteCommand.includes("exec run --format"), "remote opencode command must not regress to exec run");
assertReferenceTest(!openCodeTransportClosedBeforeTerminal(true, true, false), "exit=0 plus current final response must be terminal even without step_finish");
assertReferenceTest(openCodeTransportClosedBeforeTerminal(true, false, false), "exit=0 without final response and step_finish is not a completed turn");
assertReferenceTest(openCodeTransportClosedBeforeTerminal(false, true, true), "non-zero exit remains a transport failure even with partial text");
return {
ok: true,
cases: [
@@ -345,6 +364,10 @@ function runTracePortSelfTest(): JsonValue {
{ name: "step_boundaries_filtered", ok: true },
{ name: "reasoning_duplicate_filtered", ok: true },
{ name: "duration_preserved", ok: true, durationMs: explored?.durationMs ?? null },
{ name: "remote_opencode_exec_includes_binary", ok: true },
{ name: "opencode_exit0_final_without_step_finish_is_terminal", ok: true },
{ name: "opencode_exit0_without_final_or_step_finish_is_transport_closed", ok: true },
{ name: "opencode_nonzero_exit_is_transport_closed", ok: true },
],
transcript: transcript.filter((line) => line.seq >= 2).map((line) => ({
seq: line.seq,
@@ -11,10 +11,12 @@ export interface TaskOutputContext {
logger: (level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue) => void;
markTaskDirty: (taskId: string) => void;
nowIso: () => string;
onOutputAppended?: (task: QueueTask, output: LiveOutput, op: ArchivedLiveOutput["op"]) => void;
schedulePersistState: () => void;
}
const outputArchiveSeededTasks = new Set<string>();
const archivedOutputCache = new Map<string, { signature: string; output: LiveOutput[] }>();
let context: TaskOutputContext | null = null;
export function configureTaskOutput(runtimeContext: TaskOutputContext): void {
@@ -56,6 +58,7 @@ function ensureTaskOutputArchiveSeeded(task: QueueTask): void {
function appendOutputArchive(task: QueueTask, output: LiveOutput, op: ArchivedLiveOutput["op"], text: string): void {
try {
archivedOutputCache.delete(task.id);
mkdirSync(ctx().config.outputArchiveDir, { recursive: true });
appendFileSync(taskOutputArchivePath(task.id), serializeArchivedOutput(output, op, text), "utf8");
} catch (error) {
@@ -84,6 +87,9 @@ function archiveRecordToOutput(value: unknown): ArchivedLiveOutput | null {
function archivedTaskOutput(task: QueueTask): LiveOutput[] {
const path = taskOutputArchivePath(task.id);
if (!existsSync(path)) return [];
const signature = outputArchiveSignature(task);
const cached = archivedOutputCache.get(task.id);
if (cached?.signature === signature) return cached.output;
const bySeq = new Map<number, LiveOutput>();
try {
const text = readFileSync(path, "utf8");
@@ -107,7 +113,13 @@ function archivedTaskOutput(task: QueueTask): LiveOutput[] {
ctx().logger("warn", "codex_output_archive_read_failed", { taskId: task.id, error: ctx().errorToJson(error) });
return [];
}
return Array.from(bySeq.values()).sort((left, right) => Number(left.seq) - Number(right.seq));
const output = Array.from(bySeq.values()).sort((left, right) => Number(left.seq) - Number(right.seq));
archivedOutputCache.set(task.id, { signature, output });
while (archivedOutputCache.size > 80) {
const firstKey = archivedOutputCache.keys().next().value;
if (typeof firstKey === "string") archivedOutputCache.delete(firstKey);
}
return output;
}
function taskFullOutput(task: QueueTask): LiveOutput[] {
@@ -151,6 +163,7 @@ function appendOutput(task: QueueTask, channel: OutputChannel, text: string, met
task.updatedAt = ctx().nowIso();
ctx().markTaskDirty(task.id);
ctx().schedulePersistState();
ctx().onOutputAppended?.(task, output, archiveOp);
return output;
}
@@ -1206,15 +1206,53 @@ function fullTranscript(task: QueueTask): TranscriptLine[] {
return cachedFullTranscript(task);
}
function nonNegativeInteger(value: unknown): number | null {
const parsed = Number(value);
return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : null;
}
function retainedOutputStartsTraceStep(output: LiveOutput): boolean {
if (output.channel === "diff" || output.channel === "tool") return true;
if (output.channel !== "command") return false;
const method = String(output.method || "");
return method === "item/started" || (method !== "item/commandExecution/outputDelta" && method !== "item/completed");
}
function retainedTaskStepCount(task: QueueTask): number {
return (Array.isArray(task.output) ? task.output : []).reduce((count, output) => count + (retainedOutputStartsTraceStep(output) ? 1 : 0), 0);
}
function taskStoredStepCount(task: QueueTask): number | null {
return nonNegativeInteger(task.stepCount) ?? nonNegativeInteger(task.llmStepCount);
}
// Used by overview/list responses and SSE patches. It must stay archive-free;
// trace/detail endpoints compute exact counts from the full transcript on demand.
function taskListStepCount(task: QueueTask): number {
return taskStoredStepCount(task) ?? retainedTaskStepCount(task);
}
function taskOutputMaxSeq(task: QueueTask): number {
const output = Array.isArray(task.output) ? task.output : [];
const promptHistory = Array.isArray(task.promptHistory) ? task.promptHistory : [];
const attempts = Array.isArray(task.attempts) ? task.attempts : [];
const values = [
nonNegativeInteger(task.outputMaxSeq),
nonNegativeInteger(output.at(-1)?.seq),
nonNegativeInteger(promptHistory.at(-1)?.seq),
...attempts.map((attempt) => nonNegativeInteger(attempt.outputEndSeq)),
].filter((value): value is number => value !== null);
return values.length > 0 ? Math.max(...values) : 0;
}
function taskLlmStepCount(task: QueueTask): number {
return taskStepCountFromTranscript(task, cachedPreviewTranscript(task));
return taskStoredStepCount(task) ?? taskStepCountFromTranscript(task, cachedPreviewTranscript(task));
}
function taskForMetaResponse(task: QueueTask): JsonValue {
const fullOutput = taskFullOutput(task);
const lastOutputSeq = fullOutput.at(-1)?.seq ?? 0;
const lastOutputSeq = taskOutputMaxSeq(task);
const displayPrompt = task.basePrompt || userPromptForDisplay(task.prompt);
const stepCount = taskLlmStepCount(task);
const stepCount = taskListStepCount(task);
return {
id: task.id,
queueId: ctx().queueIdOf(task),
@@ -1260,7 +1298,7 @@ function taskForMetaResponse(task: QueueTask): JsonValue {
cancelRequested: task.cancelRequested,
terminalUnread: terminalTaskUnread(task),
nextMode: task.nextMode,
outputCount: fullOutput.length,
outputCount: task.output.length,
retainedOutputCount: task.output.length,
eventCount: task.events.length,
transcriptCount: null,
@@ -1274,8 +1312,8 @@ function taskForMetaResponse(task: QueueTask): JsonValue {
function taskForCompactMetaResponse(task: QueueTask): JsonValue {
const displayPrompt = task.basePrompt || userPromptForDisplay(task.prompt);
const lastOutputSeq = task.output.at(-1)?.seq ?? 0;
const stepCount = taskLlmStepCount(task);
const lastOutputSeq = taskOutputMaxSeq(task);
const stepCount = taskListStepCount(task);
return {
id: task.id,
queueId: ctx().queueIdOf(task),
@@ -2093,7 +2131,9 @@ export {
setAttemptInputPrompt,
statsDaysFromUrl,
taskExecutionSummary,
taskListStepCount,
taskLlmStepCount,
taskOutputMaxSeq,
taskForCompactMetaResponse,
taskForMetaResponse,
taskForResponse,
@@ -261,6 +261,9 @@ export interface QueueTask {
codexThreadId: string | null;
activeTurnId: string | null;
finalResponse: string;
stepCount?: number;
llmStepCount?: number;
outputMaxSeq?: number;
lastError: string | null;
lastJudge: JudgeResult | null;
judgeFailCount: number;