feat: add unified OA event flow

This commit is contained in:
Codex
2026-05-14 09:41:55 +00:00
parent b36d7f94d7
commit 9f483b002c
37 changed files with 3417 additions and 502 deletions
@@ -43,7 +43,13 @@ function ctx(): DevContainerContext {
return context;
}
async function startDevContainerPlan(plan: DevContainerPlan, options: { forceRecreate: boolean; verifyPing: boolean; prepareRuntime: boolean }): Promise<{
async function startDevContainerPlan(plan: DevContainerPlan, options: {
forceRecreate: boolean;
verifyPing: boolean;
prepareRuntime: boolean;
onCommandStart?: (command: { name: string; providerId: string; timeoutMs: number }) => void;
onCommandComplete?: (command: DevContainerCommandLog) => void;
}): Promise<{
ok: boolean;
providerId: string;
plan: DevContainerPlan;
@@ -52,8 +58,10 @@ async function startDevContainerPlan(plan: DevContainerPlan, options: { forceRec
}> {
const commands: DevContainerCommandLog[] = [];
const run = async (targetProviderId: string, script: string, timeoutMs: number, name: string): Promise<DevContainerCommandLog> => {
options.onCommandStart?.({ name, providerId: targetProviderId, timeoutMs });
const command = await ctx().runCodeQueueSsh(targetProviderId, script, timeoutMs, name);
commands.push(command);
options.onCommandComplete?.(command);
ctx().throwIfCommandFailed(command);
return command;
};
@@ -68,7 +76,7 @@ async function startDevContainerPlan(plan: DevContainerPlan, options: { forceRec
await run("main-server", ctx().masterProxyPrepareScript(plan), 30_000, "master-proxy-prepare");
await run(plan.providerId, ctx().containerTunnelStartScript(plan), 45_000, "remote-tunnel-start");
await run("main-server", ctx().masterProxyFinishScript(plan), 30_000, "master-proxy-finish");
if (options.prepareRuntime) await run(plan.providerId, ctx().remoteCodexRuntimePrepareScript(plan), 600_000, "remote-codex-runtime-prepare");
if (options.prepareRuntime) await run(plan.providerId, ctx().remoteCodexRuntimePrepareScript(plan), 1_800_000, "remote-codex-runtime-prepare");
const verification: Record<string, JsonValue> = {};
if (options.verifyPing) {
const before = await run("main-server", ctx().masterProxyEvidenceScript(plan), 15_000, "master-proxy-evidence-before-ping");
@@ -92,7 +100,18 @@ export async function ensureTaskExecutionContainer(task: QueueTask): Promise<voi
if (existing !== undefined) return existing;
const promise = (async () => {
ctx().appendOutput(task, "system", `ensuring provider=${plan.providerId} container=${plan.containerName} workdir=${task.cwd}\n`, "provider/container");
const result = await startDevContainerPlan(plan, { forceRecreate: false, verifyPing: false, prepareRuntime: true });
const result = await startDevContainerPlan(plan, {
forceRecreate: false,
verifyPing: false,
prepareRuntime: true,
onCommandStart: (command) => {
ctx().appendOutput(task, "system", `provider prepare start name=${command.name} target=${command.providerId} timeoutMs=${command.timeoutMs}\n`, "provider/container");
},
onCommandComplete: (command) => {
const status = command.exitCode === 0 ? "ok" : `failed exit=${command.exitCode ?? "null"} signal=${command.signal ?? "null"}`;
ctx().appendOutput(task, command.exitCode === 0 ? "system" : "error", `provider prepare ${status} name=${command.name} target=${command.providerId} durationMs=${command.durationMs}\n`, "provider/container");
},
});
ctx().appendOutput(task, "system", `provider container ready provider=${plan.providerId} container=${plan.containerName} commands=${result.commands.length}\n`, "provider/container");
ctx().logger("info", "task_provider_container_ready", {
taskId: task.id,
@@ -109,6 +109,18 @@ import {
transcriptChunkResponse,
} from "./queue-api";
import { ReferenceTaskLookupError, configureReferences, injectReferencedTaskContext, taskReferenceIds } from "./references";
import {
applyOaTraceStatsToTaskJson,
configureOaEvents,
publishCodeQueueQueueUpdated,
publishCodeQueueTaskUpdated,
publishCodeQueueTraceStatsSnapshot,
publishCodeQueueTraceStep,
outputTraceKind,
readOaTraceStatsForTask,
readOaTraceStatsForTaskAttempts,
readOaTraceStatsForTasks,
} from "./oa-events";
import { configureSelfTests, runJudgeInfraSelfTest, runQueueOrderingSelfTest, runReferenceInjectionSelfTest, runTracePortSelfTest } from "./self-tests";
import {
configureTaskView,
@@ -151,31 +163,8 @@ const queueIdPattern = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u;
const queueNameMaxLength = 80;
const config = readConfig();
type CodeQueueEventType = "hello" | "task-updated" | "task-step" | "queue-updated";
interface CodeQueueEvent {
sequence: number;
type: CodeQueueEventType;
at: string;
reason: string;
taskId?: string;
queueId?: string;
status?: TaskStatus;
updatedAt?: string;
stepCount?: number;
outputMaxSeq?: number;
}
interface CodeQueueEventClient {
enqueue: (chunk: string) => void;
close: () => void;
}
const codeQueueEventClients = new Set<CodeQueueEventClient>();
const codeQueueEventEncoder = new TextEncoder();
const taskEventSnapshots = new Map<string, { stepCount: number; outputMaxSeq: number; status: TaskStatus; updatedAt: string }>();
let codeQueueEventSequence = 0;
const logger = createLogger("code-queue", config.logFile);
configureOaEvents({ baseUrl: config.oaEventFlowBaseUrl, logger, nowIso });
const state = emptyState();
let processing = false;
const processingQueues = new Set<string>();
@@ -330,6 +319,7 @@ function readConfig(): RuntimeConfig {
turnNoActivityTimeoutMs: Math.max(60_000, Math.min(30 * 60_000, envNumber("CODEX_TURN_NO_ACTIVITY_TIMEOUT_MS", 6 * 60_000))),
databaseUrl: envRequiredString("DATABASE_URL"),
databaseFlushIntervalMs: Math.max(100, Math.min(10_000, envNumber("CODE_QUEUE_DATABASE_FLUSH_INTERVAL_MS", 1000))),
oaEventFlowBaseUrl: envString("OA_EVENT_FLOW_BASE_URL", "http://oa-event-flow:4255").replace(/\/+$/u, ""),
notifyClaudeQqEnabled: envBool("CODE_QUEUE_NOTIFY_CLAUDEQQ_ENABLED", false),
notifyClaudeQqBaseUrl: envString("CODE_QUEUE_NOTIFY_CLAUDEQQ_BASE_URL", "http://backend-core:8080/api/microservices/claudeqq/proxy").replace(/\/+$/u, ""),
notifyClaudeQqTargetType: notifyTargetTypeRaw === "group" ? "group" : "private",
@@ -610,142 +600,10 @@ function nowIso(): string {
return new Date().toISOString();
}
function sseChunk(event: CodeQueueEvent): string {
return [
`id: ${event.sequence}`,
`event: ${event.type}`,
`data: ${JSON.stringify(event)}`,
"",
"",
].join("\n");
}
function emitCodeQueueEvent(event: Omit<CodeQueueEvent, "sequence" | "at"> & { at?: string }): void {
const payload: CodeQueueEvent = {
...event,
sequence: ++codeQueueEventSequence,
at: event.at || nowIso(),
};
if (codeQueueEventClients.size === 0) return;
const chunk = sseChunk(payload);
for (const client of Array.from(codeQueueEventClients)) {
try {
client.enqueue(chunk);
} catch {
client.close();
}
}
}
function taskOutputMaxSeq(task: QueueTask): number {
return taskViewOutputMaxSeq(task);
}
function publishTaskEvent(task: QueueTask, reason: string, options: { onlyStepChange?: boolean; stepChanged?: boolean } = {}): void {
// Hot path: never build full transcripts or read output archives here.
const stepCount = taskListStepCount(task);
const outputMaxSeq = taskOutputMaxSeq(task);
const previous = taskEventSnapshots.get(task.id);
const stepChanged = options.stepChanged ?? (previous === undefined ? stepCount > 0 : stepCount > previous.stepCount);
if (options.onlyStepChange === true && !stepChanged) {
taskEventSnapshots.set(task.id, { stepCount, outputMaxSeq, status: task.status, updatedAt: task.updatedAt });
return;
}
const type: CodeQueueEventType = stepChanged ? "task-step" : "task-updated";
taskEventSnapshots.set(task.id, { stepCount, outputMaxSeq, status: task.status, updatedAt: task.updatedAt });
emitCodeQueueEvent({
type,
reason,
taskId: task.id,
queueId: queueIdOf(task),
status: task.status,
updatedAt: task.updatedAt,
stepCount,
outputMaxSeq,
});
}
function publishQueueEvent(reason: string, queueId = ""): void {
emitCodeQueueEvent({ type: "queue-updated", reason, queueId: queueId || undefined });
}
function outputCanChangeStepCount(output: LiveOutput): boolean {
return output.channel === "command" || output.channel === "diff" || output.channel === "tool";
}
function outputStartsTraceStep(output: LiveOutput): boolean {
if (output.channel === "diff" || output.channel === "tool") return true;
if (output.channel !== "command") return false;
const method = String(output.method || "");
return method === "item/started" || (method !== "item/commandExecution/outputDelta" && method !== "item/completed");
}
function recordTaskOutputMetrics(task: QueueTask, output: LiveOutput, op: "set" | "append"): boolean {
task.outputMaxSeq = Math.max(taskOutputMaxSeq(task), Number.isFinite(Number(output.seq)) ? Number(output.seq) : 0);
if (op === "append" || !outputStartsTraceStep(output)) return false;
const storedStepCount = Number(task.stepCount ?? task.llmStepCount);
const nextStepCount = Number.isFinite(storedStepCount) && storedStepCount >= 0
? Math.floor(storedStepCount) + 1
: task.output.reduce((count, item) => count + (outputStartsTraceStep(item) ? 1 : 0), 0);
task.stepCount = nextStepCount;
task.llmStepCount = nextStepCount;
return true;
}
function codeQueueEventsResponse(): Response {
let heartbeat: ReturnType<typeof setInterval> | null = null;
let client: CodeQueueEventClient | null = null;
const stream = new ReadableStream<Uint8Array>({
start(controller) {
client = {
enqueue(chunk: string) {
controller.enqueue(codeQueueEventEncoder.encode(chunk));
},
close() {
if (heartbeat !== null) {
clearInterval(heartbeat);
heartbeat = null;
}
if (client !== null) codeQueueEventClients.delete(client);
client = null;
try {
controller.close();
} catch {
// The browser may have already closed the EventSource connection.
}
},
};
codeQueueEventClients.add(client);
controller.enqueue(codeQueueEventEncoder.encode("retry: 1000\n\n"));
client.enqueue(sseChunk({
sequence: codeQueueEventSequence,
type: "hello",
at: nowIso(),
reason: "connected",
}));
heartbeat = setInterval(() => {
try {
controller.enqueue(codeQueueEventEncoder.encode(`: heartbeat ${nowIso()}\n\n`));
} catch {
client?.close();
}
}, 25_000);
heartbeat.unref?.();
},
cancel() {
client?.close();
},
});
return new Response(stream, {
headers: {
"content-type": "text/event-stream; charset=utf-8",
"cache-control": "no-store, no-transform",
"connection": "keep-alive",
"x-accel-buffering": "no",
},
});
}
function resolveReasoningEffort(model: string, explicit?: string | null): string | null {
const requested = explicit?.trim();
if (requested) return requested;
@@ -869,7 +727,7 @@ function pruneTaskHotState(task: QueueTask): void {
}
function taskRetainedTraceStepCount(task: QueueTask): number {
return task.output.reduce((count, output) => count + (outputStartsTraceStep(output) ? 1 : 0), 0);
return task.output.reduce((count, output) => count + (outputStartsTraceStep(task, output) ? 1 : 0), 0);
}
function normalizeTaskMetric(value: unknown): number | null {
@@ -939,6 +797,104 @@ function redactDatabaseUrl(value: string): string {
}
}
function publishTaskOaEvent(task: QueueTask, reason: string, options: { onlyStepChange?: boolean; stepChanged?: boolean } = {}): void {
const stepCount = taskListStepCount(task);
const outputMaxSeq = taskOutputMaxSeq(task);
if (options.onlyStepChange === true && options.stepChanged !== true) return;
const queueId = queueIdOf(task);
publishCodeQueueTaskUpdated(task, queueId, reason, stepCount, outputMaxSeq);
publishCodeQueueTraceStatsSnapshot(task, queueId, reason, stepCount, outputMaxSeq);
}
function publishQueueEvent(reason: string, queueId = ""): void {
publishCodeQueueQueueUpdated(queueId, reason);
}
function isOpenCodeStepBoundaryMethod(method: string | undefined): boolean {
return method === "opencode/step-start" || method === "opencode/step-finish";
}
function outputCanChangeStepCount(output: LiveOutput): boolean {
if (output.channel === "user" && output.method === "enqueue") return false;
return !isOpenCodeStepBoundaryMethod(output.method);
}
function commandStartedBeforeIn(outputs: LiveOutput[], output: LiveOutput): boolean {
if (typeof output.itemId !== "string") return false;
return outputs.some((item) => item !== output && item.itemId === output.itemId && item.channel === "command" && item.method === "item/started");
}
function outputStartsTraceStepInHistory(outputs: LiveOutput[], output: LiveOutput): boolean {
if (output.channel === "user" && output.method === "enqueue") return false;
if (isOpenCodeStepBoundaryMethod(output.method)) return false;
if (output.channel === "diff" || output.channel === "tool" || output.channel === "error" || output.channel === "assistant" || output.channel === "reasoning" || output.channel === "system") return true;
if (output.channel === "user") return true;
if (output.channel !== "command") return true;
const method = String(output.method || "");
if (method === "item/started") return true;
if (method === "item/commandExecution/outputDelta") return false;
if (method === "item/completed") return !commandStartedBeforeIn(outputs, output);
return true;
}
function outputStartsTraceStep(task: QueueTask, output: LiveOutput): boolean {
return outputStartsTraceStepInHistory(task.output, output);
}
function traceStatsFromOutputs(outputs: LiveOutput[]): { stepCount: number; readCount: number; editCount: number; runCount: number; errorCount: number } {
const visibleOutputs = outputs.filter((output) => outputStartsTraceStepInHistory(outputs, output));
const stats = { stepCount: visibleOutputs.length, readCount: 0, editCount: 0, runCount: 0, errorCount: 0 };
for (const output of visibleOutputs) {
const kind = outputTraceKind(output);
if (kind === "read") stats.readCount += 1;
if (kind === "edit") stats.editCount += 1;
if (kind === "run") stats.runCount += 1;
if (kind === "error") stats.errorCount += 1;
}
return stats;
}
function attemptIndexFromOutput(output: LiveOutput): number | null {
const match = String(output.text || "").match(/\battempt\s+(\d+)\s*\/\s*\d+\b/iu);
const value = Number(match?.[1] ?? NaN);
return Number.isInteger(value) && value > 0 ? value : null;
}
function outputAttemptIndexMap(outputs: LiveOutput[]): Map<number, number> {
const result = new Map<number, number>();
let currentAttempt = 0;
for (const output of outputs.slice().sort((left, right) => Number(left.seq) - Number(right.seq))) {
const startedAttempt = attemptIndexFromOutput(output);
if (startedAttempt !== null) currentAttempt = startedAttempt;
if (currentAttempt > 0 && outputStartsTraceStepInHistory(outputs, output)) result.set(output.seq, currentAttempt);
}
return result;
}
function traceAttemptIndexesForTask(task: QueueTask): number[] {
const indexes = new Set<number>();
for (const attempt of task.attempts) {
const index = Number(attempt.index);
if (Number.isInteger(index) && index > 0) indexes.add(index);
}
const currentAttempt = Number(task.currentAttempt || 0);
const maxAttempt = Math.max(currentAttempt, ...Array.from(indexes), 0);
for (let index = 1; index <= maxAttempt; index += 1) indexes.add(index);
return Array.from(indexes).sort((left, right) => left - right);
}
function recordTaskOutputMetrics(task: QueueTask, output: LiveOutput, op: "set" | "append"): boolean {
task.outputMaxSeq = Math.max(taskOutputMaxSeq(task), Number.isFinite(Number(output.seq)) ? Number(output.seq) : 0);
if (op === "append" || !outputStartsTraceStep(task, output)) return false;
const storedStepCount = Number(task.stepCount ?? task.llmStepCount);
const nextStepCount = Number.isFinite(storedStepCount) && storedStepCount >= 0
? Math.floor(storedStepCount) + 1
: task.output.reduce((count, item) => count + (outputStartsTraceStep(task, item) ? 1 : 0), 0);
task.stepCount = nextStepCount;
task.llmStepCount = nextStepCount;
return true;
}
function errorToJson(error: unknown): JsonValue {
if (error instanceof Error) return { name: error.name, message: error.message, stack: error.stack ?? null };
return String(error);
@@ -957,7 +913,7 @@ function markTaskDirty(taskId: string): void {
function persistTaskState(task: QueueTask): void {
markTaskDirty(task.id);
persistState(false);
publishTaskEvent(task, "persist");
publishTaskOaEvent(task, "persist");
}
function markQueueDirty(queueId: string): void {
@@ -1780,7 +1736,7 @@ function addEvent(task: QueueTask, event: CodexEventSummary): void {
task.events.push(event);
if (config.maxInMemoryEventRecords > 0 && task.events.length > config.maxInMemoryEventRecords) task.events.splice(0, task.events.length - config.maxInMemoryEventRecords);
markTaskDirty(task.id);
publishTaskEvent(task, "agent-event", { onlyStepChange: true });
publishTaskOaEvent(task, "agent-event", { onlyStepChange: true });
}
function taskReferencesEqual(left: string[], right: string[]): boolean {
@@ -1970,8 +1926,9 @@ configureTaskOutput({
onOutputAppended: (task, output, op) => {
const archiveOp = op === "append" ? "append" : "set";
const stepChanged = recordTaskOutputMetrics(task, output, archiveOp);
if (stepChanged) publishCodeQueueTraceStep(task, queueIdOf(task), output, taskOutputMaxSeq(task));
if (archiveOp === "append" && !outputCanChangeStepCount(output)) return;
publishTaskEvent(task, "output", { onlyStepChange: archiveOp === "append", stepChanged });
publishTaskOaEvent(task, "output", { onlyStepChange: archiveOp === "append", stepChanged });
},
schedulePersistState,
});
@@ -2050,6 +2007,7 @@ configureQueueApi({
safeQueueName,
sql,
taskQueueEnteredAt,
traceStatsForTasks: readOaTraceStatsForTasks,
tasks: () => state.tasks,
truthyParam,
});
@@ -2928,7 +2886,7 @@ async function createTasks(req: Request): Promise<Response> {
for (const task of tasks) appendOutput(task, "user", `${task.prompt}\n`, "enqueue");
state.tasks.push(...tasks);
if (tasks.length > 0) armIdleNotification();
for (const task of tasks) publishTaskEvent(task, "enqueue");
for (const task of tasks) publishTaskOaEvent(task, "enqueue");
for (const id of new Set(tasks.map(queueIdOf))) publishQueueEvent("enqueue", id);
persistState();
logger("info", "tasks_enqueued", { count: tasks.length, ids: tasks.map((task) => task.id), queueIds: Array.from(new Set(tasks.map(queueIdOf))), providerIds: Array.from(new Set(tasks.map((task) => task.providerId))) });
@@ -3057,7 +3015,7 @@ async function markTaskRead(task: QueueTask): Promise<Response> {
task.readAt = nowIso();
markTaskDirty(task.id);
persistState(false);
publishTaskEvent(task, "read");
publishTaskOaEvent(task, "read");
logger("info", "task_marked_read", { taskId: task.id, queueId: queueIdOf(task), status: task.status });
}
await flushDirtyTasksToDatabase(true);
@@ -3102,7 +3060,7 @@ async function markTaskReadById(taskId: string): Promise<Response> {
const hotTask = findTask(taskId);
if (hotTask !== null) {
hotTask.readAt = readAt;
publishTaskEvent(hotTask, "read");
publishTaskOaEvent(hotTask, "read");
}
logger("info", "task_marked_read", { taskId, queueId: safeQueueId(row.queue_id), status: row.status });
}
@@ -3148,7 +3106,7 @@ async function markTerminalTasksRead(url: URL): Promise<Response> {
for (const task of state.tasks) {
if (!ids.has(task.id)) continue;
task.readAt = readAt;
publishTaskEvent(task, "read-all");
publishTaskOaEvent(task, "read-all");
}
if (ids.size > 0) {
logger("info", "terminal_tasks_marked_read", { count: ids.size, queueId });
@@ -3161,7 +3119,7 @@ async function markTerminalTasksRead(url: URL): Promise<Response> {
if (!terminalTaskUnread(task)) continue;
task.readAt = readAt;
markTaskDirty(task.id);
publishTaskEvent(task, "read-all");
publishTaskOaEvent(task, "read-all");
count += 1;
}
if (count > 0) {
@@ -3269,6 +3227,40 @@ async function mergeDatabaseQueueTasks(sourceQueueIds: string[], targetQueueId:
return rows.map((row) => row.id);
}
async function deleteDatabaseQueues(queueIds: string[]): Promise<string[]> {
if (!databaseReady || queueIds.length === 0) return [];
const rows = await sql<Array<{ id: string }>>`
DELETE FROM unidesk_code_queue_queues
WHERE id IN ${sql(queueIds)}
RETURNING id
`;
return rows.map((row) => row.id);
}
function queueSnapshot(queueId: string, timestamp: string): QueueRecord {
const queue = knownQueue(queueId);
return queue === null
? { id: queueId, name: queueId, createdAt: timestamp, updatedAt: timestamp }
: { ...queue };
}
function deleteQueuesFromState(queueIds: string[]): QueueRecord[] {
const queueIdSet = new Set(queueIds);
const deletedQueues: QueueRecord[] = [];
const retainedQueues: QueueRecord[] = [];
for (const queue of state.queues) {
if (queueIdSet.has(queue.id)) {
deletedQueues.push({ ...queue });
dirtyDatabaseQueueIds.delete(queue.id);
} else {
retainedQueues.push(queue);
}
}
state.queues.splice(0, state.queues.length, ...retainedQueues);
for (const queueId of queueIds) dirtyDatabaseQueueIds.delete(queueId);
return deletedQueues;
}
async function mergeQueues(targetQueueIdValue: string | null, req: Request): Promise<Response> {
const body = await readJson(req);
const record = typeof body === "object" && body !== null && !Array.isArray(body) ? body as Record<string, unknown> : {};
@@ -3289,13 +3281,9 @@ async function mergeQueues(targetQueueIdValue: string | null, req: Request): Pro
const mergedAt = nowIso();
const targetQueue = ensureQueue(targetQueueId);
const sourceQueues = sourceQueueIds.map((id) => ensureQueue(id));
const sourceQueues = sourceQueueIds.map((id) => queueSnapshot(id, mergedAt));
targetQueue.updatedAt = mergedAt;
markQueueDirty(targetQueue.id);
for (const queue of sourceQueues) {
queue.updatedAt = mergedAt;
markQueueDirty(queue.id);
}
const sourceSet = new Set(sourceQueueIds);
const hotMovedTasks: QueueTask[] = [];
@@ -3306,18 +3294,22 @@ async function mergeQueues(targetQueueIdValue: string | null, req: Request): Pro
task.queueId = targetQueueId;
hotMovedTasks.push(task);
markTaskDirty(task.id);
publishTaskEvent(task, "queue-merged");
publishTaskOaEvent(task, "queue-merged");
}
const databaseMovedTaskIds = await mergeDatabaseQueueTasks(sourceQueueIds, targetQueueId);
const deletedSourceQueues = deleteQueuesFromState(sourceQueueIds);
const databaseDeletedQueueIds = await deleteDatabaseQueues(sourceQueueIds);
persistState(false);
publishQueueEvent("queue-merged", targetQueueId);
for (const sourceQueueId of sourceQueueIds) publishQueueEvent("queue-merged", sourceQueueId);
for (const sourceQueueId of sourceQueueIds) publishQueueEvent("queue-deleted-after-merge", sourceQueueId);
if (hotMovedTasks.some((task) => task.status === "queued" || task.status === "retry_wait")) armIdleNotification();
logger("info", "queues_merged", {
targetQueueId,
sourceQueueIds,
deletedSourceQueueIds: deletedSourceQueues.map((queue) => queue.id),
hotMovedTaskCount: hotMovedTasks.length,
databaseMovedTaskCount: databaseReady ? databaseMovedTaskIds.length : null,
databaseDeletedQueueIds: databaseReady ? databaseDeletedQueueIds : null,
});
for (const id of mergeQueueIds) mergingQueues.delete(id);
scheduleQueue(targetQueueId);
@@ -3339,9 +3331,11 @@ async function mergeQueues(targetQueueIdValue: string | null, req: Request): Pro
mergedTaskCount: databaseReady ? databaseMovedTaskIds.length : hotMovedTasks.length,
movedTaskIds: orderedMovedTaskIds.slice(0, 500),
targetTaskOrder: targetTaskOrder.slice(0, 500),
order: "merged tasks keep their original queueEnteredAt/createdAt ordering; source queue records are retained and never deleted",
order: "merged tasks keep their original queueEnteredAt/createdAt ordering; source queue records are deleted after merge",
targetQueue,
sourceQueues,
deletedSourceQueues: deletedSourceQueues.length > 0 ? deletedSourceQueues : sourceQueues,
deletedSourceQueueIds: sourceQueueIds,
queues: perQueueSummaries(tasks),
summary: queueSummary(false, tasks),
}, 202);
@@ -3376,13 +3370,41 @@ async function moveTaskToQueue(task: QueueTask, req: Request): Promise<Response>
return jsonResponse({ ok: true, task: taskForResponse(task), queue: await queueSummaryForResponse() }, 202);
}
async function backfillOaTraceStats(url: URL): Promise<JsonValue> {
const limitRaw = Number(url.searchParams.get("limit") ?? 2000);
const limit = Number.isInteger(limitRaw) && limitRaw > 0 ? Math.min(20_000, limitRaw) : 2000;
const taskId = url.searchParams.get("taskId")?.trim() ?? "";
const tasks = taskId.length > 0
? [await findTaskForRead(taskId)].filter((task): task is QueueTask => task !== null)
: (await loadAllTasksForRead()).slice(-limit);
const includeSteps = url.searchParams.get("steps") !== "0";
let stepEventCount = 0;
for (const task of tasks) {
const queueId = queueIdOf(task);
const outputMaxSeq = taskOutputMaxSeq(task);
const output = taskFullOutput(task);
const traceStats = traceStatsFromOutputs(output);
const attemptBySeq = outputAttemptIndexMap(output);
if (includeSteps) {
for (const item of output) {
if (!outputStartsTraceStepInHistory(output, item)) continue;
publishCodeQueueTraceStep(task, queueId, item, outputMaxSeq, attemptBySeq.get(item.seq) ?? null);
stepEventCount += 1;
}
}
publishCodeQueueTraceStatsSnapshot(task, queueId, "backfill", traceStats.stepCount, outputMaxSeq, traceStats);
}
logger("info", "oa_trace_stats_backfill_enqueued", { taskCount: tasks.length, limit, taskId: taskId || null, includeSteps, stepEventCount });
return { ok: true, taskCount: tasks.length, limit, taskId: taskId || null, includeSteps, stepEventCount, eventFlowBaseUrl: config.oaEventFlowBaseUrl } as unknown as JsonValue;
}
async function route(req: Request): Promise<Response> {
const url = new URL(req.url);
if (req.method === "OPTIONS") return jsonResponse({ ok: true });
try {
if (url.pathname === "/" || url.pathname === "/health") return jsonResponse({ ok: true, service: "code-queue", queue: await queueSummaryForHealth(), startedAt: serviceStartedAt });
if (url.pathname === "/logs") return jsonResponse({ ok: true, logs: recentLogs.slice(-parseLimit(url)) });
if (url.pathname === "/api/events" && req.method === "GET") return codeQueueEventsResponse();
if (url.pathname === "/api/events" && req.method === "GET") return jsonResponse({ ok: false, error: "Code Queue private SSE was removed; subscribe to oa-event-flow /api/events/stream with service:code-queue tags." }, 410);
if (url.pathname === "/api/dev-ready" && req.method === "GET") return jsonResponse({ ok: true, devReady: collectDevReady() });
if (url.pathname === "/api/dev-containers" && req.method === "GET") {
const plan = buildDevContainerPlan(normalizeProviderId(config.devContainerDefaultProviderId) ?? "D601", {});
@@ -3413,6 +3435,7 @@ async function route(req: Request): Promise<Response> {
if (url.pathname === "/api/queue-order/self-test" && (req.method === "GET" || req.method === "POST")) return jsonResponse(runQueueOrderingSelfTest());
if (url.pathname === "/api/reference-injection/self-test" && (req.method === "GET" || req.method === "POST")) return jsonResponse(await runReferenceInjectionSelfTest());
if (url.pathname === "/api/trace-port/self-test" && (req.method === "GET" || req.method === "POST")) return jsonResponse(runTracePortSelfTest());
if (url.pathname === "/api/oa/backfill" && (req.method === "GET" || req.method === "POST")) return jsonResponse(await backfillOaTraceStats(url));
if (url.pathname === "/api/notifications/claudeqq" && req.method === "GET") {
await loadClaudeQqNotificationOutboxFromDatabase();
const limit = parseLimit(url);
@@ -3462,7 +3485,8 @@ async function route(req: Request): Promise<Response> {
.filter((task) => taskMatchesSearch(task, searchTerms));
const limit = parseLimit(url);
const page = taskPageRows(filteredTasks, url, limit);
const tasks = page.rows.map((task) => taskForListResponse(task, lite, allTasks));
const traceStats = await readOaTraceStatsForTasks(page.rows.map((task) => task.id));
const tasks = page.rows.map((task) => applyOaTraceStatsToTaskJson(taskForListResponse(task, lite, allTasks), traceStats.get(`task:${task.id}`) ?? null));
return jsonResponse({
ok: true,
queue: queueSummary(includeDevReady, allTasks),
@@ -3507,7 +3531,8 @@ async function route(req: Request): Promise<Response> {
if (traceSummaryMatch !== null && req.method === "GET") {
const task = await findTaskForRead(decodeURIComponent(traceSummaryMatch[1] ?? ""));
if (task === null) return jsonResponse({ ok: false, error: "task not found" }, 404);
return jsonResponse({ ok: true, summary: taskTraceSummaryResponse(task) });
const traceStats = await readOaTraceStatsForTaskAttempts(task.id, traceAttemptIndexesForTask(task));
return jsonResponse({ ok: true, summary: taskTraceSummaryResponse(task, traceStats.get(`task:${task.id}`) ?? null, traceStats) });
}
const traceStepsMatch = url.pathname.match(/^\/api\/tasks\/([^/]+)\/trace-steps$/u);
if (traceStepsMatch !== null && req.method === "GET") {
@@ -3543,9 +3568,10 @@ async function route(req: Request): Promise<Response> {
if (action === "edit" && (req.method === "POST" || req.method === "PATCH")) return await editQueuedTaskPrompt(task, req);
if (action !== undefined) return jsonResponse({ ok: false, error: "not found" }, 404);
if (req.method === "GET") {
if (url.searchParams.get("meta") === "1") return jsonResponse({ ok: true, task: taskForMetaResponse(task) });
const traceStats = await readOaTraceStatsForTask(task.id);
if (url.searchParams.get("meta") === "1") return jsonResponse({ ok: true, task: applyOaTraceStatsToTaskJson(taskForMetaResponse(task), traceStats) });
const includeRaw = url.searchParams.get("raw") === "1" || url.searchParams.get("full") === "1";
return jsonResponse({ ok: true, task: taskForResponse(task, true, includeRaw) });
return jsonResponse({ ok: true, task: applyOaTraceStatsToTaskJson(taskForResponse(task, true, includeRaw), traceStats) });
}
if (req.method === "DELETE") return await interruptTask(task);
return jsonResponse({ ok: false, error: "method not allowed" }, 405);
@@ -3578,5 +3604,6 @@ const startupRecovered = queueActiveTasksForRestartRetry("Service restarted whil
if (startupRecovered > 0) logger("warn", "startup_requeued_active_tasks", { recovered: startupRecovered });
persistState();
serviceReady = true;
setTimeout(() => { void backfillOaTraceStats(new URL("http://code-queue.local/api/oa/backfill?limit=2000")).catch((error) => logger("warn", "oa_trace_stats_startup_backfill_failed", { error: errorToJson(error) })); }, 1000).unref?.();
scheduleQueue();
scheduleClaudeQqNotificationDrain(1000);
@@ -0,0 +1,370 @@
import { createHash } from "node:crypto";
import type { JsonValue, LiveOutput, QueueTask, TaskStatus } from "./types";
type JsonRecord = Record<string, JsonValue>;
interface OaEventContext {
baseUrl: string;
logger: (level: "debug" | "info" | "warn" | "error", message: string, data?: JsonValue) => void;
nowIso: () => string;
}
interface OaEventEnvelope {
eventId: string;
type: string;
createdAt: string;
sourceKind: string;
sourceId: string;
aggregateType: string;
aggregateId: string;
correlationId: string;
causationId?: string | null;
tags: string[];
payload: JsonRecord;
}
export interface OaTraceStats extends JsonRecord {
scopeId: string;
source: "oa-event-flow";
}
const postTimeoutMs = 2500;
let context: OaEventContext | null = null;
export function configureOaEvents(runtimeContext: OaEventContext): void {
const baseUrl = runtimeContext.baseUrl.replace(/\/+$/u, "");
if (baseUrl.length === 0) throw new Error("OA_EVENT_FLOW_BASE_URL is required");
context = { ...runtimeContext, baseUrl };
}
function ctx(): OaEventContext {
if (context === null) throw new Error("oa-events module is not configured");
return context;
}
function safePreview(value: string, max = 300): string {
const clean = String(value || "").replace(/\u001b\[[0-9;]*m/gu, "").trim();
return clean.length > max ? `${clean.slice(0, max)}...` : clean;
}
function hash(value: unknown): string {
return createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 20);
}
function addTag(tags: string[], value: string): void {
const tag = value.trim();
if (tag.length > 0 && !tags.includes(tag)) tags.push(tag);
}
function taskTags(task: QueueTask, queueId: string, extra: string[] = [], attemptIndexOverride: number | null = null): string[] {
const tags = ["service:code-queue", "trace", `task:${task.id}`, `queue:${queueId}`];
const attempt = Number(attemptIndexOverride ?? (task.currentAttempt || task.attempts.length || 0));
if (attempt > 0) addTag(tags, `attempt:${attempt}`);
for (const tag of extra) addTag(tags, tag);
return tags;
}
function postOaEvent(event: OaEventEnvelope): void {
const runtime = ctx();
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), postTimeoutMs);
void fetch(`${runtime.baseUrl}/api/events`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(event),
signal: controller.signal,
}).then(async (response) => {
if (!response.ok) {
runtime.logger("warn", "oa_event_publish_failed", { eventId: event.eventId, type: event.type, status: response.status, body: safePreview(await response.text(), 1000) });
}
}).catch((error) => {
runtime.logger("warn", "oa_event_publish_error", { eventId: event.eventId, type: event.type, error: error instanceof Error ? error.message : String(error) });
}).finally(() => clearTimeout(timer));
}
function normalizeCommandText(text: string): string {
const trimmed = String(text || "").trim();
if (!trimmed.startsWith("{")) return trimmed;
try {
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
const command = parsed.command ?? parsed.cmd ?? parsed.text ?? parsed.summary;
return typeof command === "string" ? command : trimmed;
} catch {
return trimmed;
}
}
function commandKind(command: string): "read" | "edit" | "run" {
if (/\b(apply_patch|git apply|cat >|tee .*<<|sed -i|python3? .*write_text|write|patch|edit|delete|create)\b/iu.test(command)) return "edit";
if (/\b(rg|grep|find|ls|cat|sed -n|tail|head|git status|git diff|ps|read|glob|search|view)\b/iu.test(command)) return "read";
return "run";
}
function openCodeToolRecord(output: LiveOutput): Record<string, unknown> | null {
if (!String(output.method || "").startsWith("opencode/")) return null;
try {
const parsed = JSON.parse(output.text) as unknown;
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
} catch {
return null;
}
}
function recordString(record: Record<string, unknown> | null, keys: string[]): string {
if (record === null) return "";
for (const key of keys) {
const value = record[key];
if (typeof value === "string") return value;
}
return "";
}
export function outputTraceKind(output: LiveOutput): "read" | "edit" | "run" | "error" | "message" | "system" {
if (output.channel === "diff") return "edit";
if (output.channel === "error") return "error";
if (output.channel === "assistant" || output.channel === "user" || output.channel === "reasoning") return "message";
if (output.channel === "tool") {
const record = openCodeToolRecord(output);
const part = record?.part && typeof record.part === "object" && !Array.isArray(record.part) ? record.part as Record<string, unknown> : null;
const state = part?.state && typeof part.state === "object" && !Array.isArray(part.state) ? part.state as Record<string, unknown> : null;
const input = state?.input && typeof state.input === "object" && !Array.isArray(state.input) ? state.input as Record<string, unknown> : null;
const tool = recordString(part, ["tool", "title"]) || recordString(record, ["type", "event", "name"]);
const command = recordString(input, ["command", "cmd", "script", "path", "pattern", "query"]);
return commandKind(`${tool} ${command}`);
}
if (output.channel === "command") return commandKind(normalizeCommandText(output.text));
return "system";
}
function outputTitle(output: LiveOutput, kind: string): string {
const method = String(output.method || "");
if (output.channel === "diff") return "Edited files";
if (output.channel === "error") return "Error";
if (output.channel === "assistant") return "Assistant message";
if (output.channel === "reasoning") return "Reasoning";
if (output.channel === "user") return method === "enqueue" ? "Submitted prompt" : "User prompt";
const command = normalizeCommandText(output.text).split(/\r?\n/u).find((line) => line.trim().length > 0)?.trim() ?? "";
if (command.length > 0) return safePreview(command, 160);
return kind === "read" ? "Read" : kind === "edit" ? "Edit" : kind === "run" ? "Run" : method || output.channel;
}
function outputSummaryLines(output: LiveOutput): string[] {
return String(output.text || "")
.replace(/\u001b\[[0-9;]*m/gu, "")
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean)
.slice(0, 4)
.map((line) => safePreview(line, 220));
}
function outputAttemptIndex(output: LiveOutput): number | null {
const match = String(output.text || "").match(/\battempt\s+(\d+)\s*\/\s*\d+\b/iu);
const value = Number(match?.[1] ?? NaN);
return Number.isInteger(value) && value > 0 ? value : null;
}
function taskOutputAttemptIndex(task: QueueTask, output: LiveOutput, attemptIndexOverride: number | null = null): number | null {
if (attemptIndexOverride !== null && Number.isInteger(attemptIndexOverride) && attemptIndexOverride > 0) return attemptIndexOverride;
const fromOutput = outputAttemptIndex(output);
if (fromOutput !== null) return fromOutput;
const current = Number(task.currentAttempt || 0);
return Number.isInteger(current) && current > 0 ? current : null;
}
function taskStatusPayload(task: QueueTask, queueId: string, reason: string, stepCount: number, outputMaxSeq: number): JsonRecord {
return {
taskId: task.id,
queueId,
reason,
status: task.status,
updatedAt: task.updatedAt,
createdAt: task.createdAt,
startedAt: task.startedAt,
finishedAt: task.finishedAt,
readAt: task.readAt,
providerId: task.providerId,
model: task.model,
currentAttempt: task.currentAttempt,
stepCount,
outputMaxSeq,
};
}
export function publishCodeQueueTaskUpdated(task: QueueTask, queueId: string, reason: string, stepCount: number, outputMaxSeq: number): void {
postOaEvent({
eventId: `code-queue:task-updated:${task.id}:${reason}:${task.updatedAt}:${outputMaxSeq}:${hash({ status: task.status, queueId, readAt: task.readAt })}`,
type: "task-updated",
createdAt: ctx().nowIso(),
sourceKind: "service",
sourceId: "code-queue",
aggregateType: "task",
aggregateId: task.id,
correlationId: task.id,
tags: taskTags(task, queueId, ["task"]),
payload: taskStatusPayload(task, queueId, reason, stepCount, outputMaxSeq),
});
}
export function publishCodeQueueTraceStatsSnapshot(
task: QueueTask,
queueId: string,
reason: string,
stepCount: number,
outputMaxSeq: number,
statsOverride: Partial<{ llmStepCount: number; traceLineCount: number; readCount: number; editCount: number; runCount: number; errorCount: number }> = {},
): void {
postOaEvent({
eventId: `code-queue:trace-stats-snapshot:${task.id}:${task.updatedAt}:${stepCount}:${outputMaxSeq}`,
type: "trace-stats-snapshot",
createdAt: ctx().nowIso(),
sourceKind: "service",
sourceId: "code-queue",
aggregateType: "task",
aggregateId: task.id,
correlationId: task.id,
tags: taskTags(task, queueId, ["stats"]),
payload: {
...taskStatusPayload(task, queueId, reason, stepCount, outputMaxSeq),
scopeId: `task:${task.id}`,
stats: {
stepCount,
llmStepCount: statsOverride.llmStepCount ?? stepCount,
traceLineCount: statsOverride.traceLineCount ?? stepCount,
outputMaxSeq,
readCount: statsOverride.readCount ?? 0,
editCount: statsOverride.editCount ?? 0,
runCount: statsOverride.runCount ?? 0,
errorCount: statsOverride.errorCount ?? (task.lastError ? 1 : 0),
attempts: task.attempts.map((attempt) => ({ index: attempt.index, mode: attempt.mode, startedAt: attempt.startedAt, finishedAt: attempt.finishedAt, terminalStatus: attempt.terminalStatus })),
},
},
});
}
export function publishCodeQueueTraceStep(task: QueueTask, queueId: string, output: LiveOutput, outputMaxSeq: number, attemptIndexOverride: number | null = null): void {
const kind = outputTraceKind(output);
const attemptIndex = taskOutputAttemptIndex(task, output, attemptIndexOverride);
const attemptScopeId = attemptIndex === null ? null : taskAttemptScopeId(task.id, attemptIndex);
postOaEvent({
eventId: `code-queue:trace-step-created:${task.id}:${output.seq}`,
type: "trace-step-created",
createdAt: output.at || ctx().nowIso(),
sourceKind: "service",
sourceId: "code-queue",
aggregateType: "task",
aggregateId: task.id,
correlationId: task.id,
tags: taskTags(task, queueId, attemptScopeId === null ? [] : [attemptScopeId], attemptIndex),
payload: {
taskId: task.id,
queueId,
scopeId: `task:${task.id}`,
attemptIndex,
attemptScopeId,
seq: output.seq,
outputSeq: output.seq,
outputMaxSeq,
kind,
channel: output.channel,
method: output.method ?? "",
itemId: output.itemId ?? "",
title: outputTitle(output, kind),
status: task.status,
summaryLines: outputSummaryLines(output),
rawSeqs: [output.seq],
},
});
}
export function publishCodeQueueQueueUpdated(queueId: string, reason: string): void {
const at = ctx().nowIso();
postOaEvent({
eventId: `code-queue:queue-updated:${queueId || "all"}:${reason}:${at}`,
type: "queue-updated",
createdAt: at,
sourceKind: "service",
sourceId: "code-queue",
aggregateType: "queue",
aggregateId: queueId || "all",
correlationId: `queue:${queueId || "all"}`,
tags: ["service:code-queue", "queue", ...(queueId ? [`queue:${queueId}`] : [])],
payload: { queueId, reason, updatedAt: at },
});
}
export async function readOaTraceStatsForScopeIds(scopeIds: string[]): Promise<Map<string, OaTraceStats>> {
const result = new Map<string, OaTraceStats>();
const uniqueScopeIds = Array.from(new Set(scopeIds.map((id) => String(id || "").trim()).filter(Boolean)));
if (uniqueScopeIds.length === 0) return result;
const runtime = ctx();
const url = new URL(`${runtime.baseUrl}/api/stats/trace`);
url.searchParams.set("scopeIds", uniqueScopeIds.join(","));
url.searchParams.set("limit", String(Math.max(100, uniqueScopeIds.length)));
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 2500);
try {
const response = await fetch(url, { signal: controller.signal });
if (!response.ok) throw new Error(`status=${response.status}`);
const body = await response.json() as Record<string, unknown>;
const stats = Array.isArray(body.stats) ? body.stats : [];
for (const item of stats) {
if (typeof item !== "object" || item === null || Array.isArray(item)) continue;
const record = item as OaTraceStats;
if (typeof record.scopeId !== "string") continue;
result.set(record.scopeId, record);
}
} catch (error) {
runtime.logger("warn", "oa_trace_stats_read_failed", { scopeCount: uniqueScopeIds.length, error: error instanceof Error ? error.message : String(error) });
} finally {
clearTimeout(timer);
}
return result;
}
export async function readOaTraceStatsForTasks(taskIds: string[]): Promise<Map<string, OaTraceStats>> {
const uniqueTaskIds = Array.from(new Set(taskIds.map((id) => String(id || "").trim()).filter(Boolean)));
return readOaTraceStatsForScopeIds(uniqueTaskIds.map((id) => taskScopeId(id)));
}
export async function readOaTraceStatsForTask(taskId: string): Promise<OaTraceStats | null> {
const map = await readOaTraceStatsForTasks([taskId]);
return map.get(taskScopeId(taskId)) ?? null;
}
export async function readOaTraceStatsForTaskAttempts(taskId: string, attemptIndexes: number[]): Promise<Map<string, OaTraceStats>> {
const uniqueAttempts = Array.from(new Set(attemptIndexes.filter((index) => Number.isInteger(index) && index > 0).map((index) => Math.floor(index))));
return readOaTraceStatsForScopeIds([taskScopeId(taskId), ...uniqueAttempts.map((index) => taskAttemptScopeId(taskId, index))]);
}
function statNumber(stats: OaTraceStats | null | undefined, key: string): number | null {
const value = Number(stats?.[key]);
return Number.isFinite(value) && value >= 0 ? Math.floor(value) : null;
}
export function applyOaTraceStatsToTaskJson(value: JsonValue, stats: OaTraceStats | null | undefined): JsonValue {
if (typeof value !== "object" || value === null || Array.isArray(value)) return value;
if (stats === null || stats === undefined) {
return { ...(value as JsonRecord), traceStats: null, statsSource: "unavailable", stepCount: null, llmStepCount: null } as unknown as JsonValue;
}
const stepCount = statNumber(stats, "stepCount");
const outputMaxSeq = statNumber(stats, "outputMaxSeq");
const patch: JsonRecord = {
traceStats: stats,
statsSource: "oa-event-flow",
};
if (stepCount !== null) {
patch.stepCount = stepCount;
patch.llmStepCount = stepCount;
}
if (outputMaxSeq !== null) patch.outputMaxSeq = outputMaxSeq;
return { ...(value as JsonRecord), ...patch };
}
export function taskScopeId(taskId: string): string {
return `task:${taskId}`;
}
export function taskAttemptScopeId(taskId: string, attemptIndex: number): string {
return `${taskScopeId(taskId)}:attempt:${Math.floor(attemptIndex)}`;
}
@@ -477,10 +477,10 @@ if command -v apt-get >/dev/null 2>&1; then
fi
fi
if ! command -v codex >/dev/null 2>&1; then
npm install -g @openai/codex@0.128.0
npm install -g --no-audit --no-fund --prefer-offline @openai/codex@0.128.0
fi
if ! command -v opencode >/dev/null 2>&1; then
npm install -g ${opencodeNpmPackage}
npm install -g --no-audit --no-fund --prefer-offline ${opencodeNpmPackage}
fi
mkdir -p "$WORKDIR" "$CODEX_HOME_DIR" "$OPENCODE_XDG_DIR/data" "$OPENCODE_XDG_DIR/config" "$OPENCODE_XDG_DIR/cache" "$OPENCODE_XDG_DIR/state"
echo "code_agent_runtime_ready codex=$(command -v codex) opencode=$(command -v opencode) cwd=$WORKDIR home=$CODEX_HOME_DIR opencodeXdg=$OPENCODE_XDG_DIR"
@@ -5,7 +5,8 @@ import { codeAgentPortForModel, codeAgentPortInfo, codeModelPorts as codeModelPo
import { claudeQqNotificationOutboxStats, notificationTargetConfigured, notificationTargetLabel } from "./notifications";
import { executionProviderOptions } from "./provider-runtime";
import { taskFullOutput } from "./task-output";
import { buildCompactTaskTranscript, buildTaskTranscript, cachedPreviewTranscript, fullTranscript, prefixPreview, safePreview, statsDaysFromUrl, taskForCompactMetaResponse, taskForMetaResponse, taskListStepCount, taskStatisticsSummary, taskTiming, timestampMs } from "./task-view";
import { applyOaTraceStatsToTaskJson, taskScopeId, type OaTraceStats } from "./oa-events";
import { buildCompactTaskTranscript, buildTaskTranscript, cachedPreviewTranscript, fullTranscript, prefixPreview, safePreview, statsDaysFromUrl, taskForCompactMetaResponse, taskForMetaResponse, taskStatisticsSummary, taskTiming, timestampMs } from "./task-view";
import { userPromptForDisplay } from "./prompts";
import type { ActiveRun, ActiveRunSlotWaiter } from "./code-agent/common";
import type { JsonValue, QueueRecord, QueuedStatusReason, QueueTask, RuntimeConfig, TaskStatus, TranscriptLine } from "./types";
@@ -43,6 +44,7 @@ export interface QueueApiContext {
safeQueueName: (value: unknown, queueId: string) => string;
sql: postgres.Sql;
taskQueueEnteredAt: (task: QueueTask) => string;
traceStatsForTasks: (taskIds: string[]) => Promise<Map<string, OaTraceStats>>;
tasks: () => QueueTask[];
truthyParam: (url: URL, name: string) => boolean;
}
@@ -140,10 +142,23 @@ function outputChunkResponse(task: QueueTask, url: URL): Response {
});
}
function taskIdsForStats(tasks: QueueTask[], selectedTask?: QueueTask | null): string[] {
const ids = tasks.map((task) => task.id);
if (selectedTask !== null && selectedTask !== undefined) ids.push(selectedTask.id);
return Array.from(new Set(ids.filter(Boolean)));
}
function statsForTask(stats: Map<string, OaTraceStats>, task: QueueTask): OaTraceStats | null {
return stats.get(taskScopeId(task.id)) ?? null;
}
function applyStats(value: JsonValue, stats: Map<string, OaTraceStats>, task: QueueTask): JsonValue {
return applyOaTraceStatsToTaskJson(value, statsForTask(stats, task));
}
function taskForListResponse(task: QueueTask, lite = false, queueTasks?: QueueTask[]): JsonValue {
const timing = taskTiming(task);
const displayPrompt = task.basePrompt || userPromptForDisplay(task.prompt);
const stepCount = taskListStepCount(task);
if (lite) {
return {
id: task.id,
@@ -157,8 +172,10 @@ function taskForListResponse(task: QueueTask, lite = false, queueTasks?: QueueTa
displayPromptChars: displayPrompt.length,
promptEditable: ctx().queuedTaskPromptEditable(task),
finalResponseChars: task.finalResponse.length,
stepCount,
llmStepCount: stepCount,
stepCount: null,
llmStepCount: null,
traceStats: null,
statsSource: "unavailable",
summaryOnly: true,
referenceTaskIds: task.referenceTaskIds,
referenceInjectionSummary: task.referenceInjection === null ? null : {
@@ -215,8 +232,10 @@ function taskForListResponse(task: QueueTask, lite = false, queueTasks?: QueueTa
displayPromptChars: displayPrompt.length,
promptEditable: ctx().queuedTaskPromptEditable(task),
finalResponseChars: task.finalResponse.length,
stepCount,
llmStepCount: stepCount,
stepCount: null,
llmStepCount: null,
traceStats: null,
statsSource: "unavailable",
summaryOnly: true,
referenceTaskIds: task.referenceTaskIds,
referenceInjection: task.referenceInjection,
@@ -1035,11 +1054,16 @@ async function databaseTasksOverviewResponse(url: URL): Promise<Response | null>
} as unknown as JsonValue;
}
const queueContextTasks = [...ctx().tasks(), ...rowsSource];
const traceStats = await ctx().traceStatsForTasks(taskIdsForStats(rowsSource, selectedTask));
if (selectedTask !== null && selected !== null && typeof selected === "object" && !Array.isArray(selected)) {
const selectedRecord = selected as Record<string, JsonValue>;
selectedRecord.task = applyStats(selectedRecord.task, traceStats, selectedTask);
}
return ctx().compactJsonResponse({
ok: true,
queue,
statistics,
tasks: rowsSource.map((task) => taskForListResponse(task, true, queueContextTasks)),
tasks: rowsSource.map((task) => applyStats(taskForListResponse(task, true, queueContextTasks), traceStats, task)),
selected,
pagination: {
limit,
@@ -1098,11 +1122,16 @@ async function tasksOverviewResponse(url: URL): Promise<Response> {
maxSeq,
} as unknown as JsonValue;
}
const traceStats = await ctx().traceStatsForTasks(taskIdsForStats(rowsSource, selectedTask));
if (selectedTask !== null && selected !== null && typeof selected === "object" && !Array.isArray(selected)) {
const selectedRecord = selected as Record<string, JsonValue>;
selectedRecord.task = applyStats(selectedRecord.task, traceStats, selectedTask);
}
return ctx().compactJsonResponse({
ok: true,
queue,
statistics: taskStatisticsSummary(filteredTasks, statsDaysFromUrl(url)),
tasks: rowsSource.map((task) => taskForListResponse(task, true, allTasks)),
tasks: rowsSource.map((task) => applyStats(taskForListResponse(task, true, allTasks), traceStats, task)),
selected,
pagination: {
limit,
@@ -1178,14 +1178,15 @@ function attemptForResponse(attempt: AttemptSummary, full = false): JsonValue {
function taskForResponse(task: QueueTask, full = false, includeRaw = full): JsonValue {
const displayPrompt = task.basePrompt || userPromptForDisplay(task.prompt);
const stepCount = taskLlmStepCount(task);
return {
...task,
agentPort: codeAgentPortForModel(task.model),
agentPortInfo: codeAgentPortInfo(codeAgentPortForModel(task.model)),
judgeFailRetryLimit,
stepCount,
llmStepCount: stepCount,
stepCount: null,
llmStepCount: null,
traceStats: null,
statsSource: "unavailable",
prompt: full ? task.prompt : safePreview(task.prompt, 2000),
basePrompt: full ? task.basePrompt : safePreview(task.basePrompt, 2000),
displayPrompt: full ? displayPrompt : safePreview(displayPrompt, 2000),
@@ -1226,8 +1227,8 @@ function taskStoredStepCount(task: QueueTask): number | null {
return nonNegativeInteger(task.stepCount) ?? nonNegativeInteger(task.llmStepCount);
}
// Used by overview/list responses and SSE patches. It must stay archive-free;
// trace/detail endpoints compute exact counts from the full transcript on demand.
// Used only as a producer-side seed for OA stats snapshots; frontend/API
// display authority is the oa_trace_stats projection applied at route edges.
function taskListStepCount(task: QueueTask): number {
return taskStoredStepCount(task) ?? retainedTaskStepCount(task);
}
@@ -1245,14 +1246,9 @@ function taskOutputMaxSeq(task: QueueTask): number {
return values.length > 0 ? Math.max(...values) : 0;
}
function taskLlmStepCount(task: QueueTask): number {
return taskStoredStepCount(task) ?? taskStepCountFromTranscript(task, cachedPreviewTranscript(task));
}
function taskForMetaResponse(task: QueueTask): JsonValue {
const lastOutputSeq = taskOutputMaxSeq(task);
const displayPrompt = task.basePrompt || userPromptForDisplay(task.prompt);
const stepCount = taskListStepCount(task);
return {
id: task.id,
queueId: ctx().queueIdOf(task),
@@ -1265,8 +1261,10 @@ function taskForMetaResponse(task: QueueTask): JsonValue {
displayPromptChars: displayPrompt.length,
promptEditable: ctx().queuedTaskPromptEditable(task),
finalResponseChars: task.finalResponse.length,
stepCount,
llmStepCount: stepCount,
stepCount: null,
llmStepCount: null,
traceStats: null,
statsSource: "unavailable",
summaryOnly: false,
referenceTaskIds: task.referenceTaskIds,
referenceInjection: task.referenceInjection,
@@ -1313,7 +1311,6 @@ function taskForMetaResponse(task: QueueTask): JsonValue {
function taskForCompactMetaResponse(task: QueueTask): JsonValue {
const displayPrompt = task.basePrompt || userPromptForDisplay(task.prompt);
const lastOutputSeq = taskOutputMaxSeq(task);
const stepCount = taskListStepCount(task);
return {
id: task.id,
queueId: ctx().queueIdOf(task),
@@ -1326,8 +1323,10 @@ function taskForCompactMetaResponse(task: QueueTask): JsonValue {
displayPromptChars: displayPrompt.length,
promptEditable: ctx().queuedTaskPromptEditable(task),
finalResponseChars: task.finalResponse.length,
stepCount,
llmStepCount: stepCount,
stepCount: null,
llmStepCount: null,
traceStats: null,
statsSource: "unavailable",
summaryOnly: true,
referenceTaskIds: task.referenceTaskIds,
referenceInjection: task.referenceInjection === null ? null : {
@@ -1517,20 +1516,10 @@ function toolStepCountsFromTranscript(lines: TranscriptLine[]): { toolCallCount:
};
}
function llmStepCountFromTranscript(lines: TranscriptLine[], _fallback = 0): number {
return toolStepCountsFromTranscript(lines).toolCallCount;
}
function realAttemptWindows(task: QueueTask, transcript: TranscriptLine[]): TraceAttemptWindow[] {
return traceAttemptWindows(task, transcript).filter((window) => window.synthetic !== true && window.index > 0);
}
function taskStepCountFromTranscript(task: QueueTask, transcript: TranscriptLine[]): number {
const windows = realAttemptWindows(task, transcript);
if (windows.length === 0) return llmStepCountFromTranscript(transcript);
return windows.reduce((sum, window) => sum + llmStepCountFromTranscript(executionLinesForAttempt(window.lines)), 0);
}
function taskExecutionTranscript(task: QueueTask, transcript: TranscriptLine[]): TranscriptLine[] {
const windows = realAttemptWindows(task, transcript);
if (windows.length === 0) return transcript;
@@ -1543,7 +1532,6 @@ function executionSummaryFromTranscript(
timing: Record<string, JsonValue>,
outputCount = taskFullOutput(task).length,
retainedOutputCount = task.output.length,
fallbackStepCount = 0,
): JsonValue {
const toolLines = transcript.filter(isToolActionLine);
const editedFiles = uniqueStrings(toolLines.flatMap((line) => line.kind === "edited" ? parseEditedFilesFromText(`${line.title}\n${line.bodyPreview ?? ""}`) : []), 40);
@@ -1571,7 +1559,7 @@ function executionSummaryFromTranscript(
}
function taskExecutionSummary(task: QueueTask, transcript = cachedPreviewTranscript(task)): JsonValue {
return executionSummaryFromTranscript(task, taskExecutionTranscript(task, transcript), taskTiming(task) as Record<string, JsonValue>, undefined, undefined, task.currentAttempt || task.attempts.length);
return executionSummaryFromTranscript(task, taskExecutionTranscript(task, transcript), taskTiming(task) as Record<string, JsonValue>);
}
function parseAttemptIndex(text: string): number | null {
@@ -1669,6 +1657,8 @@ interface TraceAttemptWindow {
label?: string;
}
type TraceStatsLookup = Map<string, JsonValue> | Record<string, JsonValue> | null | undefined;
function transcriptLineSeq(line: TranscriptLine | undefined): number | null {
const value = Number(line?.seq ?? NaN);
return Number.isFinite(value) ? value : null;
@@ -1806,7 +1796,7 @@ function executionLinesForAttempt(lines: TranscriptLine[]): TranscriptLine[] {
return lines.filter((line) => line.title !== "Submitted prompt" && line.title !== "Attempt started" && line.title !== "Judge result");
}
function taskTraceAttemptSummaries(task: QueueTask, transcript: TranscriptLine[]): JsonValue[] {
function taskTraceAttemptSummaries(task: QueueTask, transcript: TranscriptLine[], oaTraceStatsByScope: TraceStatsLookup = null): JsonValue[] {
const windows = traceAttemptWindows(task, transcript);
return windows.map((window) => {
const attempt = window.attempt;
@@ -1817,7 +1807,12 @@ function taskTraceAttemptSummaries(task: QueueTask, transcript: TranscriptLine[]
const finalResponse = synthetic ? "" : String(attempt?.finalResponse ?? attempt?.finalResponsePreview ?? (window.index === task.attempts.length ? task.finalResponse : ""));
const finalResponseChars = Number(attempt?.finalResponseChars ?? finalResponse.length);
const executionLines = executionLinesForAttempt(window.lines);
const errorCount = executionLines.filter((line) => line.kind === "error").length;
const attemptStats = synthetic || window.index <= 0 ? null : traceStatsForScope(oaTraceStatsByScope, taskAttemptScopeId(task.id, window.index));
const stepCount = traceStatsNumber(attemptStats, "stepCount");
const readCount = traceStatsNumber(attemptStats, "readCount");
const editCount = traceStatsNumber(attemptStats, "editCount");
const runCount = traceStatsNumber(attemptStats, "runCount");
const errorCount = traceStatsNumber(attemptStats, "errorCount");
const feedbackPrompt = synthetic ? null : attemptFeedbackPromptRecord(task, window.index, attempt, judge);
const inputPrompt = promptSnapshot(String(attempt?.inputPrompt ?? ""), 1200);
return {
@@ -1854,14 +1849,23 @@ function taskTraceAttemptSummaries(task: QueueTask, transcript: TranscriptLine[]
feedbackPromptSource: feedbackPrompt?.source ?? null,
feedbackPromptForAttempt: feedbackPrompt?.forAttempt ?? null,
feedbackPromptTruncated: feedbackPrompt?.truncated ?? false,
attemptScopeId: synthetic || window.index <= 0 ? null : taskAttemptScopeId(task.id, window.index),
stepCount,
readCount,
editCount,
runCount,
errorCount,
execution: executionSummaryFromTranscript(
task,
executionLines,
attemptTimingSummary(attempt, executionLines.length > 0 ? executionLines : window.lines),
window.lines.length,
window.lines.length,
synthetic || window.index <= 0 ? 0 : 1,
statsSource: attemptStats === null ? "unavailable" : "oa-event-flow",
traceStats: attemptStats,
execution: executionSummaryWithOaStats(
executionSummaryFromTranscript(
task,
executionLines,
attemptTimingSummary(attempt, executionLines.length > 0 ? executionLines : window.lines),
window.lines.length,
window.lines.length,
),
attemptStats,
),
};
}) as unknown as JsonValue[];
@@ -1905,15 +1909,82 @@ function taskTracePromptSummary(task: QueueTask): JsonValue {
} as unknown as JsonValue;
}
function taskTraceSummaryResponse(task: QueueTask): JsonValue {
function traceStatsRecord(value: JsonValue | null | undefined): Record<string, JsonValue> | null {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, JsonValue> : null;
}
function traceStatsNumber(stats: Record<string, JsonValue> | null, key: string): number | null {
const value = Number(stats?.[key]);
return Number.isFinite(value) && value >= 0 ? Math.floor(value) : null;
}
function taskAttemptScopeId(taskId: string, attemptIndex: number): string {
return `task:${taskId}:attempt:${Math.floor(attemptIndex)}`;
}
function traceStatsForScope(lookup: TraceStatsLookup, scopeId: string): Record<string, JsonValue> | null {
if (lookup === null || lookup === undefined || scopeId.length === 0) return null;
const value = lookup instanceof Map ? lookup.get(scopeId) : lookup[scopeId];
return traceStatsRecord(value);
}
function executionSummaryWithOaStats(execution: JsonValue, stats: Record<string, JsonValue> | null): JsonValue {
if (typeof execution !== "object" || execution === null || Array.isArray(execution)) return execution;
const record = execution as Record<string, JsonValue>;
if (stats === null) {
const {
stepCount: _stepCount,
llmStepCount: _llmStepCount,
toolCallCount: _toolCallCount,
readCount: _readCount,
editCount: _editCount,
runCount: _runCount,
errorCount: _errorCount,
traceLineCount: _traceLineCount,
outputMaxSeq: _outputMaxSeq,
transcriptMaxSeq: _transcriptMaxSeq,
...rest
} = record;
return {
...rest,
statsSource: "unavailable",
traceStats: null,
statsUnavailable: true,
} as unknown as JsonValue;
}
const stepCount = traceStatsNumber(stats, "stepCount");
const llmStepCount = traceStatsNumber(stats, "llmStepCount") ?? stepCount;
const readCount = traceStatsNumber(stats, "readCount");
const editCount = traceStatsNumber(stats, "editCount");
const runCount = traceStatsNumber(stats, "runCount");
const errorCount = traceStatsNumber(stats, "errorCount");
const traceLineCount = traceStatsNumber(stats, "traceLineCount");
const outputMaxSeq = traceStatsNumber(stats, "outputMaxSeq");
const toolCallCount = readCount === null && editCount === null && runCount === null ? null : (readCount ?? 0) + (editCount ?? 0) + (runCount ?? 0);
return {
...record,
...(stepCount === null ? {} : { stepCount }),
...(llmStepCount === null ? {} : { llmStepCount }),
...(toolCallCount === null ? {} : { toolCallCount }),
...(readCount === null ? {} : { readCount }),
...(editCount === null ? {} : { editCount }),
...(runCount === null ? {} : { runCount }),
...(errorCount === null ? {} : { errorCount }),
...(traceLineCount === null ? {} : { traceLineCount }),
...(outputMaxSeq === null ? {} : { outputMaxSeq, transcriptMaxSeq: outputMaxSeq }),
statsSource: "oa-event-flow",
traceStats: stats,
} as unknown as JsonValue;
}
function taskTraceSummaryResponse(task: QueueTask, oaTraceStats: JsonValue | null = null, oaTraceStatsByScope: TraceStatsLookup = null): JsonValue {
const transcript = cachedPreviewTranscript(task);
const attempts = taskTraceAttemptSummaries(task, transcript);
const stepCount = taskStepCountFromTranscript(task, transcript);
const errorCount = attempts.reduce((sum: number, attempt: JsonValue) => {
if (typeof attempt !== "object" || attempt === null || Array.isArray(attempt)) return sum;
const value = Number((attempt as { errorCount?: unknown }).errorCount || 0);
return sum + (Number.isFinite(value) && value > 0 ? value : 0);
}, 0);
const attempts = taskTraceAttemptSummaries(task, transcript, oaTraceStatsByScope);
const stats = traceStatsRecord(oaTraceStats);
const stepCount = traceStatsNumber(stats, "stepCount");
const llmStepCount = traceStatsNumber(stats, "llmStepCount") ?? stepCount;
const errorCount = traceStatsNumber(stats, "errorCount");
const execution = executionSummaryWithOaStats(taskExecutionSummary(task, transcript), stats);
return {
id: task.id,
queueId: ctx().queueIdOf(task),
@@ -1931,10 +2002,12 @@ function taskTraceSummaryResponse(task: QueueTask): JsonValue {
currentAttempt: task.currentAttempt,
maxAttempts: task.maxAttempts,
stepCount,
llmStepCount: stepCount,
llmStepCount,
promptEditable: ctx().queuedTaskPromptEditable(task),
prompt: taskTracePromptSummary(task),
execution: taskExecutionSummary(task, transcript),
execution,
traceStats: stats,
statsSource: stats === null ? "unavailable" : "oa-event-flow",
finalResponse: task.finalResponse,
finalResponseChars: task.finalResponse.length,
lastJudge: task.lastJudge,
@@ -2132,7 +2205,6 @@ export {
statsDaysFromUrl,
taskExecutionSummary,
taskListStepCount,
taskLlmStepCount,
taskOutputMaxSeq,
taskForCompactMetaResponse,
taskForMetaResponse,
@@ -63,6 +63,7 @@ export interface RuntimeConfig {
turnNoActivityTimeoutMs: number;
databaseUrl: string;
databaseFlushIntervalMs: number;
oaEventFlowBaseUrl: string;
notifyClaudeQqEnabled: boolean;
notifyClaudeQqBaseUrl: string;
notifyClaudeQqTargetType: "private" | "group";
@@ -0,0 +1,11 @@
FROM oven/bun:1-alpine
WORKDIR /app/src/components/microservices/oa-event-flow
COPY src/components/microservices/oa-event-flow/package.json ./package.json
RUN bun install --production
COPY src/components/microservices/oa-event-flow/tsconfig.json ./tsconfig.json
COPY src/components/shared /app/src/components/shared
COPY src/components/microservices/oa-event-flow/src ./src
EXPOSE 4255
CMD ["bun", "run", "src/index.ts"]
@@ -0,0 +1,15 @@
{
"lockfileVersion": 1,
"configVersion": 1,
"workspaces": {
"": {
"name": "@unidesk/oa-event-flow",
"dependencies": {
"postgres": "latest",
},
},
},
"packages": {
"postgres": ["postgres@3.4.9", "", {}, "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw=="],
}
}
@@ -0,0 +1,12 @@
{
"name": "@unidesk/oa-event-flow",
"private": true,
"type": "module",
"scripts": {
"start": "bun run src/index.ts",
"check": "tsc -p tsconfig.json --noEmit"
},
"dependencies": {
"postgres": "latest"
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,18 @@
{
"compilerOptions": {
"composite": true,
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "Bundler",
"types": ["bun", "node"],
"strict": true,
"noImplicitReturns": true,
"noFallthroughCasesInSwitch": true,
"declaration": true,
"emitDeclarationOnly": true,
"outDir": "dist",
"skipLibCheck": true
},
"include": ["src/**/*.ts"],
"references": [{ "path": "../../shared" }]
}