Codex
267 lines
17 KiB
TypeScript
267 lines
17 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { writeFileSync, unlinkSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { codexSteerTaskForTest } from "./src/code-queue";
|
|
|
|
type JsonRecord = Record<string, unknown>;
|
|
|
|
function assertCondition(condition: unknown, message: string, detail: JsonRecord = {}): void {
|
|
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
|
|
}
|
|
|
|
function runCli(args: string[], stdin?: string): { status: number | null; stdout: string; stderr: string; json: JsonRecord | null } {
|
|
const result = spawnSync("bun", ["scripts/cli.ts", ...args], {
|
|
cwd: process.cwd(),
|
|
input: stdin,
|
|
encoding: "utf8",
|
|
});
|
|
const stdout = String(result.stdout || "");
|
|
let json: JsonRecord | null = null;
|
|
try {
|
|
json = JSON.parse(stdout) as JsonRecord;
|
|
} catch {
|
|
json = null;
|
|
}
|
|
return {
|
|
status: result.status,
|
|
stdout,
|
|
stderr: String(result.stderr || ""),
|
|
json,
|
|
};
|
|
}
|
|
|
|
function nestedRecord(value: unknown, path: string[]): JsonRecord {
|
|
let current: unknown = value;
|
|
for (const key of path) {
|
|
assertCondition(current !== null && typeof current === "object" && !Array.isArray(current), "expected object while traversing JSON", { path, key, current });
|
|
current = (current as JsonRecord)[key];
|
|
}
|
|
assertCondition(current !== null && typeof current === "object" && !Array.isArray(current), "expected nested object", { path, current });
|
|
return current as JsonRecord;
|
|
}
|
|
|
|
function stringArray(value: unknown): string[] {
|
|
return Array.isArray(value) ? value.map((item) => String(item)) : [];
|
|
}
|
|
|
|
function assertDryRunPrompt(response: JsonRecord, expectedText: string): void {
|
|
assertCondition(response.ok === true, "CLI dry-run should succeed", response);
|
|
const data = nestedRecord(response.data, []);
|
|
assertCondition(data.dryRun === true, "dry-run response should expose dryRun=true", data);
|
|
const request = nestedRecord(response.data, ["request"]);
|
|
assertCondition(request.method === "POST", "dry-run should expose request method", request);
|
|
assertCondition(request.path === "/api/tasks/codex_test_task/steer", "dry-run should expose request path", request);
|
|
assertCondition(request.stableProxyPath === "/api/microservices/code-queue/proxy/api/tasks/codex_test_task/steer", "dry-run should expose stable proxy path", request);
|
|
const prompt = nestedRecord(response.data, ["request", "body", "prompt"]);
|
|
assertCondition(prompt.text === expectedText, "dry-run prompt text mismatch", prompt);
|
|
assertCondition(prompt.chars === expectedText.length, "dry-run prompt char count mismatch", prompt);
|
|
assertCondition(prompt.truncated === false, "dry-run prompt must not truncate", prompt);
|
|
const bodySummary = nestedRecord(response.data, ["request", "bodySummary"]);
|
|
assertCondition(bodySummary.promptChars === expectedText.length, "dry-run should expose body prompt char count", bodySummary);
|
|
const commands = nestedRecord(response.data, ["commands"]);
|
|
assertCondition(String(commands.rawProxy || "").includes("microservice proxy code-queue /api/tasks/codex_test_task/steer --method POST"), "dry-run should expose raw proxy equivalent", commands);
|
|
}
|
|
|
|
function assertReason(result: unknown, reason: string, status: number | null): void {
|
|
const data = nestedRecord({ data: result }, ["data"]);
|
|
assertCondition(data.ok === false, "classified steer failure should be ok=false", data);
|
|
const diagnostics = nestedRecord(data, ["diagnostics"]);
|
|
assertCondition(diagnostics.reason === reason, "unexpected steer failure reason", diagnostics);
|
|
assertCondition(diagnostics.status === status, "unexpected steer failure status", diagnostics);
|
|
assertCondition(typeof diagnostics.retryable === "boolean", "diagnostics should expose retryable boolean", diagnostics);
|
|
assertCondition(Array.isArray(diagnostics.recommendedCrossChecks), "diagnostics should expose cross-check commands", diagnostics);
|
|
}
|
|
|
|
export function runCodeQueueCliSteerContract(): JsonRecord {
|
|
const positional = runCli(["codex", "steer", "codex_test_task", "correct the running task", "--dry-run"]);
|
|
assertDryRunPrompt(positional.json ?? {}, "correct the running task");
|
|
assertCondition(String(positional.json?.command || "").includes("<prompt:redacted>"), "outer command should redact positional steer prompt", positional.json ?? {});
|
|
assertCondition(!String(positional.json?.command || "").includes("correct the running task"), "outer command must not echo positional steer prompt", positional.json ?? {});
|
|
|
|
const stdin = runCli(["codex", "steer", "codex_test_task", "--prompt-stdin", "--dry-run"], "stdin steer prompt\n");
|
|
assertDryRunPrompt(stdin.json ?? {}, "stdin steer prompt\n");
|
|
|
|
const promptFile = join(tmpdir(), `unidesk-code-queue-steer-${process.pid}.txt`);
|
|
writeFileSync(promptFile, "file steer prompt", "utf8");
|
|
try {
|
|
const fromFile = runCli(["codex", "steer", "codex_test_task", "--prompt-file", promptFile, "--dry-run"]);
|
|
assertDryRunPrompt(fromFile.json ?? {}, "file steer prompt");
|
|
} finally {
|
|
unlinkSync(promptFile);
|
|
}
|
|
|
|
const duplicateSource = runCli(["codex", "steer", "codex_test_task", "positional", "--prompt-stdin", "--dry-run"], "stdin\n");
|
|
assertCondition(duplicateSource.status !== 0, "duplicate prompt source should fail", duplicateSource.json ?? { stdout: duplicateSource.stdout });
|
|
const duplicateMessage = String(nestedRecord(duplicateSource.json, ["error"]).message || "");
|
|
assertCondition(duplicateMessage.includes("exactly one prompt source"), "duplicate prompt source error should be explicit", { duplicateMessage });
|
|
|
|
const unknownOption = runCli(["codex", "steer", "codex_test_task", "--queue", "default", "prompt", "--dry-run"]);
|
|
assertCondition(unknownOption.status !== 0, "unknown steer option should fail", unknownOption.json ?? { stdout: unknownOption.stdout });
|
|
const unknownMessage = String(nestedRecord(unknownOption.json, ["error"]).message || "");
|
|
assertCondition(unknownMessage.includes("unsupported codex steer option: --queue"), "unknown option error should name option", { unknownMessage });
|
|
|
|
const help = runCli(["codex", "help"]);
|
|
assertCondition(help.status === 0 && help.json?.ok === true, "codex help should succeed", help.json ?? { stdout: help.stdout });
|
|
const usage = stringArray(nestedRecord(help.json?.data, []).usage);
|
|
assertCondition(usage.some((line) => line.includes("codex steer <taskId>")), "codex help should list steer", { usage });
|
|
|
|
let dryRunFetchCount = 0;
|
|
const dryRunDirect = codexSteerTaskForTest("direct_task", ["do not send", "--dry-run"], () => {
|
|
dryRunFetchCount += 1;
|
|
return { ok: true, status: 200, body: { ok: true } };
|
|
});
|
|
assertCondition(dryRunFetchCount === 0, "dry-run must not call stable proxy helper", { dryRunFetchCount, dryRunDirect });
|
|
|
|
const longPrompt = `${"x".repeat(480)}-tail-secret-marker`;
|
|
const longDryRun = codexSteerTaskForTest("direct_task", [longPrompt, "--dry-run"], () => {
|
|
throw new Error("dry-run should not fetch");
|
|
}) as JsonRecord;
|
|
const longPreview = nestedRecord(longDryRun, ["request", "body", "prompt"]);
|
|
assertCondition(longPreview.truncated === true, "long dry-run prompt should be truncated", longPreview);
|
|
assertCondition(!String(longPreview.text || "").includes("tail-secret-marker"), "long dry-run must not leak prompt tail", longPreview);
|
|
|
|
let fetchPath = "";
|
|
let fetchMethod = "";
|
|
let fetchPrompt = "";
|
|
const success = codexSteerTaskForTest("direct_task", ["send this"], (path, init) => {
|
|
fetchPath = path;
|
|
fetchMethod = String(init?.method || "");
|
|
fetchPrompt = String((init?.body as JsonRecord | undefined)?.prompt || "");
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
body: {
|
|
ok: true,
|
|
task: { id: "direct_task", status: "running", prompt: "p" },
|
|
queue: { activeTaskIds: ["direct_task"] },
|
|
},
|
|
};
|
|
}) as JsonRecord;
|
|
assertCondition(fetchPath === "/api/microservices/code-queue/proxy/api/tasks/direct_task/steer", "non-dry-run should use stable proxy path", { fetchPath });
|
|
assertCondition(fetchMethod === "POST", "non-dry-run should POST", { fetchMethod });
|
|
assertCondition(fetchPrompt === "send this", "non-dry-run should send raw prompt in body", { fetchPrompt });
|
|
assertCondition(nestedRecord(success, ["steer"]).accepted === true, "successful steer should report accepted=true", success);
|
|
const successJson = JSON.stringify(success);
|
|
assertCondition(nestedRecord(success, ["steer"]).promptOmitted === true, "successful steer should mark prompt omitted", success);
|
|
assertCondition(!successJson.includes("send this"), "successful steer must not echo prompt text", success);
|
|
assertCondition(!successJson.includes("promptPreview"), "successful steer must not include promptPreview", success);
|
|
|
|
assertReason(codexSteerTaskForTest("direct_task", ["p", "--no-retry"], () => ({ ok: false, exitCode: 1, stderrTail: "Cannot connect to the Docker daemon" })), "backend-core-unreachable", null);
|
|
assertReason(codexSteerTaskForTest("direct_task", ["p", "--no-retry"], () => ({ ok: false, status: 404, body: { ok: false, error: "microservice not found: code-queue" } })), "code-queue-microservice-unregistered", 404);
|
|
assertReason(codexSteerTaskForTest("direct_task", ["p", "--no-retry"], () => ({ ok: false, status: 401, body: { ok: false, error: "unauthorized" } })), "proxy-unauthorized", 401);
|
|
assertReason(codexSteerTaskForTest("direct_task", ["p", "--no-retry"], () => ({ ok: false, status: 404, body: { ok: false, error: "proxy route not found", path: "/api/microservices/code-queue/proxy/api/tasks/direct_task/steer" } })), "proxy-404", 404);
|
|
assertReason(codexSteerTaskForTest("direct_task", ["p", "--no-retry"], () => ({ ok: false, status: 404, body: { ok: false, error: "task not found" } })), "steer-endpoint-404", 404);
|
|
assertReason(codexSteerTaskForTest("direct_task", ["p", "--no-retry"], () => ({ ok: false, status: 409, body: { ok: false, error: "task does not have an active steerable turn" } })), "upstream-runtime-rejected", 409);
|
|
assertReason(codexSteerTaskForTest("direct_task", ["p", "--no-retry"], () => ({ ok: false, status: 504, body: { ok: false, error: "provider HTTP tunnel timed out or disconnected", stage: "http-tunnel-wait" } })), "stable-proxy-failed", 504);
|
|
|
|
const abortedTunnelBody = {
|
|
ok: false,
|
|
error: "provider HTTP tunnel failed",
|
|
stage: "provider-gateway-http-fetch",
|
|
providerId: "D601",
|
|
serviceId: "code-queue",
|
|
providerError: "The operation was aborted",
|
|
retryable: false,
|
|
attempts: [{ attempt: 1, ok: false, durationMs: 30003, timeoutMs: 30000, result: { ok: false, error: "The operation was aborted" } }],
|
|
};
|
|
let retryCalls = 0;
|
|
const retryThenSuccess = codexSteerTaskForTest("direct_task", ["transient correction", "--retry-delay-ms", "0"], () => {
|
|
retryCalls += 1;
|
|
if (retryCalls === 1) return { ok: false, status: 502, body: abortedTunnelBody };
|
|
return {
|
|
ok: true,
|
|
status: 200,
|
|
body: {
|
|
ok: true,
|
|
task: { id: "direct_task", status: "running", prompt: "hidden" },
|
|
queue: { activeTaskIds: ["direct_task"] },
|
|
},
|
|
};
|
|
}) as JsonRecord;
|
|
assertCondition(retryCalls === 2, "retryable 502 tunnel abort should be retried once by default", { retryCalls, retryThenSuccess });
|
|
assertCondition(nestedRecord(retryThenSuccess, ["steer"]).accepted === true, "retry success should accept steer", retryThenSuccess);
|
|
const retrySuccessAttempts = nestedRecord(retryThenSuccess, ["steer"]).attempts;
|
|
assertCondition(Array.isArray(retrySuccessAttempts) && retrySuccessAttempts.length === 2, "retry success should expose both attempts", retryThenSuccess);
|
|
assertCondition(String(JSON.stringify(retryThenSuccess)).includes("The operation was aborted"), "retry attempts should preserve aborted tunnel evidence", retryThenSuccess);
|
|
assertCondition(!String(JSON.stringify(retryThenSuccess)).includes("transient correction"), "retry success must not echo steer prompt", retryThenSuccess);
|
|
|
|
let exhaustedCalls = 0;
|
|
const exhausted = codexSteerTaskForTest("direct_task", ["final correction", "--retry-attempts", "2", "--retry-delay-ms", "0"], () => {
|
|
exhaustedCalls += 1;
|
|
return { ok: false, status: 502, body: abortedTunnelBody };
|
|
}) as JsonRecord;
|
|
assertCondition(exhaustedCalls === 2, "retryable 502 tunnel abort should honor retry-attempts", { exhaustedCalls, exhausted });
|
|
assertReason(exhausted, "stable-proxy-failed", 502);
|
|
const exhaustedDiagnostics = nestedRecord(exhausted, ["diagnostics"]);
|
|
const exhaustedAttempts = exhaustedDiagnostics.attempts;
|
|
assertCondition(Array.isArray(exhaustedAttempts) && exhaustedAttempts.length === 2, "exhausted retry diagnostics should expose attempts", exhaustedDiagnostics);
|
|
assertCondition(String(exhaustedDiagnostics.message || "").includes("The operation was aborted"), "diagnostics should include provider abort message", exhaustedDiagnostics);
|
|
assertCondition(nestedRecord(exhaustedDiagnostics, ["operatorGuidance"]).rawProxyEquivalentIsFallback === false, "raw proxy equivalent should be diagnostic, not fallback", exhaustedDiagnostics);
|
|
assertCondition(String(nestedRecord(exhausted, ["commands"]).rawProxy || "").includes("microservice proxy code-queue /api/tasks/direct_task/steer"), "failure should still expose raw proxy diagnostic command", exhausted);
|
|
|
|
const terminalPrompt = `${"do not leak ".repeat(40)}tail-secret-marker`;
|
|
const terminalRejection = codexSteerTaskForTest("completed_task", [terminalPrompt], () => ({
|
|
ok: false,
|
|
status: 409,
|
|
body: {
|
|
ok: false,
|
|
error: "task does not have an active steerable turn",
|
|
task: {
|
|
id: "completed_task",
|
|
queueId: "default",
|
|
status: "succeeded",
|
|
terminalStatus: "completed",
|
|
currentAttempt: 1,
|
|
updatedAt: "2026-05-22T00:00:00.000Z",
|
|
finishedAt: "2026-05-22T00:00:00.000Z",
|
|
prompt: `${"hidden task prompt ".repeat(60)}tail`,
|
|
output: [{ seq: 1, text: "noisy raw task output" }],
|
|
},
|
|
},
|
|
})) as JsonRecord;
|
|
const terminalSteer = nestedRecord(terminalRejection, ["steer"]);
|
|
assertCondition(terminalRejection.ok === false, "terminal steer rejection should fail", terminalRejection);
|
|
assertCondition(terminalSteer.reason === "task-already-terminal", "terminal steer rejection should use compact terminal reason", terminalSteer);
|
|
assertCondition(terminalSteer.status === "succeeded", "terminal steer rejection should expose task status", terminalSteer);
|
|
assertCondition(terminalSteer.terminalStatus === "completed", "terminal steer rejection should expose terminal status", terminalSteer);
|
|
assertCondition(terminalSteer.lastUpdate === "2026-05-22T00:00:00.000Z", "terminal steer rejection should expose last update", terminalSteer);
|
|
assertCondition(terminalSteer.updatedAt === "2026-05-22T00:00:00.000Z", "terminal steer rejection should expose last update time", terminalSteer);
|
|
assertCondition(terminalSteer.retryable === false, "terminal steer rejection should not be retryable", terminalSteer);
|
|
const terminalCommands = nestedRecord(terminalRejection, ["commands"]);
|
|
assertCondition(String(terminalCommands.show || "").includes("codex task completed_task"), "terminal rejection should suggest show command", terminalCommands);
|
|
assertCondition(String(terminalCommands.read || "").includes("codex read completed_task"), "terminal rejection should suggest read command", terminalCommands);
|
|
assertCondition(String(terminalCommands.followUpSubmit || "").includes("codex submit --prompt-file <path> --reference-task-id completed_task"), "terminal rejection should suggest follow-up submit pattern", terminalCommands);
|
|
const terminalJson = JSON.stringify(terminalRejection);
|
|
assertCondition(!terminalJson.includes("tail-secret-marker"), "terminal rejection must not echo steer prompt", terminalRejection);
|
|
assertCondition(!terminalJson.includes("hidden task prompt"), "terminal rejection must not echo task prompt", terminalRejection);
|
|
assertCondition(!terminalJson.includes("noisy raw task output"), "terminal rejection must not echo task output", terminalRejection);
|
|
assertCondition(!("request" in terminalRejection), "terminal rejection should omit request preview", terminalRejection);
|
|
assertCondition(!("diagnostics" in terminalRejection), "terminal rejection should omit bulky diagnostics", terminalRejection);
|
|
|
|
return {
|
|
ok: true,
|
|
checks: [
|
|
"steer positional dry-run",
|
|
"steer stdin dry-run",
|
|
"steer prompt-file dry-run",
|
|
"duplicate prompt source failure",
|
|
"unsupported option failure",
|
|
"codex help lists steer",
|
|
"outer command redacts positional steer prompt",
|
|
"dry-run does not call stable proxy helper",
|
|
"dry-run prompt preview is bounded",
|
|
"non-dry-run uses stable proxy helper",
|
|
"successful steer confirms write without echoing prompt",
|
|
"steer failure classification is JSON-consumable",
|
|
"retryable tunnel aborts are retried with bounded diagnostics",
|
|
"terminal steer rejection is compact and actionable",
|
|
],
|
|
};
|
|
}
|
|
|
|
if (import.meta.main) {
|
|
const result = runCodeQueueCliSteerContract();
|
|
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
}
|