AgentRun Codex
624 lines
54 KiB
TypeScript
624 lines
54 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { access } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import os from "node:os";
|
|
import { startManagerServer } from "../../mgr/server.js";
|
|
import { MemoryAgentRunStore } from "../../mgr/store.js";
|
|
import { ManagerClient } from "../../mgr/client.js";
|
|
import { runOnce } from "../../runner/run-once.js";
|
|
import type { FailureKind, JsonRecord, TerminalStatus } from "../../common/types.js";
|
|
import { assertNoSecretLeak, createRunWithCommand, type SelfTestCase, type SelfTestContext } from "../harness.js";
|
|
|
|
const selfTest: SelfTestCase = async (context) => {
|
|
const server = await startManagerServer({ port: 0, host: "127.0.0.1", sourceCommit: "self-test", store: new MemoryAgentRunStore() });
|
|
try {
|
|
const client = new ManagerClient(server.baseUrl);
|
|
const happy = await createRunWithCommand(client, context, "hello", "selftest-turn", 15_000);
|
|
const result = await runOnce({ managerUrl: server.baseUrl, runId: happy.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome }, oneShot: true });
|
|
assert.equal(result.terminalStatus, "completed");
|
|
assert.equal(typeof (result.runner as { id?: unknown }).id, "string");
|
|
const events = await client.get(`/api/v1/runs/${happy.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.ok(events.items?.some((event) => event.type === "assistant_message"));
|
|
assert.ok(events.items?.some((event) => event.type === "backend_status" && JSON.stringify(event.payload).includes("run-claimed")));
|
|
assertNoSecretLeak(events);
|
|
const finalRun = await client.get(`/api/v1/runs/${happy.runId}`) as { terminalStatus?: string | null; status?: string };
|
|
assert.equal(finalRun.terminalStatus, "completed");
|
|
assert.equal(finalRun.status, "completed");
|
|
const finalCommand = await client.get(`/api/v1/runs/${happy.runId}/commands/${happy.commandId}`) as { state?: string };
|
|
assert.equal(finalCommand.state, "completed");
|
|
|
|
const sandboxOverride = await createRunWithCommand(client, context, "hello sandbox override", "selftest-sandbox-override", 15_000);
|
|
const sandboxOverrideResult = await runOnce({ managerUrl: server.baseUrl, runId: sandboxOverride.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_CODEX_SHELL_SANDBOX: "danger-full-access", AGENTRUN_FAKE_CODEX_MODE: "require-danger-sandbox" }, oneShot: true });
|
|
assert.equal(sandboxOverrideResult.terminalStatus, "completed");
|
|
const sandboxEvents = await client.get(`/api/v1/runs/${sandboxOverride.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: JsonRecord }> };
|
|
const sandboxStarting = sandboxEvents.items?.find((event) => event.type === "backend_status" && event.payload.phase === "codex-app-server-starting");
|
|
assert.equal(((sandboxStarting?.payload.sandbox as JsonRecord | undefined)?.requested), "workspace-write");
|
|
assert.equal(((sandboxStarting?.payload.sandbox as JsonRecord | undefined)?.effective), "danger-full-access");
|
|
assert.equal(((sandboxStarting?.payload.sandbox as JsonRecord | undefined)?.overrideSource), "AGENTRUN_CODEX_SHELL_SANDBOX");
|
|
|
|
await runLeaseConflictRecoveryCase({ client, managerUrl: server.baseUrl, context });
|
|
|
|
const projectedHome = path.join(context.tmp, "runtime-codex-home");
|
|
const projected = await createRunWithCommand(client, { workspace: context.workspace, codexHome: projectedHome }, "hello projected", "selftest-projected-codex-home", 15_000);
|
|
const projectedResult = await runOnce({ managerUrl: server.baseUrl, runId: projected.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: projectedHome, env: { CODEX_HOME: projectedHome, AGENTRUN_CODEX_SECRET_HOME: context.codexHome }, oneShot: true });
|
|
assert.equal(projectedResult.terminalStatus, "completed");
|
|
await access(path.join(projectedHome, "auth.json"));
|
|
await access(path.join(projectedHome, "config.toml"));
|
|
|
|
const deepseekHome = path.join(context.tmp, "runtime-deepseek-home");
|
|
const deepseek = await createRunWithCommand(client, { ...context, backendProfile: "deepseek" }, "hello deepseek", "selftest-deepseek-turn", 15_000);
|
|
const deepseekResult = await runOnce({ managerUrl: server.baseUrl, runId: deepseek.runId, backendProfile: "deepseek", codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: deepseekHome, env: { CODEX_HOME: deepseekHome, AGENTRUN_CODEX_SECRET_HOME: context.deepseekHome }, oneShot: true });
|
|
assert.equal(deepseekResult.terminalStatus, "completed");
|
|
await access(path.join(deepseekHome, "auth.json"));
|
|
await access(path.join(deepseekHome, "config.toml"));
|
|
const deepseekEvents = await client.get(`/api/v1/runs/${deepseek.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.ok(deepseekEvents.items?.some((event) => event.type === "backend_status" && JSON.stringify(event.payload).includes("deepseek")), "deepseek backend_status should include profile metadata");
|
|
assertNoSecretLeak(deepseekEvents);
|
|
|
|
const minimaxM3Home = path.join(context.tmp, "runtime-minimax-m3-home");
|
|
const minimaxM3 = await createRunWithCommand(client, { ...context, backendProfile: "minimax-m3" }, "hello minimax m3", "selftest-minimax-m3-turn", 15_000);
|
|
const minimaxM3Result = await runOnce({ managerUrl: server.baseUrl, runId: minimaxM3.runId, backendProfile: "minimax-m3", codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: minimaxM3Home, env: { CODEX_HOME: minimaxM3Home, AGENTRUN_CODEX_SECRET_HOME: context.minimaxM3Home }, oneShot: true });
|
|
assert.equal(minimaxM3Result.terminalStatus, "completed");
|
|
await access(path.join(minimaxM3Home, "auth.json"));
|
|
await access(path.join(minimaxM3Home, "config.toml"));
|
|
const minimaxM3Events = await client.get(`/api/v1/runs/${minimaxM3.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.ok(minimaxM3Events.items?.some((event) => event.type === "backend_status" && JSON.stringify(event.payload).includes("minimax-m3")), "minimax-m3 backend_status should include profile metadata");
|
|
assertNoSecretLeak(minimaxM3Events);
|
|
|
|
const dsflashGoHome = path.join(context.tmp, "runtime-dsflash-go-home");
|
|
const dsflashGo = await createRunWithCommand(client, { ...context, backendProfile: "dsflash-go" }, "hello dsflash-go", "selftest-dsflash-go-turn", 15_000);
|
|
const dsflashGoResult = await runOnce({ managerUrl: server.baseUrl, runId: dsflashGo.runId, backendProfile: "dsflash-go", codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: dsflashGoHome, env: { CODEX_HOME: dsflashGoHome, AGENTRUN_CODEX_SECRET_HOME: context.deepseekHome }, oneShot: true });
|
|
assert.equal(dsflashGoResult.terminalStatus, "completed");
|
|
await access(path.join(dsflashGoHome, "auth.json"));
|
|
await access(path.join(dsflashGoHome, "config.toml"));
|
|
await access(path.join(dsflashGoHome, "model-catalog.json"));
|
|
const dsflashGoEvents = await client.get(`/api/v1/runs/${dsflashGo.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.ok(dsflashGoEvents.items?.some((event) => event.type === "backend_status" && JSON.stringify(event.payload).includes("dsflash-go")), "dsflash-go backend_status should include profile metadata");
|
|
assert.ok(dsflashGoEvents.items?.some((event) => event.type === "backend_status" && JSON.stringify(event.payload).includes("\"contextWindow\":1000000")), "dsflash-go backend_status should include 1M context metadata");
|
|
assertNoSecretLeak(dsflashGoEvents);
|
|
|
|
await assert.rejects(
|
|
() => createRunWithCommand(client, { ...context, backendProfile: "deepseek", includeOnlyProfile: "codex" }, "missing deepseek", "selftest-deepseek-missing-secret", 15_000),
|
|
(error) => error instanceof Error && error.message.includes("requires a matching provider credential"),
|
|
);
|
|
await assert.rejects(
|
|
() => createRunWithCommand(client, { ...context, backendProfile: "minimax-m3", includeOnlyProfile: "deepseek" }, "missing minimax m3", "selftest-minimax-m3-missing-secret", 15_000),
|
|
(error) => error instanceof Error && error.message.includes("requires a matching provider credential"),
|
|
);
|
|
|
|
const configModel = await createRunWithCommand(client, context, "hello config model", "selftest-config-model", 15_000);
|
|
const configModelResult = await runOnce({ managerUrl: server.baseUrl, runId: configModel.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "reject-unexpected-model" }, oneShot: true });
|
|
assert.equal(configModelResult.terminalStatus, "completed", "unspecified model should be omitted so Codex config.toml remains authoritative");
|
|
|
|
const explicitModel = await createRunWithCommand(client, context, "hello explicit model placeholder", "selftest-explicit-model-placeholder", 15_000);
|
|
const explicitCommand = await client.post(`/api/v1/runs/${explicitModel.runId}/commands`, { type: "turn", payload: { prompt: "hello explicit model", model: "gpt-5.5" }, idempotencyKey: "selftest-explicit-model-command" }) as { id: string };
|
|
const explicitModelResult = await runOnce({ managerUrl: server.baseUrl, runId: explicitModel.runId, commandId: explicitCommand.id, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "require-explicit-model" }, oneShot: true });
|
|
assert.equal(explicitModelResult.terminalStatus, "completed", "explicit command payload model should still be forwarded");
|
|
|
|
const finalMessage = await createRunWithCommand(client, context, "hello final message", "selftest-final-agent-message", 15_000);
|
|
const finalMessageResult = await runOnce({ managerUrl: server.baseUrl, runId: finalMessage.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "multi-agent-message-final" }, oneShot: true });
|
|
assert.equal(finalMessageResult.terminalStatus, "completed", "multi agentMessage run should complete");
|
|
const finalMessageEnvelope = await client.get(`/api/v1/runs/${finalMessage.runId}/commands/${finalMessage.commandId}/result`) as JsonRecord;
|
|
assert.equal(finalMessageEnvelope.reply, "Final answer only.", "result reply should use the final completed agentMessage instead of concatenating progress deltas");
|
|
const finalMessageEvents = await client.get(`/api/v1/runs/${finalMessage.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
const assistantEvents = finalMessageEvents.items?.filter((event) => event.type === "assistant_message") ?? [];
|
|
const completedAssistantEvents = assistantEvents.filter((event) => eventPayload(event).source === "completed-agent-message");
|
|
assert.equal(completedAssistantEvents.length, 2, "backend should preserve each completed agentMessage as assistant_message event");
|
|
assert.equal(eventPayload(completedAssistantEvents[0] ?? { payload: {} }).text, "I am checking the workspace.");
|
|
assert.equal(eventPayload(completedAssistantEvents[0] ?? { payload: {} }).itemId, "msg_progress");
|
|
assert.equal(eventPayload(completedAssistantEvents[0] ?? { payload: {} }).replyAuthority, false);
|
|
assert.equal(eventPayload(completedAssistantEvents[1] ?? { payload: {} }).text, "Final answer only.");
|
|
assert.equal(eventPayload(completedAssistantEvents[1] ?? { payload: {} }).itemId, "msg_final");
|
|
assert.equal(eventPayload(completedAssistantEvents[1] ?? { payload: {} }).replyAuthority, false);
|
|
const finalMessageItems = finalMessageEvents.items ?? [];
|
|
const progressMessageIndex = finalMessageItems.findIndex((event) => event.type === "assistant_message" && eventPayload(event).itemId === "msg_progress");
|
|
const finalMessageIndex = finalMessageItems.findIndex((event) => event.type === "assistant_message" && eventPayload(event).itemId === "msg_final");
|
|
const turnCompletedIndex = finalMessageItems.findIndex((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/completed");
|
|
assert.ok(progressMessageIndex >= 0 && progressMessageIndex < turnCompletedIndex, "progress agentMessage should be emitted before turn/completed instead of being delayed to final response");
|
|
assert.ok(finalMessageIndex >= 0 && finalMessageIndex < turnCompletedIndex, "final agentMessage should be emitted before turn/completed instead of being delayed to final response");
|
|
assert.equal(finalMessageItems.some((event) => event.type === "backend_status" && String(eventPayload(event).phase ?? "").startsWith("item/agentMessage:")), false, "agentMessage lifecycle must not be persisted as backend_status noise");
|
|
|
|
const webSearch = await createRunWithCommand(client, context, "hello web search progress", "selftest-web-search-progress", 15_000);
|
|
const webSearchPromise = runOnce({ managerUrl: server.baseUrl, runId: webSearch.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "web-search-progress" }, oneShot: true }) as Promise<JsonRecord>;
|
|
await waitForEvent(client, webSearch.runId, (event) => event.type === "tool_call" && eventPayload(event).type === "webSearch" && eventPayload(event).method === "item/started", "webSearch tool_call start event");
|
|
await waitForEvent(client, webSearch.runId, (event) => event.type === "assistant_message" && eventPayload(event).source === "agent-message-delta-progress", "assistant delta progress event");
|
|
const webSearchResult = await webSearchPromise;
|
|
assert.equal(webSearchResult.terminalStatus, "completed", "web search progress turn should complete");
|
|
const webSearchEnvelope = await client.get(`/api/v1/runs/${webSearch.runId}/commands/${webSearch.commandId}/result`) as JsonRecord;
|
|
assert.equal(webSearchEnvelope.reply, "Final IAM recommendation.", "result reply should ignore live delta progress snapshots");
|
|
const webSearchEvents = await client.get(`/api/v1/runs/${webSearch.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
const webSearchItems = webSearchEvents.items ?? [];
|
|
assert.ok(webSearchItems.some((event) => event.type === "tool_call" && eventPayload(event).type === "webSearch" && eventPayload(event).method === "item/completed"), "webSearch completion must remain visible as a tool_call");
|
|
assert.ok(webSearchItems.some((event) => event.type === "assistant_message" && eventPayload(event).source === "agent-message-delta-progress" && eventPayload(event).progress === true), "assistant delta progress must be visible before final reply");
|
|
const webSearchStartIndex = webSearchItems.findIndex((event) => event.type === "tool_call" && eventPayload(event).type === "webSearch" && eventPayload(event).method === "item/started");
|
|
const webSearchProgressIndex = webSearchItems.findIndex((event) => event.type === "assistant_message" && eventPayload(event).source === "agent-message-delta-progress");
|
|
const webSearchCompletedIndex = webSearchItems.findIndex((event) => event.type === "tool_call" && eventPayload(event).type === "webSearch" && eventPayload(event).method === "item/completed");
|
|
const webSearchFinalIndex = webSearchItems.findIndex((event) => event.type === "assistant_message" && eventPayload(event).source === "completed-agent-message" && eventPayload(event).itemId === "msg_search");
|
|
assert.ok(webSearchStartIndex >= 0 && webSearchStartIndex < webSearchProgressIndex, "webSearch start should be visible before assistant progress");
|
|
assert.ok(webSearchProgressIndex >= 0 && webSearchProgressIndex < webSearchCompletedIndex, "assistant progress should be visible while webSearch is still running");
|
|
assert.ok(webSearchCompletedIndex >= 0 && webSearchCompletedIndex < webSearchFinalIndex, "webSearch completion should be visible before final assistant reply");
|
|
assert.equal(webSearchItems.some((event) => event.type === "tool_call" && eventPayload(event).type === "reasoning"), false, "reasoning items must still not be persisted as tool_call");
|
|
assertNoSecretLeak(webSearchEvents);
|
|
|
|
const staleThread = await createStaleThreadRun(client, context);
|
|
const staleThreadResult = await runOnce({
|
|
managerUrl: server.baseUrl,
|
|
runId: staleThread.runId,
|
|
commandId: staleThread.commandId,
|
|
codexCommand: context.fakeCodexCommand,
|
|
codexArgs: context.fakeCodexArgs,
|
|
codexHome: context.codexHome,
|
|
env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "resume-no-rollout" },
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(staleThreadResult.terminalStatus, "failed", "stale thread resume must fail instead of starting a replacement thread");
|
|
assert.equal(staleThreadResult.failureKind, "thread-resume-failed", "stale thread resume must expose thread-resume-failed as the terminal failure");
|
|
const staleEnvelope = await client.get(`/api/v1/runs/${staleThread.runId}/commands/${staleThread.commandId}/result`) as JsonRecord;
|
|
assert.equal(staleEnvelope.terminalStatus, "failed");
|
|
assert.equal(staleEnvelope.failureKind, "thread-resume-failed");
|
|
assert.equal(staleEnvelope.completed, false);
|
|
assert.equal((staleEnvelope.sessionRef as JsonRecord).threadId, "thread_missing_rollout");
|
|
const staleEvents = await client.get(`/api/v1/runs/${staleThread.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
const stalePhases = (staleEvents.items ?? []).filter((event) => event.type === "backend_status").map((event) => String(eventPayload(event).phase ?? ""));
|
|
assert.equal(staleEvents.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "thread/resume:non-resumable"), false, "stale resume must not be converted into a replacement path");
|
|
assert.equal(stalePhases.some((phase) => phase === "thread/replacement-start:completed"), false, "stale resume must not start a replacement thread");
|
|
assert.equal(staleEvents.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "thread/resume:completed"), false, "stale resume must not be reported as a successful resume");
|
|
assert.equal(staleEvents.items?.some((event) => event.type === "error" && eventPayload(event).failureKind === "thread-resume-failed"), true, "stale resume must surface terminal thread-resume-failed error");
|
|
assertNoSecretLeak({ staleThreadResult, staleEnvelope, staleEvents });
|
|
|
|
const live = await createRunWithCommand(client, context, "hello live events", "selftest-live-tool-events", 15_000);
|
|
const livePromise = runOnce({ managerUrl: server.baseUrl, runId: live.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "slow-tool-events" }, oneShot: true }) as Promise<JsonRecord>;
|
|
await waitForEvent(client, live.runId, (event) => event.type === "tool_call" && eventPayload(event).method === "item/started", "live tool_call start event");
|
|
const liveEvents = await client.get(`/api/v1/runs/${live.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
const liveToolStart = (liveEvents.items ?? []).find((event) => event.type === "tool_call" && eventPayload(event).method === "item/started") ?? { payload: {} };
|
|
assert.equal(eventPayload(liveToolStart).item, undefined, "tool_call started event must not persist raw Codex item JSON");
|
|
assert.equal(eventPayload(liveToolStart).itemPreview, undefined, "tool_call started event must not persist raw Codex item preview");
|
|
assert.equal(JSON.stringify(eventPayload(liveToolStart).summary ?? {}).includes("\\\"method\\\":\\\"item/started\\\""), false, "tool_call started event summary must not embed raw protocol JSON");
|
|
assert.equal(String(eventPayload(liveToolStart).summary ? (eventPayload(liveToolStart).summary as JsonRecord).text ?? "" : "").includes("commandExecution started:"), true, "tool_call started event summary should be human readable");
|
|
assert.equal(eventPayload(liveToolStart).toolName, "commandExecution");
|
|
assert.equal(eventPayload(liveToolStart).type, "commandExecution");
|
|
await waitForEvent(client, live.runId, (event) => event.type === "command_output" && String(eventPayload(event).text ?? "").includes("live output"), "live command output event");
|
|
const liveResult = await livePromise;
|
|
assert.equal(liveResult.terminalStatus, "completed", "slow live tool event turn should complete");
|
|
|
|
await runInterruptBeforeTurnStartResponseCase({ client, managerUrl: server.baseUrl, context });
|
|
await runToolProgressRefreshesIdleTimeoutCase({ client, managerUrl: server.baseUrl, context });
|
|
|
|
const noisy = await createRunWithCommand(client, context, "hello noisy reasoning", "selftest-noisy-reasoning-events", 15_000);
|
|
const noisyResult = await runOnce({ managerUrl: server.baseUrl, runId: noisy.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "noisy-reasoning-events" }, oneShot: true }) as JsonRecord;
|
|
assert.equal(noisyResult.terminalStatus, "completed", "noisy reasoning turn should complete");
|
|
const noisyEvents = await client.get(`/api/v1/runs/${noisy.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
const noisyItems = noisyEvents.items ?? [];
|
|
const noisyPhases = noisyItems.map((event) => eventPayload(event).phase).filter(Boolean);
|
|
assert.equal(noisyItems.some((event) => event.type === "backend_status" && eventPayload(event).phase === "item/reasoning/textDelta"), false, "reasoning textDelta must not be persisted as backend_status");
|
|
assert.equal(noisyItems.some((event) => event.type === "backend_status" && eventPayload(event).phase === "thread/tokenUsage/updated"), false, "token usage update must not be persisted as backend_status");
|
|
assert.equal(noisyItems.some((event) => event.type === "backend_status" && eventPayload(event).phase === "account/rateLimits/updated"), false, "rate limit update must not be persisted as backend_status");
|
|
assert.equal(noisyItems.some((event) => event.type === "backend_status" && eventPayload(event).phase === "warning"), false, "low value warnings must not be persisted as backend_status");
|
|
assert.equal(noisyItems.some((event) => event.type === "backend_status" && eventPayload(event).phase === "configWarning"), false, "low value config warnings must not be persisted as backend_status");
|
|
assert.equal(noisyItems.some((event) => event.type === "tool_call" && eventPayload(event).type === "reasoning"), false, "reasoning items must not be persisted as tool_call");
|
|
assert.ok(noisyItems.some((event) => event.type === "tool_call" && eventPayload(event).method === "item/started" && eventPayload(event).type === "commandExecution"), "real commandExecution tool call should remain visible");
|
|
assert.equal(noisyItems.some((event) => event.type === "tool_call" && eventPayload(event).type !== "commandExecution" && eventPayload(event).type !== "webSearch"), false, "only user-visible tool lifecycle items should be persisted as tool_call");
|
|
assert.equal(noisyItems.some((event) => event.type === "backend_status" && String(eventPayload(event).phase ?? "").startsWith("item/agentMessage:")), false, "agentMessage lifecycle must not be persisted as backend_status noise");
|
|
assert.equal(noisyPhases.includes("backend-turn-running"), false, "backend progress ticks must be summarized instead of persisted as durable trace events");
|
|
const noisyFinished = noisyItems.find((event) => event.type === "backend_status" && eventPayload(event).phase === "backend-turn-finished");
|
|
assert.equal(eventPayload(noisyFinished ?? { payload: {} }).progressEventsPrinted, false, "backend-turn-finished must declare progress ticks were not printed as events");
|
|
assert.ok(noisyItems.some((event) => event.type === "assistant_message" && eventPayload(event).text === "noise filtered final"), "final assistant_message should remain visible");
|
|
const suppression = noisyItems.find((event) => event.type === "backend_status" && eventPayload(event).phase === "codex-app-server-notifications-suppressed");
|
|
assert.ok(suppression, "suppression summary must be emitted when noisy notifications are filtered");
|
|
assert.equal(eventPayload(suppression ?? { payload: {} }).total, 8);
|
|
assert.deepEqual(countEntriesByName(eventPayload(suppression ?? { payload: {} }).methods, "method"), {
|
|
"account/rateLimits/updated": 1,
|
|
"configWarning": 1,
|
|
"item/completed": 1,
|
|
"item/reasoning/textDelta": 2,
|
|
"item/started": 1,
|
|
"thread/tokenUsage/updated": 1,
|
|
"warning": 1,
|
|
});
|
|
assert.deepEqual(countEntriesByName(eventPayload(suppression ?? { payload: {} }).itemTypes, "itemType"), {
|
|
reasoning: 4,
|
|
});
|
|
assert.equal(eventPayload(suppression ?? { payload: {} }).byMethod, undefined, "suppression summary must not use method names as JSON keys because redaction treats token-like key names as sensitive");
|
|
assert.equal(JSON.stringify(suppression).includes("thread/tokenUsage/updated"), true, "suppressed tokenUsage method name should remain visible as metadata, not redacted as a key");
|
|
assert.equal(JSON.stringify(noisyEvents).includes("internal reasoning must not become durable trace text"), false, "suppression summary must not leak reasoning text");
|
|
assertNoSecretLeak(noisyEvents);
|
|
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "missing-turn-result", expectedStatus: "failed", expectedFailureKind: "backend-response-invalid" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-401-rpc-error", expectedStatus: "failed", expectedFailureKind: "provider-auth-failed" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-429-terminal", expectedStatus: "failed", expectedFailureKind: "provider-rate-limited" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-invalid-tool-call", expectedStatus: "failed", expectedFailureKind: "provider-invalid-tool-call" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-compact-404-terminal", expectedStatus: "failed", expectedFailureKind: "provider-compact-unsupported" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-stream-disconnected-rpc-error", expectedStatus: "failed", expectedFailureKind: "provider-stream-disconnected" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-503-rpc-error", expectedStatus: "failed", expectedFailureKind: "provider-stream-disconnected" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-503-terminal", expectedStatus: "failed", expectedFailureKind: "provider-http-error" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-unavailable-terminal", expectedStatus: "failed", expectedFailureKind: "provider-unavailable" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-503-retry-event", expectedStatus: "failed", expectedFailureKind: "provider-stream-disconnected", expectRetryError: true });
|
|
await runRetryThenCompletedCase({ client, managerUrl: server.baseUrl, context });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "invalid-json", expectedStatus: "failed", expectedFailureKind: "backend-json-parse-error" });
|
|
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "missing-terminal", expectedStatus: "failed", expectedFailureKind: "backend-timeout", timeoutMs: 500 });
|
|
await runSlowProgressIdleCase({ client, managerUrl: server.baseUrl, context });
|
|
await runFailureDoesNotTerminalRunCase({ client, managerUrl: server.baseUrl, context });
|
|
await runSecretFailureCase({ client, managerUrl: server.baseUrl, context });
|
|
await runSpawnFailureCase({ client, managerUrl: server.baseUrl, context });
|
|
await runSessionStorageMountedCase({ client, managerUrl: server.baseUrl, context });
|
|
await runSessionStorageEvictedCase({ client, managerUrl: server.baseUrl, context });
|
|
await runSessionStorageSubdirCase({ client, managerUrl: server.baseUrl, context });
|
|
await runSessionStorageNoSecretLeakCase({ client, managerUrl: server.baseUrl, context });
|
|
|
|
return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "runner-lease-conflict-recovery", "codex-stdio-fake-turn", "codex-stdio-k8s-sandbox-override", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-dsflash-go-profile-fake-turn", "codex-stdio-dsflash-go-config-metadata", "codex-stdio-minimax-m3-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-minimax-m3-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-final-agent-message-only", "codex-stdio-web-search-progress", "codex-stdio-stale-thread-resume-failed", "codex-stdio-live-tool-events", "codex-stdio-interrupt-before-turn-start-response", "codex-stdio-tool-progress-refreshes-idle-timeout", "codex-stdio-noisy-reasoning-suppression", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-invalid-tool-call", "codex-stdio-provider-compact-unsupported", "codex-stdio-provider-stream-disconnected", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-unavailable", "codex-stdio-provider-503-retry-event", "codex-stdio-provider-refused-retry-recovered", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-idle-timeout-progress-refresh", "codex-stdio-command-failure-keeps-run-open", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
|
|
} finally {
|
|
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
|
}
|
|
};
|
|
|
|
async function runRetryThenCompletedCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "retry then complete", "selftest-provider-refused-retry-recovered", 15_000);
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "provider-refused-retry-then-completed" },
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(result.terminalStatus, "completed");
|
|
assert.equal(result.failureKind, null);
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.equal(events.items?.some((event) => event.type === "error" && eventPayload(event).willRetry === true && eventPayload(event).failureKind === "provider-stream-disconnected"), true);
|
|
const envelope = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}/result`) as JsonRecord;
|
|
assert.equal(envelope.terminalStatus, "completed");
|
|
assert.equal(envelope.failureKind, null);
|
|
assert.equal(envelope.failureMessage, null);
|
|
assert.equal(envelope.completed, true);
|
|
}
|
|
|
|
async function runLeaseConflictRecoveryCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "claim after stale lease", "selftest-runner-lease-conflict-recovery", 15_000);
|
|
const staleRunner = await options.client.post("/api/v1/runners/register", {
|
|
runId: item.runId,
|
|
attemptId: "attempt_stale_claim",
|
|
backendProfile: "codex",
|
|
placement: "kubernetes-job",
|
|
sourceCommit: "self-test",
|
|
id: "runner_stale_claim",
|
|
}) as JsonRecord;
|
|
await options.client.post(`/api/v1/runs/${item.runId}/claim`, { runnerId: staleRunner.id, leaseMs: 300 });
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
commandId: item.commandId,
|
|
runnerId: "runner_recovered_claim",
|
|
attemptId: "attempt_recovered_claim",
|
|
leaseMs: 1_000,
|
|
claimRetryTimeoutMs: 2_000,
|
|
claimRetryIntervalMs: 50,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: { CODEX_HOME: options.context.codexHome },
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(result.terminalStatus, "completed", "replacement runner should claim after stale lease expiry and finish the pending command");
|
|
const run = await options.client.get(`/api/v1/runs/${item.runId}`) as JsonRecord;
|
|
assert.equal(run.claimedBy, "runner_recovered_claim");
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.equal(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "runner-claim-waiting-for-stale-lease"), true, "lease conflict recovery should be visible while waiting");
|
|
assert.equal(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "runner-claim-recovered"), true, "lease conflict recovery should be visible after claim succeeds");
|
|
assertNoSecretLeak({ result, run, events });
|
|
}
|
|
|
|
async function runSlowProgressIdleCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 250);
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "slow-progress-before-terminal" },
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(result.terminalStatus, "completed", "activity before idle deadline must refresh the turn idle timeout");
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.equal(events.items?.some((event) => event.type === "error" && eventPayload(event).failureKind === "backend-timeout"), false, "progressing turns must not fail on total elapsed time");
|
|
}
|
|
|
|
async function runInterruptBeforeTurnStartResponseCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "interrupt hanging tool before turn/start response", "selftest-interrupt-before-turn-start-response", 1_000);
|
|
const runPromise = runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-hangs-before-turn-start-response" },
|
|
oneShot: true,
|
|
}) as Promise<JsonRecord>;
|
|
await waitForEvent(options.client, item.runId, (event) => event.type === "tool_call" && eventPayload(event).itemId === "tool_hang_before_response", "tool_call before turn/start response");
|
|
const interrupt = await options.client.post(`/api/v1/runs/${item.runId}/commands`, { type: "interrupt", payload: { reason: "self-test interrupt hanging tool" }, idempotencyKey: "selftest-interrupt-before-response-command" }) as { id: string };
|
|
const result = await runPromise;
|
|
assert.equal(result.terminalStatus, "cancelled", "interrupt should cancel the active turn even if turn/start response has not returned");
|
|
assert.equal(result.failureKind, "cancelled");
|
|
const interruptCommand = await options.client.get(`/api/v1/runs/${item.runId}/commands/${interrupt.id}`) as { state?: string };
|
|
assert.equal(interruptCommand.state, "completed", "interrupt control command should be terminal completed after delivery");
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "active-turn-control-ready"), "turn/started notification should expose active turn control before turn/start response");
|
|
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "interrupt-command-acknowledged"), "interrupt command acknowledgement should be visible");
|
|
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/interrupt:completed"), "interrupt delivery result should be visible");
|
|
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/start:interrupted-before-response"), "turn/start pending request should be bypassed after interrupt terminal");
|
|
assertNoSecretLeak({ result, events });
|
|
}
|
|
|
|
async function runToolProgressRefreshesIdleTimeoutCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "tool progress refreshes idle timeout", "selftest-tool-progress-refreshes-idle", 120);
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-progress-refreshes-idle" },
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(result.terminalStatus, "completed", "tool progress should refresh idle timeout until terminal completion");
|
|
assert.equal(result.failureKind, null);
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.ok(events.items?.some((event) => event.type === "command_output" && String(eventPayload(event).text ?? "").includes("progress 3")), "progress output should stay visible while idle timeout is refreshed");
|
|
assert.equal(events.items?.some((event) => event.type === "error" && eventPayload(event).failureKind === "backend-timeout"), false, "progressing tool output must not fail on wall-clock elapsed time");
|
|
const command = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}`) as { state?: string };
|
|
assert.equal(command.state, "completed", "command should complete after progress-delayed terminal status");
|
|
assertNoSecretLeak({ result, events });
|
|
}
|
|
|
|
async function runFailureDoesNotTerminalRunCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "first command fails", "selftest-command-failure-keeps-run-open", 3_000);
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "provider-503-terminal" },
|
|
idleTimeoutMs: 100,
|
|
pollIntervalMs: 25,
|
|
}) as JsonRecord;
|
|
assert.equal(result.stopped, "idle-timeout", "non one-shot runner should remain alive after a failed command until idle timeout");
|
|
assert.equal(result.terminalStatus, "failed");
|
|
assert.equal(result.failureKind, "provider-http-error");
|
|
const command = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}`) as { state?: string };
|
|
assert.equal(command.state, "failed");
|
|
const run = await options.client.get(`/api/v1/runs/${item.runId}`) as { status?: string; terminalStatus?: string | null; failureKind?: string | null };
|
|
assert.equal(["claimed", "running"].includes(String(run.status)), true, "command failure must keep the reusable run/session non-terminal");
|
|
assert.equal(run.terminalStatus, null);
|
|
assert.equal(run.failureKind, null);
|
|
const runEnvelope = await options.client.get(`/api/v1/runs/${item.runId}/result?commandId=${item.commandId}`) as JsonRecord;
|
|
assert.equal(runEnvelope.failureKind, "provider-http-error", "run result must inherit terminal command failureKind even when reusable run stays non-terminal");
|
|
const commandEnvelope = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}/result`) as JsonRecord;
|
|
assert.equal(commandEnvelope.failureKind, "provider-http-error", "command result must expose provider HTTP failureKind from terminal command/events");
|
|
}
|
|
|
|
async function runFailureCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext; mode: string; expectedStatus: TerminalStatus; expectedFailureKind: FailureKind; timeoutMs?: number; expectRetryError?: boolean }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, `failure ${options.mode}`, `selftest-${options.mode}`, options.timeoutMs ?? 3_000);
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: options.mode },
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(result.terminalStatus, options.expectedStatus, options.mode);
|
|
assert.equal(result.failureKind, options.expectedFailureKind, options.mode);
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.ok(events.items?.some((event) => event.type === "error"), options.mode);
|
|
assert.ok(events.items?.some((event) => event.type === "error" && eventPayload(event).failureKind === options.expectedFailureKind), `${options.mode} expected error event failureKind ${options.expectedFailureKind}`);
|
|
if (options.expectRetryError) {
|
|
assert.ok(events.items?.some((event) => {
|
|
const payload = eventPayload(event);
|
|
return event.type === "error" && payload.willRetry === true && payload.failureKind === options.expectedFailureKind;
|
|
}), `${options.mode} expected retry error event failureKind ${options.expectedFailureKind}`);
|
|
}
|
|
const command = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}`) as { state?: string };
|
|
assert.equal(command.state, "failed", options.mode);
|
|
assertNoSecretLeak(events);
|
|
}
|
|
|
|
function eventPayload(event: { payload: unknown }): JsonRecord {
|
|
return typeof event.payload === "object" && event.payload !== null && !Array.isArray(event.payload) ? event.payload as JsonRecord : {};
|
|
}
|
|
|
|
function countEntriesByName(value: unknown, keyName: "method" | "itemType"): Record<string, number> {
|
|
const output: Record<string, number> = {};
|
|
if (!Array.isArray(value)) return output;
|
|
for (const entry of value) {
|
|
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) continue;
|
|
const record = entry as JsonRecord;
|
|
if (typeof record[keyName] === "string" && typeof record.count === "number") output[record[keyName]] = record.count;
|
|
}
|
|
return output;
|
|
}
|
|
|
|
async function waitForEvent(client: ManagerClient, runId: string, predicate: (event: { type: string; payload: unknown }) => boolean, label: string): Promise<void> {
|
|
const deadline = Date.now() + 3_000;
|
|
while (Date.now() < deadline) {
|
|
const events = await client.get(`/api/v1/runs/${runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
if ((events.items ?? []).some(predicate)) return;
|
|
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
}
|
|
assert.fail(`timed out waiting for ${label}`);
|
|
}
|
|
|
|
async function createStaleThreadRun(client: ManagerClient, context: SelfTestContext): Promise<{ runId: string; commandId: string }> {
|
|
const run = await client.post("/api/v1/runs", {
|
|
tenantId: "unidesk",
|
|
projectId: "pikasTech/unidesk",
|
|
workspaceRef: { kind: "host-path", path: context.workspace },
|
|
sessionRef: { sessionId: "selftest-stale-thread-session", conversationId: "selftest-stale-thread-session", threadId: "thread_missing_rollout" },
|
|
providerId: "G14",
|
|
backendProfile: "codex",
|
|
executionPolicy: {
|
|
sandbox: "workspace-write",
|
|
approval: "never",
|
|
timeoutMs: 15_000,
|
|
network: "default",
|
|
secretScope: {
|
|
allowCredentialEcho: false,
|
|
providerCredentials: [{ profile: "codex", secretRef: { name: "agentrun-v01-provider-codex", keys: ["auth.json", "config.toml"], mountPath: context.codexHome } }],
|
|
},
|
|
},
|
|
traceSink: null,
|
|
}) as { id: string };
|
|
const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt: "hello stale thread" }, idempotencyKey: "selftest-stale-thread-resume-fails" }) as { id: string };
|
|
return { runId: run.id, commandId: command.id };
|
|
}
|
|
|
|
async function runSecretFailureCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "failure missing secret files", "selftest-secret-unavailable", 3_000);
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: path.join(options.context.tmp, "missing-codex-home"),
|
|
env: { CODEX_HOME: path.join(options.context.tmp, "missing-codex-home") },
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(result.terminalStatus, "blocked", "secret unavailable");
|
|
assert.equal(result.failureKind, "secret-unavailable", "secret unavailable");
|
|
const command = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}`) as { state?: string };
|
|
assert.equal(command.state, "failed", "secret unavailable command state");
|
|
const envelope = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}/result`) as JsonRecord;
|
|
assert.equal(envelope.terminalStatus, "failed", "secret unavailable result terminal");
|
|
assert.equal(envelope.failureKind, "secret-unavailable", "secret unavailable result kind");
|
|
}
|
|
|
|
async function runSpawnFailureCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "failure spawn", "selftest-spawn-failure", 3_000);
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: path.join(os.tmpdir(), `agentrun-missing-codex-${process.pid}`),
|
|
codexArgs: [],
|
|
codexHome: options.context.codexHome,
|
|
env: { CODEX_HOME: options.context.codexHome },
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(result.terminalStatus, "failed", "spawn failure");
|
|
assert.equal(result.failureKind, "backend-spawn-failed", "spawn failure");
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assert.ok(events.items?.some((event) => event.type === "error"), "spawn failure");
|
|
const command = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}`) as { state?: string };
|
|
assert.equal(command.state, "failed", "spawn failure");
|
|
const commandEnvelope = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}/result`) as JsonRecord;
|
|
assert.equal(commandEnvelope.failureKind, "backend-spawn-failed", "spawn failure command result kind");
|
|
const runEnvelope = await options.client.get(`/api/v1/runs/${item.runId}/result?commandId=${item.commandId}`) as JsonRecord;
|
|
assert.equal(runEnvelope.failureKind, "backend-spawn-failed", "spawn failure run result kind");
|
|
assertNoSecretLeak(events);
|
|
}
|
|
|
|
|
|
async function runSessionStorageMountedCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "session storage mounted", "selftest-session-storage-mounted", 3_000);
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: {
|
|
CODEX_HOME: options.context.codexHome,
|
|
|
|
AGENTRUN_SESSION_PVC_NAME: "agentrun-v01-session-selftest-mounted",
|
|
AGENTRUN_SESSION_PVC_NAMESPACE: "agentrun-v01",
|
|
AGENTRUN_SESSION_PVC_MOUNT_PATH: "/home/agentrun/.codex-codex/sessions",
|
|
AGENTRUN_CODEX_ROLLOUT_SUBDIR: "sessions",
|
|
},
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(result.terminalStatus, "completed");
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
const mounted = (events.items ?? []).find((event) => event.type === "backend_status" && eventPayload(event).phase === "codex-rollout-storage-mounted");
|
|
assert.ok(mounted, "codex-rollout-storage-mounted event must be emitted when AGENTRUN_SESSION_PVC_NAME is set");
|
|
const payload = eventPayload(mounted);
|
|
assert.equal(payload.pvcName, "agentrun-v01-session-selftest-mounted");
|
|
assert.equal(payload.pvcNamespace, "agentrun-v01");
|
|
assert.equal(payload.mountPath, "/home/agentrun/.codex-codex/sessions");
|
|
assert.equal(payload.codexRolloutSubdir, "sessions");
|
|
assert.equal(payload.valuesPrinted, false);
|
|
}
|
|
|
|
async function runSessionStorageEvictedCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const stale = await createStaleThreadRun(options.client, options.context);
|
|
const result = await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: stale.runId,
|
|
commandId: stale.commandId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: {
|
|
CODEX_HOME: options.context.codexHome,
|
|
AGENTRUN_FAKE_CODEX_MODE: "resume-no-rollout",
|
|
AGENTRUN_SESSION_PVC_NAME: "agentrun-v01-session-selftest-evicted",
|
|
AGENTRUN_SESSION_PVC_NAMESPACE: "agentrun-v01",
|
|
AGENTRUN_SESSION_PVC_MOUNT_PATH: "/home/agentrun/.codex-codex/sessions",
|
|
AGENTRUN_CODEX_ROLLOUT_SUBDIR: "sessions",
|
|
},
|
|
oneShot: true,
|
|
}) as JsonRecord;
|
|
assert.equal(result.terminalStatus, "failed");
|
|
assert.equal(result.failureKind, "session-store-evicted", "with PVC env set, no rollout found must be classified as session-store-evicted");
|
|
}
|
|
|
|
async function runSessionStorageSubdirCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "session storage subdir", "selftest-session-storage-subdir", 3_000);
|
|
await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: {
|
|
CODEX_HOME: options.context.codexHome,
|
|
|
|
AGENTRUN_SESSION_PVC_NAME: "agentrun-v01-session-selftest-subdir",
|
|
AGENTRUN_SESSION_PVC_NAMESPACE: "agentrun-v01",
|
|
AGENTRUN_SESSION_PVC_MOUNT_PATH: "/home/agentrun/.codex-deepseek/custom",
|
|
AGENTRUN_CODEX_ROLLOUT_SUBDIR: "custom",
|
|
},
|
|
oneShot: true,
|
|
});
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
const mounted = (events.items ?? []).find((event) => event.type === "backend_status" && eventPayload(event).phase === "codex-rollout-storage-mounted");
|
|
assert.ok(mounted, "storage-mounted event must fire for custom subdir");
|
|
const payload = eventPayload(mounted);
|
|
assert.equal(payload.codexRolloutSubdir, "custom", "AGENTRUN_CODEX_ROLLOUT_SUBDIR must be observed in the storage-mounted event");
|
|
assert.equal(payload.mountPath, "/home/agentrun/.codex-deepseek/custom", "mount path must use the rollout subdir suffix");
|
|
}
|
|
|
|
async function runSessionStorageNoSecretLeakCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
|
const item = await createRunWithCommand(options.client, options.context, "session storage no leak", "selftest-session-storage-no-leak", 3_000);
|
|
await runOnce({
|
|
managerUrl: options.managerUrl,
|
|
runId: item.runId,
|
|
codexCommand: options.context.fakeCodexCommand,
|
|
codexArgs: options.context.fakeCodexArgs,
|
|
codexHome: options.context.codexHome,
|
|
env: {
|
|
CODEX_HOME: options.context.codexHome,
|
|
|
|
AGENTRUN_SESSION_PVC_NAME: "agentrun-v01-session-selftest-leak",
|
|
AGENTRUN_SESSION_PVC_NAMESPACE: "agentrun-v01",
|
|
AGENTRUN_SESSION_PVC_MOUNT_PATH: "/home/agentrun/.codex-codex/sessions",
|
|
AGENTRUN_CODEX_ROLLOUT_SUBDIR: "sessions",
|
|
},
|
|
oneShot: true,
|
|
});
|
|
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
|
|
assertNoSecretLeak(events);
|
|
}
|
|
|
|
export default selfTest;
|