mirrors/pikasTech-agentrun
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use idle timeout for codex turns

Browse Source
This commit is contained in:
AgentRun Codex
2026-06-11 12:52:46 +08:00
parent dafcfc7ce4
commit f49695a821
11 changed files with 115 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/selftest/cases/30-codex-stdio.ts
+10 -12
View File
@@ -182,7 +182,7 @@ const selfTest: SelfTestCase = async (context) => {
assert.equal(liveResult.terminalStatus, "completed", "slow live tool event turn should complete");
await runInterruptBeforeTurnStartResponseCase({ client, managerUrl: server.baseUrl, context });
await runHardTimeoutDuringToolProgressCase({ client, managerUrl: server.baseUrl, context });
await runToolProgressRefreshesIdleTimeoutCase({ client, managerUrl: server.baseUrl, context });
const noisy = await createRunWithCommand(client, context, "hello noisy reasoning", "selftest-noisy-reasoning-events", 15_000);
const noisyResult = await runOnce({ managerUrl: server.baseUrl, runId: noisy.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "noisy-reasoning-events" }, oneShot: true }) as JsonRecord;
@@ -245,7 +245,7 @@ const selfTest: SelfTestCase = async (context) => {
await runSessionStorageSubdirCase({ client, managerUrl: server.baseUrl, context });
await runSessionStorageNoSecretLeakCase({ client, managerUrl: server.baseUrl, context });
return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "runner-lease-conflict-recovery", "codex-stdio-fake-turn", "codex-stdio-k8s-sandbox-override", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-dsflash-go-profile-fake-turn", "codex-stdio-dsflash-go-config-metadata", "codex-stdio-minimax-m3-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-minimax-m3-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-final-agent-message-only", "codex-stdio-web-search-progress", "codex-stdio-stale-thread-resume-failed", "codex-stdio-live-tool-events", "codex-stdio-interrupt-before-turn-start-response", "codex-stdio-hard-timeout-during-tool-progress", "codex-stdio-noisy-reasoning-suppression", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-invalid-tool-call", "codex-stdio-provider-compact-unsupported", "codex-stdio-provider-stream-disconnected", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-unavailable", "codex-stdio-provider-503-retry-event", "codex-stdio-provider-refused-retry-recovered", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-idle-timeout-progress-refresh", "codex-stdio-command-failure-keeps-run-open", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "runner-lease-conflict-recovery", "codex-stdio-fake-turn", "codex-stdio-k8s-sandbox-override", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-dsflash-go-profile-fake-turn", "codex-stdio-dsflash-go-config-metadata", "codex-stdio-minimax-m3-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-minimax-m3-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-final-agent-message-only", "codex-stdio-web-search-progress", "codex-stdio-stale-thread-resume-failed", "codex-stdio-live-tool-events", "codex-stdio-interrupt-before-turn-start-response", "codex-stdio-tool-progress-refreshes-idle-timeout", "codex-stdio-noisy-reasoning-suppression", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-invalid-tool-call", "codex-stdio-provider-compact-unsupported", "codex-stdio-provider-stream-disconnected", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-unavailable", "codex-stdio-provider-503-retry-event", "codex-stdio-provider-refused-retry-recovered", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-idle-timeout-progress-refresh", "codex-stdio-command-failure-keeps-run-open", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
} finally {
await new Promise<void>((resolve) => server.server.close(() => resolve()));
}
@@ -350,26 +350,24 @@ async function runInterruptBeforeTurnStartResponseCase(options: { client: Manage
assertNoSecretLeak({ result, events });
}
async function runHardTimeoutDuringToolProgressCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
const item = await createRunWithCommand(options.client, options.context, "hard timeout during tool progress", "selftest-hard-timeout-tool-progress", 120);
async function runToolProgressRefreshesIdleTimeoutCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
const item = await createRunWithCommand(options.client, options.context, "tool progress refreshes idle timeout", "selftest-tool-progress-refreshes-idle", 120);
const result = await runOnce({
managerUrl: options.managerUrl,
runId: item.runId,
codexCommand: options.context.fakeCodexCommand,
codexArgs: options.context.fakeCodexArgs,
codexHome: options.context.codexHome,
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "hard-timeout-tool-progress" },
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-progress-refreshes-idle" },
oneShot: true,
}) as JsonRecord;
assert.equal(result.terminalStatus, "failed", "hard timeout should fail even while the tool keeps producing progress");
assert.equal(result.failureKind, "backend-timeout");
assert.equal(result.terminalStatus, "completed", "tool progress should refresh idle timeout until terminal completion");
assert.equal(result.failureKind, null);
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
assert.ok(events.items?.some((event) => event.type === "command_output" && String(eventPayload(event).text ?? "").includes("progress")), "progress output should be visible before hard timeout");
assert.ok(events.items?.some((event) => event.type === "error" && eventPayload(event).phase === "turn:hard-timeout"), "hard timeout should be recorded as an error event");
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn-interrupt-requested"), "timeout should request backend interrupt before process teardown");
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/interrupt:completed"), "timeout interrupt result should be visible");
assert.ok(events.items?.some((event) => event.type === "command_output" && String(eventPayload(event).text ?? "").includes("progress 3")), "progress output should stay visible while idle timeout is refreshed");
assert.equal(events.items?.some((event) => event.type === "error" && eventPayload(event).failureKind === "backend-timeout"), false, "progressing tool output must not fail on wall-clock elapsed time");
const command = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}`) as { state?: string };
assert.equal(command.state, "failed", "hard timed out command should be failed");
assert.equal(command.state, "completed", "command should complete after progress-delayed terminal status");
assertNoSecretLeak({ result, events });
}
src/selftest/cases/50-hwlab-manual-dispatch.ts
+5 -2
View File
@@ -97,7 +97,7 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin
const resultBundleTargets = (((resultEnvelope.resourceBundleRef as JsonRecord).bundles as JsonRecord).items as JsonRecord[]).map((item) => item.targetPath);
assert.deepEqual(resultBundleTargets, ["tools", ".agents/skills"]);
const materialized = ((resultEnvelope.resourceBundleRef as JsonRecord).materialized as JsonRecord);
assert.deepEqual(((materialized.tools as JsonRecord).names), ["hwpod"]);
assert.deepEqual(((materialized.tools as JsonRecord).names), ["apply_patch", "hwpod", "tran", "trans"]);
assert.equal(((materialized.tools as JsonRecord).installed), true);
assert.deepEqual(((materialized.skillDirs as JsonRecord).names), ["dad-dev", "hwpod-cli", "hwpod-ctl"]);
const requiredSkillItems = ((materialized.requiredSkills as JsonRecord).items as JsonRecord[]);
@@ -275,6 +275,9 @@ async function createLocalGitBundle(context: SelfTestContext, repoName = "bundle
await mkdir(path.join(repo, "tools"), { recursive: true });
await mkdir(path.join(repo, "tools", "src"), { recursive: true });
await writeFile(path.join(repo, "tools", "hwpod"), "#!/usr/bin/env sh\nexec bun \"$(dirname \"$0\")/hwpod-cli.ts\" \"$@\"\n", "utf8");
await writeFile(path.join(repo, "tools", "tran"), "#!/usr/bin/env sh\necho tran-selftest\n", "utf8");
await writeFile(path.join(repo, "tools", "trans"), "#!/usr/bin/env sh\necho trans-selftest\n", "utf8");
await writeFile(path.join(repo, "tools", "apply_patch"), "#!/usr/bin/env sh\necho apply-patch-selftest\n", "utf8");
await writeFile(path.join(repo, "tools", "hwpod-cli.ts"), "import { hwpodSelftestName } from './src/hwpod-harness-lib.ts';\nconsole.log(JSON.stringify({ ok: true, cli: hwpodSelftestName(), argv: process.argv.slice(2) }));\n", "utf8");
await writeFile(path.join(repo, "tools", "src", "hwpod-harness-lib.ts"), "export function hwpodSelftestName() { return 'hwpod-selftest'; }\n", "utf8");
await writeFile(path.join(repo, "tools", "hwpod-node.test.ts"), "console.log('test-only source file without shebang');\n", "utf8");
@@ -315,7 +318,7 @@ async function createLocalGitBundle(context: SelfTestContext, repoName = "bundle
"Use hwpod-ctl for HWPOD runtime inspection and control-plane state.",
].join("\n"), "utf8");
await writeFile(path.join(repo, "skills", "hwpod-ctl", "scripts", "hwpod-ctl.mjs"), "console.log(JSON.stringify({ ok: true, cli: 'hwpod-ctl-skill-selftest' }));\n", "utf8");
await execFile("git", ["add", "README.md", "tools/hwpod", "tools/hwpod-cli.ts", "tools/src/hwpod-harness-lib.ts", "tools/hwpod-node.test.ts", "internal/agent/prompts/hwlab-v02-runtime.md", "skills/dad-dev/SKILL.md", "skills/hwpod-cli/SKILL.md", "skills/hwpod-cli/scripts/hwpod-cli.mjs", "skills/hwpod-ctl/SKILL.md", "skills/hwpod-ctl/scripts/hwpod-ctl.mjs"], { cwd: repo });
await execFile("git", ["add", "README.md", "tools/hwpod", "tools/tran", "tools/trans", "tools/apply_patch", "tools/hwpod-cli.ts", "tools/src/hwpod-harness-lib.ts", "tools/hwpod-node.test.ts", "internal/agent/prompts/hwlab-v02-runtime.md", "skills/dad-dev/SKILL.md", "skills/hwpod-cli/SKILL.md", "skills/hwpod-cli/scripts/hwpod-cli.mjs", "skills/hwpod-ctl/SKILL.md", "skills/hwpod-ctl/scripts/hwpod-ctl.mjs"], { cwd: repo });
await execFile("git", ["-c", "user.email=selftest@example.invalid", "-c", "user.name=AgentRun SelfTest", "commit", "-m", "bundle selftest"], { cwd: repo });
const { stdout } = await execFile("git", ["rev-parse", "HEAD"], { cwd: repo });
return { repoUrl: repo, commitId: stdout.trim(), requiredSkills: [{ name: "dad-dev" }] };
src/selftest/cases/55-timeout-liveness.ts
+14 -10
View File
@@ -33,18 +33,21 @@ const selfTest: SelfTestCase = async (context: SelfTestContext) => {
await sleep(36);
const inactiveLive = (await commandResult(client, inactive)).liveness as JsonRecord;
assert.equal(inactiveLive.phase, "runner-stdio-inactive");
assert.ok(["approaching-hard-timeout", "overdue"].includes(String((inactiveLive.timeoutBudget as JsonRecord).state)));
assert.ok(["approaching-idle-timeout", "overdue"].includes(String((inactiveLive.timeoutBudget as JsonRecord).state)));
assert.equal(((inactiveLive.timeoutBudget as JsonRecord).timeoutKind), "idle");
const terminal = await createActiveRun(client, context, "timeout-liveness-terminal", 50);
await client.post(`/api/v1/runs/${terminal.runId}/events`, { type: "error", payload: { commandId: terminal.commandId, failureKind: "backend-timeout", phase: "turn:hard-timeout", message: "codex stdio turn hard timed out after 50ms" } });
await client.patch(`/api/v1/commands/${terminal.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
await client.patch(`/api/v1/runs/${terminal.runId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
await client.post(`/api/v1/runs/${terminal.runId}/events`, { type: "error", payload: { commandId: terminal.commandId, failureKind: "backend-timeout", phase: "turn:idle-timeout", message: "codex stdio turn idle timed out after 50ms without activity" } });
await client.patch(`/api/v1/commands/${terminal.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn idle timed out after 50ms without activity" });
await client.patch(`/api/v1/runs/${terminal.runId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn idle timed out after 50ms without activity" });
const terminalResult = await commandResult(client, terminal);
const terminalLive = terminalResult.liveness as JsonRecord;
assert.equal(terminalResult.terminalStatus, "failed");
assert.equal(terminalLive.phase, "terminal");
assert.equal(((terminalLive.timeoutBudget as JsonRecord).state), "timed-out");
assert.equal(((terminalResult.terminalClassification as JsonRecord).category), "execution-hard-timeout");
assert.equal(((terminalLive.timeoutBudget as JsonRecord).timeoutKind), "idle");
assert.equal(((terminalResult.terminalClassification as JsonRecord).category), "execution-idle-timeout");
assert.equal(((terminalResult.terminalClassification as JsonRecord).idleTimeout), true);
assert.equal(((terminalResult.terminalClassification as JsonRecord).providerEvidence), "insufficient");
assert.equal(((terminalLive.terminalClassification as JsonRecord).providerInterruptionKnown), false);
assert.ok((terminalLive.recoveryActions as JsonRecord[]).some((action) => action.action === "resume-session"));
@@ -52,13 +55,13 @@ const selfTest: SelfTestCase = async (context: SelfTestContext) => {
const noSession = await createActiveRun(client, context, "timeout-liveness-no-session", 50, { session: false });
await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "backend_status", payload: { commandId: noSession.commandId, phase: "codex-app-server-closed", message: "stdio closed before terminal result" } });
await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "terminal_status", payload: { commandId: noSession.commandId, terminalStatus: "failed", failureKind: "backend-timeout", message: "codex stdio turn hard timed out after 50ms" } });
await client.patch(`/api/v1/commands/${noSession.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "terminal_status", payload: { commandId: noSession.commandId, terminalStatus: "failed", failureKind: "backend-timeout", message: "codex stdio turn idle timed out after 50ms without activity" } });
await client.patch(`/api/v1/commands/${noSession.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn idle timed out after 50ms without activity" });
const noSessionResult = await commandResult(client, noSession);
const noSessionLive = noSessionResult.liveness as JsonRecord;
const noSessionClassification = noSessionResult.terminalClassification as JsonRecord;
const noSessionDiagnosis = noSessionResult.diagnosis as JsonRecord;
assert.equal(noSessionClassification.category, "execution-hard-timeout");
assert.equal(noSessionClassification.category, "execution-idle-timeout");
assert.equal(noSessionClassification.providerEvidence, "observed-transport-disconnect");
assert.equal(noSessionClassification.providerInterruptionKnown, false);
assert.equal(noSessionDiagnosis.category, "terminal-command-open-run");
@@ -111,12 +114,13 @@ const selfTest: SelfTestCase = async (context: SelfTestContext) => {
const commander = await client.get("/api/v1/queue/commander?queue=timeout-liveness&readerId=timeout-liveness") as JsonRecord;
const commanderItem = ((commander.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord;
assert.equal(((commanderItem.supervisor as JsonRecord).phase), "terminal");
assert.equal((((commanderItem.supervisor as JsonRecord).diagnosis as JsonRecord).category), "execution-hard-timeout");
assert.equal((((commanderItem.supervisor as JsonRecord).diagnosis as JsonRecord).category), "execution-idle-timeout");
assert.equal((((commanderItem.supervisor as JsonRecord).timeoutBudget as JsonRecord).state), "timed-out");
assert.equal((((commanderItem.supervisor as JsonRecord).timeoutBudget as JsonRecord).timeoutKind), "idle");
const commanderSummary = summarizeQueueCommanderSnapshot(commander, { limit: 5 });
const summaryItem = ((commanderSummary.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord;
assert.equal(((summaryItem.supervisor as JsonRecord).phase), "terminal");
assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).category), "execution-hard-timeout");
assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).category), "execution-idle-timeout");
assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).providerEvidence), "insufficient");
assert.equal(JSON.stringify(commanderSummary).includes("hwpod workspace apply-patch"), false, "commander summary must stay compact and avoid dumping command bodies");
assert.equal(JSON.stringify(summaryItem).includes("fullRecordBytes"), false, "commander item must not add bookkeeping noise");
src/selftest/cases/76-aipod-spec.ts
+3
View File
@@ -38,6 +38,9 @@ const selfTest: SelfTestCase = async (context) => {
assert.equal(task.backendProfile, "sub2api");
assert.equal(task.providerId, "G14");
assert.equal(task.idempotencyKey, "selftest-aipod-artificer");
const sessionRef = task.sessionRef as JsonRecord;
assert.match(String(sessionRef.sessionId), /^sess_artificer_[a-f0-9]{24}$/u, "Artificer queue task should default to a resumable session");
assert.equal(sessionRef.conversationId, sessionRef.sessionId, "default Artificer conversation should match the generated session");
const taskImageRef = ((task.metadata as JsonRecord).aipodImageRef as JsonRecord);
assert.equal(taskImageRef.kind, "env-image-dockerfile");
assert.equal(taskImageRef.valuesPrinted, false);
src/selftest/fake-codex-app-server.ts
+12 -8
View File
@@ -262,17 +262,21 @@ for await (const line of rl) {
activeSteerTurn = { id: turn.id, completed: false, timer: setTimeout(() => undefined, 60_000) };
continue;
}
if (mode === "hard-timeout-tool-progress") {
if (mode === "tool-progress-refreshes-idle") {
turnCounter += 1;
const turn = { id: `turn_selftest_${turnCounter}`, status: "running" };
notify("turn/started", { turn });
notify("item/started", { item: { id: "tool_hard_timeout", type: "commandExecution", command: "hwpod cmd long-running", status: "running", processId: process.pid } });
const turn = { id: `turn_selftest_${turnCounter}`, status: "completed" };
notify("turn/started", { turn: { id: turn.id, status: "running" } });
notify("item/started", { item: { id: "tool_idle_refresh", type: "commandExecution", command: "hwpod cmd long-running", status: "running", processId: process.pid } });
respond(message.id, { turn });
activeSteerTurn = { id: turn.id, completed: false, timer: null };
let ticks = 0;
activeSteerTurn.timer = setInterval(() => {
const timer = setInterval(() => {
ticks += 1;
notify("item/commandExecution/outputDelta", { itemId: "tool_hard_timeout", delta: `progress ${ticks}\n` });
notify("item/commandExecution/outputDelta", { itemId: "tool_idle_refresh", delta: `progress ${ticks}\n` });
if (ticks >= 4) {
clearInterval(timer);
notify("item/completed", { item: { id: "tool_idle_refresh", type: "commandExecution", command: "hwpod cmd long-running", status: "completed" } });
notify("turn/completed", { turn });
}
}, 25);
continue;
}
@@ -329,7 +333,7 @@ for await (const line of rl) {
continue;
}
if (message.method === "turn/interrupt") {
if ((mode !== "tool-hangs-before-turn-start-response" && mode !== "hard-timeout-tool-progress" && mode !== "steer-waits") || !activeSteerTurn) {
if ((mode !== "tool-hangs-before-turn-start-response" && mode !== "steer-waits") || !activeSteerTurn) {
respond(message.id, null, { code: -32000, message: "no active fake turn for interrupt" });
continue;
}