mirrors/pikasTech-agentrun
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 支持长工具调用 interrupt 与硬超时回收

Browse Source
This commit is contained in:
Codex
2026-06-10 00:25:54 +08:00
parent da218772ea
commit ff2b5dce34
4 changed files with 213 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/selftest/cases/30-codex-stdio.ts
+54 -2
View File
@@ -181,6 +181,9 @@ const selfTest: SelfTestCase = async (context) => {
const liveResult = await livePromise;
assert.equal(liveResult.terminalStatus, "completed", "slow live tool event turn should complete");
await runInterruptBeforeTurnStartResponseCase({ client, managerUrl: server.baseUrl, context });
await runHardTimeoutDuringToolProgressCase({ client, managerUrl: server.baseUrl, context });
const noisy = await createRunWithCommand(client, context, "hello noisy reasoning", "selftest-noisy-reasoning-events", 15_000);
const noisyResult = await runOnce({ managerUrl: server.baseUrl, runId: noisy.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "noisy-reasoning-events" }, oneShot: true }) as JsonRecord;
assert.equal(noisyResult.terminalStatus, "completed", "noisy reasoning turn should complete");
@@ -239,7 +242,7 @@ const selfTest: SelfTestCase = async (context) => {
await runSessionStorageSubdirCase({ client, managerUrl: server.baseUrl, context });
await runSessionStorageNoSecretLeakCase({ client, managerUrl: server.baseUrl, context });
return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "runner-lease-conflict-recovery", "codex-stdio-fake-turn", "codex-stdio-k8s-sandbox-override", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-dsflash-go-profile-fake-turn", "codex-stdio-dsflash-go-config-metadata", "codex-stdio-minimax-m3-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-minimax-m3-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-final-agent-message-only", "codex-stdio-web-search-progress", "codex-stdio-stale-thread-resume-failed", "codex-stdio-live-tool-events", "codex-stdio-noisy-reasoning-suppression", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-invalid-tool-call", "codex-stdio-provider-compact-unsupported", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-503-retry-event", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-idle-timeout-progress-refresh", "codex-stdio-command-failure-keeps-run-open", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "runner-lease-conflict-recovery", "codex-stdio-fake-turn", "codex-stdio-k8s-sandbox-override", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-dsflash-go-profile-fake-turn", "codex-stdio-dsflash-go-config-metadata", "codex-stdio-minimax-m3-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-minimax-m3-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-final-agent-message-only", "codex-stdio-web-search-progress", "codex-stdio-stale-thread-resume-failed", "codex-stdio-live-tool-events", "codex-stdio-interrupt-before-turn-start-response", "codex-stdio-hard-timeout-during-tool-progress", "codex-stdio-noisy-reasoning-suppression", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-invalid-tool-call", "codex-stdio-provider-compact-unsupported", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-503-retry-event", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-idle-timeout-progress-refresh", "codex-stdio-command-failure-keeps-run-open", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
} finally {
await new Promise<void>((resolve) => server.server.close(() => resolve()));
}
@@ -281,7 +284,7 @@ async function runLeaseConflictRecoveryCase(options: { client: ManagerClient; ma
}
async function runSlowProgressIdleCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 60);
const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 250);
const result = await runOnce({
managerUrl: options.managerUrl,
runId: item.runId,
@@ -296,6 +299,55 @@ async function runSlowProgressIdleCase(options: { client: ManagerClient; manager
assert.equal(events.items?.some((event) => event.type === "error" && eventPayload(event).failureKind === "backend-timeout"), false, "progressing turns must not fail on total elapsed time");
}
async function runInterruptBeforeTurnStartResponseCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
const item = await createRunWithCommand(options.client, options.context, "interrupt hanging tool before turn/start response", "selftest-interrupt-before-turn-start-response", 1_000);
const runPromise = runOnce({
managerUrl: options.managerUrl,
runId: item.runId,
codexCommand: options.context.fakeCodexCommand,
codexArgs: options.context.fakeCodexArgs,
codexHome: options.context.codexHome,
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-hangs-before-turn-start-response" },
oneShot: true,
}) as Promise<JsonRecord>;
await waitForEvent(options.client, item.runId, (event) => event.type === "tool_call" && eventPayload(event).itemId === "tool_hang_before_response", "tool_call before turn/start response");
const interrupt = await options.client.post(`/api/v1/runs/${item.runId}/commands`, { type: "interrupt", payload: { reason: "self-test interrupt hanging tool" }, idempotencyKey: "selftest-interrupt-before-response-command" }) as { id: string };
const result = await runPromise;
assert.equal(result.terminalStatus, "cancelled", "interrupt should cancel the active turn even if turn/start response has not returned");
assert.equal(result.failureKind, "cancelled");
const interruptCommand = await options.client.get(`/api/v1/runs/${item.runId}/commands/${interrupt.id}`) as { state?: string };
assert.equal(interruptCommand.state, "completed", "interrupt control command should be terminal completed after delivery");
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "active-turn-control-ready"), "turn/started notification should expose active turn control before turn/start response");
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "interrupt-command-acknowledged"), "interrupt command acknowledgement should be visible");
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/interrupt:completed"), "interrupt delivery result should be visible");
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/start:interrupted-before-response"), "turn/start pending request should be bypassed after interrupt terminal");
assertNoSecretLeak({ result, events });
}
async function runHardTimeoutDuringToolProgressCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
const item = await createRunWithCommand(options.client, options.context, "hard timeout during tool progress", "selftest-hard-timeout-tool-progress", 120);
const result = await runOnce({
managerUrl: options.managerUrl,
runId: item.runId,
codexCommand: options.context.fakeCodexCommand,
codexArgs: options.context.fakeCodexArgs,
codexHome: options.context.codexHome,
env: { CODEX_HOME: options.context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "hard-timeout-tool-progress" },
oneShot: true,
}) as JsonRecord;
assert.equal(result.terminalStatus, "failed", "hard timeout should fail even while the tool keeps producing progress");
assert.equal(result.failureKind, "backend-timeout");
const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
assert.ok(events.items?.some((event) => event.type === "command_output" && String(eventPayload(event).text ?? "").includes("progress")), "progress output should be visible before hard timeout");
assert.ok(events.items?.some((event) => event.type === "error" && eventPayload(event).phase === "turn:hard-timeout"), "hard timeout should be recorded as an error event");
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn-interrupt-requested"), "timeout should request backend interrupt before process teardown");
assert.ok(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "turn/interrupt:completed"), "timeout interrupt result should be visible");
const command = await options.client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}`) as { state?: string };
assert.equal(command.state, "failed", "hard timed out command should be failed");
assertNoSecretLeak({ result, events });
}
async function runFailureDoesNotTerminalRunCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
const item = await createRunWithCommand(options.client, options.context, "first command fails", "selftest-command-failure-keeps-run-open", 3_000);
const result = await runOnce({
src/selftest/fake-codex-app-server.ts
+35 -2
View File
@@ -224,6 +224,29 @@ for await (const line of rl) {
}, 50);
continue;
}
if (mode === "tool-hangs-before-turn-start-response") {
turnCounter += 1;
const turn = { id: `turn_selftest_${turnCounter}`, status: "running" };
notify("turn/started", { turn });
notify("item/started", { item: { id: "tool_hang_before_response", type: "commandExecution", command: "hwpod cmd git clone", status: "running", processId: process.pid } });
notify("item/commandExecution/outputDelta", { itemId: "tool_hang_before_response", delta: "clone started\n" });
activeSteerTurn = { id: turn.id, completed: false, timer: setTimeout(() => undefined, 60_000) };
continue;
}
if (mode === "hard-timeout-tool-progress") {
turnCounter += 1;
const turn = { id: `turn_selftest_${turnCounter}`, status: "running" };
notify("turn/started", { turn });
notify("item/started", { item: { id: "tool_hard_timeout", type: "commandExecution", command: "hwpod cmd long-running", status: "running", processId: process.pid } });
respond(message.id, { turn });
activeSteerTurn = { id: turn.id, completed: false, timer: null };
let ticks = 0;
activeSteerTurn.timer = setInterval(() => {
ticks += 1;
notify("item/commandExecution/outputDelta", { itemId: "tool_hard_timeout", delta: `progress ${ticks}\n` });
}, 25);
continue;
}
if (mode === "steer-waits") {
turnCounter += 1;
const turn = { id: `turn_selftest_${turnCounter}`, status: "running" };
@@ -276,6 +299,16 @@ for await (const line of rl) {
setTimeout(() => completeActiveSteerTurn("steer-applied"), 20);
continue;
}
if (message.method === "turn/interrupt") {
if ((mode !== "tool-hangs-before-turn-start-response" && mode !== "hard-timeout-tool-progress" && mode !== "steer-waits") || !activeSteerTurn) {
respond(message.id, null, { code: -32000, message: "no active fake turn for interrupt" });
continue;
}
notify("item/completed", { item: { id: "tool_interrupted", type: "commandExecution", command: "hwpod cmd interrupted", status: "cancelled" } });
respond(message.id, { interrupted: true });
setTimeout(() => completeActiveSteerTurn("interrupt-applied", "cancelled"), 20);
continue;
}
respond(message.id, null, { code: -32601, message: `unsupported fake method ${message.method ?? "unknown"}` });
}
@@ -288,11 +321,11 @@ function notify(method: string, params: unknown): void {
process.stdout.write(`${JSON.stringify({ method, params })}\n`);
}
function completeActiveSteerTurn(reason: string): void {
function completeActiveSteerTurn(reason: string, status = "completed"): void {
if (!activeSteerTurn || activeSteerTurn.completed) return;
activeSteerTurn.completed = true;
if (activeSteerTurn.timer) clearTimeout(activeSteerTurn.timer);
const turn = { id: activeSteerTurn.id, status: "completed", reason };
const turn = { id: activeSteerTurn.id, status, reason };
notify("turn/completed", { turn });
}