AgentRun Codex
158 lines
12 KiB
TypeScript
158 lines
12 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
import { ManagerClient } from "../../mgr/client.js";
|
|
import { startManagerServer } from "../../mgr/server.js";
|
|
import { MemoryAgentRunStore } from "../../mgr/store.js";
|
|
import type { JsonRecord } from "../../common/types.js";
|
|
import { assertNoSecretLeak, type SelfTestCase, type SelfTestContext } from "../harness.js";
|
|
import { summarizeQueueCommanderSnapshot } from "../../../scripts/src/cli.js";
|
|
|
|
const selfTest: SelfTestCase = async (context: SelfTestContext) => {
|
|
const store = new MemoryAgentRunStore();
|
|
const server = await startManagerServer({ port: 0, host: "127.0.0.1", sourceCommit: "self-test", store });
|
|
try {
|
|
const client = new ManagerClient(server.baseUrl);
|
|
|
|
const tool = await createActiveRun(client, context, "timeout-liveness-tool", 120_000);
|
|
await client.post(`/api/v1/runs/${tool.runId}/events`, { type: "tool_call", payload: { commandId: tool.commandId, itemId: "tool_live", toolName: "commandExecution", status: "running", command: "hwpod workspace apply-patch" } });
|
|
const toolResult = await commandResult(client, tool);
|
|
const toolLive = toolResult.liveness as JsonRecord;
|
|
assert.equal(toolLive.phase, "waiting-tool");
|
|
assert.equal(((toolLive.lastActivity as JsonRecord).activityKind), "tool-in-flight");
|
|
assert.equal(((toolLive.lastActivity as JsonRecord).sourceSeq), 4);
|
|
assert.equal(((toolLive.timeoutBudget as JsonRecord).state), "within-budget");
|
|
assert.ok(Array.isArray(toolLive.recoveryActions));
|
|
|
|
const assistant = await createActiveRun(client, context, "timeout-liveness-assistant", 120_000);
|
|
await client.post(`/api/v1/runs/${assistant.runId}/events`, { type: "assistant_message", payload: { commandId: assistant.commandId, itemId: "msg_progress", progress: true, text: "正在生成 apply-patch，先汇总待改文件。" } });
|
|
const assistantLive = (await commandResult(client, assistant)).liveness as JsonRecord;
|
|
assert.equal(assistantLive.phase, "waiting-model-output");
|
|
assert.equal(((assistantLive.lastActivity as JsonRecord).activityKind), "assistant-progress");
|
|
|
|
const inactive = await createActiveRun(client, context, "timeout-liveness-inactive", 40);
|
|
await sleep(36);
|
|
const inactiveLive = (await commandResult(client, inactive)).liveness as JsonRecord;
|
|
assert.equal(inactiveLive.phase, "runner-stdio-inactive");
|
|
assert.ok(["approaching-hard-timeout", "overdue"].includes(String((inactiveLive.timeoutBudget as JsonRecord).state)));
|
|
|
|
const terminal = await createActiveRun(client, context, "timeout-liveness-terminal", 50);
|
|
await client.post(`/api/v1/runs/${terminal.runId}/events`, { type: "error", payload: { commandId: terminal.commandId, failureKind: "backend-timeout", phase: "turn:hard-timeout", message: "codex stdio turn hard timed out after 50ms" } });
|
|
await client.patch(`/api/v1/commands/${terminal.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
|
|
await client.patch(`/api/v1/runs/${terminal.runId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
|
|
const terminalResult = await commandResult(client, terminal);
|
|
const terminalLive = terminalResult.liveness as JsonRecord;
|
|
assert.equal(terminalResult.terminalStatus, "failed");
|
|
assert.equal(terminalLive.phase, "terminal");
|
|
assert.equal(((terminalLive.timeoutBudget as JsonRecord).state), "timed-out");
|
|
assert.equal(((terminalResult.terminalClassification as JsonRecord).category), "execution-hard-timeout");
|
|
assert.equal(((terminalResult.terminalClassification as JsonRecord).providerEvidence), "insufficient");
|
|
assert.equal(((terminalLive.terminalClassification as JsonRecord).providerInterruptionKnown), false);
|
|
assert.ok((terminalLive.recoveryActions as JsonRecord[]).some((action) => action.action === "resume-session"));
|
|
assert.ok((terminalLive.recoveryActions as JsonRecord[]).some((action) => action.action === "split-task"));
|
|
|
|
const noSession = await createActiveRun(client, context, "timeout-liveness-no-session", 50, { session: false });
|
|
await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "backend_status", payload: { commandId: noSession.commandId, phase: "codex-app-server-closed", message: "stdio closed before terminal result" } });
|
|
await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "terminal_status", payload: { commandId: noSession.commandId, terminalStatus: "failed", failureKind: "backend-timeout", message: "codex stdio turn hard timed out after 50ms" } });
|
|
await client.patch(`/api/v1/commands/${noSession.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
|
|
await client.patch(`/api/v1/runs/${noSession.runId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
|
|
const noSessionResult = await commandResult(client, noSession);
|
|
const noSessionLive = noSessionResult.liveness as JsonRecord;
|
|
const noSessionClassification = noSessionResult.terminalClassification as JsonRecord;
|
|
assert.equal(noSessionClassification.category, "execution-hard-timeout");
|
|
assert.equal(noSessionClassification.providerEvidence, "observed-transport-disconnect");
|
|
assert.equal(noSessionClassification.providerInterruptionKnown, false);
|
|
assert.match(String(noSessionClassification.providerInterruptionReason), /cannot distinguish provider outage/u);
|
|
assert.equal((noSessionLive.transportDisconnect as JsonRecord).sourceSeq, 4);
|
|
assert.equal((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "resume-session"), false, "sessionId=null must not suggest session-only resume");
|
|
assert.equal((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "poll-output"), false, "sessionId=null must not suggest session output path");
|
|
assert.ok((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "poll-trace" && String(action.command).includes("runs events")));
|
|
|
|
assert.ok(terminal.sessionId, "terminal fixture must have a session id");
|
|
const terminalSessionId = terminal.sessionId;
|
|
const session = await client.get(`/api/v1/sessions/${terminalSessionId}?readerId=timeout-liveness`) as JsonRecord;
|
|
assert.equal(((session.liveness as JsonRecord).phase), "terminal");
|
|
assert.ok(Array.isArray(((session.supervisor as JsonRecord).recoveryActions)), "session show must keep terminal recovery actions");
|
|
|
|
const task = await client.post("/api/v1/queue/tasks", queueTask(context, terminalSessionId, 50)) as JsonRecord;
|
|
store.updateQueueTaskAttempt(String(task.id), {
|
|
state: "running",
|
|
latestAttempt: { attemptId: "attempt_timeout_liveness", state: "running", runId: terminal.runId, commandId: terminal.commandId, runnerJobId: null, sessionId: terminalSessionId, sessionPath: `/api/v1/sessions/${terminalSessionId}` },
|
|
sessionPath: `/api/v1/sessions/${terminalSessionId}`,
|
|
});
|
|
const commander = await client.get("/api/v1/queue/commander?queue=timeout-liveness&readerId=timeout-liveness") as JsonRecord;
|
|
const commanderItem = ((commander.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord;
|
|
assert.equal(((commanderItem.supervisor as JsonRecord).phase), "terminal");
|
|
assert.equal((((commanderItem.supervisor as JsonRecord).timeoutBudget as JsonRecord).state), "timed-out");
|
|
const commanderSummary = summarizeQueueCommanderSnapshot(commander, { limit: 5 });
|
|
const summaryItem = ((commanderSummary.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord;
|
|
assert.equal(((summaryItem.supervisor as JsonRecord).phase), "terminal");
|
|
assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).category), "execution-hard-timeout");
|
|
assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).providerEvidence), "insufficient");
|
|
assert.equal(JSON.stringify(commanderSummary).includes("hwpod workspace apply-patch"), false, "commander summary must stay compact and avoid dumping command bodies");
|
|
assert.equal(JSON.stringify(summaryItem).includes("fullRecordBytes"), false, "commander item must not add bookkeeping noise");
|
|
assertNoSecretLeak({ toolResult, assistantLive, inactiveLive, terminalResult, noSessionResult, session, commanderSummary });
|
|
|
|
return { name: "timeout-liveness", tests: ["tool-in-flight-liveness", "assistant-progress-liveness", "stdio-inactive-timeout-budget", "terminal-timeout-recovery", "no-session-drilldown", "terminal-classification", "queue-commander-supervisor"] };
|
|
} finally {
|
|
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
|
}
|
|
};
|
|
|
|
async function createActiveRun(client: ManagerClient, context: SelfTestContext, sessionSuffix: string, timeoutMs: number, options: { session?: boolean } = {}): Promise<{ runId: string; commandId: string; sessionId: string | null }> {
|
|
const sessionId = `selftest-${sessionSuffix}`;
|
|
const run = await client.post("/api/v1/runs", runBody(context, options.session === false ? null : sessionId, timeoutMs)) as JsonRecord;
|
|
const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt: sessionSuffix }, idempotencyKey: sessionSuffix }) as JsonRecord;
|
|
await client.post(`/api/v1/runs/${run.id}/claim`, { runnerId: `runner_${sessionSuffix}`, leaseMs: 60_000 });
|
|
await client.post(`/api/v1/commands/${command.id}/ack`, {});
|
|
return { runId: String(run.id), commandId: String(command.id), sessionId: options.session === false ? null : sessionId };
|
|
}
|
|
|
|
async function commandResult(client: ManagerClient, item: { runId: string; commandId: string }): Promise<JsonRecord> {
|
|
return await client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}/result`) as JsonRecord;
|
|
}
|
|
|
|
function runBody(context: SelfTestContext, sessionId: string | null, timeoutMs: number): JsonRecord {
|
|
return {
|
|
tenantId: "unidesk",
|
|
projectId: "pikasTech/agentrun",
|
|
workspaceRef: { kind: "host-path", path: context.workspace },
|
|
sessionRef: sessionId ? { sessionId, conversationId: sessionId } : null,
|
|
providerId: "G14",
|
|
backendProfile: "codex",
|
|
executionPolicy: executionPolicy(timeoutMs, context.codexHome),
|
|
traceSink: null,
|
|
};
|
|
}
|
|
|
|
function queueTask(context: SelfTestContext, sessionId: string, timeoutMs: number): JsonRecord {
|
|
return {
|
|
tenantId: "unidesk",
|
|
projectId: "pikasTech/agentrun",
|
|
queue: "timeout-liveness",
|
|
lane: "selftest",
|
|
title: "timeout liveness commander",
|
|
priority: 1,
|
|
backendProfile: "codex",
|
|
providerId: "G14",
|
|
workspaceRef: { kind: "host-path", path: context.workspace },
|
|
sessionRef: { sessionId, conversationId: sessionId },
|
|
executionPolicy: executionPolicy(timeoutMs, context.codexHome),
|
|
resourceBundleRef: null,
|
|
payload: { prompt: "timeout liveness commander" },
|
|
references: [],
|
|
metadata: {},
|
|
};
|
|
}
|
|
|
|
function executionPolicy(timeoutMs: number, codexHome: string): JsonRecord {
|
|
return {
|
|
sandbox: "workspace-write",
|
|
approval: "never",
|
|
timeoutMs,
|
|
network: "default",
|
|
secretScope: { allowCredentialEcho: false, providerCredentials: [{ profile: "codex", secretRef: { name: "agentrun-v01-provider-codex", keys: ["auth.json", "config.toml"], mountPath: codexHome } }] },
|
|
};
|
|
}
|
|
|
|
export default selfTest;
|