import assert from "node:assert/strict"; import { setTimeout as sleep } from "node:timers/promises"; import { ManagerClient } from "../../mgr/client.js"; import { startManagerServer } from "../../mgr/server.js"; import { MemoryAgentRunStore } from "../../mgr/store.js"; import type { JsonRecord } from "../../common/types.js"; import { assertNoSecretLeak, type SelfTestCase, type SelfTestContext } from "../harness.js"; import { summarizeQueueCommanderSnapshot } from "../../../scripts/src/cli.js"; const selfTest: SelfTestCase = async (context: SelfTestContext) => { const store = new MemoryAgentRunStore(); const server = await startManagerServer({ port: 0, host: "127.0.0.1", sourceCommit: "self-test", store }); try { const client = new ManagerClient(server.baseUrl); const tool = await createActiveRun(client, context, "timeout-liveness-tool", 120_000); await client.post(`/api/v1/runs/${tool.runId}/events`, { type: "tool_call", payload: { commandId: tool.commandId, itemId: "tool_live", toolName: "commandExecution", status: "running", command: "hwpod workspace apply-patch" } }); const toolResult = await commandResult(client, tool); const toolLive = toolResult.liveness as JsonRecord; assert.equal(toolLive.phase, "waiting-tool"); assert.equal(((toolLive.lastActivity as JsonRecord).activityKind), "tool-in-flight"); assert.equal(((toolLive.lastActivity as JsonRecord).sourceSeq), 4); assert.equal(((toolLive.timeoutBudget as JsonRecord).state), "within-budget"); assert.ok(Array.isArray(toolLive.recoveryActions)); const assistant = await createActiveRun(client, context, "timeout-liveness-assistant", 120_000); await client.post(`/api/v1/runs/${assistant.runId}/events`, { type: "assistant_message", payload: { commandId: assistant.commandId, itemId: "msg_progress", progress: true, text: "正在生成 apply-patch,先汇总待改文件。" } }); const assistantLive = (await commandResult(client, assistant)).liveness as JsonRecord; assert.equal(assistantLive.phase, "waiting-model-output"); assert.equal(((assistantLive.lastActivity as JsonRecord).activityKind), "assistant-progress"); const inactive = await createActiveRun(client, context, "timeout-liveness-inactive", 40); await sleep(36); const inactiveLive = (await commandResult(client, inactive)).liveness as JsonRecord; assert.equal(inactiveLive.phase, "runner-stdio-inactive"); assert.ok(["approaching-idle-timeout", "overdue"].includes(String((inactiveLive.timeoutBudget as JsonRecord).state))); assert.equal(((inactiveLive.timeoutBudget as JsonRecord).timeoutKind), "idle"); const terminal = await createActiveRun(client, context, "timeout-liveness-terminal", 50); await client.post(`/api/v1/runs/${terminal.runId}/events`, { type: "error", payload: { commandId: terminal.commandId, failureKind: "backend-timeout", phase: "turn:idle-timeout", message: "codex stdio turn idle timed out after 50ms without activity" } }); await client.patch(`/api/v1/commands/${terminal.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn idle timed out after 50ms without activity" }); await client.patch(`/api/v1/runs/${terminal.runId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn idle timed out after 50ms without activity" }); const terminalResult = await commandResult(client, terminal); const terminalLive = terminalResult.liveness as JsonRecord; assert.equal(terminalResult.terminalStatus, "failed"); assert.equal(terminalLive.phase, "terminal"); assert.equal(((terminalLive.timeoutBudget as JsonRecord).state), "timed-out"); assert.equal(((terminalLive.timeoutBudget as JsonRecord).timeoutKind), "idle"); assert.equal(((terminalResult.terminalClassification as JsonRecord).category), "execution-idle-timeout"); assert.equal(((terminalResult.terminalClassification as JsonRecord).idleTimeout), true); assert.equal(((terminalResult.terminalClassification as JsonRecord).providerEvidence), "insufficient"); assert.equal(((terminalLive.terminalClassification as JsonRecord).providerInterruptionKnown), false); assert.ok((terminalLive.recoveryActions as JsonRecord[]).some((action) => action.action === "resume-session")); assert.ok((terminalLive.recoveryActions as JsonRecord[]).some((action) => action.action === "split-task")); const noSession = await createActiveRun(client, context, "timeout-liveness-no-session", 50, { session: false }); await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "backend_status", payload: { commandId: noSession.commandId, phase: "codex-app-server-closed", message: "stdio closed before terminal result" } }); await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "terminal_status", payload: { commandId: noSession.commandId, terminalStatus: "failed", failureKind: "backend-timeout", message: "codex stdio turn idle timed out after 50ms without activity" } }); await client.patch(`/api/v1/commands/${noSession.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn idle timed out after 50ms without activity" }); const noSessionResult = await commandResult(client, noSession); const noSessionLive = noSessionResult.liveness as JsonRecord; const noSessionClassification = noSessionResult.terminalClassification as JsonRecord; const noSessionDiagnosis = noSessionResult.diagnosis as JsonRecord; assert.equal(noSessionClassification.category, "execution-idle-timeout"); assert.equal(noSessionClassification.providerEvidence, "observed-transport-disconnect"); assert.equal(noSessionClassification.providerInterruptionKnown, false); assert.equal(noSessionDiagnosis.category, "terminal-command-open-run"); assert.equal(noSessionDiagnosis.providerEvidence, "observed-transport-disconnect"); assert.equal(((noSessionDiagnosis.session as JsonRecord).sessionRefNull), true); assert.ok((noSessionDiagnosis.recoveryActions as JsonRecord[]).some((action) => action.action === "session-unavailable")); assert.match(String(noSessionClassification.providerInterruptionReason), /cannot distinguish provider outage/u); assert.equal((noSessionLive.transportDisconnect as JsonRecord).sourceSeq, 4); assert.equal((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "resume-session"), false, "sessionId=null must not suggest session-only resume"); assert.equal((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "poll-output"), false, "sessionId=null must not suggest session output path"); assert.ok((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "poll-trace" && String(action.command).includes("runs events"))); const stale = await createActiveRun(client, context, "timeout-liveness-stale-claimed", 120_000, { session: false, leaseMs: 1 }); await store.saveRunnerJob({ runId: stale.runId, commandId: stale.commandId, idempotencyKey: "stale-runner-lost", payloadHash: "hash-stale-runner-lost", attemptId: "attempt_stale_runner_lost", runnerId: "runner_stale_claimed", namespace: "agentrun-v01", jobName: "agentrun-v01-runner-stale", managerUrl: server.baseUrl, image: "127.0.0.1:5000/agentrun/selftest:stale", sourceCommit: "self-test", serviceAccountName: "agentrun-v01-runner", result: { runner: { logPath: "kubectl -n agentrun-v01 logs job/agentrun-v01-runner-stale" }, kubernetes: { created: true, valuesPrinted: false }, valuesPrinted: false }, }); await sleep(5); const staleResult = await commandResult(client, stale); const staleDiagnosis = staleResult.diagnosis as JsonRecord; assert.equal(staleDiagnosis.category, "runner-lost"); assert.equal(staleDiagnosis.staleClaimed, true); assert.equal(staleDiagnosis.runnerLost, true); assert.equal(((staleDiagnosis.runnerJob as JsonRecord).phase), "created"); assert.equal(((staleDiagnosis.session as JsonRecord).sessionRefNull), true); assert.ok(terminal.sessionId, "terminal fixture must have a session id"); const terminalSessionId = terminal.sessionId; const session = await client.get(`/api/v1/sessions/${terminalSessionId}?readerId=timeout-liveness`) as JsonRecord; assert.equal(((session.liveness as JsonRecord).phase), "terminal"); assert.ok(Array.isArray(((session.supervisor as JsonRecord).recoveryActions)), "session show must keep terminal recovery actions"); const task = await client.post("/api/v1/queue/tasks", queueTask(context, terminalSessionId, 50)) as JsonRecord; store.updateQueueTaskAttempt(String(task.id), { state: "running", latestAttempt: { attemptId: "attempt_timeout_liveness", state: "running", runId: terminal.runId, commandId: terminal.commandId, runnerJobId: null, sessionId: terminalSessionId, sessionPath: `/api/v1/sessions/${terminalSessionId}` }, sessionPath: `/api/v1/sessions/${terminalSessionId}`, }); const commander = await client.get("/api/v1/queue/commander?queue=timeout-liveness&readerId=timeout-liveness") as JsonRecord; const commanderItem = ((commander.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord; assert.equal(((commanderItem.supervisor as JsonRecord).phase), "terminal"); assert.equal((((commanderItem.supervisor as JsonRecord).diagnosis as JsonRecord).category), "execution-idle-timeout"); assert.equal((((commanderItem.supervisor as JsonRecord).timeoutBudget as JsonRecord).state), "timed-out"); assert.equal((((commanderItem.supervisor as JsonRecord).timeoutBudget as JsonRecord).timeoutKind), "idle"); const commanderSummary = summarizeQueueCommanderSnapshot(commander, { limit: 5 }); const summaryItem = ((commanderSummary.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord; assert.equal(((summaryItem.supervisor as JsonRecord).phase), "terminal"); assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).category), "execution-idle-timeout"); assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).providerEvidence), "insufficient"); assert.equal(JSON.stringify(commanderSummary).includes("hwpod workspace apply-patch"), false, "commander summary must stay compact and avoid dumping command bodies"); assert.equal(JSON.stringify(summaryItem).includes("fullRecordBytes"), false, "commander item must not add bookkeeping noise"); assertNoSecretLeak({ toolResult, assistantLive, inactiveLive, terminalResult, noSessionResult, staleResult, session, commanderSummary }); return { name: "timeout-liveness", tests: ["tool-in-flight-liveness", "assistant-progress-liveness", "stdio-inactive-timeout-budget", "terminal-timeout-recovery", "no-session-drilldown", "terminal-classification", "queue-commander-supervisor", "diagnosis-visibility", "stale-claimed-runner-lost"] }; } finally { await new Promise((resolve) => server.server.close(() => resolve())); } }; async function createActiveRun(client: ManagerClient, context: SelfTestContext, sessionSuffix: string, timeoutMs: number, options: { session?: boolean; leaseMs?: number } = {}): Promise<{ runId: string; commandId: string; sessionId: string | null }> { const sessionId = `selftest-${sessionSuffix}`; const run = await client.post("/api/v1/runs", runBody(context, options.session === false ? null : sessionId, timeoutMs)) as JsonRecord; const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt: sessionSuffix }, idempotencyKey: sessionSuffix }) as JsonRecord; await client.post(`/api/v1/runs/${run.id}/claim`, { runnerId: `runner_${sessionSuffix}`, leaseMs: options.leaseMs ?? 60_000 }); await client.post(`/api/v1/commands/${command.id}/ack`, {}); return { runId: String(run.id), commandId: String(command.id), sessionId: options.session === false ? null : sessionId }; } async function commandResult(client: ManagerClient, item: { runId: string; commandId: string }): Promise { return await client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}/result`) as JsonRecord; } function runBody(context: SelfTestContext, sessionId: string | null, timeoutMs: number): JsonRecord { return { tenantId: "unidesk", projectId: "pikasTech/agentrun", workspaceRef: { kind: "host-path", path: context.workspace }, sessionRef: sessionId ? { sessionId, conversationId: sessionId } : null, providerId: "G14", backendProfile: "codex", executionPolicy: executionPolicy(timeoutMs, context.codexHome), traceSink: null, }; } function queueTask(context: SelfTestContext, sessionId: string, timeoutMs: number): JsonRecord { return { tenantId: "unidesk", projectId: "pikasTech/agentrun", queue: "timeout-liveness", lane: "selftest", title: "timeout liveness commander", priority: 1, backendProfile: "codex", providerId: "G14", workspaceRef: { kind: "host-path", path: context.workspace }, sessionRef: { sessionId, conversationId: sessionId }, executionPolicy: executionPolicy(timeoutMs, context.codexHome), resourceBundleRef: null, payload: { prompt: "timeout liveness commander" }, references: [], metadata: {}, }; } function executionPolicy(timeoutMs: number, codexHome: string): JsonRecord { return { sandbox: "workspace-write", approval: "never", timeoutMs, network: "default", secretScope: { allowCredentialEcho: false, providerCredentials: [{ profile: "codex", secretRef: { name: "agentrun-v01-provider-codex", keys: ["auth.json", "config.toml"], mountPath: codexHome } }] }, }; } export default selfTest;