mirrors/pikasTech-agentrun
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: expose stale runner diagnostics

Browse Source
This commit is contained in:
AgentRun Codex
2026-06-11 09:19:43 +08:00
parent f2fe9f1a2b
commit ec62cc70dd
8 changed files with 279 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/selftest/cases/55-timeout-liveness.ts
+35 -5
View File
@@ -54,19 +54,48 @@ const selfTest: SelfTestCase = async (context: SelfTestContext) => {
await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "backend_status", payload: { commandId: noSession.commandId, phase: "codex-app-server-closed", message: "stdio closed before terminal result" } });
await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "terminal_status", payload: { commandId: noSession.commandId, terminalStatus: "failed", failureKind: "backend-timeout", message: "codex stdio turn hard timed out after 50ms" } });
await client.patch(`/api/v1/commands/${noSession.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
await client.patch(`/api/v1/runs/${noSession.runId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
const noSessionResult = await commandResult(client, noSession);
const noSessionLive = noSessionResult.liveness as JsonRecord;
const noSessionClassification = noSessionResult.terminalClassification as JsonRecord;
const noSessionDiagnosis = noSessionResult.diagnosis as JsonRecord;
assert.equal(noSessionClassification.category, "execution-hard-timeout");
assert.equal(noSessionClassification.providerEvidence, "observed-transport-disconnect");
assert.equal(noSessionClassification.providerInterruptionKnown, false);
assert.equal(noSessionDiagnosis.category, "terminal-command-open-run");
assert.equal(noSessionDiagnosis.providerEvidence, "observed-transport-disconnect");
assert.equal(((noSessionDiagnosis.session as JsonRecord).sessionRefNull), true);
assert.ok((noSessionDiagnosis.recoveryActions as JsonRecord[]).some((action) => action.action === "session-unavailable"));
assert.match(String(noSessionClassification.providerInterruptionReason), /cannot distinguish provider outage/u);
assert.equal((noSessionLive.transportDisconnect as JsonRecord).sourceSeq, 4);
assert.equal((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "resume-session"), false, "sessionId=null must not suggest session-only resume");
assert.equal((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "poll-output"), false, "sessionId=null must not suggest session output path");
assert.ok((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "poll-trace" && String(action.command).includes("runs events")));
const stale = await createActiveRun(client, context, "timeout-liveness-stale-claimed", 120_000, { session: false, leaseMs: 1 });
await store.saveRunnerJob({
runId: stale.runId,
commandId: stale.commandId,
idempotencyKey: "stale-runner-lost",
payloadHash: "hash-stale-runner-lost",
attemptId: "attempt_stale_runner_lost",
runnerId: "runner_stale_claimed",
namespace: "agentrun-v01",
jobName: "agentrun-v01-runner-stale",
managerUrl: server.baseUrl,
image: "127.0.0.1:5000/agentrun/selftest:stale",
sourceCommit: "self-test",
serviceAccountName: "agentrun-v01-runner",
result: { runner: { logPath: "kubectl -n agentrun-v01 logs job/agentrun-v01-runner-stale" }, kubernetes: { created: true, valuesPrinted: false }, valuesPrinted: false },
});
await sleep(5);
const staleResult = await commandResult(client, stale);
const staleDiagnosis = staleResult.diagnosis as JsonRecord;
assert.equal(staleDiagnosis.category, "runner-lost");
assert.equal(staleDiagnosis.staleClaimed, true);
assert.equal(staleDiagnosis.runnerLost, true);
assert.equal(((staleDiagnosis.runnerJob as JsonRecord).phase), "created");
assert.equal(((staleDiagnosis.session as JsonRecord).sessionRefNull), true);
assert.ok(terminal.sessionId, "terminal fixture must have a session id");
const terminalSessionId = terminal.sessionId;
const session = await client.get(`/api/v1/sessions/${terminalSessionId}?readerId=timeout-liveness`) as JsonRecord;
@@ -82,6 +111,7 @@ const selfTest: SelfTestCase = async (context: SelfTestContext) => {
const commander = await client.get("/api/v1/queue/commander?queue=timeout-liveness&readerId=timeout-liveness") as JsonRecord;
const commanderItem = ((commander.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord;
assert.equal(((commanderItem.supervisor as JsonRecord).phase), "terminal");
assert.equal((((commanderItem.supervisor as JsonRecord).diagnosis as JsonRecord).category), "execution-hard-timeout");
assert.equal((((commanderItem.supervisor as JsonRecord).timeoutBudget as JsonRecord).state), "timed-out");
const commanderSummary = summarizeQueueCommanderSnapshot(commander, { limit: 5 });
const summaryItem = ((commanderSummary.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord;
@@ -90,19 +120,19 @@ const selfTest: SelfTestCase = async (context: SelfTestContext) => {
assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).providerEvidence), "insufficient");
assert.equal(JSON.stringify(commanderSummary).includes("hwpod workspace apply-patch"), false, "commander summary must stay compact and avoid dumping command bodies");
assert.equal(JSON.stringify(summaryItem).includes("fullRecordBytes"), false, "commander item must not add bookkeeping noise");
assertNoSecretLeak({ toolResult, assistantLive, inactiveLive, terminalResult, noSessionResult, session, commanderSummary });
assertNoSecretLeak({ toolResult, assistantLive, inactiveLive, terminalResult, noSessionResult, staleResult, session, commanderSummary });
return { name: "timeout-liveness", tests: ["tool-in-flight-liveness", "assistant-progress-liveness", "stdio-inactive-timeout-budget", "terminal-timeout-recovery", "no-session-drilldown", "terminal-classification", "queue-commander-supervisor"] };
return { name: "timeout-liveness", tests: ["tool-in-flight-liveness", "assistant-progress-liveness", "stdio-inactive-timeout-budget", "terminal-timeout-recovery", "no-session-drilldown", "terminal-classification", "queue-commander-supervisor", "diagnosis-visibility", "stale-claimed-runner-lost"] };
} finally {
await new Promise<void>((resolve) => server.server.close(() => resolve()));
}
};
async function createActiveRun(client: ManagerClient, context: SelfTestContext, sessionSuffix: string, timeoutMs: number, options: { session?: boolean } = {}): Promise<{ runId: string; commandId: string; sessionId: string | null }> {
async function createActiveRun(client: ManagerClient, context: SelfTestContext, sessionSuffix: string, timeoutMs: number, options: { session?: boolean; leaseMs?: number } = {}): Promise<{ runId: string; commandId: string; sessionId: string | null }> {
const sessionId = `selftest-${sessionSuffix}`;
const run = await client.post("/api/v1/runs", runBody(context, options.session === false ? null : sessionId, timeoutMs)) as JsonRecord;
const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt: sessionSuffix }, idempotencyKey: sessionSuffix }) as JsonRecord;
await client.post(`/api/v1/runs/${run.id}/claim`, { runnerId: `runner_${sessionSuffix}`, leaseMs: 60_000 });
await client.post(`/api/v1/runs/${run.id}/claim`, { runnerId: `runner_${sessionSuffix}`, leaseMs: options.leaseMs ?? 60_000 });
await client.post(`/api/v1/commands/${command.id}/ack`, {});
return { runId: String(run.id), commandId: String(command.id), sessionId: options.session === false ? null : sessionId };
}
src/selftest/cases/60-hwlab-baseline-contract.ts
+3
View File
@@ -122,10 +122,13 @@ async function assertRunnerJobStatus(client: ManagerClient, context: SelfTestCon
assert.equal(status.runId, item.runId);
assert.equal(status.commandId, item.commandId);
assert.equal(status.phase, "created");
assert.equal(((status.diagnosis as JsonRecord).category), "runner-job-not-started");
assert.equal(((status.diagnosis as JsonRecord).runnerLostSuspected), true);
assert.equal(status.valuesPrinted, false);
assert.equal(typeof status.logPath, "string");
const single = await client.get(`/api/v1/runs/${item.runId}/runner-jobs/${String(status.id)}`) as JsonRecord;
assert.equal(single.jobName, status.jobName);
assert.equal(((single.diagnosis as JsonRecord).phase), "created");
assertNoSecretLeak({ list, single });
}