fix: 增强终态失败分类可见性
This commit is contained in:
AgentRun Codex
@@ -25,6 +25,17 @@ interface AssistantReplySummary {
|
||||
outputTruncated: boolean;
|
||||
}
|
||||
|
||||
interface TerminalClassificationInput {
|
||||
terminal: TerminalStatus | null;
|
||||
terminalSource: string;
|
||||
failureKind: FailureKind | null;
|
||||
failureMessage: string | null;
|
||||
timeoutBudget: JsonRecord;
|
||||
transportDisconnect: RunEvent | null;
|
||||
lastActivity: JsonRecord | null;
|
||||
command: CommandRecord | null;
|
||||
}
|
||||
|
||||
export async function buildRunResult(store: AgentRunStore, runId: string, commandId?: string): Promise<JsonRecord> {
|
||||
const run = await store.getRun(runId);
|
||||
const command = await selectCommand(store, runId, commandId);
|
||||
@@ -44,6 +55,7 @@ export async function buildRunResult(store: AgentRunStore, runId: string, comman
|
||||
const reply = assistantReply(scopedEvents);
|
||||
const blocker = terminal === "blocked" || terminal === "failed" ? { failureKind, message: failureMessage, details: failureDetails } : null;
|
||||
const liveness = livenessSnapshot(run, command, events, scopedEvents, terminal, failureKind, failureMessage);
|
||||
const terminalClassification = terminalClassificationSummary({ terminal, terminalSource, failureKind, failureMessage, liveness });
|
||||
const steerDelivery = command?.type === "steer" ? steerDeliverySummary(events, command.id) : null;
|
||||
return {
|
||||
runId: run.id,
|
||||
@@ -75,6 +87,7 @@ export async function buildRunResult(store: AgentRunStore, runId: string, comman
|
||||
failureKind,
|
||||
failureMessage,
|
||||
failureDetails,
|
||||
terminalClassification,
|
||||
blocker,
|
||||
liveness,
|
||||
...(steerDelivery ? { steerDelivery } : {}),
|
||||
@@ -103,6 +116,7 @@ function livenessSnapshot(run: RunRecord, command: CommandRecord | null, events:
|
||||
const transportDisconnect = latestTransportDisconnect(scopedEvents);
|
||||
const lastActivity = livenessActivitySummary(lastCommandActivity, nowMs);
|
||||
const timeoutBudget = timeoutBudgetSummary(run, command, terminal, failureKind, nowMs);
|
||||
const terminalClassification = terminalClassificationFromEvidence({ terminal, terminalSource: "liveness", failureKind, failureMessage, timeoutBudget, transportDisconnect, lastActivity, command });
|
||||
const phase = livenessPhase({ active, command, lastVisibleActivity, leaseExpired: lease.leaseExpired, transportDisconnect, timeoutBudget, lastActivity });
|
||||
const afterSeq = lastEvent?.seq ?? 0;
|
||||
return {
|
||||
@@ -119,6 +133,7 @@ function livenessSnapshot(run: RunRecord, command: CommandRecord | null, events:
|
||||
lastActivity,
|
||||
lastCommandActivity: lastActivity,
|
||||
timeoutBudget,
|
||||
terminalClassification,
|
||||
lease,
|
||||
transportDisconnect: transportDisconnect ? livenessActivitySummary(transportDisconnect, nowMs) : null,
|
||||
recoveryActions: recoveryActions({ run, command, afterSeq, active, terminal, failureKind, failureMessage }),
|
||||
@@ -146,6 +161,103 @@ function livenessPhase(input: { active: boolean; command: CommandRecord | null;
|
||||
return "waiting-model";
|
||||
}
|
||||
|
||||
function terminalClassificationSummary(input: { terminal: TerminalStatus | null; terminalSource: string; failureKind: FailureKind | null; failureMessage: string | null; liveness: JsonRecord }): JsonRecord {
|
||||
const livenessClassification = jsonRecordValue(input.liveness.terminalClassification);
|
||||
return {
|
||||
...(livenessClassification ?? {}),
|
||||
terminalStatus: input.terminal,
|
||||
terminalSource: input.terminalSource,
|
||||
failureKind: input.failureKind,
|
||||
failureMessage: input.failureMessage ? boundedTextSummary(input.failureMessage, { limitChars: 240 }).text as string : null,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function terminalClassificationFromEvidence(input: TerminalClassificationInput): JsonRecord {
|
||||
const timeoutState = stringJsonValue(input.timeoutBudget.state);
|
||||
const hardTimeout = input.failureKind === "backend-timeout" || timeoutState === "timed-out";
|
||||
const providerKind = providerFailureCategory(input.failureKind);
|
||||
const cancelled = input.terminal === "cancelled" || input.failureKind === "cancelled";
|
||||
const taskFailure = input.terminal === "failed" && input.failureKind !== null && !hardTimeout && !providerKind && !infrastructureFailureKind(input.failureKind);
|
||||
let category = "unknown";
|
||||
let confidence = "low";
|
||||
let providerEvidence = "not-applicable";
|
||||
let reason = "terminal state is not yet available";
|
||||
|
||||
if (input.terminal === "completed") {
|
||||
category = "completed";
|
||||
confidence = "high";
|
||||
reason = "command completed successfully";
|
||||
} else if (cancelled) {
|
||||
category = "cancelled";
|
||||
confidence = "high";
|
||||
reason = "terminal status or failureKind is cancelled";
|
||||
} else if (providerKind) {
|
||||
category = providerKind;
|
||||
confidence = "high";
|
||||
providerEvidence = "failure-kind";
|
||||
reason = `failureKind ${input.failureKind} is provider-specific`;
|
||||
} else if (hardTimeout && input.transportDisconnect) {
|
||||
category = "execution-hard-timeout";
|
||||
confidence = "medium";
|
||||
providerEvidence = "observed-transport-disconnect";
|
||||
reason = "hard timeout is terminal and a backend transport/app-server close event was observed, but existing events do not prove the model provider caused it";
|
||||
} else if (hardTimeout) {
|
||||
category = "execution-hard-timeout";
|
||||
confidence = "high";
|
||||
providerEvidence = "insufficient";
|
||||
reason = "hard timeout is terminal; no provider-specific failure event was recorded";
|
||||
} else if (input.terminal === "blocked") {
|
||||
category = "blocked";
|
||||
confidence = "high";
|
||||
reason = `terminal status is blocked${input.failureKind ? ` with failureKind ${input.failureKind}` : ""}`;
|
||||
} else if (taskFailure) {
|
||||
category = "task-failed";
|
||||
confidence = "medium";
|
||||
reason = `terminal failure is not timeout, cancellation, provider-specific, or infrastructure-classified${input.failureKind ? `; failureKind=${input.failureKind}` : ""}`;
|
||||
} else if (input.terminal === "failed" && infrastructureFailureKind(input.failureKind)) {
|
||||
category = "infrastructure-failed";
|
||||
confidence = "medium";
|
||||
reason = `failureKind ${input.failureKind} is infrastructure/backend classified`;
|
||||
}
|
||||
|
||||
return {
|
||||
category,
|
||||
confidence,
|
||||
providerEvidence,
|
||||
providerInterruption: providerEvidence === "failure-kind" || providerEvidence === "observed-transport-disconnect" ? providerEvidence : "not-established",
|
||||
providerInterruptionKnown: providerEvidence === "failure-kind",
|
||||
providerInterruptionReason: providerEvidence === "failure-kind"
|
||||
? "provider-specific failureKind is authoritative"
|
||||
: providerEvidence === "observed-transport-disconnect"
|
||||
? "transport disconnect was observed, but current events cannot distinguish provider outage from runner/backend shutdown during timeout"
|
||||
: providerEvidence === "insufficient"
|
||||
? "no provider-specific error or disconnect evidence was recorded"
|
||||
: null,
|
||||
hardTimeout,
|
||||
timeoutState,
|
||||
transportDisconnectObserved: Boolean(input.transportDisconnect),
|
||||
transportDisconnectSeq: input.transportDisconnect?.seq ?? null,
|
||||
lastActivityKind: stringJsonValue(input.lastActivity?.activityKind),
|
||||
lastActivitySeq: numberJsonValue(input.lastActivity?.sourceSeq),
|
||||
commandId: input.command?.id ?? null,
|
||||
reason,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function providerFailureCategory(failureKind: FailureKind | null): string | null {
|
||||
if (!failureKind) return null;
|
||||
if (failureKind === "provider-stream-disconnected") return "provider-interrupted";
|
||||
if (failureKind.startsWith("provider-")) return "provider-failed";
|
||||
return null;
|
||||
}
|
||||
|
||||
function infrastructureFailureKind(failureKind: FailureKind | null): boolean {
|
||||
if (!failureKind) return false;
|
||||
return failureKind.startsWith("backend-") || failureKind === "runner-lease-conflict" || failureKind === "infra-failed" || failureKind === "thread-resume-failed";
|
||||
}
|
||||
|
||||
function timeoutBudgetSummary(run: RunRecord, command: CommandRecord | null, terminal: TerminalStatus | null, failureKind: FailureKind | null, nowMs: number): JsonRecord {
|
||||
const timeoutMs = typeof run.executionPolicy.timeoutMs === "number" && Number.isFinite(run.executionPolicy.timeoutMs) && run.executionPolicy.timeoutMs > 0 ? Math.trunc(run.executionPolicy.timeoutMs) : null;
|
||||
const startedAt = command?.acknowledgedAt ?? command?.createdAt ?? run.updatedAt ?? run.createdAt;
|
||||
@@ -295,6 +407,14 @@ function numberJsonValue(value: JsonValue | undefined): number | null {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function stringJsonValue(value: JsonValue | undefined): string | null {
|
||||
return typeof value === "string" && value.length > 0 ? value : null;
|
||||
}
|
||||
|
||||
function jsonRecordValue(value: unknown): JsonRecord | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as JsonRecord : null;
|
||||
}
|
||||
|
||||
function steerDeliverySummary(events: RunEvent[], commandId: string): JsonRecord {
|
||||
const related = events.filter((event) => event.payload.commandId === commandId);
|
||||
const completed = latestPhaseEvent(related, "turn/steer:completed");
|
||||
|
||||
@@ -144,12 +144,14 @@ async function queueTaskSupervisor(store: AgentRunStore, task: JsonRecord): Prom
|
||||
const liveness = asJsonRecord(result.liveness);
|
||||
const lastActivity = asJsonRecord(liveness?.lastActivity ?? liveness?.lastCommandActivity);
|
||||
const timeoutBudget = asJsonRecord(liveness?.timeoutBudget);
|
||||
const terminalClassification = asJsonRecord(result.terminalClassification ?? liveness?.terminalClassification);
|
||||
return {
|
||||
runId: stringJsonValue(result.runId),
|
||||
commandId: stringJsonValue(result.commandId),
|
||||
status: stringJsonValue(result.status),
|
||||
terminalStatus: stringJsonValue(result.terminalStatus),
|
||||
failureKind: stringJsonValue(result.failureKind),
|
||||
terminalClassification: terminalClassification ? compactTerminalClassification(terminalClassification) : null,
|
||||
phase: stringJsonValue(liveness?.phase),
|
||||
active: liveness?.active === true,
|
||||
lastSeq: numberJsonValue(liveness?.lastSeq ?? result.lastSeq),
|
||||
@@ -190,6 +192,22 @@ function compactTimeoutBudget(budget: JsonRecord): JsonRecord {
|
||||
};
|
||||
}
|
||||
|
||||
function compactTerminalClassification(record: JsonRecord): JsonRecord {
|
||||
return {
|
||||
category: stringJsonValue(record.category),
|
||||
confidence: stringJsonValue(record.confidence),
|
||||
providerEvidence: stringJsonValue(record.providerEvidence),
|
||||
providerInterruption: stringJsonValue(record.providerInterruption),
|
||||
providerInterruptionKnown: record.providerInterruptionKnown === true,
|
||||
providerInterruptionReason: boundedJsonString(record.providerInterruptionReason, 240),
|
||||
hardTimeout: record.hardTimeout === true,
|
||||
transportDisconnectObserved: record.transportDisconnectObserved === true,
|
||||
transportDisconnectSeq: numberJsonValue(record.transportDisconnectSeq),
|
||||
reason: boundedJsonString(record.reason, 240),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function compactRecoveryActions(value: JsonValue | undefined): JsonValue[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.slice(0, 5).map((item) => {
|
||||
|
||||
@@ -44,18 +44,40 @@ const selfTest: SelfTestCase = async (context: SelfTestContext) => {
|
||||
assert.equal(terminalResult.terminalStatus, "failed");
|
||||
assert.equal(terminalLive.phase, "terminal");
|
||||
assert.equal(((terminalLive.timeoutBudget as JsonRecord).state), "timed-out");
|
||||
assert.equal(((terminalResult.terminalClassification as JsonRecord).category), "execution-hard-timeout");
|
||||
assert.equal(((terminalResult.terminalClassification as JsonRecord).providerEvidence), "insufficient");
|
||||
assert.equal(((terminalLive.terminalClassification as JsonRecord).providerInterruptionKnown), false);
|
||||
assert.ok((terminalLive.recoveryActions as JsonRecord[]).some((action) => action.action === "resume-session"));
|
||||
assert.ok((terminalLive.recoveryActions as JsonRecord[]).some((action) => action.action === "split-task"));
|
||||
|
||||
const session = await client.get(`/api/v1/sessions/${terminal.sessionId}?readerId=timeout-liveness`) as JsonRecord;
|
||||
const noSession = await createActiveRun(client, context, "timeout-liveness-no-session", 50, { session: false });
|
||||
await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "backend_status", payload: { commandId: noSession.commandId, phase: "codex-app-server-closed", message: "stdio closed before terminal result" } });
|
||||
await client.post(`/api/v1/runs/${noSession.runId}/events`, { type: "terminal_status", payload: { commandId: noSession.commandId, terminalStatus: "failed", failureKind: "backend-timeout", message: "codex stdio turn hard timed out after 50ms" } });
|
||||
await client.patch(`/api/v1/commands/${noSession.commandId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
|
||||
await client.patch(`/api/v1/runs/${noSession.runId}/status`, { terminalStatus: "failed", failureKind: "backend-timeout", failureMessage: "codex stdio turn hard timed out after 50ms" });
|
||||
const noSessionResult = await commandResult(client, noSession);
|
||||
const noSessionLive = noSessionResult.liveness as JsonRecord;
|
||||
const noSessionClassification = noSessionResult.terminalClassification as JsonRecord;
|
||||
assert.equal(noSessionClassification.category, "execution-hard-timeout");
|
||||
assert.equal(noSessionClassification.providerEvidence, "observed-transport-disconnect");
|
||||
assert.equal(noSessionClassification.providerInterruptionKnown, false);
|
||||
assert.match(String(noSessionClassification.providerInterruptionReason), /cannot distinguish provider outage/u);
|
||||
assert.equal((noSessionLive.transportDisconnect as JsonRecord).sourceSeq, 4);
|
||||
assert.equal((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "resume-session"), false, "sessionId=null must not suggest session-only resume");
|
||||
assert.equal((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "poll-output"), false, "sessionId=null must not suggest session output path");
|
||||
assert.ok((noSessionLive.recoveryActions as JsonRecord[]).some((action) => action.action === "poll-trace" && String(action.command).includes("runs events")));
|
||||
|
||||
assert.ok(terminal.sessionId, "terminal fixture must have a session id");
|
||||
const terminalSessionId = terminal.sessionId;
|
||||
const session = await client.get(`/api/v1/sessions/${terminalSessionId}?readerId=timeout-liveness`) as JsonRecord;
|
||||
assert.equal(((session.liveness as JsonRecord).phase), "terminal");
|
||||
assert.ok(Array.isArray(((session.supervisor as JsonRecord).recoveryActions)), "session show must keep terminal recovery actions");
|
||||
|
||||
const task = await client.post("/api/v1/queue/tasks", queueTask(context, terminal.sessionId, 50)) as JsonRecord;
|
||||
const task = await client.post("/api/v1/queue/tasks", queueTask(context, terminalSessionId, 50)) as JsonRecord;
|
||||
store.updateQueueTaskAttempt(String(task.id), {
|
||||
state: "running",
|
||||
latestAttempt: { attemptId: "attempt_timeout_liveness", state: "running", runId: terminal.runId, commandId: terminal.commandId, runnerJobId: null, sessionId: terminal.sessionId, sessionPath: `/api/v1/sessions/${terminal.sessionId}` },
|
||||
sessionPath: `/api/v1/sessions/${terminal.sessionId}`,
|
||||
latestAttempt: { attemptId: "attempt_timeout_liveness", state: "running", runId: terminal.runId, commandId: terminal.commandId, runnerJobId: null, sessionId: terminalSessionId, sessionPath: `/api/v1/sessions/${terminalSessionId}` },
|
||||
sessionPath: `/api/v1/sessions/${terminalSessionId}`,
|
||||
});
|
||||
const commander = await client.get("/api/v1/queue/commander?queue=timeout-liveness&readerId=timeout-liveness") as JsonRecord;
|
||||
const commanderItem = ((commander.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord;
|
||||
@@ -64,35 +86,37 @@ const selfTest: SelfTestCase = async (context: SelfTestContext) => {
|
||||
const commanderSummary = summarizeQueueCommanderSnapshot(commander, { limit: 5 });
|
||||
const summaryItem = ((commanderSummary.items as JsonRecord[]) ?? []).find((item) => item.id === task.id) as JsonRecord;
|
||||
assert.equal(((summaryItem.supervisor as JsonRecord).phase), "terminal");
|
||||
assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).category), "execution-hard-timeout");
|
||||
assert.equal((((summaryItem.supervisor as JsonRecord).terminalClassification as JsonRecord).providerEvidence), "insufficient");
|
||||
assert.equal(JSON.stringify(commanderSummary).includes("hwpod workspace apply-patch"), false, "commander summary must stay compact and avoid dumping command bodies");
|
||||
assert.equal(JSON.stringify(summaryItem).includes("fullRecordBytes"), false, "commander item must not add bookkeeping noise");
|
||||
assertNoSecretLeak({ toolResult, assistantLive, inactiveLive, terminalResult, session, commanderSummary });
|
||||
assertNoSecretLeak({ toolResult, assistantLive, inactiveLive, terminalResult, noSessionResult, session, commanderSummary });
|
||||
|
||||
return { name: "timeout-liveness", tests: ["tool-in-flight-liveness", "assistant-progress-liveness", "stdio-inactive-timeout-budget", "terminal-timeout-recovery", "queue-commander-supervisor"] };
|
||||
return { name: "timeout-liveness", tests: ["tool-in-flight-liveness", "assistant-progress-liveness", "stdio-inactive-timeout-budget", "terminal-timeout-recovery", "no-session-drilldown", "terminal-classification", "queue-commander-supervisor"] };
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
||||
}
|
||||
};
|
||||
|
||||
async function createActiveRun(client: ManagerClient, context: SelfTestContext, sessionSuffix: string, timeoutMs: number): Promise<{ runId: string; commandId: string; sessionId: string }> {
|
||||
async function createActiveRun(client: ManagerClient, context: SelfTestContext, sessionSuffix: string, timeoutMs: number, options: { session?: boolean } = {}): Promise<{ runId: string; commandId: string; sessionId: string | null }> {
|
||||
const sessionId = `selftest-${sessionSuffix}`;
|
||||
const run = await client.post("/api/v1/runs", runBody(context, sessionId, timeoutMs)) as JsonRecord;
|
||||
const run = await client.post("/api/v1/runs", runBody(context, options.session === false ? null : sessionId, timeoutMs)) as JsonRecord;
|
||||
const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt: sessionSuffix }, idempotencyKey: sessionSuffix }) as JsonRecord;
|
||||
await client.post(`/api/v1/runs/${run.id}/claim`, { runnerId: `runner_${sessionSuffix}`, leaseMs: 60_000 });
|
||||
await client.post(`/api/v1/commands/${command.id}/ack`, {});
|
||||
return { runId: String(run.id), commandId: String(command.id), sessionId };
|
||||
return { runId: String(run.id), commandId: String(command.id), sessionId: options.session === false ? null : sessionId };
|
||||
}
|
||||
|
||||
async function commandResult(client: ManagerClient, item: { runId: string; commandId: string }): Promise<JsonRecord> {
|
||||
return await client.get(`/api/v1/runs/${item.runId}/commands/${item.commandId}/result`) as JsonRecord;
|
||||
}
|
||||
|
||||
function runBody(context: SelfTestContext, sessionId: string, timeoutMs: number): JsonRecord {
|
||||
function runBody(context: SelfTestContext, sessionId: string | null, timeoutMs: number): JsonRecord {
|
||||
return {
|
||||
tenantId: "unidesk",
|
||||
projectId: "pikasTech/agentrun",
|
||||
workspaceRef: { kind: "host-path", path: context.workspace },
|
||||
sessionRef: { sessionId, conversationId: sessionId },
|
||||
sessionRef: sessionId ? { sessionId, conversationId: sessionId } : null,
|
||||
providerId: "G14",
|
||||
backendProfile: "codex",
|
||||
executionPolicy: executionPolicy(timeoutMs, context.codexHome),
|
||||
|
||||
Reference in New Issue
Block a user