AgentRun Codex
751 lines
37 KiB
TypeScript
751 lines
37 KiB
TypeScript
import type { AgentRunStore } from "./store.js";
|
|
import type { CommandRecord, FailureKind, JsonRecord, JsonValue, RunEvent, RunRecord, RunnerJobRecord, TerminalStatus } from "../common/types.js";
|
|
import { boundedTextSummary, outputBytesFromPayload, outputTruncatedFromPayload } from "../common/output.js";
|
|
|
|
const maxToolCallSummaryItems = 40;
|
|
const toolCallCommandLimitChars = 600;
|
|
const toolCallFieldLimitChars = 200;
|
|
|
|
const RESULT_EVENT_PAGE_LIMIT = 500;
|
|
const RESULT_EVENT_MAX_PAGES = 200;
|
|
|
|
interface ResultEventPage {
|
|
events: RunEvent[];
|
|
capped: boolean;
|
|
nextAfterSeq: number | null;
|
|
}
|
|
|
|
interface AssistantReplySummary {
|
|
text: string;
|
|
seq: number | null;
|
|
source: string | null;
|
|
final: boolean;
|
|
replyAuthority: boolean;
|
|
textTruncated: boolean;
|
|
outputTruncated: boolean;
|
|
}
|
|
|
|
interface TerminalClassificationInput {
|
|
terminal: TerminalStatus | null;
|
|
terminalSource: string;
|
|
failureKind: FailureKind | null;
|
|
failureMessage: string | null;
|
|
timeoutBudget: JsonRecord;
|
|
transportDisconnect: RunEvent | null;
|
|
lastActivity: JsonRecord | null;
|
|
command: CommandRecord | null;
|
|
}
|
|
|
|
export async function buildRunResult(store: AgentRunStore, runId: string, commandId?: string): Promise<JsonRecord> {
|
|
const run = await store.getRun(runId);
|
|
const command = await selectCommand(store, runId, commandId);
|
|
const eventPage = await listResultEvents(store, runId);
|
|
const events = eventPage.events;
|
|
const scopedEvents = command ? eventsForCommand(events, command.id) : events;
|
|
const jobs = await store.listRunnerJobs(runId, command?.id);
|
|
const latestJob = jobs.at(-1) ?? null;
|
|
const commandTerminal = command ? terminalFromCommand(command) : null;
|
|
const terminalEventStatus = terminalFromEvents(scopedEvents);
|
|
const preliminaryTerminal = commandTerminal ?? terminalEventStatus ?? run.terminalStatus;
|
|
const failureKind = resultFailureKind(run, command, scopedEvents, preliminaryTerminal);
|
|
const terminal = resultTerminal(commandTerminal, terminalEventStatus, run.terminalStatus, failureKind);
|
|
const terminalSource = resultTerminalSource(commandTerminal, terminalEventStatus, run.terminalStatus, failureKind);
|
|
const failureMessage = resultFailureMessage(run, command, scopedEvents, terminal);
|
|
const failureDetails = resultFailureDetails(scopedEvents, terminal);
|
|
const reply = assistantReply(scopedEvents);
|
|
const blocker = terminal === "blocked" || terminal === "failed" ? { failureKind, message: failureMessage, details: failureDetails } : null;
|
|
const liveness = livenessSnapshot(run, command, events, scopedEvents, terminal, failureKind, failureMessage);
|
|
const terminalClassification = terminalClassificationSummary({ terminal, terminalSource, failureKind, failureMessage, liveness });
|
|
const steerDelivery = command?.type === "steer" ? steerDeliverySummary(events, command.id) : null;
|
|
return {
|
|
runId: run.id,
|
|
commandId: command?.id ?? commandId ?? null,
|
|
attemptId: latestJob?.attemptId ?? attemptFromEvents(events),
|
|
runnerId: latestJob?.runnerId ?? null,
|
|
jobName: latestJob?.jobName ?? null,
|
|
namespace: latestJob?.namespace ?? null,
|
|
status: command?.state ?? run.status,
|
|
runStatus: run.status,
|
|
commandState: command?.state ?? null,
|
|
terminalStatus: terminal,
|
|
terminalSource,
|
|
completed: terminal === "completed",
|
|
reply: reply.text,
|
|
finalResponse: {
|
|
text: reply.text,
|
|
seq: reply.seq,
|
|
source: reply.source,
|
|
final: reply.final,
|
|
replyAuthority: reply.replyAuthority,
|
|
textTruncated: reply.textTruncated,
|
|
outputTruncated: reply.outputTruncated,
|
|
},
|
|
finalAssistantSeq: reply.seq,
|
|
finalAssistantSource: reply.source,
|
|
finalAssistantTextTruncated: reply.textTruncated,
|
|
finalAssistantOutputTruncated: reply.outputTruncated,
|
|
failureKind,
|
|
failureMessage,
|
|
failureDetails,
|
|
terminalClassification,
|
|
blocker,
|
|
liveness,
|
|
...(steerDelivery ? { steerDelivery } : {}),
|
|
lastSeq: events.at(-1)?.seq ?? 0,
|
|
eventCount: events.length,
|
|
eventsCapped: eventPage.capped,
|
|
nextAfterSeq: eventPage.nextAfterSeq,
|
|
scopedEventCount: scopedEvents.length,
|
|
scopedLastSeq: scopedEvents.at(-1)?.seq ?? 0,
|
|
artifactSummary: artifactSummary(scopedEvents),
|
|
toolCallSummary: toolCallSummary(scopedEvents),
|
|
sessionRef: sessionSummary(run),
|
|
resourceBundleRef: resourceBundleSummary(run, events),
|
|
runnerJobCount: jobs.length,
|
|
};
|
|
}
|
|
|
|
function livenessSnapshot(run: RunRecord, command: CommandRecord | null, events: RunEvent[], scopedEvents: RunEvent[], terminal: TerminalStatus | null, failureKind: FailureKind | null, failureMessage: string | null): JsonRecord {
|
|
const nowMs = Date.now();
|
|
const active = terminal === null && !runIsTerminal(run) && !commandIsTerminal(command);
|
|
const lastEvent = events.at(-1) ?? null;
|
|
const lastVisibleActivity = latestVisibleActivity(scopedEvents);
|
|
const lastBusinessActivity = latestBusinessActivity(scopedEvents);
|
|
const lastCommandActivity = lastBusinessActivity ?? latestLivenessActivity(scopedEvents);
|
|
const lease = leaseSummary(run, nowMs);
|
|
const transportDisconnect = latestTransportDisconnect(scopedEvents);
|
|
const lastActivity = livenessActivitySummary(lastCommandActivity, nowMs);
|
|
const timeoutBudget = timeoutBudgetSummary(run, command, terminal, failureKind, nowMs);
|
|
const terminalClassification = terminalClassificationFromEvidence({ terminal, terminalSource: "liveness", failureKind, failureMessage, timeoutBudget, transportDisconnect, lastActivity, command });
|
|
const phase = livenessPhase({ active, command, lastVisibleActivity, leaseExpired: lease.leaseExpired, transportDisconnect, timeoutBudget, lastActivity });
|
|
const afterSeq = lastEvent?.seq ?? 0;
|
|
return {
|
|
phase,
|
|
active,
|
|
observedAt: new Date(nowMs).toISOString(),
|
|
runStatus: run.status,
|
|
commandId: command?.id ?? null,
|
|
commandType: command?.type ?? null,
|
|
commandState: command?.state ?? null,
|
|
lastSeq: lastEvent?.seq ?? 0,
|
|
lastEventAt: lastEvent?.createdAt ?? null,
|
|
lastEventAgeMs: lastEvent ? ageMs(lastEvent.createdAt, nowMs) : null,
|
|
lastActivity,
|
|
lastCommandActivity: lastActivity,
|
|
timeoutBudget,
|
|
terminalClassification,
|
|
lease,
|
|
transportDisconnect: transportDisconnect ? livenessActivitySummary(transportDisconnect, nowMs) : null,
|
|
recoveryActions: recoveryActions({ run, command, afterSeq, active, terminal, failureKind, failureMessage }),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function livenessPhase(input: { active: boolean; command: CommandRecord | null; lastVisibleActivity: RunEvent | null; leaseExpired: boolean | null; transportDisconnect: RunEvent | null; timeoutBudget: JsonRecord; lastActivity: JsonRecord | null }): string {
|
|
if (!input.active) return "terminal";
|
|
if (input.command?.state === "pending") return "waiting-runner";
|
|
if (input.leaseExpired === true) return "runner-heartbeat-stale";
|
|
if (input.transportDisconnect) return "transport-disconnected";
|
|
if (input.lastVisibleActivity?.type === "tool_call") {
|
|
const status = input.lastVisibleActivity.payload.status;
|
|
if (status === "inProgress" || status === "running") return "waiting-tool";
|
|
if (status === "completed") return "idle-after-tool";
|
|
}
|
|
if (input.lastVisibleActivity?.type === "command_output") return "waiting-tool-output";
|
|
if (input.lastVisibleActivity?.type === "assistant_message") return "waiting-model-output";
|
|
const remainingMs = numberJsonValue(input.timeoutBudget.remainingMs);
|
|
const timeoutMs = numberJsonValue(input.timeoutBudget.timeoutMs);
|
|
const activityAgeMs = numberJsonValue(input.lastActivity?.ageMs);
|
|
const inactiveThresholdMs = timeoutMs === null ? 300_000 : Math.min(300_000, Math.max(60_000, Math.floor(timeoutMs / 4)));
|
|
if (remainingMs !== null && remainingMs <= Math.min(120_000, Math.max(10_000, Math.floor((timeoutMs ?? 120_000) / 10))) && (activityAgeMs === null || activityAgeMs >= inactiveThresholdMs)) return "runner-stdio-inactive";
|
|
return "waiting-model";
|
|
}
|
|
|
|
function terminalClassificationSummary(input: { terminal: TerminalStatus | null; terminalSource: string; failureKind: FailureKind | null; failureMessage: string | null; liveness: JsonRecord }): JsonRecord {
|
|
const livenessClassification = jsonRecordValue(input.liveness.terminalClassification);
|
|
return {
|
|
...(livenessClassification ?? {}),
|
|
terminalStatus: input.terminal,
|
|
terminalSource: input.terminalSource,
|
|
failureKind: input.failureKind,
|
|
failureMessage: input.failureMessage ? boundedTextSummary(input.failureMessage, { limitChars: 240 }).text as string : null,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function terminalClassificationFromEvidence(input: TerminalClassificationInput): JsonRecord {
|
|
const timeoutState = stringJsonValue(input.timeoutBudget.state);
|
|
const hardTimeout = input.failureKind === "backend-timeout" || timeoutState === "timed-out";
|
|
const providerKind = providerFailureCategory(input.failureKind);
|
|
const cancelled = input.terminal === "cancelled" || input.failureKind === "cancelled";
|
|
const taskFailure = input.terminal === "failed" && input.failureKind !== null && !hardTimeout && !providerKind && !infrastructureFailureKind(input.failureKind);
|
|
let category = "unknown";
|
|
let confidence = "low";
|
|
let providerEvidence = "not-applicable";
|
|
let reason = "terminal state is not yet available";
|
|
|
|
if (input.terminal === "completed") {
|
|
category = "completed";
|
|
confidence = "high";
|
|
reason = "command completed successfully";
|
|
} else if (cancelled) {
|
|
category = "cancelled";
|
|
confidence = "high";
|
|
reason = "terminal status or failureKind is cancelled";
|
|
} else if (providerKind) {
|
|
category = providerKind;
|
|
confidence = "high";
|
|
providerEvidence = "failure-kind";
|
|
reason = `failureKind ${input.failureKind} is provider-specific`;
|
|
} else if (hardTimeout && input.transportDisconnect) {
|
|
category = "execution-hard-timeout";
|
|
confidence = "medium";
|
|
providerEvidence = "observed-transport-disconnect";
|
|
reason = "hard timeout is terminal and a backend transport/app-server close event was observed, but existing events do not prove the model provider caused it";
|
|
} else if (hardTimeout) {
|
|
category = "execution-hard-timeout";
|
|
confidence = "high";
|
|
providerEvidence = "insufficient";
|
|
reason = "hard timeout is terminal; no provider-specific failure event was recorded";
|
|
} else if (input.terminal === "blocked") {
|
|
category = "blocked";
|
|
confidence = "high";
|
|
reason = `terminal status is blocked${input.failureKind ? ` with failureKind ${input.failureKind}` : ""}`;
|
|
} else if (taskFailure) {
|
|
category = "task-failed";
|
|
confidence = "medium";
|
|
reason = `terminal failure is not timeout, cancellation, provider-specific, or infrastructure-classified${input.failureKind ? `; failureKind=${input.failureKind}` : ""}`;
|
|
} else if (input.terminal === "failed" && infrastructureFailureKind(input.failureKind)) {
|
|
category = "infrastructure-failed";
|
|
confidence = "medium";
|
|
reason = `failureKind ${input.failureKind} is infrastructure/backend classified`;
|
|
}
|
|
|
|
return {
|
|
category,
|
|
confidence,
|
|
providerEvidence,
|
|
providerInterruption: providerEvidence === "failure-kind" || providerEvidence === "observed-transport-disconnect" ? providerEvidence : "not-established",
|
|
providerInterruptionKnown: providerEvidence === "failure-kind",
|
|
providerInterruptionReason: providerEvidence === "failure-kind"
|
|
? "provider-specific failureKind is authoritative"
|
|
: providerEvidence === "observed-transport-disconnect"
|
|
? "transport disconnect was observed, but current events cannot distinguish provider outage from runner/backend shutdown during timeout"
|
|
: providerEvidence === "insufficient"
|
|
? "no provider-specific error or disconnect evidence was recorded"
|
|
: null,
|
|
hardTimeout,
|
|
timeoutState,
|
|
transportDisconnectObserved: Boolean(input.transportDisconnect),
|
|
transportDisconnectSeq: input.transportDisconnect?.seq ?? null,
|
|
lastActivityKind: stringJsonValue(input.lastActivity?.activityKind),
|
|
lastActivitySeq: numberJsonValue(input.lastActivity?.sourceSeq),
|
|
commandId: input.command?.id ?? null,
|
|
reason,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function providerFailureCategory(failureKind: FailureKind | null): string | null {
|
|
if (!failureKind) return null;
|
|
if (failureKind === "provider-stream-disconnected") return "provider-interrupted";
|
|
if (failureKind.startsWith("provider-")) return "provider-failed";
|
|
return null;
|
|
}
|
|
|
|
function infrastructureFailureKind(failureKind: FailureKind | null): boolean {
|
|
if (!failureKind) return false;
|
|
return failureKind.startsWith("backend-") || failureKind === "runner-lease-conflict" || failureKind === "infra-failed" || failureKind === "thread-resume-failed";
|
|
}
|
|
|
|
function timeoutBudgetSummary(run: RunRecord, command: CommandRecord | null, terminal: TerminalStatus | null, failureKind: FailureKind | null, nowMs: number): JsonRecord {
|
|
const timeoutMs = typeof run.executionPolicy.timeoutMs === "number" && Number.isFinite(run.executionPolicy.timeoutMs) && run.executionPolicy.timeoutMs > 0 ? Math.trunc(run.executionPolicy.timeoutMs) : null;
|
|
const startedAt = command?.acknowledgedAt ?? command?.createdAt ?? run.updatedAt ?? run.createdAt;
|
|
const startedMs = Date.parse(startedAt);
|
|
const elapsedMs = timeoutMs !== null && Number.isFinite(startedMs) ? Math.max(0, nowMs - startedMs) : null;
|
|
const remainingMs = timeoutMs !== null && elapsedMs !== null ? Math.max(0, timeoutMs - elapsedMs) : null;
|
|
const approachingThresholdMs = timeoutMs === null ? null : Math.min(120_000, Math.max(10_000, Math.floor(timeoutMs / 10)));
|
|
let state = "unknown";
|
|
if (timeoutMs !== null && elapsedMs !== null) {
|
|
if (terminal !== null) state = failureKind === "backend-timeout" ? "timed-out" : "terminal";
|
|
else if (remainingMs === 0) state = "overdue";
|
|
else if (approachingThresholdMs !== null && remainingMs !== null && remainingMs <= approachingThresholdMs) state = "approaching-hard-timeout";
|
|
else state = "within-budget";
|
|
}
|
|
return {
|
|
timeoutMs,
|
|
source: "executionPolicy.timeoutMs",
|
|
startedAt,
|
|
elapsedMs,
|
|
remainingMs,
|
|
approachingThresholdMs,
|
|
state,
|
|
hardTimeout: true,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function leaseSummary(run: RunRecord, nowMs: number): JsonRecord & { leaseExpired: boolean | null } {
|
|
const leaseExpiresMs = run.leaseExpiresAt ? Date.parse(run.leaseExpiresAt) : NaN;
|
|
const hasLease = Boolean(run.claimedBy && run.leaseExpiresAt && Number.isFinite(leaseExpiresMs));
|
|
const leaseExpired = run.claimedBy ? (hasLease ? leaseExpiresMs <= nowMs : true) : null;
|
|
return {
|
|
claimedBy: run.claimedBy,
|
|
leaseExpiresAt: run.leaseExpiresAt,
|
|
leaseExpired,
|
|
leaseRemainingMs: hasLease ? Math.max(0, leaseExpiresMs - nowMs) : null,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function latestLivenessActivity(events: RunEvent[]): RunEvent | null {
|
|
return [...events].reverse().find(isLivenessActivityEvent) ?? null;
|
|
}
|
|
|
|
function latestVisibleActivity(events: RunEvent[]): RunEvent | null {
|
|
return [...events].reverse().find((event) => event.type === "assistant_message" || event.type === "tool_call" || event.type === "command_output" || event.type === "diff" || event.type === "error" || event.type === "terminal_status") ?? null;
|
|
}
|
|
|
|
function latestBusinessActivity(events: RunEvent[]): RunEvent | null {
|
|
return [...events].reverse().find((event) => event.type === "assistant_message" || event.type === "tool_call" || event.type === "command_output" || event.type === "diff") ?? null;
|
|
}
|
|
|
|
function isLivenessActivityEvent(event: RunEvent): boolean {
|
|
if (event.type === "assistant_message" || event.type === "tool_call" || event.type === "command_output" || event.type === "diff" || event.type === "error" || event.type === "terminal_status") return true;
|
|
if (event.type !== "backend_status") return false;
|
|
const phase = typeof event.payload.phase === "string" ? event.payload.phase : "";
|
|
return ["backend-turn-started", "thread/start:completed", "thread/resume:completed", "turn/start:completed", "backend-turn-finished", "codex-app-server-closed"].includes(phase);
|
|
}
|
|
|
|
function latestTransportDisconnect(events: RunEvent[]): RunEvent | null {
|
|
return [...events].reverse().find((event) => {
|
|
const phase = typeof event.payload.phase === "string" ? event.payload.phase : "";
|
|
return (event.type === "error" && phase.includes("transport")) || (event.type === "backend_status" && phase === "codex-app-server-closed");
|
|
}) ?? null;
|
|
}
|
|
|
|
function livenessActivitySummary(event: RunEvent | null, nowMs: number): JsonRecord | null {
|
|
if (!event) return null;
|
|
return {
|
|
seq: event.seq,
|
|
sourceSeq: event.seq,
|
|
eventId: event.id,
|
|
type: event.type,
|
|
activityKind: activityKind(event),
|
|
phase: typeof event.payload.phase === "string" ? event.payload.phase : null,
|
|
status: typeof event.payload.status === "string" ? event.payload.status : null,
|
|
toolName: typeof event.payload.toolName === "string" ? event.payload.toolName : null,
|
|
itemId: typeof event.payload.itemId === "string" ? event.payload.itemId : null,
|
|
exitCode: normalizedExitCode(event.payload.exitCode),
|
|
commandId: typeof event.payload.commandId === "string" ? event.payload.commandId : null,
|
|
createdAt: event.createdAt,
|
|
ageMs: ageMs(event.createdAt, nowMs),
|
|
summary: activityTextSummary(event),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function activityKind(event: RunEvent): string {
|
|
if (event.type === "tool_call") {
|
|
const status = typeof event.payload.status === "string" ? event.payload.status : "";
|
|
if (status === "running" || status === "inProgress") return "tool-in-flight";
|
|
if (status === "completed") return "tool-completed";
|
|
if (status === "cancelled") return "tool-cancelled";
|
|
if (status === "failed") return "tool-failed";
|
|
return "tool";
|
|
}
|
|
if (event.type === "command_output") return "tool-output";
|
|
if (event.type === "assistant_message") return event.payload.progress === true ? "assistant-progress" : "assistant-output";
|
|
if (event.type === "diff") return "diff";
|
|
if (event.type === "error") return "error";
|
|
if (event.type === "terminal_status") return "terminal";
|
|
return "backend-activity";
|
|
}
|
|
|
|
function activityTextSummary(event: RunEvent): string | null {
|
|
if (event.type === "tool_call") {
|
|
const name = typeof event.payload.toolName === "string" ? event.payload.toolName : typeof event.payload.type === "string" ? event.payload.type : "tool";
|
|
const status = typeof event.payload.status === "string" ? event.payload.status : "observed";
|
|
return `${name} ${status}`;
|
|
}
|
|
const fromSummary = typeof event.payload.outputSummary === "string" ? event.payload.outputSummary : typeof event.payload.summary === "string" ? event.payload.summary : null;
|
|
const text = fromSummary ?? textPayload(event.payload) ?? (typeof event.payload.phase === "string" ? event.payload.phase : null);
|
|
if (!text) return null;
|
|
return boundedTextSummary(text.replace(/\s+/gu, " ").trim(), { limitChars: 240 }).text as string;
|
|
}
|
|
|
|
function ageMs(value: string, nowMs: number): number | null {
|
|
const parsed = Date.parse(value);
|
|
return Number.isFinite(parsed) ? Math.max(0, nowMs - parsed) : null;
|
|
}
|
|
|
|
function recoveryActions(input: { run: RunRecord; command: CommandRecord | null; afterSeq: number; active: boolean; terminal: TerminalStatus | null; failureKind: FailureKind | null; failureMessage: string | null }): JsonRecord[] {
|
|
const { run, command, afterSeq, active, terminal, failureKind, failureMessage } = input;
|
|
const sessionId = run.sessionRef?.sessionId ?? null;
|
|
const traceCommand = sessionId ? `./scripts/agentrun sessions trace ${sessionId} --after-seq ${afterSeq} --limit 100 --run-id ${run.id}` : `./scripts/agentrun runs events ${run.id} --after-seq ${afterSeq} --limit 100 --summary`;
|
|
const outputCommand = sessionId ? `./scripts/agentrun sessions output ${sessionId} --after-seq ${afterSeq} --limit 100 --run-id ${run.id}` : null;
|
|
const actions: JsonRecord[] = [
|
|
{ action: "poll-trace", runId: run.id, commandId: command?.id ?? null, afterSeq, command: traceCommand, valuesPrinted: false },
|
|
];
|
|
if (outputCommand) actions.push({ action: "poll-output", runId: run.id, commandId: command?.id ?? null, afterSeq, command: outputCommand, valuesPrinted: false });
|
|
if (active) {
|
|
if (sessionId) actions.push({ action: "steer-session", sessionId, runId: run.id, commandId: command?.id ?? null, command: `./scripts/agentrun sessions steer ${sessionId} --prompt-stdin`, valuesPrinted: false });
|
|
if (command) actions.push({ action: "cancel-command", runId: run.id, commandId: command.id, command: `./scripts/agentrun commands cancel ${command.id} --reason <reason>`, valuesPrinted: false });
|
|
else actions.push({ action: "cancel-run", runId: run.id, command: `./scripts/agentrun runs cancel ${run.id} --reason <reason>`, valuesPrinted: false });
|
|
return actions;
|
|
}
|
|
if (terminal === "failed" || terminal === "blocked" || terminal === "cancelled") {
|
|
if (command) actions.push({ action: "inspect-result", runId: run.id, commandId: command.id, command: `./scripts/agentrun commands result ${command.id} --run-id ${run.id}`, valuesPrinted: false });
|
|
if (sessionId) actions.push({ action: "resume-session", sessionId, command: `./scripts/agentrun sessions turn ${sessionId} --prompt-stdin`, valuesPrinted: false });
|
|
if (failureKind === "backend-timeout") actions.push({ action: "split-task", reason: "backend-timeout", hint: "把大 patch / 长工具链拆成更短 turn 后用同一 session 续跑", failureMessage: failureMessage ? boundedTextSummary(failureMessage, { limitChars: 200 }).text as string : null, valuesPrinted: false });
|
|
else actions.push({ action: "retry-or-split", reason: failureKind ?? "terminal", hint: "先读 trace/output 的 detail id，再决定 steer、重跑或拆分", valuesPrinted: false });
|
|
}
|
|
return actions;
|
|
}
|
|
|
|
function numberJsonValue(value: JsonValue | undefined): number | null {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
function stringJsonValue(value: JsonValue | undefined): string | null {
|
|
return typeof value === "string" && value.length > 0 ? value : null;
|
|
}
|
|
|
|
function jsonRecordValue(value: unknown): JsonRecord | null {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as JsonRecord : null;
|
|
}
|
|
|
|
function steerDeliverySummary(events: RunEvent[], commandId: string): JsonRecord {
|
|
const related = events.filter((event) => event.payload.commandId === commandId);
|
|
const completed = latestPhaseEvent(related, "turn/steer:completed");
|
|
const acknowledged = latestPhaseEvent(related, "steer-command-acknowledged");
|
|
const failed = [...related].reverse().find((event) => event.type === "error") ?? null;
|
|
const deliverySeq = completed?.seq ?? failed?.seq ?? acknowledged?.seq ?? related.at(-1)?.seq ?? null;
|
|
const targetCommandId = targetCommandIdFromEvents(related);
|
|
const targetFollowUp = targetCommandId && deliverySeq !== null ? targetFollowUpAfterSeq(events, targetCommandId, deliverySeq) : null;
|
|
return {
|
|
commandId,
|
|
targetCommandId,
|
|
deliveryState: failed ? "failed" : completed ? "forwarded-to-backend" : acknowledged ? "acknowledged-by-runner" : "not-observed",
|
|
backendAccepted: Boolean(completed),
|
|
targetFollowUpObserved: targetFollowUp?.observed ?? null,
|
|
targetFollowUpSeq: targetFollowUp?.seq ?? null,
|
|
targetFollowUpType: targetFollowUp?.type ?? null,
|
|
targetFollowUpPhase: targetFollowUp?.phase ?? null,
|
|
semantics: "steer completion means the backend accepted the turn/steer RPC; target turn progress is reported by the target command liveness and is not implied by the steer command terminal state",
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function latestPhaseEvent(events: RunEvent[], phase: string): RunEvent | null {
|
|
return [...events].reverse().find((event) => event.payload.phase === phase) ?? null;
|
|
}
|
|
|
|
function targetCommandIdFromEvents(events: RunEvent[]): string | null {
|
|
for (const event of [...events].reverse()) {
|
|
const targetCommandId = event.payload.targetCommandId;
|
|
if (typeof targetCommandId === "string" && targetCommandId.length > 0) return targetCommandId;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function targetFollowUpAfterSeq(events: RunEvent[], targetCommandId: string, afterSeq: number): JsonRecord {
|
|
const event = events.find((item) => item.seq > afterSeq && item.payload.commandId === targetCommandId && isTargetFollowUpEvent(item));
|
|
return {
|
|
observed: Boolean(event),
|
|
seq: event?.seq ?? null,
|
|
type: event?.type ?? null,
|
|
phase: typeof event?.payload.phase === "string" ? event.payload.phase : null,
|
|
};
|
|
}
|
|
|
|
function isTargetFollowUpEvent(event: RunEvent): boolean {
|
|
return event.type === "assistant_message" || event.type === "tool_call" || event.type === "command_output" || event.type === "diff" || event.type === "error" || event.type === "terminal_status";
|
|
}
|
|
|
|
function runIsTerminal(run: RunRecord): boolean {
|
|
return run.status === "completed" || run.status === "failed" || run.status === "blocked" || run.status === "cancelled";
|
|
}
|
|
|
|
function commandIsTerminal(command: CommandRecord | null): boolean {
|
|
return command ? terminalFromCommand(command) !== null : false;
|
|
}
|
|
|
|
async function listResultEvents(store: AgentRunStore, runId: string): Promise<ResultEventPage> {
|
|
const events: RunEvent[] = [];
|
|
let afterSeq = 0;
|
|
for (let pageIndex = 0; pageIndex < RESULT_EVENT_MAX_PAGES; pageIndex += 1) {
|
|
const page = await store.listEvents(runId, afterSeq, RESULT_EVENT_PAGE_LIMIT);
|
|
if (page.length === 0) return { events, capped: false, nextAfterSeq: null };
|
|
events.push(...page);
|
|
const nextAfterSeq = page.at(-1)?.seq ?? afterSeq;
|
|
if (nextAfterSeq <= afterSeq) return { events, capped: true, nextAfterSeq: afterSeq };
|
|
afterSeq = nextAfterSeq;
|
|
if (page.length < RESULT_EVENT_PAGE_LIMIT) return { events, capped: false, nextAfterSeq: null };
|
|
}
|
|
return { events, capped: true, nextAfterSeq: events.at(-1)?.seq ?? afterSeq };
|
|
}
|
|
|
|
async function selectCommand(store: AgentRunStore, runId: string, commandId?: string): Promise<CommandRecord | null> {
|
|
if (commandId) {
|
|
const command = await store.getCommand(commandId);
|
|
return command.runId === runId ? command : null;
|
|
}
|
|
const commands = await store.listCommands(runId, 0, 100);
|
|
return commands.at(-1) ?? null;
|
|
}
|
|
|
|
function terminalFromEvents(events: RunEvent[]): TerminalStatus | null {
|
|
for (const event of [...events].reverse()) {
|
|
if (event.type !== "terminal_status") continue;
|
|
const value = event.payload.terminalStatus;
|
|
if (value === "completed" || value === "failed" || value === "blocked" || value === "cancelled") return value;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function terminalFromCommand(command: CommandRecord): TerminalStatus | null {
|
|
if (command.state === "completed") return "completed";
|
|
if (command.state === "failed") return "failed";
|
|
if (command.state === "cancelled") return "cancelled";
|
|
return null;
|
|
}
|
|
|
|
function resultTerminal(commandTerminal: TerminalStatus | null, terminalEventStatus: TerminalStatus | null, runTerminalStatus: TerminalStatus | null, failureKind: FailureKind | null): TerminalStatus | null {
|
|
if (commandTerminal === "failed" && terminalEventStatus === "blocked" && failureKind === "required-skill-unavailable") return "blocked";
|
|
return commandTerminal ?? terminalEventStatus ?? runTerminalStatus;
|
|
}
|
|
|
|
function resultTerminalSource(commandTerminal: TerminalStatus | null, terminalEventStatus: TerminalStatus | null, runTerminalStatus: TerminalStatus | null, failureKind: FailureKind | null): string {
|
|
if (commandTerminal === "failed" && terminalEventStatus === "blocked" && failureKind === "required-skill-unavailable") return "terminal_status-event";
|
|
if (commandTerminal) return "command-record";
|
|
if (terminalEventStatus) return "terminal_status-event";
|
|
if (runTerminalStatus) return "run-record";
|
|
return "none";
|
|
}
|
|
|
|
function eventsForCommand(events: RunEvent[], commandId: string): RunEvent[] {
|
|
const scoped = events.filter((event) => event.payload.commandId === commandId || typeof event.payload.commandId !== "string");
|
|
return scoped.length > 0 ? scoped : events;
|
|
}
|
|
|
|
function failureKindFromEvents(events: RunEvent[]): FailureKind | null {
|
|
for (const event of [...events].reverse()) {
|
|
const value = event.payload.failureKind;
|
|
if (isFailureKind(value)) return value;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function resultFailureKind(run: RunRecord, command: CommandRecord | null, events: RunEvent[], terminal: TerminalStatus | null): FailureKind | null {
|
|
if (terminal === "completed") return null;
|
|
if (command) return failureKindFromEvents(events);
|
|
return run.failureKind ?? failureKindFromEvents(events);
|
|
}
|
|
|
|
function isFailureKind(value: unknown): value is FailureKind {
|
|
return typeof value === "string" && [
|
|
"cancelled",
|
|
"tenant-policy-denied",
|
|
"secret-unavailable",
|
|
"prompt-unavailable",
|
|
"prompt-too-large",
|
|
"required-skill-unavailable",
|
|
"skill-unavailable",
|
|
"runner-lease-conflict",
|
|
"backend-failed",
|
|
"backend-timeout",
|
|
"backend-response-invalid",
|
|
"thread-resume-failed",
|
|
"provider-auth-failed",
|
|
"provider-invalid-tool-call",
|
|
"provider-compact-unsupported",
|
|
"provider-rate-limited",
|
|
"provider-unavailable",
|
|
"provider-stream-disconnected",
|
|
"provider-refused-retry-recovered",
|
|
"infra-failed",
|
|
"schema-invalid",
|
|
].includes(value);
|
|
}
|
|
|
|
function messageFromEvents(events: RunEvent[]): string | null {
|
|
for (const event of [...events].reverse()) {
|
|
const value = event.payload.message;
|
|
if (typeof value === "string" && value.length > 0) return value;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function resultFailureMessage(run: RunRecord, command: CommandRecord | null, events: RunEvent[], terminal: TerminalStatus | null): string | null {
|
|
if (terminal === "completed") return null;
|
|
if (command) return messageFromEvents(events);
|
|
return run.failureMessage ?? messageFromEvents(events);
|
|
}
|
|
|
|
function detailsFromEvents(events: RunEvent[]): JsonRecord | null {
|
|
for (const event of [...events].reverse()) {
|
|
const value = event.payload.details;
|
|
if (typeof value === "object" && value !== null && !Array.isArray(value)) return value as JsonRecord;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function resultFailureDetails(events: RunEvent[], terminal: TerminalStatus | null): JsonRecord | null {
|
|
return terminal === "completed" ? null : detailsFromEvents(events);
|
|
}
|
|
|
|
function assistantReply(events: RunEvent[]): AssistantReplySummary {
|
|
const assistantEvents = events.filter((event) => event.type === "assistant_message");
|
|
const latestAuthoritative = [...assistantEvents].reverse().find((event) => (event.payload.replyAuthority === true || event.payload.final === true) && textPayload(event.payload).length > 0);
|
|
if (latestAuthoritative) return assistantReplySummary(latestAuthoritative);
|
|
const latestAssistant = [...assistantEvents].reverse().find((event) => textPayload(event.payload).length > 0);
|
|
if (latestAssistant) return assistantReplySummary(latestAssistant);
|
|
return { text: "", seq: null, source: null, final: false, replyAuthority: false, textTruncated: false, outputTruncated: false };
|
|
}
|
|
|
|
function assistantReplySummary(event: RunEvent): AssistantReplySummary {
|
|
return {
|
|
text: textPayload(event.payload),
|
|
seq: event.seq,
|
|
source: typeof event.payload.source === "string" ? event.payload.source : null,
|
|
final: event.payload.final === true,
|
|
replyAuthority: event.payload.replyAuthority === true,
|
|
textTruncated: event.payload.textTruncated === true,
|
|
outputTruncated: event.payload.outputTruncated === true,
|
|
};
|
|
}
|
|
|
|
function textPayload(payload: JsonRecord): string {
|
|
for (const key of ["text", "content", "delta"]) {
|
|
const value = payload[key];
|
|
if (typeof value === "string") return value;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
function artifactSummary(events: RunEvent[]): JsonRecord {
|
|
let commandOutputEvents = 0;
|
|
let diffEvents = 0;
|
|
let toolCallEvents = 0;
|
|
let outputChars = 0;
|
|
let outputBytes = 0;
|
|
let outputTruncatedEvents = 0;
|
|
const streamSummary: Record<string, { events: number; outputBytes: number; outputTruncated: boolean }> = {
|
|
stdout: { events: 0, outputBytes: 0, outputTruncated: false },
|
|
stderr: { events: 0, outputBytes: 0, outputTruncated: false },
|
|
};
|
|
for (const event of events) {
|
|
if (event.type === "command_output") {
|
|
commandOutputEvents += 1;
|
|
const text = textPayload(event.payload);
|
|
outputChars += text.length;
|
|
const bytes = outputBytesFromPayload(event.payload);
|
|
outputBytes += bytes;
|
|
const truncated = outputTruncatedFromPayload(event.payload);
|
|
if (truncated) outputTruncatedEvents += 1;
|
|
const stream = event.payload.stream === "stderr" ? "stderr" : "stdout";
|
|
streamSummary[stream].events += 1;
|
|
streamSummary[stream].outputBytes += bytes;
|
|
streamSummary[stream].outputTruncated ||= truncated;
|
|
}
|
|
if (event.type === "diff") diffEvents += 1;
|
|
if (event.type === "tool_call") toolCallEvents += 1;
|
|
}
|
|
return { commandOutputEvents, diffEvents, toolCallEvents, outputChars, outputBytes, truncatedEvents: outputTruncatedEvents, outputTruncatedEvents, stdoutSummary: streamSummary.stdout, stderrSummary: streamSummary.stderr };
|
|
}
|
|
|
|
function toolCallSummary(events: RunEvent[]): JsonRecord {
|
|
const toolCallEvents = events.filter((event) => event.type === "tool_call");
|
|
const statusCounts: Record<string, number> = {};
|
|
const exitCodeCounts: Record<string, number> = {};
|
|
for (const event of toolCallEvents) {
|
|
const status = typeof event.payload.status === "string" && event.payload.status.length > 0 ? event.payload.status : "unknown";
|
|
statusCounts[status] = (statusCounts[status] ?? 0) + 1;
|
|
const exitCode = normalizedExitCode(event.payload.exitCode);
|
|
if (exitCode !== null) exitCodeCounts[String(exitCode)] = (exitCodeCounts[String(exitCode)] ?? 0) + 1;
|
|
}
|
|
const window = toolCallEvents.slice(-maxToolCallSummaryItems);
|
|
return {
|
|
count: toolCallEvents.length,
|
|
statusCounts,
|
|
exitCodeCounts,
|
|
items: window.map((event) => toolCallItemSummary(event)),
|
|
itemsOmitted: Math.max(0, toolCallEvents.length - window.length),
|
|
itemWindow: "latest",
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function toolCallItemSummary(event: RunEvent): JsonRecord {
|
|
const payload = event.payload;
|
|
return {
|
|
seq: event.seq,
|
|
createdAt: event.createdAt,
|
|
method: boundedOptionalString(payload.method, toolCallFieldLimitChars),
|
|
toolName: boundedOptionalString(payload.toolName, toolCallFieldLimitChars),
|
|
type: boundedOptionalString(payload.type, toolCallFieldLimitChars),
|
|
itemId: boundedOptionalString(payload.itemId, toolCallFieldLimitChars),
|
|
status: boundedOptionalString(payload.status, toolCallFieldLimitChars),
|
|
exitCode: normalizedExitCode(payload.exitCode),
|
|
processId: boundedOptionalString(payload.processId, toolCallFieldLimitChars),
|
|
cwd: boundedOptionalString(payload.cwd, toolCallFieldLimitChars),
|
|
command: boundedOptionalString(payload.command, toolCallCommandLimitChars),
|
|
commandTruncated: optionalStringTruncated(payload.command, toolCallCommandLimitChars),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function boundedOptionalString(value: JsonValue | undefined, limitChars: number): string | null {
|
|
if (typeof value !== "string") return null;
|
|
return boundedTextSummary(value, { limitChars }).text as string;
|
|
}
|
|
|
|
function optionalStringTruncated(value: JsonValue | undefined, limitChars: number): boolean {
|
|
if (typeof value !== "string") return false;
|
|
return boundedTextSummary(value, { limitChars }).outputTruncated === true;
|
|
}
|
|
|
|
function normalizedExitCode(value: JsonValue | undefined): number | null {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
function attemptFromEvents(events: RunEvent[]): string | null {
|
|
for (const event of [...events].reverse()) {
|
|
const value = event.payload.attemptId;
|
|
if (typeof value === "string") return value;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function sessionSummary(run: RunRecord): JsonRecord | null {
|
|
if (!run.sessionRef) return null;
|
|
return {
|
|
sessionId: run.sessionRef.sessionId,
|
|
conversationId: run.sessionRef.conversationId ?? null,
|
|
threadId: run.sessionRef.threadId ?? null,
|
|
expiresAt: run.sessionRef.expiresAt ?? null,
|
|
metadataKeys: Object.keys(run.sessionRef.metadata ?? {}).sort(),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
function resourceBundleSummary(run: RunRecord, events: RunEvent[]): JsonRecord | null {
|
|
if (!run.resourceBundleRef) return null;
|
|
const materialized = events.find((event) => event.type === "backend_status" && event.payload.phase === "resource-bundle-materialized")?.payload ?? null;
|
|
return {
|
|
kind: run.resourceBundleRef.kind,
|
|
repoUrl: run.resourceBundleRef.repoUrl,
|
|
commitId: run.resourceBundleRef.commitId ?? null,
|
|
ref: run.resourceBundleRef.ref ?? null,
|
|
bundles: {
|
|
count: run.resourceBundleRef.bundles.length,
|
|
items: run.resourceBundleRef.bundles.map((item) => ({ name: item.name ?? null, repoUrl: item.repoUrl ?? run.resourceBundleRef?.repoUrl ?? null, commitId: item.commitId ?? run.resourceBundleRef?.commitId ?? null, ref: item.ref ?? run.resourceBundleRef?.ref ?? null, subpath: item.subpath, targetPath: item.targetPath, valuesPrinted: false })),
|
|
valuesPrinted: false,
|
|
},
|
|
promptRefs: run.resourceBundleRef.promptRefs ? { count: run.resourceBundleRef.promptRefs.length, names: run.resourceBundleRef.promptRefs.map((item) => item.name), required: run.resourceBundleRef.promptRefs.filter((item) => item.required === true).map((item) => item.name), valuesPrinted: false } : { count: 0, names: [], required: [], valuesPrinted: false },
|
|
requiredSkills: run.resourceBundleRef.requiredSkills ? { count: run.resourceBundleRef.requiredSkills.length, names: run.resourceBundleRef.requiredSkills.map((item) => item.name), valuesPrinted: false } : { count: 0, names: [], valuesPrinted: false },
|
|
materialized: materialized as JsonValue,
|
|
};
|
|
}
|