fix: use idle timeout for codex turns
This commit is contained in:
AgentRun Codex
@@ -59,7 +59,7 @@ export function runDiagnosis(input: RunDiagnosisInput): JsonRecord {
|
||||
} : null,
|
||||
runnerJob,
|
||||
session,
|
||||
timeoutBudget: timeoutBudget ? compactRecord(timeoutBudget, ["state", "timeoutMs", "elapsedMs", "remainingMs", "startedAt", "source"]) : null,
|
||||
timeoutBudget: timeoutBudget ? compactRecord(timeoutBudget, ["state", "timeoutKind", "timeoutMs", "elapsedMs", "idleElapsedMs", "remainingMs", "startedAt", "idleStartedAt", "lastActivityAt", "lastActivitySeq", "commandElapsedMs", "runElapsedMs", "source"]) : null,
|
||||
recoveryActions,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
@@ -116,8 +116,8 @@ function recoveryActionsForDiagnosis(input: { run: RunRecord; command: CommandRe
|
||||
actions.push({ action: "poll-events", runId: input.run.id, afterSeq: input.lastSeq, command: `./scripts/agentrun runs events ${input.run.id} --after-seq ${input.lastSeq} --limit 100 --tail-summary`, valuesPrinted: false });
|
||||
const sessionId = stringValue(input.session.sessionId);
|
||||
if (sessionId) actions.push({ action: "resume-session", sessionId, command: `./scripts/agentrun sessions turn ${sessionId} --prompt-stdin`, valuesPrinted: false });
|
||||
else actions.push({ action: "session-unavailable", reason: "sessionRef=null", hint: "当前 run 没有 sessionRef，只能从 run/events/command/runner-job 继续 drill-down 或重新提交任务。", valuesPrinted: false });
|
||||
if (input.runnerLost || input.staleClaimed || input.terminalCommandOpenRun) actions.push({ action: "refresh-queue-or-resubmit", reason: input.failureKind ?? "stale-runner-state", hint: "先用 queue refresh/show 对齐 attempt，再按任务边界决定重新派发或拆分续跑。", valuesPrinted: false });
|
||||
else actions.push({ action: "session-unavailable", reason: "sessionRef=null", hint: "当前 run 没有 sessionRef，管理者只能从 run/events/command/runner-job 读取 trace 后重新提交；这表示该任务不可同 session 续跑。", valuesPrinted: false });
|
||||
if (input.runnerLost || input.staleClaimed || input.terminalCommandOpenRun) actions.push({ action: "refresh-queue-or-resubmit", reason: input.failureKind ?? "stale-runner-state", hint: "先用 queue refresh/show 对齐 attempt；有 sessionId 时继续同一 session，没有 sessionId 才重新派发。", valuesPrinted: false });
|
||||
return actions.slice(0, 6);
|
||||
}
|
||||
|
||||
|
||||
+44
-18
@@ -118,7 +118,7 @@ function livenessSnapshot(run: RunRecord, command: CommandRecord | null, events:
|
||||
const lease = leaseSummary(run, nowMs);
|
||||
const transportDisconnect = latestTransportDisconnect(scopedEvents);
|
||||
const lastActivity = livenessActivitySummary(lastCommandActivity, nowMs);
|
||||
const timeoutBudget = timeoutBudgetSummary(run, command, terminal, failureKind, nowMs);
|
||||
const timeoutBudget = timeoutBudgetSummary(run, command, terminal, failureKind, nowMs, lastActivity);
|
||||
const terminalClassification = terminalClassificationFromEvidence({ terminal, terminalSource: "liveness", failureKind, failureMessage, timeoutBudget, transportDisconnect, lastActivity, command });
|
||||
const phase = livenessPhase({ active, command, lastVisibleActivity, leaseExpired: lease.leaseExpired, transportDisconnect, timeoutBudget, lastActivity });
|
||||
const afterSeq = lastEvent?.seq ?? 0;
|
||||
@@ -178,10 +178,13 @@ function terminalClassificationSummary(input: { terminal: TerminalStatus | null;
|
||||
|
||||
function terminalClassificationFromEvidence(input: TerminalClassificationInput): JsonRecord {
|
||||
const timeoutState = stringJsonValue(input.timeoutBudget.state);
|
||||
const hardTimeout = input.failureKind === "backend-timeout" || timeoutState === "timed-out";
|
||||
const timeoutFailure = input.failureKind === "backend-timeout" || timeoutState === "timed-out";
|
||||
const timeoutKind = stringJsonValue(input.timeoutBudget.timeoutKind) ?? (input.timeoutBudget.hardTimeout === true ? "hard" : "idle");
|
||||
const idleTimeout = timeoutFailure && timeoutKind === "idle";
|
||||
const hardTimeout = timeoutFailure && timeoutKind === "hard";
|
||||
const providerKind = providerFailureCategory(input.failureKind);
|
||||
const cancelled = input.terminal === "cancelled" || input.failureKind === "cancelled";
|
||||
const taskFailure = input.terminal === "failed" && input.failureKind !== null && !hardTimeout && !providerKind && !infrastructureFailureKind(input.failureKind);
|
||||
const taskFailure = input.terminal === "failed" && input.failureKind !== null && !timeoutFailure && !providerKind && !infrastructureFailureKind(input.failureKind);
|
||||
let category = "unknown";
|
||||
let confidence = "low";
|
||||
let providerEvidence = "not-applicable";
|
||||
@@ -200,16 +203,18 @@ function terminalClassificationFromEvidence(input: TerminalClassificationInput):
|
||||
confidence = "high";
|
||||
providerEvidence = "failure-kind";
|
||||
reason = `failureKind ${input.failureKind} is provider-specific`;
|
||||
} else if (hardTimeout && input.transportDisconnect) {
|
||||
category = "execution-hard-timeout";
|
||||
} else if (timeoutFailure && input.transportDisconnect) {
|
||||
category = idleTimeout ? "execution-idle-timeout" : "execution-hard-timeout";
|
||||
confidence = "medium";
|
||||
providerEvidence = "observed-transport-disconnect";
|
||||
reason = "hard timeout is terminal and a backend transport/app-server close event was observed, but existing events do not prove the model provider caused it";
|
||||
} else if (hardTimeout) {
|
||||
category = "execution-hard-timeout";
|
||||
reason = idleTimeout
|
||||
? "idle timeout is terminal and a backend transport/app-server close event was observed, but existing events do not prove the model provider caused it"
|
||||
: "hard timeout is terminal and a backend transport/app-server close event was observed, but existing events do not prove the model provider caused it";
|
||||
} else if (timeoutFailure) {
|
||||
category = idleTimeout ? "execution-idle-timeout" : "execution-hard-timeout";
|
||||
confidence = "high";
|
||||
providerEvidence = "insufficient";
|
||||
reason = "hard timeout is terminal; no provider-specific failure event was recorded";
|
||||
reason = idleTimeout ? "idle timeout is terminal; no provider-specific failure event was recorded" : "hard timeout is terminal; no provider-specific failure event was recorded";
|
||||
} else if (input.terminal === "blocked") {
|
||||
category = "blocked";
|
||||
confidence = "high";
|
||||
@@ -238,6 +243,9 @@ function terminalClassificationFromEvidence(input: TerminalClassificationInput):
|
||||
? "no provider-specific error or disconnect evidence was recorded"
|
||||
: null,
|
||||
hardTimeout,
|
||||
idleTimeout,
|
||||
timeoutFailure,
|
||||
timeoutKind,
|
||||
timeoutState,
|
||||
transportDisconnectObserved: Boolean(input.transportDisconnect),
|
||||
transportDisconnectSeq: input.transportDisconnect?.seq ?? null,
|
||||
@@ -261,29 +269,47 @@ function infrastructureFailureKind(failureKind: FailureKind | null): boolean {
|
||||
return failureKind.startsWith("backend-") || failureKind === "runner-lease-conflict" || failureKind === "infra-failed" || failureKind === "thread-resume-failed";
|
||||
}
|
||||
|
||||
function timeoutBudgetSummary(run: RunRecord, command: CommandRecord | null, terminal: TerminalStatus | null, failureKind: FailureKind | null, nowMs: number): JsonRecord {
|
||||
function timeoutBudgetSummary(run: RunRecord, command: CommandRecord | null, terminal: TerminalStatus | null, failureKind: FailureKind | null, nowMs: number, lastActivity: JsonRecord | null): JsonRecord {
|
||||
const timeoutMs = typeof run.executionPolicy.timeoutMs === "number" && Number.isFinite(run.executionPolicy.timeoutMs) && run.executionPolicy.timeoutMs > 0 ? Math.trunc(run.executionPolicy.timeoutMs) : null;
|
||||
const startedAt = command?.acknowledgedAt ?? command?.createdAt ?? run.updatedAt ?? run.createdAt;
|
||||
const startedMs = Date.parse(startedAt);
|
||||
const elapsedMs = timeoutMs !== null && Number.isFinite(startedMs) ? Math.max(0, nowMs - startedMs) : null;
|
||||
const commandStartedAt = command?.acknowledgedAt ?? command?.createdAt ?? null;
|
||||
const fallbackStartedAt = commandStartedAt ?? run.updatedAt ?? run.createdAt;
|
||||
const fallbackStartedMs = Date.parse(fallbackStartedAt);
|
||||
const lastActivityAt = stringJsonValue(lastActivity?.createdAt);
|
||||
const lastActivityMs = lastActivityAt ? Date.parse(lastActivityAt) : NaN;
|
||||
const idleStartedAt = Number.isFinite(lastActivityMs) ? lastActivityAt : fallbackStartedAt;
|
||||
const idleStartedMs = Number.isFinite(lastActivityMs) ? lastActivityMs : fallbackStartedMs;
|
||||
const elapsedMs = timeoutMs !== null && Number.isFinite(idleStartedMs) ? Math.max(0, nowMs - idleStartedMs) : null;
|
||||
const remainingMs = timeoutMs !== null && elapsedMs !== null ? Math.max(0, timeoutMs - elapsedMs) : null;
|
||||
const approachingThresholdMs = timeoutMs === null ? null : Math.min(120_000, Math.max(10_000, Math.floor(timeoutMs / 10)));
|
||||
const commandStartedMs = commandStartedAt ? Date.parse(commandStartedAt) : NaN;
|
||||
const commandElapsedMs = Number.isFinite(commandStartedMs) ? Math.max(0, nowMs - commandStartedMs) : null;
|
||||
const runStartedMs = Date.parse(run.createdAt);
|
||||
const runElapsedMs = Number.isFinite(runStartedMs) ? Math.max(0, nowMs - runStartedMs) : null;
|
||||
let state = "unknown";
|
||||
if (timeoutMs !== null && elapsedMs !== null) {
|
||||
if (terminal !== null) state = failureKind === "backend-timeout" ? "timed-out" : "terminal";
|
||||
else if (remainingMs === 0) state = "overdue";
|
||||
else if (approachingThresholdMs !== null && remainingMs !== null && remainingMs <= approachingThresholdMs) state = "approaching-hard-timeout";
|
||||
else if (approachingThresholdMs !== null && remainingMs !== null && remainingMs <= approachingThresholdMs) state = "approaching-idle-timeout";
|
||||
else state = "within-budget";
|
||||
}
|
||||
return {
|
||||
timeoutMs,
|
||||
source: "executionPolicy.timeoutMs",
|
||||
startedAt,
|
||||
source: "executionPolicy.timeoutMs idle activity",
|
||||
timeoutKind: "idle",
|
||||
startedAt: idleStartedAt,
|
||||
idleStartedAt,
|
||||
lastActivityAt: Number.isFinite(lastActivityMs) ? lastActivityAt : null,
|
||||
lastActivitySeq: numberJsonValue(lastActivity?.sourceSeq ?? lastActivity?.seq),
|
||||
elapsedMs,
|
||||
idleElapsedMs: elapsedMs,
|
||||
remainingMs,
|
||||
approachingThresholdMs,
|
||||
state,
|
||||
hardTimeout: true,
|
||||
hardTimeout: false,
|
||||
commandStartedAt,
|
||||
commandElapsedMs,
|
||||
runStartedAt: run.createdAt,
|
||||
runElapsedMs,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
@@ -400,7 +426,7 @@ function recoveryActions(input: { run: RunRecord; command: CommandRecord | null;
|
||||
if (terminal === "failed" || terminal === "blocked" || terminal === "cancelled") {
|
||||
if (command) actions.push({ action: "inspect-result", runId: run.id, commandId: command.id, command: `./scripts/agentrun commands result ${command.id} --run-id ${run.id}`, valuesPrinted: false });
|
||||
if (sessionId) actions.push({ action: "resume-session", sessionId, command: `./scripts/agentrun sessions turn ${sessionId} --prompt-stdin`, valuesPrinted: false });
|
||||
if (failureKind === "backend-timeout") actions.push({ action: "split-task", reason: "backend-timeout", hint: "把大 patch / 长工具链拆成更短 turn 后用同一 session 续跑", failureMessage: failureMessage ? boundedTextSummary(failureMessage, { limitChars: 200 }).text as string : null, valuesPrinted: false });
|
||||
if (failureKind === "backend-timeout") actions.push({ action: "split-task", reason: "backend-timeout", hint: "先由管理者读取 trace/result，总结下一步，再把后续 prompt 发到同一 session；必要时把大 patch / 长工具链拆成更短 turn。", failureMessage: failureMessage ? boundedTextSummary(failureMessage, { limitChars: 200 }).text as string : null, valuesPrinted: false });
|
||||
else actions.push({ action: "retry-or-split", reason: failureKind ?? "terminal", hint: "先读 trace/output 的 detail id，再决定 steer、重跑或拆分", valuesPrinted: false });
|
||||
}
|
||||
return actions;
|
||||
|
||||
@@ -184,10 +184,17 @@ function compactActivity(activity: JsonRecord): JsonRecord {
|
||||
function compactTimeoutBudget(budget: JsonRecord): JsonRecord {
|
||||
return {
|
||||
state: stringJsonValue(budget.state),
|
||||
timeoutKind: stringJsonValue(budget.timeoutKind),
|
||||
timeoutMs: numberJsonValue(budget.timeoutMs),
|
||||
elapsedMs: numberJsonValue(budget.elapsedMs),
|
||||
idleElapsedMs: numberJsonValue(budget.idleElapsedMs),
|
||||
remainingMs: numberJsonValue(budget.remainingMs),
|
||||
startedAt: stringJsonValue(budget.startedAt),
|
||||
idleStartedAt: stringJsonValue(budget.idleStartedAt),
|
||||
lastActivityAt: stringJsonValue(budget.lastActivityAt),
|
||||
lastActivitySeq: numberJsonValue(budget.lastActivitySeq),
|
||||
commandElapsedMs: numberJsonValue(budget.commandElapsedMs),
|
||||
runElapsedMs: numberJsonValue(budget.runElapsedMs),
|
||||
source: stringJsonValue(budget.source),
|
||||
valuesPrinted: false,
|
||||
};
|
||||
@@ -202,6 +209,8 @@ function compactTerminalClassification(record: JsonRecord): JsonRecord {
|
||||
providerInterruptionKnown: record.providerInterruptionKnown === true,
|
||||
providerInterruptionReason: boundedJsonString(record.providerInterruptionReason, 240),
|
||||
hardTimeout: record.hardTimeout === true,
|
||||
idleTimeout: record.idleTimeout === true,
|
||||
timeoutKind: stringJsonValue(record.timeoutKind),
|
||||
transportDisconnectObserved: record.transportDisconnectObserved === true,
|
||||
transportDisconnectSeq: numberJsonValue(record.transportDisconnectSeq),
|
||||
reason: boundedJsonString(record.reason, 240),
|
||||
|
||||
Reference in New Issue
Block a user