fix(codex): remove missing-terminal-after-tool authority (#244)
This commit is contained in:
@@ -471,7 +471,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
|||||||
const turnStartedAt = Date.now();
|
const turnStartedAt = Date.now();
|
||||||
let lastActivityAt = turnStartedAt;
|
let lastActivityAt = turnStartedAt;
|
||||||
let lastToolCall: JsonRecord | null = null;
|
let lastToolCall: JsonRecord | null = null;
|
||||||
let missingTerminalAfterToolReported = false;
|
|
||||||
let threadId: string | undefined = options.threadId;
|
let threadId: string | undefined = options.threadId;
|
||||||
let turnId: string | undefined;
|
let turnId: string | undefined;
|
||||||
let terminal: { status: TerminalStatus; failureKind: FailureKind | null; message: string | null } | null = null;
|
let terminal: { status: TerminalStatus; failureKind: FailureKind | null; message: string | null } | null = null;
|
||||||
@@ -567,11 +566,9 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
|||||||
const turnHardTimeoutMs = positiveTimeout(options.timeoutMs);
|
const turnHardTimeoutMs = positiveTimeout(options.timeoutMs);
|
||||||
const turnIdleTimeoutMs = turnHardTimeoutMs;
|
const turnIdleTimeoutMs = turnHardTimeoutMs;
|
||||||
const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs);
|
const idleWarningMs = codexIdleWarningMs(env, turnIdleTimeoutMs);
|
||||||
const missingTerminalAfterToolTimeoutMs = codexMissingTerminalAfterToolTimeoutMs(env, turnIdleTimeoutMs);
|
|
||||||
let hardTimeout: NodeJS.Timeout | null = null;
|
let hardTimeout: NodeJS.Timeout | null = null;
|
||||||
let idleTimeout: NodeJS.Timeout | null = null;
|
let idleTimeout: NodeJS.Timeout | null = null;
|
||||||
let idleWarningTimeout: NodeJS.Timeout | null = null;
|
let idleWarningTimeout: NodeJS.Timeout | null = null;
|
||||||
let missingTerminalAfterToolTimeout: NodeJS.Timeout | null = null;
|
|
||||||
let lastToolCallAt: number | null = null;
|
let lastToolCallAt: number | null = null;
|
||||||
const turnHardTimeoutAttrs = (): JsonRecord => ({
|
const turnHardTimeoutAttrs = (): JsonRecord => ({
|
||||||
waitingFor,
|
waitingFor,
|
||||||
@@ -604,46 +601,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
|||||||
hardTimeout = setTimeout(failTurnHardTimeout, turnHardTimeoutMs);
|
hardTimeout = setTimeout(failTurnHardTimeout, turnHardTimeoutMs);
|
||||||
hardTimeout.unref?.();
|
hardTimeout.unref?.();
|
||||||
};
|
};
|
||||||
const missingTerminalAfterToolAttrs = (): JsonRecord => ({
|
|
||||||
waitingFor,
|
|
||||||
idleMs: Math.max(0, Date.now() - lastActivityAt),
|
|
||||||
timeoutMs: missingTerminalAfterToolTimeoutMs,
|
|
||||||
lastNotificationMethod,
|
|
||||||
threadId: threadId ?? null,
|
|
||||||
turnId: turnId ?? null,
|
|
||||||
terminalStatus: terminal?.status ?? null,
|
|
||||||
retryable: false,
|
|
||||||
retryAttempt: null,
|
|
||||||
retryMaxAttempts: 0,
|
|
||||||
retryExhausted: true,
|
|
||||||
lastToolCall,
|
|
||||||
});
|
|
||||||
const reportMissingTerminalAfterTool = (): void => {
|
|
||||||
if (!lastToolCall || missingTerminalAfterToolReported) return;
|
|
||||||
missingTerminalAfterToolReported = true;
|
|
||||||
emitCodexOtelSpan("codex_stdio.missing_terminal_after_tool", options, env, missingTerminalAfterToolAttrs());
|
|
||||||
};
|
|
||||||
const clearMissingTerminalAfterToolTimeout = (): void => {
|
|
||||||
if (!missingTerminalAfterToolTimeout) return;
|
|
||||||
clearTimeout(missingTerminalAfterToolTimeout);
|
|
||||||
missingTerminalAfterToolTimeout = null;
|
|
||||||
};
|
|
||||||
const failMissingTerminalAfterTool = (): void => {
|
|
||||||
if (terminal || !lastToolCall) return;
|
|
||||||
reportMissingTerminalAfterTool();
|
|
||||||
terminal = { status: "failed", failureKind: "backend-timeout", message: `codex app-server did not emit turn/completed within ${missingTerminalAfterToolTimeoutMs}ms after tool activity` };
|
|
||||||
const attrs = { ...missingTerminalAfterToolAttrs(), terminalStatus: terminal.status, failureKind: terminal.failureKind };
|
|
||||||
emitEvent({ type: "error", payload: { failureKind: terminal.failureKind, message: terminal.message, phase: "turn:missing-terminal-after-tool-timeout", timeoutMs: missingTerminalAfterToolTimeoutMs, retryable: false, retryAttempt: null, retryMaxAttempts: 0, retryExhausted: true, lastToolCall } });
|
|
||||||
emitCodexOtelSpan("codex_stdio.missing_terminal_after_tool_timeout", options, env, attrs, { status: "error", error: terminal.message });
|
|
||||||
beginInterruptAndStop("missing terminal after tool timeout", "turn:missing-terminal-after-tool-timeout");
|
|
||||||
resolveTerminalNow();
|
|
||||||
};
|
|
||||||
const scheduleMissingTerminalAfterToolTimeout = (): void => {
|
|
||||||
if (!lastToolCall) return;
|
|
||||||
clearMissingTerminalAfterToolTimeout();
|
|
||||||
missingTerminalAfterToolTimeout = setTimeout(failMissingTerminalAfterTool, missingTerminalAfterToolTimeoutMs);
|
|
||||||
missingTerminalAfterToolTimeout.unref?.();
|
|
||||||
};
|
|
||||||
const scheduleIdleWarning = (): void => {
|
const scheduleIdleWarning = (): void => {
|
||||||
if (idleWarningTimeout) clearTimeout(idleWarningTimeout);
|
if (idleWarningTimeout) clearTimeout(idleWarningTimeout);
|
||||||
idleWarningTimeout = setTimeout(() => {
|
idleWarningTimeout = setTimeout(() => {
|
||||||
@@ -651,7 +608,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
|||||||
const idleMs = Math.max(0, Date.now() - lastActivityAt);
|
const idleMs = Math.max(0, Date.now() - lastActivityAt);
|
||||||
const attrs = { waitingFor, idleMs, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, terminalStatus: null };
|
const attrs = { waitingFor, idleMs, lastNotificationMethod, threadId: threadId ?? null, turnId: turnId ?? null, terminalStatus: null };
|
||||||
emitCodexOtelSpan("codex_stdio.idle_warning", options, env, attrs);
|
emitCodexOtelSpan("codex_stdio.idle_warning", options, env, attrs);
|
||||||
reportMissingTerminalAfterTool();
|
|
||||||
}, idleWarningMs);
|
}, idleWarningMs);
|
||||||
idleWarningTimeout.unref?.();
|
idleWarningTimeout.unref?.();
|
||||||
};
|
};
|
||||||
@@ -678,7 +634,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
|||||||
idleTimeout = null;
|
idleTimeout = null;
|
||||||
if (idleWarningTimeout) clearTimeout(idleWarningTimeout);
|
if (idleWarningTimeout) clearTimeout(idleWarningTimeout);
|
||||||
idleWarningTimeout = null;
|
idleWarningTimeout = null;
|
||||||
clearMissingTerminalAfterToolTimeout();
|
|
||||||
};
|
};
|
||||||
scheduleTurnHardTimeout();
|
scheduleTurnHardTimeout();
|
||||||
refreshTurnActivity();
|
refreshTurnActivity();
|
||||||
@@ -694,8 +649,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
|||||||
if (toolSummary?.status === "completed" || toolSummary?.status === "failed") {
|
if (toolSummary?.status === "completed" || toolSummary?.status === "failed") {
|
||||||
lastToolCall = toolSummary;
|
lastToolCall = toolSummary;
|
||||||
lastToolCallAt = Date.now();
|
lastToolCallAt = Date.now();
|
||||||
missingTerminalAfterToolReported = false;
|
|
||||||
scheduleMissingTerminalAfterToolTimeout();
|
|
||||||
}
|
}
|
||||||
exposeActiveTurn(normalized.turnId ? "turn-notification" : "notification");
|
exposeActiveTurn(normalized.turnId ? "turn-notification" : "notification");
|
||||||
emitEvents(normalized.events);
|
emitEvents(normalized.events);
|
||||||
@@ -813,7 +766,6 @@ async function runCodexStdioTurnWithSession(options: CodexStdioTurnOptions, sess
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (!terminal) {
|
if (!terminal) {
|
||||||
reportMissingTerminalAfterTool();
|
|
||||||
terminal = { status: "failed", failureKind: "backend-response-invalid", message: "codex app-server did not emit turn/completed" };
|
terminal = { status: "failed", failureKind: "backend-response-invalid", message: "codex app-server did not emit turn/completed" };
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -1542,12 +1494,6 @@ function codexIdleWarningMs(env: NodeJS.ProcessEnv, turnTimeoutMs: number): numb
|
|||||||
return Math.max(250, Math.floor(turnTimeoutMs / 2));
|
return Math.max(250, Math.floor(turnTimeoutMs / 2));
|
||||||
}
|
}
|
||||||
|
|
||||||
function codexMissingTerminalAfterToolTimeoutMs(env: NodeJS.ProcessEnv, turnTimeoutMs: number): number {
|
|
||||||
const configured = Number(env.AGENTRUN_CODEX_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS);
|
|
||||||
if (Number.isFinite(configured) && configured > 0) return Math.max(250, Math.floor(configured));
|
|
||||||
return positiveTimeout(turnTimeoutMs);
|
|
||||||
}
|
|
||||||
|
|
||||||
function codexTerminalNotificationDrainMs(env: NodeJS.ProcessEnv): number {
|
function codexTerminalNotificationDrainMs(env: NodeJS.ProcessEnv): number {
|
||||||
const configured = Number(env.AGENTRUN_CODEX_TERMINAL_NOTIFICATION_DRAIN_MS);
|
const configured = Number(env.AGENTRUN_CODEX_TERMINAL_NOTIFICATION_DRAIN_MS);
|
||||||
if (Number.isFinite(configured) && configured >= 0) return Math.floor(configured);
|
if (Number.isFinite(configured) && configured >= 0) return Math.floor(configured);
|
||||||
|
|||||||
@@ -49,7 +49,6 @@ export interface RunnerJobDefaults {
|
|||||||
jobNamePrefix?: string;
|
jobNamePrefix?: string;
|
||||||
lane?: string;
|
lane?: string;
|
||||||
runnerIdleTimeoutMs?: number;
|
runnerIdleTimeoutMs?: number;
|
||||||
missingTerminalAfterToolTimeoutMs?: number;
|
|
||||||
backendRetryMaxAttempts?: number;
|
backendRetryMaxAttempts?: number;
|
||||||
backendRetryInitialBackoffMs?: number;
|
backendRetryInitialBackoffMs?: number;
|
||||||
backendRetryMaxBackoffMs?: number;
|
backendRetryMaxBackoffMs?: number;
|
||||||
@@ -68,7 +67,6 @@ export interface CreateRunnerJobInput extends JsonRecord {
|
|||||||
sourceCommit?: string;
|
sourceCommit?: string;
|
||||||
serviceAccountName?: string;
|
serviceAccountName?: string;
|
||||||
runnerIdleTimeoutMs?: number;
|
runnerIdleTimeoutMs?: number;
|
||||||
missingTerminalAfterToolTimeoutMs?: number;
|
|
||||||
backendRetryMaxAttempts?: number;
|
backendRetryMaxAttempts?: number;
|
||||||
backendRetryInitialBackoffMs?: number;
|
backendRetryInitialBackoffMs?: number;
|
||||||
backendRetryMaxBackoffMs?: number;
|
backendRetryMaxBackoffMs?: number;
|
||||||
@@ -103,7 +101,6 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore;
|
|||||||
const attemptId = optionalString(options.input.attemptId) ?? `attempt_${Date.now().toString(36)}`;
|
const attemptId = optionalString(options.input.attemptId) ?? `attempt_${Date.now().toString(36)}`;
|
||||||
const runnerId = optionalString(options.input.runnerId);
|
const runnerId = optionalString(options.input.runnerId);
|
||||||
const runnerIdleTimeoutMs = optionalPositiveInteger(options.input.runnerIdleTimeoutMs, "runnerIdleTimeoutMs") ?? options.defaults.runnerIdleTimeoutMs;
|
const runnerIdleTimeoutMs = optionalPositiveInteger(options.input.runnerIdleTimeoutMs, "runnerIdleTimeoutMs") ?? options.defaults.runnerIdleTimeoutMs;
|
||||||
const missingTerminalAfterToolTimeoutMs = optionalPositiveInteger(options.input.missingTerminalAfterToolTimeoutMs, "missingTerminalAfterToolTimeoutMs") ?? options.defaults.missingTerminalAfterToolTimeoutMs;
|
|
||||||
const backendRetryMaxAttempts = optionalPositiveInteger(options.input.backendRetryMaxAttempts, "backendRetryMaxAttempts") ?? options.defaults.backendRetryMaxAttempts;
|
const backendRetryMaxAttempts = optionalPositiveInteger(options.input.backendRetryMaxAttempts, "backendRetryMaxAttempts") ?? options.defaults.backendRetryMaxAttempts;
|
||||||
const backendRetryInitialBackoffMs = optionalPositiveInteger(options.input.backendRetryInitialBackoffMs, "backendRetryInitialBackoffMs") ?? options.defaults.backendRetryInitialBackoffMs;
|
const backendRetryInitialBackoffMs = optionalPositiveInteger(options.input.backendRetryInitialBackoffMs, "backendRetryInitialBackoffMs") ?? options.defaults.backendRetryInitialBackoffMs;
|
||||||
const backendRetryMaxBackoffMs = optionalPositiveInteger(options.input.backendRetryMaxBackoffMs, "backendRetryMaxBackoffMs") ?? options.defaults.backendRetryMaxBackoffMs;
|
const backendRetryMaxBackoffMs = optionalPositiveInteger(options.input.backendRetryMaxBackoffMs, "backendRetryMaxBackoffMs") ?? options.defaults.backendRetryMaxBackoffMs;
|
||||||
@@ -120,7 +117,6 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore;
|
|||||||
attemptId: optionalString(options.input.attemptId) ?? null,
|
attemptId: optionalString(options.input.attemptId) ?? null,
|
||||||
runnerId: optionalString(options.input.runnerId) ?? null,
|
runnerId: optionalString(options.input.runnerId) ?? null,
|
||||||
runnerIdleTimeoutMs: runnerIdleTimeoutMs ?? null,
|
runnerIdleTimeoutMs: runnerIdleTimeoutMs ?? null,
|
||||||
missingTerminalAfterToolTimeoutMs: missingTerminalAfterToolTimeoutMs ?? null,
|
|
||||||
backendRetryMaxAttempts: backendRetryMaxAttempts ?? null,
|
backendRetryMaxAttempts: backendRetryMaxAttempts ?? null,
|
||||||
backendRetryInitialBackoffMs: backendRetryInitialBackoffMs ?? null,
|
backendRetryInitialBackoffMs: backendRetryInitialBackoffMs ?? null,
|
||||||
backendRetryMaxBackoffMs: backendRetryMaxBackoffMs ?? null,
|
backendRetryMaxBackoffMs: backendRetryMaxBackoffMs ?? null,
|
||||||
@@ -205,7 +201,6 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore;
|
|||||||
sourceCommit,
|
sourceCommit,
|
||||||
transientEnv: renderTransientEnv,
|
transientEnv: renderTransientEnv,
|
||||||
...(runnerIdleTimeoutMs !== undefined ? { runnerIdleTimeoutMs } : {}),
|
...(runnerIdleTimeoutMs !== undefined ? { runnerIdleTimeoutMs } : {}),
|
||||||
...(missingTerminalAfterToolTimeoutMs !== undefined ? { missingTerminalAfterToolTimeoutMs } : {}),
|
|
||||||
...(backendRetryMaxAttempts !== undefined ? { backendRetryMaxAttempts } : {}),
|
...(backendRetryMaxAttempts !== undefined ? { backendRetryMaxAttempts } : {}),
|
||||||
...(backendRetryInitialBackoffMs !== undefined ? { backendRetryInitialBackoffMs } : {}),
|
...(backendRetryInitialBackoffMs !== undefined ? { backendRetryInitialBackoffMs } : {}),
|
||||||
...(backendRetryMaxBackoffMs !== undefined ? { backendRetryMaxBackoffMs } : {}),
|
...(backendRetryMaxBackoffMs !== undefined ? { backendRetryMaxBackoffMs } : {}),
|
||||||
|
|||||||
@@ -56,7 +56,6 @@ function runnerJobDefaultsForRequest(defaults: ManagerServerOptions["runnerJobDe
|
|||||||
jobNamePrefix,
|
jobNamePrefix,
|
||||||
lane,
|
lane,
|
||||||
...(defaults?.runnerIdleTimeoutMs !== undefined ? { runnerIdleTimeoutMs: defaults.runnerIdleTimeoutMs } : optionalPositiveIntegerRecord("runnerIdleTimeoutMs", process.env.AGENTRUN_RUNNER_IDLE_TIMEOUT_MS)),
|
...(defaults?.runnerIdleTimeoutMs !== undefined ? { runnerIdleTimeoutMs: defaults.runnerIdleTimeoutMs } : optionalPositiveIntegerRecord("runnerIdleTimeoutMs", process.env.AGENTRUN_RUNNER_IDLE_TIMEOUT_MS)),
|
||||||
...(defaults?.missingTerminalAfterToolTimeoutMs !== undefined ? { missingTerminalAfterToolTimeoutMs: defaults.missingTerminalAfterToolTimeoutMs } : optionalPositiveIntegerRecord("missingTerminalAfterToolTimeoutMs", process.env.AGENTRUN_RUNNER_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS)),
|
|
||||||
...(defaults?.backendRetryMaxAttempts !== undefined ? { backendRetryMaxAttempts: defaults.backendRetryMaxAttempts } : optionalPositiveIntegerRecord("backendRetryMaxAttempts", process.env.AGENTRUN_BACKEND_RETRY_MAX_ATTEMPTS)),
|
...(defaults?.backendRetryMaxAttempts !== undefined ? { backendRetryMaxAttempts: defaults.backendRetryMaxAttempts } : optionalPositiveIntegerRecord("backendRetryMaxAttempts", process.env.AGENTRUN_BACKEND_RETRY_MAX_ATTEMPTS)),
|
||||||
...(defaults?.backendRetryInitialBackoffMs !== undefined ? { backendRetryInitialBackoffMs: defaults.backendRetryInitialBackoffMs } : optionalPositiveIntegerRecord("backendRetryInitialBackoffMs", process.env.AGENTRUN_BACKEND_RETRY_INITIAL_BACKOFF_MS)),
|
...(defaults?.backendRetryInitialBackoffMs !== undefined ? { backendRetryInitialBackoffMs: defaults.backendRetryInitialBackoffMs } : optionalPositiveIntegerRecord("backendRetryInitialBackoffMs", process.env.AGENTRUN_BACKEND_RETRY_INITIAL_BACKOFF_MS)),
|
||||||
...(defaults?.backendRetryMaxBackoffMs !== undefined ? { backendRetryMaxBackoffMs: defaults.backendRetryMaxBackoffMs } : optionalPositiveIntegerRecord("backendRetryMaxBackoffMs", process.env.AGENTRUN_BACKEND_RETRY_MAX_BACKOFF_MS)),
|
...(defaults?.backendRetryMaxBackoffMs !== undefined ? { backendRetryMaxBackoffMs: defaults.backendRetryMaxBackoffMs } : optionalPositiveIntegerRecord("backendRetryMaxBackoffMs", process.env.AGENTRUN_BACKEND_RETRY_MAX_BACKOFF_MS)),
|
||||||
@@ -114,7 +113,6 @@ export interface ManagerServerOptions {
|
|||||||
jobNamePrefix?: string;
|
jobNamePrefix?: string;
|
||||||
lane?: string;
|
lane?: string;
|
||||||
runnerIdleTimeoutMs?: number;
|
runnerIdleTimeoutMs?: number;
|
||||||
missingTerminalAfterToolTimeoutMs?: number;
|
|
||||||
backendRetryMaxAttempts?: number;
|
backendRetryMaxAttempts?: number;
|
||||||
backendRetryInitialBackoffMs?: number;
|
backendRetryInitialBackoffMs?: number;
|
||||||
backendRetryMaxBackoffMs?: number;
|
backendRetryMaxBackoffMs?: number;
|
||||||
|
|||||||
+4
-13
@@ -58,7 +58,6 @@ export interface RunnerJobRenderOptions {
|
|||||||
backoffLimit?: number;
|
backoffLimit?: number;
|
||||||
ttlSecondsAfterFinished?: number;
|
ttlSecondsAfterFinished?: number;
|
||||||
runnerIdleTimeoutMs?: number;
|
runnerIdleTimeoutMs?: number;
|
||||||
missingTerminalAfterToolTimeoutMs?: number;
|
|
||||||
backendRetryMaxAttempts?: number;
|
backendRetryMaxAttempts?: number;
|
||||||
backendRetryInitialBackoffMs?: number;
|
backendRetryInitialBackoffMs?: number;
|
||||||
backendRetryMaxBackoffMs?: number;
|
backendRetryMaxBackoffMs?: number;
|
||||||
@@ -156,7 +155,7 @@ export function renderRunnerJobDryRun(options: RunnerJobRenderOptions): JsonReco
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { manifest: JsonRecord; namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; serviceAccountName: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; warnings: string[]; ttlSecondsAfterFinished: number; ttlPolicy: JsonRecord; runnerIdleTimeoutMs: number; missingTerminalAfterToolTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number } {
|
export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { manifest: JsonRecord; namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; serviceAccountName: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; warnings: string[]; ttlSecondsAfterFinished: number; ttlPolicy: JsonRecord; runnerIdleTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number } {
|
||||||
const namespace = options.namespace ?? "agentrun-v01";
|
const namespace = options.namespace ?? "agentrun-v01";
|
||||||
const attemptId = options.attemptId ?? `attempt_${Date.now().toString(36)}`;
|
const attemptId = options.attemptId ?? `attempt_${Date.now().toString(36)}`;
|
||||||
const runnerId = options.runnerId ?? `runner_${shortHash(`${options.run.id}:${attemptId}:${options.commandId}`)}`;
|
const runnerId = options.runnerId ?? `runner_${shortHash(`${options.run.id}:${attemptId}:${options.commandId}`)}`;
|
||||||
@@ -169,7 +168,6 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani
|
|||||||
const ttlSecondsAfterFinished = normalizeTtlSecondsAfterFinished(options.ttlSecondsAfterFinished, warnings);
|
const ttlSecondsAfterFinished = normalizeTtlSecondsAfterFinished(options.ttlSecondsAfterFinished, warnings);
|
||||||
const ttlPolicy = terminalArtifactTtlPolicy(ttlSecondsAfterFinished);
|
const ttlPolicy = terminalArtifactTtlPolicy(ttlSecondsAfterFinished);
|
||||||
const runnerIdleTimeoutMs = normalizeRunnerIdleTimeoutMs(options.runnerIdleTimeoutMs);
|
const runnerIdleTimeoutMs = normalizeRunnerIdleTimeoutMs(options.runnerIdleTimeoutMs);
|
||||||
const missingTerminalAfterToolTimeoutMs = normalizeMissingTerminalAfterToolTimeoutMs(options.missingTerminalAfterToolTimeoutMs, runnerIdleTimeoutMs);
|
|
||||||
const backendRetryMaxAttempts = normalizeBackendRetryMaxAttempts(options.backendRetryMaxAttempts);
|
const backendRetryMaxAttempts = normalizeBackendRetryMaxAttempts(options.backendRetryMaxAttempts);
|
||||||
const backendRetryInitialBackoffMs = normalizeBackendRetryBackoffMs(options.backendRetryInitialBackoffMs, "backendRetryInitialBackoffMs", 1000);
|
const backendRetryInitialBackoffMs = normalizeBackendRetryBackoffMs(options.backendRetryInitialBackoffMs, "backendRetryInitialBackoffMs", 1000);
|
||||||
const backendRetryMaxBackoffMs = normalizeBackendRetryBackoffMs(options.backendRetryMaxBackoffMs, "backendRetryMaxBackoffMs", 30000);
|
const backendRetryMaxBackoffMs = normalizeBackendRetryBackoffMs(options.backendRetryMaxBackoffMs, "backendRetryMaxBackoffMs", 30000);
|
||||||
@@ -178,7 +176,7 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani
|
|||||||
const toolCredentials = toolCredentialProjections(options.run, namespace);
|
const toolCredentials = toolCredentialProjections(options.run, namespace);
|
||||||
const sessionPvc = options.sessionPvc;
|
const sessionPvc = options.sessionPvc;
|
||||||
if (secretRefs.length === 0) warnings.push("run executionPolicy.secretScope 未声明 provider SecretRef;runner 将按 secret-unavailable 上报,而不会降级直连外部凭据");
|
if (secretRefs.length === 0) warnings.push("run executionPolicy.secretScope 未声明 provider SecretRef;runner 将按 secret-unavailable 上报,而不会降级直连外部凭据");
|
||||||
const env = runnerEnv(options, { namespace, jobName, runnerJobId, runnerId, attemptId, sourceCommit, secretRefs, toolCredentials, sessionPvc, runnerIdleTimeoutMs, missingTerminalAfterToolTimeoutMs, backendRetryMaxAttempts, backendRetryInitialBackoffMs, backendRetryMaxBackoffMs });
|
const env = runnerEnv(options, { namespace, jobName, runnerJobId, runnerId, attemptId, sourceCommit, secretRefs, toolCredentials, sessionPvc, runnerIdleTimeoutMs, backendRetryMaxAttempts, backendRetryInitialBackoffMs, backendRetryMaxBackoffMs });
|
||||||
const manifest: JsonRecord = {
|
const manifest: JsonRecord = {
|
||||||
apiVersion: "batch/v1",
|
apiVersion: "batch/v1",
|
||||||
kind: "Job",
|
kind: "Job",
|
||||||
@@ -245,10 +243,10 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
return { manifest, namespace, jobName, runnerJobId, runnerId, attemptId, sourceCommit, serviceAccountName, secretRefs, toolCredentials, warnings, ttlSecondsAfterFinished, ttlPolicy, runnerIdleTimeoutMs, missingTerminalAfterToolTimeoutMs, backendRetryMaxAttempts, backendRetryInitialBackoffMs, backendRetryMaxBackoffMs };
|
return { manifest, namespace, jobName, runnerJobId, runnerId, attemptId, sourceCommit, serviceAccountName, secretRefs, toolCredentials, warnings, ttlSecondsAfterFinished, ttlPolicy, runnerIdleTimeoutMs, backendRetryMaxAttempts, backendRetryInitialBackoffMs, backendRetryMaxBackoffMs };
|
||||||
}
|
}
|
||||||
|
|
||||||
function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; sessionPvc: RunnerSessionPvcOptions | undefined; runnerIdleTimeoutMs: number; missingTerminalAfterToolTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number }): JsonRecord[] {
|
function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; sessionPvc: RunnerSessionPvcOptions | undefined; runnerIdleTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number }): JsonRecord[] {
|
||||||
const selectedSecret = context.secretRefs.find((item) => item.profile === options.run.backendProfile);
|
const selectedSecret = context.secretRefs.find((item) => item.profile === options.run.backendProfile);
|
||||||
const codexHome = runnerCodexHome(options.run.backendProfile, selectedSecret, context.sessionPvc);
|
const codexHome = runnerCodexHome(options.run.backendProfile, selectedSecret, context.sessionPvc);
|
||||||
const bootRepoUrl = optionalString(options.bootRepoUrl) ?? defaultBootRepoUrl;
|
const bootRepoUrl = optionalString(options.bootRepoUrl) ?? defaultBootRepoUrl;
|
||||||
@@ -278,7 +276,6 @@ function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string
|
|||||||
{ name: "AGENTRUN_WORK_READY_VERSION", value: String(staticWorkReadyCapabilitySummary().version) },
|
{ name: "AGENTRUN_WORK_READY_VERSION", value: String(staticWorkReadyCapabilitySummary().version) },
|
||||||
{ name: "AGENTRUN_PROJECT_DEPENDENCY_POLICY", value: "explicit-cache-or-derived-image-only" },
|
{ name: "AGENTRUN_PROJECT_DEPENDENCY_POLICY", value: "explicit-cache-or-derived-image-only" },
|
||||||
{ name: "AGENTRUN_RUNNER_IDLE_TIMEOUT_MS", value: String(context.runnerIdleTimeoutMs) },
|
{ name: "AGENTRUN_RUNNER_IDLE_TIMEOUT_MS", value: String(context.runnerIdleTimeoutMs) },
|
||||||
{ name: "AGENTRUN_CODEX_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS", value: String(context.missingTerminalAfterToolTimeoutMs) },
|
|
||||||
{ name: "AGENTRUN_BACKEND_RETRY_MAX_ATTEMPTS", value: String(context.backendRetryMaxAttempts) },
|
{ name: "AGENTRUN_BACKEND_RETRY_MAX_ATTEMPTS", value: String(context.backendRetryMaxAttempts) },
|
||||||
{ name: "AGENTRUN_BACKEND_RETRY_INITIAL_BACKOFF_MS", value: String(context.backendRetryInitialBackoffMs) },
|
{ name: "AGENTRUN_BACKEND_RETRY_INITIAL_BACKOFF_MS", value: String(context.backendRetryInitialBackoffMs) },
|
||||||
{ name: "AGENTRUN_BACKEND_RETRY_MAX_BACKOFF_MS", value: String(context.backendRetryMaxBackoffMs) },
|
{ name: "AGENTRUN_BACKEND_RETRY_MAX_BACKOFF_MS", value: String(context.backendRetryMaxBackoffMs) },
|
||||||
@@ -309,12 +306,6 @@ function normalizeRunnerIdleTimeoutMs(value: number | undefined): number {
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeMissingTerminalAfterToolTimeoutMs(value: number | undefined, runnerIdleTimeoutMs: number): number {
|
|
||||||
if (value === undefined) return runnerIdleTimeoutMs;
|
|
||||||
if (!Number.isInteger(value) || value <= 0) throw new Error("missingTerminalAfterToolTimeoutMs must be a positive integer");
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
function normalizeBackendRetryMaxAttempts(value: number | undefined): number {
|
function normalizeBackendRetryMaxAttempts(value: number | undefined): number {
|
||||||
if (value === undefined) return 1;
|
if (value === undefined) return 1;
|
||||||
if (!Number.isInteger(value) || value <= 0) throw new Error("backendRetryMaxAttempts must be a positive integer");
|
if (!Number.isInteger(value) || value <= 0) throw new Error("backendRetryMaxAttempts must be a positive integer");
|
||||||
|
|||||||
@@ -263,18 +263,18 @@ process.exit(1);
|
|||||||
assert.ok(steerEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "steer-command-acknowledged" && event.payload?.commandId === steerCommand.id && event.payload?.targetCommandId === steerRun.commandId));
|
assert.ok(steerEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "steer-command-acknowledged" && event.payload?.commandId === steerCommand.id && event.payload?.targetCommandId === steerRun.commandId));
|
||||||
assert.ok(steerEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "turn/steer:completed" && event.payload?.commandId === steerCommand.id && event.payload?.targetCommandId === steerRun.commandId && event.payload.deliveryState === "forwarded-to-backend" && event.payload.targetEffect === "not-guaranteed"));
|
assert.ok(steerEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "turn/steer:completed" && event.payload?.commandId === steerCommand.id && event.payload?.targetCommandId === steerRun.commandId && event.payload.deliveryState === "forwarded-to-backend" && event.payload.targetEffect === "not-guaranteed"));
|
||||||
|
|
||||||
const idleAfterTool = await createHwlabRun(client, context, bundle, "hwlab-session-idle-after-tool", "complete a tool and then fail without terminal", "hwlab-command-idle-after-tool", 10_000);
|
const noEventWatchdog = await createHwlabRun(client, context, bundle, "hwlab-session-no-event-watchdog", "complete a tool and then produce no more events", "hwlab-command-no-event-watchdog", 10_000);
|
||||||
const idleAfterToolRunner = runOnce({ managerUrl: server.baseUrl, runId: idleAfterTool.runId, commandId: idleAfterTool.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-completes-without-terminal", AGENTRUN_CODEX_MISSING_TERMINAL_AFTER_TOOL_TIMEOUT_MS: "300", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-idle-after-tool") }, oneShot: true, pollIntervalMs: 50 });
|
const noEventWatchdogRunner = runOnce({ managerUrl: server.baseUrl, runId: noEventWatchdog.runId, commandId: noEventWatchdog.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-completes-without-terminal", AGENTRUN_RUNNER_IDLE_TIMEOUT_MS: "300", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-no-event-watchdog") }, oneShot: true, pollIntervalMs: 50 });
|
||||||
await waitForCommandState(client, idleAfterTool.runId, idleAfterTool.commandId, "acknowledged");
|
await waitForCommandState(client, noEventWatchdog.runId, noEventWatchdog.commandId, "acknowledged");
|
||||||
await waitForEvent(client, idleAfterTool.runId, (event) => event.type === "tool_call" && (event.payload as JsonRecord).status === "completed", "tool_call completed without terminal");
|
await waitForEvent(client, noEventWatchdog.runId, (event) => event.type === "tool_call" && (event.payload as JsonRecord).status === "completed", "tool_call completed before no-event watchdog");
|
||||||
await waitForCommandState(client, idleAfterTool.runId, idleAfterTool.commandId, "failed");
|
await waitForCommandState(client, noEventWatchdog.runId, noEventWatchdog.commandId, "failed");
|
||||||
const idleAfterToolResult = await idleAfterToolRunner as JsonRecord;
|
const noEventWatchdogResult = await noEventWatchdogRunner as JsonRecord;
|
||||||
assert.equal(idleAfterToolResult.terminalStatus, "failed");
|
assert.equal(noEventWatchdogResult.terminalStatus, "failed");
|
||||||
assert.equal(idleAfterToolResult.failureKind, "backend-timeout");
|
assert.equal(noEventWatchdogResult.failureKind, "backend-timeout");
|
||||||
const idleEnvelope = await client.get(`/api/v1/runs/${idleAfterTool.runId}/commands/${idleAfterTool.commandId}/result`) as JsonRecord;
|
const idleEnvelope = await client.get(`/api/v1/runs/${noEventWatchdog.runId}/commands/${noEventWatchdog.commandId}/result`) as JsonRecord;
|
||||||
assert.equal(idleEnvelope.terminalStatus, "failed");
|
assert.equal(idleEnvelope.terminalStatus, "failed");
|
||||||
assert.equal(idleEnvelope.failureKind, "backend-timeout");
|
assert.equal(idleEnvelope.failureKind, "backend-timeout");
|
||||||
assert.match(String(idleEnvelope.failureMessage ?? idleEnvelope.message ?? ""), /did not emit turn\/completed/u);
|
assert.match(String(idleEnvelope.failureMessage ?? idleEnvelope.message ?? ""), /idle timed out/u);
|
||||||
|
|
||||||
const outputWithoutTerminal = await createHwlabRun(client, context, bundle, "hwlab-session-output-without-terminal", "start a tool that keeps printing without terminal", "hwlab-command-output-without-terminal", 500);
|
const outputWithoutTerminal = await createHwlabRun(client, context, bundle, "hwlab-session-output-without-terminal", "start a tool that keeps printing without terminal", "hwlab-command-output-without-terminal", 500);
|
||||||
const outputWithoutTerminalRunner = runOnce({ managerUrl: server.baseUrl, runId: outputWithoutTerminal.runId, commandId: outputWithoutTerminal.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-output-without-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-output-without-terminal") }, oneShot: true, pollIntervalMs: 50 });
|
const outputWithoutTerminalRunner = runOnce({ managerUrl: server.baseUrl, runId: outputWithoutTerminal.runId, commandId: outputWithoutTerminal.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-output-without-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-output-without-terminal") }, oneShot: true, pollIntervalMs: 50 });
|
||||||
@@ -296,7 +296,7 @@ process.exit(1);
|
|||||||
const runningResult = await running;
|
const runningResult = await running;
|
||||||
assert.equal(runningResult.terminalStatus, "cancelled");
|
assert.equal(runningResult.terminalStatus, "cancelled");
|
||||||
|
|
||||||
return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-blocker", "same-run-runner-multiturn", "running-steer", "missing-terminal-after-tool-auto-stop", "tool-output-hard-timeout", "running-cancel"] };
|
return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-blocker", "same-run-runner-multiturn", "running-steer", "no-event-watchdog-after-tool", "tool-output-hard-timeout", "running-cancel"] };
|
||||||
} finally {
|
} finally {
|
||||||
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user