import { RunnerManagerApi, failureKindFromError, terminalStatusForFailure, errorMessage } from "./manager-api.js"; import { createBackendSession, runBackendTurn, type BackendActiveTurnControl, type BackendAdapterOptions, type BackendSession } from "../backend/adapter.js"; import { materializeResourceBundle } from "./resource-bundle.js"; import type { BackendEvent, BackendProfile, CommandRecord, FailureKind, InitialPromptAssembly, JsonRecord, RunRecord, RunnerRecord, TerminalStatus } from "../common/types.js"; import { AgentRunError } from "../common/errors.js"; export interface RunnerOnceOptions extends BackendAdapterOptions { managerUrl: string; runId: string; commandId?: string; runnerId?: string; attemptId?: string; leaseMs?: number; backendProfile?: BackendProfile; placement?: "host-process" | "kubernetes-job"; sourceCommit?: string; jobName?: string; podName?: string; logPath?: string; idleTimeoutMs?: number; pollIntervalMs?: number; claimRetryTimeoutMs?: number; claimRetryIntervalMs?: number; oneShot?: boolean; } interface CommandExecutionResult extends JsonRecord { commandId: string; terminalStatus: TerminalStatus; failureKind: FailureKind | null; } export async function runOnce(options: RunnerOnceOptions): Promise { const api = new RunnerManagerApi(options.managerUrl); const targetRun = await api.getRun(options.runId); if (isTerminalRun(targetRun)) return { terminalStatus: targetRun.terminalStatus, failureKind: targetRun.failureKind, run: targetRun, skipped: "run-terminal" } as JsonRecord; if (options.backendProfile && options.backendProfile !== targetRun.backendProfile) { throw new AgentRunError("schema-invalid", `runner backendProfile ${options.backendProfile} does not match run backendProfile ${targetRun.backendProfile}`, { httpStatus: 400 }); } const leaseMs = options.leaseMs ?? 60_000; const attemptId = options.attemptId ?? `attempt_${Date.now().toString(36)}`; const runner = await api.register({ runId: options.runId, attemptId, backendProfile: targetRun.backendProfile, placement: options.placement ?? "host-process", sourceCommit: options.sourceCommit ?? process.env.AGENTRUN_SOURCE_COMMIT ?? "unknown", ...(options.runnerId ? { runnerId: options.runnerId } : {}), ...(options.jobName ? { jobName: options.jobName } : {}), ...(options.podName ? { podName: options.podName } : {}), ...(options.logPath ? { logPath: options.logPath } : {}), }) as RunnerRecord; let claimed: RunRecord; try { claimed = await claimRunWithLeaseRecovery(api, options, runner, attemptId, leaseMs); await api.heartbeat(options.runId, runner.id, leaseMs); } catch (error) { const failureKind = failureKindFromError(error); if (failureKind !== "runner-lease-conflict") { await api.reportFailure(options.runId, { terminalStatus: terminalStatusForFailure(failureKind), failureKind, failureMessage: errorMessage(error) }); } throw error; } const stopHeartbeat = startHeartbeat(api, options.runId, runner.id, leaseMs); const idleTimeoutMs = options.idleTimeoutMs ?? 120_000; const pollIntervalMs = normalizePollIntervalMs(options.pollIntervalMs); const commandResults: CommandExecutionResult[] = []; let workspacePath: string | undefined; let resourceEnv: NodeJS.ProcessEnv | undefined; let initialPrompt: InitialPromptAssembly | undefined; let materializationAttempted = false; let materializationFailure: { failureKind: FailureKind; terminalStatus: TerminalStatus; message: string } | null = null; let backendSession: BackendSession | null = null; try { let idleSince = Date.now(); let firstPoll = true; while (true) { const currentRun = await api.getRun(options.runId); if (isTerminalRun(currentRun)) return { runner, claimed, terminalStatus: currentRun.terminalStatus, failureKind: currentRun.failureKind, run: currentRun, commandsProcessed: commandResults.length, commandResults, stopped: "run-terminal" } as JsonRecord; const commandsResponse = await api.pollCommands(options.runId, { afterSeq: 0, limit: 20, ...(firstPoll && options.commandId ? { commandId: options.commandId } : {}) }); firstPoll = false; const command = commandsResponse.selected; if (!command) { await failPendingSteerWithoutActiveTurn(api, options.runId, commandsResponse.items, runner.id, attemptId); if (options.oneShot === true) return noPendingResult(runner, claimed, commandResults, commandsResponse.items.length, "one-shot-no-pending"); if (Date.now() - idleSince >= idleTimeoutMs) return noPendingResult(runner, claimed, commandResults, commandsResponse.items.length, "idle-timeout"); await sleep(pollIntervalMs); continue; } idleSince = Date.now(); if (!materializationAttempted) { materializationAttempted = true; try { const materialized = await materializeResourceBundle(claimed.resourceBundleRef ?? null, resourceMaterializationEnv(options.env ?? process.env, options.runId, attemptId, claimed.workspaceRef)); if (materialized) { workspacePath = materialized.workspacePath; resourceEnv = resourceEnvForMaterialized(options.env ?? process.env, materialized); initialPrompt = materialized.initialPrompt; await api.appendEvent(options.runId, { type: "backend_status", payload: { ...materialized.event, commandId: command.id, attemptId, runnerId: runner.id } }); } } catch (error) { const failureKind = failureKindFromError(error); materializationFailure = { failureKind, terminalStatus: terminalStatusForFailure(failureKind), message: errorMessage(error) }; } } const result = materializationFailure ? await reportCommandFailure(api, options.runId, command.id, runner, attemptId, materializationFailure, "runner:resource-bundle", { terminalRun: true }) : await executeCommand(api, withResourceAssembly(options, resourceEnv, initialPrompt), command, runner, attemptId, workspacePath, backendSession ?? (backendSession = createBackendSession(currentRun, withResourceAssembly(options, resourceEnv, initialPrompt)))); commandResults.push(result); if (options.oneShot === true) { const run = await api.reportStatus(options.runId, { terminalStatus: result.terminalStatus, failureKind: result.failureKind, failureMessage: null }); return { runner, commandId: command.id, terminalStatus: result.terminalStatus, failureKind: result.failureKind, run, commandsProcessed: commandResults.length, commandResults, stopped: "one-shot" } as JsonRecord; } } } catch (error) { const failureKind = failureKindFromError(error); const terminalStatus = terminalStatusForFailure(failureKind); const message = errorMessage(error); await api.appendEvent(options.runId, { type: "error", payload: { failureKind, message, phase: "runner:loop", attemptId, runnerId: runner.id } }); const finalRun = await api.reportStatus(options.runId, { terminalStatus, failureKind, failureMessage: message }) as RunRecord; return { runner, terminalStatus, failureKind, run: finalRun, commandsProcessed: commandResults.length, commandResults } as JsonRecord; } finally { if (backendSession) { try { const closeEvents = await backendSession.close(); for (const event of closeEvents) await api.appendEvent(options.runId, annotateCommandEvent(event, "runner-shutdown", attemptId, runner.id)); } catch { // Runner shutdown must not hide the terminal result already reported for the command. } } stopHeartbeat(); } } function withResourceAssembly(options: RunnerOnceOptions, resourceEnv: NodeJS.ProcessEnv | undefined, initialPrompt: InitialPromptAssembly | undefined): RunnerOnceOptions { return { ...options, ...(resourceEnv ? { env: resourceEnv } : {}), ...(initialPrompt ? { initialPrompt } : {}), }; } function resourceMaterializationEnv(env: NodeJS.ProcessEnv, runId: string, attemptId: string, workspaceRef: RunRecord["workspaceRef"]): NodeJS.ProcessEnv { const workspaceBranch = typeof workspaceRef.branch === "string" && workspaceRef.branch.trim().length > 0 ? workspaceRef.branch.trim() : undefined; return { ...env, AGENTRUN_RUN_ID: env.AGENTRUN_RUN_ID ?? runId, AGENTRUN_ATTEMPT_ID: env.AGENTRUN_ATTEMPT_ID ?? attemptId, ...(workspaceBranch ? { AGENTRUN_WORKSPACE_BRANCH: env.AGENTRUN_WORKSPACE_BRANCH ?? workspaceBranch, AGENTRUN_WORKSPACE_REF: env.AGENTRUN_WORKSPACE_REF ?? workspaceBranch } : {}), }; } function resourceEnvForMaterialized(env: NodeJS.ProcessEnv, materialized: Awaited>): NodeJS.ProcessEnv | undefined { if (!materialized) return undefined; let next: NodeJS.ProcessEnv | undefined; if (materialized.binPath) next = prependPath(env, materialized.binPath); if (materialized.skillsDir) { const base = next ?? { ...env }; const previous = base.AGENTRUN_SKILLS_DIRS; next = { ...base, AGENTRUN_SKILLS_DIRS: previous && previous.trim().length > 0 ? `${materialized.skillsDir}${pathDelimiter()}${previous}` : materialized.skillsDir, HWLAB_CODE_AGENT_SKILLS_DIRS: materialized.skillsDir, }; } return next; } function prependPath(env: NodeJS.ProcessEnv, binPath: string): NodeJS.ProcessEnv { return { ...env, PATH: `${binPath}${pathDelimiter()}${env.PATH ?? process.env.PATH ?? ""}` }; } function pathDelimiter(): string { return process.platform === "win32" ? ";" : ":"; } async function executeCommand(api: RunnerManagerApi, options: RunnerOnceOptions, command: CommandRecord, runner: RunnerRecord, attemptId: string, workspacePath: string | undefined, backendSession: BackendSession | null): Promise { await api.ackCommand(command.id); const acked = await api.getCommand(options.runId, command.id); if (acked.state === "cancelled") return await reportCancelled(api, options.runId, command.id, runner, attemptId, "command cancelled before backend start"); await assertNotCancelled(api, options.runId, command.id); await api.appendEvent(options.runId, { type: "backend_status", payload: { phase: "backend-turn-started", commandId: command.id, attemptId, runnerId: runner.id, backendProfile: options.backendProfile ?? null, workspaceReady: Boolean(workspacePath) } }); const abortController = new AbortController(); const stopCancelWatch = watchCancellation(api, options.runId, command.id, abortController); const backendProgress = startBackendProgress(); let stopSteerWatch: (() => void) | undefined; try { const latestRun = await api.getRun(options.runId); const backendOptions = { ...options, ...(workspacePath ? { workspacePath } : {}), abortSignal: abortController.signal, onEvent: async (event: BackendEvent) => { await appendBestEffort(api, options.runId, annotateCommandEvent(event, command.id, attemptId, runner.id)); }, onActiveTurn: (control: BackendActiveTurnControl) => { stopSteerWatch = startSteerWatch(api, options.runId, command.id, attemptId, runner.id, control, options.pollIntervalMs); return () => { stopSteerWatch?.(); stopSteerWatch = undefined; }; }, }; const result = backendSession ? await backendSession.runTurn(latestRun, command, backendOptions) : await runBackendTurn(latestRun, command, backendOptions); for (const event of result.events) { await api.appendEvent(options.runId, annotateCommandEvent(event, command.id, attemptId, runner.id)); } await api.reportCommandStatus(command.id, { terminalStatus: result.terminalStatus, failureKind: result.failureKind, failureMessage: result.failureMessage, ...(result.threadId ? { threadId: result.threadId } : {}), ...(result.turnId ? { turnId: result.turnId } : {}) }); return { commandId: command.id, terminalStatus: result.terminalStatus, failureKind: result.failureKind } as CommandExecutionResult; } catch (error) { const failureKind = failureKindFromError(error); const failure = { failureKind, terminalStatus: terminalStatusForFailure(failureKind), message: errorMessage(error) }; return await reportCommandFailure(api, options.runId, command.id, runner, attemptId, failure, "runner:execute"); } finally { stopSteerWatch?.(); const progressSummary = backendProgress.stop(); await appendBestEffort(api, options.runId, { type: "backend_status", payload: { phase: "backend-turn-finished", commandId: command.id, attemptId, runnerId: runner.id, ...progressSummary } }); stopCancelWatch(); } } function startSteerWatch(api: RunnerManagerApi, runId: string, targetCommandId: string, attemptId: string, runnerId: string, control: BackendActiveTurnControl, pollIntervalMs: number | undefined): () => void { let stopped = false; let polling = false; const seen = new Set(); const intervalMs = normalizePollIntervalMs(pollIntervalMs); const poll = async (): Promise => { if (stopped || polling) return; polling = true; try { const commands = await api.listCommands(runId, { afterSeq: 0, limit: 100 }); const pendingSteer = commands.filter((item) => item.type === "steer" && item.state === "pending" && !seen.has(item.id)); for (const steerCommand of pendingSteer) { seen.add(steerCommand.id); await handleSteerCommand(api, runId, steerCommand, targetCommandId, attemptId, runnerId, control); } } catch { // The active backend turn remains authoritative; missed steer commands stay pending for the next poll. } finally { polling = false; } }; const timer = setInterval(() => { void poll(); }, intervalMs); void poll(); return () => { stopped = true; clearInterval(timer); }; } async function handleSteerCommand(api: RunnerManagerApi, runId: string, command: CommandRecord, targetCommandId: string, attemptId: string, runnerId: string, control: BackendActiveTurnControl): Promise { const acked = await api.ackCommand(command.id); if (acked.state === "cancelled") { await api.reportCommandStatus(command.id, { terminalStatus: "cancelled", failureKind: "cancelled", failureMessage: "steer command cancelled before delivery", threadId: control.threadId, turnId: control.turnId }); return; } const prompt = steerPrompt(command.payload); if (!prompt) { await reportNonTerminalCommandFailure(api, runId, command.id, attemptId, runnerId, { terminalStatus: "blocked", failureKind: "schema-invalid", message: "steer command payload requires a non-empty prompt, message, or text" }, "runner:steer:payload", control); return; } await appendBestEffort(api, runId, { type: "backend_status", payload: { phase: "steer-command-acknowledged", commandId: command.id, commandType: "steer", targetCommandId, attemptId, runnerId, threadId: control.threadId, turnId: control.turnId } }); try { await control.steer(prompt); await appendBestEffort(api, runId, { type: "backend_status", payload: { phase: "turn/steer:completed", commandId: command.id, commandType: "steer", targetCommandId, attemptId, runnerId, threadId: control.threadId, turnId: control.turnId } }); await api.reportCommandStatus(command.id, { terminalStatus: "completed", failureKind: null, failureMessage: null, threadId: control.threadId, turnId: control.turnId }); } catch (error) { const failureKind = failureKindFromError(error); await reportNonTerminalCommandFailure(api, runId, command.id, attemptId, runnerId, { terminalStatus: terminalStatusForFailure(failureKind), failureKind, message: errorMessage(error) }, "runner:steer", control); } } async function failPendingSteerWithoutActiveTurn(api: RunnerManagerApi, runId: string, commands: CommandRecord[], runnerId: string, attemptId: string): Promise { for (const command of commands.filter((item) => item.type === "steer" && item.state === "pending")) { const acked = await api.ackCommand(command.id); if (acked.state === "cancelled") continue; await reportNonTerminalCommandFailure(api, runId, command.id, attemptId, runnerId, { terminalStatus: "blocked", failureKind: "schema-invalid", message: "steer command requires an active turn" }, "runner:steer:no-active-turn"); } } async function reportNonTerminalCommandFailure(api: RunnerManagerApi, runId: string, commandId: string, attemptId: string, runnerId: string, failure: { terminalStatus: TerminalStatus; failureKind: FailureKind; message: string }, phase: string, control?: BackendActiveTurnControl): Promise { await appendBestEffort(api, runId, { type: "error", payload: { failureKind: failure.failureKind, message: failure.message, phase, commandId, attemptId, runnerId, ...(control ? { threadId: control.threadId, turnId: control.turnId } : {}) } }); await api.reportCommandStatus(commandId, { terminalStatus: failure.terminalStatus, failureKind: failure.failureKind, failureMessage: failure.message, ...(control ? { threadId: control.threadId, turnId: control.turnId } : {}) }); } function steerPrompt(payload: JsonRecord): string | null { for (const key of ["prompt", "message", "text"]) { const value = payload[key]; if (typeof value === "string" && value.trim().length > 0) return value.trim(); } return null; } function isTerminalRun(run: RunRecord): boolean { return run.status === "completed" || run.status === "failed" || run.status === "blocked" || run.status === "cancelled"; } async function assertNotCancelled(api: RunnerManagerApi, runId: string, commandId: string): Promise { const [run, command] = await Promise.all([api.getRun(runId), api.getCommand(runId, commandId)]); if (run.status === "cancelled" || command.state === "cancelled") throw new AgentRunError("cancelled", "run or command was cancelled", { httpStatus: 409 }); } function watchCancellation(api: RunnerManagerApi, runId: string, commandId: string, controller: AbortController): () => void { let stopped = false; const check = async (): Promise => { if (stopped || controller.signal.aborted) return; try { const [run, command] = await Promise.all([api.getRun(runId), api.getCommand(runId, commandId)]); if (run.status === "cancelled" || command.state === "cancelled") controller.abort(); } catch { // Cancellation polling must not hide the backend's own terminal result. } }; const timer = setInterval(() => { void check(); }, 2_000); void check(); return () => { stopped = true; clearInterval(timer); }; } function startHeartbeat(api: RunnerManagerApi, runId: string, runnerId: string, leaseMs: number): () => void { let stopped = false; const beat = async (): Promise => { if (stopped) return; try { await api.heartbeat(runId, runnerId, leaseMs); } catch { // The next manager call will surface lease or run-terminal details. } }; const timer = setInterval(() => { void beat(); }, Math.max(1_000, Math.floor(leaseMs / 3))); return () => { stopped = true; clearInterval(timer); }; } function startBackendProgress(): { stop: () => JsonRecord } { let stopped = false; let ticks = 0; const startedAt = Date.now(); const tick = (): void => { if (stopped) return; ticks += 1; }; const timer = setInterval(tick, 10_000); return { stop: () => { stopped = true; clearInterval(timer); return { elapsedMs: Date.now() - startedAt, progressTicks: ticks, progressEventsPrinted: false }; }, }; } async function appendBestEffort(api: RunnerManagerApi, runId: string, event: BackendEvent): Promise { try { await api.appendEvent(runId, event); } catch { // Visibility events are best effort; terminal command reporting remains authoritative. } } async function claimRunWithLeaseRecovery(api: RunnerManagerApi, options: RunnerOnceOptions, runner: RunnerRecord, attemptId: string, leaseMs: number): Promise { const startedAt = Date.now(); const timeoutMs = normalizeClaimRetryTimeoutMs(options.claimRetryTimeoutMs, leaseMs); const intervalMs = normalizeClaimRetryIntervalMs(options.claimRetryIntervalMs); let waitingEventWritten = false; let lastError: unknown = null; while (Date.now() - startedAt <= timeoutMs) { try { const claimed = await api.claim(options.runId, runner.id, leaseMs); if (waitingEventWritten) { await appendBestEffort(api, options.runId, { type: "backend_status", payload: { phase: "runner-claim-recovered", attemptId, runnerId: runner.id, waitedMs: Date.now() - startedAt }, }); } return claimed; } catch (error) { if (failureKindFromError(error) !== "runner-lease-conflict") throw error; lastError = error; const run = await getRunBestEffort(api, options.runId); if (!waitingEventWritten) { waitingEventWritten = true; await appendBestEffort(api, options.runId, { type: "backend_status", payload: { phase: "runner-claim-waiting-for-stale-lease", attemptId, runnerId: runner.id, claimedBy: run?.claimedBy ?? null, leaseExpiresAt: run?.leaseExpiresAt ?? null, retryTimeoutMs: timeoutMs, }, }); } const remainingMs = timeoutMs - (Date.now() - startedAt); if (remainingMs <= 0) break; await sleep(Math.min(remainingMs, claimRetryDelayMs(run, intervalMs))); } } throw lastError ?? new AgentRunError("runner-lease-conflict", `run ${options.runId} could not be claimed before retry timeout`, { httpStatus: 409 }); } async function getRunBestEffort(api: RunnerManagerApi, runId: string): Promise { try { return await api.getRun(runId); } catch { return null; } } function claimRetryDelayMs(run: RunRecord | null, intervalMs: number): number { const leaseExpiresAt = run?.leaseExpiresAt ? Date.parse(run.leaseExpiresAt) : NaN; if (Number.isFinite(leaseExpiresAt)) { const untilExpiryMs = leaseExpiresAt - Date.now() + 100; if (untilExpiryMs > 0) return Math.max(25, Math.min(intervalMs, untilExpiryMs)); } return intervalMs; } function normalizeClaimRetryTimeoutMs(value: number | undefined, leaseMs: number): number { if (Number.isFinite(value ?? NaN)) return Math.max(0, Math.floor(value!)); return Math.max(leaseMs + Math.max(5_000, Math.floor(leaseMs / 3)), leaseMs); } function normalizeClaimRetryIntervalMs(value: number | undefined): number { if (!Number.isFinite(value ?? NaN)) return 5_000; return Math.max(25, Math.min(30_000, Math.floor(value!))); } function annotateCommandEvent(event: BackendEvent, commandId: string, attemptId: string, runnerId: string): BackendEvent { return { ...event, payload: { ...event.payload, commandId, attemptId, runnerId } }; } async function reportCommandFailure(api: RunnerManagerApi, runId: string, commandId: string, runner: RunnerRecord, attemptId: string, failure: { terminalStatus: TerminalStatus; failureKind: FailureKind; message: string }, phase: string, options: { terminalRun?: boolean } = {}): Promise { await api.appendEvent(runId, { type: "error", payload: { failureKind: failure.failureKind, message: failure.message, phase, commandId, attemptId, runnerId: runner.id } }); await api.appendEvent(runId, { type: "terminal_status", payload: { terminalStatus: failure.terminalStatus, failureKind: failure.failureKind, message: failure.message, commandId, attemptId, runnerId: runner.id } }); await api.reportCommandStatus(commandId, { terminalStatus: failure.terminalStatus, failureKind: failure.failureKind, failureMessage: failure.message }); if (options.terminalRun === true) await api.reportStatus(runId, { terminalStatus: failure.terminalStatus, failureKind: failure.failureKind, failureMessage: failure.message }); return { commandId, terminalStatus: failure.terminalStatus, failureKind: failure.failureKind } as CommandExecutionResult; } async function reportCancelled(api: RunnerManagerApi, runId: string, commandId: string, runner: RunnerRecord, attemptId: string, message: string): Promise { await api.reportCommandStatus(commandId, { terminalStatus: "cancelled", failureKind: "cancelled", failureMessage: message }); await api.reportStatus(runId, { terminalStatus: "cancelled", failureKind: "cancelled", failureMessage: message }); await api.appendEvent(runId, { type: "backend_status", payload: { phase: "turn-cancelled", commandId, attemptId, runnerId: runner.id, failureKind: "cancelled", message } }); await api.appendEvent(runId, { type: "terminal_status", payload: { terminalStatus: "cancelled", failureKind: "cancelled", message, commandId, attemptId, runnerId: runner.id } }); return { commandId, terminalStatus: "cancelled", failureKind: "cancelled" }; } function noPendingResult(runner: RunnerRecord, claimed: RunRecord, commandResults: CommandExecutionResult[], polledCommands: number, stopped: string): JsonRecord { if (commandResults.length > 0) { const last = commandResults.at(-1)!; return { runner, claimed, terminalStatus: last.terminalStatus, failureKind: last.failureKind, commandsProcessed: commandResults.length, commandResults, polledCommands, stopped }; } return { runner, claimed, terminalStatus: "blocked", failureKind: "schema-invalid", commandsProcessed: 0, commandResults, polledCommands, stopped }; } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } function normalizePollIntervalMs(value: number | undefined): number { if (!Number.isFinite(value ?? NaN)) return 250; return Math.max(50, Math.min(2_000, Math.floor(value!))); }