import { spawn } from "node:child_process"; import { AgentRunError } from "../common/errors.js"; import { redactJson, redactText } from "../common/redaction.js"; import { isTerminalCommandState, isTerminalRunStatus, summarizeResourceBundleRef, summarizeSessionRef } from "./store.js"; import type { AgentRunStore } from "./store.js"; import type { JsonRecord } from "../common/types.js"; import { stableHash, validateEnvName } from "../common/validation.js"; import { renderRunnerJobManifest } from "../runner/k8s-job.js"; import type { RunnerTransientEnv } from "../runner/k8s-job.js"; export interface RunnerJobDefaults { namespace: string; managerUrl: string; image: string; sourceCommit: string; serviceAccountName?: string; kubectlCommand?: string; } export interface CreateRunnerJobInput extends JsonRecord { commandId: string; managerUrl?: string; image?: string; namespace?: string; attemptId?: string; runnerId?: string; sourceCommit?: string; serviceAccountName?: string; idempotencyKey?: string; transientEnv?: JsonRecord[]; } export async function createKubernetesRunnerJob(options: { store: AgentRunStore; runId: string; input: CreateRunnerJobInput; defaults: RunnerJobDefaults }): Promise { const commandId = stringField(options.input, "commandId"); const run = await options.store.getRun(options.runId); const command = await options.store.getCommand(commandId); if (command.runId !== run.id) throw new AgentRunError("schema-invalid", `command ${commandId} does not belong to run ${run.id}`, { httpStatus: 400 }); if (command.type !== "turn") throw new AgentRunError("schema-invalid", `command ${commandId} is not a turn command`, { httpStatus: 400 }); const image = optionalString(options.input.image) ?? options.defaults.image; if (!image) throw new AgentRunError("schema-invalid", "runner job image is required; set --image or AGENTRUN_RUNNER_IMAGE", { httpStatus: 400 }); const namespace = optionalString(options.input.namespace) ?? options.defaults.namespace; const managerUrl = optionalString(options.input.managerUrl) ?? options.defaults.managerUrl; const sourceCommit = optionalString(options.input.sourceCommit) ?? options.defaults.sourceCommit; const serviceAccountName = optionalString(options.input.serviceAccountName) ?? options.defaults.serviceAccountName; const idempotencyKey = optionalString(options.input.idempotencyKey); const transientEnv = transientEnvField(options.input.transientEnv); const normalizedPayload = { commandId, image, namespace, managerUrl, sourceCommit, serviceAccountName: serviceAccountName ?? null, attemptId: optionalString(options.input.attemptId) ?? null, runnerId: optionalString(options.input.runnerId) ?? null, transientEnv: transientEnv.map((item) => ({ name: item.name, valueHash: stableHash(item.value), sensitive: true })), }; const payloadHash = stableHash(normalizedPayload); if (idempotencyKey) { const existing = await options.store.getRunnerJobByIdempotencyKey(run.id, idempotencyKey, payloadHash); if (existing) return { ...existing.result, idempotentReplay: true }; } if (isTerminalRunStatus(run.status)) throw new AgentRunError(run.failureKind ?? (run.status === "cancelled" ? "cancelled" : "schema-invalid"), `run ${run.id} is already terminal: ${run.status}`, { httpStatus: 409 }); if (isTerminalCommandState(command.state) || command.state !== "pending") throw new AgentRunError(command.state === "cancelled" ? "cancelled" : "schema-invalid", `command ${commandId} is not pending: ${command.state}`, { httpStatus: 409 }); const renderOptions = { run, commandId, managerUrl, image, namespace, sourceCommit, transientEnv, ...(serviceAccountName ? { serviceAccountName } : {}), }; const attemptId = optionalString(options.input.attemptId); const runnerId = optionalString(options.input.runnerId); const render = renderRunnerJobManifest({ ...renderOptions, ...(attemptId ? { attemptId } : {}), ...(runnerId ? { runnerId } : {}) }); const created = await kubectlCreate(render.manifest, options.defaults.kubectlCommand ?? "kubectl"); const response = { action: "create-kubernetes-job", mutation: true, runId: run.id, commandId, attemptId: render.attemptId, runnerId: render.runnerId, namespace: render.namespace, jobName: render.jobName, jobIdentity: { kind: "Job", namespace: render.namespace, name: render.jobName, serviceAccountName: render.serviceAccountName, uid: objectPath(created, ["metadata", "uid"]), }, runner: { runId: run.id, commandId, attemptId: render.attemptId, runnerId: render.runnerId, backendProfile: run.backendProfile, managerUrl, sourceCommit, placement: "kubernetes-job", logPath: `kubectl -n ${render.namespace} logs job/${render.jobName}`, }, secretRefs: render.secretRefs.map((item) => ({ profile: item.profile, name: item.secretRef.name, namespace: item.secretRef.namespace ?? render.namespace, keys: item.secretRef.keys ?? [], mountPath: item.runtimeMountPath, projectionPath: item.projectionMountPath, writableCopy: true, valuesPrinted: false })), toolCredentials: summarizeToolCredentials(render.toolCredentials, render.namespace), transientEnv: summarizeTransientEnv(transientEnv), retention: { ttlSecondsAfterFinished: render.ttlSecondsAfterFinished, }, pollCommands: { run: `./scripts/agentrun runs show ${run.id} --manager-url ${managerUrl}`, command: `./scripts/agentrun commands show ${commandId} --run-id ${run.id} --manager-url ${managerUrl}`, events: `./scripts/agentrun runs events ${run.id} --manager-url ${managerUrl} --after-seq 0 --limit 100`, }, warnings: render.warnings, kubernetes: { created: true, valuesPrinted: false, apiVersion: objectPath(created, ["apiVersion"]), kind: objectPath(created, ["kind"]), resourceVersion: objectPath(created, ["metadata", "resourceVersion"]), }, } satisfies JsonRecord; const saved = await options.store.saveRunnerJob({ runId: run.id, commandId, idempotencyKey: idempotencyKey ?? null, payloadHash, attemptId: render.attemptId, runnerId: render.runnerId, namespace: render.namespace, jobName: render.jobName, managerUrl, image, sourceCommit, serviceAccountName: serviceAccountName ?? null, result: response, }); await options.store.appendEvent(run.id, "backend_status", { phase: "runner-job-created", commandId, attemptId: saved.attemptId, runnerId: saved.runnerId, namespace: saved.namespace, jobName: saved.jobName, idempotencyKey: idempotencyKey ? "present" : null, transientEnv: summarizeTransientEnv(transientEnv), toolCredentials: summarizeToolCredentials(render.toolCredentials, render.namespace), sessionRef: summarizeSessionRef(run.sessionRef ?? null), resourceBundleRef: summarizeResourceBundleRef(run.resourceBundleRef ?? null), }); return response; } function transientEnvField(value: unknown): RunnerTransientEnv[] { if (value === undefined) return []; if (!Array.isArray(value)) throw new AgentRunError("schema-invalid", "transientEnv must be an array", { httpStatus: 400 }); if (value.length > 8) throw new AgentRunError("schema-invalid", "transientEnv must contain at most 8 entries", { httpStatus: 400 }); const seen = new Set(); return value.map((entry, index) => { if (!entry || typeof entry !== "object" || Array.isArray(entry)) throw new AgentRunError("schema-invalid", `transientEnv[${index}] must be an object`, { httpStatus: 400 }); const record = entry as JsonRecord; const name = stringField(record, "name"); validateEnvName(name, `transientEnv[${index}].name`); if (name === "GH_TOKEN" || name === "GITHUB_TOKEN" || name === "OPENAI_API_KEY" || name === "CODEX_API_KEY") throw new AgentRunError("tenant-policy-denied", `transientEnv ${name} must use tool/provider credential assembly instead`, { httpStatus: 403 }); if (seen.has(name)) throw new AgentRunError("schema-invalid", `transientEnv name ${name} is duplicated`, { httpStatus: 400 }); seen.add(name); const rawValue = record.value; if (typeof rawValue !== "string" || rawValue.length === 0) throw new AgentRunError("schema-invalid", `transientEnv[${index}].value must be a non-empty string`, { httpStatus: 400 }); if (Buffer.byteLength(rawValue, "utf8") > 8192) throw new AgentRunError("schema-invalid", `transientEnv[${index}].value is too large`, { httpStatus: 400 }); return { name, value: rawValue, sensitive: true }; }); } function summarizeToolCredentials(items: Array<{ tool: string; purpose: string | null; secretRef: { namespace?: string; name: string; keys?: string[] }; envName: string; secretKey: string }>, namespace: string): JsonRecord { return { count: items.length, items: items.map((item) => ({ tool: item.tool, purpose: item.purpose, name: item.secretRef.name, namespace: item.secretRef.namespace ?? namespace, keys: item.secretRef.keys ?? [], projection: { kind: "env", envName: item.envName, secretKey: item.secretKey }, valuesPrinted: false, })), valuesPrinted: false, }; } function summarizeTransientEnv(items: RunnerTransientEnv[]): JsonRecord { return { count: items.length, names: items.map((item) => item.name), valuesPrinted: false, }; } async function kubectlCreate(manifest: JsonRecord, kubectlCommand: string): Promise { const child = spawn(kubectlCommand, ["create", "-f", "-", "-o", "json"], { stdio: ["pipe", "pipe", "pipe"] }); let stdout = ""; let stderr = ""; child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); child.stdout.on("data", (chunk) => { stdout += String(chunk); }); child.stderr.on("data", (chunk) => { stderr += String(chunk); }); child.stdin.end(`${JSON.stringify(manifest)}\n`); const result = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve, reject) => { child.on("error", reject); child.on("close", (code, signal) => resolve({ code, signal })); }).catch((error: unknown) => { throw new AgentRunError("infra-failed", `failed to start kubectl: ${error instanceof Error ? error.message : String(error)}`, { httpStatus: 503 }); }); if (result.code !== 0) { throw new AgentRunError("infra-failed", `kubectl create runner job failed with code ${result.code}`, { httpStatus: 502, details: redactJson({ stderr: redactText(stderr.slice(-4000)), stdout: redactText(stdout.slice(-2000)), signal: result.signal }) }); } try { const parsed = JSON.parse(stdout) as unknown; if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) return redactJson(parsed as JsonRecord); } catch (error) { throw new AgentRunError("infra-failed", `kubectl returned invalid JSON: ${error instanceof Error ? error.message : String(error)}`, { httpStatus: 502, details: { stdoutPreview: redactText(stdout.slice(0, 1000)) } }); } throw new AgentRunError("infra-failed", "kubectl returned non-object JSON", { httpStatus: 502 }); } function stringField(record: JsonRecord, key: string): string { const value = record[key]; if (typeof value !== "string" || value.trim().length === 0) throw new AgentRunError("schema-invalid", `${key} is required`, { httpStatus: 400 }); return value.trim(); } function optionalString(value: unknown): string | undefined { return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; } function objectPath(value: unknown, path: string[]): string | null { let current: unknown = value; for (const key of path) { if (typeof current !== "object" || current === null || Array.isArray(current)) return null; current = (current as Record)[key]; } return typeof current === "string" ? current : null; }