fix: prepare writable codex home for runner jobs

src/backend/codex-stdio.ts

@@ -1,6 +1,7 @@
 import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
 import { createHash } from "node:crypto";
 import { accessSync, constants as fsConstants } from "node:fs";
+import { chmod, copyFile, mkdir } from "node:fs/promises";
 import path from "node:path";
 import * as readline from "node:readline";
 import type { BackendEvent, BackendTurnResult, FailureKind, JsonRecord, JsonValue, TerminalStatus } from "../common/types.js";
@@ -256,6 +257,8 @@ export class CodexStdioClient {
 
 export async function runCodexStdioTurn(options: CodexStdioTurnOptions): Promise<BackendTurnResult> {
   const codexHome = resolveCodexHome(options);
+  const projectionFailure = await prepareProjectedCodexHome(codexHome, options.env?.AGENTRUN_CODEX_SECRET_HOME ?? process.env.AGENTRUN_CODEX_SECRET_HOME);
+  if (projectionFailure) return projectionFailure;
   const secretFailure = codexHomeReadiness(codexHome);
   if (secretFailure) return secretFailure;
   const env = childEnv(options, codexHome);
@@ -347,6 +350,38 @@ export async function runCodexStdioTurn(options: CodexStdioTurnOptions): Promise
   return { terminalStatus: terminal.status, failureKind: terminal.failureKind, failureMessage: terminal.message, events: events.map((event) => ({ ...event, payload: redactJson(event.payload) })), ...(threadId ? { threadId } : {}), ...(turnId ? { turnId } : {}) };
 }
 
+async function prepareProjectedCodexHome(codexHome: string, projectedHome: string | undefined): Promise<BackendTurnResult | null> {
+  if (!projectedHome || projectedHome.trim().length === 0) return null;
+  if (path.resolve(projectedHome) === path.resolve(codexHome)) return null;
+  try {
+    await mkdir(codexHome, { recursive: true, mode: 0o700 });
+    for (const fileName of ["auth.json", "config.toml"]) {
+      await copyFile(path.join(projectedHome, fileName), path.join(codexHome, fileName));
+      await chmod(path.join(codexHome, fileName), 0o600);
+    }
+    return null;
+  } catch (error) {
+    const payload = {
+      failureKind: "secret-unavailable",
+      projection: {
+        source: pathSummary(projectedHome),
+        destination: pathSummary(codexHome),
+        valuesPrinted: false,
+      },
+      message: error instanceof Error ? redactText(error.message) : "failed to prepare writable Codex home",
+    } satisfies JsonRecord;
+    return {
+      terminalStatus: "blocked",
+      failureKind: "secret-unavailable",
+      failureMessage: "Codex Secret projection could not be copied to writable CODEX_HOME",
+      events: [
+        { type: "error", payload },
+        { type: "terminal_status", payload: { terminalStatus: "blocked", failureKind: "secret-unavailable" } },
+      ],
+    };
+  }
+}
+
 function codexHomeReadiness(codexHome: string): BackendTurnResult | null {
   const auth = fileReadable(`${codexHome}/auth.json`);
   const config = fileReadable(`${codexHome}/config.toml`);

src/mgr/kubernetes-runner-job.ts

@@ -73,7 +73,7 @@ export async function createKubernetesRunnerJob(options: { store: AgentRunStore;
       placement: "kubernetes-job",
       logPath: `kubectl -n ${render.namespace} logs job/${render.jobName}`,
     },
-    secretRefs: render.secretRefs.map((item) => ({ profile: item.profile, name: item.secretRef.name, namespace: item.secretRef.namespace ?? render.namespace, keys: item.secretRef.keys ?? [], mountPath: item.mountPath, valuesPrinted: false })),
+    secretRefs: render.secretRefs.map((item) => ({ profile: item.profile, name: item.secretRef.name, namespace: item.secretRef.namespace ?? render.namespace, keys: item.secretRef.keys ?? [], mountPath: item.runtimeMountPath, projectionPath: item.projectionMountPath, writableCopy: true, valuesPrinted: false })),
     pollCommands: {
       run: `bun scripts/agentrun-cli.ts runs show ${run.id} --manager-url ${managerUrl}`,
       command: `bun scripts/agentrun-cli.ts commands show ${commandId} --run-id ${run.id} --manager-url ${managerUrl}`,

src/runner/k8s-job.ts

@@ -21,7 +21,8 @@ interface CredentialProjection {
   profile: BackendProfile | string;
   secretRef: SecretRef;
   volumeName: string;
-  mountPath: string;
+  runtimeMountPath: string;
+  projectionMountPath: string;
 }
 
 export function renderRunnerJobDryRun(options: RunnerJobRenderOptions): JsonRecord {
@@ -45,7 +46,7 @@ export function renderRunnerJobDryRun(options: RunnerJobRenderOptions): JsonReco
       managerUrl: options.managerUrl,
       sourceCommit: render.sourceCommit,
     },
-    secretRefs: render.secretRefs.map((item) => ({ profile: item.profile, name: item.secretRef.name, namespace: item.secretRef.namespace ?? render.namespace, keys: item.secretRef.keys ?? [], mountPath: item.mountPath, valuesPrinted: false })),
+    secretRefs: render.secretRefs.map((item) => ({ profile: item.profile, name: item.secretRef.name, namespace: item.secretRef.namespace ?? render.namespace, keys: item.secretRef.keys ?? [], mountPath: item.runtimeMountPath, projectionPath: item.projectionMountPath, writableCopy: true, valuesPrinted: false })),
     pollCommands: {
       run: `bun scripts/agentrun-cli.ts runs show ${options.run.id} --manager-url ${options.managerUrl}`,
       events: `bun scripts/agentrun-cli.ts runs events ${options.run.id} --manager-url ${options.managerUrl} --after-seq 0 --limit 100`,
@@ -101,7 +102,10 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani
               imagePullPolicy: options.imagePullPolicy ?? "IfNotPresent",
               command: ["bun", "src/runner/main.ts"],
               env,
-              volumeMounts: secretRefs.map((item) => ({ name: item.volumeName, mountPath: item.mountPath, readOnly: true })),
+              volumeMounts: [
+                { name: "runner-home", mountPath: "/home/agentrun" },
+                ...secretRefs.map((item) => ({ name: item.volumeName, mountPath: item.projectionMountPath, readOnly: true })),
+              ],
               resources: {
                 requests: { cpu: "250m", memory: "512Mi" },
                 limits: { cpu: "2", memory: "4Gi" },
@@ -113,7 +117,7 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani
               },
             },
           ],
-          volumes: secretRefs.map(secretVolume),
+          volumes: [{ name: "runner-home", emptyDir: {} }, ...secretRefs.map(secretVolume)],
         },
       },
     },
@@ -122,7 +126,8 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani
 }
 
 function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string; jobName: string; runnerId: string; attemptId: string; sourceCommit: string; secretRefs: CredentialProjection[] }): JsonRecord[] {
-  const codexMount = context.secretRefs.find((item) => item.profile === "codex")?.mountPath ?? "/home/agentrun/.codex";
+  const codexSecret = context.secretRefs.find((item) => item.profile === "codex");
+  const codexHome = codexSecret?.runtimeMountPath ?? "/home/agentrun/.codex";
   return [
     { name: "AGENTRUN_MGR_URL", value: options.managerUrl },
     { name: "AGENTRUN_RUN_ID", value: options.run.id },
@@ -136,7 +141,8 @@ function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string
     { name: "AGENTRUN_K8S_JOB_NAME", value: context.jobName },
     { name: "AGENTRUN_LOG_PATH", value: "/tmp/agentrun-runner.jsonl" },
     { name: "HOME", value: "/home/agentrun" },
-    { name: "CODEX_HOME", value: codexMount },
+    { name: "CODEX_HOME", value: codexHome },
+    ...(codexSecret ? [{ name: "AGENTRUN_CODEX_SECRET_HOME", value: codexSecret.projectionMountPath }] : []),
   ];
 }
 
@@ -147,7 +153,8 @@ function credentialProjections(run: RunRecord, namespace: string): CredentialPro
     profile: item.profile,
     secretRef: item.secretRef.namespace ? item.secretRef : { ...item.secretRef, namespace },
     volumeName: sanitizeVolumeName(`${String(item.profile)}-${index}`),
-    mountPath: normalizeMountPath(item.secretRef.mountPath),
+    runtimeMountPath: normalizeMountPath(item.secretRef.mountPath),
+    projectionMountPath: `/var/run/agentrun/secrets/${sanitizeVolumeName(`${String(item.profile)}-${index}`)}`,
   }));
 }
 

src/selftest/cases/20-runner-k8s-job.ts

@@ -5,7 +5,7 @@ import { startManagerServer } from "../../mgr/server.js";
 import { MemoryAgentRunStore } from "../../mgr/store.js";
 import { ManagerClient } from "../../mgr/client.js";
 import { renderRunnerJobDryRun } from "../../runner/k8s-job.js";
-import type { RunRecord } from "../../common/types.js";
+import type { JsonRecord, RunRecord } from "../../common/types.js";
 import { assertNoSecretLeak, createRunWithCommand, type SelfTestCase } from "../harness.js";
 
 const selfTest: SelfTestCase = async (context) => {
@@ -24,6 +24,7 @@ const selfTest: SelfTestCase = async (context) => {
     assert.equal(rendered.dryRun, true);
     assert.equal(rendered.mutation, false);
     assert.equal((rendered.jobIdentity as { serviceAccountName?: string }).serviceAccountName, "agentrun-v01-runner");
+    assertRunnerJobUsesWritableCodexHome(rendered.manifest as JsonRecord, context.codexHome);
     assertNoSecretLeak(rendered);
 
     const fakeKubectl = path.join(context.tmp, "fake-kubectl.js");
@@ -66,3 +67,24 @@ console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kin
 };
 
 export default selfTest;
+
+function assertRunnerJobUsesWritableCodexHome(manifest: JsonRecord, expectedCodexHome: string): void {
+  const spec = manifest.spec as JsonRecord;
+  const template = spec.template as JsonRecord;
+  const podSpec = template.spec as JsonRecord;
+  const volumes = podSpec.volumes as JsonRecord[];
+  assert.ok(volumes.some((volume) => volume.name === "runner-home" && typeof volume.emptyDir === "object"), "runner home must be writable emptyDir");
+
+  const containers = podSpec.containers as JsonRecord[];
+  const runner = containers[0] as JsonRecord;
+  const mounts = runner.volumeMounts as JsonRecord[];
+  assert.ok(mounts.some((mount) => mount.name === "runner-home" && mount.mountPath === "/home/agentrun"), "runner-home must mount at /home/agentrun");
+  assert.ok(mounts.some((mount) => mount.name === "codex-0" && mount.mountPath === "/var/run/agentrun/secrets/codex-0" && mount.readOnly === true), "Codex Secret must mount read-only outside CODEX_HOME");
+
+  const env = runner.env as JsonRecord[];
+  const value = (name: string): unknown => env.find((item) => item.name === name)?.value;
+  assert.equal(value("HOME"), "/home/agentrun");
+  assert.equal(value("CODEX_HOME"), expectedCodexHome);
+  assert.equal(value("AGENTRUN_CODEX_SECRET_HOME"), "/var/run/agentrun/secrets/codex-0");
+  assert.notEqual(value("CODEX_HOME"), value("AGENTRUN_CODEX_SECRET_HOME"));
+}

src/selftest/cases/30-codex-stdio.ts

@@ -1,4 +1,5 @@
 import assert from "node:assert/strict";
+import { access } from "node:fs/promises";
 import path from "node:path";
 import os from "node:os";
 import { startManagerServer } from "../../mgr/server.js";
@@ -25,12 +26,19 @@ const selfTest: SelfTestCase = async (context) => {
     const finalCommand = await client.get(`/api/v1/runs/${happy.runId}/commands/${happy.commandId}`) as { state?: string };
     assert.equal(finalCommand.state, "completed");
 
+    const projectedHome = path.join(context.tmp, "runtime-codex-home");
+    const projected = await createRunWithCommand(client, { workspace: context.workspace, codexHome: projectedHome }, "hello projected", "selftest-projected-codex-home", 15_000);
+    const projectedResult = await runOnce({ managerUrl: server.baseUrl, runId: projected.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: projectedHome, env: { CODEX_HOME: projectedHome, AGENTRUN_CODEX_SECRET_HOME: context.codexHome } });
+    assert.equal(projectedResult.terminalStatus, "completed");
+    await access(path.join(projectedHome, "auth.json"));
+    await access(path.join(projectedHome, "config.toml"));
+
     await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "missing-turn-result", expectedStatus: "failed", expectedFailureKind: "backend-response-invalid" });
     await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "invalid-json", expectedStatus: "failed", expectedFailureKind: "backend-json-parse-error" });
     await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "missing-terminal", expectedStatus: "failed", expectedFailureKind: "backend-timeout", timeoutMs: 500 });
     await runSpawnFailureCase({ client, managerUrl: server.baseUrl, context });
 
-    return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "codex-stdio-fake-turn", "codex-stdio-missing-turn-result", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-spawn-failure"] };
+    return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "codex-stdio-fake-turn", "codex-stdio-projected-writable-home", "codex-stdio-missing-turn-result", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-spawn-failure"] };
   } finally {
     await new Promise<void>((resolve) => server.server.close(() => resolve()));
   }