fix(runner): persist codex home in session pvc (#242)
This commit is contained in:
@@ -1,7 +1,7 @@
|
|||||||
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
|
||||||
import { createHash } from "node:crypto";
|
import { createHash } from "node:crypto";
|
||||||
import { accessSync, constants as fsConstants, readdirSync, readFileSync } from "node:fs";
|
import { accessSync, constants as fsConstants, readdirSync, readFileSync } from "node:fs";
|
||||||
import { chmod, copyFile, mkdir } from "node:fs/promises";
|
import { chmod, copyFile, mkdir, writeFile } from "node:fs/promises";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
import * as readline from "node:readline";
|
import * as readline from "node:readline";
|
||||||
import type { BackendEvent, BackendProfile, BackendTurnResult, CommandRecord, FailureKind, InitialPromptAssembly, JsonRecord, JsonValue, RunRecord, TerminalStatus } from "../common/types.js";
|
import type { BackendEvent, BackendProfile, BackendTurnResult, CommandRecord, FailureKind, InitialPromptAssembly, JsonRecord, JsonValue, RunRecord, TerminalStatus } from "../common/types.js";
|
||||||
@@ -852,6 +852,7 @@ async function prepareProjectedCodexHome(codexHome: string, projectedHome: strin
|
|||||||
await copyFile(path.join(projectedHome, fileName), path.join(codexHome, fileName));
|
await copyFile(path.join(projectedHome, fileName), path.join(codexHome, fileName));
|
||||||
await chmod(path.join(codexHome, fileName), 0o600);
|
await chmod(path.join(codexHome, fileName), 0o600);
|
||||||
}
|
}
|
||||||
|
await normalizeCopiedCodexConfig(codexHome);
|
||||||
return null;
|
return null;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const payload = {
|
const payload = {
|
||||||
@@ -875,6 +876,29 @@ async function prepareProjectedCodexHome(codexHome: string, projectedHome: strin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function normalizeCopiedCodexConfig(codexHome: string): Promise<void> {
|
||||||
|
const configPath = path.join(codexHome, "config.toml");
|
||||||
|
const modelCatalogPath = path.join(codexHome, "model-catalog.json");
|
||||||
|
if (!fileReadable(modelCatalogPath).readable) return;
|
||||||
|
let configToml = "";
|
||||||
|
try {
|
||||||
|
configToml = readFileSync(configPath, "utf8");
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const normalized = configToml.replace(
|
||||||
|
/(^\s*model_catalog_json\s*=\s*")([^"]+)("\s*$)/mu,
|
||||||
|
(_match, prefix: string, _oldValue: string, suffix: string) => `${prefix}${tomlBasicStringValue(modelCatalogPath)}${suffix}`,
|
||||||
|
);
|
||||||
|
if (normalized === configToml) return;
|
||||||
|
await writeFile(configPath, normalized, { mode: 0o600 });
|
||||||
|
await chmod(configPath, 0o600);
|
||||||
|
}
|
||||||
|
|
||||||
|
function tomlBasicStringValue(value: string): string {
|
||||||
|
return value.replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"");
|
||||||
|
}
|
||||||
|
|
||||||
function codexHomeReadiness(codexHome: string, profile: BackendProfile): BackendTurnResult | null {
|
function codexHomeReadiness(codexHome: string, profile: BackendProfile): BackendTurnResult | null {
|
||||||
const auth = fileReadable(`${codexHome}/auth.json`);
|
const auth = fileReadable(`${codexHome}/auth.json`);
|
||||||
const config = fileReadable(`${codexHome}/config.toml`);
|
const config = fileReadable(`${codexHome}/config.toml`);
|
||||||
|
|||||||
@@ -250,7 +250,7 @@ export function renderRunnerJobManifest(options: RunnerJobRenderOptions): { mani
|
|||||||
|
|
||||||
function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; sessionPvc: RunnerSessionPvcOptions | undefined; runnerIdleTimeoutMs: number; missingTerminalAfterToolTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number }): JsonRecord[] {
|
function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string; jobName: string; runnerJobId: string; runnerId: string; attemptId: string; sourceCommit: string; secretRefs: CredentialProjection[]; toolCredentials: ToolCredentialProjection[]; sessionPvc: RunnerSessionPvcOptions | undefined; runnerIdleTimeoutMs: number; missingTerminalAfterToolTimeoutMs: number; backendRetryMaxAttempts: number; backendRetryInitialBackoffMs: number; backendRetryMaxBackoffMs: number }): JsonRecord[] {
|
||||||
const selectedSecret = context.secretRefs.find((item) => item.profile === options.run.backendProfile);
|
const selectedSecret = context.secretRefs.find((item) => item.profile === options.run.backendProfile);
|
||||||
const codexHome = selectedSecret?.runtimeMountPath ?? defaultRuntimeHome(options.run.backendProfile);
|
const codexHome = runnerCodexHome(options.run.backendProfile, selectedSecret, context.sessionPvc);
|
||||||
const bootRepoUrl = optionalString(options.bootRepoUrl) ?? defaultBootRepoUrl;
|
const bootRepoUrl = optionalString(options.bootRepoUrl) ?? defaultBootRepoUrl;
|
||||||
return dedupeEnvVars([
|
return dedupeEnvVars([
|
||||||
{ name: "AGENTRUN_MGR_URL", value: options.managerUrl },
|
{ name: "AGENTRUN_MGR_URL", value: options.managerUrl },
|
||||||
@@ -285,6 +285,7 @@ function runnerEnv(options: RunnerJobRenderOptions, context: { namespace: string
|
|||||||
{ name: "AGENTRUN_RUNNER_POLL_INTERVAL_MS", value: "250" },
|
{ name: "AGENTRUN_RUNNER_POLL_INTERVAL_MS", value: "250" },
|
||||||
{ name: "HOME", value: "/home/agentrun" },
|
{ name: "HOME", value: "/home/agentrun" },
|
||||||
{ name: "CODEX_HOME", value: codexHome },
|
{ name: "CODEX_HOME", value: codexHome },
|
||||||
|
{ name: "AGENTRUN_CODEX_HOME_STORAGE_KIND", value: context.sessionPvc ? "session-pvc" : "runner-home" },
|
||||||
...runnerOtelEnvVars(process.env),
|
...runnerOtelEnvVars(process.env),
|
||||||
...(selectedSecret ? [{ name: "AGENTRUN_CODEX_SECRET_HOME", value: selectedSecret.projectionMountPath }] : []),
|
...(selectedSecret ? [{ name: "AGENTRUN_CODEX_SECRET_HOME", value: selectedSecret.projectionMountPath }] : []),
|
||||||
...(context.sessionPvc ? [
|
...(context.sessionPvc ? [
|
||||||
@@ -580,6 +581,11 @@ function normalizeMountPath(value: string | undefined, profile: string): string
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function runnerCodexHome(profile: string, selectedSecret: CredentialProjection | undefined, sessionPvc: RunnerSessionPvcOptions | undefined): string {
|
||||||
|
if (!sessionPvc) return selectedSecret?.runtimeMountPath ?? defaultRuntimeHome(profile);
|
||||||
|
return `${sessionPvc.mountPath}/codex-home/${sanitizeVolumeName(profile)}`;
|
||||||
|
}
|
||||||
|
|
||||||
function defaultRuntimeHome(profile: string): string {
|
function defaultRuntimeHome(profile: string): string {
|
||||||
return `/home/agentrun/.codex-${sanitizeVolumeName(profile)}`;
|
return `/home/agentrun/.codex-${sanitizeVolumeName(profile)}`;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user