pikasTech-agentrun/src/selftest/cases/20-runner-k8s-job.ts

import assert from "node:assert/strict";
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { startManagerServer } from "../../mgr/server.js";
import { MemoryAgentRunStore } from "../../mgr/store.js";
import { ManagerClient } from "../../mgr/client.js";
import { renderRunnerJobDryRun } from "../../runner/k8s-job.js";
import type { JsonRecord, RunRecord } from "../../common/types.js";
import { assertNoSecretLeak, createRunWithCommand, type SelfTestCase } from "../harness.js";

const selfTest: SelfTestCase = async (context) => {
  const server = await startManagerServer({ port: 0, host: "127.0.0.1", sourceCommit: "self-test", store: new MemoryAgentRunStore() });
  try {
    const client = new ManagerClient(server.baseUrl);
    const githubToolCredentials = [{
      tool: "github",
      purpose: "pull-request",
      secretRef: { name: "agentrun-v01-tool-github-pr", keys: ["GH_TOKEN"] },
      projection: { kind: "env", envName: "GH_TOKEN", secretKey: "GH_TOKEN" },
    }];
    const unideskSshToolCredentials = [{
      tool: "unidesk-ssh",
      purpose: "ssh-passthrough",
      secretRef: { name: "agentrun-v01-tool-unidesk-ssh", keys: ["UNIDESK_SSH_CLIENT_TOKEN"] },
      projection: { kind: "env", envName: "UNIDESK_SSH_CLIENT_TOKEN", secretKey: "UNIDESK_SSH_CLIENT_TOKEN" },
    }];
    const githubSshToolCredentials = [{
      tool: "github",
      purpose: "github-ssh",
      secretRef: { name: "agentrun-v01-tool-github-ssh", keys: ["id_ed25519", "known_hosts", "config"] },
      projection: { kind: "volume", mountPath: "/home/agentrun/.ssh" },
    }];
    const combinedToolCredentials = [...githubToolCredentials, ...unideskSshToolCredentials, ...githubSshToolCredentials];
    const item = await createRunWithCommand(client, { ...context, toolCredentials: combinedToolCredentials }, "job smoke", "selftest-job-render", 15_000);
    const rendered = renderRunnerJobDryRun({
      run: await client.get(`/api/v1/runs/${item.runId}`) as RunRecord,
      commandId: item.commandId,
      managerUrl: server.baseUrl,
      image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
      attemptId: "attempt_selftest",
      sourceCommit: "self-test",
      transientEnv: [{ name: "HWLAB_API_KEY", value: "hwl_live_selftest", sensitive: true }],
    });
    assert.equal(rendered.dryRun, true);
    assert.equal(rendered.mutation, false);
    assert.equal(((rendered.retention as JsonRecord).ttlSecondsAfterFinished), 86_400);
    assert.equal((rendered.jobIdentity as { serviceAccountName?: string }).serviceAccountName, "agentrun-v01-runner");
    assertRunnerJobUsesWritableCodexHome(rendered.manifest as JsonRecord, context.codexHome, "codex-0", "/var/run/agentrun/secrets/codex-0");
    assertRunnerJobUsesToolCredential(rendered, "GH_TOKEN", "agentrun-v01-tool-github-pr", "GH_TOKEN");
    assertRunnerJobUsesToolCredential(rendered, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN");
    assertRunnerJobUsesToolCredentialVolume(rendered, "agentrun-v01-tool-github-ssh", "/home/agentrun/.ssh", ["id_ed25519", "known_hosts", "config"]);
    assertRunnerJobUsesGithubSshCommand(rendered.manifest as JsonRecord, "/home/agentrun/.ssh");
    assertRunnerJobUsesG14EgressProxy(rendered.manifest as JsonRecord);
    assert.equal(runnerEnvValue(rendered.manifest as JsonRecord, "AGENTRUN_CODEX_SHELL_SANDBOX"), "danger-full-access");
    assert.equal(runnerEnvValue(rendered.manifest as JsonRecord, "HWLAB_API_KEY"), "REDACTED");
    assert.deepEqual((((rendered.transientEnv as JsonRecord).names) as string[]), ["HWLAB_API_KEY"]);
    assertNoSecretLeak(rendered);

    await assert.rejects(
      () => createRunWithCommand(client, {
        ...context,
        toolCredentials: [{
          tool: "unidesk-ssh",
          purpose: "ssh-passthrough",
          secretRef: { name: "agentrun-v01-tool-unidesk-ssh", keys: ["UNIDESK_SSH_CLIENT_TOKEN"] },
          projection: { kind: "env", envName: "UNIDESK_SSH_TOKEN", secretKey: "UNIDESK_SSH_CLIENT_TOKEN" },
        }],
      }, "bad unidesk ssh projection", "selftest-bad-unidesk-ssh-projection", 15_000),
      (error) => error instanceof Error && error.message.includes("unidesk-ssh must project UNIDESK_SSH_CLIENT_TOKEN"),
    );

    const deepseekItem = await createRunWithCommand(client, { ...context, backendProfile: "deepseek" }, "deepseek job smoke", "selftest-deepseek-job-render", 15_000);
    const deepseekRendered = renderRunnerJobDryRun({
      run: await client.get(`/api/v1/runs/${deepseekItem.runId}`) as RunRecord,
      commandId: deepseekItem.commandId,
      managerUrl: server.baseUrl,
      image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
      attemptId: "attempt_selftest_deepseek",
      sourceCommit: "self-test",
    });
    assertRunnerJobUsesWritableCodexHome(deepseekRendered.manifest as JsonRecord, context.deepseekHome, "deepseek-0", "/var/run/agentrun/secrets/deepseek-0");
    assertRunnerJobDoesNotMountProfile(deepseekRendered.manifest as JsonRecord, "codex-0");
    assertNoSecretLeak(deepseekRendered);

    const minimaxItem = await createRunWithCommand(client, { ...context, backendProfile: "minimax-m3" }, "minimax m3 job smoke", "selftest-minimax-m3-job-render", 15_000);
    const minimaxRendered = renderRunnerJobDryRun({
      run: await client.get(`/api/v1/runs/${minimaxItem.runId}`) as RunRecord,
      commandId: minimaxItem.commandId,
      managerUrl: server.baseUrl,
      image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
      attemptId: "attempt_selftest_minimax_m3",
      sourceCommit: "self-test",
    });
    assertRunnerJobUsesWritableCodexHome(minimaxRendered.manifest as JsonRecord, context.minimaxM3Home, "minimax-m3-0", "/var/run/agentrun/secrets/minimax-m3-0");
    assertRunnerJobDoesNotMountProfile(minimaxRendered.manifest as JsonRecord, "codex-0");
    assertRunnerJobDoesNotMountProfile(minimaxRendered.manifest as JsonRecord, "deepseek-0");
    assertNoSecretLeak(minimaxRendered);

    const dsflashGoItem = await createRunWithCommand(client, { ...context, backendProfile: "dsflash-go" }, "dsflash-go job smoke", "selftest-dsflash-go-job-render", 15_000);
    const dsflashGoRendered = renderRunnerJobDryRun({
      run: await client.get(`/api/v1/runs/${dsflashGoItem.runId}`) as RunRecord,
      commandId: dsflashGoItem.commandId,
      managerUrl: server.baseUrl,
      image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
      attemptId: "attempt_selftest_dsflash_go",
      sourceCommit: "self-test",
    });
    assertRunnerJobUsesWritableCodexHome(dsflashGoRendered.manifest as JsonRecord, context.deepseekHome, "dsflash-go-0", "/var/run/agentrun/secrets/dsflash-go-0");
    assertRunnerJobSecretKeys(dsflashGoRendered, "dsflash-go", ["auth.json", "config.toml", "model-catalog.json"]);
    assertRunnerJobDoesNotMountProfile(dsflashGoRendered.manifest as JsonRecord, "codex-0");
    assertRunnerJobDoesNotMountProfile(dsflashGoRendered.manifest as JsonRecord, "deepseek-0");
    assertRunnerJobDoesNotMountProfile(dsflashGoRendered.manifest as JsonRecord, "minimax-m3-0");
    assertNoSecretLeak(dsflashGoRendered);

    const legacyDsflashRun = await client.post("/api/v1/runs", {
      tenantId: "unidesk",
      projectId: "pikasTech/unidesk",
      workspaceRef: { kind: "host-path", path: context.workspace },
      providerId: "G14",
      backendProfile: "dsflash-go",
      executionPolicy: {
        sandbox: "workspace-write",
        approval: "never",
        timeoutMs: 15_000,
        network: "default",
        secretScope: { allowCredentialEcho: false, providerCredentials: [{ profile: "dsflash-go", secretRef: { name: "agentrun-v01-provider-dsflash-go", keys: ["auth.json", "config.toml"], mountPath: context.deepseekHome } }] },
      },
      traceSink: null,
    }) as RunRecord;
    const legacyDsflashCredential = legacyDsflashRun.executionPolicy.secretScope.providerCredentials?.[0];
    assert.deepEqual(legacyDsflashCredential?.secretRef.keys, ["auth.json", "config.toml", "model-catalog.json"]);

    const fakeKubectl = path.join(context.tmp, "fake-kubectl.js");
    const createdManifest = path.join(context.tmp, "created-runner-job.json");
    const createdTransientEnvSecret = path.join(context.tmp, "created-transient-env-secret.json");
    const patchedTransientEnvSecret = path.join(context.tmp, "patched-transient-env-secret.json");
    await writeFile(fakeKubectl, `#!/usr/bin/env bun
const args = Bun.argv.slice(2);
async function readStdin() {
  const chunks = [];
  for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
  return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
}
if (args[0] === "create") {
  const text = await readStdin();
  const manifest = JSON.parse(text);
  if (manifest.kind === "Secret") await Bun.write(${JSON.stringify(createdTransientEnvSecret)}, text);
  if (manifest.kind === "Job") await Bun.write(${JSON.stringify(createdManifest)}, text);
  const uid = manifest.kind === "Job" ? "job-uid-selftest" : "secret-uid-selftest";
  console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid, resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
  process.exit(0);
}
if (args[0] === "patch" && args[1] === "secret") {
  await Bun.write(${JSON.stringify(patchedTransientEnvSecret)}, JSON.stringify({ args }, null, 2));
  console.log(JSON.stringify({ apiVersion: "v1", kind: "Secret", metadata: { uid: "secret-uid-selftest", resourceVersion: "2", name: args[2], namespace: args[4] } }));
  process.exit(0);
}
if (args[0] === "delete" && args[1] === "secret") {
  console.log(JSON.stringify({ kind: "Status", status: "Success" }));
  process.exit(0);
}
console.error("unsupported fake kubectl args: " + args.join(" "));
process.exit(1);
`);
    await chmod(fakeKubectl, 0o755);
    await mkdir(path.dirname(fakeKubectl), { recursive: true });
    const serverWithKubectl = await startManagerServer({
      port: 0,
      host: "127.0.0.1",
      sourceCommit: "self-test",
      store: new MemoryAgentRunStore(),
      runnerJobDefaults: {
        namespace: "agentrun-v01",
        managerUrl: "http://agentrun-mgr.agentrun-v01.svc.cluster.local:8080",
        image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
        kubectlCommand: fakeKubectl,
        unideskSshEndpointEnv: { name: "UNIDESK_MAIN_SERVER_IP", value: "https://unidesk.default.example.test" },
      },
    });
    try {
      const jobClient = new ManagerClient(serverWithKubectl.baseUrl);
      const jobItem = await createRunWithCommand(jobClient, { ...context, toolCredentials: combinedToolCredentials }, "job create smoke", "selftest-job-create", 15_000);
      const created = await jobClient.post(`/api/v1/runs/${jobItem.runId}/runner-jobs`, {
        commandId: jobItem.commandId,
        attemptId: "attempt_selftest_create",
        transientEnv: [
          { name: "HWLAB_API_KEY", value: "hwl_live_selftest", sensitive: true },
          { name: "HWLAB_RUNTIME_API_URL", value: "http://runtime-api.test", sensitive: true },
          { name: "HWLAB_RUNTIME_WEB_URL", value: "http://runtime-web.test", sensitive: true },
          { name: "HWLAB_RUNTIME_NAMESPACE", value: "hwlab-v02", sensitive: true },
          { name: "HWLAB_RUNTIME_LANE", value: "v02", sensitive: true },
          { name: "HWLAB_RUNTIME_ENDPOINT_SOURCE", value: "runtime-namespace", sensitive: true },
          { name: "HWLAB_RUNTIME_ENDPOINT_LOCKED", value: "1", sensitive: true },
          { name: "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", value: "1", sensitive: true },
          { name: "UNIDESK_MAIN_SERVER_IP", value: "https://unidesk.example.test", sensitive: true },
        ],
      });
      assert.equal((created as { mutation?: unknown }).mutation, true);
      assert.equal(((created as JsonRecord).retention as JsonRecord).ttlSecondsAfterFinished, 86_400);
      assert.deepEqual((((created as JsonRecord).transientEnv as JsonRecord).names) as string[], ["HWLAB_API_KEY", "HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL", "HWLAB_RUNTIME_NAMESPACE", "HWLAB_RUNTIME_LANE", "HWLAB_RUNTIME_ENDPOINT_SOURCE", "HWLAB_RUNTIME_ENDPOINT_LOCKED", "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", "UNIDESK_MAIN_SERVER_IP"]);
      const transientEnvSecret = (created as JsonRecord).transientEnvSecret as JsonRecord;
      assert.match(String(transientEnvSecret.name), /^agentrun-v01-runner-env-[a-f0-9]{20}$/u);
      assert.equal(transientEnvSecret.namespace, "agentrun-v01");
      assert.equal(transientEnvSecret.valuesPrinted, false);
      assert.deepEqual(transientEnvSecret.keys as string[], ["HWLAB_API_KEY", "HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL", "HWLAB_RUNTIME_NAMESPACE", "HWLAB_RUNTIME_LANE", "HWLAB_RUNTIME_ENDPOINT_SOURCE", "HWLAB_RUNTIME_ENDPOINT_LOCKED", "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", "UNIDESK_MAIN_SERVER_IP"]);
      assert.equal(((transientEnvSecret.ownerReference as JsonRecord).attached), true);
      const secretManifest = JSON.parse(await readFile(createdTransientEnvSecret, "utf8")) as JsonRecord;
      assert.equal(secretManifest.kind, "Secret");
      assert.equal(((secretManifest.metadata as JsonRecord).name), transientEnvSecret.name);
      assert.equal(Object.keys((secretManifest.stringData as JsonRecord)).length, 9);
      const ownerPatch = JSON.parse(await readFile(patchedTransientEnvSecret, "utf8")) as JsonRecord;
      assert.deepEqual((ownerPatch.args as string[]).slice(0, 3), ["patch", "secret", String(transientEnvSecret.name)]);
      const manifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord;
      assert.equal((manifest.spec as JsonRecord).ttlSecondsAfterFinished, 86_400);
      assertRunnerJobUsesG14EgressProxy(manifest);
      assertRunnerJobUsesTransientEnvSecret(manifest, "HWLAB_API_KEY", String(transientEnvSecret.name));
      assertRunnerJobUsesTransientEnvSecret(manifest, "HWLAB_RUNTIME_API_URL", String(transientEnvSecret.name));
      assertRunnerJobUsesTransientEnvSecret(manifest, "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", String(transientEnvSecret.name));
      assertRunnerJobUsesTransientEnvSecret(manifest, "UNIDESK_MAIN_SERVER_IP", String(transientEnvSecret.name));
      assertRunnerJobUsesToolCredential({ manifest, toolCredentials: (created as JsonRecord).toolCredentials } as JsonRecord, "GH_TOKEN", "agentrun-v01-tool-github-pr", "GH_TOKEN");
      assertRunnerJobUsesToolCredential({ manifest, toolCredentials: (created as JsonRecord).toolCredentials } as JsonRecord, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN");
      assertRunnerJobUsesToolCredentialVolume({ manifest, toolCredentials: (created as JsonRecord).toolCredentials } as JsonRecord, "agentrun-v01-tool-github-ssh", "/home/agentrun/.ssh", ["id_ed25519", "known_hosts", "config"]);
      assertRunnerJobUsesGithubSshCommand(manifest, "/home/agentrun/.ssh");
      assertNoSecretLeak(created);
      const defaultEndpointJobItem = await createRunWithCommand(jobClient, { ...context, toolCredentials: unideskSshToolCredentials }, "job create unidesk ssh default endpoint", "selftest-job-create-unidesk-ssh-default-endpoint", 15_000);
      const defaultEndpointCreated = await jobClient.post(`/api/v1/runs/${defaultEndpointJobItem.runId}/runner-jobs`, {
        commandId: defaultEndpointJobItem.commandId,
        attemptId: "attempt_selftest_unidesk_ssh_default_endpoint",
      }) as JsonRecord;
      assert.deepEqual(((defaultEndpointCreated.transientEnv as JsonRecord).names) as string[], ["UNIDESK_MAIN_SERVER_IP"]);
      const defaultEndpointManifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord;
      const defaultEndpointSecret = defaultEndpointCreated.transientEnvSecret as JsonRecord;
      assertRunnerJobUsesTransientEnvSecret(defaultEndpointManifest, "UNIDESK_MAIN_SERVER_IP", String(defaultEndpointSecret.name));
      assertRunnerJobUsesG14EgressProxy(defaultEndpointManifest);
      assertRunnerJobUsesToolCredential({ manifest: defaultEndpointManifest, toolCredentials: defaultEndpointCreated.toolCredentials } as JsonRecord, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN");
      assertNoSecretLeak(defaultEndpointCreated);
      await assert.rejects(
        () => jobClient.post(`/api/v1/runs/${jobItem.runId}/runner-jobs`, {
          commandId: jobItem.commandId,
          attemptId: "attempt_selftest_bad_unidesk_ssh_transient",
          transientEnv: [{ name: "UNIDESK_SSH_CLIENT_TOKEN", value: "test-unidesk-ssh-client-token", sensitive: true }],
        }),
        (error) => error instanceof Error && error.message.includes("must use tool/provider credential assembly instead"),
      );
    } finally {
      await new Promise<void>((resolve) => serverWithKubectl.server.close(() => resolve()));
    }
    const sessionRunRecord: RunRecord = {
      id: "run-selftest-session-pvc",
      tenantId: "unidesk",
      projectId: "pikasTech/unidesk",
      workspaceRef: { kind: "host-path", path: context.workspace },
      sessionRef: { sessionId: "sess-selftest-runner-001" },
      resourceBundleRef: null,
      providerId: "G14",
      backendProfile: "codex",
      executionPolicy: { sandbox: "workspace-write", approval: "never", timeoutMs: 15_000, network: "default", secretScope: { allowCredentialEcho: false, providerCredentials: [{ profile: "codex", secretRef: { name: "agentrun-v01-provider-codex", keys: ["auth.json", "config.toml"] } }] } },
      traceSink: null,
      status: "pending",
      terminalStatus: null,
      failureKind: null,
      failureMessage: null,
      createdAt: new Date().toISOString(),
      updatedAt: new Date().toISOString(),
      claimedBy: null,
      leaseExpiresAt: null,
    };
    const sessionPvcRendered = renderRunnerJobDryRun({
      run: sessionRunRecord,
      commandId: "cmd-selftest-session-pvc",
      managerUrl: server.baseUrl,
      image: "127.0.0.1:5000/agentrun/agentrun-mgr-env:self-test",
      sessionPvc: { pvcName: "agentrun-v01-session-sess-selftest-runner-001", namespace: "agentrun-v01", mountPath: "/home/agentrun/.codex-codex/sessions", codexRolloutSubdir: "sessions" },
    });
    const sessionPvcManifest = sessionPvcRendered.manifest as JsonRecord;
    const sessionPvcSpec = sessionPvcManifest.spec as JsonRecord;
    const sessionPvcTemplate = sessionPvcSpec.template as JsonRecord;
    const sessionPvcPodSpec = sessionPvcTemplate.spec as JsonRecord;
    const sessionPvcContainers = Array.isArray(sessionPvcPodSpec.containers) ? sessionPvcPodSpec.containers as JsonRecord[] : [];
    const sessionPvcMounts = Array.isArray(sessionPvcContainers[0]?.volumeMounts) ? sessionPvcContainers[0].volumeMounts as JsonRecord[] : [];
    const sessionPvcVols = Array.isArray(sessionPvcPodSpec.volumes) ? sessionPvcPodSpec.volumes as JsonRecord[] : [];
    assert.ok(sessionPvcMounts.some((m) => m.name === "agentrun-sessions" && m.mountPath === "/home/agentrun/.codex-codex/sessions" && m.readOnly === false), "session pvc volume mount must be present");
    assert.ok(sessionPvcVols.some((v) => typeof v === "object" && v !== null && (v as JsonRecord).persistentVolumeClaim !== undefined && ((v as JsonRecord).persistentVolumeClaim as JsonRecord).claimName === "agentrun-v01-session-sess-selftest-runner-001"), "session pvc volume must reference the per-session PVC");
    const sessionPvcEnv = Array.isArray(sessionPvcContainers[0]?.env) ? sessionPvcContainers[0].env as JsonRecord[] : [];
    const envMap = new Map(sessionPvcEnv.map((e) => [String(e.name), String(e.value)]));
    assert.equal(envMap.get("AGENTRUN_SESSION_PVC_NAME"), "agentrun-v01-session-sess-selftest-runner-001");
    assert.equal(envMap.get("AGENTRUN_SESSION_PVC_NAMESPACE"), "agentrun-v01");
    assert.equal(envMap.get("AGENTRUN_SESSION_PVC_MOUNT_PATH"), "/home/agentrun/.codex-codex/sessions");
    assert.equal(envMap.get("AGENTRUN_CODEX_ROLLOUT_SUBDIR"), "sessions");
    return { name: "runner-k8s-job", tests: ["runner-k8s-job-dry-run", "runner-k8s-job-codex-shell-sandbox-env", "runner-k8s-job-g14-egress-proxy-env", "runner-k8s-job-deepseek-profile-dry-run", "runner-k8s-job-minimax-m3-profile-dry-run", "runner-k8s-job-dsflash-go-profile-dry-run", "runner-k8s-job-dsflash-go-legacy-secretref-normalized", "runner-k8s-job-create-api", "runner-k8s-job-retention-ttl", "runner-job-transient-env", "runner-job-transient-env-secretref", "runner-job-tool-credential-env", "runner-job-unidesk-ssh-tool-credential-env", "runner-job-tool-credential-volume", "runner-job-unidesk-ssh-endpoint-auto-env", "runner-job-unidesk-ssh-transient-env-denied", "runner-k8s-job-session-pvc-volume-and-env"] };
  } finally {
    await new Promise<void>((resolve) => server.server.close(() => resolve()));
  }
};

export default selfTest;

function runnerEnvEntry(manifest: JsonRecord, name: string): JsonRecord | undefined {
  const spec = manifest.spec as JsonRecord;
  const template = spec.template as JsonRecord;
  const podSpec = template.spec as JsonRecord;
  const containers = podSpec.containers as JsonRecord[];
  const runner = containers[0] as JsonRecord;
  const env = runner.env as JsonRecord[];
  return env.find((item) => item.name === name) as JsonRecord | undefined;
}

function runnerEnvValue(manifest: JsonRecord, name: string): unknown {
  return runnerEnvEntry(manifest, name)?.value;
}

function assertRunnerJobUsesTransientEnvSecret(manifest: JsonRecord, envName: string, secretName: string): void {
  const entry = runnerEnvEntry(manifest, envName);
  assert.ok(entry, `${envName} env should be present`);
  assert.equal(entry.value, undefined);
  const valueFrom = entry.valueFrom as JsonRecord;
  const secretKeyRef = valueFrom.secretKeyRef as JsonRecord;
  assert.equal(secretKeyRef.name, secretName);
  assert.equal(secretKeyRef.key, envName);
}

function assertRunnerJobUsesG14EgressProxy(manifest: JsonRecord): void {
  const proxy = "http://g14-provider-egress-proxy.unidesk.svc.cluster.local:18789";
  assert.equal(runnerEnvValue(manifest, "HTTP_PROXY"), proxy);
  assert.equal(runnerEnvValue(manifest, "HTTPS_PROXY"), proxy);
  assert.equal(runnerEnvValue(manifest, "ALL_PROXY"), proxy);
  assert.equal(runnerEnvValue(manifest, "http_proxy"), proxy);
  assert.equal(runnerEnvValue(manifest, "https_proxy"), proxy);
  assert.equal(runnerEnvValue(manifest, "all_proxy"), proxy);
  const noProxy = String(runnerEnvValue(manifest, "NO_PROXY"));
  assert.equal(runnerEnvValue(manifest, "no_proxy"), noProxy);
  assert.ok(noProxy.includes("hyueapi.com"), "NO_PROXY must keep hyueapi.com direct");
  assert.ok(noProxy.includes(".hyueapi.com"), "NO_PROXY must keep .hyueapi.com direct");
  assert.ok(noProxy.includes("g14-provider-egress-proxy.unidesk.svc.cluster.local"), "NO_PROXY must include the proxy Service itself");
  assert.ok(noProxy.includes(".svc"), "NO_PROXY must include Kubernetes Service domains");
}

function assertRunnerJobUsesGithubSshCommand(manifest: JsonRecord, mountPath: string): void {
  const value = String(runnerEnvValue(manifest, "GIT_SSH_COMMAND"));
  assert.ok(value.includes(`-F ${mountPath}/config`), "GIT_SSH_COMMAND must use the mounted SSH config");
  assert.ok(value.includes(`-i ${mountPath}/id_ed25519`), "GIT_SSH_COMMAND must use the mounted identity by absolute path");
  assert.ok(value.includes(`UserKnownHostsFile=${mountPath}/known_hosts`), "GIT_SSH_COMMAND must use mounted known_hosts by absolute path");
  assert.ok(value.includes("StrictHostKeyChecking=yes"), "GIT_SSH_COMMAND must keep strict host key checking enabled");
}

function assertRunnerJobUsesToolCredential(rendered: JsonRecord, envName: string, secretName: string, secretKey: string): void {
  const manifest = rendered.manifest as JsonRecord;
  const spec = manifest.spec as JsonRecord;
  const template = spec.template as JsonRecord;
  const podSpec = template.spec as JsonRecord;
  const containers = podSpec.containers as JsonRecord[];
  const runner = containers[0] as JsonRecord;
  const env = runner.env as JsonRecord[];
  const entry = env.find((item) => item.name === envName) as JsonRecord | undefined;
  assert.ok(entry, `${envName} env should be projected from a SecretRef`);
  assert.equal(entry.value, undefined);
  const valueFrom = entry.valueFrom as JsonRecord;
  const secretKeyRef = valueFrom.secretKeyRef as JsonRecord;
  assert.equal(secretKeyRef.name, secretName);
  assert.equal(secretKeyRef.key, secretKey);

  const summary = rendered.toolCredentials as JsonRecord;
  assert.equal(summary.valuesPrinted, false);
  assert.ok(Number(summary.count) >= 1);
  const items = summary.items as JsonRecord[];
  const summaryEntry = items.find((item) => {
    const projection = item.projection as JsonRecord;
    return item.name === secretName && projection.envName === envName && projection.secretKey === secretKey;
  });
  assert.ok(summaryEntry, `${envName} tool credential summary should include its SecretRef and projection`);
  assert.equal(summaryEntry.valuesPrinted, false);
}

function assertRunnerJobUsesToolCredentialVolume(rendered: JsonRecord, secretName: string, mountPath: string, secretKeys: string[]): void {
  const manifest = rendered.manifest as JsonRecord;
  const spec = manifest.spec as JsonRecord;
  const template = spec.template as JsonRecord;
  const podSpec = template.spec as JsonRecord;
  const containers = podSpec.containers as JsonRecord[];
  const runner = containers[0] as JsonRecord;
  const mounts = runner.volumeMounts as JsonRecord[];
  const mount = mounts.find((item) => item.mountPath === mountPath) as JsonRecord | undefined;
  assert.ok(mount, `${mountPath} tool credential volume mount should be present`);
  assert.equal(mount.readOnly, true);
  const volumes = podSpec.volumes as JsonRecord[];
  const volume = volumes.find((item) => item.name === mount.name) as JsonRecord | undefined;
  assert.ok(volume, `${mountPath} tool credential volume should be present`);
  const secret = volume.secret as JsonRecord;
  assert.equal(secret.secretName, secretName);
  assert.deepEqual((secret.items as JsonRecord[]).map((item) => item.key), secretKeys);

  const summary = rendered.toolCredentials as JsonRecord;
  assert.equal(summary.valuesPrinted, false);
  const items = summary.items as JsonRecord[];
  const summaryEntry = items.find((item) => {
    const projection = item.projection as JsonRecord;
    return item.name === secretName && projection.kind === "volume" && projection.mountPath === mountPath;
  });
  assert.ok(summaryEntry, `${mountPath} tool credential summary should include its SecretRef and volume projection`);
  assert.equal(summaryEntry.valuesPrinted, false);
}

function assertRunnerJobUsesWritableCodexHome(manifest: JsonRecord, expectedCodexHome: string, volumeName: string, projectionPath: string): void {
  const spec = manifest.spec as JsonRecord;
  const template = spec.template as JsonRecord;
  const podSpec = template.spec as JsonRecord;
  const volumes = podSpec.volumes as JsonRecord[];
  assert.ok(volumes.some((volume) => volume.name === "runner-home" && typeof volume.emptyDir === "object"), "runner home must be writable emptyDir");

  const containers = podSpec.containers as JsonRecord[];
  const runner = containers[0] as JsonRecord;
  const mounts = runner.volumeMounts as JsonRecord[];
  assert.ok(mounts.some((mount) => mount.name === "runner-home" && mount.mountPath === "/home/agentrun"), "runner-home must mount at /home/agentrun");
  assert.ok(mounts.some((mount) => mount.name === volumeName && mount.mountPath === projectionPath && mount.readOnly === true), "Codex Secret must mount read-only outside CODEX_HOME");

  const env = runner.env as JsonRecord[];
  const value = (name: string): unknown => env.find((item) => item.name === name)?.value;
  assert.equal(value("HOME"), "/home/agentrun");
  assert.equal(value("CODEX_HOME"), expectedCodexHome);
  assert.equal(value("AGENTRUN_CODEX_SECRET_HOME"), projectionPath);
  assert.equal(value("AGENTRUN_RUNNER_IDLE_TIMEOUT_MS"), "600000");
  assert.equal(value("AGENTRUN_RUNNER_POLL_INTERVAL_MS"), "250");
  assert.equal(value("AGENTRUN_RUNNER_ONE_SHOT"), undefined);
  assert.notEqual(value("CODEX_HOME"), value("AGENTRUN_CODEX_SECRET_HOME"));
}

function assertRunnerJobSecretKeys(rendered: JsonRecord, profile: string, expectedKeys: string[]): void {
  const refs = rendered.secretRefs as JsonRecord[];
  const ref = refs.find((item) => item.profile === profile);
  assert.ok(ref, `${profile} SecretRef summary must be present`);
  assert.deepEqual(ref.keys, expectedKeys);
}

function assertRunnerJobDoesNotMountProfile(manifest: JsonRecord, volumeName: string): void {
  const spec = manifest.spec as JsonRecord;
  const template = spec.template as JsonRecord;
  const podSpec = template.spec as JsonRecord;
  const volumes = podSpec.volumes as JsonRecord[];
  const containers = podSpec.containers as JsonRecord[];
  const runner = containers[0] as JsonRecord;
  const mounts = runner.volumeMounts as JsonRecord[];
  assert.equal(volumes.some((volume) => volume.name === volumeName), false, `${volumeName} volume must not be mounted for another backendProfile`);
  assert.equal(mounts.some((mount) => mount.name === volumeName), false, `${volumeName} mount must not exist for another backendProfile`);
}