pikasTech-agentrun/src/selftest/cases/20-runner-k8s-job.ts

import assert from "node:assert/strict";
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import { startManagerServer } from "../../mgr/server.js";
import { MemoryAgentRunStore } from "../../mgr/store.js";
import { ManagerClient } from "../../mgr/client.js";
import { renderRunnerJobDryRun } from "../../runner/k8s-job.js";
import type { JsonRecord, RunRecord } from "../../common/types.js";
import { assertNoSecretLeak, createRunWithCommand, type SelfTestCase } from "../harness.js";

const selfTest: SelfTestCase = async (context) => {
  const server = await startManagerServer({ port: 0, host: "127.0.0.1", sourceCommit: "self-test", store: new MemoryAgentRunStore() });
  try {
    const client = new ManagerClient(server.baseUrl);
    const githubToolCredentials = [{
      tool: "github",
      purpose: "pull-request",
      secretRef: { name: "agentrun-v01-tool-github-pr", keys: ["GH_TOKEN"] },
      projection: { kind: "env", envName: "GH_TOKEN", secretKey: "GH_TOKEN" },
    }];
    const unideskSshToolCredentials = [{
      tool: "unidesk-ssh",
      purpose: "ssh-passthrough",
      secretRef: { name: "agentrun-v01-tool-unidesk-ssh", keys: ["UNIDESK_SSH_CLIENT_TOKEN"] },
      projection: { kind: "env", envName: "UNIDESK_SSH_CLIENT_TOKEN", secretKey: "UNIDESK_SSH_CLIENT_TOKEN" },
    }];
    const combinedToolCredentials = [...githubToolCredentials, ...unideskSshToolCredentials];
    const item = await createRunWithCommand(client, { ...context, toolCredentials: combinedToolCredentials }, "job smoke", "selftest-job-render", 15_000);
    const rendered = renderRunnerJobDryRun({
      run: await client.get(`/api/v1/runs/${item.runId}`) as RunRecord,
      commandId: item.commandId,
      managerUrl: server.baseUrl,
      image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
      attemptId: "attempt_selftest",
      sourceCommit: "self-test",
      transientEnv: [{ name: "HWLAB_API_KEY", value: "hwl_live_selftest", sensitive: true }],
    });
    assert.equal(rendered.dryRun, true);
    assert.equal(rendered.mutation, false);
    assert.equal(((rendered.retention as JsonRecord).ttlSecondsAfterFinished), 86_400);
    assert.equal((rendered.jobIdentity as { serviceAccountName?: string }).serviceAccountName, "agentrun-v01-runner");
    assertRunnerJobUsesWritableCodexHome(rendered.manifest as JsonRecord, context.codexHome, "codex-0", "/var/run/agentrun/secrets/codex-0");
    assertRunnerJobUsesToolCredential(rendered, "GH_TOKEN", "agentrun-v01-tool-github-pr", "GH_TOKEN");
    assertRunnerJobUsesToolCredential(rendered, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN");
    assert.equal(runnerEnvValue(rendered.manifest as JsonRecord, "HWLAB_API_KEY"), "REDACTED");
    assert.deepEqual((((rendered.transientEnv as JsonRecord).names) as string[]), ["HWLAB_API_KEY"]);
    assertNoSecretLeak(rendered);

    await assert.rejects(
      () => createRunWithCommand(client, {
        ...context,
        toolCredentials: [{
          tool: "unidesk-ssh",
          purpose: "ssh-passthrough",
          secretRef: { name: "agentrun-v01-tool-unidesk-ssh", keys: ["UNIDESK_SSH_CLIENT_TOKEN"] },
          projection: { kind: "env", envName: "UNIDESK_SSH_TOKEN", secretKey: "UNIDESK_SSH_CLIENT_TOKEN" },
        }],
      }, "bad unidesk ssh projection", "selftest-bad-unidesk-ssh-projection", 15_000),
      (error) => error instanceof Error && error.message.includes("unidesk-ssh must project UNIDESK_SSH_CLIENT_TOKEN"),
    );

    const deepseekItem = await createRunWithCommand(client, { ...context, backendProfile: "deepseek" }, "deepseek job smoke", "selftest-deepseek-job-render", 15_000);
    const deepseekRendered = renderRunnerJobDryRun({
      run: await client.get(`/api/v1/runs/${deepseekItem.runId}`) as RunRecord,
      commandId: deepseekItem.commandId,
      managerUrl: server.baseUrl,
      image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
      attemptId: "attempt_selftest_deepseek",
      sourceCommit: "self-test",
    });
    assertRunnerJobUsesWritableCodexHome(deepseekRendered.manifest as JsonRecord, context.deepseekHome, "deepseek-0", "/var/run/agentrun/secrets/deepseek-0");
    assertRunnerJobDoesNotMountProfile(deepseekRendered.manifest as JsonRecord, "codex-0");
    assertNoSecretLeak(deepseekRendered);

    const minimaxItem = await createRunWithCommand(client, { ...context, backendProfile: "minimax-m3" }, "minimax m3 job smoke", "selftest-minimax-m3-job-render", 15_000);
    const minimaxRendered = renderRunnerJobDryRun({
      run: await client.get(`/api/v1/runs/${minimaxItem.runId}`) as RunRecord,
      commandId: minimaxItem.commandId,
      managerUrl: server.baseUrl,
      image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
      attemptId: "attempt_selftest_minimax_m3",
      sourceCommit: "self-test",
    });
    assertRunnerJobUsesWritableCodexHome(minimaxRendered.manifest as JsonRecord, context.minimaxM3Home, "minimax-m3-0", "/var/run/agentrun/secrets/minimax-m3-0");
    assertRunnerJobDoesNotMountProfile(minimaxRendered.manifest as JsonRecord, "codex-0");
    assertRunnerJobDoesNotMountProfile(minimaxRendered.manifest as JsonRecord, "deepseek-0");
    assertNoSecretLeak(minimaxRendered);

    const fakeKubectl = path.join(context.tmp, "fake-kubectl.js");
    const createdManifest = path.join(context.tmp, "created-runner-job.json");
    await writeFile(fakeKubectl, `#!/usr/bin/env bun
const chunks = [];
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
const text = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
await Bun.write(${JSON.stringify(createdManifest)}, text);
const manifest = JSON.parse(text);
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "job-uid-selftest", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
`);
    await chmod(fakeKubectl, 0o755);
    await mkdir(path.dirname(fakeKubectl), { recursive: true });
    const serverWithKubectl = await startManagerServer({
      port: 0,
      host: "127.0.0.1",
      sourceCommit: "self-test",
      store: new MemoryAgentRunStore(),
      runnerJobDefaults: {
        namespace: "agentrun-v01",
        managerUrl: "http://agentrun-mgr.agentrun-v01.svc.cluster.local:8080",
        image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
        kubectlCommand: fakeKubectl,
      },
    });
    try {
      const jobClient = new ManagerClient(serverWithKubectl.baseUrl);
      const jobItem = await createRunWithCommand(jobClient, { ...context, toolCredentials: combinedToolCredentials }, "job create smoke", "selftest-job-create", 15_000);
      const created = await jobClient.post(`/api/v1/runs/${jobItem.runId}/runner-jobs`, {
        commandId: jobItem.commandId,
        attemptId: "attempt_selftest_create",
        transientEnv: [
          { name: "HWLAB_API_KEY", value: "hwl_live_selftest", sensitive: true },
          { name: "HWLAB_RUNTIME_API_URL", value: "http://runtime-api.test", sensitive: true },
          { name: "HWLAB_RUNTIME_WEB_URL", value: "http://runtime-web.test", sensitive: true },
          { name: "HWLAB_RUNTIME_NAMESPACE", value: "hwlab-v02", sensitive: true },
          { name: "HWLAB_RUNTIME_LANE", value: "v02", sensitive: true },
          { name: "HWLAB_RUNTIME_ENDPOINT_SOURCE", value: "runtime-namespace", sensitive: true },
          { name: "HWLAB_RUNTIME_ENDPOINT_LOCKED", value: "1", sensitive: true },
          { name: "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", value: "1", sensitive: true },
          { name: "UNIDESK_MAIN_SERVER_IP", value: "https://unidesk.example.test", sensitive: true },
        ],
      });
      assert.equal((created as { mutation?: unknown }).mutation, true);
      assert.equal(((created as JsonRecord).retention as JsonRecord).ttlSecondsAfterFinished, 86_400);
      assert.deepEqual((((created as JsonRecord).transientEnv as JsonRecord).names) as string[], ["HWLAB_API_KEY", "HWLAB_RUNTIME_API_URL", "HWLAB_RUNTIME_WEB_URL", "HWLAB_RUNTIME_NAMESPACE", "HWLAB_RUNTIME_LANE", "HWLAB_RUNTIME_ENDPOINT_SOURCE", "HWLAB_RUNTIME_ENDPOINT_LOCKED", "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", "UNIDESK_MAIN_SERVER_IP"]);
      const manifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord;
      assert.equal((manifest.spec as JsonRecord).ttlSecondsAfterFinished, 86_400);
      assert.equal(runnerEnvValue(manifest, "HWLAB_API_KEY"), "hwl_live_selftest");
      assert.equal(runnerEnvValue(manifest, "HWLAB_RUNTIME_API_URL"), "http://runtime-api.test");
      assert.equal(runnerEnvValue(manifest, "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME"), "1");
      assert.equal(runnerEnvValue(manifest, "UNIDESK_MAIN_SERVER_IP"), "https://unidesk.example.test");
      assertRunnerJobUsesToolCredential({ manifest, toolCredentials: (created as JsonRecord).toolCredentials } as JsonRecord, "GH_TOKEN", "agentrun-v01-tool-github-pr", "GH_TOKEN");
      assertRunnerJobUsesToolCredential({ manifest, toolCredentials: (created as JsonRecord).toolCredentials } as JsonRecord, "UNIDESK_SSH_CLIENT_TOKEN", "agentrun-v01-tool-unidesk-ssh", "UNIDESK_SSH_CLIENT_TOKEN");
      assertNoSecretLeak(created);
      await assert.rejects(
        () => jobClient.post(`/api/v1/runs/${jobItem.runId}/runner-jobs`, {
          commandId: jobItem.commandId,
          attemptId: "attempt_selftest_bad_unidesk_ssh_transient",
          transientEnv: [{ name: "UNIDESK_SSH_CLIENT_TOKEN", value: "test-unidesk-ssh-client-token", sensitive: true }],
        }),
        (error) => error instanceof Error && error.message.includes("must use tool/provider credential assembly instead"),
      );
    } finally {
      await new Promise<void>((resolve) => serverWithKubectl.server.close(() => resolve()));
    }
    const sessionRunRecord: RunRecord = {
      id: "run-selftest-session-pvc",
      tenantId: "unidesk",
      projectId: "pikasTech/unidesk",
      workspaceRef: { kind: "host-path", path: context.workspace },
      sessionRef: { sessionId: "sess-selftest-runner-001" },
      resourceBundleRef: null,
      providerId: "G14",
      backendProfile: "codex",
      executionPolicy: { sandbox: "workspace-write", approval: "never", timeoutMs: 15_000, network: "default", secretScope: { allowCredentialEcho: false, providerCredentials: [{ profile: "codex", secretRef: { name: "agentrun-v01-provider-codex", keys: ["auth.json", "config.toml"] } }] } },
      traceSink: null,
      status: "pending",
      terminalStatus: null,
      failureKind: null,
      failureMessage: null,
      createdAt: new Date().toISOString(),
      updatedAt: new Date().toISOString(),
      claimedBy: null,
      leaseExpiresAt: null,
    };
    const sessionPvcRendered = renderRunnerJobDryRun({
      run: sessionRunRecord,
      commandId: "cmd-selftest-session-pvc",
      managerUrl: server.baseUrl,
      image: "127.0.0.1:5000/agentrun/agentrun-mgr-env:self-test",
      sessionPvc: { pvcName: "agentrun-v01-session-sess-selftest-runner-001", namespace: "agentrun-v01", mountPath: "/home/agentrun/.codex-codex/sessions", codexRolloutSubdir: "sessions" },
    });
    const sessionPvcManifest = sessionPvcRendered.manifest as JsonRecord;
    const sessionPvcSpec = sessionPvcManifest.spec as JsonRecord;
    const sessionPvcTemplate = sessionPvcSpec.template as JsonRecord;
    const sessionPvcPodSpec = sessionPvcTemplate.spec as JsonRecord;
    const sessionPvcContainers = Array.isArray(sessionPvcPodSpec.containers) ? sessionPvcPodSpec.containers as JsonRecord[] : [];
    const sessionPvcMounts = Array.isArray(sessionPvcContainers[0]?.volumeMounts) ? sessionPvcContainers[0].volumeMounts as JsonRecord[] : [];
    const sessionPvcVols = Array.isArray(sessionPvcPodSpec.volumes) ? sessionPvcPodSpec.volumes as JsonRecord[] : [];
    assert.ok(sessionPvcMounts.some((m) => m.name === "agentrun-sessions" && m.mountPath === "/home/agentrun/.codex-codex/sessions" && m.readOnly === false), "session pvc volume mount must be present");
    assert.ok(sessionPvcVols.some((v) => typeof v === "object" && v !== null && (v as JsonRecord).persistentVolumeClaim !== undefined && ((v as JsonRecord).persistentVolumeClaim as JsonRecord).claimName === "agentrun-v01-session-sess-selftest-runner-001"), "session pvc volume must reference the per-session PVC");
    const sessionPvcEnv = Array.isArray(sessionPvcContainers[0]?.env) ? sessionPvcContainers[0].env as JsonRecord[] : [];
    const envMap = new Map(sessionPvcEnv.map((e) => [String(e.name), String(e.value)]));
    assert.equal(envMap.get("AGENTRUN_SESSION_PVC_NAME"), "agentrun-v01-session-sess-selftest-runner-001");
    assert.equal(envMap.get("AGENTRUN_SESSION_PVC_NAMESPACE"), "agentrun-v01");
    assert.equal(envMap.get("AGENTRUN_SESSION_PVC_MOUNT_PATH"), "/home/agentrun/.codex-codex/sessions");
    assert.equal(envMap.get("AGENTRUN_CODEX_ROLLOUT_SUBDIR"), "sessions");
    return { name: "runner-k8s-job", tests: ["runner-k8s-job-dry-run", "runner-k8s-job-deepseek-profile-dry-run", "runner-k8s-job-minimax-m3-profile-dry-run", "runner-k8s-job-create-api", "runner-k8s-job-retention-ttl", "runner-job-transient-env", "runner-job-tool-credential-env", "runner-job-unidesk-ssh-tool-credential-env", "runner-job-unidesk-ssh-transient-env-denied", "runner-k8s-job-session-pvc-volume-and-env"] };
  } finally {
    await new Promise<void>((resolve) => server.server.close(() => resolve()));
  }
};

export default selfTest;

function runnerEnvValue(manifest: JsonRecord, name: string): unknown {
  const spec = manifest.spec as JsonRecord;
  const template = spec.template as JsonRecord;
  const podSpec = template.spec as JsonRecord;
  const containers = podSpec.containers as JsonRecord[];
  const runner = containers[0] as JsonRecord;
  const env = runner.env as JsonRecord[];
  return env.find((item) => item.name === name)?.value;
}

function assertRunnerJobUsesToolCredential(rendered: JsonRecord, envName: string, secretName: string, secretKey: string): void {
  const manifest = rendered.manifest as JsonRecord;
  const spec = manifest.spec as JsonRecord;
  const template = spec.template as JsonRecord;
  const podSpec = template.spec as JsonRecord;
  const containers = podSpec.containers as JsonRecord[];
  const runner = containers[0] as JsonRecord;
  const env = runner.env as JsonRecord[];
  const entry = env.find((item) => item.name === envName) as JsonRecord | undefined;
  assert.ok(entry, `${envName} env should be projected from a SecretRef`);
  assert.equal(entry.value, undefined);
  const valueFrom = entry.valueFrom as JsonRecord;
  const secretKeyRef = valueFrom.secretKeyRef as JsonRecord;
  assert.equal(secretKeyRef.name, secretName);
  assert.equal(secretKeyRef.key, secretKey);

  const summary = rendered.toolCredentials as JsonRecord;
  assert.equal(summary.valuesPrinted, false);
  assert.ok(Number(summary.count) >= 1);
  const items = summary.items as JsonRecord[];
  const summaryEntry = items.find((item) => {
    const projection = item.projection as JsonRecord;
    return item.name === secretName && projection.envName === envName && projection.secretKey === secretKey;
  });
  assert.ok(summaryEntry, `${envName} tool credential summary should include its SecretRef and projection`);
  assert.equal(summaryEntry.valuesPrinted, false);
}

function assertRunnerJobUsesWritableCodexHome(manifest: JsonRecord, expectedCodexHome: string, volumeName: string, projectionPath: string): void {
  const spec = manifest.spec as JsonRecord;
  const template = spec.template as JsonRecord;
  const podSpec = template.spec as JsonRecord;
  const volumes = podSpec.volumes as JsonRecord[];
  assert.ok(volumes.some((volume) => volume.name === "runner-home" && typeof volume.emptyDir === "object"), "runner home must be writable emptyDir");

  const containers = podSpec.containers as JsonRecord[];
  const runner = containers[0] as JsonRecord;
  const mounts = runner.volumeMounts as JsonRecord[];
  assert.ok(mounts.some((mount) => mount.name === "runner-home" && mount.mountPath === "/home/agentrun"), "runner-home must mount at /home/agentrun");
  assert.ok(mounts.some((mount) => mount.name === volumeName && mount.mountPath === projectionPath && mount.readOnly === true), "Codex Secret must mount read-only outside CODEX_HOME");

  const env = runner.env as JsonRecord[];
  const value = (name: string): unknown => env.find((item) => item.name === name)?.value;
  assert.equal(value("HOME"), "/home/agentrun");
  assert.equal(value("CODEX_HOME"), expectedCodexHome);
  assert.equal(value("AGENTRUN_CODEX_SECRET_HOME"), projectionPath);
  assert.equal(value("AGENTRUN_RUNNER_IDLE_TIMEOUT_MS"), "600000");
  assert.equal(value("AGENTRUN_RUNNER_POLL_INTERVAL_MS"), "250");
  assert.equal(value("AGENTRUN_RUNNER_ONE_SHOT"), undefined);
  assert.notEqual(value("CODEX_HOME"), value("AGENTRUN_CODEX_SECRET_HOME"));
}

function assertRunnerJobDoesNotMountProfile(manifest: JsonRecord, volumeName: string): void {
  const spec = manifest.spec as JsonRecord;
  const template = spec.template as JsonRecord;
  const podSpec = template.spec as JsonRecord;
  const volumes = podSpec.volumes as JsonRecord[];
  const containers = podSpec.containers as JsonRecord[];
  const runner = containers[0] as JsonRecord;
  const mounts = runner.volumeMounts as JsonRecord[];
  assert.equal(volumes.some((volume) => volume.name === volumeName), false, `${volumeName} volume must not be mounted for another backendProfile`);
  assert.equal(mounts.some((mount) => mount.name === volumeName), false, `${volumeName} mount must not exist for another backendProfile`);
}