pikasTech-agentrun/src/selftest/harness.ts

import { chmod, mkdtemp, mkdir, readFile, writeFile, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import assert from "node:assert/strict";
import { ManagerClient } from "../mgr/client.js";
import type { AipodImageRef, BackendProfile, JsonRecord } from "../common/types.js";
import { backendProfileSpec } from "../common/backend-profiles.js";
import { parseAipodSpecYaml } from "../common/aipod-specs.js";
import { dsflashGoModelCatalogJson } from "../common/model-catalogs.js";

export interface SelfTestContext {
  root: string;
  tmp: string;
  codexHome: string;
  deepseekHome: string;
  minimaxM3Home: string;
  workspace: string;
  fakeCodexPath: string;
  fakeCodexCommand: string;
  fakeCodexArgs: string[];
  cleanup(): Promise<void>;
}

export interface SelfTestResult {
  name?: string;
  tests?: string[];
}

export type SelfTestCase = (context: SelfTestContext) => Promise<SelfTestResult> | SelfTestResult;

type SelfTestRunContext = Pick<SelfTestContext, "workspace" | "codexHome"> & Partial<Pick<SelfTestContext, "deepseekHome" | "minimaxM3Home">> & { backendProfile?: BackendProfile; includeOnlyProfile?: BackendProfile; toolCredentials?: JsonRecord[] };

export async function createSelfTestContext(root: string): Promise<SelfTestContext> {
  const tmp = await mkdtemp(path.join(os.tmpdir(), "agentrun-selftest-"));
  const previousSelftestWorkReadyBinPath = process.env.AGENTRUN_SELFTEST_WORK_READY_BIN_PATH;
  const codexHome = path.join(tmp, "codex-home");
  const deepseekHome = path.join(tmp, "deepseek-home");
  const minimaxM3Home = path.join(tmp, "minimax-m3-home");
  const workReadyBin = path.join(tmp, "work-ready-bin");
  const workspace = path.join(tmp, "workspace");
  await mkdir(codexHome, { recursive: true });
  await mkdir(deepseekHome, { recursive: true });
  await mkdir(minimaxM3Home, { recursive: true });
  await mkdir(workReadyBin, { recursive: true });
  await mkdir(workspace, { recursive: true });
  await writeFakeWorkReadyTools(workReadyBin);
  process.env.AGENTRUN_SELFTEST_WORK_READY_BIN_PATH = workReadyBin;
  await writeFile(path.join(codexHome, "auth.json"), JSON.stringify({ token: "test-token-material" }));
  await writeFile(path.join(codexHome, "config.toml"), "model = \"gpt-test\"\n");
  await writeFile(path.join(deepseekHome, "auth.json"), JSON.stringify({ token: "test-token-material-deepseek" }));
  await writeFile(path.join(deepseekHome, "config.toml"), "model_provider = \"opencode\"\nmodel = \"deepseek-v4-flash\"\nreview_model = \"deepseek-v4-flash\"\nmodel_context_window = 1000000\nmodel_auto_compact_token_limit = 900000\nmodel_catalog_json = \"model-catalog.json\"\n[model_providers.opencode]\nname = \"OpenCode\"\nbase_url = \"http://hwlab-deepseek-proxy.hwlab-v02.svc.cluster.local:4000/v1\"\nwire_api = \"responses\"\nrequires_openai_auth = true\n");
  await writeFile(path.join(deepseekHome, "model-catalog.json"), dsflashGoModelCatalogJson());
  await writeFile(path.join(minimaxM3Home, "auth.json"), JSON.stringify({ token: "test-token-material-minimax-m3" }));
  await writeFile(path.join(minimaxM3Home, "config.toml"), "model = \"MiniMax-M3\"\nmodel_provider = \"minimax\"\n[model_providers.minimax]\nname = \"MiniMax\"\nbase_url = \"https://api.minimaxi.com/v1\"\nenv_key = \"MINIMAX_API_KEY\"\nwire_api = \"responses\"\n");
  await writeFile(path.join(workspace, "README.md"), "self-test workspace\n");
  const fakeCodexPath = path.join(root, "src/selftest/fake-codex-app-server.ts");
  return {
    root,
    tmp,
    codexHome,
    deepseekHome,
    minimaxM3Home,
    workspace,
    fakeCodexPath,
    fakeCodexCommand: process.env.AGENTRUN_SELFTEST_CODEX_COMMAND ?? defaultFakeCommand(),
    fakeCodexArgs: process.env.AGENTRUN_SELFTEST_CODEX_ARGS ? JSON.parse(process.env.AGENTRUN_SELFTEST_CODEX_ARGS) as string[] : defaultFakeArgs(fakeCodexPath),
    cleanup: async () => {
      if (previousSelftestWorkReadyBinPath === undefined) delete process.env.AGENTRUN_SELFTEST_WORK_READY_BIN_PATH;
      else process.env.AGENTRUN_SELFTEST_WORK_READY_BIN_PATH = previousSelftestWorkReadyBinPath;
      await rm(tmp, { recursive: true, force: true });
    },
  };
}

async function writeFakeWorkReadyTools(dir: string): Promise<void> {
  for (const tool of ["bun", "node", "npm", "git", "ssh", "gh", "rg", "curl", "kubectl"]) {
    const file = path.join(dir, tool);
    await writeFile(file, `#!/bin/sh\necho ${tool}-selftest-version\n`, "utf8");
    await chmod(file, 0o755);
  }
}

export async function createRunWithCommand(client: ManagerClient, context: SelfTestRunContext, prompt: string, idempotencyKey: string, timeoutMs: number): Promise<{ runId: string; commandId: string }> {
  const backendProfile = context.backendProfile ?? "codex";
  const run = await client.post("/api/v1/runs", {
    tenantId: "unidesk",
    projectId: "pikasTech/unidesk",
    workspaceRef: { kind: "host-path", path: context.workspace },
    providerId: "G14",
    backendProfile,
    executionPolicy: {
      sandbox: "workspace-write",
      approval: "never",
      timeoutMs,
      network: "default",
      secretScope: { allowCredentialEcho: false, providerCredentials: providerCredentials(context, backendProfile), ...toolCredentialScope(context) },
    },
    traceSink: null,
  }) as { id: string };
  const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt }, idempotencyKey }) as { id: string };
  const duplicate = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt }, idempotencyKey }) as { id: string };
  assert.equal(duplicate.id, command.id);
  return { runId: run.id, commandId: command.id };
}

function toolCredentialScope(context: { toolCredentials?: JsonRecord[] }): JsonRecord {
  return context.toolCredentials ? { toolCredentials: context.toolCredentials } : {};
}

function providerCredentials(context: Pick<SelfTestContext, "codexHome"> & Partial<Pick<SelfTestContext, "deepseekHome" | "minimaxM3Home">> & { includeOnlyProfile?: BackendProfile }, backendProfile: BackendProfile): JsonRecord[] {
  const profiles: BackendProfile[] = context.includeOnlyProfile ? [context.includeOnlyProfile] : [backendProfile];
  return profiles.map((profile) => ({
    profile,
    secretRef: {
      name: backendProfileSpec(profile)?.defaultSecretName ?? `agentrun-v01-provider-${profile}`,
      keys: [...(backendProfileSpec(profile)?.requiredSecretKeys ?? ["auth.json", "config.toml"])],
      mountPath: profileSecretHome(context, profile),
    },
  }));
}

export function assertNoSecretLeak(value: unknown): void {
  const text = JSON.stringify(value);
  assert.equal(text.includes("test-token-material"), false);
  assert.equal(text.includes("test-token-material-deepseek"), false);
  assert.equal(text.includes("test-token-material-minimax-m3"), false);
  assert.equal(text.includes("Bearer test-token"), false);
}

export function profileSecretHome(context: Pick<SelfTestContext, "codexHome"> & Partial<Pick<SelfTestContext, "deepseekHome" | "minimaxM3Home">>, profile: BackendProfile): string {
  if (profile === "deepseek") return context.deepseekHome ?? context.codexHome;
  if (profile === "dsflash-go") return context.deepseekHome ?? context.codexHome;
  if (profile === "minimax-m3") return context.minimaxM3Home ?? context.codexHome;
  return context.codexHome;
}

export async function loadArtificerImageRef(root: string): Promise<AipodImageRef> {
  const text = await readFile(path.join(root, "config", "aipods", "artificer.yaml"), "utf8");
  const spec = parseAipodSpecYaml(text, "selftest-artificer-image-ref");
  return spec.spec.imageRef;
}

function defaultFakeCommand(): string {
  return process.versions.bun ? process.execPath : "npx";
}

function defaultFakeArgs(fakePath: string): string[] {
  return process.versions.bun ? [fakePath] : ["tsx", fakePath];
}