Codex
216 lines
16 KiB
TypeScript
216 lines
16 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { execFile as execFileCallback } from "node:child_process";
|
|
import { promisify } from "node:util";
|
|
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { startManagerServer } from "../../mgr/server.js";
|
|
import { ManagerClient } from "../../mgr/client.js";
|
|
import { MemoryAgentRunStore } from "../../mgr/store.js";
|
|
import { runOnce } from "../../runner/run-once.js";
|
|
import type { JsonRecord, ResourceBundleRef } from "../../common/types.js";
|
|
import { assertNoSecretLeak, type SelfTestCase, type SelfTestContext } from "../harness.js";
|
|
|
|
const execFile = promisify(execFileCallback);
|
|
type LocalBundle = { repoUrl: string; commitId: string; toolAliases?: ResourceBundleRef["toolAliases"] };
|
|
|
|
const selfTest: SelfTestCase = async (context) => {
|
|
const containerfile = await readFile(path.join(context.root, "deploy/container/Containerfile"), "utf8");
|
|
assert.ok(containerfile.includes(" git ") && containerfile.includes(" openssh-client"), "runtime image must include git and openssh-client for ResourceBundleRef checkout");
|
|
assert.ok(containerfile.includes("deploy/runtime/boot") && containerfile.includes("agentrun-mgr.sh"), "runtime image must boot through the env-reuse source checkout script");
|
|
assert.ok(!containerfile.includes("COPY src ./src"), "runtime env image must not bake source files into every source commit image");
|
|
const fakeKubectl = path.join(context.tmp, "fake-kubectl-hwlab.js");
|
|
const createdManifest = path.join(context.tmp, "created-hwlab-runner-job.json");
|
|
await writeFile(fakeKubectl, `#!/usr/bin/env bun
|
|
const chunks = [];
|
|
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
|
|
const text = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
|
|
await Bun.write(${JSON.stringify(createdManifest)}, text);
|
|
const manifest = JSON.parse(text);
|
|
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "job-uid-hwlab", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
|
|
`);
|
|
await chmod(fakeKubectl, 0o755);
|
|
const store = new MemoryAgentRunStore();
|
|
const server = await startManagerServer({
|
|
port: 0,
|
|
host: "127.0.0.1",
|
|
sourceCommit: "self-test",
|
|
store,
|
|
runnerJobDefaults: {
|
|
namespace: "agentrun-v01",
|
|
managerUrl: "http://agentrun-mgr.agentrun-v01.svc.cluster.local:8080",
|
|
image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
|
kubectlCommand: fakeKubectl,
|
|
},
|
|
});
|
|
try {
|
|
const client = new ManagerClient(server.baseUrl);
|
|
const bundle = await createLocalGitBundle(context);
|
|
const first = await createHwlabRun(client, context, bundle, "hwlab-session-1", "hello bundle", "hwlab-command-1");
|
|
const created = await client.post(`/api/v1/runs/${first.runId}/runner-jobs`, { commandId: first.commandId, idempotencyKey: "hwlab-trace-1" }) as JsonRecord;
|
|
const replay = await client.post(`/api/v1/runs/${first.runId}/runner-jobs`, { commandId: first.commandId, idempotencyKey: "hwlab-trace-1" }) as JsonRecord;
|
|
assert.equal(replay.idempotentReplay, true);
|
|
assert.equal(replay.jobName, created.jobName);
|
|
assert.equal(replay.attemptId, created.attemptId);
|
|
await assert.rejects(
|
|
() => client.post(`/api/v1/runs/${first.runId}/runner-jobs`, { commandId: first.commandId, idempotencyKey: "hwlab-trace-1", runnerId: "runner_changed" }),
|
|
(error) => error instanceof Error && error.message.includes("idempotency key reused"),
|
|
);
|
|
const manifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord;
|
|
assert.ok(JSON.stringify(manifest).includes("AGENTRUN_RESOURCE_BUNDLE_JSON"));
|
|
assert.equal(runnerEnvValue(manifest, "AGENTRUN_RESOURCE_BIN_PATH"), "/usr/local/bin");
|
|
assert.ok(JSON.stringify(manifest).includes("/opt/agentrun/deploy/runtime/boot/agentrun-runner.sh"));
|
|
assert.ok(JSON.stringify(manifest).includes("AGENTRUN_BOOT_COMMIT"));
|
|
assertNoSecretLeak(created);
|
|
|
|
const pendingCancel = await createHwlabRun(client, context, bundle, "hwlab-session-cancel-pending", "cancel pending", "hwlab-command-cancel-pending");
|
|
const cancelledRun = await client.post(`/api/v1/runs/${pendingCancel.runId}/cancel`, { reason: "self-test pending cancel" }) as { status?: string; terminalStatus?: string; failureKind?: string };
|
|
assert.equal(cancelledRun.status, "cancelled");
|
|
assert.equal(cancelledRun.terminalStatus, "cancelled");
|
|
assert.equal(cancelledRun.failureKind, "cancelled");
|
|
const cancelledCommand = await client.get(`/api/v1/runs/${pendingCancel.runId}/commands/${pendingCancel.commandId}`) as { state?: string };
|
|
assert.equal(cancelledCommand.state, "cancelled");
|
|
await assert.rejects(
|
|
() => client.post(`/api/v1/runs/${pendingCancel.runId}/runner-jobs`, { commandId: pendingCancel.commandId, idempotencyKey: "hwlab-cancelled-job" }),
|
|
(error) => error instanceof Error && error.message.includes("already terminal"),
|
|
);
|
|
|
|
const sessionRun = await createHwlabRun(client, context, bundle, "hwlab-session-resume", "hello session", "hwlab-command-session");
|
|
const resourceBin = path.join(context.tmp, "resource-bin");
|
|
const runResult = await runOnce({ managerUrl: server.baseUrl, runId: sessionRun.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces"), AGENTRUN_RESOURCE_BIN_PATH: resourceBin }, oneShot: true });
|
|
assert.equal(runResult.terminalStatus, "completed");
|
|
const hwpod = await execFile(path.join(resourceBin, "hwpod"), ["profile", "list"]);
|
|
assert.match(hwpod.stdout, /"argv":\["profile","list"\]/u);
|
|
await writeFile(path.join(resourceBin, "blocked"), "#!/usr/bin/env sh\necho existing\n", "utf8");
|
|
const blockedRun = await createHwlabRun(client, context, { ...bundle, toolAliases: [{ name: "blocked", path: "tools/device-pod-cli.mjs", kind: "node-script" }] }, "hwlab-session-blocked-alias", "blocked alias", "hwlab-command-blocked-alias");
|
|
const blockedResult = await runOnce({ managerUrl: server.baseUrl, runId: blockedRun.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-blocked"), AGENTRUN_RESOURCE_BIN_PATH: resourceBin }, oneShot: true });
|
|
assert.equal(blockedResult.terminalStatus, "blocked");
|
|
assert.equal(blockedResult.failureKind, "schema-invalid");
|
|
const session = await store.getSession("hwlab-session-resume");
|
|
assert.equal(session?.threadId, "thread_selftest_1");
|
|
const resultEnvelope = await client.get(`/api/v1/runs/${sessionRun.runId}/commands/${sessionRun.commandId}/result`) as JsonRecord;
|
|
assert.equal(resultEnvelope.terminalStatus, "completed");
|
|
assert.equal(resultEnvelope.reply, "fake codex stdio reply");
|
|
assert.equal(((resultEnvelope.sessionRef as JsonRecord).threadId), "thread_selftest_1");
|
|
assert.equal(((resultEnvelope.resourceBundleRef as JsonRecord).commitId), bundle.commitId);
|
|
assert.deepEqual(((resultEnvelope.resourceBundleRef as JsonRecord).toolAliases as JsonRecord).names, ["hwpod"]);
|
|
const materialized = ((resultEnvelope.resourceBundleRef as JsonRecord).materialized as JsonRecord);
|
|
assert.deepEqual(((materialized.toolAliases as JsonRecord).names), ["hwpod"]);
|
|
assertNoSecretLeak(resultEnvelope);
|
|
|
|
const resumed = await createHwlabRun(client, context, bundle, "hwlab-session-resume", "hello resumed", "hwlab-command-session-resumed");
|
|
const resumedRun = await client.get(`/api/v1/runs/${resumed.runId}`) as JsonRecord;
|
|
assert.equal(((resumedRun.sessionRef as JsonRecord).threadId), "thread_selftest_1");
|
|
|
|
const multiTurn = await createHwlabRun(client, context, bundle, "hwlab-session-multiturn", "hello first turn", "hwlab-command-multiturn-1");
|
|
const fakeCodexStartFile = path.join(context.tmp, "fake-codex-starts-multiturn.txt");
|
|
const multiturnRunner = runOnce({ managerUrl: server.baseUrl, runId: multiTurn.runId, commandId: multiTurn.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-multiturn"), AGENTRUN_FAKE_CODEX_START_FILE: fakeCodexStartFile }, idleTimeoutMs: 500, pollIntervalMs: 50 });
|
|
await waitForCommandState(client, multiTurn.runId, multiTurn.commandId, "completed");
|
|
const secondCommand = await client.post(`/api/v1/runs/${multiTurn.runId}/commands`, { type: "turn", payload: { prompt: "hello second turn", traceId: "hwlab-command-multiturn-2" }, idempotencyKey: "hwlab-command-multiturn-2" }) as { id: string };
|
|
await waitForCommandState(client, multiTurn.runId, secondCommand.id, "completed");
|
|
const multiturnResult = await multiturnRunner as JsonRecord;
|
|
assert.equal(multiturnResult.commandsProcessed, 2);
|
|
const starts = (await readTextIfExists(fakeCodexStartFile)).trim().split("\n").filter(Boolean);
|
|
assert.equal(starts.length, 1, "same-run multiturn runner must keep one codex app-server process alive instead of restarting per command");
|
|
const multiEventsResponse = await client.get(`/api/v1/runs/${multiTurn.runId}/events?afterSeq=0&limit=200`) as { items?: Array<{ type?: string; payload?: JsonRecord }> };
|
|
const multiEvents = multiEventsResponse.items ?? [];
|
|
const secondCommandCreatedAt = await commandCreatedAt(client, multiTurn.runId, secondCommand.id);
|
|
const reuseEvent = multiEvents.find((event) => event.type === "backend_status" && event.payload?.phase === "codex-app-server:reused" && event.payload?.commandId === secondCommand.id);
|
|
assert.ok(secondCommandCreatedAt, "second command must expose createdAt for pickup latency assertion");
|
|
assert.ok(reuseEvent?.payload, "second command must produce codex-app-server:reused event");
|
|
assert.ok(Date.parse(String((reuseEvent as JsonRecord).createdAt ?? "")) - Date.parse(secondCommandCreatedAt) < 1_000, "same-run runner should pick up the next command without multi-second polling latency");
|
|
assert.equal(multiEvents.filter((event) => event.type === "backend_status" && event.payload?.phase === "codex-app-server-starting").length, 1);
|
|
assert.equal(multiEvents.filter((event) => event.type === "backend_status" && event.payload?.phase === "codex-app-server:reused").length, 1);
|
|
assert.equal(multiEvents.filter((event) => event.type === "backend_status" && event.payload?.phase === "resource-bundle-materialized").length, 1);
|
|
for (const commandId of [multiTurn.commandId, secondCommand.id]) {
|
|
assert.ok(multiEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "backend-turn-started" && event.payload?.commandId === commandId), `command ${commandId} must emit backend-turn-started before waiting on Codex`);
|
|
assert.ok(multiEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "backend-turn-running" && event.payload?.commandId === commandId), `command ${commandId} must emit backend-turn-running while Codex is active`);
|
|
assert.ok(multiEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "backend-turn-finished" && event.payload?.commandId === commandId), `command ${commandId} must emit backend-turn-finished after Codex returns`);
|
|
}
|
|
assert.equal(multiEvents.filter((event) => event.type === "backend_status" && event.payload?.phase === "command-terminal").length, 2);
|
|
const secondEnvelope = await client.get(`/api/v1/runs/${multiTurn.runId}/commands/${secondCommand.id}/result`) as JsonRecord;
|
|
assert.equal(secondEnvelope.terminalStatus, "completed");
|
|
assert.equal(secondEnvelope.reply, "fake codex stdio reply");
|
|
|
|
const runningCancel = await createHwlabRun(client, context, bundle, "hwlab-session-cancel-running", "cancel running", "hwlab-command-cancel-running", 10_000);
|
|
const running = runOnce({ managerUrl: server.baseUrl, runId: runningCancel.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "missing-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-running-cancel") }, oneShot: true });
|
|
await waitForCommandState(client, runningCancel.runId, runningCancel.commandId, "acknowledged");
|
|
await client.post(`/api/v1/commands/${runningCancel.commandId}/cancel`, { reason: "self-test running cancel" });
|
|
const runningResult = await running;
|
|
assert.equal(runningResult.terminalStatus, "cancelled");
|
|
|
|
return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-bundle-materialization", "resource-bundle-tool-alias", "same-run-runner-multiturn", "running-cancel"] };
|
|
} finally {
|
|
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
|
}
|
|
};
|
|
|
|
async function createLocalGitBundle(context: SelfTestContext): Promise<LocalBundle> {
|
|
const repo = path.join(context.tmp, "bundle-repo");
|
|
await mkdir(repo, { recursive: true });
|
|
await execFile("git", ["init"], { cwd: repo });
|
|
await writeFile(path.join(repo, "README.md"), "HWLAB bundle self-test\n", "utf8");
|
|
await mkdir(path.join(repo, "tools"), { recursive: true });
|
|
await writeFile(path.join(repo, "tools", "device-pod-cli.mjs"), "console.log(JSON.stringify({ ok: true, cli: 'hwpod-selftest', argv: process.argv.slice(2) }));\n", "utf8");
|
|
await execFile("git", ["add", "README.md", "tools/device-pod-cli.mjs"], { cwd: repo });
|
|
await execFile("git", ["-c", "user.email=selftest@example.invalid", "-c", "user.name=AgentRun SelfTest", "commit", "-m", "bundle selftest"], { cwd: repo });
|
|
const { stdout } = await execFile("git", ["rev-parse", "HEAD"], { cwd: repo });
|
|
return { repoUrl: repo, commitId: stdout.trim() };
|
|
}
|
|
|
|
async function createHwlabRun(client: ManagerClient, context: SelfTestContext, bundle: LocalBundle, sessionId: string, prompt: string, idempotencyKey: string, timeoutMs = 15_000): Promise<{ runId: string; commandId: string }> {
|
|
const toolAliases = bundle.toolAliases ?? [{ name: "hwpod", path: "tools/device-pod-cli.mjs", kind: "node-script" }];
|
|
const run = await client.post("/api/v1/runs", {
|
|
tenantId: "hwlab",
|
|
projectId: "pikasTech/HWLAB",
|
|
workspaceRef: { kind: "opaque", repo: "pikasTech/HWLAB" },
|
|
sessionRef: { sessionId, conversationId: sessionId },
|
|
resourceBundleRef: { kind: "git", repoUrl: bundle.repoUrl, commitId: bundle.commitId, toolAliases, submodules: false, lfs: false },
|
|
providerId: "G14",
|
|
backendProfile: "codex",
|
|
executionPolicy: {
|
|
sandbox: "workspace-write",
|
|
approval: "never",
|
|
timeoutMs,
|
|
network: "default",
|
|
secretScope: { allowCredentialEcho: false, providerCredentials: [{ profile: "codex", secretRef: { name: "agentrun-v01-provider-codex", keys: ["auth.json", "config.toml"], mountPath: context.codexHome } }] },
|
|
},
|
|
traceSink: { kind: "hwlab", traceId: idempotencyKey },
|
|
}) as { id: string };
|
|
const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt, traceId: idempotencyKey }, idempotencyKey }) as { id: string };
|
|
return { runId: run.id, commandId: command.id };
|
|
}
|
|
|
|
async function waitForCommandState(client: ManagerClient, runId: string, commandId: string, state: string): Promise<void> {
|
|
const deadline = Date.now() + 5_000;
|
|
while (Date.now() < deadline) {
|
|
const command = await client.get(`/api/v1/runs/${runId}/commands/${commandId}`) as { state?: string };
|
|
if (command.state === state) return;
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
}
|
|
throw new Error(`command ${commandId} did not reach ${state}`);
|
|
}
|
|
|
|
async function commandCreatedAt(client: ManagerClient, runId: string, commandId: string): Promise<string> {
|
|
const command = await client.get(`/api/v1/runs/${runId}/commands/${commandId}`) as { createdAt?: string };
|
|
return command.createdAt ?? "";
|
|
}
|
|
|
|
async function readTextIfExists(filePath: string): Promise<string> {
|
|
try {
|
|
return await readFile(filePath, "utf8");
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
function runnerEnvValue(manifest: JsonRecord, name: string): unknown {
|
|
const spec = manifest.spec as JsonRecord;
|
|
const template = spec.template as JsonRecord;
|
|
const podSpec = template.spec as JsonRecord;
|
|
const containers = podSpec.containers as JsonRecord[];
|
|
const env = containers[0]?.env as JsonRecord[];
|
|
return env.find((entry) => entry.name === name)?.value;
|
|
}
|
|
|
|
export default selfTest;
|