AgentRun Codex
231 lines
14 KiB
TypeScript
231 lines
14 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { execFile as execFileCallback } from "node:child_process";
|
|
import { promisify } from "node:util";
|
|
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { startManagerServer } from "../../mgr/server.js";
|
|
import { MemoryAgentRunStore } from "../../mgr/store.js";
|
|
import { ManagerClient } from "../../mgr/client.js";
|
|
import { runOnce } from "../../runner/run-once.js";
|
|
import { eventContractSummary } from "../../common/events.js";
|
|
import type { BackendProfile, JsonRecord, RunEvent } from "../../common/types.js";
|
|
import { backendTurnOptions } from "../../backend/adapter.js";
|
|
import { assertNoSecretLeak, createRunWithCommand, profileSecretHome, type SelfTestCase, type SelfTestContext } from "../harness.js";
|
|
|
|
const execFile = promisify(execFileCallback);
|
|
|
|
const selfTest: SelfTestCase = async (context) => {
|
|
const fakeKubectl = path.join(context.tmp, "fake-kubectl-contract.js");
|
|
const createdManifest = path.join(context.tmp, "created-contract-runner-job.json");
|
|
await writeFile(fakeKubectl, `#!/usr/bin/env bun
|
|
const chunks = [];
|
|
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
|
|
const text = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
|
|
await Bun.write(${JSON.stringify(createdManifest)}, text);
|
|
const manifest = JSON.parse(text);
|
|
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "job-uid-contract", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
|
|
`);
|
|
await chmod(fakeKubectl, 0o755);
|
|
const store = new MemoryAgentRunStore();
|
|
const server = await startManagerServer({
|
|
port: 0,
|
|
host: "127.0.0.1",
|
|
sourceCommit: "self-test",
|
|
store,
|
|
runnerJobDefaults: {
|
|
namespace: "agentrun-v01",
|
|
managerUrl: "http://agentrun-mgr.agentrun-v01.svc.cluster.local:8080",
|
|
image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
|
kubectlCommand: fakeKubectl,
|
|
},
|
|
});
|
|
try {
|
|
const client = new ManagerClient(server.baseUrl);
|
|
await assertBackendPreflight(client);
|
|
await assertEventContractAndCompletedSemantics(client, context, server.baseUrl);
|
|
await assertRunnerJobStatus(client, context);
|
|
assertThreadResumeStandard(context);
|
|
await assertSessionProfileIsolation(client, context);
|
|
await assertResourceBundleFailure(client, context, server.baseUrl);
|
|
return { name: "hwlab-baseline-contract", tests: ["event-contract", "result-completed-terminal-only", "bounded-output-summary", "runner-job-status", "thread-resume-standard", "backend-preflight-redacted", "session-profile-isolation", "resource-bundle-failure-kind"] };
|
|
} finally {
|
|
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
|
}
|
|
};
|
|
|
|
async function assertBackendPreflight(client: ManagerClient): Promise<void> {
|
|
const response = await client.get("/api/v1/backends") as { items?: JsonRecord[] };
|
|
const items = response.items ?? [];
|
|
assert.ok(items.length >= 4, "codex/deepseek/minimax-m3/dsflash-go backend capabilities should be visible");
|
|
assert.ok(items.some((item) => item.profile === "minimax-m3"), "minimax-m3 backend capability should be visible");
|
|
assert.ok(items.some((item) => item.profile === "dsflash-go"), "dsflash-go backend capability should be visible");
|
|
for (const item of items) {
|
|
const preflight = item.preflight as JsonRecord;
|
|
const defaultSecretRef = item.defaultSecretRef as JsonRecord;
|
|
assert.equal(preflight.valuesPrinted, false);
|
|
assert.equal(preflight.credentialValuesPrinted, false);
|
|
assert.equal(defaultSecretRef.valuesPrinted, false);
|
|
assert.ok(Array.isArray(preflight.requiredSecretKeys));
|
|
}
|
|
assertNoSecretLeak(response);
|
|
}
|
|
|
|
async function assertEventContractAndCompletedSemantics(client: ManagerClient, context: SelfTestContext, managerUrl: string): Promise<void> {
|
|
const happy = await createRunWithCommand(client, context, "hello event contract", "selftest-event-contract", 15_000);
|
|
await client.post(`/api/v1/runs/${happy.runId}/events`, { type: "tool_call", payload: { method: "item/completed", type: "commandExecution", toolName: "commandExecution", itemId: "call_selftest_hwpod", command: "authorization=Bearer selftest-redacted-value hwpod build --hwpod-id d601-f103-v2", cwd: "/workspace/hwlab", status: "completed", exitCode: 0, processId: "1234" } });
|
|
await client.post(`/api/v1/runs/${happy.runId}/events`, { type: "diff", payload: { filesChanged: 1, summary: "selftest diff" } });
|
|
const result = await runOnce({ managerUrl, runId: happy.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome }, oneShot: true });
|
|
assert.equal(result.terminalStatus, "completed");
|
|
const eventsResponse = await client.get(`/api/v1/runs/${happy.runId}/events?afterSeq=0&limit=100`) as { items?: RunEvent[] };
|
|
const events = eventsResponse.items ?? [];
|
|
const summary = eventContractSummary(events);
|
|
assert.equal(summary.ok, true, JSON.stringify(summary.issues));
|
|
const types = new Set(events.map((event) => event.type));
|
|
for (const type of ["backend_status", "assistant_message", "tool_call", "command_output", "diff", "terminal_status"]) assert.ok(types.has(type as never), `missing event type ${type}`);
|
|
const envelope = await client.get(`/api/v1/runs/${happy.runId}/commands/${happy.commandId}/result`) as JsonRecord;
|
|
assert.equal(envelope.completed, true);
|
|
assert.equal(envelope.terminalStatus, "completed");
|
|
assert.equal(envelope.terminalSource, "command-record");
|
|
const toolCallSummary = envelope.toolCallSummary as JsonRecord;
|
|
assert.equal(toolCallSummary.count, 1);
|
|
assert.deepEqual(toolCallSummary.statusCounts, { completed: 1 });
|
|
assert.deepEqual(toolCallSummary.exitCodeCounts, { "0": 1 });
|
|
const toolCallItems = toolCallSummary.items as JsonRecord[];
|
|
assert.equal(toolCallItems[0]?.status, "completed");
|
|
assert.equal(toolCallItems[0]?.exitCode, 0);
|
|
assert.match(String(toolCallItems[0]?.command), /hwpod build/u);
|
|
assert.doesNotMatch(String(toolCallItems[0]?.command), /selftest-redacted-value/u);
|
|
assertNoSecretLeak({ eventsResponse, envelope });
|
|
|
|
const partial = await createRunWithCommand(client, context, "partial should not complete", "selftest-partial-not-completed", 15_000);
|
|
const largeOutput = `${"x".repeat(6_000)}\nAuthorization: Bearer test-token\n`;
|
|
await client.post(`/api/v1/runs/${partial.runId}/events`, { type: "assistant_message", payload: { text: "partial assistant only" } });
|
|
await client.post(`/api/v1/runs/${partial.runId}/events`, { type: "command_output", payload: { stream: "stdout", text: largeOutput } });
|
|
const partialEnvelope = await client.get(`/api/v1/runs/${partial.runId}/commands/${partial.commandId}/result`) as JsonRecord;
|
|
const artifactSummary = partialEnvelope.artifactSummary as JsonRecord;
|
|
const stdoutSummary = artifactSummary.stdoutSummary as JsonRecord;
|
|
assert.equal(partialEnvelope.completed, false);
|
|
assert.equal(partialEnvelope.terminalStatus, null);
|
|
assert.equal(stdoutSummary.outputTruncated, true);
|
|
assert.equal(typeof stdoutSummary.outputBytes, "number");
|
|
assertNoSecretLeak(partialEnvelope);
|
|
}
|
|
|
|
async function assertRunnerJobStatus(client: ManagerClient, context: SelfTestContext): Promise<void> {
|
|
const item = await createRunWithCommand(client, context, "runner job status", "selftest-runner-job-status", 15_000);
|
|
await client.post(`/api/v1/runs/${item.runId}/runner-jobs`, { commandId: item.commandId, idempotencyKey: "selftest-runner-job-status" }) as JsonRecord;
|
|
const manifest = JSON.parse(await readFile(path.join(context.tmp, "created-contract-runner-job.json"), "utf8")) as JsonRecord;
|
|
assert.equal(manifest.kind, "Job");
|
|
const list = await client.get(`/api/v1/runs/${item.runId}/runner-jobs`) as { items?: JsonRecord[]; count?: number };
|
|
assert.equal(list.count, 1);
|
|
const status = list.items?.[0] ?? {};
|
|
assert.equal(status.runId, item.runId);
|
|
assert.equal(status.commandId, item.commandId);
|
|
assert.equal(status.phase, "created");
|
|
assert.equal(((status.diagnosis as JsonRecord).category), "runner-job-created");
|
|
assert.equal(((status.diagnosis as JsonRecord).runnerLostSuspected), true);
|
|
assert.equal(status.valuesPrinted, false);
|
|
assert.equal(typeof status.logPath, "string");
|
|
const single = await client.get(`/api/v1/runs/${item.runId}/runner-jobs/${String(status.id)}`) as JsonRecord;
|
|
assert.equal(single.jobName, status.jobName);
|
|
assert.equal(((single.diagnosis as JsonRecord).phase), "created");
|
|
assertNoSecretLeak({ list, single });
|
|
}
|
|
|
|
function assertThreadResumeStandard(context: SelfTestContext): void {
|
|
const run = {
|
|
...runPayload(context, "codex", "selftest-thread-standard-session"),
|
|
id: "run_thread_standard",
|
|
status: "claimed",
|
|
terminalStatus: null,
|
|
terminalFailureKind: null,
|
|
terminalFailureMessage: null,
|
|
sessionRef: {
|
|
sessionId: "selftest-thread-standard-session",
|
|
conversationId: "selftest-thread-standard-session",
|
|
threadId: "thread_from_session_ref",
|
|
},
|
|
resourceBundleRef: null,
|
|
claimedBy: null,
|
|
leaseExpiresAt: null,
|
|
createdAt: "2026-06-01T00:00:00.000Z",
|
|
updatedAt: "2026-06-01T00:00:00.000Z",
|
|
} as any;
|
|
const command = {
|
|
id: "cmd_thread_standard",
|
|
runId: run.id,
|
|
type: "turn",
|
|
payload: { prompt: "resume session thread" },
|
|
state: "pending",
|
|
seq: 1,
|
|
createdAt: run.createdAt,
|
|
updatedAt: run.updatedAt,
|
|
} as any;
|
|
assert.equal(backendTurnOptions(run, command).threadId, "thread_from_session_ref");
|
|
assert.equal(backendTurnOptions(run, { ...command, payload: { prompt: "explicit wins", threadId: "thread_explicit" } }).threadId, "thread_explicit");
|
|
assert.equal(backendTurnOptions({ ...run, sessionRef: { sessionId: "selftest-thread-standard-session" } }, command).threadId, undefined);
|
|
}
|
|
|
|
async function assertSessionProfileIsolation(client: ManagerClient, context: SelfTestContext): Promise<void> {
|
|
const first = await client.post("/api/v1/runs", runPayload(context, "codex", "selftest-profile-boundary-session")) as { id: string };
|
|
await client.patch(`/api/v1/runs/${first.id}/status`, { terminalStatus: "completed", failureKind: null, failureMessage: null, threadId: "thread_codex_profile_boundary", turnId: "turn_profile_boundary" });
|
|
await assert.rejects(
|
|
() => client.post("/api/v1/runs", runPayload(context, "deepseek", "selftest-profile-boundary-session")),
|
|
(error) => error instanceof Error && error.message.includes("backendProfile boundary"),
|
|
);
|
|
}
|
|
|
|
async function assertResourceBundleFailure(client: ManagerClient, context: SelfTestContext, managerUrl: string): Promise<void> {
|
|
const repo = await createLocalGitRepo(context);
|
|
const run = await client.post("/api/v1/runs", {
|
|
...runPayload(context, "codex", "selftest-bad-bundle-session"),
|
|
resourceBundleRef: { kind: "gitbundle", repoUrl: repo.repoUrl, commitId: "0000000000000000000000000000000000000000", bundles: [{ name: "tools", subpath: "tools", targetPath: "tools" }], submodules: false, lfs: false },
|
|
}) as { id: string };
|
|
const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt: "bad bundle" }, idempotencyKey: "selftest-bad-bundle" }) as { id: string };
|
|
const result = await runOnce({ managerUrl, runId: run.id, commandId: command.id, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "bad-bundle-workspaces") }, oneShot: true }) as JsonRecord;
|
|
assert.equal(result.terminalStatus, "failed");
|
|
assert.equal(result.failureKind, "infra-failed");
|
|
const envelope = await client.get(`/api/v1/runs/${run.id}/commands/${command.id}/result`) as JsonRecord;
|
|
assert.equal(envelope.completed, false);
|
|
assert.equal(envelope.failureKind, "infra-failed");
|
|
const commandRecord = await client.get(`/api/v1/runs/${run.id}/commands/${command.id}`) as { state?: string };
|
|
assert.equal(commandRecord.state, "failed");
|
|
const runRecord = await client.get(`/api/v1/runs/${run.id}`) as { status?: string; terminalStatus?: string; failureKind?: string };
|
|
assert.equal(runRecord.status, "failed");
|
|
assert.equal(runRecord.terminalStatus, "failed");
|
|
assert.equal(runRecord.failureKind, "infra-failed");
|
|
}
|
|
|
|
function runPayload(context: SelfTestContext, backendProfile: BackendProfile, sessionId: string): JsonRecord {
|
|
const secretHome = profileSecretHome(context, backendProfile);
|
|
return {
|
|
tenantId: "hwlab",
|
|
projectId: "pikasTech/HWLAB",
|
|
workspaceRef: { kind: "opaque", repo: "pikasTech/HWLAB" },
|
|
sessionRef: { sessionId, conversationId: sessionId },
|
|
providerId: "G14",
|
|
backendProfile,
|
|
executionPolicy: {
|
|
sandbox: "workspace-write",
|
|
approval: "never",
|
|
timeoutMs: 15_000,
|
|
network: "default",
|
|
secretScope: { allowCredentialEcho: false, providerCredentials: [{ profile: backendProfile, secretRef: { name: `agentrun-v01-provider-${backendProfile}`, keys: ["auth.json", "config.toml"], mountPath: secretHome } }] },
|
|
},
|
|
traceSink: { kind: "hwlab", traceId: sessionId },
|
|
};
|
|
}
|
|
|
|
async function createLocalGitRepo(context: SelfTestContext): Promise<{ repoUrl: string; commitId: string }> {
|
|
const repo = path.join(context.tmp, "contract-bundle-repo");
|
|
await mkdir(repo, { recursive: true });
|
|
await execFile("git", ["init"], { cwd: repo });
|
|
await writeFile(path.join(repo, "README.md"), "AgentRun contract bundle self-test\n", "utf8");
|
|
await execFile("git", ["add", "README.md"], { cwd: repo });
|
|
await execFile("git", ["-c", "user.email=selftest@example.invalid", "-c", "user.name=AgentRun SelfTest", "commit", "-m", "contract bundle selftest"], { cwd: repo });
|
|
const { stdout } = await execFile("git", ["rev-parse", "HEAD"], { cwd: repo });
|
|
return { repoUrl: repo, commitId: stdout.trim() };
|
|
}
|
|
|
|
export default selfTest;
|