Lyon
460 lines
38 KiB
TypeScript
460 lines
38 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { execFile as execFileCallback } from "node:child_process";
|
|
import { promisify } from "node:util";
|
|
import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { startManagerServer } from "../../mgr/server.js";
|
|
import { ManagerClient } from "../../mgr/client.js";
|
|
import { MemoryAgentRunStore } from "../../mgr/store.js";
|
|
import { runOnce } from "../../runner/run-once.js";
|
|
import type { JsonRecord, ResourceBundleRef } from "../../common/types.js";
|
|
import { assertNoSecretLeak, type SelfTestCase, type SelfTestContext } from "../harness.js";
|
|
|
|
const execFile = promisify(execFileCallback);
|
|
type LocalBundle = { repoUrl: string; commitId: string; branch?: string; bundles?: ResourceBundleRef["bundles"]; promptRefs?: ResourceBundleRef["promptRefs"]; requiredSkills?: ResourceBundleRef["requiredSkills"] };
|
|
type RefResolvedBundle = LocalBundle & { branch: string; latestCommitId: string };
|
|
|
|
const selfTest: SelfTestCase = async (context) => {
|
|
const containerfile = await readFile(path.join(context.root, "deploy/container/Containerfile"), "utf8");
|
|
assert.ok(containerfile.includes(" git ") && containerfile.includes(" openssh-client"), "runtime image must include git and openssh-client for ResourceBundleRef checkout");
|
|
assert.ok(containerfile.includes("deploy/runtime/boot") && containerfile.includes("agentrun-mgr.sh"), "runtime image must boot through the env-reuse source checkout script");
|
|
assert.ok(!containerfile.includes("COPY src ./src"), "runtime env image must not bake source files into every source commit image");
|
|
const fakeKubectl = path.join(context.tmp, "fake-kubectl-hwlab.js");
|
|
const createdManifest = path.join(context.tmp, "created-hwlab-runner-job.json");
|
|
await writeFile(fakeKubectl, `#!/usr/bin/env bun
|
|
const args = Bun.argv.slice(2);
|
|
async function readStdin() {
|
|
const chunks = [];
|
|
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
|
|
return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
|
|
}
|
|
if (args[0] === "get" && args[1] === "pvc") {
|
|
console.error('Error from server (NotFound): persistentvolumeclaims "' + args[2] + '" not found');
|
|
process.exit(1);
|
|
}
|
|
if (args[0] === "create") {
|
|
const text = await readStdin();
|
|
const manifest = JSON.parse(text);
|
|
if (manifest.kind === "PersistentVolumeClaim") {
|
|
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "pvc-uid-hwlab", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace }, status: { phase: "Bound" } }));
|
|
process.exit(0);
|
|
}
|
|
await Bun.write(${JSON.stringify(createdManifest)}, text);
|
|
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "job-uid-hwlab", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
|
|
process.exit(0);
|
|
}
|
|
console.error("unsupported fake kubectl args: " + args.join(" "));
|
|
process.exit(1);
|
|
`);
|
|
await chmod(fakeKubectl, 0o755);
|
|
const store = new MemoryAgentRunStore();
|
|
const server = await startManagerServer({
|
|
port: 0,
|
|
host: "127.0.0.1",
|
|
sourceCommit: "self-test",
|
|
store,
|
|
runnerJobDefaults: {
|
|
namespace: "agentrun-v01",
|
|
managerUrl: "http://agentrun-mgr.agentrun-v01.svc.cluster.local:8080",
|
|
image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
|
kubectlCommand: fakeKubectl,
|
|
},
|
|
});
|
|
try {
|
|
const client = new ManagerClient(server.baseUrl);
|
|
const bundle = await createLocalGitBundle(context);
|
|
const assemblyBundle: LocalBundle = {
|
|
...bundle,
|
|
promptRefs: [{ name: "hwlab-v02-runtime", path: "internal/agent/prompts/hwlab-v02-runtime.md", inject: "thread-start", required: true }],
|
|
};
|
|
const first = await createHwlabRun(client, context, bundle, "hwlab-session-1", "hello bundle", "hwlab-command-1");
|
|
const created = await client.post(`/api/v1/runs/${first.runId}/runner-jobs`, { commandId: first.commandId, idempotencyKey: "hwlab-trace-1" }) as JsonRecord;
|
|
const replay = await client.post(`/api/v1/runs/${first.runId}/runner-jobs`, { commandId: first.commandId, idempotencyKey: "hwlab-trace-1" }) as JsonRecord;
|
|
assert.equal(replay.idempotentReplay, true);
|
|
assert.equal(replay.jobName, created.jobName);
|
|
assert.equal(replay.attemptId, created.attemptId);
|
|
await assert.rejects(
|
|
() => client.post(`/api/v1/runs/${first.runId}/runner-jobs`, { commandId: first.commandId, idempotencyKey: "hwlab-trace-1", runnerId: "runner_changed" }),
|
|
(error) => error instanceof Error && error.message.includes("idempotency key reused"),
|
|
);
|
|
const manifest = JSON.parse(await readFile(createdManifest, "utf8")) as JsonRecord;
|
|
assert.ok(JSON.stringify(manifest).includes("AGENTRUN_RESOURCE_BUNDLE_JSON"));
|
|
assert.ok(JSON.stringify(manifest).includes("/opt/agentrun/deploy/runtime/boot/agentrun-runner.sh"));
|
|
assert.ok(JSON.stringify(manifest).includes("AGENTRUN_BOOT_COMMIT"));
|
|
assertNoSecretLeak(created);
|
|
|
|
const pendingCancel = await createHwlabRun(client, context, bundle, "hwlab-session-cancel-pending", "cancel pending", "hwlab-command-cancel-pending");
|
|
const cancelledRun = await client.post(`/api/v1/runs/${pendingCancel.runId}/cancel`, { reason: "self-test pending cancel" }) as { status?: string; terminalStatus?: string; failureKind?: string };
|
|
assert.equal(cancelledRun.status, "cancelled");
|
|
assert.equal(cancelledRun.terminalStatus, "cancelled");
|
|
assert.equal(cancelledRun.failureKind, "cancelled");
|
|
const cancelledCommand = await client.get(`/api/v1/runs/${pendingCancel.runId}/commands/${pendingCancel.commandId}`) as { state?: string };
|
|
assert.equal(cancelledCommand.state, "cancelled");
|
|
await assert.rejects(
|
|
() => client.post(`/api/v1/runs/${pendingCancel.runId}/runner-jobs`, { commandId: pendingCancel.commandId, idempotencyKey: "hwlab-cancelled-job" }),
|
|
(error) => error instanceof Error && error.message.includes("already terminal"),
|
|
);
|
|
|
|
const sessionRun = await createHwlabRun(client, context, bundle, "hwlab-session-resume", "hello session", "hwlab-command-session");
|
|
const resourceBinPath = path.join(context.tmp, "resource-bin");
|
|
const runResult = await runOnce({ managerUrl: server.baseUrl, runId: sessionRun.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces"), AGENTRUN_RESOURCE_BIN_PATH: resourceBinPath }, oneShot: true });
|
|
assert.equal(runResult.terminalStatus, "completed");
|
|
await access(path.join(resourceBinPath, "hwpod"));
|
|
const resourceBinExec = await execFile(path.join(resourceBinPath, "hwpod"), ["--selftest"]);
|
|
assert.match(resourceBinExec.stdout, /hwpod-selftest/u);
|
|
await access(path.join(resourceBinPath, "agentrun-git"));
|
|
const agentrunGitHelp = JSON.parse((await execFile(path.join(resourceBinPath, "agentrun-git"), ["--help"])).stdout) as JsonRecord;
|
|
assert.equal(agentrunGitHelp.tool, "agentrun-git");
|
|
const session = await store.getSession("hwlab-session-resume");
|
|
assert.equal(session?.threadId, "thread_selftest_1");
|
|
const resultEnvelope = await client.get(`/api/v1/runs/${sessionRun.runId}/commands/${sessionRun.commandId}/result`) as JsonRecord;
|
|
assert.equal(resultEnvelope.terminalStatus, "completed");
|
|
assert.equal(resultEnvelope.reply, "fake codex stdio reply");
|
|
assert.equal(((resultEnvelope.sessionRef as JsonRecord).threadId), "thread_selftest_1");
|
|
assert.equal(((resultEnvelope.resourceBundleRef as JsonRecord).commitId), bundle.commitId);
|
|
assert.equal(((resultEnvelope.resourceBundleRef as JsonRecord).kind), "gitbundle");
|
|
assert.deepEqual((((resultEnvelope.resourceBundleRef as JsonRecord).requiredSkills as JsonRecord).names), ["dad-dev"]);
|
|
const resultBundleTargets = (((resultEnvelope.resourceBundleRef as JsonRecord).bundles as JsonRecord).items as JsonRecord[]).map((item) => item.targetPath);
|
|
assert.deepEqual(resultBundleTargets, ["tools", ".agents/skills"]);
|
|
const materialized = ((resultEnvelope.resourceBundleRef as JsonRecord).materialized as JsonRecord);
|
|
assert.deepEqual(((materialized.tools as JsonRecord).names), ["apply_patch", "hwpod", "tran", "trans"]);
|
|
assert.equal(((materialized.tools as JsonRecord).installed), true);
|
|
const runtimeToolNames = (((materialized.tools as JsonRecord).runtimeTools as JsonRecord).names) as string[];
|
|
assert.ok(runtimeToolNames.includes("agentrun-git"));
|
|
const runtimeToolItems = (((materialized.tools as JsonRecord).runtimeTools as JsonRecord).items as JsonRecord[]);
|
|
const agentrunGitRuntimeTool = runtimeToolItems.find((item) => item.name === "agentrun-git");
|
|
assert.equal(agentrunGitRuntimeTool?.overridesResourceTool, false);
|
|
assert.deepEqual(((materialized.skillDirs as JsonRecord).names), ["dad-dev", "hwpod-cli", "hwpod-ctl"]);
|
|
const requiredSkillItems = ((materialized.requiredSkills as JsonRecord).items as JsonRecord[]);
|
|
assert.deepEqual(((materialized.requiredSkills as JsonRecord).names), ["dad-dev"]);
|
|
assert.equal(requiredSkillItems[0]?.status, "materialized");
|
|
assert.equal(typeof requiredSkillItems[0]?.manifestSha256, "string");
|
|
assert.equal(typeof requiredSkillItems[0]?.manifestBytes, "number");
|
|
assert.equal(((requiredSkillItems[0]?.sourceBundle as JsonRecord).commitId), bundle.commitId);
|
|
assertNoSecretLeak(resultEnvelope);
|
|
|
|
const refBundle = await createRefResolvedLocalGitBundle(context);
|
|
const refRun = await createHwlabRun(client, context, refBundle, "hwlab-session-ref-resolution", "resolve bundle from branch ref", "hwlab-command-ref-resolution");
|
|
const refRunResult = await runOnce({ managerUrl: server.baseUrl, runId: refRun.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-ref-resolution") }, oneShot: true }) as JsonRecord;
|
|
assert.equal(refRunResult.terminalStatus, "completed");
|
|
const refEnvelope = await client.get(`/api/v1/runs/${refRun.runId}/commands/${refRun.commandId}/result`) as JsonRecord;
|
|
const refResource = refEnvelope.resourceBundleRef as JsonRecord;
|
|
assert.equal(refResource.commitId, refBundle.commitId, "result summary keeps the request commit as a non-authoritative hint");
|
|
const refMaterialized = refResource.materialized as JsonRecord;
|
|
assert.equal(refMaterialized.commitId, refBundle.latestCommitId, "materialized bundle must resolve the current workspaceRef.branch commit");
|
|
assert.equal(refMaterialized.requestedCommitId, refBundle.commitId);
|
|
assert.equal(refMaterialized.requestedRef, refBundle.branch);
|
|
const refBundleCommits = (((refMaterialized.bundles as JsonRecord).items as JsonRecord[]).map((item) => item.commitId));
|
|
assert.deepEqual(refBundleCommits, [refBundle.latestCommitId, refBundle.latestCommitId]);
|
|
assertNoSecretLeak(refEnvelope);
|
|
|
|
const assemblyRun = await createHwlabRun(client, context, assemblyBundle, "hwlab-session-assembly", "list visible bundle skills without tools", "hwlab-command-assembly-1");
|
|
const assemblyInputFile = path.join(context.tmp, "fake-codex-turn-input-assembly.jsonl");
|
|
const assemblyRunner = runOnce({ managerUrl: server.baseUrl, runId: assemblyRun.runId, commandId: assemblyRun.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-assembly"), AGENTRUN_FAKE_CODEX_TURN_INPUT_FILE: assemblyInputFile }, idleTimeoutMs: 500, pollIntervalMs: 50 });
|
|
await waitForCommandState(client, assemblyRun.runId, assemblyRun.commandId, "completed");
|
|
const assemblySecond = await client.post(`/api/v1/runs/${assemblyRun.runId}/commands`, { type: "turn", payload: { prompt: "second turn should resume without initial prompt", traceId: "hwlab-command-assembly-2" }, idempotencyKey: "hwlab-command-assembly-2" }) as { id: string };
|
|
await waitForCommandState(client, assemblyRun.runId, assemblySecond.id, "completed");
|
|
const assemblyRunnerResult = await assemblyRunner as JsonRecord;
|
|
assert.equal(assemblyRunnerResult.commandsProcessed, 2);
|
|
const assemblyInputs = (await readTextIfExists(assemblyInputFile)).trim().split("\n").filter(Boolean).map((line) => JSON.parse(line) as JsonRecord);
|
|
assert.equal(assemblyInputs.length, 2);
|
|
const firstAssemblyInput = turnInputText(assemblyInputs[0]);
|
|
const secondAssemblyInput = turnInputText(assemblyInputs[1]);
|
|
assert.match(firstAssemblyInput, /HWLAB v0\.2 runtime prompt self-test/u);
|
|
assert.match(firstAssemblyInput, /dad-dev/u);
|
|
assert.match(firstAssemblyInput, /hwpod-cli/u);
|
|
assert.match(firstAssemblyInput, /hwpod-ctl/u);
|
|
assert.match(firstAssemblyInput, /hwpod-compiler-cli/u);
|
|
assert.match(firstAssemblyInput, /hwpod-node-ops/u);
|
|
assert.match(firstAssemblyInput, /hwpod/u);
|
|
assert.match(firstAssemblyInput, /list visible bundle skills/u);
|
|
assert.match(secondAssemblyInput, /second turn should resume/u);
|
|
assert.doesNotMatch(secondAssemblyInput, /HWLAB v0\.2 runtime prompt self-test/u);
|
|
assert.doesNotMatch(secondAssemblyInput, /Resource Skills/u);
|
|
const assemblyEventsResponse = await client.get(`/api/v1/runs/${assemblyRun.runId}/events?afterSeq=0&limit=200`) as { items?: Array<{ type?: string; payload?: JsonRecord }> };
|
|
const assemblyEvents = assemblyEventsResponse.items ?? [];
|
|
assert.ok(assemblyEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "initial-prompt-assembly" && event.payload?.commandId === assemblyRun.commandId && event.payload?.initialPromptInjected === true));
|
|
assert.ok(assemblyEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "initial-prompt-assembly" && event.payload?.commandId === assemblySecond.id && event.payload?.initialPromptInjected === false && event.payload?.reason === "thread-resume"));
|
|
const assemblyEnvelope = await client.get(`/api/v1/runs/${assemblyRun.runId}/commands/${assemblyRun.commandId}/result`) as JsonRecord;
|
|
const assemblyResource = assemblyEnvelope.resourceBundleRef as JsonRecord;
|
|
assert.deepEqual(((assemblyResource.promptRefs as JsonRecord).names), ["hwlab-v02-runtime"]);
|
|
const assemblyBundleTargets = ((assemblyResource.bundles as JsonRecord).items as JsonRecord[]).map((item) => item.targetPath);
|
|
assert.deepEqual(assemblyBundleTargets, ["tools", ".agents/skills"]);
|
|
const assemblyMaterialized = assemblyResource.materialized as JsonRecord;
|
|
assert.deepEqual(((assemblyMaterialized.promptRefs as JsonRecord).names), ["hwlab-v02-runtime"]);
|
|
assert.deepEqual(((assemblyMaterialized.skillDirs as JsonRecord).names), ["dad-dev", "hwpod-cli", "hwpod-ctl"]);
|
|
assert.deepEqual(((assemblyMaterialized.requiredSkills as JsonRecord).names), ["dad-dev"]);
|
|
assert.equal(((assemblyMaterialized.initialPrompt as JsonRecord).available), true);
|
|
assertNoSecretLeak(assemblyEnvelope);
|
|
|
|
const missingPromptRun = await createHwlabRun(client, context, { ...bundle, promptRefs: [{ name: "missing-prompt", path: "internal/agent/prompts/missing.md", inject: "thread-start", required: true }] }, "hwlab-session-missing-prompt", "missing prompt", "hwlab-command-missing-prompt");
|
|
const missingPromptResult = await runOnce({ managerUrl: server.baseUrl, runId: missingPromptRun.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-missing-prompt") }, oneShot: true }) as JsonRecord;
|
|
assert.equal(missingPromptResult.terminalStatus, "blocked");
|
|
assert.equal(missingPromptResult.failureKind, "prompt-unavailable");
|
|
|
|
const missingSkillRun = await createHwlabRun(client, context, { ...bundle, bundles: [{ name: "hwlab-tools", subpath: "tools", targetPath: "tools" }], requiredSkills: [{ name: "dad-dev" }] }, "hwlab-session-missing-skill", "missing required skill", "hwlab-command-missing-skill");
|
|
const missingSkillResult = await runOnce({ managerUrl: server.baseUrl, runId: missingSkillRun.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-missing-skill") }, oneShot: true }) as JsonRecord;
|
|
assert.equal(missingSkillResult.terminalStatus, "blocked");
|
|
assert.equal(missingSkillResult.failureKind, "required-skill-unavailable");
|
|
const missingSkillEnvelope = await client.get(`/api/v1/runs/${missingSkillRun.runId}/commands/${missingSkillRun.commandId}/result`) as JsonRecord;
|
|
assert.equal(missingSkillEnvelope.terminalStatus, "blocked");
|
|
assert.equal(missingSkillEnvelope.terminalSource, "terminal_status-event");
|
|
assert.equal(missingSkillEnvelope.failureKind, "required-skill-unavailable");
|
|
assert.deepEqual((((missingSkillEnvelope.resourceBundleRef as JsonRecord).requiredSkills as JsonRecord).names), ["dad-dev"]);
|
|
const missingSkillBlocker = missingSkillEnvelope.blocker as JsonRecord;
|
|
assert.equal(missingSkillBlocker.failureKind, "required-skill-unavailable");
|
|
assert.deepEqual(((missingSkillBlocker.details as JsonRecord).required as string[]), ["dad-dev"]);
|
|
assert.deepEqual((((missingSkillBlocker.details as JsonRecord).missing as JsonRecord[]).map((item) => item.name)), ["dad-dev"]);
|
|
assertNoSecretLeak(missingSkillEnvelope);
|
|
|
|
const resumed = await createHwlabRun(client, context, bundle, "hwlab-session-resume", "hello resumed", "hwlab-command-session-resumed");
|
|
const resumedRun = await client.get(`/api/v1/runs/${resumed.runId}`) as JsonRecord;
|
|
assert.equal(((resumedRun.sessionRef as JsonRecord).threadId), "thread_selftest_1");
|
|
|
|
const multiTurn = await createHwlabRun(client, context, bundle, "hwlab-session-multiturn", "hello first turn", "hwlab-command-multiturn-1");
|
|
const fakeCodexStartFile = path.join(context.tmp, "fake-codex-starts-multiturn.txt");
|
|
const multiturnRunner = runOnce({ managerUrl: server.baseUrl, runId: multiTurn.runId, commandId: multiTurn.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-multiturn"), AGENTRUN_FAKE_CODEX_START_FILE: fakeCodexStartFile }, idleTimeoutMs: 500, pollIntervalMs: 50 });
|
|
await waitForCommandState(client, multiTurn.runId, multiTurn.commandId, "completed");
|
|
const secondCommand = await client.post(`/api/v1/runs/${multiTurn.runId}/commands`, { type: "turn", payload: { prompt: "hello second turn", traceId: "hwlab-command-multiturn-2" }, idempotencyKey: "hwlab-command-multiturn-2" }) as { id: string };
|
|
await waitForCommandState(client, multiTurn.runId, secondCommand.id, "completed");
|
|
const multiturnResult = await multiturnRunner as JsonRecord;
|
|
assert.equal(multiturnResult.commandsProcessed, 2);
|
|
const starts = (await readTextIfExists(fakeCodexStartFile)).trim().split("\n").filter(Boolean);
|
|
assert.equal(starts.length, 1, "same-run multiturn runner must keep one codex app-server process alive instead of restarting per command");
|
|
const multiEventsResponse = await client.get(`/api/v1/runs/${multiTurn.runId}/events?afterSeq=0&limit=200`) as { items?: Array<{ type?: string; payload?: JsonRecord }> };
|
|
const multiEvents = multiEventsResponse.items ?? [];
|
|
const secondCommandCreatedAt = await commandCreatedAt(client, multiTurn.runId, secondCommand.id);
|
|
const reuseEvent = multiEvents.find((event) => event.type === "backend_status" && event.payload?.phase === "codex-app-server:reused" && event.payload?.commandId === secondCommand.id);
|
|
assert.ok(secondCommandCreatedAt, "second command must expose createdAt for pickup latency assertion");
|
|
assert.ok(reuseEvent?.payload, "second command must produce codex-app-server:reused event");
|
|
assert.ok(Date.parse(String((reuseEvent as JsonRecord).createdAt ?? "")) - Date.parse(secondCommandCreatedAt) < 1_000, "same-run runner should pick up the next command without multi-second polling latency");
|
|
assert.equal(multiEvents.filter((event) => event.type === "backend_status" && event.payload?.phase === "codex-app-server-starting").length, 1);
|
|
assert.equal(multiEvents.filter((event) => event.type === "backend_status" && event.payload?.phase === "codex-app-server:reused").length, 1);
|
|
assert.equal(multiEvents.filter((event) => event.type === "backend_status" && event.payload?.phase === "resource-bundle-materialized").length, 1);
|
|
for (const commandId of [multiTurn.commandId, secondCommand.id]) {
|
|
assert.ok(multiEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "backend-turn-started" && event.payload?.commandId === commandId), `command ${commandId} must emit backend-turn-started before waiting on Codex`);
|
|
assert.ok(multiEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "backend-turn-finished" && event.payload?.commandId === commandId), `command ${commandId} must emit backend-turn-finished after Codex returns`);
|
|
}
|
|
assert.equal(multiEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "backend-turn-running"), false, "backend-turn-running ticks must not be persisted as durable trace events");
|
|
assert.equal(multiEvents.filter((event) => event.type === "backend_status" && event.payload?.phase === "command-terminal").length, 2);
|
|
const secondEnvelope = await client.get(`/api/v1/runs/${multiTurn.runId}/commands/${secondCommand.id}/result`) as JsonRecord;
|
|
assert.equal(secondEnvelope.terminalStatus, "completed");
|
|
assert.equal(secondEnvelope.reply, "fake codex stdio reply");
|
|
|
|
const steerRun = await createHwlabRun(client, context, bundle, "hwlab-session-steer", "start a turn that waits for steer", "hwlab-command-steer-turn", 10_000);
|
|
const steerRunner = runOnce({ managerUrl: server.baseUrl, runId: steerRun.runId, commandId: steerRun.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "steer-waits", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-steer") }, oneShot: true, pollIntervalMs: 50 });
|
|
await waitForCommandState(client, steerRun.runId, steerRun.commandId, "acknowledged");
|
|
const steerCommand = await client.post(`/api/v1/runs/${steerRun.runId}/commands`, { type: "steer", payload: { prompt: "STEER_MARK_SELFTEST", traceId: "hwlab-command-steer" }, idempotencyKey: "hwlab-command-steer" }) as { id: string };
|
|
await waitForCommandState(client, steerRun.runId, steerCommand.id, "completed");
|
|
const steerRunnerResult = await steerRunner as JsonRecord;
|
|
assert.equal(steerRunnerResult.terminalStatus, "completed");
|
|
const steerTurnEnvelope = await client.get(`/api/v1/runs/${steerRun.runId}/commands/${steerRun.commandId}/result`) as JsonRecord;
|
|
assert.equal(steerTurnEnvelope.terminalStatus, "completed");
|
|
assert.match(String(steerTurnEnvelope.reply), /steered:STEER_MARK_SELFTEST/u);
|
|
const steerCommandEnvelope = await client.get(`/api/v1/runs/${steerRun.runId}/commands/${steerCommand.id}/result`) as JsonRecord;
|
|
assert.equal(steerCommandEnvelope.terminalStatus, "completed");
|
|
const steerDelivery = steerCommandEnvelope.steerDelivery as JsonRecord;
|
|
assert.equal(steerDelivery.deliveryState, "forwarded-to-backend");
|
|
assert.equal(steerDelivery.backendAccepted, true);
|
|
assert.equal(steerDelivery.targetCommandId, steerRun.commandId);
|
|
assert.match(String(steerDelivery.semantics), /target command liveness/u);
|
|
const steerEventsResponse = await client.get(`/api/v1/runs/${steerRun.runId}/events?afterSeq=0&limit=200`) as { items?: Array<{ type?: string; payload?: JsonRecord }> };
|
|
const steerEvents = steerEventsResponse.items ?? [];
|
|
assert.ok(steerEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "steer-command-acknowledged" && event.payload?.commandId === steerCommand.id && event.payload?.targetCommandId === steerRun.commandId));
|
|
assert.ok(steerEvents.some((event) => event.type === "backend_status" && event.payload?.phase === "turn/steer:completed" && event.payload?.commandId === steerCommand.id && event.payload?.targetCommandId === steerRun.commandId && event.payload.deliveryState === "forwarded-to-backend" && event.payload.targetEffect === "not-guaranteed"));
|
|
|
|
const noEventWatchdog = await createHwlabRun(client, context, bundle, "hwlab-session-no-event-watchdog", "complete a tool and then produce no more events", "hwlab-command-no-event-watchdog", 10_000);
|
|
const noEventWatchdogRunner = runOnce({ managerUrl: server.baseUrl, runId: noEventWatchdog.runId, commandId: noEventWatchdog.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-completes-without-terminal", AGENTRUN_RUNNER_IDLE_TIMEOUT_MS: "300", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-no-event-watchdog") }, oneShot: true, pollIntervalMs: 50 });
|
|
await waitForCommandState(client, noEventWatchdog.runId, noEventWatchdog.commandId, "acknowledged");
|
|
await waitForEvent(client, noEventWatchdog.runId, (event) => event.type === "tool_call" && (event.payload as JsonRecord).status === "completed", "tool_call completed before no-event watchdog");
|
|
await waitForCommandState(client, noEventWatchdog.runId, noEventWatchdog.commandId, "failed");
|
|
const noEventWatchdogResult = await noEventWatchdogRunner as JsonRecord;
|
|
assert.equal(noEventWatchdogResult.terminalStatus, "failed");
|
|
assert.equal(noEventWatchdogResult.failureKind, "backend-timeout");
|
|
const idleEnvelope = await client.get(`/api/v1/runs/${noEventWatchdog.runId}/commands/${noEventWatchdog.commandId}/result`) as JsonRecord;
|
|
assert.equal(idleEnvelope.terminalStatus, "failed");
|
|
assert.equal(idleEnvelope.failureKind, "backend-timeout");
|
|
assert.match(String(idleEnvelope.failureMessage ?? idleEnvelope.message ?? ""), /idle timed out/u);
|
|
|
|
const outputWithoutTerminal = await createHwlabRun(client, context, bundle, "hwlab-session-output-without-terminal", "start a tool that keeps printing without terminal", "hwlab-command-output-without-terminal", 500);
|
|
const outputWithoutTerminalRunner = runOnce({ managerUrl: server.baseUrl, runId: outputWithoutTerminal.runId, commandId: outputWithoutTerminal.commandId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "tool-output-without-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-output-without-terminal") }, oneShot: true, pollIntervalMs: 50 });
|
|
await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "acknowledged");
|
|
await waitForEvent(client, outputWithoutTerminal.runId, (event) => event.type === "command_output", "command output without terminal");
|
|
await waitForCommandState(client, outputWithoutTerminal.runId, outputWithoutTerminal.commandId, "failed");
|
|
const outputWithoutTerminalResult = await outputWithoutTerminalRunner as JsonRecord;
|
|
assert.equal(outputWithoutTerminalResult.terminalStatus, "failed");
|
|
assert.equal(outputWithoutTerminalResult.failureKind, "backend-timeout");
|
|
const outputWithoutTerminalEnvelope = await client.get(`/api/v1/runs/${outputWithoutTerminal.runId}/commands/${outputWithoutTerminal.commandId}/result`) as JsonRecord;
|
|
assert.equal(outputWithoutTerminalEnvelope.terminalStatus, "failed");
|
|
assert.equal(outputWithoutTerminalEnvelope.failureKind, "backend-timeout");
|
|
assert.match(String(outputWithoutTerminalEnvelope.failureMessage ?? outputWithoutTerminalEnvelope.message ?? ""), /hard timed out/u);
|
|
|
|
const runningCancel = await createHwlabRun(client, context, bundle, "hwlab-session-cancel-running", "cancel running", "hwlab-command-cancel-running", 10_000);
|
|
const running = runOnce({ managerUrl: server.baseUrl, runId: runningCancel.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_FAKE_CODEX_MODE: "missing-terminal", AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "workspaces-running-cancel") }, oneShot: true });
|
|
await waitForCommandState(client, runningCancel.runId, runningCancel.commandId, "acknowledged");
|
|
await client.post(`/api/v1/commands/${runningCancel.commandId}/cancel`, { reason: "self-test running cancel" });
|
|
const runningResult = await running;
|
|
assert.equal(runningResult.terminalStatus, "cancelled");
|
|
|
|
return { name: "hwlab-manual-dispatch", tests: ["runner-job-idempotency", "pending-cancel", "result-envelope", "session-ref-resume", "resource-gitbundle-materialization", "gitbundle-ref-resolution", "gitbundle-tools-path", "gitbundle-skill-dir-assembly", "resource-prompt-required-blocker", "resource-required-skill-blocker", "same-run-runner-multiturn", "running-steer", "no-event-watchdog-after-tool", "tool-output-hard-timeout", "running-cancel"] };
|
|
} finally {
|
|
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
|
}
|
|
};
|
|
|
|
async function createLocalGitBundle(context: SelfTestContext, repoName = "bundle-repo"): Promise<LocalBundle> {
|
|
const repo = path.join(context.tmp, repoName);
|
|
await mkdir(repo, { recursive: true });
|
|
await execFile("git", ["init"], { cwd: repo });
|
|
await writeFile(path.join(repo, "README.md"), "HWLAB bundle self-test\n", "utf8");
|
|
await mkdir(path.join(repo, "tools"), { recursive: true });
|
|
await mkdir(path.join(repo, "tools", "src"), { recursive: true });
|
|
await writeFile(path.join(repo, "tools", "hwpod"), "#!/usr/bin/env sh\nexec bun \"$(dirname \"$0\")/hwpod-cli.ts\" \"$@\"\n", "utf8");
|
|
await writeFile(path.join(repo, "tools", "tran"), "#!/usr/bin/env sh\necho tran-selftest\n", "utf8");
|
|
await writeFile(path.join(repo, "tools", "trans"), "#!/usr/bin/env sh\necho trans-selftest\n", "utf8");
|
|
await writeFile(path.join(repo, "tools", "apply_patch"), "#!/usr/bin/env sh\necho apply-patch-selftest\n", "utf8");
|
|
await writeFile(path.join(repo, "tools", "hwpod-cli.ts"), "import { hwpodSelftestName } from './src/hwpod-harness-lib.ts';\nconsole.log(JSON.stringify({ ok: true, cli: hwpodSelftestName(), argv: process.argv.slice(2) }));\n", "utf8");
|
|
await writeFile(path.join(repo, "tools", "src", "hwpod-harness-lib.ts"), "export function hwpodSelftestName() { return 'hwpod-selftest'; }\n", "utf8");
|
|
await writeFile(path.join(repo, "tools", "hwpod-node.test.ts"), "console.log('test-only source file without shebang');\n", "utf8");
|
|
await mkdir(path.join(repo, "internal", "agent", "prompts"), { recursive: true });
|
|
await writeFile(path.join(repo, "internal", "agent", "prompts", "hwlab-v02-runtime.md"), [
|
|
"HWLAB v0.2 runtime prompt self-test",
|
|
"HWPOD means target device, workspace, debug probe, and io probe.",
|
|
"Use hwpod-cli and hwpod-ctl through the hwpod alias for HWLAB work.",
|
|
"Compile D601-F103-V2 through hwpod-cli -> hwpod-compiler-cli -> /v1/hwpod-node-ops -> hwpod-node.",
|
|
"Do not invent fallback hardware paths or legacy profile routes.",
|
|
].join("\n"), "utf8");
|
|
await mkdir(path.join(repo, "skills", "dad-dev"), { recursive: true });
|
|
await writeFile(path.join(repo, "skills", "dad-dev", "SKILL.md"), [
|
|
"---",
|
|
"name: dad-dev",
|
|
"description: Track dad-dev issue progress and validate delivery evidence for AgentRun work.",
|
|
"---",
|
|
"# dad-dev",
|
|
"Record P1-P4 evidence and remaining risk for AgentRun issue work.",
|
|
].join("\n"), "utf8");
|
|
await mkdir(path.join(repo, "skills", "hwpod-cli", "scripts"), { recursive: true });
|
|
await writeFile(path.join(repo, "skills", "hwpod-cli", "SKILL.md"), [
|
|
"---",
|
|
"name: hwpod-cli",
|
|
"description: Use hwpod for HWLAB HWPOD compile, status, job polling, and output inspection.",
|
|
"---",
|
|
"# hwpod-cli",
|
|
"Run `hwpod` from PATH for HWPOD compile and execution operations.",
|
|
].join("\n"), "utf8");
|
|
await writeFile(path.join(repo, "skills", "hwpod-cli", "scripts", "hwpod-cli.mjs"), "console.log(JSON.stringify({ ok: true, cli: 'hwpod-cli-skill-selftest' }));\n", "utf8");
|
|
await mkdir(path.join(repo, "skills", "hwpod-ctl", "scripts"), { recursive: true });
|
|
await writeFile(path.join(repo, "skills", "hwpod-ctl", "SKILL.md"), [
|
|
"---",
|
|
"name: hwpod-ctl",
|
|
"description: Inspect and control HWLAB HWPOD runtime state, node jobs, probes, and traces.",
|
|
"---",
|
|
"# hwpod-ctl",
|
|
"Use hwpod-ctl for HWPOD runtime inspection and control-plane state.",
|
|
].join("\n"), "utf8");
|
|
await writeFile(path.join(repo, "skills", "hwpod-ctl", "scripts", "hwpod-ctl.mjs"), "console.log(JSON.stringify({ ok: true, cli: 'hwpod-ctl-skill-selftest' }));\n", "utf8");
|
|
await execFile("git", ["add", "README.md", "tools/hwpod", "tools/tran", "tools/trans", "tools/apply_patch", "tools/hwpod-cli.ts", "tools/src/hwpod-harness-lib.ts", "tools/hwpod-node.test.ts", "internal/agent/prompts/hwlab-v02-runtime.md", "skills/dad-dev/SKILL.md", "skills/hwpod-cli/SKILL.md", "skills/hwpod-cli/scripts/hwpod-cli.mjs", "skills/hwpod-ctl/SKILL.md", "skills/hwpod-ctl/scripts/hwpod-ctl.mjs"], { cwd: repo });
|
|
await execFile("git", ["-c", "user.email=selftest@example.invalid", "-c", "user.name=AgentRun SelfTest", "commit", "-m", "bundle selftest"], { cwd: repo });
|
|
const { stdout } = await execFile("git", ["rev-parse", "HEAD"], { cwd: repo });
|
|
return { repoUrl: repo, commitId: stdout.trim(), requiredSkills: [{ name: "dad-dev" }] };
|
|
}
|
|
|
|
async function createRefResolvedLocalGitBundle(context: SelfTestContext): Promise<RefResolvedBundle> {
|
|
const first = await createLocalGitBundle(context, "bundle-ref-repo");
|
|
const { stdout: branchStdout } = await execFile("git", ["branch", "--show-current"], { cwd: first.repoUrl });
|
|
const branch = branchStdout.trim() || "master";
|
|
await writeFile(path.join(first.repoUrl, "README.md"), "HWLAB bundle self-test latest ref\n", "utf8");
|
|
await execFile("git", ["add", "README.md"], { cwd: first.repoUrl });
|
|
await execFile("git", ["-c", "user.email=selftest@example.invalid", "-c", "user.name=AgentRun SelfTest", "commit", "-m", "bundle selftest latest ref"], { cwd: first.repoUrl });
|
|
const { stdout } = await execFile("git", ["rev-parse", "HEAD"], { cwd: first.repoUrl });
|
|
const latestCommitId = stdout.trim();
|
|
assert.notEqual(latestCommitId, first.commitId);
|
|
return { ...first, branch, latestCommitId };
|
|
}
|
|
|
|
async function createHwlabRun(client: ManagerClient, context: SelfTestContext, bundle: LocalBundle, sessionId: string, prompt: string, idempotencyKey: string, timeoutMs = 15_000): Promise<{ runId: string; commandId: string }> {
|
|
const resourceBundleRef: ResourceBundleRef = { kind: "gitbundle", repoUrl: bundle.repoUrl, commitId: bundle.commitId, bundles: bundle.bundles ?? defaultGitBundles(), submodules: false, lfs: false };
|
|
if (bundle.promptRefs) resourceBundleRef.promptRefs = bundle.promptRefs;
|
|
if (bundle.requiredSkills) resourceBundleRef.requiredSkills = bundle.requiredSkills;
|
|
const run = await client.post("/api/v1/runs", {
|
|
tenantId: "hwlab",
|
|
projectId: "pikasTech/HWLAB",
|
|
workspaceRef: { kind: "opaque", repo: "pikasTech/HWLAB", ...(bundle.branch ? { branch: bundle.branch } : {}) },
|
|
sessionRef: { sessionId, conversationId: sessionId },
|
|
resourceBundleRef,
|
|
providerId: "G14",
|
|
backendProfile: "codex",
|
|
executionPolicy: {
|
|
sandbox: "workspace-write",
|
|
approval: "never",
|
|
timeoutMs,
|
|
network: "default",
|
|
secretScope: { allowCredentialEcho: false, providerCredentials: [{ profile: "codex", secretRef: { name: "agentrun-v01-provider-codex", keys: ["auth.json", "config.toml"], mountPath: context.codexHome } }] },
|
|
},
|
|
traceSink: { kind: "hwlab", traceId: idempotencyKey },
|
|
}) as { id: string };
|
|
const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt, traceId: idempotencyKey }, idempotencyKey }) as { id: string };
|
|
return { runId: run.id, commandId: command.id };
|
|
}
|
|
|
|
function defaultGitBundles(): ResourceBundleRef["bundles"] {
|
|
return [
|
|
{ name: "hwlab-tools", subpath: "tools", targetPath: "tools" },
|
|
{ name: "hwlab-agent-skills", subpath: "skills", targetPath: ".agents/skills" },
|
|
];
|
|
}
|
|
|
|
async function waitForCommandState(client: ManagerClient, runId: string, commandId: string, state: string): Promise<void> {
|
|
const deadline = Date.now() + 5_000;
|
|
while (Date.now() < deadline) {
|
|
const command = await client.get(`/api/v1/runs/${runId}/commands/${commandId}`) as { state?: string };
|
|
if (command.state === state) return;
|
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
}
|
|
throw new Error(`command ${commandId} did not reach ${state}`);
|
|
}
|
|
|
|
async function waitForEvent(client: ManagerClient, runId: string, predicate: (event: { type?: string; payload?: JsonRecord }) => boolean, label: string): Promise<void> {
|
|
const deadline = Date.now() + 5_000;
|
|
while (Date.now() < deadline) {
|
|
const events = await client.get(`/api/v1/runs/${runId}/events?afterSeq=0&limit=200`) as { items?: Array<{ type?: string; payload?: JsonRecord }> };
|
|
if ((events.items ?? []).some(predicate)) return;
|
|
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
}
|
|
throw new Error(`timed out waiting for ${label}`);
|
|
}
|
|
|
|
async function commandCreatedAt(client: ManagerClient, runId: string, commandId: string): Promise<string> {
|
|
const command = await client.get(`/api/v1/runs/${runId}/commands/${commandId}`) as { createdAt?: string };
|
|
return command.createdAt ?? "";
|
|
}
|
|
|
|
async function readTextIfExists(filePath: string): Promise<string> {
|
|
try {
|
|
return await readFile(filePath, "utf8");
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
function turnInputText(record: JsonRecord | undefined): string {
|
|
const input = record?.input;
|
|
if (!Array.isArray(input)) return "";
|
|
return input.flatMap((item) => {
|
|
if (typeof item !== "object" || item === null || Array.isArray(item)) return [];
|
|
const text = (item as JsonRecord).text;
|
|
return typeof text === "string" ? [text] : [];
|
|
}).join("\n");
|
|
}
|
|
|
|
function runnerEnvValue(manifest: JsonRecord, name: string): unknown {
|
|
const spec = manifest.spec as JsonRecord;
|
|
const template = spec.template as JsonRecord;
|
|
const podSpec = template.spec as JsonRecord;
|
|
const containers = podSpec.containers as JsonRecord[];
|
|
const env = containers[0]?.env as JsonRecord[];
|
|
return env.find((entry) => entry.name === name)?.value;
|
|
}
|
|
|
|
export default selfTest;
|