Codex
364 lines
16 KiB
TypeScript
364 lines
16 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { rmSync } from "node:fs";
|
|
import { describe, expect, test } from "bun:test";
|
|
|
|
import { runAgentRunCommand } from "./agentrun";
|
|
|
|
const FIXTURE_RUN_ID = "run_event_summary_fixture";
|
|
const HIDDEN_OUTPUT_MARKER = "HIDDEN_TOOL_OUTPUT_MUST_NOT_REACH_DEFAULT_RENDER";
|
|
const HIDDEN_STDERR_MARKER = "HIDDEN_STDERR_MUST_NOT_REACH_DEFAULT_RENDER";
|
|
const HIDDEN_SECOND_COMMAND_LINE = "HIDDEN_SECOND_COMMAND_LINE";
|
|
const HIDDEN_CREDENTIAL_MARKER = "HIDDEN_EVENT_CREDENTIAL_MUST_NOT_REACH_DETAIL";
|
|
const HIDDEN_URL_CREDENTIAL = "HIDDEN_URL_CREDENTIAL_MUST_NOT_REACH_DETAIL";
|
|
const SCP_LIKE_REMOTE = `${HIDDEN_URL_CREDENTIAL}@mirror.example.invalid:pikasTech/unidesk.git?token=${HIDDEN_CREDENTIAL_MARKER}`;
|
|
const SCP_LIKE_REMOTE_SHA256 = createHash("sha256").update(SCP_LIKE_REMOTE).digest("hex");
|
|
const LARGE_TOOL_OUTPUT = `${HIDDEN_OUTPUT_MARKER}\n${"skill body and command output\n".repeat(2_400)}`;
|
|
const LARGE_OUTPUT_BYTES = Buffer.byteLength(LARGE_TOOL_OUTPUT, "utf8");
|
|
const LARGE_OUTPUT_SHA256 = `sha256:${createHash("sha256").update(LARGE_TOOL_OUTPUT).digest("hex")}`;
|
|
|
|
const agentRunClientYaml = [
|
|
"version: 1",
|
|
"kind: AgentRunConfig",
|
|
"metadata:",
|
|
" name: agentrun",
|
|
"manager:",
|
|
" baseUrl: __BASE_URL__",
|
|
" timeoutMs: 15000",
|
|
"auth:",
|
|
" env: HWLAB_API_KEY",
|
|
" file: /tmp/hwlab-api-key.env",
|
|
" header: Authorization",
|
|
" scheme: Bearer",
|
|
"client:",
|
|
" role: render-only",
|
|
" transport: direct-http",
|
|
" sessionPolicy:",
|
|
" tenantId: unidesk",
|
|
" projectId: test",
|
|
" providerId: NC01",
|
|
" backendProfile: codex",
|
|
" workspaceRef:",
|
|
" kind: opaque",
|
|
" executionPolicy:",
|
|
" sandbox: workspace-write",
|
|
" approval: never",
|
|
" timeoutMs: 900000",
|
|
" network: enabled",
|
|
" secretScope:",
|
|
" allowCredentialEcho: false",
|
|
].join("\n");
|
|
|
|
function fixtureEvents(): Array<Record<string, unknown>> {
|
|
const items: Array<Record<string, unknown>> = [];
|
|
for (let seq = 22; seq <= 47; seq += 1) {
|
|
if (seq <= 41) {
|
|
items.push({
|
|
id: `evt_${seq}`,
|
|
seq,
|
|
type: "tool_call",
|
|
payload: {
|
|
commandId: "cmd_fixture",
|
|
itemId: `tool_${seq}`,
|
|
toolName: "commandExecution",
|
|
type: "commandExecution",
|
|
method: "item/completed",
|
|
status: "completed",
|
|
command: `find . -maxdepth 3 -name package.json\n${HIDDEN_SECOND_COMMAND_LINE}`,
|
|
exitCode: 0,
|
|
durationMs: 17,
|
|
outputSummary: LARGE_TOOL_OUTPUT,
|
|
stderr: HIDDEN_STDERR_MARKER,
|
|
outputBytes: LARGE_OUTPUT_BYTES,
|
|
outputTruncated: true,
|
|
summary: {
|
|
text: `commandExecution completed output=${LARGE_TOOL_OUTPUT}`,
|
|
textBytes: LARGE_OUTPUT_BYTES,
|
|
textTruncated: true,
|
|
},
|
|
},
|
|
});
|
|
continue;
|
|
}
|
|
if (seq === 42) {
|
|
items.push({
|
|
id: `evt_${seq}`,
|
|
seq,
|
|
type: "backend_status",
|
|
payload: {
|
|
commandId: "cmd_fixture",
|
|
phase: "resource-bundle-materialized",
|
|
kind: "gitbundle",
|
|
repoUrl: `https://fixture:${HIDDEN_URL_CREDENTIAL}@example.invalid/pikasTech/unidesk.git?access_token=${HIDDEN_CREDENTIAL_MARKER}`,
|
|
fetchRepoUrl: SCP_LIKE_REMOTE,
|
|
mirrorUsed: true,
|
|
commitId: "1".repeat(40),
|
|
requestedCommitId: null,
|
|
requestedRef: "refs/heads/master",
|
|
treeId: "2".repeat(40),
|
|
workspaceLifecycle: "first-materialization",
|
|
resolutionAuthority: "REDACTED",
|
|
primaryWorkspace: {
|
|
status: "materialized",
|
|
sourceContract: "resourceBundleRef",
|
|
repoUrl: `https://fixture:${HIDDEN_URL_CREDENTIAL}@example.invalid/pikasTech/unidesk.git?access_token=${HIDDEN_CREDENTIAL_MARKER}`,
|
|
fetchRepoUrl: SCP_LIKE_REMOTE,
|
|
mirrorUsed: true,
|
|
requestedRef: "refs/heads/master",
|
|
requestedCommitId: null,
|
|
sourceCommit: "1".repeat(40),
|
|
headCommit: "3".repeat(40),
|
|
headIsFrozenSource: false,
|
|
treeId: "2".repeat(40),
|
|
headTreeId: "4".repeat(40),
|
|
sourceBranch: "master",
|
|
branch: "fix/cadence",
|
|
targetPath: ".",
|
|
workspaceLifecycle: "continuation-reuse",
|
|
resolutionAuthority: "REDACTED",
|
|
assemblyContractHash: "sha256:assembly-fixture",
|
|
bundleContractHash: "sha256:bundle-contract-fixture",
|
|
bundleProvenanceHash: "sha256:bundle-provenance-fixture",
|
|
provenanceHash: "sha256:workspace-provenance-fixture",
|
|
writable: true,
|
|
credential: HIDDEN_CREDENTIAL_MARKER,
|
|
valuesPrinted: false,
|
|
},
|
|
bundles: {
|
|
count: 6,
|
|
items: Array.from({ length: 6 }, (_, index) => ({ name: `bundle-${index}`, credential: HIDDEN_CREDENTIAL_MARKER })),
|
|
valuesPrinted: false,
|
|
},
|
|
tools: { count: 4, items: [{ name: "git", output: LARGE_TOOL_OUTPUT }], valuesPrinted: false },
|
|
skillDirs: { count: 2, items: [{ path: ".agents/skills", body: LARGE_TOOL_OUTPUT }], valuesPrinted: false },
|
|
requiredSkills: { count: 11, items: [{ name: "unidesk-gh", body: LARGE_TOOL_OUTPUT }], valuesPrinted: false },
|
|
promptRefs: { count: 0, items: [], valuesPrinted: false },
|
|
initialPrompt: {
|
|
available: true,
|
|
bytes: LARGE_OUTPUT_BYTES,
|
|
sha256: LARGE_OUTPUT_SHA256,
|
|
promptRefCount: 0,
|
|
skillCount: 11,
|
|
text: LARGE_TOOL_OUTPUT,
|
|
valuesPrinted: false,
|
|
},
|
|
credentialRef: { name: "fixture-secret", value: HIDDEN_CREDENTIAL_MARKER },
|
|
valuesPrinted: false,
|
|
},
|
|
});
|
|
continue;
|
|
}
|
|
items.push({
|
|
id: `evt_${seq}`,
|
|
seq,
|
|
type: "backend_status",
|
|
payload: { commandId: "cmd_fixture", phase: `phase-${seq}`, message: "bounded status" },
|
|
});
|
|
}
|
|
return items;
|
|
}
|
|
|
|
function renderedText(value: Awaited<ReturnType<typeof runAgentRunCommand>>): string {
|
|
if (!("renderedText" in value) || typeof value.renderedText !== "string") throw new Error("expected rendered AgentRun CLI output");
|
|
return value.renderedText;
|
|
}
|
|
|
|
describe("AgentRun events render-only progressive disclosure", () => {
|
|
test("default human/json/yaml stay bounded while full/raw retain the exact event page", async () => {
|
|
const events = fixtureEvents();
|
|
const observedLimits: number[] = [];
|
|
const server = Bun.serve({
|
|
port: 0,
|
|
fetch(request) {
|
|
const url = new URL(request.url);
|
|
expect(request.headers.get("authorization")).toBe("Bearer fixture-secret");
|
|
expect(url.pathname).toBe(`/api/v1/runs/${FIXTURE_RUN_ID}/events`);
|
|
const afterSeq = Number(url.searchParams.get("afterSeq") ?? "0");
|
|
const limit = Number(url.searchParams.get("limit") ?? "100");
|
|
observedLimits.push(limit);
|
|
return Response.json({
|
|
ok: true,
|
|
data: {
|
|
items: events.filter((item) => Number(item.seq) > afterSeq).slice(0, limit),
|
|
},
|
|
});
|
|
},
|
|
});
|
|
const previousConfig = process.env.AGENTRUN_CLIENT_CONFIG;
|
|
const previousKey = process.env.HWLAB_API_KEY;
|
|
const configPath = `/tmp/unidesk-agentrun-events-${process.pid}-${Date.now()}.yaml`;
|
|
process.env.AGENTRUN_CLIENT_CONFIG = configPath;
|
|
process.env.HWLAB_API_KEY = "fixture-secret";
|
|
await Bun.write(configPath, agentRunClientYaml.replace("__BASE_URL__", server.url.href.replace(/\/$/u, "")));
|
|
try {
|
|
const args = ["events", `run/${FIXTURE_RUN_ID}`, "--after-seq", "21", "--limit", "100"];
|
|
const jsonResult = await runAgentRunCommand(null, [...args, "-o", "json"]);
|
|
const jsonText = renderedText(jsonResult);
|
|
const json = JSON.parse(jsonText) as {
|
|
kind: string;
|
|
count: number;
|
|
requestedLimit: number;
|
|
effectiveLimit: number;
|
|
fetchedCount: number;
|
|
hasMore: boolean;
|
|
nextAfterSeq: number;
|
|
items: Array<Record<string, unknown>>;
|
|
};
|
|
expect(json.kind).toBe("EventList");
|
|
expect(json.count).toBe(20);
|
|
expect(json.requestedLimit).toBe(100);
|
|
expect(json.effectiveLimit).toBe(20);
|
|
expect(json.fetchedCount).toBe(21);
|
|
expect(json.hasMore).toBe(true);
|
|
expect(json.nextAfterSeq).toBe(41);
|
|
expect(Buffer.byteLength(jsonText, "utf8")).toBeLessThan(10_240);
|
|
expect(jsonText).not.toContain(HIDDEN_OUTPUT_MARKER);
|
|
expect(jsonText).not.toContain(HIDDEN_STDERR_MARKER);
|
|
expect(jsonText).not.toContain(HIDDEN_SECOND_COMMAND_LINE);
|
|
|
|
const completed = json.items.find((item) => item.seq === 24);
|
|
expect(completed).toMatchObject({
|
|
identity: "event/evt_24",
|
|
type: "tool_call",
|
|
status: "completed",
|
|
commandId: "cmd_fixture",
|
|
tool: "commandExecution",
|
|
command: "find . -maxdepth 3 -name package.json",
|
|
exitCode: 0,
|
|
durationMs: 17,
|
|
outputBytes: LARGE_OUTPUT_BYTES,
|
|
outputSha256: LARGE_OUTPUT_SHA256,
|
|
outputTruncated: true,
|
|
outputOmitted: true,
|
|
});
|
|
|
|
const humanText = renderedText(await runAgentRunCommand(null, args));
|
|
expect(Buffer.byteLength(humanText, "utf8")).toBeLessThan(10_240);
|
|
expect(humanText).toContain("commandExecution find . -maxdepth 3 -name package.json");
|
|
expect(humanText).toContain(`${LARGE_OUTPUT_BYTES}B`);
|
|
expect(humanText).toContain("truncated");
|
|
expect(humanText).toContain("Detail: bun scripts/cli.ts agentrun events");
|
|
expect(humanText).not.toContain(HIDDEN_OUTPUT_MARKER);
|
|
expect(humanText).not.toContain(HIDDEN_STDERR_MARKER);
|
|
expect(humanText).not.toContain(HIDDEN_SECOND_COMMAND_LINE);
|
|
|
|
const yamlText = renderedText(await runAgentRunCommand(null, [...args, "-o", "yaml"]));
|
|
const yaml = Bun.YAML.parse(yamlText) as { kind?: string; items?: Array<Record<string, unknown>> };
|
|
expect(yaml.kind).toBe("EventList");
|
|
expect(yaml.items?.find((item) => item.seq === 24)?.outputSha256).toBe(LARGE_OUTPUT_SHA256);
|
|
expect(Buffer.byteLength(yamlText, "utf8")).toBeLessThan(10_240);
|
|
expect(yamlText).not.toContain(HIDDEN_OUTPUT_MARKER);
|
|
|
|
for (const disclosure of ["--full", "--raw"]) {
|
|
const fullText = renderedText(await runAgentRunCommand(null, [...args, disclosure]));
|
|
const full = JSON.parse(fullText) as { data?: { items?: Array<{ payload?: { outputSummary?: string; stderr?: string } }> } };
|
|
expect(full.data?.items?.length).toBe(26);
|
|
expect(full.data?.items?.[2]?.payload?.outputSummary).toBe(LARGE_TOOL_OUTPUT);
|
|
expect(full.data?.items?.[2]?.payload?.stderr).toBe(HIDDEN_STDERR_MARKER);
|
|
expect(fullText).toContain(HIDDEN_OUTPUT_MARKER);
|
|
expect(fullText).toContain(HIDDEN_SECOND_COMMAND_LINE);
|
|
}
|
|
|
|
const detailArgs = ["events", `run/${FIXTURE_RUN_ID}`, "--detail-seq", "42"];
|
|
const detailJsonText = renderedText(await runAgentRunCommand(null, [...detailArgs, "-o", "json"]));
|
|
const detailJson = JSON.parse(detailJsonText) as {
|
|
kind?: string;
|
|
source?: string;
|
|
seq?: number;
|
|
phase?: string;
|
|
correlation?: { runId?: string; commandId?: string };
|
|
evidence?: {
|
|
primaryWorkspace?: Record<string, unknown>;
|
|
counts?: Record<string, unknown>;
|
|
initialPrompt?: Record<string, unknown>;
|
|
};
|
|
redaction?: Record<string, unknown>;
|
|
valuesPrinted?: boolean;
|
|
};
|
|
expect(detailJson.kind).toBe("EventDetail");
|
|
expect(detailJson.source).toBe(`run/${FIXTURE_RUN_ID}`);
|
|
expect(detailJson.seq).toBe(42);
|
|
expect(detailJson.phase).toBe("resource-bundle-materialized");
|
|
expect(detailJson.correlation).toMatchObject({ runId: FIXTURE_RUN_ID, commandId: "cmd_fixture" });
|
|
expect(detailJson.evidence?.primaryWorkspace).toMatchObject({
|
|
sourceContract: "resourceBundleRef",
|
|
repoUrl: "https://example.invalid/pikasTech/unidesk.git",
|
|
fetchRepoUrl: `<scp-like-remote;path-omitted;sha256:${SCP_LIKE_REMOTE_SHA256}>`,
|
|
sourceCommit: "1".repeat(40),
|
|
headCommit: "3".repeat(40),
|
|
workspaceLifecycle: "continuation-reuse",
|
|
resolutionAuthority: null,
|
|
resolutionAuthoritySource: "durable-event",
|
|
resolutionAuthorityRedacted: true,
|
|
resolutionModeInference: {
|
|
mode: "runner-first-materialization",
|
|
source: "durable-event requestedCommitId",
|
|
authoritative: false,
|
|
valuesPrinted: false,
|
|
},
|
|
writable: true,
|
|
assemblyContractHash: "sha256:assembly-fixture",
|
|
bundleContractHash: "sha256:bundle-contract-fixture",
|
|
bundleProvenanceHash: "sha256:bundle-provenance-fixture",
|
|
provenanceHash: "sha256:workspace-provenance-fixture",
|
|
valuesPrinted: false,
|
|
});
|
|
expect(detailJson.evidence?.counts).toEqual({ bundles: 6, tools: 4, skillDirs: 2, requiredSkills: 11, promptRefs: 0 });
|
|
expect(detailJson.evidence?.initialPrompt).toMatchObject({
|
|
available: true,
|
|
bytes: LARGE_OUTPUT_BYTES,
|
|
sha256: LARGE_OUTPUT_SHA256,
|
|
promptRefCount: 0,
|
|
skillCount: 11,
|
|
contentOmitted: true,
|
|
valuesPrinted: false,
|
|
});
|
|
expect(detailJson.redaction).toEqual({ credentialValues: "omitted", urlCredentials: "removed", secretRefsOnly: true, valuesPrinted: false });
|
|
expect(detailJson.valuesPrinted).toBe(false);
|
|
expect(Buffer.byteLength(detailJsonText, "utf8")).toBeLessThan(10_240);
|
|
|
|
const detailYamlText = renderedText(await runAgentRunCommand(null, [...detailArgs, "-o", "yaml"]));
|
|
const detailYaml = Bun.YAML.parse(detailYamlText) as typeof detailJson;
|
|
expect(detailYaml.evidence).toEqual(detailJson.evidence);
|
|
const detailHumanText = renderedText(await runAgentRunCommand(null, detailArgs));
|
|
expect(detailHumanText).toContain("workspaceLifecycle");
|
|
expect(detailHumanText).toContain("sha256:workspace-provenance-fixture");
|
|
expect(detailHumanText).toContain("--after-seq 41 --limit 1 --full -o json");
|
|
|
|
for (const boundedText of [detailJsonText, detailYamlText, detailHumanText]) {
|
|
expect(Buffer.byteLength(boundedText, "utf8")).toBeLessThan(10_240);
|
|
expect(boundedText).not.toContain(HIDDEN_OUTPUT_MARKER);
|
|
expect(boundedText).not.toContain(HIDDEN_CREDENTIAL_MARKER);
|
|
expect(boundedText).not.toContain(HIDDEN_URL_CREDENTIAL);
|
|
expect(boundedText).not.toContain("access_token");
|
|
}
|
|
|
|
const requestsBeforeInvalid = observedLimits.length;
|
|
const invalid = await runAgentRunCommand(null, [...detailArgs, "--full", "-o", "json"]);
|
|
expect(invalid.ok).toBe(false);
|
|
expect(renderedText(invalid)).toContain("validation-failed");
|
|
expect(renderedText(invalid)).toContain("bounded EventDetail and complete event disclosure are separate read modes");
|
|
expect(observedLimits.length).toBe(requestsBeforeInvalid);
|
|
|
|
const wrongVerb = await runAgentRunCommand(null, ["describe", `run/${FIXTURE_RUN_ID}`, "--detail-seq", "42", "-o", "json"]);
|
|
expect(wrongVerb.ok).toBe(false);
|
|
expect(renderedText(wrongVerb)).toContain("--detail-seq is supported only by agentrun events");
|
|
expect(observedLimits.length).toBe(requestsBeforeInvalid);
|
|
|
|
const missing = await runAgentRunCommand(null, ["events", `run/${FIXTURE_RUN_ID}`, "--detail-seq", "48", "-o", "json"]);
|
|
expect(missing.ok).toBe(false);
|
|
expect(renderedText(missing)).toContain("not-found");
|
|
expect(renderedText(missing)).toContain("no durable event at seq 48");
|
|
|
|
expect(observedLimits.slice(0, 5)).toEqual([21, 21, 21, 100, 100]);
|
|
expect(observedLimits.slice(5)).toEqual([1, 1, 1, 1]);
|
|
} finally {
|
|
server.stop(true);
|
|
if (previousConfig === undefined) delete process.env.AGENTRUN_CLIENT_CONFIG;
|
|
else process.env.AGENTRUN_CLIENT_CONFIG = previousConfig;
|
|
if (previousKey === undefined) delete process.env.HWLAB_API_KEY;
|
|
else process.env.HWLAB_API_KEY = previousKey;
|
|
rmSync(configPath, { force: true });
|
|
}
|
|
});
|
|
});
|