feat: 补齐 HWLAB 基线 AgentRun 执行元语
This commit is contained in:
Codex
@@ -58,14 +58,17 @@ const selfTest: SelfTestCase = async (context) => {
|
||||
assert.equal(explicitModelResult.terminalStatus, "completed", "explicit command payload model should still be forwarded");
|
||||
|
||||
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "missing-turn-result", expectedStatus: "failed", expectedFailureKind: "backend-response-invalid" });
|
||||
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-401-rpc-error", expectedStatus: "failed", expectedFailureKind: "provider-auth-failed" });
|
||||
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-429-terminal", expectedStatus: "failed", expectedFailureKind: "provider-rate-limited" });
|
||||
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-503-rpc-error", expectedStatus: "failed", expectedFailureKind: "provider-unavailable" });
|
||||
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-503-terminal", expectedStatus: "failed", expectedFailureKind: "provider-unavailable" });
|
||||
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "provider-503-retry-event", expectedStatus: "failed", expectedFailureKind: "provider-unavailable", expectRetryError: true });
|
||||
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "invalid-json", expectedStatus: "failed", expectedFailureKind: "backend-json-parse-error" });
|
||||
await runFailureCase({ client, managerUrl: server.baseUrl, context, mode: "missing-terminal", expectedStatus: "failed", expectedFailureKind: "backend-timeout", timeoutMs: 500 });
|
||||
await runSecretFailureCase({ client, managerUrl: server.baseUrl, context });
|
||||
await runSpawnFailureCase({ client, managerUrl: server.baseUrl, context });
|
||||
|
||||
return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "codex-stdio-fake-turn", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-missing-turn-result", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-503-retry-event", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-spawn-failure"] };
|
||||
return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "codex-stdio-fake-turn", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-503-retry-event", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
||||
}
|
||||
@@ -101,6 +104,23 @@ function eventPayload(event: { payload: unknown }): JsonRecord {
|
||||
return typeof event.payload === "object" && event.payload !== null && !Array.isArray(event.payload) ? event.payload as JsonRecord : {};
|
||||
}
|
||||
|
||||
async function runSecretFailureCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
||||
const item = await createRunWithCommand(options.client, options.context, "failure missing secret files", "selftest-secret-unavailable", 3_000);
|
||||
const result = await runOnce({
|
||||
managerUrl: options.managerUrl,
|
||||
runId: item.runId,
|
||||
codexCommand: options.context.fakeCodexCommand,
|
||||
codexArgs: options.context.fakeCodexArgs,
|
||||
codexHome: path.join(options.context.tmp, "missing-codex-home"),
|
||||
env: { CODEX_HOME: path.join(options.context.tmp, "missing-codex-home") },
|
||||
}) as JsonRecord;
|
||||
assert.equal(result.terminalStatus, "blocked", "secret unavailable");
|
||||
assert.equal(result.failureKind, "secret-unavailable", "secret unavailable");
|
||||
const run = await options.client.get(`/api/v1/runs/${item.runId}`) as { status?: string; failureKind?: string };
|
||||
assert.equal(run.status, "blocked", "secret unavailable");
|
||||
assert.equal(run.failureKind, "secret-unavailable", "secret unavailable");
|
||||
}
|
||||
|
||||
async function runSpawnFailureCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
|
||||
const item = await createRunWithCommand(options.client, options.context, "failure spawn", "selftest-spawn-failure", 3_000);
|
||||
const result = await runOnce({
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile as execFileCallback } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { startManagerServer } from "../../mgr/server.js";
|
||||
import { MemoryAgentRunStore } from "../../mgr/store.js";
|
||||
import { ManagerClient } from "../../mgr/client.js";
|
||||
import { runOnce } from "../../runner/run-once.js";
|
||||
import { eventContractSummary } from "../../common/events.js";
|
||||
import type { BackendProfile, JsonRecord, RunEvent } from "../../common/types.js";
|
||||
import { assertNoSecretLeak, createRunWithCommand, type SelfTestCase, type SelfTestContext } from "../harness.js";
|
||||
|
||||
const execFile = promisify(execFileCallback);
|
||||
|
||||
const selfTest: SelfTestCase = async (context) => {
|
||||
const fakeKubectl = path.join(context.tmp, "fake-kubectl-contract.js");
|
||||
const createdManifest = path.join(context.tmp, "created-contract-runner-job.json");
|
||||
await writeFile(fakeKubectl, `#!/usr/bin/env bun
|
||||
const chunks = [];
|
||||
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
|
||||
const text = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
|
||||
await Bun.write(${JSON.stringify(createdManifest)}, text);
|
||||
const manifest = JSON.parse(text);
|
||||
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "job-uid-contract", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
|
||||
`);
|
||||
await chmod(fakeKubectl, 0o755);
|
||||
const store = new MemoryAgentRunStore();
|
||||
const server = await startManagerServer({
|
||||
port: 0,
|
||||
host: "127.0.0.1",
|
||||
sourceCommit: "self-test",
|
||||
store,
|
||||
runnerJobDefaults: {
|
||||
namespace: "agentrun-v01",
|
||||
managerUrl: "http://agentrun-mgr.agentrun-v01.svc.cluster.local:8080",
|
||||
image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
||||
kubectlCommand: fakeKubectl,
|
||||
},
|
||||
});
|
||||
try {
|
||||
const client = new ManagerClient(server.baseUrl);
|
||||
await assertBackendPreflight(client);
|
||||
await assertEventContractAndCompletedSemantics(client, context, server.baseUrl);
|
||||
await assertRunnerJobStatus(client, context);
|
||||
await assertSessionProfileIsolation(client, context);
|
||||
await assertResourceBundleFailure(client, context, server.baseUrl);
|
||||
return { name: "hwlab-baseline-contract", tests: ["event-contract", "result-completed-terminal-only", "bounded-output-summary", "runner-job-status", "backend-preflight-redacted", "session-profile-isolation", "resource-bundle-failure-kind"] };
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
||||
}
|
||||
};
|
||||
|
||||
async function assertBackendPreflight(client: ManagerClient): Promise<void> {
|
||||
const response = await client.get("/api/v1/backends") as { items?: JsonRecord[] };
|
||||
const items = response.items ?? [];
|
||||
assert.ok(items.length >= 2, "codex/deepseek backend capabilities should be visible");
|
||||
for (const item of items) {
|
||||
const preflight = item.preflight as JsonRecord;
|
||||
const defaultSecretRef = item.defaultSecretRef as JsonRecord;
|
||||
assert.equal(preflight.valuesPrinted, false);
|
||||
assert.equal(preflight.credentialValuesPrinted, false);
|
||||
assert.equal(defaultSecretRef.valuesPrinted, false);
|
||||
assert.ok(Array.isArray(preflight.requiredSecretKeys));
|
||||
}
|
||||
assertNoSecretLeak(response);
|
||||
}
|
||||
|
||||
async function assertEventContractAndCompletedSemantics(client: ManagerClient, context: SelfTestContext, managerUrl: string): Promise<void> {
|
||||
const happy = await createRunWithCommand(client, context, "hello event contract", "selftest-event-contract", 15_000);
|
||||
await client.post(`/api/v1/runs/${happy.runId}/events`, { type: "tool_call", payload: { method: "selftest/tool", item: { command: "echo ok" } } });
|
||||
await client.post(`/api/v1/runs/${happy.runId}/events`, { type: "diff", payload: { filesChanged: 1, summary: "selftest diff" } });
|
||||
const result = await runOnce({ managerUrl, runId: happy.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome } });
|
||||
assert.equal(result.terminalStatus, "completed");
|
||||
const eventsResponse = await client.get(`/api/v1/runs/${happy.runId}/events?afterSeq=0&limit=100`) as { items?: RunEvent[] };
|
||||
const events = eventsResponse.items ?? [];
|
||||
const summary = eventContractSummary(events);
|
||||
assert.equal(summary.ok, true, JSON.stringify(summary.issues));
|
||||
const types = new Set(events.map((event) => event.type));
|
||||
for (const type of ["backend_status", "assistant_message", "tool_call", "command_output", "diff", "terminal_status"]) assert.ok(types.has(type as never), `missing event type ${type}`);
|
||||
const envelope = await client.get(`/api/v1/runs/${happy.runId}/commands/${happy.commandId}/result`) as JsonRecord;
|
||||
assert.equal(envelope.completed, true);
|
||||
assert.equal(envelope.terminalStatus, "completed");
|
||||
assert.equal(envelope.terminalSource, "terminal_status-event");
|
||||
assertNoSecretLeak({ eventsResponse, envelope });
|
||||
|
||||
const partial = await createRunWithCommand(client, context, "partial should not complete", "selftest-partial-not-completed", 15_000);
|
||||
const largeOutput = `${"x".repeat(6_000)}\nAuthorization: Bearer test-token\n`;
|
||||
await client.post(`/api/v1/runs/${partial.runId}/events`, { type: "assistant_message", payload: { text: "partial assistant only" } });
|
||||
await client.post(`/api/v1/runs/${partial.runId}/events`, { type: "command_output", payload: { stream: "stdout", text: largeOutput } });
|
||||
const partialEnvelope = await client.get(`/api/v1/runs/${partial.runId}/commands/${partial.commandId}/result`) as JsonRecord;
|
||||
const artifactSummary = partialEnvelope.artifactSummary as JsonRecord;
|
||||
const stdoutSummary = artifactSummary.stdoutSummary as JsonRecord;
|
||||
assert.equal(partialEnvelope.completed, false);
|
||||
assert.equal(partialEnvelope.terminalStatus, null);
|
||||
assert.equal(stdoutSummary.outputTruncated, true);
|
||||
assert.equal(typeof stdoutSummary.outputBytes, "number");
|
||||
assertNoSecretLeak(partialEnvelope);
|
||||
}
|
||||
|
||||
async function assertRunnerJobStatus(client: ManagerClient, context: SelfTestContext): Promise<void> {
|
||||
const item = await createRunWithCommand(client, context, "runner job status", "selftest-runner-job-status", 15_000);
|
||||
await client.post(`/api/v1/runs/${item.runId}/runner-jobs`, { commandId: item.commandId, idempotencyKey: "selftest-runner-job-status" }) as JsonRecord;
|
||||
const manifest = JSON.parse(await readFile(path.join(context.tmp, "created-contract-runner-job.json"), "utf8")) as JsonRecord;
|
||||
assert.equal(manifest.kind, "Job");
|
||||
const list = await client.get(`/api/v1/runs/${item.runId}/runner-jobs`) as { items?: JsonRecord[]; count?: number };
|
||||
assert.equal(list.count, 1);
|
||||
const status = list.items?.[0] ?? {};
|
||||
assert.equal(status.runId, item.runId);
|
||||
assert.equal(status.commandId, item.commandId);
|
||||
assert.equal(status.phase, "created");
|
||||
assert.equal(status.valuesPrinted, false);
|
||||
assert.equal(typeof status.logPath, "string");
|
||||
const single = await client.get(`/api/v1/runs/${item.runId}/runner-jobs/${String(status.id)}`) as JsonRecord;
|
||||
assert.equal(single.jobName, status.jobName);
|
||||
assertNoSecretLeak({ list, single });
|
||||
}
|
||||
|
||||
async function assertSessionProfileIsolation(client: ManagerClient, context: SelfTestContext): Promise<void> {
|
||||
const first = await client.post("/api/v1/runs", runPayload(context, "codex", "selftest-profile-boundary-session")) as { id: string };
|
||||
await client.patch(`/api/v1/runs/${first.id}/status`, { terminalStatus: "completed", failureKind: null, failureMessage: null, threadId: "thread_codex_profile_boundary", turnId: "turn_profile_boundary" });
|
||||
await assert.rejects(
|
||||
() => client.post("/api/v1/runs", runPayload(context, "deepseek", "selftest-profile-boundary-session")),
|
||||
(error) => error instanceof Error && error.message.includes("backendProfile boundary"),
|
||||
);
|
||||
}
|
||||
|
||||
async function assertResourceBundleFailure(client: ManagerClient, context: SelfTestContext, managerUrl: string): Promise<void> {
|
||||
const repo = await createLocalGitRepo(context);
|
||||
const run = await client.post("/api/v1/runs", {
|
||||
...runPayload(context, "codex", "selftest-bad-bundle-session"),
|
||||
resourceBundleRef: { kind: "git", repoUrl: repo.repoUrl, commitId: "0000000000000000000000000000000000000000", submodules: false, lfs: false },
|
||||
}) as { id: string };
|
||||
const command = await client.post(`/api/v1/runs/${run.id}/commands`, { type: "turn", payload: { prompt: "bad bundle" }, idempotencyKey: "selftest-bad-bundle" }) as { id: string };
|
||||
const result = await runOnce({ managerUrl, runId: run.id, commandId: command.id, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: context.codexHome, env: { CODEX_HOME: context.codexHome, AGENTRUN_WORKSPACE_ROOT: path.join(context.tmp, "bad-bundle-workspaces") } }) as JsonRecord;
|
||||
assert.equal(result.terminalStatus, "failed");
|
||||
assert.equal(result.failureKind, "infra-failed");
|
||||
const envelope = await client.get(`/api/v1/runs/${run.id}/commands/${command.id}/result`) as JsonRecord;
|
||||
assert.equal(envelope.completed, false);
|
||||
assert.equal(envelope.failureKind, "infra-failed");
|
||||
const commandRecord = await client.get(`/api/v1/runs/${run.id}/commands/${command.id}`) as { state?: string };
|
||||
assert.equal(commandRecord.state, "failed");
|
||||
}
|
||||
|
||||
function runPayload(context: SelfTestContext, backendProfile: BackendProfile, sessionId: string): JsonRecord {
|
||||
const secretHome = backendProfile === "deepseek" ? context.deepseekHome : context.codexHome;
|
||||
return {
|
||||
tenantId: "hwlab",
|
||||
projectId: "pikasTech/HWLAB",
|
||||
workspaceRef: { kind: "opaque", repo: "pikasTech/HWLAB" },
|
||||
sessionRef: { sessionId, conversationId: sessionId },
|
||||
providerId: "G14",
|
||||
backendProfile,
|
||||
executionPolicy: {
|
||||
sandbox: "workspace-write",
|
||||
approval: "never",
|
||||
timeoutMs: 15_000,
|
||||
network: "default",
|
||||
secretScope: { allowCredentialEcho: false, providerCredentials: [{ profile: backendProfile, secretRef: { name: `agentrun-v01-provider-${backendProfile}`, keys: ["auth.json", "config.toml"], mountPath: secretHome } }] },
|
||||
},
|
||||
traceSink: { kind: "hwlab", traceId: sessionId },
|
||||
};
|
||||
}
|
||||
|
||||
async function createLocalGitRepo(context: SelfTestContext): Promise<{ repoUrl: string; commitId: string }> {
|
||||
const repo = path.join(context.tmp, "contract-bundle-repo");
|
||||
await mkdir(repo, { recursive: true });
|
||||
await execFile("git", ["init"], { cwd: repo });
|
||||
await writeFile(path.join(repo, "README.md"), "AgentRun contract bundle self-test\n", "utf8");
|
||||
await execFile("git", ["add", "README.md"], { cwd: repo });
|
||||
await execFile("git", ["-c", "user.email=selftest@example.invalid", "-c", "user.name=AgentRun SelfTest", "commit", "-m", "contract bundle selftest"], { cwd: repo });
|
||||
const { stdout } = await execFile("git", ["rev-parse", "HEAD"], { cwd: repo });
|
||||
return { repoUrl: repo, commitId: stdout.trim() };
|
||||
}
|
||||
|
||||
export default selfTest;
|
||||
@@ -66,6 +66,10 @@ for await (const line of rl) {
|
||||
respond(message.id, null, { code: -32000, message: "responseStreamDisconnected: HTTP 503 Service Unavailable from provider" });
|
||||
continue;
|
||||
}
|
||||
if (mode === "provider-401-rpc-error") {
|
||||
respond(message.id, null, { code: -32000, message: "HTTP 401 Unauthorized: invalid api key" });
|
||||
continue;
|
||||
}
|
||||
if (mode === "missing-terminal") {
|
||||
turnCounter += 1;
|
||||
const turn = { id: `turn_selftest_${turnCounter}`, status: "running" };
|
||||
@@ -81,6 +85,14 @@ for await (const line of rl) {
|
||||
respond(message.id, { turn });
|
||||
continue;
|
||||
}
|
||||
if (mode === "provider-429-terminal") {
|
||||
turnCounter += 1;
|
||||
const turn = { id: `turn_selftest_${turnCounter}`, status: "failed", error: { message: "HTTP 429 Too Many Requests: rate limit exceeded" } };
|
||||
notify("turn/started", { turn: { id: turn.id, status: "running" } });
|
||||
notify("turn/completed", { turn });
|
||||
respond(message.id, { turn });
|
||||
continue;
|
||||
}
|
||||
if (mode === "provider-503-retry-event") {
|
||||
turnCounter += 1;
|
||||
const turn = {
|
||||
|
||||
Reference in New Issue
Block a user