Codex
195 lines
11 KiB
TypeScript
195 lines
11 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import { codexPromptLiveAuthorizationLintForTest } from "./src/code-queue";
|
|
|
|
type JsonRecord = Record<string, unknown>;
|
|
|
|
function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
|
|
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
|
|
}
|
|
|
|
function asRecord(value: unknown): JsonRecord {
|
|
assertCondition(typeof value === "object" && value !== null && !Array.isArray(value), "expected JSON object", { value });
|
|
return value as JsonRecord;
|
|
}
|
|
|
|
function nestedRecord(value: unknown, path: string[]): JsonRecord {
|
|
let current: unknown = value;
|
|
for (const key of path) {
|
|
current = asRecord(current)[key];
|
|
}
|
|
return asRecord(current);
|
|
}
|
|
|
|
function stringArray(value: unknown): string[] {
|
|
return Array.isArray(value) ? value.map((item) => String(item)) : [];
|
|
}
|
|
|
|
function assertLegacyFrozenWrite(result: { status: number | null; stdout: string; stderr: string; json: JsonRecord | null }, command: string): void {
|
|
assertCondition(result.status !== 0 && result.json?.ok === false, `${command} should be frozen`, result.json ?? { stdout: result.stdout, stderr: result.stderr });
|
|
const data = nestedRecord(result.json?.data, []);
|
|
assertCondition(data.frozen === true, `${command} frozen payload should expose frozen=true`, data);
|
|
assertCondition(data.mutation === false, `${command} frozen payload should be non-mutating`, data);
|
|
assertCondition(data.degradedReason === "legacy-code-queue-frozen", `${command} should use the legacy frozen reason`, data);
|
|
const replacement = nestedRecord(data, ["replacement"]);
|
|
assertCondition(String(replacement.queueSubmit || "").includes("agentrun queue submit"), `${command} should point to AgentRun queue submit`, replacement);
|
|
assertCondition(String(replacement.sessionsSteer || "").includes("agentrun sessions steer"), `${command} should point to AgentRun sessions steer`, replacement);
|
|
}
|
|
|
|
function runCli(args: string[], stdin?: string): { status: number | null; stdout: string; stderr: string; json: JsonRecord | null } {
|
|
const result = spawnSync("bun", ["scripts/cli.ts", ...args], {
|
|
cwd: process.cwd(),
|
|
input: stdin,
|
|
encoding: "utf8",
|
|
});
|
|
const stdout = String(result.stdout || "");
|
|
let json: JsonRecord | null = null;
|
|
try {
|
|
json = JSON.parse(stdout) as JsonRecord;
|
|
} catch {
|
|
json = null;
|
|
}
|
|
return {
|
|
status: result.status,
|
|
stdout,
|
|
stderr: String(result.stderr || ""),
|
|
json,
|
|
};
|
|
}
|
|
|
|
const readOnlyPrompt = `
|
|
DEV test class: read-only
|
|
允许的 live mutation: none
|
|
禁止动作: prod mutation, secret values, database patch, Code Queue backend restart, interrupt/cancel.
|
|
closeout fields: report class, live mutation false, commands, target environment, evidence, residual risk.
|
|
|
|
Task: update docs and run contract tests only.
|
|
Validation: git diff --check and bun scripts/code-queue-prompt-lint-contract-test.ts.
|
|
`;
|
|
|
|
const unclassifiedM3SmokePrompt = `
|
|
Run the HWLAB M3 smoke on DEV and verify res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.
|
|
Do not touch production.
|
|
`;
|
|
|
|
const liveReadPrompt = `
|
|
DEV test class: live-read
|
|
允许的 live mutation: none
|
|
禁止动作: no POST/PUT/PATCH/DELETE, no deploy, no task trigger, no prod mutation.
|
|
closeout fields: report class, live mutation false, endpoints read, namespace/service, evidence, residual risk.
|
|
|
|
Read live DEV health and status only: GET /health and kubectl get pods.
|
|
`;
|
|
|
|
const liveMutatingPrompt = `
|
|
DEV test class: live-mutating
|
|
允许的 live mutation: run exactly one DEV HWLAB M3 operation for res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1, record operation/audit/evidence ids, then observe recovery.
|
|
禁止动作: no prod mutation, no secret values, no database manual patch, no Code Queue backend restart, no unrelated interrupt/cancel.
|
|
closeout fields: report class, live mutation true, command summary, DEV target, operation/audit/evidence ids, rollback observation, residual risk.
|
|
`;
|
|
|
|
const secretBearingPrompt = `
|
|
DEV test class: live-mutating
|
|
允许的 live mutation: POST /api/run?token=ds-secret-must-not-print once in DEV only.
|
|
禁止动作: no prod mutation, no database patch.
|
|
closeout fields: report class, live mutation true, command summary and evidence id.
|
|
`;
|
|
|
|
function assertLintShape(lint: JsonRecord): void {
|
|
assertCondition(lint.dryRun === true, "lint must be dry-run", lint);
|
|
assertCondition(lint.mutation === false, "lint must be non-mutating", lint);
|
|
assertCondition(asRecord(lint.policy).printsPromptText === false, "lint policy must not print full prompt", lint);
|
|
assertCondition(asRecord(lint.promptShape).textEchoed === false, "lint shape must not echo prompt text", lint);
|
|
assertCondition(Array.isArray(lint.signals), "lint must expose signals", lint);
|
|
const json = JSON.stringify(lint);
|
|
assertCondition(!json.includes("ds-secret-must-not-print"), "lint must not print secret marker", lint);
|
|
}
|
|
|
|
export function runCodeQueuePromptLintContract(): JsonRecord {
|
|
const readOnly = asRecord(codexPromptLiveAuthorizationLintForTest(readOnlyPrompt));
|
|
assertLintShape(readOnly);
|
|
assertCondition(readOnly.ok === true, "well-formed read-only prompt should pass", readOnly);
|
|
assertCondition(readOnly.declaredClass === "read-only", "read-only prompt should declare read-only", readOnly);
|
|
assertCondition(readOnly.effectiveClass === "read-only", "read-only effective class mismatch", readOnly);
|
|
assertCondition(readOnly.requiredClass === "read-only", "read-only required class mismatch", readOnly);
|
|
assertCondition(readOnly.dispatchDisposition === "ready", "read-only prompt should be dispatch-ready", readOnly);
|
|
|
|
const liveRead = asRecord(codexPromptLiveAuthorizationLintForTest(liveReadPrompt));
|
|
assertLintShape(liveRead);
|
|
assertCondition(liveRead.ok === true, "well-formed live-read prompt should pass", liveRead);
|
|
assertCondition(liveRead.declaredClass === "live-read", "live-read prompt should declare live-read", liveRead);
|
|
assertCondition(liveRead.requiredClass === "live-read", "live-read required class mismatch", liveRead);
|
|
assertCondition(liveRead.dispatchDisposition === "ready", "live-read prompt should be dispatch-ready", liveRead);
|
|
|
|
const unclassifiedM3 = asRecord(codexPromptLiveAuthorizationLintForTest(unclassifiedM3SmokePrompt));
|
|
assertLintShape(unclassifiedM3);
|
|
assertCondition(unclassifiedM3.ok === false, "unclassified M3 smoke should fail lint", unclassifiedM3);
|
|
assertCondition(unclassifiedM3.declaredClass === null, "unclassified prompt should have no declared class", unclassifiedM3);
|
|
assertCondition(unclassifiedM3.effectiveClass === "read-only", "unclassified prompt should default to read-only", unclassifiedM3);
|
|
assertCondition(unclassifiedM3.requiredClass === "live-mutating", "M3 smoke should require live-mutating", unclassifiedM3);
|
|
assertCondition(unclassifiedM3.dispatchDisposition === "needs-authorization", "unclassified live mutation should need authorization", unclassifiedM3);
|
|
assertCondition(stringArray(unclassifiedM3.missingOrContradictory).some((item) => item.includes("missing DEV test class")), "unclassified prompt should report missing class", unclassifiedM3);
|
|
|
|
const liveMutating = asRecord(codexPromptLiveAuthorizationLintForTest(liveMutatingPrompt));
|
|
assertLintShape(liveMutating);
|
|
assertCondition(liveMutating.ok === true, "well-formed live-mutating prompt should pass", liveMutating);
|
|
assertCondition(liveMutating.declaredClass === "live-mutating", "live-mutating prompt should declare live-mutating", liveMutating);
|
|
assertCondition(liveMutating.requiredClass === "live-mutating", "live-mutating required class mismatch", liveMutating);
|
|
assertCondition(liveMutating.liveMutationAuthorized === true, "live-mutating prompt should be authorized when allowed mutation is enumerated", liveMutating);
|
|
|
|
const secretBearing = asRecord(codexPromptLiveAuthorizationLintForTest(secretBearingPrompt));
|
|
assertLintShape(secretBearing);
|
|
assertCondition(secretBearing.requiredClass === "live-mutating", "secret-bearing live mutation should still classify", secretBearing);
|
|
assertCondition(!JSON.stringify(secretBearing).includes("ds-secret-must-not-print"), "prompt lint evidence must redact secret-looking values", secretBearing);
|
|
|
|
const tmp = mkdtempSync(join(tmpdir(), "unidesk-code-queue-prompt-lint-"));
|
|
const promptFile = join(tmp, "prompt.md");
|
|
writeFileSync(promptFile, liveMutatingPrompt, "utf8");
|
|
try {
|
|
const cliLint = runCli(["codex", "prompt-lint", "--prompt-file", promptFile]);
|
|
assertCondition(cliLint.status === 0 && cliLint.json?.ok === true, "prompt-lint CLI should succeed for authorized live-mutating prompt", cliLint.json ?? { stdout: cliLint.stdout });
|
|
const lintData = nestedRecord(cliLint.json?.data, []);
|
|
assertCondition(lintData.dryRun === true && lintData.mutation === false, "prompt-lint CLI should be dry-run and non-mutating", lintData);
|
|
assertCondition(lintData.declaredClass === "live-mutating", "prompt-lint CLI should classify live-mutating", lintData);
|
|
assertCondition(!JSON.stringify(lintData).includes("run exactly one DEV HWLAB M3 operation"), "prompt-lint CLI should not echo full prompt text", lintData);
|
|
} finally {
|
|
rmSync(tmp, { recursive: true, force: true });
|
|
}
|
|
|
|
const submitDryRun = runCli(["codex", "submit", "--prompt-stdin", "--dry-run"], unclassifiedM3SmokePrompt);
|
|
assertLegacyFrozenWrite(submitDryRun, "codex submit");
|
|
assertCondition(!submitDryRun.stdout.includes("res_boxsimu_1"), "frozen submit must not echo prompt text", { stdout: submitDryRun.stdout });
|
|
|
|
const steerDryRun = runCli(["codex", "steer", "codex_test_task", "--prompt-stdin", "--dry-run"], unclassifiedM3SmokePrompt);
|
|
assertLegacyFrozenWrite(steerDryRun, "codex steer");
|
|
assertCondition(!steerDryRun.stdout.includes("res_boxsimu_1"), "frozen steer must not echo prompt text", { stdout: steerDryRun.stdout });
|
|
|
|
const help = runCli(["codex", "help"]);
|
|
assertCondition(help.status === 0 && help.json?.ok === true, "codex help should succeed", help.json ?? { stdout: help.stdout });
|
|
const helpData = nestedRecord(help.json?.data, []);
|
|
const usage = stringArray(helpData.usage);
|
|
assertCondition(usage.some((line) => line.includes("codex prompt-lint")), "codex help should list prompt-lint", helpData);
|
|
const authorizationHelp = nestedRecord(helpData, ["promptLiveAuthorization"]);
|
|
assertCondition(stringArray(authorizationHelp.classes).includes("live-mutating"), "help should document live-mutating class", authorizationHelp);
|
|
assertCondition(authorizationHelp.defaultWhenMissing === "read-only", "help should document read-only default", authorizationHelp);
|
|
|
|
return {
|
|
ok: true,
|
|
checks: [
|
|
"prompt-lint classifies read-only/live-read/live-mutating prompts",
|
|
"unclassified HWLAB M3 smoke defaults read-only but requires live-mutating authorization",
|
|
"prompt-lint evidence redacts secret-looking values",
|
|
"prompt-lint CLI is dry-run, non-mutating, and does not echo full prompt text",
|
|
"legacy submit --dry-run is frozen and points to AgentRun",
|
|
"legacy steer --dry-run is frozen and points to AgentRun",
|
|
"codex help documents prompt-lint and authorization classes",
|
|
],
|
|
};
|
|
}
|
|
|
|
if (import.meta.main) {
|
|
process.stdout.write(`${JSON.stringify(runCodeQueuePromptLintContract(), null, 2)}\n`);
|
|
}
|