fix: enforce runner retention before job create
This commit is contained in:
lyon
@@ -265,6 +265,118 @@ process.exit(1);
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => serverWithKubectl.server.close(() => resolve()));
|
||||
}
|
||||
const retentionKubectl = path.join(context.tmp, "fake-kubectl-retention.js");
|
||||
const retentionCreatedManifest = path.join(context.tmp, "created-retention-runner-job.json");
|
||||
const retentionDeleted = path.join(context.tmp, "retention-deleted.json");
|
||||
await writeFile(retentionDeleted, "[]\n");
|
||||
await writeFile(retentionKubectl, `#!/usr/bin/env bun
|
||||
const args = Bun.argv.slice(2);
|
||||
const deletedPath = ${JSON.stringify(retentionDeleted)};
|
||||
async function readDeleted() {
|
||||
try { return JSON.parse(await Bun.file(deletedPath).text()); } catch { return []; }
|
||||
}
|
||||
async function writeDeleted(items) { await Bun.write(deletedPath, JSON.stringify(items, null, 2) + "\\n"); }
|
||||
async function readStdin() {
|
||||
const chunks = [];
|
||||
for await (const chunk of Bun.stdin.stream()) chunks.push(chunk);
|
||||
return Buffer.concat(chunks.map((chunk) => Buffer.from(chunk))).toString("utf8");
|
||||
}
|
||||
function runnerJob(name, createdAt, commandId) {
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: {
|
||||
name,
|
||||
namespace: "agentrun-v02",
|
||||
creationTimestamp: createdAt,
|
||||
labels: { "app.kubernetes.io/part-of": "agentrun", "app.kubernetes.io/name": "agentrun-runner", "app.kubernetes.io/component": "runner", "agentrun.pikastech.local/lane": "v0.2", "job-name": name },
|
||||
annotations: { "agentrun.pikastech.local/run-id": "run-old", "agentrun.pikastech.local/command-id": commandId, "agentrun.pikastech.local/runner-id": "runner-old" },
|
||||
},
|
||||
status: {},
|
||||
};
|
||||
}
|
||||
if (args[0] === "get" && args[1] === "jobs") {
|
||||
const deleted = await readDeleted();
|
||||
const deletedJobs = new Set(deleted.filter((item) => item.resource === "job").map((item) => item.name));
|
||||
const items = [
|
||||
runnerJob("agentrun-v02-runner-old-a", "2026-01-01T00:00:00Z", "cmd-old-a"),
|
||||
runnerJob("agentrun-v02-runner-old-b", "2026-01-01T00:01:00Z", "cmd-old-b"),
|
||||
].filter((item) => !deletedJobs.has(item.metadata.name));
|
||||
console.log(JSON.stringify({ apiVersion: "v1", kind: "List", items }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (args[0] === "get" && args[1] === "pods") {
|
||||
console.log(JSON.stringify({ apiVersion: "v1", kind: "List", items: [] }));
|
||||
process.exit(0);
|
||||
}
|
||||
if (args[0] === "delete") {
|
||||
const deleted = await readDeleted();
|
||||
const resource = args[1];
|
||||
const name = args[2] && !args[2].startsWith("-") ? args[2] : args.join(" ");
|
||||
deleted.push({ resource, name, args });
|
||||
await writeDeleted(deleted);
|
||||
console.log(String(resource) + "/" + String(name).replace(/[^a-z0-9-]+/g, "-") + " deleted");
|
||||
process.exit(0);
|
||||
}
|
||||
if (args[0] === "create") {
|
||||
const text = await readStdin();
|
||||
const manifest = JSON.parse(text);
|
||||
if (manifest.kind === "Job") await Bun.write(${JSON.stringify(retentionCreatedManifest)}, text);
|
||||
console.log(JSON.stringify({ apiVersion: manifest.apiVersion, kind: manifest.kind, metadata: { uid: "retention-job-uid", resourceVersion: "1", name: manifest.metadata.name, namespace: manifest.metadata.namespace } }));
|
||||
process.exit(0);
|
||||
}
|
||||
console.error("unsupported fake retention kubectl args: " + args.join(" "));
|
||||
process.exit(1);
|
||||
`);
|
||||
await chmod(retentionKubectl, 0o755);
|
||||
const serverWithRetention = await startManagerServer({
|
||||
port: 0,
|
||||
host: "127.0.0.1",
|
||||
sourceCommit: "self-test",
|
||||
store: new MemoryAgentRunStore(),
|
||||
runnerJobDefaults: {
|
||||
namespace: "agentrun-v02",
|
||||
managerUrl: "http://agentrun-mgr.agentrun-v02.svc.cluster.local:8080",
|
||||
image: "127.0.0.1:5000/agentrun/agentrun-mgr@sha256:1111111111111111111111111111111111111111111111111111111111111111",
|
||||
serviceAccountName: "agentrun-v02-runner",
|
||||
jobNamePrefix: "agentrun-v02-runner",
|
||||
lane: "v0.2",
|
||||
kubectlCommand: retentionKubectl,
|
||||
retention: {
|
||||
namespace: "agentrun-v02",
|
||||
maxRunners: 1,
|
||||
cleanupOrder: "oldest-inactive-last-active-first",
|
||||
activeHeartbeatMaxAgeMs: 900_000,
|
||||
matchLabels: { "app.kubernetes.io/part-of": "agentrun", "app.kubernetes.io/name": "agentrun-runner", "app.kubernetes.io/component": "runner" },
|
||||
jobNamePrefixes: ["agentrun-v02-runner"],
|
||||
ageBasedCleanup: { enabled: false, maxAgeHours: null },
|
||||
},
|
||||
},
|
||||
});
|
||||
try {
|
||||
const retentionClient = new ManagerClient(serverWithRetention.baseUrl);
|
||||
const retentionItem = await createRunWithCommand(retentionClient, context, "retention job create", "selftest-runner-retention", 15_000);
|
||||
const retentionCreated = await retentionClient.post(`/api/v1/runs/${retentionItem.runId}/runner-jobs`, { commandId: retentionItem.commandId, attemptId: "attempt_selftest_retention" }) as JsonRecord;
|
||||
const retention = (retentionCreated.retention as JsonRecord).preCreateCleanup as JsonRecord;
|
||||
assert.equal(retention.liveRunnerCountBefore, 2);
|
||||
assert.equal(retention.selectedRunnerCount, 2);
|
||||
assert.equal(retention.deletedRunnerJobCount, 2);
|
||||
assert.equal(retention.liveRunnerCountAfter, 0);
|
||||
assert.equal(retention.maxRunners, 1);
|
||||
assert.equal(retention.valuesPrinted, false);
|
||||
assert.match(String(retentionCreated.jobName), /^agentrun-v02-runner-[a-f0-9]{12}$/u);
|
||||
const retentionManifest = JSON.parse(await readFile(retentionCreatedManifest, "utf8")) as JsonRecord;
|
||||
const metadata = retentionManifest.metadata as JsonRecord;
|
||||
const labels = metadata.labels as JsonRecord;
|
||||
assert.equal(labels["agentrun.pikastech.local/lane"], "v0.2");
|
||||
assert.match(String(metadata.name), /^agentrun-v02-runner-[a-f0-9]{12}$/u);
|
||||
const deletedItems = JSON.parse(await readFile(retentionDeleted, "utf8")) as JsonRecord[];
|
||||
assert.equal(deletedItems.filter((item) => item.resource === "job").length, 2);
|
||||
assert.ok(deletedItems.some((item) => String((item.args as string[]).join(" ")).includes("agentrun.pikastech.local/runner-job=agentrun-v02-runner-old-a")), "old runner associated resources should be selected by runner-job label");
|
||||
assertNoSecretLeak(retentionCreated);
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => serverWithRetention.server.close(() => resolve()));
|
||||
}
|
||||
const sessionRunRecord: RunRecord = {
|
||||
id: "run-selftest-session-pvc",
|
||||
tenantId: "unidesk",
|
||||
@@ -307,7 +419,7 @@ process.exit(1);
|
||||
assert.equal(envMap.get("AGENTRUN_SESSION_PVC_NAMESPACE"), "agentrun-v01");
|
||||
assert.equal(envMap.get("AGENTRUN_SESSION_PVC_MOUNT_PATH"), "/home/agentrun/.codex-codex/sessions");
|
||||
assert.equal(envMap.get("AGENTRUN_CODEX_ROLLOUT_SUBDIR"), "sessions");
|
||||
return { name: "runner-k8s-job", tests: ["runner-k8s-job-dry-run", "runner-k8s-job-codex-shell-sandbox-env", "runner-k8s-job-g14-egress-proxy-env", "runner-k8s-job-bounded-git-transport-env", "runner-k8s-job-deepseek-profile-dry-run", "runner-k8s-job-minimax-m3-profile-dry-run", "runner-k8s-job-dsflash-go-profile-dry-run", "runner-k8s-job-dsflash-go-legacy-secretref-normalized", "runner-k8s-job-create-api", "runner-k8s-job-retention-ttl", "runner-job-transient-env", "runner-job-transient-env-secretref", "runner-job-tool-credential-env", "runner-job-unidesk-ssh-tool-credential-env", "runner-job-tool-credential-volume", "runner-job-unidesk-ssh-endpoint-auto-env", "runner-job-unidesk-ssh-transient-env-denied", "runner-k8s-job-session-pvc-volume-and-env"] };
|
||||
return { name: "runner-k8s-job", tests: ["runner-k8s-job-dry-run", "runner-k8s-job-codex-shell-sandbox-env", "runner-k8s-job-g14-egress-proxy-env", "runner-k8s-job-bounded-git-transport-env", "runner-k8s-job-deepseek-profile-dry-run", "runner-k8s-job-minimax-m3-profile-dry-run", "runner-k8s-job-dsflash-go-profile-dry-run", "runner-k8s-job-dsflash-go-legacy-secretref-normalized", "runner-k8s-job-create-api", "runner-k8s-job-retention-ttl", "runner-job-transient-env", "runner-job-transient-env-secretref", "runner-job-tool-credential-env", "runner-job-unidesk-ssh-tool-credential-env", "runner-job-tool-credential-volume", "runner-job-unidesk-ssh-endpoint-auto-env", "runner-job-unidesk-ssh-transient-env-denied", "runner-job-pre-create-retention", "runner-k8s-job-session-pvc-volume-and-env"] };
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.server.close(() => resolve()));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user