feat: add controlled AgentRun CI cleanup
This commit is contained in:
Codex
+549
-2
@@ -22,7 +22,7 @@ const mirrorToolsImage = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine
|
||||
|
||||
export function agentRunHelp(): unknown {
|
||||
return {
|
||||
command: "agentrun v01 control-plane status|trigger-current|refresh | git-mirror status|sync|flush",
|
||||
command: "agentrun v01 control-plane status|trigger-current|refresh|cleanup-runs|cleanup-released-pvs | git-mirror status|sync|flush",
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts agentrun v01 control-plane status",
|
||||
@@ -31,12 +31,16 @@ export function agentRunHelp(): unknown {
|
||||
"bun scripts/cli.ts agentrun v01 control-plane trigger-current --confirm",
|
||||
"bun scripts/cli.ts agentrun v01 control-plane refresh --dry-run",
|
||||
"bun scripts/cli.ts agentrun v01 control-plane refresh --confirm",
|
||||
"bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
|
||||
"bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit 200 --confirm",
|
||||
"bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit 200 --dry-run",
|
||||
"bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit 200 --confirm",
|
||||
"bun scripts/cli.ts agentrun v01 git-mirror status",
|
||||
"bun scripts/cli.ts agentrun v01 git-mirror status --full",
|
||||
"bun scripts/cli.ts agentrun v01 git-mirror sync --confirm",
|
||||
"bun scripts/cli.ts agentrun v01 git-mirror flush --confirm",
|
||||
],
|
||||
description: "Operate AgentRun v0.1 Tekton/Argo control plane and devops-infra git mirror through G14 routes; status is read-only and trigger-current pre-syncs mirror refs before creating the PipelineRun.",
|
||||
description: "Operate AgentRun v0.1 Tekton/Argo control plane and devops-infra git mirror through G14 routes; status is read-only, trigger-current pre-syncs mirror refs before creating the PipelineRun, and cleanup-runs/cleanup-released-pvs provide controlled completed CI workspace retention.",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -47,6 +51,8 @@ export async function runAgentRunCommand(config: UniDeskConfig, args: string[]):
|
||||
if (action === "status") return await status(config, parseDisclosureOptions(args.slice(3)));
|
||||
if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(args.slice(3)));
|
||||
if (action === "refresh") return await refresh(config, parseConfirmOptions(args.slice(3)));
|
||||
if (action === "cleanup-runs") return await cleanupRuns(config, parseCleanupRunsOptions(args.slice(3)));
|
||||
if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(args.slice(3)));
|
||||
}
|
||||
if (group === "git-mirror") {
|
||||
if (action === "status") return await gitMirrorStatus(config, parseDisclosureOptions(args.slice(3)));
|
||||
@@ -74,6 +80,17 @@ interface GitMirrorOptions extends ConfirmOptions {
|
||||
wait: boolean;
|
||||
}
|
||||
|
||||
interface CleanupRunsOptions extends ConfirmOptions {
|
||||
minAgeMinutes: number;
|
||||
limit: number;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
interface CleanupReleasedPvOptions extends ConfirmOptions {
|
||||
limit: number;
|
||||
timeoutSeconds: number;
|
||||
}
|
||||
|
||||
interface DisclosureOptions {
|
||||
full: boolean;
|
||||
raw: boolean;
|
||||
@@ -112,6 +129,57 @@ function parseGitMirrorOptions(args: string[]): GitMirrorOptions {
|
||||
return { ...base, timeoutSeconds, wait: args.includes("--wait") };
|
||||
}
|
||||
|
||||
function parseCleanupRunsOptions(args: string[]): CleanupRunsOptions {
|
||||
validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--min-age-minutes", "--limit", "--timeout-seconds"]));
|
||||
const base = parseConfirmOptions(args);
|
||||
return {
|
||||
...base,
|
||||
minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
|
||||
limit: positiveIntegerOption(args, "--limit", 20, 500),
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 600),
|
||||
};
|
||||
}
|
||||
|
||||
function parseCleanupReleasedPvOptions(args: string[]): CleanupReleasedPvOptions {
|
||||
validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--limit", "--timeout-seconds"]));
|
||||
const base = parseConfirmOptions(args);
|
||||
return {
|
||||
...base,
|
||||
limit: positiveIntegerOption(args, "--limit", 20, 500),
|
||||
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
|
||||
};
|
||||
}
|
||||
|
||||
function validateOptions(args: string[], booleanOptions: Set<string>, valueOptions: Set<string>): void {
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const arg = args[index];
|
||||
if (booleanOptions.has(arg)) continue;
|
||||
if (valueOptions.has(arg)) {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unsupported option: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
function optionValue(args: string[], name: string): string | undefined {
|
||||
const index = args.indexOf(name);
|
||||
if (index === -1) return undefined;
|
||||
const value = args[index + 1];
|
||||
if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
||||
const raw = optionValue(args, name);
|
||||
if (raw === undefined) return defaultValue;
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
|
||||
return Math.min(value, maxValue);
|
||||
}
|
||||
|
||||
async function status(config: UniDeskConfig, options: DisclosureOptions): Promise<Record<string, unknown>> {
|
||||
const sourceProbe = await timedStatusStage("source", () => capture(config, g14SourceRoute, ["script", "--", [
|
||||
"cd /root/agentrun-v01",
|
||||
@@ -360,6 +428,469 @@ async function refresh(config: UniDeskConfig, options: ConfirmOptions): Promise<
|
||||
};
|
||||
}
|
||||
|
||||
async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise<Record<string, unknown>> {
|
||||
const result = await capture(config, g14K3sRoute, ["script", "--", cleanupRunsScript(options)]);
|
||||
const payload = captureJsonPayload(result);
|
||||
const ok = result.exitCode === 0 && payload.ok !== false;
|
||||
const base = {
|
||||
...payload,
|
||||
ok,
|
||||
command: "agentrun v01 control-plane cleanup-runs",
|
||||
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
||||
namespace: ciNamespace,
|
||||
minAgeMinutes: options.minAgeMinutes,
|
||||
limit: options.limit,
|
||||
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
||||
};
|
||||
if (options.dryRun || !options.confirm) {
|
||||
return {
|
||||
...base,
|
||||
dryRun: true,
|
||||
mutation: false,
|
||||
next: {
|
||||
confirm: `bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
dryRun: false,
|
||||
mutation: true,
|
||||
followUp: {
|
||||
status: "bun scripts/cli.ts agentrun v01 control-plane status",
|
||||
releasedPvs: `bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit ${options.limit} --dry-run`,
|
||||
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise<Record<string, unknown>> {
|
||||
const result = await capture(config, g14K3sRoute, ["script", "--", cleanupReleasedPvsScript(options)]);
|
||||
const payload = captureJsonPayload(result);
|
||||
const ok = result.exitCode === 0 && payload.ok !== false;
|
||||
const base = {
|
||||
...payload,
|
||||
ok,
|
||||
command: "agentrun v01 control-plane cleanup-released-pvs",
|
||||
mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
|
||||
namespace: ciNamespace,
|
||||
limit: options.limit,
|
||||
probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
|
||||
};
|
||||
if (options.dryRun || !options.confirm) {
|
||||
return {
|
||||
...base,
|
||||
dryRun: true,
|
||||
mutation: false,
|
||||
next: {
|
||||
confirm: `bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit ${options.limit} --confirm`,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
dryRun: false,
|
||||
mutation: true,
|
||||
followUp: {
|
||||
cleanupRuns: `bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit ${options.limit} --dry-run`,
|
||||
diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function cleanupRunsScript(options: CleanupRunsOptions): string {
|
||||
return [
|
||||
"set -eu",
|
||||
`namespace=${shQuote(ciNamespace)}`,
|
||||
`min_age_minutes=${String(options.minAgeMinutes)}`,
|
||||
`limit=${String(options.limit)}`,
|
||||
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
||||
"tmp_dir=$(mktemp -d)",
|
||||
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
||||
"kubectl -n \"$namespace\" get pipelinerun -o json > \"$tmp_dir/pipelineruns.json\"",
|
||||
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"",
|
||||
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
|
||||
"kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"",
|
||||
"NAMESPACE=\"$namespace\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
||||
cleanupRunsPlanNodeScript(),
|
||||
"NODE",
|
||||
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
||||
" cat \"$tmp_dir/plan.json\"",
|
||||
" exit 0",
|
||||
"fi",
|
||||
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPipelineRuns)?plan.selectedPipelineRuns:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-names.txt\"",
|
||||
"delete_exit=0",
|
||||
"if [ -s \"$tmp_dir/selected-names.txt\" ]; then",
|
||||
" xargs -r kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-names.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
||||
"else",
|
||||
" : > \"$tmp_dir/delete.out\"",
|
||||
" : > \"$tmp_dir/delete.err\"",
|
||||
"fi",
|
||||
"kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs-after.json\"",
|
||||
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
||||
cleanupRunsFinalizeNodeScript(),
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions): string {
|
||||
return [
|
||||
"set -eu",
|
||||
`namespace=${shQuote(ciNamespace)}`,
|
||||
`limit=${String(options.limit)}`,
|
||||
`timeout_seconds=${String(options.timeoutSeconds)}`,
|
||||
"tmp_dir=$(mktemp -d)",
|
||||
"trap 'rm -rf \"$tmp_dir\"' EXIT",
|
||||
"kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
|
||||
"NAMESPACE=\"$namespace\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
|
||||
cleanupReleasedPvsPlanNodeScript(),
|
||||
"NODE",
|
||||
"if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
|
||||
" cat \"$tmp_dir/plan.json\"",
|
||||
" exit 0",
|
||||
"fi",
|
||||
"node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPersistentVolumes)?plan.selectedPersistentVolumes:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-pvs.txt\"",
|
||||
"delete_exit=0",
|
||||
"if [ -s \"$tmp_dir/selected-pvs.txt\" ]; then",
|
||||
" xargs -r kubectl delete pv --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-pvs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
|
||||
"else",
|
||||
" : > \"$tmp_dir/delete.out\"",
|
||||
" : > \"$tmp_dir/delete.err\"",
|
||||
"fi",
|
||||
"kubectl get pv -o json > \"$tmp_dir/pvs-after.json\"",
|
||||
"DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
|
||||
cleanupReleasedPvsFinalizeNodeScript(),
|
||||
"NODE",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function cleanupRunsPlanNodeScript(): string {
|
||||
return String.raw`
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const cp = require("node:child_process");
|
||||
const tmp = process.env.TMP_DIR;
|
||||
const namespace = process.env.NAMESPACE;
|
||||
const minAgeMinutes = Number(process.env.MIN_AGE_MINUTES || 60);
|
||||
const limit = Number(process.env.LIMIT || 20);
|
||||
const now = Date.now();
|
||||
|
||||
function readJson(name) {
|
||||
return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
|
||||
}
|
||||
|
||||
function conditionOf(item) {
|
||||
const conditions = Array.isArray(item?.status?.conditions) ? item.status.conditions : [];
|
||||
return conditions.find((entry) => entry.type === "Succeeded") || conditions[0] || {};
|
||||
}
|
||||
|
||||
function ageMinutes(createdAt) {
|
||||
const createdMs = Date.parse(createdAt || "");
|
||||
return Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;
|
||||
}
|
||||
|
||||
function localPathOf(pv) {
|
||||
return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null;
|
||||
}
|
||||
|
||||
function duBytes(hostPath) {
|
||||
if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null;
|
||||
try {
|
||||
const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
||||
const bytes = Number(String(out).trim().split(/\s+/)[0]);
|
||||
return Number.isFinite(bytes) ? bytes : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (!Number.isFinite(bytes)) return null;
|
||||
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
|
||||
}
|
||||
|
||||
const pipelineRuns = readJson("pipelineruns.json");
|
||||
const pvcs = readJson("pvcs.json");
|
||||
const pvs = readJson("pvs.json");
|
||||
const pods = readJson("pods.json");
|
||||
const pvByName = new Map((Array.isArray(pvs.items) ? pvs.items : []).map((item) => [item?.metadata?.name, item]));
|
||||
const activeClaimPods = new Map();
|
||||
for (const pod of Array.isArray(pods.items) ? pods.items : []) {
|
||||
const phase = pod?.status?.phase || "";
|
||||
if (phase === "Succeeded" || phase === "Failed") continue;
|
||||
for (const volume of Array.isArray(pod?.spec?.volumes) ? pod.spec.volumes : []) {
|
||||
const claimName = volume?.persistentVolumeClaim?.claimName;
|
||||
if (!claimName) continue;
|
||||
const entry = activeClaimPods.get(claimName) || [];
|
||||
entry.push(pod?.metadata?.name || null);
|
||||
activeClaimPods.set(claimName, entry.filter(Boolean));
|
||||
}
|
||||
}
|
||||
|
||||
const pvcsByOwner = new Map();
|
||||
for (const pvc of Array.isArray(pvcs.items) ? pvcs.items : []) {
|
||||
const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun");
|
||||
if (!owner?.name) continue;
|
||||
const entry = pvcsByOwner.get(owner.name) || [];
|
||||
entry.push(pvc);
|
||||
pvcsByOwner.set(owner.name, entry);
|
||||
}
|
||||
|
||||
const allPipelineRuns = (Array.isArray(pipelineRuns.items) ? pipelineRuns.items : [])
|
||||
.map((item) => {
|
||||
const condition = conditionOf(item);
|
||||
return {
|
||||
name: item?.metadata?.name || "",
|
||||
createdAt: item?.metadata?.creationTimestamp || null,
|
||||
ageMinutes: ageMinutes(item?.metadata?.creationTimestamp),
|
||||
status: condition.status || null,
|
||||
reason: condition.reason || null,
|
||||
};
|
||||
})
|
||||
.filter((item) => item.name.startsWith("agentrun-v01-ci-"));
|
||||
|
||||
const protectedActivePipelineRuns = allPipelineRuns
|
||||
.filter((item) => item.status !== "True" && item.status !== "False")
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)));
|
||||
const protectedLatestPipelineRun = allPipelineRuns
|
||||
.filter((item) => item.status === "True" || item.status === "False")
|
||||
.sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))[0]?.name || null;
|
||||
|
||||
const candidates = allPipelineRuns
|
||||
.filter((item) => item.status === "True" || item.status === "False")
|
||||
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= minAgeMinutes)
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||||
.slice(0, limit)
|
||||
.map((item) => {
|
||||
const owned = pvcsByOwner.get(item.name) || [];
|
||||
const activeMountPods = owned.flatMap((pvc) => activeClaimPods.get(pvc?.metadata?.name) || []);
|
||||
const protectedLatest = item.name === protectedLatestPipelineRun;
|
||||
return {
|
||||
...item,
|
||||
selected: activeMountPods.length === 0 && !protectedLatest,
|
||||
selectedReason: protectedLatest ? "protected-latest-pipelinerun" : activeMountPods.length === 0 ? "terminal-and-unmounted" : "owned-pvc-active-mounted",
|
||||
ownedPvcCount: owned.length,
|
||||
activeMountPods,
|
||||
};
|
||||
});
|
||||
|
||||
const selectedPipelineRuns = candidates.filter((item) => item.selected).map((item) => item.name);
|
||||
const selectedSet = new Set(selectedPipelineRuns);
|
||||
const ownedPvcs = [];
|
||||
const protectedOwnedPvcs = [];
|
||||
for (const [owner, items] of pvcsByOwner.entries()) {
|
||||
if (!selectedSet.has(owner) && !candidates.some((item) => item.name === owner)) continue;
|
||||
for (const pvc of items) {
|
||||
const volume = pvc?.spec?.volumeName || null;
|
||||
const pv = volume ? pvByName.get(volume) : null;
|
||||
const hostPath = localPathOf(pv);
|
||||
const activeMountPods = activeClaimPods.get(pvc?.metadata?.name) || [];
|
||||
const entry = {
|
||||
name: pvc?.metadata?.name || null,
|
||||
volume,
|
||||
phase: pvc?.status?.phase || null,
|
||||
ownerKind: "PipelineRun",
|
||||
owner,
|
||||
storageClass: pv?.spec?.storageClassName || null,
|
||||
reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null,
|
||||
hostPath,
|
||||
estimatedBytes: duBytes(hostPath),
|
||||
activeMountPods,
|
||||
};
|
||||
if (selectedSet.has(owner)) ownedPvcs.push(entry);
|
||||
else protectedOwnedPvcs.push(entry);
|
||||
}
|
||||
}
|
||||
|
||||
const estimatedReclaimBytes = ownedPvcs.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0);
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
planKind: "agentrun-ci-completed-pipelinerun-workspace-retention",
|
||||
generatedAt: new Date().toISOString(),
|
||||
namespace,
|
||||
criteria: { prefix: "agentrun-v01-ci-", terminalStatuses: ["True", "False"], minAgeMinutes, limit },
|
||||
candidates,
|
||||
candidateCount: candidates.length,
|
||||
protectedActivePipelineRuns,
|
||||
protectedActivePipelineRunCount: protectedActivePipelineRuns.length,
|
||||
protectedLatestPipelineRun,
|
||||
selectedPipelineRuns,
|
||||
selectedPipelineRunCount: selectedPipelineRuns.length,
|
||||
ownedPvcs,
|
||||
ownedPvcCount: ownedPvcs.length,
|
||||
protectedOwnedPvcs,
|
||||
protectedOwnedPvcCount: protectedOwnedPvcs.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatBytes(estimatedReclaimBytes),
|
||||
}));
|
||||
`;
|
||||
}
|
||||
|
||||
function cleanupRunsFinalizeNodeScript(): string {
|
||||
return String.raw`
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const tmp = process.env.TMP_DIR;
|
||||
const deleteExit = Number(process.env.DELETE_EXIT || 0);
|
||||
const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8"));
|
||||
const pvcsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvcs-after.json"), "utf8"));
|
||||
const selected = new Set(Array.isArray(plan.selectedPipelineRuns) ? plan.selectedPipelineRuns : []);
|
||||
const remainingOwnedPvcs = (Array.isArray(pvcsAfter.items) ? pvcsAfter.items : [])
|
||||
.map((pvc) => {
|
||||
const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun");
|
||||
return {
|
||||
name: pvc?.metadata?.name || null,
|
||||
volume: pvc?.spec?.volumeName || null,
|
||||
phase: pvc?.status?.phase || null,
|
||||
ownerKind: owner?.kind || null,
|
||||
owner: owner?.name || null,
|
||||
};
|
||||
})
|
||||
.filter((item) => item.owner && selected.has(item.owner));
|
||||
|
||||
function tail(name) {
|
||||
try {
|
||||
const text = fs.readFileSync(path.join(tmp, name), "utf8");
|
||||
return text.length > 3000 ? text.slice(-3000) : text;
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
...plan,
|
||||
ok: deleteExit === 0,
|
||||
deletedPipelineRuns: Array.from(selected),
|
||||
deletedPipelineRunCount: selected.size,
|
||||
deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") },
|
||||
remainingOwnedPvcs,
|
||||
remainingOwnedPvcCount: remainingOwnedPvcs.length,
|
||||
}));
|
||||
`;
|
||||
}
|
||||
|
||||
function cleanupReleasedPvsPlanNodeScript(): string {
|
||||
return String.raw`
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const cp = require("node:child_process");
|
||||
const tmp = process.env.TMP_DIR;
|
||||
const namespace = process.env.NAMESPACE;
|
||||
const limit = Number(process.env.LIMIT || 20);
|
||||
|
||||
function readJson(name) {
|
||||
return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
|
||||
}
|
||||
|
||||
function localPathOf(pv) {
|
||||
return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null;
|
||||
}
|
||||
|
||||
function duBytes(hostPath) {
|
||||
if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null;
|
||||
try {
|
||||
const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
|
||||
const bytes = Number(String(out).trim().split(/\s+/)[0]);
|
||||
return Number.isFinite(bytes) ? bytes : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function formatBytes(bytes) {
|
||||
if (!Number.isFinite(bytes)) return null;
|
||||
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit += 1;
|
||||
}
|
||||
return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
|
||||
}
|
||||
|
||||
const pvs = readJson("pvs.json");
|
||||
const candidates = (Array.isArray(pvs.items) ? pvs.items : [])
|
||||
.map((pv) => {
|
||||
const hostPath = localPathOf(pv);
|
||||
return {
|
||||
name: pv?.metadata?.name || "",
|
||||
createdAt: pv?.metadata?.creationTimestamp || null,
|
||||
phase: pv?.status?.phase || null,
|
||||
storageClass: pv?.spec?.storageClassName || null,
|
||||
reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null,
|
||||
claimNamespace: pv?.spec?.claimRef?.namespace || null,
|
||||
claimName: pv?.spec?.claimRef?.name || null,
|
||||
capacity: pv?.spec?.capacity?.storage || null,
|
||||
hostPath,
|
||||
estimatedBytes: duBytes(hostPath),
|
||||
};
|
||||
})
|
||||
.filter((item) => item.phase === "Released")
|
||||
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
||||
.filter((item) => item.claimNamespace === namespace)
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||||
.slice(0, limit);
|
||||
|
||||
const selectedPersistentVolumes = candidates.map((item) => item.name);
|
||||
const estimatedReclaimBytes = candidates.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0);
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
planKind: "agentrun-ci-released-local-path-pv-retention",
|
||||
generatedAt: new Date().toISOString(),
|
||||
namespace,
|
||||
criteria: { phase: "Released", storageClass: "local-path", reclaimPolicy: "Delete", claimNamespace: namespace, limit },
|
||||
candidates,
|
||||
candidateCount: candidates.length,
|
||||
selectedPersistentVolumes,
|
||||
selectedPersistentVolumeCount: selectedPersistentVolumes.length,
|
||||
estimatedReclaimBytes,
|
||||
estimatedReclaimHuman: formatBytes(estimatedReclaimBytes),
|
||||
}));
|
||||
`;
|
||||
}
|
||||
|
||||
function cleanupReleasedPvsFinalizeNodeScript(): string {
|
||||
return String.raw`
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const tmp = process.env.TMP_DIR;
|
||||
const deleteExit = Number(process.env.DELETE_EXIT || 0);
|
||||
const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8"));
|
||||
const pvsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvs-after.json"), "utf8"));
|
||||
const selected = new Set(Array.isArray(plan.selectedPersistentVolumes) ? plan.selectedPersistentVolumes : []);
|
||||
const remainingPersistentVolumes = (Array.isArray(pvsAfter.items) ? pvsAfter.items : [])
|
||||
.filter((pv) => selected.has(pv?.metadata?.name))
|
||||
.map((pv) => ({ name: pv?.metadata?.name || null, phase: pv?.status?.phase || null, claimNamespace: pv?.spec?.claimRef?.namespace || null, claimName: pv?.spec?.claimRef?.name || null }));
|
||||
|
||||
function tail(name) {
|
||||
try {
|
||||
const text = fs.readFileSync(path.join(tmp, name), "utf8");
|
||||
return text.length > 3000 ? text.slice(-3000) : text;
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
...plan,
|
||||
ok: deleteExit === 0,
|
||||
deletedPersistentVolumes: Array.from(selected),
|
||||
deletedPersistentVolumeCount: selected.size,
|
||||
deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") },
|
||||
remainingPersistentVolumes,
|
||||
remainingPersistentVolumeCount: remainingPersistentVolumes.length,
|
||||
}));
|
||||
`;
|
||||
}
|
||||
|
||||
function statusScript(pipelineRun: string | null): string {
|
||||
const pr = pipelineRun ?? "";
|
||||
return [
|
||||
@@ -969,6 +1500,22 @@ function labeledJson(text: string, label: string): Record<string, unknown> {
|
||||
}
|
||||
}
|
||||
|
||||
function captureJsonPayload(result: SshCaptureResult): Record<string, unknown> {
|
||||
const trimmed = result.stdout.trim();
|
||||
if (trimmed.length === 0) return {};
|
||||
try {
|
||||
return record(JSON.parse(trimmed) as unknown);
|
||||
} catch {
|
||||
const lastJsonLine = trimmed.split(/\r?\n/u).reverse().find((line) => line.trim().startsWith("{") && line.trim().endsWith("}"));
|
||||
if (lastJsonLine === undefined) return {};
|
||||
try {
|
||||
return record(JSON.parse(lastJsonLine) as unknown);
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user