feat: add controlled AgentRun CI cleanup

scripts/agentrun-cli-contract-test.ts

@@ -0,0 +1,35 @@
+import { agentRunHelp } from "./src/agentrun";
+import { rootHelp } from "./src/help";
+
+function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
+  if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
+}
+
+const agentRunUsage = Array.isArray((agentRunHelp() as { usage?: unknown }).usage)
+  ? ((agentRunHelp() as { usage: unknown[] }).usage).map(String)
+  : [];
+
+assertCondition(
+  agentRunUsage.some((line) => line.includes("control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run"))
+    && agentRunUsage.some((line) => line.includes("control-plane cleanup-runs --min-age-minutes 30 --limit 200 --confirm"))
+    && agentRunUsage.some((line) => line.includes("control-plane cleanup-released-pvs --limit 200 --dry-run"))
+    && agentRunUsage.some((line) => line.includes("control-plane cleanup-released-pvs --limit 200 --confirm")),
+  "AgentRun help must expose controlled CI workspace retention commands",
+  agentRunUsage,
+);
+
+const globalHelp = JSON.stringify(rootHelp());
+
+assertCondition(
+  globalHelp.includes("agentrun v01 control-plane status|trigger-current|refresh|cleanup-runs|cleanup-released-pvs"),
+  "global help must index AgentRun cleanup entrypoints",
+  rootHelp(),
+);
+
+console.log(JSON.stringify({
+  ok: true,
+  checks: [
+    "AgentRun command help exposes cleanup-runs and cleanup-released-pvs",
+    "global help indexes AgentRun cleanup entrypoints",
+  ],
+}));

scripts/gc-remote-registry-maintenance-contract-test.ts

@@ -0,0 +1,34 @@
+const sourceText = await Bun.file(new URL("./src/gc-remote.ts", import.meta.url)).text();
+
+function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
+  if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
+}
+
+function functionBody(name: string): string {
+  const marker = `def ${name}():`;
+  const start = sourceText.indexOf(marker);
+  if (start < 0) return "";
+  const next = sourceText.indexOf("\ndef ", start + marker.length);
+  return sourceText.slice(start, next < 0 ? sourceText.length : next);
+}
+
+for (const name of ["execute_registry_retention", "execute_registry_garbage_collect_only"]) {
+  const body = functionBody(name);
+  const suspendIndex = body.indexOf("patch_cronjob_suspend(name, True)");
+  const waitIndex = body.indexOf("idle_after_suspend = wait_no_active_hwlab_ci(180)");
+  const refusalIndex = body.indexOf("refusing registry maintenance because hwlab-ci did not become idle after suspend");
+  const preRefusalIndex = body.indexOf("refusing registry maintenance while hwlab-ci PipelineRun/TaskRun is active");
+  assertCondition(
+    suspendIndex >= 0 && waitIndex > suspendIndex && refusalIndex > waitIndex && preRefusalIndex < 0,
+    "registry maintenance must suspend poller CronJobs before refusing on active hwlab-ci objects",
+    { name, suspendIndex, waitIndex, refusalIndex, preRefusalIndex },
+  );
+}
+
+console.log(JSON.stringify({
+  ok: true,
+  checks: [
+    "registry retention suspends poller CronJobs before active-CI idle wait",
+    "registry GC-only suspends poller CronJobs before active-CI idle wait",
+  ],
+}));

scripts/hwlab-g14-contract-test.ts

@@ -394,6 +394,11 @@ assertCondition(
     && sourceText.indexOf("if (args.includes(\"--status\")) return monitorStatus(options);") < sourceText.indexOf("const command = [\"bun\", \"scripts/cli.ts\", \"hwlab\", \"g14\", \"monitor-prs\""),
   "monitor-prs --status must be a read-only query before async monitor startJob",
 );
+assertCondition(
+  sourceText.includes("protectedLatestByPrefix")
+    && sourceText.includes("protected-latest-pipelinerun"),
+  "control-plane cleanup-runs must protect the latest PipelineRun per lane by default",
+);
 
 const staleSuccessAlignment = v02CommitAlignment({
   expectedSourceHead: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",

scripts/src/agentrun.ts

@@ -22,7 +22,7 @@ const mirrorToolsImage = "127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine
 
 export function agentRunHelp(): unknown {
   return {
-    command: "agentrun v01 control-plane status|trigger-current|refresh | git-mirror status|sync|flush",
+    command: "agentrun v01 control-plane status|trigger-current|refresh|cleanup-runs|cleanup-released-pvs | git-mirror status|sync|flush",
     output: "json",
     usage: [
       "bun scripts/cli.ts agentrun v01 control-plane status",
@@ -31,12 +31,16 @@ export function agentRunHelp(): unknown {
       "bun scripts/cli.ts agentrun v01 control-plane trigger-current --confirm",
       "bun scripts/cli.ts agentrun v01 control-plane refresh --dry-run",
       "bun scripts/cli.ts agentrun v01 control-plane refresh --confirm",
+      "bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
+      "bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit 200 --confirm",
+      "bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit 200 --dry-run",
+      "bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit 200 --confirm",
       "bun scripts/cli.ts agentrun v01 git-mirror status",
       "bun scripts/cli.ts agentrun v01 git-mirror status --full",
       "bun scripts/cli.ts agentrun v01 git-mirror sync --confirm",
       "bun scripts/cli.ts agentrun v01 git-mirror flush --confirm",
     ],
-    description: "Operate AgentRun v0.1 Tekton/Argo control plane and devops-infra git mirror through G14 routes; status is read-only and trigger-current pre-syncs mirror refs before creating the PipelineRun.",
+    description: "Operate AgentRun v0.1 Tekton/Argo control plane and devops-infra git mirror through G14 routes; status is read-only, trigger-current pre-syncs mirror refs before creating the PipelineRun, and cleanup-runs/cleanup-released-pvs provide controlled completed CI workspace retention.",
   };
 }
 
@@ -47,6 +51,8 @@ export async function runAgentRunCommand(config: UniDeskConfig, args: string[]):
     if (action === "status") return await status(config, parseDisclosureOptions(args.slice(3)));
     if (action === "trigger-current") return await triggerCurrent(config, parseTriggerOptions(args.slice(3)));
     if (action === "refresh") return await refresh(config, parseConfirmOptions(args.slice(3)));
+    if (action === "cleanup-runs") return await cleanupRuns(config, parseCleanupRunsOptions(args.slice(3)));
+    if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(args.slice(3)));
   }
   if (group === "git-mirror") {
     if (action === "status") return await gitMirrorStatus(config, parseDisclosureOptions(args.slice(3)));
@@ -74,6 +80,17 @@ interface GitMirrorOptions extends ConfirmOptions {
   wait: boolean;
 }
 
+interface CleanupRunsOptions extends ConfirmOptions {
+  minAgeMinutes: number;
+  limit: number;
+  timeoutSeconds: number;
+}
+
+interface CleanupReleasedPvOptions extends ConfirmOptions {
+  limit: number;
+  timeoutSeconds: number;
+}
+
 interface DisclosureOptions {
   full: boolean;
   raw: boolean;
@@ -112,6 +129,57 @@ function parseGitMirrorOptions(args: string[]): GitMirrorOptions {
   return { ...base, timeoutSeconds, wait: args.includes("--wait") };
 }
 
+function parseCleanupRunsOptions(args: string[]): CleanupRunsOptions {
+  validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--min-age-minutes", "--limit", "--timeout-seconds"]));
+  const base = parseConfirmOptions(args);
+  return {
+    ...base,
+    minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080),
+    limit: positiveIntegerOption(args, "--limit", 20, 500),
+    timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 600),
+  };
+}
+
+function parseCleanupReleasedPvOptions(args: string[]): CleanupReleasedPvOptions {
+  validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--limit", "--timeout-seconds"]));
+  const base = parseConfirmOptions(args);
+  return {
+    ...base,
+    limit: positiveIntegerOption(args, "--limit", 20, 500),
+    timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600),
+  };
+}
+
+function validateOptions(args: string[], booleanOptions: Set<string>, valueOptions: Set<string>): void {
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index];
+    if (booleanOptions.has(arg)) continue;
+    if (valueOptions.has(arg)) {
+      const value = args[index + 1];
+      if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value`);
+      index += 1;
+      continue;
+    }
+    throw new Error(`unsupported option: ${arg}`);
+  }
+}
+
+function optionValue(args: string[], name: string): string | undefined {
+  const index = args.indexOf(name);
+  if (index === -1) return undefined;
+  const value = args[index + 1];
+  if (value === undefined || value.startsWith("--")) throw new Error(`${name} requires a value`);
+  return value;
+}
+
+function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
+  const raw = optionValue(args, name);
+  if (raw === undefined) return defaultValue;
+  const value = Number(raw);
+  if (!Number.isInteger(value) || value < 0) throw new Error(`${name} must be a non-negative integer`);
+  return Math.min(value, maxValue);
+}
+
 async function status(config: UniDeskConfig, options: DisclosureOptions): Promise<Record<string, unknown>> {
   const sourceProbe = await timedStatusStage("source", () => capture(config, g14SourceRoute, ["script", "--", [
     "cd /root/agentrun-v01",
@@ -360,6 +428,469 @@ async function refresh(config: UniDeskConfig, options: ConfirmOptions): Promise<
   };
 }
 
+async function cleanupRuns(config: UniDeskConfig, options: CleanupRunsOptions): Promise<Record<string, unknown>> {
+  const result = await capture(config, g14K3sRoute, ["script", "--", cleanupRunsScript(options)]);
+  const payload = captureJsonPayload(result);
+  const ok = result.exitCode === 0 && payload.ok !== false;
+  const base = {
+    ...payload,
+    ok,
+    command: "agentrun v01 control-plane cleanup-runs",
+    mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
+    namespace: ciNamespace,
+    minAgeMinutes: options.minAgeMinutes,
+    limit: options.limit,
+    probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
+  };
+  if (options.dryRun || !options.confirm) {
+    return {
+      ...base,
+      dryRun: true,
+      mutation: false,
+      next: {
+        confirm: `bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes ${options.minAgeMinutes} --limit ${options.limit} --confirm`,
+      },
+    };
+  }
+  return {
+    ...base,
+    dryRun: false,
+    mutation: true,
+    followUp: {
+      status: "bun scripts/cli.ts agentrun v01 control-plane status",
+      releasedPvs: `bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit ${options.limit} --dry-run`,
+      diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
+    },
+  };
+}
+
+async function cleanupReleasedPvs(config: UniDeskConfig, options: CleanupReleasedPvOptions): Promise<Record<string, unknown>> {
+  const result = await capture(config, g14K3sRoute, ["script", "--", cleanupReleasedPvsScript(options)]);
+  const payload = captureJsonPayload(result);
+  const ok = result.exitCode === 0 && payload.ok !== false;
+  const base = {
+    ...payload,
+    ok,
+    command: "agentrun v01 control-plane cleanup-released-pvs",
+    mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
+    namespace: ciNamespace,
+    limit: options.limit,
+    probe: compactCapture(result, { full: result.exitCode !== 0, stdoutTailChars: 3000, stderrTailChars: 3000 }),
+  };
+  if (options.dryRun || !options.confirm) {
+    return {
+      ...base,
+      dryRun: true,
+      mutation: false,
+      next: {
+        confirm: `bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit ${options.limit} --confirm`,
+      },
+    };
+  }
+  return {
+    ...base,
+    dryRun: false,
+    mutation: true,
+    followUp: {
+      cleanupRuns: `bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit ${options.limit} --dry-run`,
+      diskPressure: "trans G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
+    },
+  };
+}
+
+function cleanupRunsScript(options: CleanupRunsOptions): string {
+  return [
+    "set -eu",
+    `namespace=${shQuote(ciNamespace)}`,
+    `min_age_minutes=${String(options.minAgeMinutes)}`,
+    `limit=${String(options.limit)}`,
+    `timeout_seconds=${String(options.timeoutSeconds)}`,
+    "tmp_dir=$(mktemp -d)",
+    "trap 'rm -rf \"$tmp_dir\"' EXIT",
+    "kubectl -n \"$namespace\" get pipelinerun -o json > \"$tmp_dir/pipelineruns.json\"",
+    "kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs.json\"",
+    "kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
+    "kubectl -n \"$namespace\" get pod -o json > \"$tmp_dir/pods.json\"",
+    "NAMESPACE=\"$namespace\" MIN_AGE_MINUTES=\"$min_age_minutes\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
+    cleanupRunsPlanNodeScript(),
+    "NODE",
+    "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
+    "  cat \"$tmp_dir/plan.json\"",
+    "  exit 0",
+    "fi",
+    "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPipelineRuns)?plan.selectedPipelineRuns:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-names.txt\"",
+    "delete_exit=0",
+    "if [ -s \"$tmp_dir/selected-names.txt\" ]; then",
+    "  xargs -r kubectl -n \"$namespace\" delete pipelinerun --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-names.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
+    "else",
+    "  : > \"$tmp_dir/delete.out\"",
+    "  : > \"$tmp_dir/delete.err\"",
+    "fi",
+    "kubectl -n \"$namespace\" get pvc -o json > \"$tmp_dir/pvcs-after.json\"",
+    "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
+    cleanupRunsFinalizeNodeScript(),
+    "NODE",
+  ].join("\n");
+}
+
+function cleanupReleasedPvsScript(options: CleanupReleasedPvOptions): string {
+  return [
+    "set -eu",
+    `namespace=${shQuote(ciNamespace)}`,
+    `limit=${String(options.limit)}`,
+    `timeout_seconds=${String(options.timeoutSeconds)}`,
+    "tmp_dir=$(mktemp -d)",
+    "trap 'rm -rf \"$tmp_dir\"' EXIT",
+    "kubectl get pv -o json > \"$tmp_dir/pvs.json\"",
+    "NAMESPACE=\"$namespace\" LIMIT=\"$limit\" TMP_DIR=\"$tmp_dir\" node <<'NODE' > \"$tmp_dir/plan.json\"",
+    cleanupReleasedPvsPlanNodeScript(),
+    "NODE",
+    "if [ " + shQuote(options.confirm && !options.dryRun ? "true" : "false") + " != true ]; then",
+    "  cat \"$tmp_dir/plan.json\"",
+    "  exit 0",
+    "fi",
+    "node -e 'const fs=require(\"node:fs\"); const plan=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const names=Array.isArray(plan.selectedPersistentVolumes)?plan.selectedPersistentVolumes:[]; fs.writeFileSync(process.argv[2], names.join(\"\\n\") + (names.length>0?\"\\n\":\"\"));' \"$tmp_dir/plan.json\" \"$tmp_dir/selected-pvs.txt\"",
+    "delete_exit=0",
+    "if [ -s \"$tmp_dir/selected-pvs.txt\" ]; then",
+    "  xargs -r kubectl delete pv --ignore-not-found=true --wait=true --timeout=\"${timeout_seconds}s\" < \"$tmp_dir/selected-pvs.txt\" > \"$tmp_dir/delete.out\" 2> \"$tmp_dir/delete.err\" || delete_exit=$?",
+    "else",
+    "  : > \"$tmp_dir/delete.out\"",
+    "  : > \"$tmp_dir/delete.err\"",
+    "fi",
+    "kubectl get pv -o json > \"$tmp_dir/pvs-after.json\"",
+    "DELETE_EXIT=\"$delete_exit\" TMP_DIR=\"$tmp_dir\" node <<'NODE'",
+    cleanupReleasedPvsFinalizeNodeScript(),
+    "NODE",
+  ].join("\n");
+}
+
+function cleanupRunsPlanNodeScript(): string {
+  return String.raw`
+const fs = require("node:fs");
+const path = require("node:path");
+const cp = require("node:child_process");
+const tmp = process.env.TMP_DIR;
+const namespace = process.env.NAMESPACE;
+const minAgeMinutes = Number(process.env.MIN_AGE_MINUTES || 60);
+const limit = Number(process.env.LIMIT || 20);
+const now = Date.now();
+
+function readJson(name) {
+  return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
+}
+
+function conditionOf(item) {
+  const conditions = Array.isArray(item?.status?.conditions) ? item.status.conditions : [];
+  return conditions.find((entry) => entry.type === "Succeeded") || conditions[0] || {};
+}
+
+function ageMinutes(createdAt) {
+  const createdMs = Date.parse(createdAt || "");
+  return Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;
+}
+
+function localPathOf(pv) {
+  return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null;
+}
+
+function duBytes(hostPath) {
+  if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null;
+  try {
+    const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    const bytes = Number(String(out).trim().split(/\s+/)[0]);
+    return Number.isFinite(bytes) ? bytes : null;
+  } catch {
+    return null;
+  }
+}
+
+function formatBytes(bytes) {
+  if (!Number.isFinite(bytes)) return null;
+  const units = ["B", "KiB", "MiB", "GiB", "TiB"];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit += 1;
+  }
+  return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
+}
+
+const pipelineRuns = readJson("pipelineruns.json");
+const pvcs = readJson("pvcs.json");
+const pvs = readJson("pvs.json");
+const pods = readJson("pods.json");
+const pvByName = new Map((Array.isArray(pvs.items) ? pvs.items : []).map((item) => [item?.metadata?.name, item]));
+const activeClaimPods = new Map();
+for (const pod of Array.isArray(pods.items) ? pods.items : []) {
+  const phase = pod?.status?.phase || "";
+  if (phase === "Succeeded" || phase === "Failed") continue;
+  for (const volume of Array.isArray(pod?.spec?.volumes) ? pod.spec.volumes : []) {
+    const claimName = volume?.persistentVolumeClaim?.claimName;
+    if (!claimName) continue;
+    const entry = activeClaimPods.get(claimName) || [];
+    entry.push(pod?.metadata?.name || null);
+    activeClaimPods.set(claimName, entry.filter(Boolean));
+  }
+}
+
+const pvcsByOwner = new Map();
+for (const pvc of Array.isArray(pvcs.items) ? pvcs.items : []) {
+  const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun");
+  if (!owner?.name) continue;
+  const entry = pvcsByOwner.get(owner.name) || [];
+  entry.push(pvc);
+  pvcsByOwner.set(owner.name, entry);
+}
+
+const allPipelineRuns = (Array.isArray(pipelineRuns.items) ? pipelineRuns.items : [])
+  .map((item) => {
+    const condition = conditionOf(item);
+    return {
+      name: item?.metadata?.name || "",
+      createdAt: item?.metadata?.creationTimestamp || null,
+      ageMinutes: ageMinutes(item?.metadata?.creationTimestamp),
+      status: condition.status || null,
+      reason: condition.reason || null,
+    };
+  })
+  .filter((item) => item.name.startsWith("agentrun-v01-ci-"));
+
+const protectedActivePipelineRuns = allPipelineRuns
+  .filter((item) => item.status !== "True" && item.status !== "False")
+  .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)));
+const protectedLatestPipelineRun = allPipelineRuns
+  .filter((item) => item.status === "True" || item.status === "False")
+  .sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))[0]?.name || null;
+
+const candidates = allPipelineRuns
+  .filter((item) => item.status === "True" || item.status === "False")
+  .filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= minAgeMinutes)
+  .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
+  .slice(0, limit)
+  .map((item) => {
+    const owned = pvcsByOwner.get(item.name) || [];
+    const activeMountPods = owned.flatMap((pvc) => activeClaimPods.get(pvc?.metadata?.name) || []);
+    const protectedLatest = item.name === protectedLatestPipelineRun;
+    return {
+      ...item,
+      selected: activeMountPods.length === 0 && !protectedLatest,
+      selectedReason: protectedLatest ? "protected-latest-pipelinerun" : activeMountPods.length === 0 ? "terminal-and-unmounted" : "owned-pvc-active-mounted",
+      ownedPvcCount: owned.length,
+      activeMountPods,
+    };
+  });
+
+const selectedPipelineRuns = candidates.filter((item) => item.selected).map((item) => item.name);
+const selectedSet = new Set(selectedPipelineRuns);
+const ownedPvcs = [];
+const protectedOwnedPvcs = [];
+for (const [owner, items] of pvcsByOwner.entries()) {
+  if (!selectedSet.has(owner) && !candidates.some((item) => item.name === owner)) continue;
+  for (const pvc of items) {
+    const volume = pvc?.spec?.volumeName || null;
+    const pv = volume ? pvByName.get(volume) : null;
+    const hostPath = localPathOf(pv);
+    const activeMountPods = activeClaimPods.get(pvc?.metadata?.name) || [];
+    const entry = {
+      name: pvc?.metadata?.name || null,
+      volume,
+      phase: pvc?.status?.phase || null,
+      ownerKind: "PipelineRun",
+      owner,
+      storageClass: pv?.spec?.storageClassName || null,
+      reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null,
+      hostPath,
+      estimatedBytes: duBytes(hostPath),
+      activeMountPods,
+    };
+    if (selectedSet.has(owner)) ownedPvcs.push(entry);
+    else protectedOwnedPvcs.push(entry);
+  }
+}
+
+const estimatedReclaimBytes = ownedPvcs.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0);
+console.log(JSON.stringify({
+  ok: true,
+  planKind: "agentrun-ci-completed-pipelinerun-workspace-retention",
+  generatedAt: new Date().toISOString(),
+  namespace,
+  criteria: { prefix: "agentrun-v01-ci-", terminalStatuses: ["True", "False"], minAgeMinutes, limit },
+  candidates,
+  candidateCount: candidates.length,
+  protectedActivePipelineRuns,
+  protectedActivePipelineRunCount: protectedActivePipelineRuns.length,
+  protectedLatestPipelineRun,
+  selectedPipelineRuns,
+  selectedPipelineRunCount: selectedPipelineRuns.length,
+  ownedPvcs,
+  ownedPvcCount: ownedPvcs.length,
+  protectedOwnedPvcs,
+  protectedOwnedPvcCount: protectedOwnedPvcs.length,
+  estimatedReclaimBytes,
+  estimatedReclaimHuman: formatBytes(estimatedReclaimBytes),
+}));
+`;
+}
+
+function cleanupRunsFinalizeNodeScript(): string {
+  return String.raw`
+const fs = require("node:fs");
+const path = require("node:path");
+const tmp = process.env.TMP_DIR;
+const deleteExit = Number(process.env.DELETE_EXIT || 0);
+const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8"));
+const pvcsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvcs-after.json"), "utf8"));
+const selected = new Set(Array.isArray(plan.selectedPipelineRuns) ? plan.selectedPipelineRuns : []);
+const remainingOwnedPvcs = (Array.isArray(pvcsAfter.items) ? pvcsAfter.items : [])
+  .map((pvc) => {
+    const owner = (Array.isArray(pvc?.metadata?.ownerReferences) ? pvc.metadata.ownerReferences : []).find((entry) => entry.kind === "PipelineRun");
+    return {
+      name: pvc?.metadata?.name || null,
+      volume: pvc?.spec?.volumeName || null,
+      phase: pvc?.status?.phase || null,
+      ownerKind: owner?.kind || null,
+      owner: owner?.name || null,
+    };
+  })
+  .filter((item) => item.owner && selected.has(item.owner));
+
+function tail(name) {
+  try {
+    const text = fs.readFileSync(path.join(tmp, name), "utf8");
+    return text.length > 3000 ? text.slice(-3000) : text;
+  } catch {
+    return "";
+  }
+}
+
+console.log(JSON.stringify({
+  ...plan,
+  ok: deleteExit === 0,
+  deletedPipelineRuns: Array.from(selected),
+  deletedPipelineRunCount: selected.size,
+  deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") },
+  remainingOwnedPvcs,
+  remainingOwnedPvcCount: remainingOwnedPvcs.length,
+}));
+`;
+}
+
+function cleanupReleasedPvsPlanNodeScript(): string {
+  return String.raw`
+const fs = require("node:fs");
+const path = require("node:path");
+const cp = require("node:child_process");
+const tmp = process.env.TMP_DIR;
+const namespace = process.env.NAMESPACE;
+const limit = Number(process.env.LIMIT || 20);
+
+function readJson(name) {
+  return JSON.parse(fs.readFileSync(path.join(tmp, name), "utf8"));
+}
+
+function localPathOf(pv) {
+  return pv?.spec?.local?.path || pv?.spec?.hostPath?.path || null;
+}
+
+function duBytes(hostPath) {
+  if (typeof hostPath !== "string" || !hostPath.startsWith("/var/lib/rancher/k3s/storage/")) return null;
+  try {
+    const out = cp.execFileSync("du", ["-sB1", hostPath], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    const bytes = Number(String(out).trim().split(/\s+/)[0]);
+    return Number.isFinite(bytes) ? bytes : null;
+  } catch {
+    return null;
+  }
+}
+
+function formatBytes(bytes) {
+  if (!Number.isFinite(bytes)) return null;
+  const units = ["B", "KiB", "MiB", "GiB", "TiB"];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit += 1;
+  }
+  return value.toFixed(unit === 0 ? 0 : 1) + units[unit];
+}
+
+const pvs = readJson("pvs.json");
+const candidates = (Array.isArray(pvs.items) ? pvs.items : [])
+  .map((pv) => {
+    const hostPath = localPathOf(pv);
+    return {
+      name: pv?.metadata?.name || "",
+      createdAt: pv?.metadata?.creationTimestamp || null,
+      phase: pv?.status?.phase || null,
+      storageClass: pv?.spec?.storageClassName || null,
+      reclaimPolicy: pv?.spec?.persistentVolumeReclaimPolicy || null,
+      claimNamespace: pv?.spec?.claimRef?.namespace || null,
+      claimName: pv?.spec?.claimRef?.name || null,
+      capacity: pv?.spec?.capacity?.storage || null,
+      hostPath,
+      estimatedBytes: duBytes(hostPath),
+    };
+  })
+  .filter((item) => item.phase === "Released")
+  .filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
+  .filter((item) => item.claimNamespace === namespace)
+  .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
+  .slice(0, limit);
+
+const selectedPersistentVolumes = candidates.map((item) => item.name);
+const estimatedReclaimBytes = candidates.reduce((sum, item) => sum + (Number.isFinite(item.estimatedBytes) ? item.estimatedBytes : 0), 0);
+console.log(JSON.stringify({
+  ok: true,
+  planKind: "agentrun-ci-released-local-path-pv-retention",
+  generatedAt: new Date().toISOString(),
+  namespace,
+  criteria: { phase: "Released", storageClass: "local-path", reclaimPolicy: "Delete", claimNamespace: namespace, limit },
+  candidates,
+  candidateCount: candidates.length,
+  selectedPersistentVolumes,
+  selectedPersistentVolumeCount: selectedPersistentVolumes.length,
+  estimatedReclaimBytes,
+  estimatedReclaimHuman: formatBytes(estimatedReclaimBytes),
+}));
+`;
+}
+
+function cleanupReleasedPvsFinalizeNodeScript(): string {
+  return String.raw`
+const fs = require("node:fs");
+const path = require("node:path");
+const tmp = process.env.TMP_DIR;
+const deleteExit = Number(process.env.DELETE_EXIT || 0);
+const plan = JSON.parse(fs.readFileSync(path.join(tmp, "plan.json"), "utf8"));
+const pvsAfter = JSON.parse(fs.readFileSync(path.join(tmp, "pvs-after.json"), "utf8"));
+const selected = new Set(Array.isArray(plan.selectedPersistentVolumes) ? plan.selectedPersistentVolumes : []);
+const remainingPersistentVolumes = (Array.isArray(pvsAfter.items) ? pvsAfter.items : [])
+  .filter((pv) => selected.has(pv?.metadata?.name))
+  .map((pv) => ({ name: pv?.metadata?.name || null, phase: pv?.status?.phase || null, claimNamespace: pv?.spec?.claimRef?.namespace || null, claimName: pv?.spec?.claimRef?.name || null }));
+
+function tail(name) {
+  try {
+    const text = fs.readFileSync(path.join(tmp, name), "utf8");
+    return text.length > 3000 ? text.slice(-3000) : text;
+  } catch {
+    return "";
+  }
+}
+
+console.log(JSON.stringify({
+  ...plan,
+  ok: deleteExit === 0,
+  deletedPersistentVolumes: Array.from(selected),
+  deletedPersistentVolumeCount: selected.size,
+  deletion: { exitCode: deleteExit, stdoutTail: tail("delete.out"), stderrTail: tail("delete.err") },
+  remainingPersistentVolumes,
+  remainingPersistentVolumeCount: remainingPersistentVolumes.length,
+}));
+`;
+}
+
 function statusScript(pipelineRun: string | null): string {
   const pr = pipelineRun ?? "";
   return [
@@ -969,6 +1500,22 @@ function labeledJson(text: string, label: string): Record<string, unknown> {
   }
 }
 
+function captureJsonPayload(result: SshCaptureResult): Record<string, unknown> {
+  const trimmed = result.stdout.trim();
+  if (trimmed.length === 0) return {};
+  try {
+    return record(JSON.parse(trimmed) as unknown);
+  } catch {
+    const lastJsonLine = trimmed.split(/\r?\n/u).reverse().find((line) => line.trim().startsWith("{") && line.trim().endsWith("}"));
+    if (lastJsonLine === undefined) return {};
+    try {
+      return record(JSON.parse(lastJsonLine) as unknown);
+    } catch {
+      return {};
+    }
+  }
+}
+
 function record(value: unknown): Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
 }

scripts/src/gc-remote.ts

@@ -1014,9 +1014,6 @@ def execute_registry_retention():
     deployment = registry_deployment_preflight()
     if not deployment.get("ok"):
         raise RuntimeError("registry deployment preflight failed: %s" % deployment.get("reason"))
-    active = active_hwlab_ci_writes()
-    if not active.get("ok") or int(active.get("activeCount") or 0) > 0:
-        raise RuntimeError("refusing registry maintenance while hwlab-ci PipelineRun/TaskRun is active")
     plan = plan_registry_retention()
     delete_rows = plan.get("deleteRows") or []
     delete_revision_rows = plan.get("deleteRevisionRows") or []
@@ -1144,9 +1141,6 @@ def execute_registry_garbage_collect_only():
     deployment = registry_deployment_preflight()
     if not deployment.get("ok"):
         raise RuntimeError("registry deployment preflight failed: %s" % deployment.get("reason"))
-    active = active_hwlab_ci_writes()
-    if not active.get("ok") or int(active.get("activeCount") or 0) > 0:
-        raise RuntimeError("refusing registry maintenance while hwlab-ci PipelineRun/TaskRun is active")
     cronjobs = ["hwlab-g14-branch-poller", "hwlab-v02-branch-poller"]
     original_crons = cronjob_suspend_states(cronjobs)
     before = du_size(REGISTRY_ROOT, 60) or 0

scripts/src/help.ts

@@ -59,7 +59,7 @@ export function rootHelp(): unknown {
       { command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." },
       { command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." },
       { command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions through UniDesk G14 routes; long confirmed trigger/sync/flush actions return async jobs by default." },
-      { command: "agentrun v01 control-plane status|trigger-current|refresh", description: "Run bounded AgentRun v0.1 Tekton/Argo status, manual PipelineRun trigger, and Argo refresh operations through UniDesk G14 routes." },
+      { command: "agentrun v01 control-plane status|trigger-current|refresh|cleanup-runs|cleanup-released-pvs", description: "Run bounded AgentRun v0.1 Tekton/Argo status, manual PipelineRun trigger, Argo refresh, and completed CI workspace retention through UniDesk G14 routes." },
       { command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." },
       { command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
       { command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },

scripts/src/hwlab-g14.ts

@@ -2510,7 +2510,7 @@ function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string
   const targetPipelineRun = options.pipelineRun ?? (options.sourceCommit === undefined ? undefined : v02PipelineRunName(options.sourceCommit));
   const prefixes = pipelinePrefixesForLane(options.lane);
   const now = Date.now();
-  const candidates = statusText(result)
+  const terminalRuns = statusText(result)
     .split(/\r?\n/u)
     .map((line) => line.trim())
     .filter(Boolean)
@@ -2521,10 +2521,24 @@ function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string
       return { name, createdAt, ageMinutes, status: status || null, reason: reason || null };
     })
     .filter((item) => item.name.length > 0 && prefixes.some((prefix) => item.name.startsWith(prefix)))
-    .filter((item) => targetPipelineRun === undefined || item.name === targetPipelineRun)
     .filter((item) => item.status === "True" || item.status === "False")
+    .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)));
+  const protectedLatestByPrefix = new Map<string, string>();
+  for (const prefix of prefixes) {
+    const latest = terminalRuns
+      .filter((item) => item.name.startsWith(prefix))
+      .sort((a, b) => String(b.createdAt).localeCompare(String(a.createdAt)))[0];
+    if (latest !== undefined) protectedLatestByPrefix.set(prefix, latest.name);
+  }
+  const candidates = terminalRuns
+    .filter((item) => targetPipelineRun === undefined || item.name === targetPipelineRun)
     .filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
-    .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
+    .map((item) => {
+      const protectedLatest = targetPipelineRun === undefined && [...protectedLatestByPrefix.values()].includes(item.name);
+      return protectedLatest
+        ? { ...item, selected: false, selectedReason: "protected-latest-pipelinerun" }
+        : item;
+    })
     .slice(0, options.limit);
   if (targetPipelineRun !== undefined && candidates.length === 0) {
     return [{