fix: add JD01 GC retention controls

2026-07-05 04:18:22 +00:00
parent e956a0ec2a
commit ab3566435c
18 changed files with 2205 additions and 1240 deletions
@@ -0,0 +1,102 @@
+import { execFileSync, spawnSync } from "node:child_process";
+
+function runJson(args) {
+  return JSON.parse(execFileSync("kubectl", args, { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }));
+}
+
+function duBytes(path) {
+  if (!path || !path.startsWith("/var/lib/rancher/k3s/storage/")) return null;
+  const result = spawnSync("du", ["-sb", path], { encoding: "utf8", timeout: 8000 });
+  if (result.status !== 0) return null;
+  const value = Number(result.stdout.trim().split(/\s+/u)[0]);
+  return Number.isFinite(value) ? value : null;
+}
+
+const namespace = process.env.NAMESPACE;
+const confirm = process.env.CONFIRM === "true";
+const enabled = process.env.ENABLED === "true";
+const limit = Math.max(1, Math.min(Number(process.env.LIMIT || "100"), 1000));
+const prefixes = JSON.parse(Buffer.from(process.env.PREFIXES_JSON_B64 || "W10=", "base64").toString("utf8"));
+
+if (!enabled) {
+  console.log(JSON.stringify({ ok: false, error: "session-pvc-retention-disabled", selectedPvcCount: 0, mutation: false }));
+  process.exit(0);
+}
+if (!namespace || !Array.isArray(prefixes) || prefixes.length === 0) throw new Error("session PVC cleanup requires namespace and YAML prefixes");
+
+const pvData = runJson(["get", "pv", "-o", "json"]);
+const pvcData = runJson(["-n", namespace, "get", "pvc", "-o", "json"]);
+const podData = runJson(["-n", namespace, "get", "pod", "-o", "json"]);
+const pvs = new Map((pvData.items || []).map((pv) => [pv.metadata?.name, pv]));
+const activeClaims = new Map();
+for (const pod of podData.items || []) {
+  const phase = pod.status?.phase;
+  if (phase === "Succeeded" || phase === "Failed") continue;
+  for (const volume of pod.spec?.volumes || []) {
+    const claim = volume.persistentVolumeClaim?.claimName;
+    if (!claim) continue;
+    const list = activeClaims.get(claim) || [];
+    list.push(pod.metadata?.name);
+    activeClaims.set(claim, list);
+  }
+}
+
+const candidates = [];
+const protectedRows = [];
+for (const pvc of pvcData.items || []) {
+  const name = pvc.metadata?.name || "";
+  const matchedPrefix = prefixes.find((prefix) => name.startsWith(prefix));
+  if (!matchedPrefix) continue;
+  const activeMountPods = activeClaims.get(name) || [];
+  const pv = pvs.get(pvc.spec?.volumeName);
+  const storageClass = pvc.spec?.storageClassName || pv?.spec?.storageClassName || null;
+  const reclaimPolicy = pv?.spec?.persistentVolumeReclaimPolicy || null;
+  const hostPath = pv?.spec?.hostPath?.path || pv?.spec?.local?.path || null;
+  const row = {
+    namespace,
+    pvc: name,
+    volume: pvc.spec?.volumeName || null,
+    matchedPrefix,
+    phase: pvc.status?.phase || null,
+    pvPhase: pv?.status?.phase || null,
+    storageClass,
+    reclaimPolicy,
+    activeMountCount: activeMountPods.length,
+    activeMountPods: activeMountPods.slice(0, 5),
+    estimatedBytes: duBytes(hostPath),
+  };
+  if (activeMountPods.length > 0 || storageClass !== "local-path" || reclaimPolicy !== "Delete") {
+    protectedRows.push({ ...row, reason: activeMountPods.length > 0 ? "active-mount" : "not-local-path-delete" });
+  } else {
+    candidates.push(row);
+  }
+}
+
+candidates.sort((a, b) => (b.estimatedBytes || 0) - (a.estimatedBytes || 0));
+const selected = candidates.slice(0, limit);
+const result = {
+  ok: true,
+  planKind: "agentrun-session-pvc-retention",
+  namespace,
+  dryRun: !confirm,
+  mutation: confirm,
+  criteria: { prefixes, storageClass: "local-path", reclaimPolicy: "Delete", requireNoActiveMount: true, limit },
+  candidatePvcCount: candidates.length,
+  selectedPvcCount: selected.length,
+  protectedPvcCount: protectedRows.length,
+  estimatedReclaimBytes: selected.reduce((sum, item) => sum + (item.estimatedBytes || 0), 0),
+  selectedPreview: selected.slice(0, 12),
+  protectedPreview: protectedRows.slice(0, 12),
+  deletedPvcCount: 0,
+  valuesPrinted: false,
+};
+
+if (confirm && selected.length > 0) {
+  for (let index = 0; index < selected.length; index += 50) {
+    execFileSync("kubectl", ["-n", namespace, "delete", "pvc", "--wait=false", ...selected.slice(index, index + 50).map((item) => item.pvc)], { encoding: "utf8", maxBuffer: 1024 * 1024 });
+  }
+  result.deletedPvcCount = selected.length;
+  result.deleteMode = "submit-only-wait-false";
+}
+
+console.log(JSON.stringify(result));
@@ -33,7 +33,7 @@ import {
 } from "../agentrun-manifests";
 import { sha256Fingerprint } from "../platform-infra-ops-library";

-import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, ConfirmOptions, GitMirrorOptions, LaneConfirmOptions, RefreshOptions, SecretSyncOptions, StatusOptions } from "./options";
+import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, CleanupSessionPvcsOptions, ConfirmOptions, GitMirrorOptions, LaneConfirmOptions, RefreshOptions, SecretSyncOptions, StatusOptions } from "./options";
 import { agentRunControlPlaneStatusCommand } from "./public-exposure";
 import { applyYamlScript, manifestObjectRef, yamlLaneGitMirrorStatusScript } from "./secrets";
 import { compactAgentRunLaneStatusTarget, compactLaneSecretsStatus } from "./trigger";
@@ -193,6 +193,18 @@ export function parseCleanupReleasedPvOptions(args: string[]): CleanupReleasedPv
  };
 }

+export function parseCleanupSessionPvcsOptions(args: string[]): CleanupSessionPvcsOptions {
+  validateOptions(args, new Set(["--confirm", "--dry-run"]), new Set(["--limit", "--timeout-seconds", "--node", "--lane"]));
+  const base = parseConfirmOptions(args);
+  return {
+    ...base,
+    node: optionValue(args, "--node") ?? null,
+    lane: optionValue(args, "--lane") ?? null,
+    limit: positiveIntegerOption(args, "--limit", 100, 1000),
+    timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
+  };
+}
+
 export function validateOptions(args: string[], booleanOptions: Set<string>, valueOptions: Set<string>): void {
  for (let index = 0; index < args.length; index += 1) {
    const arg = args[index];
@@ -34,7 +34,7 @@ import {
 import { sha256Fingerprint } from "../platform-infra-ops-library";

 import type { AgentRunResourceVerb, AgentRunRestCompatGroup } from "./utils";
-import { controlPlaneApply, controlPlanePlan, parseCleanupReleasedPvOptions, parseCleanupRunnersOptions, parseCleanupRunsOptions, parseConfirmOptions, parseGitMirrorOptions, parseLaneConfirmOptions, parseRefreshOptions, parseSecretSyncOptions, status } from "./control-plane";
+import { controlPlaneApply, controlPlanePlan, parseCleanupReleasedPvOptions, parseCleanupRunnersOptions, parseCleanupRunsOptions, parseCleanupSessionPvcsOptions, parseConfirmOptions, parseGitMirrorOptions, parseLaneConfirmOptions, parseRefreshOptions, parseSecretSyncOptions, status } from "./control-plane";
 import { gitMirrorStatus } from "./git-mirror";
 import { agentRunExplain, isRecord, parseGitMirrorStatusOptions, parseStatusOptions, parseTriggerOptions } from "./options";
 import { renderAgentRunControlPlaneActionSummary, renderAgentRunControlPlanePlanSummary, renderAgentRunControlPlaneStatusSummary } from "./public-exposure";
@@ -43,7 +43,7 @@ import { agentRunGetKindHelp, runAgentRunResourceCommand } from "./resource-acti
 import { runAgentRunRestCompatCommand, runGitMirrorJob, startAsyncAgentRunJob } from "./rest-bridge";
 import { exposeAgentRun, restartYamlLane, secretSync, triggerCurrent } from "./trigger";
 import { unsupported } from "./utils";
-import { cleanupReleasedPvs, cleanupRunners, cleanupRuns, refresh } from "./yaml-lane";
+import { cleanupReleasedPvs, cleanupRunners, cleanupRuns, cleanupSessionPvcs, refresh } from "./yaml-lane";

 export function agentRunHelp(): unknown {
  return {
@@ -143,6 +143,9 @@ export async function runAgentRunCommand(config: UniDeskConfig | null, args: str
      return options.full || options.raw ? result : renderAgentRunControlPlaneActionSummary(result, "AGENTRUN RUNNER CLEANUP");
    }
    if (action === "cleanup-runs") return await cleanupRuns(config, parseCleanupRunsOptions(actionArgs));
+    if (action === "cleanup-session-pvcs") {
+      return await cleanupSessionPvcs(config, parseCleanupSessionPvcsOptions(actionArgs));
+    }
    if (action === "cleanup-released-pvs") return await cleanupReleasedPvs(config, parseCleanupReleasedPvOptions(actionArgs));
  }
  if (group === "git-mirror") {
@@ -271,7 +274,7 @@ export function agentRunHelpText(args: string[]): string {
    return [
      "Usage: bun scripts/cli.ts agentrun control-plane <action> [options]",
      "",
-      "Actions: plan, apply, status, secret-sync, expose, trigger-current, refresh, cleanup-runners, cleanup-runs, cleanup-released-pvs",
+      "Actions: plan, apply, status, secret-sync, expose, trigger-current, refresh, cleanup-runners, cleanup-runs, cleanup-session-pvcs, cleanup-released-pvs",
      "Examples:",
      "  bun scripts/cli.ts agentrun control-plane plan --node D601 --lane v02",
      "  bun scripts/cli.ts agentrun control-plane apply --node D601 --lane v02 --dry-run",
@@ -283,6 +286,7 @@ export function agentRunHelpText(args: string[]): string {
      "  bun scripts/cli.ts agentrun control-plane expose --dry-run",
      "  bun scripts/cli.ts agentrun control-plane trigger-current --dry-run",
      "  bun scripts/cli.ts agentrun control-plane cleanup-runners --node D601 --lane v02 --dry-run",
+      "  bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node JD01 --lane jd01-v02 --dry-run",
      "  bun scripts/cli.ts agentrun control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
    ].join("\n");
  }
@@ -265,6 +265,13 @@ export interface CleanupReleasedPvOptions extends ConfirmOptions {
  timeoutSeconds: number;
 }

+export interface CleanupSessionPvcsOptions extends ConfirmOptions {
+  node: string | null;
+  lane: string | null;
+  limit: number;
+  timeoutSeconds: number;
+}
+
 export interface DisclosureOptions {
  full: boolean;
  raw: boolean;
@@ -35,7 +35,7 @@ import {
 } from "../agentrun-manifests";
 import { sha256Fingerprint } from "../platform-infra-ops-library";

-import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, RefreshOptions } from "./options";
+import type { CleanupReleasedPvOptions, CleanupRunnersOptions, CleanupRunsOptions, CleanupSessionPvcsOptions, RefreshOptions } from "./options";
 import { cleanupReleasedPvsFinalizeNodeScript, cleanupReleasedPvsPlanNodeScript, cleanupRunnersFinalizeNodeScript, cleanupRunsFinalizeNodeScript, cleanupRunsPlanNodeScript, refreshYamlLaneScript } from "./git-mirror";
 import { cleanupRunnersFactsNodeScript, cleanupRunnersPlanNodeScript, collectLaneSecretSources, createYamlLaneJobScript, yamlLaneGitopsPublishJobManifest, yamlLaneGitopsPublishPayloadFromProbe, yamlLaneJobProbeScript } from "./secrets";
 import { capture, captureJsonPayload, compactCapture, progressEvent, shQuote, sleep, stringOrNull } from "./utils";
@@ -204,6 +204,55 @@ export async function cleanupReleasedPvs(config: UniDeskConfig, options: Cleanup
  };
 }

+export async function cleanupSessionPvcs(config: UniDeskConfig, options: CleanupSessionPvcsOptions): Promise<Record<string, unknown>> {
+  const { configPath, spec } = resolveAgentRunLaneTarget(options);
+  const result = await capture(config, spec.nodeKubeRoute, ["sh", "--", cleanupSessionPvcsScript(options, spec)]);
+  const payload = captureJsonPayload(result);
+  const ok = result.exitCode === 0 && payload.ok !== false;
+  const base = {
+    ...payload,
+    ok,
+    command: "agentrun control-plane cleanup-session-pvcs",
+    configPath,
+    target: { node: spec.nodeId, lane: spec.lane, namespace: spec.runtime.namespace },
+    mode: options.dryRun || !options.confirm ? "dry-run" : "confirmed-cleanup",
+    namespace: spec.runtime.namespace,
+    retention: spec.deployment.runner.retention.sessionPvcRetention,
+    probe: result.exitCode === 0 ? undefined : compactCapture(result, { full: true, stdoutTailChars: 3000, stderrTailChars: 3000 }),
+  };
+  if (options.dryRun || !options.confirm) {
+    return { ...base, dryRun: true, mutation: false, next: { confirm: `bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --confirm` } };
+  }
+  return {
+    ...base,
+    dryRun: false,
+    mutation: true,
+    followUp: {
+      dryRun: `bun scripts/cli.ts agentrun control-plane cleanup-session-pvcs --node ${spec.nodeId} --lane ${spec.lane} --limit ${options.limit} --dry-run`,
+      diskPressure: `bun scripts/cli.ts gc remote ${spec.nodeId} status --limit 20`,
+    },
+  };
+}
+
+export function cleanupSessionPvcsScript(options: CleanupSessionPvcsOptions, spec: AgentRunLaneSpec): string {
+  const retention = spec.deployment.runner.retention.sessionPvcRetention;
+  const script = readFileSync(rootPath("scripts/src/agentrun/cleanup-session-pvcs.mjs"), "utf8");
+  return [
+    "set -eu",
+    `namespace=${shQuote(spec.runtime.namespace)}`,
+    `confirm=${options.confirm && !options.dryRun ? "true" : "false"}`,
+    `limit=${String(Math.min(options.limit, retention.maxDeletePerRun))}`,
+    `enabled=${retention.enabled ? "true" : "false"}`,
+    `prefixes_json_b64=${shQuote(Buffer.from(JSON.stringify(retention.prefixes), "utf8").toString("base64"))}`,
+    "tmp_dir=$(mktemp -d)",
+    "trap 'rm -rf \"$tmp_dir\"' EXIT",
+    "cat > \"$tmp_dir/cleanup-session-pvcs.mjs\" <<'NODE'",
+    script,
+    "NODE",
+    "env NAMESPACE=\"$namespace\" CONFIRM=\"$confirm\" LIMIT=\"$limit\" ENABLED=\"$enabled\" PREFIXES_JSON_B64=\"$prefixes_json_b64\" node \"$tmp_dir/cleanup-session-pvcs.mjs\"",
+  ].join("\n");
+}
+
 export function cleanupRunnersScript(options: CleanupRunnersOptions, spec: AgentRunLaneSpec): string {
  const retention = spec.deployment.runner.retention;
  const matchLabelsB64 = Buffer.from(JSON.stringify(retention.selectors.matchLabels), "utf8").toString("base64");