def pv_host_path(pv): spec = (pv or {}).get("spec") or {} host_path = (spec.get("hostPath") or {}).get("path") if isinstance(host_path, str) and host_path: return host_path local_path = (spec.get("local") or {}).get("path") if isinstance(local_path, str) and local_path: return local_path return None def pvc_owner_group(namespace, owner): owner = str(owner or "") if namespace == "agentrun-ci": return "agentrun" if namespace == "hwlab-ci": if owner.startswith("agentrun-"): return "agentrun" return "hwlab" if namespace.startswith("hwlab-"): return "hwlab-runtime" return "other" def parse_k8s_quantity(value): if value is None: return None raw = str(value).strip() match = re.match(r"^([0-9]+(?:\.[0-9]+)?)(Ki|Mi|Gi|Ti|K|M|G|T)?$", raw) if not match: return None multiplier = { None: 1, "K": 1000, "M": 1000**2, "G": 1000**3, "T": 1000**4, "Ki": 1024, "Mi": 1024**2, "Gi": 1024**3, "Ti": 1024**4, }.get(match.group(2), 1) return int(float(match.group(1)) * multiplier) def metadata_owner(meta): refs = meta.get("ownerReferences") or [] if refs: first = refs[0] or {} return first.get("kind"), first.get("name"), [{"kind": item.get("kind"), "name": item.get("name")} for item in refs[:5]] labels = meta.get("labels") or {} annotations = meta.get("annotations") or {} for key in [ "tekton.dev/pipelineRun", "tekton.dev/taskRun", "agentrun.unidesk/run-id", "hwlab.unidesk/run-id", "app.kubernetes.io/instance", ]: value = labels.get(key) or annotations.get(key) if value: return "Label", value, [] return None, None, [] def ci_storage_snapshot(): namespaces = set(config_list(PVC_CONFIG, "namespaces", ["hwlab-ci", "agentrun-ci"])) candidate_namespaces = set(config_list(PVC_CONFIG, "candidateNamespaces", [])) hwlab_node = config_str(PVC_CONFIG, "hwlabNode", PROVIDER_ID) hwlab_lane = config_str(PVC_CONFIG, "hwlabLane", "v03") agentrun_node = config_str(PVC_CONFIG, "agentrunNode", PROVIDER_ID) agentrun_lane = config_str(PVC_CONFIG, "agentrunLane", "v02") limit = config_int(PVC_CONFIG, "limit", int(OPTIONS.get("limit") or 50), minimum=1, maximum=5000) pv_data = kubectl_json(["get", "pv"], 30) or {} pvc_data = kubectl_json(["get", "pvc", "-A"], 30) or {} pod_data = kubectl_json(["get", "pod", "-A"], 30) or {} pvs = {} for pv in pv_data.get("items") or []: meta = pv.get("metadata") or {} name = meta.get("name") if name: pvs[name] = pv mounts = {} for pod in pod_data.get("items") or []: meta = pod.get("metadata") or {} ns = str(meta.get("namespace") or "") pod_name = str(meta.get("name") or "") phase = str(((pod.get("status") or {}).get("phase")) or "") if phase in set(["Succeeded", "Failed"]): continue spec = pod.get("spec") or {} for vol in spec.get("volumes") or []: claim = (vol.get("persistentVolumeClaim") or {}).get("claimName") if claim: mounts.setdefault((ns, claim), []).append(pod_name) rows = [] for pvc in pvc_data.get("items") or []: meta = pvc.get("metadata") or {} spec = pvc.get("spec") or {} status = pvc.get("status") or {} ns = str(meta.get("namespace") or "") name = str(meta.get("name") or "") if ns not in namespaces: continue volume = str(spec.get("volumeName") or "") pv = pvs.get(volume) or {} pv_spec = pv.get("spec") or {} pv_meta = pv.get("metadata") or {} owner_kind, owner_name, owner_refs = metadata_owner(meta) requested = parse_k8s_quantity((((spec.get("resources") or {}).get("requests") or {}).get("storage"))) host_path = pv_host_path(pv) active = sorted(mounts.get((ns, name), [])) estimated = du_size(host_path, 8) if host_path else None candidate_reasons = [] if not active: candidate_reasons.append("no-active-mount-observed") if status.get("phase") != "Bound": candidate_reasons.append("pvc-not-bound") if (pv.get("status") or {}).get("phase") == "Released": candidate_reasons.append("pv-released") review_candidate = ns in candidate_namespaces and len(candidate_reasons) > 0 rows.append({ "namespace": ns, "pvc": name, "volume": volume or None, "phase": status.get("phase"), "pvPhase": (pv.get("status") or {}).get("phase"), "ownerKind": owner_kind, "owner": owner_name, "ownerRefs": owner_refs, "ownerGroup": pvc_owner_group(ns, owner_name), "storageClass": spec.get("storageClassName") or pv_spec.get("storageClassName"), "reclaimPolicy": pv_spec.get("persistentVolumeReclaimPolicy"), "requestedBytes": requested, "requestedHuman": fmt_bytes(requested or 0), "hostPath": host_path, "pvCreatedAt": (pv_meta.get("creationTimestamp") if isinstance(pv_meta, dict) else None), "pvcCreatedAt": meta.get("creationTimestamp"), "activeMountPods": active, "estimatedBytes": estimated, "estimatedHuman": fmt_bytes(estimated or 0), "reviewCandidate": review_candidate, "reviewReasons": candidate_reasons, "dryRunOnly": True, }) rows.sort(key=lambda item: safe_int(item.get("estimatedBytes")), reverse=True) by_namespace = {} by_owner_group = {} for row in rows: for bucket, key in [(by_namespace, row.get("namespace") or "unknown"), (by_owner_group, row.get("ownerGroup") or "unknown")]: current = bucket.setdefault(key, {"count": 0, "estimatedBytes": 0, "activeMountCount": 0}) current["count"] += 1 current["estimatedBytes"] += safe_int(row.get("estimatedBytes")) current["activeMountCount"] += len(row.get("activeMountPods") or []) current["estimatedHuman"] = fmt_bytes(current["estimatedBytes"]) review_candidates = [row for row in rows if row.get("reviewCandidate")] return { "scope": "YAML-configured PVC namespaces", "configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.pvcAttribution" % PROVIDER_ID, "namespaces": sorted(namespaces), "candidateNamespaces": sorted(candidate_namespaces), "pvcCount": len(rows), "reviewCandidateCount": len(review_candidates), "estimatedBytes": sum(safe_int(row.get("estimatedBytes")) for row in rows), "estimatedHuman": fmt_bytes(sum(safe_int(row.get("estimatedBytes")) for row in rows)), "requestedBytes": sum(safe_int(row.get("requestedBytes")) for row in rows), "requestedHuman": fmt_bytes(sum(safe_int(row.get("requestedBytes")) for row in rows)), "byNamespace": by_namespace, "byOwnerGroup": by_owner_group, "topPvcs": rows[:limit], "reviewCandidates": review_candidates[:limit], "handoff": { "hwlab": { "dryRun": "bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node %s --lane %s --min-age-minutes 30 --limit 200 --dry-run" % (hwlab_node, hwlab_lane), "releasedPvs": "bun scripts/cli.ts hwlab nodes control-plane cleanup-released-pvs --node %s --lane %s --limit 200 --dry-run" % (hwlab_node, hwlab_lane), }, "agentrun": { "dryRun": "bun scripts/cli.ts agentrun control-plane cleanup-runs --node %s --lane %s --min-age-minutes 30 --limit 200 --dry-run" % (agentrun_node, agentrun_lane), "releasedPvs": "bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node %s --lane %s --limit 200 --dry-run" % (agentrun_node, agentrun_lane), }, }, "policy": "analysis-only; remote GC never deletes PVC/PV/local-path data and only hands off to owner-aware retention commands", } def compact_pvc_row(row): return { "namespace": row.get("namespace"), "pvc": row.get("pvc"), "phase": row.get("phase"), "pvPhase": row.get("pvPhase"), "ownerKind": row.get("ownerKind"), "owner": row.get("owner"), "ownerGroup": row.get("ownerGroup"), "estimatedBytes": row.get("estimatedBytes"), "estimatedHuman": row.get("estimatedHuman"), "activeMountCount": len(row.get("activeMountPods") or []), "reviewCandidate": row.get("reviewCandidate"), "reviewReasons": row.get("reviewReasons"), } def compact_pvc_attribution(payload): if bool(OPTIONS.get("full")): return payload top = payload.get("topPvcs") or [] review = payload.get("reviewCandidates") or [] compact_top = [compact_pvc_row(row) for row in top[:8] if isinstance(row, dict)] return { "configSource": payload.get("configSource"), "candidateNamespaces": payload.get("candidateNamespaces"), "pvcCount": payload.get("pvcCount"), "reviewCandidateCount": payload.get("reviewCandidateCount"), "estimatedBytes": payload.get("estimatedBytes"), "estimatedHuman": payload.get("estimatedHuman"), "byNamespace": payload.get("byNamespace"), "byOwnerGroup": payload.get("byOwnerGroup"), "topPvcs": compact_top, "reviewCandidates": [compact_pvc_row(row) for row in review[:2] if isinstance(row, dict)], "handoff": payload.get("handoff"), "compacted": True, "fullDisclosure": "rerun with --full for hostPath, creation timestamps and complete row details", } def compact_ci_storage_summary(payload): return { "scope": payload.get("scope"), "configSource": payload.get("configSource"), "pvcCount": payload.get("pvcCount"), "reviewCandidateCount": payload.get("reviewCandidateCount"), "estimatedBytes": payload.get("estimatedBytes"), "estimatedHuman": payload.get("estimatedHuman"), "requestedBytes": payload.get("requestedBytes"), "requestedHuman": payload.get("requestedHuman"), "compacted": True, "fullDisclosure": "use pvcAttribution or --full for row-level details", } def local_path_storage_root(): root = config_str(LOCAL_PATH_CONFIG, "root", "") if not root: return "" return os.path.realpath(os.path.abspath(root)) def local_path_orphan_prefixes(): return config_list(LOCAL_PATH_CONFIG, "orphanDirPrefixes", []) def is_direct_local_path_child(root, path): resolved = os.path.realpath(os.path.abspath(path)) return os.path.dirname(resolved) == root and resolved.startswith(root.rstrip("/") + "/") def local_path_referenced_paths(root): pv_data = kubectl_json(["get", "pv"], 30) or {} referenced = set() for pv in pv_data.get("items") or []: host_path = pv_host_path(pv) if not host_path: continue resolved = os.path.realpath(os.path.abspath(host_path)) if resolved == root or resolved.startswith(root.rstrip("/") + "/"): referenced.add(resolved) return referenced def assert_local_path_orphan(path, referenced=None): root = local_path_storage_root() if not root: raise RuntimeError("localPathStorage.root is not configured") prefixes = local_path_orphan_prefixes() resolved = os.path.realpath(os.path.abspath(path)) name = os.path.basename(resolved) if not is_direct_local_path_child(root, resolved): raise RuntimeError("refusing to remove local-path orphan outside configured direct storage root: %s" % path) if os.path.islink(path) or not os.path.isdir(resolved): raise RuntimeError("refusing to remove non-directory or symlink local-path orphan: %s" % path) if not prefixes or not any(name.startswith(prefix) for prefix in prefixes): raise RuntimeError("refusing to remove local-path orphan outside YAML prefix allowlist: %s" % path) refs = referenced if referenced is not None else local_path_referenced_paths(root) for ref in refs: if resolved == ref or ref.startswith(resolved.rstrip("/") + "/") or resolved.startswith(ref.rstrip("/") + "/"): raise RuntimeError("refusing to remove local-path path still referenced by PV: %s" % path) if path_has_open_fd(resolved): raise RuntimeError("refusing to remove local-path orphan with open fd/cwd reference: %s" % path) return resolved def local_path_orphan_rows(): if not config_bool(LOCAL_PATH_CONFIG, "enabled", False): return [], {"ok": False, "reason": "local-path-orphan-cleanup-disabled"} root = local_path_storage_root() prefixes = local_path_orphan_prefixes() if not root or not os.path.isdir(root) or os.path.islink(root): return [], {"ok": False, "reason": "local-path-root-unavailable", "root": root} if not prefixes: return [], {"ok": False, "reason": "local-path-prefix-allowlist-empty", "root": root} referenced = local_path_referenced_paths(root) min_age_minutes = config_float(LOCAL_PATH_CONFIG, "orphanMinAgeMinutes", 0.0, minimum=0.0) cutoff = time.time() - min_age_minutes * 60.0 rows = [] protected = [] for name in sorted(os.listdir(root)): path = os.path.join(root, name) resolved = os.path.realpath(os.path.abspath(path)) try: stat = os.lstat(path) except OSError: continue if not os.path.isdir(path) or os.path.islink(path) or not any(name.startswith(prefix) for prefix in prefixes): continue row = {"path": resolved, "name": name, "sizeBytes": 0, "estimatedReclaimBytes": 0} if not is_direct_local_path_child(root, resolved): protected.append({**row, "reason": "not-direct-child"}) continue if stat.st_mtime >= cutoff: protected.append({**row, "reason": "younger-than-min-age"}) continue referenced_by = [ref for ref in referenced if resolved == ref or ref.startswith(resolved.rstrip("/") + "/") or resolved.startswith(ref.rstrip("/") + "/")] if referenced_by: protected.append({**row, "reason": "pv-referenced", "referencedCount": len(referenced_by)}) continue if path_has_open_fd(resolved): protected.append({**row, "reason": "open-fd"}) continue size = du_size(resolved, 10) or path_size(resolved) rows.append({**row, "sizeBytes": size, "estimatedReclaimBytes": size}) rows.sort(key=lambda item: safe_int(item.get("estimatedReclaimBytes")), reverse=True) return rows, { "ok": True, "root": root, "prefixes": prefixes, "referencedPathCount": len(referenced), "protectedCount": len(protected), "protectedPreview": protected[:8], "minAgeMinutes": min_age_minutes, } def local_path_orphan_candidate(): rows, meta = local_path_orphan_rows() if not meta.get("ok"): return { "id": "k3s-local-path-orphans:unavailable", "kind": "k3s-local-path-orphans-unavailable", "risk": "blocked", "description": "K3s local-path orphan cleanup is unavailable or disabled by YAML", "estimatedReclaimBytes": 0, "diagnostic": meta, } limit = int(OPTIONS.get("limit") or 50) selected = rows[:limit] estimated = sum(safe_int(row.get("estimatedReclaimBytes")) for row in selected) if estimated <= 0: return None return { "id": "k3s-local-path-orphans:delete", "kind": "k3s-local-path-orphans-delete", "risk": "medium", "description": "Delete YAML-allowlisted k3s local-path storage directories that no PV references and no process has open", "path": meta.get("root"), "sizeBytes": estimated, "estimatedReclaimBytes": estimated, "orphanCount": len(rows), "selectedOrphanCount": len(selected), "protectedCount": meta.get("protectedCount"), "referencedPathCount": meta.get("referencedPathCount"), "selectedPreview": [{"name": row.get("name"), "path": row.get("path"), "estimatedReclaimBytes": row.get("estimatedReclaimBytes")} for row in selected[:8]], "protectedPreview": meta.get("protectedPreview"), "action": {"op": "rm-recursive", "allowlist": "yaml-local-path-orphan", "root": meta.get("root"), "limit": limit}, } def execute_local_path_orphan_cleanup(): rows, meta = local_path_orphan_rows() if not meta.get("ok"): raise RuntimeError("local-path orphan cleanup unavailable: %s" % meta.get("reason")) limit = int(OPTIONS.get("limit") or 50) selected = rows[:limit] referenced = local_path_referenced_paths(local_path_storage_root()) reclaimed = 0 deleted = [] for row in selected: path = assert_local_path_orphan(row.get("path"), referenced) before = du_size(path, 10) or path_size(path) shutil.rmtree(path, ignore_errors=True) reclaimed += before deleted.append({"name": row.get("name"), "path": path, "reclaimedBytes": before}) return { "reclaimedBytes": reclaimed, "deletedOrphanCount": len(deleted), "deletedPreview": deleted[:12], "root": meta.get("root"), "protectedCount": meta.get("protectedCount"), }