Codex
384 lines
17 KiB
Python
384 lines
17 KiB
Python
def pv_host_path(pv):
|
|
spec = (pv or {}).get("spec") or {}
|
|
host_path = (spec.get("hostPath") or {}).get("path")
|
|
if isinstance(host_path, str) and host_path:
|
|
return host_path
|
|
local_path = (spec.get("local") or {}).get("path")
|
|
if isinstance(local_path, str) and local_path:
|
|
return local_path
|
|
return None
|
|
|
|
def pvc_owner_group(namespace, owner):
|
|
owner = str(owner or "")
|
|
if namespace == "agentrun-ci":
|
|
return "agentrun"
|
|
if namespace == "hwlab-ci":
|
|
if owner.startswith("agentrun-"):
|
|
return "agentrun"
|
|
return "hwlab"
|
|
if namespace.startswith("hwlab-"):
|
|
return "hwlab-runtime"
|
|
return "other"
|
|
|
|
def parse_k8s_quantity(value):
|
|
if value is None:
|
|
return None
|
|
raw = str(value).strip()
|
|
match = re.match(r"^([0-9]+(?:\.[0-9]+)?)(Ki|Mi|Gi|Ti|K|M|G|T)?$", raw)
|
|
if not match:
|
|
return None
|
|
multiplier = {
|
|
None: 1,
|
|
"K": 1000,
|
|
"M": 1000**2,
|
|
"G": 1000**3,
|
|
"T": 1000**4,
|
|
"Ki": 1024,
|
|
"Mi": 1024**2,
|
|
"Gi": 1024**3,
|
|
"Ti": 1024**4,
|
|
}.get(match.group(2), 1)
|
|
return int(float(match.group(1)) * multiplier)
|
|
|
|
def metadata_owner(meta):
|
|
refs = meta.get("ownerReferences") or []
|
|
if refs:
|
|
first = refs[0] or {}
|
|
return first.get("kind"), first.get("name"), [{"kind": item.get("kind"), "name": item.get("name")} for item in refs[:5]]
|
|
labels = meta.get("labels") or {}
|
|
annotations = meta.get("annotations") or {}
|
|
for key in [
|
|
"tekton.dev/pipelineRun",
|
|
"tekton.dev/taskRun",
|
|
"agentrun.unidesk/run-id",
|
|
"hwlab.unidesk/run-id",
|
|
"app.kubernetes.io/instance",
|
|
]:
|
|
value = labels.get(key) or annotations.get(key)
|
|
if value:
|
|
return "Label", value, []
|
|
return None, None, []
|
|
|
|
def ci_storage_snapshot():
|
|
namespaces = set(config_list(PVC_CONFIG, "namespaces", ["hwlab-ci", "agentrun-ci"]))
|
|
candidate_namespaces = set(config_list(PVC_CONFIG, "candidateNamespaces", []))
|
|
hwlab_node = config_str(PVC_CONFIG, "hwlabNode", PROVIDER_ID)
|
|
hwlab_lane = config_str(PVC_CONFIG, "hwlabLane", "v03")
|
|
agentrun_node = config_str(PVC_CONFIG, "agentrunNode", PROVIDER_ID)
|
|
agentrun_lane = config_str(PVC_CONFIG, "agentrunLane", "v02")
|
|
limit = config_int(PVC_CONFIG, "limit", int(OPTIONS.get("limit") or 50), minimum=1, maximum=5000)
|
|
pv_data = kubectl_json(["get", "pv"], 30) or {}
|
|
pvc_data = kubectl_json(["get", "pvc", "-A"], 30) or {}
|
|
pod_data = kubectl_json(["get", "pod", "-A"], 30) or {}
|
|
pvs = {}
|
|
for pv in pv_data.get("items") or []:
|
|
meta = pv.get("metadata") or {}
|
|
name = meta.get("name")
|
|
if name:
|
|
pvs[name] = pv
|
|
mounts = {}
|
|
for pod in pod_data.get("items") or []:
|
|
meta = pod.get("metadata") or {}
|
|
ns = str(meta.get("namespace") or "")
|
|
pod_name = str(meta.get("name") or "")
|
|
phase = str(((pod.get("status") or {}).get("phase")) or "")
|
|
if phase in set(["Succeeded", "Failed"]):
|
|
continue
|
|
spec = pod.get("spec") or {}
|
|
for vol in spec.get("volumes") or []:
|
|
claim = (vol.get("persistentVolumeClaim") or {}).get("claimName")
|
|
if claim:
|
|
mounts.setdefault((ns, claim), []).append(pod_name)
|
|
rows = []
|
|
for pvc in pvc_data.get("items") or []:
|
|
meta = pvc.get("metadata") or {}
|
|
spec = pvc.get("spec") or {}
|
|
status = pvc.get("status") or {}
|
|
ns = str(meta.get("namespace") or "")
|
|
name = str(meta.get("name") or "")
|
|
if ns not in namespaces:
|
|
continue
|
|
volume = str(spec.get("volumeName") or "")
|
|
pv = pvs.get(volume) or {}
|
|
pv_spec = pv.get("spec") or {}
|
|
pv_meta = pv.get("metadata") or {}
|
|
owner_kind, owner_name, owner_refs = metadata_owner(meta)
|
|
requested = parse_k8s_quantity((((spec.get("resources") or {}).get("requests") or {}).get("storage")))
|
|
host_path = pv_host_path(pv)
|
|
active = sorted(mounts.get((ns, name), []))
|
|
estimated = du_size(host_path, 8) if host_path else None
|
|
candidate_reasons = []
|
|
if not active:
|
|
candidate_reasons.append("no-active-mount-observed")
|
|
if status.get("phase") != "Bound":
|
|
candidate_reasons.append("pvc-not-bound")
|
|
if (pv.get("status") or {}).get("phase") == "Released":
|
|
candidate_reasons.append("pv-released")
|
|
review_candidate = ns in candidate_namespaces and len(candidate_reasons) > 0
|
|
rows.append({
|
|
"namespace": ns,
|
|
"pvc": name,
|
|
"volume": volume or None,
|
|
"phase": status.get("phase"),
|
|
"pvPhase": (pv.get("status") or {}).get("phase"),
|
|
"ownerKind": owner_kind,
|
|
"owner": owner_name,
|
|
"ownerRefs": owner_refs,
|
|
"ownerGroup": pvc_owner_group(ns, owner_name),
|
|
"storageClass": spec.get("storageClassName") or pv_spec.get("storageClassName"),
|
|
"reclaimPolicy": pv_spec.get("persistentVolumeReclaimPolicy"),
|
|
"requestedBytes": requested,
|
|
"requestedHuman": fmt_bytes(requested or 0),
|
|
"hostPath": host_path,
|
|
"pvCreatedAt": (pv_meta.get("creationTimestamp") if isinstance(pv_meta, dict) else None),
|
|
"pvcCreatedAt": meta.get("creationTimestamp"),
|
|
"activeMountPods": active,
|
|
"estimatedBytes": estimated,
|
|
"estimatedHuman": fmt_bytes(estimated or 0),
|
|
"reviewCandidate": review_candidate,
|
|
"reviewReasons": candidate_reasons,
|
|
"dryRunOnly": True,
|
|
})
|
|
rows.sort(key=lambda item: safe_int(item.get("estimatedBytes")), reverse=True)
|
|
by_namespace = {}
|
|
by_owner_group = {}
|
|
for row in rows:
|
|
for bucket, key in [(by_namespace, row.get("namespace") or "unknown"), (by_owner_group, row.get("ownerGroup") or "unknown")]:
|
|
current = bucket.setdefault(key, {"count": 0, "estimatedBytes": 0, "activeMountCount": 0})
|
|
current["count"] += 1
|
|
current["estimatedBytes"] += safe_int(row.get("estimatedBytes"))
|
|
current["activeMountCount"] += len(row.get("activeMountPods") or [])
|
|
current["estimatedHuman"] = fmt_bytes(current["estimatedBytes"])
|
|
review_candidates = [row for row in rows if row.get("reviewCandidate")]
|
|
return {
|
|
"scope": "YAML-configured PVC namespaces",
|
|
"configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.pvcAttribution" % PROVIDER_ID,
|
|
"namespaces": sorted(namespaces),
|
|
"candidateNamespaces": sorted(candidate_namespaces),
|
|
"pvcCount": len(rows),
|
|
"reviewCandidateCount": len(review_candidates),
|
|
"estimatedBytes": sum(safe_int(row.get("estimatedBytes")) for row in rows),
|
|
"estimatedHuman": fmt_bytes(sum(safe_int(row.get("estimatedBytes")) for row in rows)),
|
|
"requestedBytes": sum(safe_int(row.get("requestedBytes")) for row in rows),
|
|
"requestedHuman": fmt_bytes(sum(safe_int(row.get("requestedBytes")) for row in rows)),
|
|
"byNamespace": by_namespace,
|
|
"byOwnerGroup": by_owner_group,
|
|
"topPvcs": rows[:limit],
|
|
"reviewCandidates": review_candidates[:limit],
|
|
"handoff": {
|
|
"hwlab": {
|
|
"dryRun": "bun scripts/cli.ts hwlab nodes control-plane cleanup-runs --node %s --lane %s --min-age-minutes 30 --limit 200 --dry-run" % (hwlab_node, hwlab_lane),
|
|
"releasedPvs": "bun scripts/cli.ts hwlab nodes control-plane cleanup-released-pvs --node %s --lane %s --limit 200 --dry-run" % (hwlab_node, hwlab_lane),
|
|
},
|
|
"agentrun": {
|
|
"dryRun": "bun scripts/cli.ts agentrun control-plane cleanup-runs --node %s --lane %s --min-age-minutes 30 --limit 200 --dry-run" % (agentrun_node, agentrun_lane),
|
|
"releasedPvs": "bun scripts/cli.ts agentrun control-plane cleanup-released-pvs --node %s --lane %s --limit 200 --dry-run" % (agentrun_node, agentrun_lane),
|
|
},
|
|
},
|
|
"policy": "analysis-only; remote GC never deletes PVC/PV/local-path data and only hands off to owner-aware retention commands",
|
|
}
|
|
|
|
def compact_pvc_row(row):
|
|
return {
|
|
"namespace": row.get("namespace"),
|
|
"pvc": row.get("pvc"),
|
|
"phase": row.get("phase"),
|
|
"pvPhase": row.get("pvPhase"),
|
|
"ownerKind": row.get("ownerKind"),
|
|
"owner": row.get("owner"),
|
|
"ownerGroup": row.get("ownerGroup"),
|
|
"estimatedBytes": row.get("estimatedBytes"),
|
|
"estimatedHuman": row.get("estimatedHuman"),
|
|
"activeMountCount": len(row.get("activeMountPods") or []),
|
|
"reviewCandidate": row.get("reviewCandidate"),
|
|
"reviewReasons": row.get("reviewReasons"),
|
|
}
|
|
|
|
def compact_pvc_attribution(payload):
|
|
if bool(OPTIONS.get("full")):
|
|
return payload
|
|
top = payload.get("topPvcs") or []
|
|
review = payload.get("reviewCandidates") or []
|
|
compact_top = [compact_pvc_row(row) for row in top[:8] if isinstance(row, dict)]
|
|
return {
|
|
"configSource": payload.get("configSource"),
|
|
"candidateNamespaces": payload.get("candidateNamespaces"),
|
|
"pvcCount": payload.get("pvcCount"),
|
|
"reviewCandidateCount": payload.get("reviewCandidateCount"),
|
|
"estimatedBytes": payload.get("estimatedBytes"),
|
|
"estimatedHuman": payload.get("estimatedHuman"),
|
|
"byNamespace": payload.get("byNamespace"),
|
|
"byOwnerGroup": payload.get("byOwnerGroup"),
|
|
"topPvcs": compact_top,
|
|
"reviewCandidates": [compact_pvc_row(row) for row in review[:2] if isinstance(row, dict)],
|
|
"handoff": payload.get("handoff"),
|
|
"compacted": True,
|
|
"fullDisclosure": "rerun with --full for hostPath, creation timestamps and complete row details",
|
|
}
|
|
|
|
def compact_ci_storage_summary(payload):
|
|
return {
|
|
"scope": payload.get("scope"),
|
|
"configSource": payload.get("configSource"),
|
|
"pvcCount": payload.get("pvcCount"),
|
|
"reviewCandidateCount": payload.get("reviewCandidateCount"),
|
|
"estimatedBytes": payload.get("estimatedBytes"),
|
|
"estimatedHuman": payload.get("estimatedHuman"),
|
|
"requestedBytes": payload.get("requestedBytes"),
|
|
"requestedHuman": payload.get("requestedHuman"),
|
|
"compacted": True,
|
|
"fullDisclosure": "use pvcAttribution or --full for row-level details",
|
|
}
|
|
|
|
def local_path_storage_root():
|
|
root = config_str(LOCAL_PATH_CONFIG, "root", "")
|
|
if not root:
|
|
return ""
|
|
return os.path.realpath(os.path.abspath(root))
|
|
|
|
def local_path_orphan_prefixes():
|
|
return config_list(LOCAL_PATH_CONFIG, "orphanDirPrefixes", [])
|
|
|
|
def is_direct_local_path_child(root, path):
|
|
resolved = os.path.realpath(os.path.abspath(path))
|
|
return os.path.dirname(resolved) == root and resolved.startswith(root.rstrip("/") + "/")
|
|
|
|
def local_path_referenced_paths(root):
|
|
pv_data = kubectl_json(["get", "pv"], 30) or {}
|
|
referenced = set()
|
|
for pv in pv_data.get("items") or []:
|
|
host_path = pv_host_path(pv)
|
|
if not host_path:
|
|
continue
|
|
resolved = os.path.realpath(os.path.abspath(host_path))
|
|
if resolved == root or resolved.startswith(root.rstrip("/") + "/"):
|
|
referenced.add(resolved)
|
|
return referenced
|
|
|
|
def assert_local_path_orphan(path, referenced=None):
|
|
root = local_path_storage_root()
|
|
if not root:
|
|
raise RuntimeError("localPathStorage.root is not configured")
|
|
prefixes = local_path_orphan_prefixes()
|
|
resolved = os.path.realpath(os.path.abspath(path))
|
|
name = os.path.basename(resolved)
|
|
if not is_direct_local_path_child(root, resolved):
|
|
raise RuntimeError("refusing to remove local-path orphan outside configured direct storage root: %s" % path)
|
|
if os.path.islink(path) or not os.path.isdir(resolved):
|
|
raise RuntimeError("refusing to remove non-directory or symlink local-path orphan: %s" % path)
|
|
if not prefixes or not any(name.startswith(prefix) for prefix in prefixes):
|
|
raise RuntimeError("refusing to remove local-path orphan outside YAML prefix allowlist: %s" % path)
|
|
refs = referenced if referenced is not None else local_path_referenced_paths(root)
|
|
for ref in refs:
|
|
if resolved == ref or ref.startswith(resolved.rstrip("/") + "/") or resolved.startswith(ref.rstrip("/") + "/"):
|
|
raise RuntimeError("refusing to remove local-path path still referenced by PV: %s" % path)
|
|
if path_has_open_fd(resolved):
|
|
raise RuntimeError("refusing to remove local-path orphan with open fd/cwd reference: %s" % path)
|
|
return resolved
|
|
|
|
def local_path_orphan_rows():
|
|
if not config_bool(LOCAL_PATH_CONFIG, "enabled", False):
|
|
return [], {"ok": False, "reason": "local-path-orphan-cleanup-disabled"}
|
|
root = local_path_storage_root()
|
|
prefixes = local_path_orphan_prefixes()
|
|
if not root or not os.path.isdir(root) or os.path.islink(root):
|
|
return [], {"ok": False, "reason": "local-path-root-unavailable", "root": root}
|
|
if not prefixes:
|
|
return [], {"ok": False, "reason": "local-path-prefix-allowlist-empty", "root": root}
|
|
referenced = local_path_referenced_paths(root)
|
|
min_age_minutes = config_float(LOCAL_PATH_CONFIG, "orphanMinAgeMinutes", 0.0, minimum=0.0)
|
|
cutoff = time.time() - min_age_minutes * 60.0
|
|
rows = []
|
|
protected = []
|
|
for name in sorted(os.listdir(root)):
|
|
path = os.path.join(root, name)
|
|
resolved = os.path.realpath(os.path.abspath(path))
|
|
try:
|
|
stat = os.lstat(path)
|
|
except OSError:
|
|
continue
|
|
if not os.path.isdir(path) or os.path.islink(path) or not any(name.startswith(prefix) for prefix in prefixes):
|
|
continue
|
|
row = {"path": resolved, "name": name, "sizeBytes": 0, "estimatedReclaimBytes": 0}
|
|
if not is_direct_local_path_child(root, resolved):
|
|
protected.append({**row, "reason": "not-direct-child"})
|
|
continue
|
|
if stat.st_mtime >= cutoff:
|
|
protected.append({**row, "reason": "younger-than-min-age"})
|
|
continue
|
|
referenced_by = [ref for ref in referenced if resolved == ref or ref.startswith(resolved.rstrip("/") + "/") or resolved.startswith(ref.rstrip("/") + "/")]
|
|
if referenced_by:
|
|
protected.append({**row, "reason": "pv-referenced", "referencedCount": len(referenced_by)})
|
|
continue
|
|
if path_has_open_fd(resolved):
|
|
protected.append({**row, "reason": "open-fd"})
|
|
continue
|
|
size = du_size(resolved, 10) or path_size(resolved)
|
|
rows.append({**row, "sizeBytes": size, "estimatedReclaimBytes": size})
|
|
rows.sort(key=lambda item: safe_int(item.get("estimatedReclaimBytes")), reverse=True)
|
|
return rows, {
|
|
"ok": True,
|
|
"root": root,
|
|
"prefixes": prefixes,
|
|
"referencedPathCount": len(referenced),
|
|
"protectedCount": len(protected),
|
|
"protectedPreview": protected[:8],
|
|
"minAgeMinutes": min_age_minutes,
|
|
}
|
|
|
|
def local_path_orphan_candidate():
|
|
rows, meta = local_path_orphan_rows()
|
|
if not meta.get("ok"):
|
|
return {
|
|
"id": "k3s-local-path-orphans:unavailable",
|
|
"kind": "k3s-local-path-orphans-unavailable",
|
|
"risk": "blocked",
|
|
"description": "K3s local-path orphan cleanup is unavailable or disabled by YAML",
|
|
"estimatedReclaimBytes": 0,
|
|
"diagnostic": meta,
|
|
}
|
|
limit = int(OPTIONS.get("limit") or 50)
|
|
selected = rows[:limit]
|
|
estimated = sum(safe_int(row.get("estimatedReclaimBytes")) for row in selected)
|
|
if estimated <= 0:
|
|
return None
|
|
return {
|
|
"id": "k3s-local-path-orphans:delete",
|
|
"kind": "k3s-local-path-orphans-delete",
|
|
"risk": "medium",
|
|
"description": "Delete YAML-allowlisted k3s local-path storage directories that no PV references and no process has open",
|
|
"path": meta.get("root"),
|
|
"sizeBytes": estimated,
|
|
"estimatedReclaimBytes": estimated,
|
|
"orphanCount": len(rows),
|
|
"selectedOrphanCount": len(selected),
|
|
"protectedCount": meta.get("protectedCount"),
|
|
"referencedPathCount": meta.get("referencedPathCount"),
|
|
"selectedPreview": [{"name": row.get("name"), "path": row.get("path"), "estimatedReclaimBytes": row.get("estimatedReclaimBytes")} for row in selected[:8]],
|
|
"protectedPreview": meta.get("protectedPreview"),
|
|
"action": {"op": "rm-recursive", "allowlist": "yaml-local-path-orphan", "root": meta.get("root"), "limit": limit},
|
|
}
|
|
|
|
def execute_local_path_orphan_cleanup():
|
|
rows, meta = local_path_orphan_rows()
|
|
if not meta.get("ok"):
|
|
raise RuntimeError("local-path orphan cleanup unavailable: %s" % meta.get("reason"))
|
|
limit = int(OPTIONS.get("limit") or 50)
|
|
selected = rows[:limit]
|
|
referenced = local_path_referenced_paths(local_path_storage_root())
|
|
reclaimed = 0
|
|
deleted = []
|
|
for row in selected:
|
|
path = assert_local_path_orphan(row.get("path"), referenced)
|
|
before = du_size(path, 10) or path_size(path)
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
reclaimed += before
|
|
deleted.append({"name": row.get("name"), "path": path, "reclaimedBytes": before})
|
|
return {
|
|
"reclaimedBytes": reclaimed,
|
|
"deletedOrphanCount": len(deleted),
|
|
"deletedPreview": deleted[:12],
|
|
"root": meta.get("root"),
|
|
"protectedCount": meta.get("protectedCount"),
|
|
}
|