Codex
478 lines
20 KiB
Python
478 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
|
|
def now_iso():
|
|
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def command(args, timeout=60):
|
|
started = time.time()
|
|
try:
|
|
result = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
|
return {
|
|
"exitCode": result.returncode,
|
|
"elapsedMs": int((time.time() - started) * 1000),
|
|
"stdoutTail": result.stdout[-1200:],
|
|
"stderrTail": result.stderr[-1200:],
|
|
}
|
|
except subprocess.TimeoutExpired as exc:
|
|
return {
|
|
"exitCode": 124,
|
|
"elapsedMs": int((time.time() - started) * 1000),
|
|
"stdoutTail": (exc.stdout or "")[-1200:] if isinstance(exc.stdout, str) else "",
|
|
"stderrTail": (exc.stderr or "")[-1200:] if isinstance(exc.stderr, str) else "",
|
|
"timedOut": True,
|
|
}
|
|
|
|
|
|
def load_config(path):
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
return json.load(handle)
|
|
|
|
|
|
def safe_int(value, default=0):
|
|
try:
|
|
return int(value)
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def du_size(path):
|
|
result = command(["du", "-sb", path], 15)
|
|
if result["exitCode"] != 0:
|
|
return 0
|
|
try:
|
|
return int((result["stdoutTail"] or "0").split()[0])
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def path_size(path):
|
|
try:
|
|
stat = os.lstat(path)
|
|
except OSError:
|
|
return 0
|
|
if os.path.isdir(path) and not os.path.islink(path):
|
|
return du_size(path)
|
|
return int(getattr(stat, "st_blocks", 0)) * 512
|
|
|
|
|
|
def path_has_open_fd(path):
|
|
resolved = os.path.realpath(path)
|
|
prefix = resolved.rstrip("/") + "/"
|
|
try:
|
|
pids = [name for name in os.listdir("/proc") if name.isdigit()]
|
|
except OSError:
|
|
return True
|
|
for pid in pids:
|
|
base = os.path.join("/proc", pid)
|
|
for name in ["cwd", "root"]:
|
|
try:
|
|
target = os.path.realpath(os.readlink(os.path.join(base, name)))
|
|
except OSError:
|
|
continue
|
|
if target == resolved or target.startswith(prefix):
|
|
return True
|
|
fd_dir = os.path.join(base, "fd")
|
|
try:
|
|
fds = os.listdir(fd_dir)
|
|
except OSError:
|
|
continue
|
|
for fd in fds:
|
|
try:
|
|
target = os.path.realpath(os.readlink(os.path.join(fd_dir, fd)))
|
|
except OSError:
|
|
continue
|
|
if target == resolved or target.startswith(prefix):
|
|
return True
|
|
return False
|
|
|
|
|
|
def run_json(args, timeout=45):
|
|
started = time.time()
|
|
try:
|
|
process = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
|
except subprocess.TimeoutExpired as exc:
|
|
return None, {
|
|
"exitCode": 124,
|
|
"elapsedMs": int((time.time() - started) * 1000),
|
|
"stdoutTail": (exc.stdout or "")[-1200:] if isinstance(exc.stdout, str) else "",
|
|
"stderrTail": (exc.stderr or "")[-1200:] if isinstance(exc.stderr, str) else "",
|
|
"timedOut": True,
|
|
}
|
|
result = {
|
|
"exitCode": process.returncode,
|
|
"elapsedMs": int((time.time() - started) * 1000),
|
|
"stdoutTail": process.stdout[-1200:],
|
|
"stderrTail": process.stderr[-1200:],
|
|
}
|
|
if process.returncode != 0:
|
|
return None, result
|
|
try:
|
|
return json.loads(process.stdout or "{}"), result
|
|
except Exception:
|
|
return None, result
|
|
|
|
|
|
def write_state(config, payload):
|
|
state_path = config.get("statePath")
|
|
if not state_path:
|
|
return
|
|
os.makedirs(os.path.dirname(state_path), mode=0o755, exist_ok=True)
|
|
tmp = "%s.tmp.%d" % (state_path, os.getpid())
|
|
with open(tmp, "w", encoding="utf-8") as handle:
|
|
json.dump(payload, handle, ensure_ascii=False, sort_keys=True)
|
|
handle.write("\n")
|
|
os.replace(tmp, state_path)
|
|
|
|
|
|
def run_journal(config):
|
|
target = safe_int(config.get("journalTargetBytes"), 268435456)
|
|
result = command(["journalctl", "--vacuum-size=%d" % target], 30)
|
|
return {"id": "journal", "ok": result["exitCode"] == 0, "command": result}
|
|
|
|
|
|
def run_apt(config):
|
|
if not config.get("includeAptCache"):
|
|
return {"id": "apt-cache", "ok": True, "skipped": True}
|
|
before = du_size("/var/cache/apt/archives")
|
|
result = command(["apt-get", "clean"], 30)
|
|
after = du_size("/var/cache/apt/archives")
|
|
return {"id": "apt-cache", "ok": result["exitCode"] == 0, "reclaimedBytes": max(0, before - after), "command": result}
|
|
|
|
|
|
def run_tmp(config):
|
|
prefixes = list(config.get("tmpPrefixAllowlist") or [])
|
|
protected = set(config.get("tmpExactProtect") or [])
|
|
cutoff = time.time() - float(config.get("tmpMinAgeHours") or 24) * 3600.0
|
|
deleted = 0
|
|
reclaimed = 0
|
|
if not os.path.isdir("/tmp"):
|
|
return {"id": "tmp-allowlist", "ok": True, "deletedCount": 0, "reclaimedBytes": 0}
|
|
for name in os.listdir("/tmp"):
|
|
path = os.path.join("/tmp", name)
|
|
if path in protected or not any(name.startswith(prefix) for prefix in prefixes):
|
|
continue
|
|
try:
|
|
stat = os.lstat(path)
|
|
except OSError:
|
|
continue
|
|
if stat.st_mtime >= cutoff or path_has_open_fd(path):
|
|
continue
|
|
before = path_size(path)
|
|
if os.path.isdir(path) and not os.path.islink(path):
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
else:
|
|
try:
|
|
os.unlink(path)
|
|
except FileNotFoundError:
|
|
pass
|
|
deleted += 1
|
|
reclaimed += before
|
|
return {"id": "tmp-allowlist", "ok": True, "deletedCount": deleted, "reclaimedBytes": reclaimed}
|
|
|
|
|
|
def run_tool_caches(config):
|
|
paths = list(config.get("toolCachePaths") or [])
|
|
reclaimed = 0
|
|
deleted = 0
|
|
for path in paths:
|
|
resolved = os.path.abspath(path)
|
|
if resolved != path or os.path.islink(resolved) or resolved in ["/", "/root", "/root/.npm", "/root/.bun"]:
|
|
continue
|
|
before = path_size(resolved)
|
|
if os.path.isdir(resolved):
|
|
shutil.rmtree(resolved, ignore_errors=True)
|
|
elif os.path.exists(resolved):
|
|
try:
|
|
os.unlink(resolved)
|
|
except FileNotFoundError:
|
|
pass
|
|
else:
|
|
continue
|
|
deleted += 1
|
|
reclaimed += before
|
|
return {"id": "tool-caches", "ok": True, "deletedCount": deleted, "reclaimedBytes": reclaimed}
|
|
|
|
|
|
def pid_alive(pid):
|
|
try:
|
|
value = int(pid)
|
|
except Exception:
|
|
return False
|
|
return value > 0 and os.path.exists("/proc/%d" % value)
|
|
|
|
|
|
def read_json_file(path):
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
return json.load(handle)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def run_web_observe(config):
|
|
cfg = config.get("webObserve") or {}
|
|
if not cfg.get("enabled"):
|
|
return {"id": "web-observe-artifacts", "ok": True, "skipped": True}
|
|
roots = [os.path.abspath(item) for item in (cfg.get("observeStateRoots") or cfg.get("webObserveRoots") or []) if isinstance(item, str) and item.startswith("/")]
|
|
stale_hours = float(cfg.get("staleRunMaxAgeHours") or 6)
|
|
cutoff = time.time() - stale_hours * 3600.0
|
|
deleted = 0
|
|
reclaimed = 0
|
|
protected = 0
|
|
for root in roots:
|
|
if not os.path.isdir(root):
|
|
continue
|
|
for name in os.listdir(root):
|
|
path = os.path.abspath(os.path.join(root, name))
|
|
if os.path.dirname(path) != root or os.path.islink(path) or not os.path.isdir(path):
|
|
continue
|
|
manifest = read_json_file(os.path.join(path, "manifest.json"))
|
|
heartbeat = read_json_file(os.path.join(path, "heartbeat.json"))
|
|
pid = manifest.get("pid") or heartbeat.get("pid") or read_json_file(os.path.join(path, "pid")).get("pid")
|
|
try:
|
|
stat = os.lstat(path)
|
|
except OSError:
|
|
continue
|
|
completed = bool(manifest.get("completedAt") or heartbeat.get("completedAt") or manifest.get("status") in ["completed", "failed", "blocked"])
|
|
stale = completed or stat.st_mtime < cutoff
|
|
if pid_alive(pid) or not stale or path_has_open_fd(path):
|
|
protected += 1
|
|
continue
|
|
before = path_size(path)
|
|
shutil.rmtree(path, ignore_errors=True)
|
|
deleted += 1
|
|
reclaimed += before
|
|
return {"id": "web-observe-artifacts", "ok": True, "deletedCount": deleted, "protectedCount": protected, "reclaimedBytes": reclaimed}
|
|
|
|
|
|
def active_claims(namespace):
|
|
pods, _ = run_json(["kubectl", "-n", namespace, "get", "pod", "-o", "json"], 30)
|
|
claims = {}
|
|
for pod in (pods or {}).get("items") or []:
|
|
if ((pod.get("status") or {}).get("phase")) in ["Succeeded", "Failed"]:
|
|
continue
|
|
pod_name = ((pod.get("metadata") or {}).get("name")) or ""
|
|
for volume in ((pod.get("spec") or {}).get("volumes") or []):
|
|
claim = (volume.get("persistentVolumeClaim") or {}).get("claimName")
|
|
if claim:
|
|
claims.setdefault(claim, []).append(pod_name)
|
|
return claims
|
|
|
|
|
|
def run_session_pvcs(config):
|
|
cfg = config.get("agentrunSessionPvcs") or {}
|
|
if not cfg.get("enabled"):
|
|
return {"id": "agentrun-session-pvcs", "ok": True, "skipped": True}
|
|
namespace = cfg.get("namespace")
|
|
prefixes = list(cfg.get("prefixes") or [])
|
|
limit = max(1, min(safe_int(cfg.get("maxDeletePerRun"), 100), 1000))
|
|
if not namespace or not prefixes:
|
|
return {"id": "agentrun-session-pvcs", "ok": False, "error": "namespace-or-prefixes-missing"}
|
|
pv_data, pv_cmd = run_json(["kubectl", "get", "pv", "-o", "json"], 30)
|
|
pvc_data, pvc_cmd = run_json(["kubectl", "-n", namespace, "get", "pvc", "-o", "json"], 30)
|
|
if pv_data is None or pvc_data is None:
|
|
return {"id": "agentrun-session-pvcs", "ok": False, "pvCommand": pv_cmd, "pvcCommand": pvc_cmd}
|
|
pvs = {((pv.get("metadata") or {}).get("name")): pv for pv in pv_data.get("items") or []}
|
|
active = active_claims(namespace)
|
|
selected = []
|
|
protected = 0
|
|
for pvc in pvc_data.get("items") or []:
|
|
name = ((pvc.get("metadata") or {}).get("name")) or ""
|
|
if not any(name.startswith(prefix) for prefix in prefixes):
|
|
continue
|
|
pv = pvs.get((pvc.get("spec") or {}).get("volumeName")) or {}
|
|
storage_class = (pvc.get("spec") or {}).get("storageClassName") or (pv.get("spec") or {}).get("storageClassName")
|
|
reclaim_policy = (pv.get("spec") or {}).get("persistentVolumeReclaimPolicy")
|
|
if active.get(name) or storage_class != "local-path" or reclaim_policy != "Delete":
|
|
protected += 1
|
|
continue
|
|
selected.append(name)
|
|
selected = selected[:limit]
|
|
if selected:
|
|
result = command(["kubectl", "-n", namespace, "delete", "pvc", "--wait=false"] + selected, 45)
|
|
ok = result["exitCode"] == 0
|
|
else:
|
|
result = None
|
|
ok = True
|
|
return {"id": "agentrun-session-pvcs", "ok": ok, "deletedPvcCount": len(selected), "protectedCount": protected, "command": result}
|
|
|
|
|
|
def ci_active(config):
|
|
namespaces = list((config.get("k3sImageCache") or {}).get("ciNamespaces") or [])
|
|
active = []
|
|
for namespace in namespaces:
|
|
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pipelinerun,taskrun,job -n %s --no-headers 2>/dev/null | awk '$2 != \"True\" && $2 != \"False\" && $2 != \"Complete\" && $2 != \"Failed\" {print}' | head -20" % namespace], 15)
|
|
for line in (result.get("stdoutTail") or "").splitlines():
|
|
if line.strip():
|
|
active.append({"namespace": namespace, "line": line.strip()})
|
|
return active
|
|
|
|
|
|
def run_k3s_images(config):
|
|
cfg = config.get("k3sImageCache") or {}
|
|
if not cfg.get("enabled"):
|
|
return {"id": "k3s-image-cache", "ok": True, "skipped": True}
|
|
active = ci_active(config)
|
|
if active:
|
|
return {"id": "k3s-image-cache", "ok": True, "skipped": True, "reason": "ci-active", "activePreview": active[:5]}
|
|
endpoint = cfg.get("runtimeEndpoint") or "unix:///run/k3s/containerd/containerd.sock"
|
|
before = du_size("/var/lib/rancher/k3s/agent/containerd")
|
|
result = command(["crictl", "--runtime-endpoint", endpoint, "rmi", "--prune"], 300)
|
|
after = du_size("/var/lib/rancher/k3s/agent/containerd")
|
|
return {"id": "k3s-image-cache", "ok": result["exitCode"] == 0, "reclaimedBytes": max(0, before - after), "command": result}
|
|
|
|
|
|
def table_data_lines(stdout, header_prefix):
|
|
return [line.strip() for line in str(stdout or "").splitlines() if line.strip() and not line.startswith(header_prefix)]
|
|
|
|
|
|
def host_containerd_empty(config):
|
|
cfg = config.get("hostContainerdCache") or {}
|
|
address = cfg.get("address") or "/run/containerd/containerd.sock"
|
|
namespaces = cfg.get("namespaces") or ["default"]
|
|
active = []
|
|
for namespace in namespaces:
|
|
base = ["ctr", "--address", address, "-n", namespace]
|
|
for kind, args, header in [
|
|
("image", ["images", "list", "-q"], ""),
|
|
("container", ["containers", "list", "-q"], ""),
|
|
("task", ["tasks", "list", "-q"], ""),
|
|
("lease", ["leases", "list", "-q"], ""),
|
|
("snapshot", ["snapshots", "list"], "KEY"),
|
|
("content", ["content", "list"], "DIGEST"),
|
|
]:
|
|
result = command(base + args, 20)
|
|
if result["exitCode"] != 0:
|
|
active.append({"namespace": namespace, "kind": kind, "state": "unknown"})
|
|
continue
|
|
lines = table_data_lines(result.get("stdoutTail") or "", header) if header else [line for line in (result.get("stdoutTail") or "").splitlines() if line.strip()]
|
|
for line in lines:
|
|
active.append({"namespace": namespace, "kind": kind, "name": line.split()[0] if line.split() else line})
|
|
return active
|
|
|
|
|
|
def direct_child_rows(root, predicate):
|
|
rows = []
|
|
if not root or not os.path.isdir(root) or os.path.islink(root):
|
|
return rows
|
|
real_root = os.path.realpath(os.path.abspath(root))
|
|
for name in sorted(os.listdir(real_root)):
|
|
path = os.path.realpath(os.path.abspath(os.path.join(real_root, name)))
|
|
if os.path.dirname(path) != real_root or not predicate(name, path):
|
|
continue
|
|
rows.append({"name": name, "path": path, "sizeBytes": path_size(path)})
|
|
rows.sort(key=lambda item: item["sizeBytes"], reverse=True)
|
|
return rows
|
|
|
|
|
|
def run_host_containerd(config):
|
|
cfg = config.get("hostContainerdCache") or {}
|
|
if not cfg.get("enabled"):
|
|
return {"id": "host-containerd-orphans", "ok": True, "skipped": True}
|
|
active = host_containerd_empty(config)
|
|
if active:
|
|
return {"id": "host-containerd-orphans", "ok": True, "skipped": True, "reason": "metadata-not-empty", "activePreview": active[:8]}
|
|
orphan = cfg.get("orphanCleanup") or {}
|
|
if not orphan.get("enabled"):
|
|
return {"id": "host-containerd-orphans", "ok": True, "skipped": True, "reason": "orphan-cleanup-disabled"}
|
|
overlay_root = os.path.realpath(os.path.abspath(orphan.get("overlaySnapshotsRoot") or ""))
|
|
content_root = os.path.realpath(os.path.abspath(orphan.get("contentBlobRoot") or ""))
|
|
root = os.path.realpath(os.path.abspath(cfg.get("root") or ""))
|
|
if not root or not overlay_root.startswith(root + "/") or not content_root.startswith(root + "/"):
|
|
return {"id": "host-containerd-orphans", "ok": False, "error": "orphan-root-outside-containerd-root"}
|
|
for root_path in [overlay_root, content_root]:
|
|
if os.path.exists(root_path) and path_has_open_fd(root_path):
|
|
return {"id": "host-containerd-orphans", "ok": True, "skipped": True, "reason": "open-fd", "root": root_path}
|
|
rows = direct_child_rows(overlay_root, lambda name, path: os.path.isdir(path) and not os.path.islink(path) and re.match(r"^[0-9]+$", name) is not None)
|
|
rows += direct_child_rows(content_root, lambda name, path: os.path.isfile(path) and not os.path.islink(path) and re.match(r"^[0-9a-f]{64}$", name) is not None)
|
|
rows.sort(key=lambda item: item["sizeBytes"], reverse=True)
|
|
limit = max(1, min(safe_int(orphan.get("maxDeletePerRun"), 100), 1000))
|
|
reclaimed = 0
|
|
deleted = 0
|
|
for row in rows[:limit]:
|
|
before = path_size(row["path"])
|
|
if os.path.isdir(row["path"]):
|
|
shutil.rmtree(row["path"], ignore_errors=True)
|
|
else:
|
|
try:
|
|
os.unlink(row["path"])
|
|
except FileNotFoundError:
|
|
continue
|
|
reclaimed += before
|
|
deleted += 1
|
|
return {"id": "host-containerd-orphans", "ok": True, "deletedCount": deleted, "candidateCount": len(rows), "reclaimedBytes": reclaimed}
|
|
|
|
|
|
def run_local_path_orphans(config):
|
|
cfg = config.get("localPathStorage") or {}
|
|
if not cfg.get("enabled"):
|
|
return {"id": "local-path-orphans", "ok": True, "skipped": True}
|
|
root = os.path.realpath(os.path.abspath(cfg.get("root") or ""))
|
|
prefixes = list(cfg.get("orphanDirPrefixes") or [])
|
|
if not root or not prefixes or not os.path.isdir(root):
|
|
return {"id": "local-path-orphans", "ok": False, "error": "root-or-prefix-missing"}
|
|
pv_data, pv_cmd = run_json(["kubectl", "get", "pv", "-o", "json"], 30)
|
|
if pv_data is None:
|
|
return {"id": "local-path-orphans", "ok": False, "pvCommand": pv_cmd}
|
|
referenced = set()
|
|
for pv in pv_data.get("items") or []:
|
|
spec = pv.get("spec") or {}
|
|
path = ((spec.get("hostPath") or {}).get("path")) or ((spec.get("local") or {}).get("path"))
|
|
if path:
|
|
referenced.add(os.path.realpath(os.path.abspath(path)))
|
|
rows = []
|
|
for name in os.listdir(root):
|
|
path = os.path.realpath(os.path.abspath(os.path.join(root, name)))
|
|
if os.path.dirname(path) != root or not any(name.startswith(prefix) for prefix in prefixes) or path in referenced or path_has_open_fd(path):
|
|
continue
|
|
if os.path.isdir(path) and not os.path.islink(path):
|
|
rows.append({"name": name, "path": path, "sizeBytes": path_size(path)})
|
|
rows.sort(key=lambda item: item["sizeBytes"], reverse=True)
|
|
limit = max(1, min(safe_int(cfg.get("maxDeletePerRun"), 100), 1000))
|
|
reclaimed = 0
|
|
deleted = 0
|
|
for row in rows[:limit]:
|
|
before = path_size(row["path"])
|
|
shutil.rmtree(row["path"], ignore_errors=True)
|
|
reclaimed += before
|
|
deleted += 1
|
|
return {"id": "local-path-orphans", "ok": True, "deletedCount": deleted, "candidateCount": len(rows), "reclaimedBytes": reclaimed}
|
|
|
|
|
|
def main():
|
|
config_path = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
config = load_config(config_path)
|
|
results = []
|
|
for fn in [run_journal, run_apt, run_tmp, run_tool_caches, run_web_observe, run_session_pvcs, run_k3s_images, run_host_containerd, run_local_path_orphans]:
|
|
try:
|
|
results.append(fn(config))
|
|
except Exception as exc:
|
|
results.append({"id": getattr(fn, "__name__", "stage"), "ok": False, "error": str(exc)})
|
|
payload = {
|
|
"ok": all(item.get("ok") for item in results),
|
|
"observedAt": now_iso(),
|
|
"providerId": config.get("providerId"),
|
|
"unitName": config.get("unitName"),
|
|
"resultCount": len(results),
|
|
"failedCount": len([item for item in results if not item.get("ok")]),
|
|
"reclaimedBytes": sum(safe_int(item.get("reclaimedBytes")) for item in results),
|
|
"results": results,
|
|
}
|
|
write_state(config, payload)
|
|
print(json.dumps(payload, ensure_ascii=False))
|
|
return 0 if payload["ok"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|