fix: add JD01 GC retention controls
This commit is contained in:
Codex
@@ -0,0 +1,386 @@
|
||||
def k3s_crictl_base():
|
||||
endpoint = str(CONTAINERD_CONFIG.get("runtimeEndpoint") or "unix:///run/k3s/containerd/containerd.sock")
|
||||
return ["crictl", "--runtime-endpoint", endpoint]
|
||||
|
||||
def shell_single_quote(value):
|
||||
return "'" + str(value).replace("'", "'\"'\"'") + "'"
|
||||
|
||||
def k3s_crictl_json(args, timeout=30):
|
||||
result = command(k3s_crictl_base() + args + ["-o", "json"], timeout)
|
||||
if result["exitCode"] != 0:
|
||||
return None, result
|
||||
try:
|
||||
return json.loads(result["stdout"] or "{}"), result
|
||||
except Exception:
|
||||
return None, result
|
||||
|
||||
def ci_activity_snapshot_for_prune():
|
||||
namespaces = config_list(CONTAINERD_CONFIG, "ciNamespaces", ["hwlab-ci", "agentrun-ci"])
|
||||
active = []
|
||||
commands = []
|
||||
for namespace in namespaces:
|
||||
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pipelinerun,taskrun,job -n %s --no-headers 2>/dev/null | awk '$2 != \"True\" && $2 != \"False\" && $2 != \"Complete\" && $2 != \"Failed\" {print}' | head -20" % shell_single_quote(namespace)], 15)
|
||||
commands.append({"namespace": namespace, "command": bounded(result)})
|
||||
for line in (result.get("stdout") or "").splitlines():
|
||||
if line.strip():
|
||||
active.append({"namespace": namespace, "line": line.strip()})
|
||||
return {"ok": True, "activeCount": len(active), "activePreview": active[:20], "commands": commands}
|
||||
|
||||
def compact_ci_activity(activity):
|
||||
return {
|
||||
"ok": activity.get("ok"),
|
||||
"activeCount": activity.get("activeCount"),
|
||||
"activePreview": activity.get("activePreview") or [],
|
||||
}
|
||||
|
||||
def compact_image_ref(ref):
|
||||
ref = str(ref or "")
|
||||
return ref if len(ref) <= 120 else ref[:117] + "..."
|
||||
|
||||
def k3s_cri_image_rows():
|
||||
images, image_cmd = k3s_crictl_json(["images"], 45)
|
||||
containers, container_cmd = k3s_crictl_json(["ps", "-a"], 30)
|
||||
if images is None:
|
||||
return None, {"ok": False, "reason": "crictl-images-failed", "command": bounded(image_cmd)}
|
||||
if containers is None:
|
||||
return None, {"ok": False, "reason": "crictl-ps-failed", "command": bounded(container_cmd)}
|
||||
used = set()
|
||||
for container in containers.get("containers") or []:
|
||||
for key in ["imageRef", "image", "imageId"]:
|
||||
value = container.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
used.add(value)
|
||||
image = container.get("image") or {}
|
||||
if isinstance(image, dict):
|
||||
for key in ["image", "annotations", "userSpecifiedImage"]:
|
||||
value = image.get(key)
|
||||
if isinstance(value, str) and value:
|
||||
used.add(value)
|
||||
rows = []
|
||||
for image in images.get("images") or []:
|
||||
refs = []
|
||||
for key in ["repoTags", "repoDigests"]:
|
||||
value = image.get(key)
|
||||
if isinstance(value, list):
|
||||
refs.extend([str(item) for item in value if item])
|
||||
image_id = str(image.get("id") or "")
|
||||
pinned = bool(image.get("pinned"))
|
||||
size = safe_int(image.get("size_") or image.get("size") or 0)
|
||||
in_use = pinned or image_id in used or any(ref in used for ref in refs)
|
||||
rows.append({"id": image_id, "refs": refs, "sizeBytes": size, "inUse": in_use, "pinned": pinned})
|
||||
return rows, {"ok": True, "imageCommand": bounded(image_cmd), "containerCommand": bounded(container_cmd)}
|
||||
|
||||
def k3s_image_cache_candidate():
|
||||
if not config_bool(CONTAINERD_CONFIG, "enabled", False):
|
||||
return {
|
||||
"id": "k3s-cri-image-prune:disabled",
|
||||
"kind": "k3s-cri-image-prune-disabled",
|
||||
"risk": "blocked",
|
||||
"description": "K3s CRI image prune is disabled in YAML",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.containerdImageCache.enabled" % PROVIDER_ID,
|
||||
}
|
||||
activity = ci_activity_snapshot_for_prune()
|
||||
if int(activity.get("activeCount") or 0) > 0:
|
||||
return {
|
||||
"id": "k3s-cri-image-prune:ci-active",
|
||||
"kind": "k3s-cri-image-prune-blocked",
|
||||
"risk": "blocked",
|
||||
"description": "K3s CRI image prune is blocked while CI workloads are active",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"ciActivity": compact_ci_activity(activity),
|
||||
}
|
||||
rows, meta = k3s_cri_image_rows()
|
||||
if rows is None:
|
||||
return {
|
||||
"id": "k3s-cri-image-prune:unavailable",
|
||||
"kind": "k3s-cri-image-prune-unavailable",
|
||||
"risk": "blocked",
|
||||
"description": "K3s CRI image list is unavailable",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"diagnostic": meta,
|
||||
}
|
||||
unused = [row for row in rows if not row.get("inUse")]
|
||||
estimated = sum(safe_int(row.get("sizeBytes")) for row in unused)
|
||||
if estimated <= 0:
|
||||
return None
|
||||
return {
|
||||
"id": "k3s-cri-image-prune:unused",
|
||||
"kind": "k3s-cri-image-prune",
|
||||
"risk": "medium",
|
||||
"description": "Prune unused k3s CRI images through crictl rmi --prune; no containerd paths are deleted directly",
|
||||
"sizeBytes": estimated,
|
||||
"estimatedReclaimBytes": estimated,
|
||||
"imageCount": len(rows),
|
||||
"unusedImageCount": len(unused),
|
||||
"unusedPreview": [{"id": row.get("id"), "refs": [compact_image_ref(ref) for ref in (row.get("refs") or [])[:2]], "sizeBytes": row.get("sizeBytes")} for row in unused[:3]],
|
||||
"ciActivity": compact_ci_activity(activity),
|
||||
"action": {"command": k3s_crictl_base() + ["rmi", "--prune"], "mode": "cri-unused-images-only"},
|
||||
}
|
||||
|
||||
def execute_k3s_image_cache_prune():
|
||||
activity = ci_activity_snapshot_for_prune()
|
||||
if int(activity.get("activeCount") or 0) > 0:
|
||||
raise RuntimeError("refusing k3s image prune while CI workloads are active")
|
||||
before = du_size("/var/lib/rancher/k3s/agent/containerd", 45) or 0
|
||||
result = command(k3s_crictl_base() + ["rmi", "--prune"], 300)
|
||||
if result["exitCode"] != 0:
|
||||
raise RuntimeError((result["stderr"] or result["stdout"] or "crictl rmi --prune failed").strip())
|
||||
after = du_size("/var/lib/rancher/k3s/agent/containerd", 45) or 0
|
||||
return {"reclaimedBytes": max(0, before - after), "commandOutput": bounded(result), "ciActivity": compact_ci_activity(activity)}
|
||||
|
||||
def host_ctr_base(namespace=None):
|
||||
address = config_str(HOST_CONTAINERD_CONFIG, "address", "")
|
||||
args = ["ctr"]
|
||||
if address:
|
||||
args.extend(["--address", address])
|
||||
if namespace:
|
||||
args.extend(["-n", namespace])
|
||||
return args
|
||||
|
||||
def host_ctr(args, timeout=30, namespace=None):
|
||||
return command(host_ctr_base(namespace) + args, timeout)
|
||||
|
||||
def host_containerd_namespaces():
|
||||
configured = config_list(HOST_CONTAINERD_CONFIG, "namespaces", [])
|
||||
if configured:
|
||||
return configured, {"source": "yaml", "command": None}
|
||||
result = host_ctr(["namespaces", "list", "-q"], 20)
|
||||
if result["exitCode"] != 0:
|
||||
return [], {"source": "ctr", "command": bounded(result), "error": "ctr-namespaces-failed"}
|
||||
return [line.strip() for line in (result.get("stdout") or "").splitlines() if line.strip()], {"source": "ctr", "command": bounded(result)}
|
||||
|
||||
def host_containerd_activity():
|
||||
if not config_bool(HOST_CONTAINERD_CONFIG, "enabled", False):
|
||||
return {"ok": False, "reason": "host-containerd-cache-disabled", "activeCount": 0}
|
||||
root = config_str(HOST_CONTAINERD_CONFIG, "root", "")
|
||||
if not root or not os.path.isdir(root):
|
||||
return {"ok": False, "reason": "host-containerd-root-unavailable", "root": root, "activeCount": 0}
|
||||
namespaces, namespace_meta = host_containerd_namespaces()
|
||||
active = []
|
||||
commands = []
|
||||
for namespace in namespaces:
|
||||
task_result = host_ctr(["tasks", "list", "-q"], 20, namespace)
|
||||
container_result = host_ctr(["containers", "list", "-q"], 20, namespace)
|
||||
image_result = host_ctr(["images", "list", "-q"], 20, namespace)
|
||||
lease_result = host_ctr(["leases", "list", "-q"], 20, namespace)
|
||||
snapshot_result = host_ctr(["snapshots", "list"], 20, namespace)
|
||||
content_result = host_ctr(["content", "list"], 20, namespace)
|
||||
snapshot_lines = table_data_lines(snapshot_result.get("stdout") or "", "KEY")
|
||||
content_lines = table_data_lines(content_result.get("stdout") or "", "DIGEST")
|
||||
commands.append({
|
||||
"namespace": namespace,
|
||||
"tasks": bounded(task_result),
|
||||
"containers": bounded(container_result),
|
||||
"images": bounded(image_result),
|
||||
"leases": bounded(lease_result),
|
||||
"snapshots": bounded(snapshot_result),
|
||||
"content": bounded(content_result),
|
||||
})
|
||||
for kind, result in [("task", task_result), ("container", container_result), ("lease", lease_result)]:
|
||||
if result["exitCode"] != 0:
|
||||
active.append({"namespace": namespace, "kind": kind, "state": "unknown", "reason": "ctr-list-failed"})
|
||||
continue
|
||||
for line in (result.get("stdout") or "").splitlines():
|
||||
if line.strip():
|
||||
active.append({"namespace": namespace, "kind": kind, "name": line.strip()})
|
||||
if snapshot_result["exitCode"] != 0:
|
||||
active.append({"namespace": namespace, "kind": "snapshot", "state": "unknown", "reason": "ctr-list-failed"})
|
||||
for line in snapshot_lines:
|
||||
active.append({"namespace": namespace, "kind": "snapshot", "name": line.split()[0] if line.split() else line})
|
||||
if content_result["exitCode"] != 0:
|
||||
active.append({"namespace": namespace, "kind": "content", "state": "unknown", "reason": "ctr-list-failed"})
|
||||
for line in content_lines:
|
||||
active.append({"namespace": namespace, "kind": "content", "name": line.split()[0] if line.split() else line})
|
||||
return {
|
||||
"ok": True,
|
||||
"root": root,
|
||||
"namespaces": namespaces,
|
||||
"namespaceMeta": namespace_meta,
|
||||
"activeCount": len(active),
|
||||
"activePreview": active[:20],
|
||||
"commands": commands,
|
||||
}
|
||||
|
||||
def compact_host_containerd_activity(activity):
|
||||
return {
|
||||
"ok": activity.get("ok"),
|
||||
"reason": activity.get("reason"),
|
||||
"root": activity.get("root"),
|
||||
"namespaces": activity.get("namespaces"),
|
||||
"activeCount": activity.get("activeCount"),
|
||||
"activePreview": activity.get("activePreview") or [],
|
||||
}
|
||||
|
||||
def table_data_lines(stdout, header_prefix):
|
||||
lines = [line.strip() for line in str(stdout or "").splitlines() if line.strip()]
|
||||
return [line for line in lines if not line.startswith(header_prefix)]
|
||||
|
||||
def host_containerd_orphan_config():
|
||||
value = HOST_CONTAINERD_CONFIG.get("orphanCleanup") if isinstance(HOST_CONTAINERD_CONFIG, dict) else None
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
def direct_child_paths(root, predicate):
|
||||
if not root or not os.path.isdir(root) or os.path.islink(root):
|
||||
return []
|
||||
rows = []
|
||||
for name in sorted(os.listdir(root)):
|
||||
path = os.path.realpath(os.path.abspath(os.path.join(root, name)))
|
||||
if os.path.dirname(path) != os.path.realpath(os.path.abspath(root)):
|
||||
continue
|
||||
if not predicate(name, path):
|
||||
continue
|
||||
rows.append({"name": name, "path": path, "estimatedReclaimBytes": du_size(path, 10) or path_size(path)})
|
||||
return rows
|
||||
|
||||
def host_containerd_orphan_rows(activity):
|
||||
cfg = host_containerd_orphan_config()
|
||||
if not config_bool(cfg, "enabled", False):
|
||||
return [], {"ok": False, "reason": "host-containerd-orphan-cleanup-disabled"}
|
||||
if not activity.get("ok") or int(activity.get("activeCount") or 0) > 0:
|
||||
return [], {"ok": False, "reason": "host-containerd-metadata-not-empty", "activity": compact_host_containerd_activity(activity)}
|
||||
overlay_root = os.path.realpath(os.path.abspath(config_str(cfg, "overlaySnapshotsRoot", "")))
|
||||
content_root = os.path.realpath(os.path.abspath(config_str(cfg, "contentBlobRoot", "")))
|
||||
root = os.path.realpath(os.path.abspath(config_str(HOST_CONTAINERD_CONFIG, "root", "")))
|
||||
if not root or not overlay_root.startswith(root.rstrip("/") + "/") or not content_root.startswith(root.rstrip("/") + "/"):
|
||||
return [], {"ok": False, "reason": "host-containerd-orphan-root-outside-containerd-root", "root": root}
|
||||
open_roots = []
|
||||
for candidate_root in [overlay_root, content_root]:
|
||||
if os.path.exists(candidate_root) and path_has_open_fd(candidate_root):
|
||||
open_roots.append(candidate_root)
|
||||
if open_roots:
|
||||
return [], {"ok": False, "reason": "host-containerd-orphan-root-open-fd", "openRoots": open_roots}
|
||||
overlay_rows = direct_child_paths(overlay_root, lambda name, path: os.path.isdir(path) and not os.path.islink(path) and re.match(r"^[0-9]+$", name) is not None)
|
||||
content_rows = direct_child_paths(content_root, lambda name, path: os.path.isfile(path) and not os.path.islink(path) and re.match(r"^[0-9a-f]{64}$", name) is not None)
|
||||
safe_rows = []
|
||||
for kind, rows in [("overlay-snapshot-dir", overlay_rows), ("content-blob-file", content_rows)]:
|
||||
for row in rows:
|
||||
safe_rows.append({**row, "kind": kind})
|
||||
safe_rows.sort(key=lambda item: safe_int(item.get("estimatedReclaimBytes")), reverse=True)
|
||||
return safe_rows, {
|
||||
"ok": True,
|
||||
"root": root,
|
||||
"overlaySnapshotsRoot": overlay_root,
|
||||
"contentBlobRoot": content_root,
|
||||
"overlayCandidateCount": len(overlay_rows),
|
||||
"contentCandidateCount": len(content_rows),
|
||||
"protectedCount": 0,
|
||||
"protectedPreview": [],
|
||||
}
|
||||
|
||||
def host_containerd_orphan_candidate(activity):
|
||||
rows, meta = host_containerd_orphan_rows(activity)
|
||||
if not meta.get("ok"):
|
||||
return None
|
||||
limit = int(OPTIONS.get("limit") or 50)
|
||||
selected = rows[:limit]
|
||||
estimated = sum(safe_int(row.get("estimatedReclaimBytes")) for row in selected)
|
||||
if estimated <= 0:
|
||||
return None
|
||||
return {
|
||||
"id": "host-containerd-orphan-state:delete",
|
||||
"kind": "host-containerd-orphan-state-delete",
|
||||
"risk": "medium",
|
||||
"description": "Delete YAML-allowlisted host containerd orphan snapshot/content files only when ctr metadata has no tasks, containers, leases, images, snapshots or content",
|
||||
"path": meta.get("root"),
|
||||
"sizeBytes": estimated,
|
||||
"estimatedReclaimBytes": estimated,
|
||||
"orphanCount": len(rows),
|
||||
"selectedOrphanCount": len(selected),
|
||||
"overlayCandidateCount": meta.get("overlayCandidateCount"),
|
||||
"contentCandidateCount": meta.get("contentCandidateCount"),
|
||||
"protectedCount": meta.get("protectedCount"),
|
||||
"selectedPreview": [{"kind": row.get("kind"), "name": row.get("name"), "estimatedReclaimBytes": row.get("estimatedReclaimBytes")} for row in selected[:8]],
|
||||
"protectedPreview": meta.get("protectedPreview"),
|
||||
"action": {"op": "remove-yaml-allowlisted-host-containerd-orphans", "limit": limit},
|
||||
}
|
||||
|
||||
def host_containerd_cache_candidate():
|
||||
activity = host_containerd_activity()
|
||||
if not activity.get("ok"):
|
||||
return {
|
||||
"id": "host-containerd-cache:unavailable",
|
||||
"kind": "host-containerd-cache-unavailable",
|
||||
"risk": "blocked",
|
||||
"description": "Host containerd cache cleanup is disabled or unavailable by YAML",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"diagnostic": compact_host_containerd_activity(activity),
|
||||
}
|
||||
if int(activity.get("activeCount") or 0) > 0:
|
||||
return {
|
||||
"id": "host-containerd-cache:active",
|
||||
"kind": "host-containerd-cache-blocked",
|
||||
"risk": "blocked",
|
||||
"description": "Host containerd cache prune is blocked while host containerd tasks or containers exist",
|
||||
"estimatedReclaimBytes": 0,
|
||||
"activity": compact_host_containerd_activity(activity),
|
||||
}
|
||||
orphan = host_containerd_orphan_candidate(activity)
|
||||
if orphan:
|
||||
return orphan
|
||||
root = activity.get("root") or ""
|
||||
size = du_size(root, 45) or 0
|
||||
if size <= 0:
|
||||
return None
|
||||
return {
|
||||
"id": "host-containerd-cache:prune-unused",
|
||||
"kind": "host-containerd-cache-prune",
|
||||
"risk": "medium",
|
||||
"description": "Prune host containerd images in YAML-selected namespaces only when no host containerd tasks or containers exist",
|
||||
"path": root,
|
||||
"sizeBytes": size,
|
||||
"estimatedReclaimBytes": size,
|
||||
"activity": compact_host_containerd_activity(activity),
|
||||
"action": {"command": "ctr images prune --all per namespace", "mode": "host-containerd-unused-images-only"},
|
||||
}
|
||||
|
||||
def execute_host_containerd_cache_prune():
|
||||
activity = host_containerd_activity()
|
||||
if not activity.get("ok"):
|
||||
raise RuntimeError("host containerd cache cleanup unavailable: %s" % activity.get("reason"))
|
||||
if int(activity.get("activeCount") or 0) > 0:
|
||||
raise RuntimeError("refusing host containerd prune while tasks or containers exist")
|
||||
root = activity.get("root") or ""
|
||||
before = du_size(root, 45) or 0
|
||||
results = []
|
||||
for namespace in activity.get("namespaces") or []:
|
||||
result = host_ctr(["images", "prune", "--all"], 300, namespace)
|
||||
results.append({"namespace": namespace, "imagesPrune": bounded(result)})
|
||||
if result["exitCode"] != 0:
|
||||
raise RuntimeError("host containerd image prune failed in namespace %s: %s" % (namespace, (result.get("stderr") or result.get("stdout") or "").strip()))
|
||||
after = du_size(root, 45) or 0
|
||||
return {
|
||||
"reclaimedBytes": max(0, before - after),
|
||||
"activity": compact_host_containerd_activity(activity),
|
||||
"commandResults": results[:8],
|
||||
}
|
||||
|
||||
def execute_host_containerd_orphan_cleanup():
|
||||
activity = host_containerd_activity()
|
||||
rows, meta = host_containerd_orphan_rows(activity)
|
||||
if not meta.get("ok"):
|
||||
raise RuntimeError("host containerd orphan cleanup unavailable: %s" % meta.get("reason"))
|
||||
for root_path in [meta.get("overlaySnapshotsRoot"), meta.get("contentBlobRoot")]:
|
||||
if root_path and os.path.exists(root_path) and path_has_open_fd(root_path):
|
||||
raise RuntimeError("refusing host containerd orphan cleanup with open fd/cwd under root: %s" % root_path)
|
||||
limit = int(OPTIONS.get("limit") or 50)
|
||||
selected = rows[:limit]
|
||||
reclaimed = 0
|
||||
deleted = []
|
||||
for row in selected:
|
||||
path = row.get("path")
|
||||
before = du_size(path, 10) or path_size(path)
|
||||
if row.get("kind") == "overlay-snapshot-dir":
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
elif row.get("kind") == "content-blob-file":
|
||||
os.unlink(path)
|
||||
else:
|
||||
raise RuntimeError("unsupported host containerd orphan kind: %s" % row.get("kind"))
|
||||
reclaimed += before
|
||||
deleted.append({"kind": row.get("kind"), "name": row.get("name"), "reclaimedBytes": before})
|
||||
return {
|
||||
"reclaimedBytes": reclaimed,
|
||||
"deletedOrphanCount": len(deleted),
|
||||
"deletedPreview": deleted[:12],
|
||||
"root": meta.get("root"),
|
||||
}
|
||||
Reference in New Issue
Block a user