feat: add platform infra codex pool config
This commit is contained in:
Codex
+452
-3
@@ -25,6 +25,8 @@ interface RemoteGcOptions {
|
||||
limit: number;
|
||||
resultLimit: number;
|
||||
full: boolean;
|
||||
historyLimit: number;
|
||||
saveSnapshot: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
|
||||
@@ -47,6 +49,8 @@ const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
|
||||
limit: 50,
|
||||
resultLimit: 50,
|
||||
full: false,
|
||||
historyLimit: 12,
|
||||
saveSnapshot: true,
|
||||
};
|
||||
|
||||
export async function runRemoteGcCommand(config: UniDeskConfig, providerId: string | undefined, action: string | undefined, args: string[]): Promise<unknown> {
|
||||
@@ -54,12 +58,14 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
return {
|
||||
ok: false,
|
||||
error: "gc-remote-provider-required",
|
||||
usage: "bun scripts/cli.ts gc remote <providerId> plan|run|status [--confirm]",
|
||||
usage: "bun scripts/cli.ts gc remote <providerId> plan|snapshot|trend|run|status [--confirm]",
|
||||
};
|
||||
}
|
||||
const subaction = action ?? "plan";
|
||||
const options = parseRemoteGcOptions(args);
|
||||
if (subaction === "plan" || subaction === "dry-run") return await runRemoteGc(config, providerId, "plan", options);
|
||||
if (subaction === "snapshot" || subaction === "growth") return await runRemoteGc(config, providerId, "snapshot", options);
|
||||
if (subaction === "trend") return await runRemoteGc(config, providerId, "trend", options);
|
||||
if (subaction === "status") return await runRemoteGc(config, providerId, "status", options);
|
||||
if (subaction === "run") {
|
||||
if (!options.confirm) {
|
||||
@@ -79,7 +85,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
|
||||
ok: false,
|
||||
error: "unsupported-gc-remote-action",
|
||||
action: subaction,
|
||||
supportedActions: ["plan", "run", "status"],
|
||||
supportedActions: ["plan", "snapshot", "trend", "run", "status"],
|
||||
};
|
||||
}
|
||||
|
||||
@@ -126,6 +132,10 @@ function parseRemoteGcOptions(args: string[]): RemoteGcOptions {
|
||||
const value = parseNonNegativeNumber(arg, args[++index]);
|
||||
if (!Number.isInteger(value) || value <= 0) throw new Error("--result-limit must be a positive integer");
|
||||
options.resultLimit = Math.min(value, 5000);
|
||||
} else if (arg === "--history-limit") {
|
||||
const value = parseNonNegativeNumber(arg, args[++index]);
|
||||
if (!Number.isInteger(value) || value <= 1) throw new Error("--history-limit must be an integer greater than 1");
|
||||
options.historyLimit = Math.min(value, 200);
|
||||
} else if (arg === "--no-journal") {
|
||||
options.journal = false;
|
||||
} else if (arg === "--no-docker-logs") {
|
||||
@@ -145,6 +155,8 @@ function parseRemoteGcOptions(args: string[]): RemoteGcOptions {
|
||||
options.registryGcOnly = true;
|
||||
} else if (arg === "--full" || arg === "--raw") {
|
||||
options.full = true;
|
||||
} else if (arg === "--no-save" || arg === "--no-snapshot-save") {
|
||||
options.saveSnapshot = false;
|
||||
} else {
|
||||
throw new Error(`unknown gc remote option: ${arg}`);
|
||||
}
|
||||
@@ -178,7 +190,7 @@ function parseSize(raw: string): number | null {
|
||||
return Number.isFinite(bytes) ? bytes : null;
|
||||
}
|
||||
|
||||
async function runRemoteGc(config: UniDeskConfig, providerId: string, action: "plan" | "run" | "status", options: RemoteGcOptions): Promise<unknown> {
|
||||
async function runRemoteGc(config: UniDeskConfig, providerId: string, action: "plan" | "snapshot" | "trend" | "run" | "status", options: RemoteGcOptions): Promise<unknown> {
|
||||
const scriptConfig = Buffer.from(JSON.stringify({ providerId, action, options }), "utf8").toString("base64");
|
||||
const result = await runSshCommandCapture(config, providerId, ["py"], remoteGcPython(scriptConfig));
|
||||
if (result.exitCode !== 0) {
|
||||
@@ -210,6 +222,7 @@ async function runRemoteGc(config: UniDeskConfig, providerId: string, action: "p
|
||||
function remoteGcPython(configBase64: string): string {
|
||||
return String.raw`
|
||||
import base64
|
||||
import calendar
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -281,6 +294,7 @@ REGISTRY_PROTECTED_TAGS = set([
|
||||
|
||||
EXPECTED_G14_NODE = "ubuntu-rog-zephyrus-g14-ga401iv-ga401iv"
|
||||
REMOTE_GC_JOB_DIR = "/tmp/unidesk-gc-remote/jobs"
|
||||
REMOTE_GROWTH_SNAPSHOT_DIR = "/tmp/unidesk-gc-remote/growth-snapshots"
|
||||
REMOTE_STDOUT_JSON_LIMIT = 256 * 1024
|
||||
|
||||
def now_iso():
|
||||
@@ -496,6 +510,391 @@ def du_size(path, timeout=20):
|
||||
except Exception:
|
||||
return path_size(path)
|
||||
|
||||
def safe_int(value, default=0):
|
||||
try:
|
||||
if value is None:
|
||||
return default
|
||||
return int(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def iso_to_epoch(value):
|
||||
try:
|
||||
return calendar.timegm(time.strptime(str(value), "%Y-%m-%dT%H:%M:%SZ"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def growth_snapshot_path():
|
||||
os.makedirs(REMOTE_GROWTH_SNAPSHOT_DIR, exist_ok=True)
|
||||
provider_slug = re.sub(r"[^A-Za-z0-9._-]+", "-", PROVIDER_ID.lower()).strip("-") or "provider"
|
||||
return os.path.join(REMOTE_GROWTH_SNAPSHOT_DIR, "%s.jsonl" % provider_slug)
|
||||
|
||||
def read_growth_snapshots(limit=None):
|
||||
path = growth_snapshot_path()
|
||||
if not os.path.isfile(path):
|
||||
return []
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
lines = handle.readlines()
|
||||
except OSError:
|
||||
return []
|
||||
rows = []
|
||||
for line in lines[-max(1, int(limit or 200)):]:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(item, dict):
|
||||
rows.append(item)
|
||||
return rows
|
||||
|
||||
def append_growth_snapshot(snapshot):
|
||||
path = growth_snapshot_path()
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(snapshot, ensure_ascii=False, sort_keys=True))
|
||||
handle.write("\n")
|
||||
return path
|
||||
|
||||
def source_size_item(source_id, label, path, cleanup_owner, timeout=20):
|
||||
size = du_size(path, timeout) if os.path.exists(path) else None
|
||||
return {
|
||||
"id": source_id,
|
||||
"label": label,
|
||||
"path": path,
|
||||
"exists": size is not None,
|
||||
"sizeBytes": size,
|
||||
"sizeHuman": fmt_bytes(size or 0),
|
||||
"cleanupOwner": cleanup_owner,
|
||||
}
|
||||
|
||||
def disk_source_snapshot():
|
||||
sources = [
|
||||
source_size_item("hwlab-host-data", "HWLAB host data", "/var/lib/hwlab", "hwlab-registry-retention", 60),
|
||||
source_size_item("hwlab-registry", "HWLAB registry", REGISTRY_ROOT, "gc-remote-hwlab-registry", 60),
|
||||
source_size_item("k3s-storage", "k3s local-path storage", "/var/lib/rancher/k3s/storage", "owner-aware-pvc-retention", 45),
|
||||
source_size_item("k3s-containerd", "k3s containerd", "/var/lib/rancher/k3s/agent/containerd", "observation-only", 45),
|
||||
source_size_item("host-containerd", "host containerd", "/var/lib/containerd", "observation-only", 30),
|
||||
source_size_item("kubelet", "kubelet state", "/var/lib/kubelet", "protected-runtime", 20),
|
||||
source_size_item("var-log", "host logs", "/var/log", "gc-remote-logs-journald", 20),
|
||||
source_size_item("tmp", "allowlisted tmp and other tmp", "/tmp", "gc-remote-tmp-allowlist", 20),
|
||||
source_size_item("apt-cache", "apt archives", "/var/cache/apt/archives", "gc-remote-apt-cache", 10),
|
||||
source_size_item("hwlab-v02-source", "HWLAB v0.2 source workspace", "/root/hwlab-v02", "protected-source", 20),
|
||||
source_size_item("agentrun-source", "AgentRun source workspace", "/root/agentrun-v01", "protected-source", 20),
|
||||
]
|
||||
return [item for item in sources if item.get("exists")]
|
||||
|
||||
def containerd_breakdown_snapshot():
|
||||
rows = [
|
||||
source_size_item("k3s-containerd-content", "k3s containerd content store", "/var/lib/rancher/k3s/agent/containerd/io.containerd.content.v1.content", "observation-only", 30),
|
||||
source_size_item("k3s-containerd-overlayfs", "k3s containerd overlay snapshots", "/var/lib/rancher/k3s/agent/containerd/io.containerd.snapshotter.v1.overlayfs", "observation-only", 30),
|
||||
source_size_item("host-containerd-content", "host containerd content store", "/var/lib/containerd/io.containerd.content.v1.content", "observation-only", 20),
|
||||
source_size_item("host-containerd-overlayfs", "host containerd overlay snapshots", "/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs", "observation-only", 20),
|
||||
]
|
||||
rows = [item for item in rows if item.get("exists")]
|
||||
return {
|
||||
"state": "observation-only",
|
||||
"cleanupSupported": False,
|
||||
"reason": "containerd cleanup still requires a reference-safe image/content classifier; this snapshot only classifies growth sources",
|
||||
"breakdown": rows,
|
||||
}
|
||||
|
||||
def pv_host_path(pv):
|
||||
spec = (pv or {}).get("spec") or {}
|
||||
host_path = (spec.get("hostPath") or {}).get("path")
|
||||
if isinstance(host_path, str) and host_path:
|
||||
return host_path
|
||||
local_path = (spec.get("local") or {}).get("path")
|
||||
if isinstance(local_path, str) and local_path:
|
||||
return local_path
|
||||
return None
|
||||
|
||||
def pvc_owner_group(namespace, owner):
|
||||
owner = str(owner or "")
|
||||
if namespace == "agentrun-ci":
|
||||
return "agentrun"
|
||||
if namespace == "hwlab-ci":
|
||||
if owner.startswith("agentrun-"):
|
||||
return "agentrun"
|
||||
return "hwlab"
|
||||
if namespace.startswith("hwlab-"):
|
||||
return "hwlab-runtime"
|
||||
return "other"
|
||||
|
||||
def ci_storage_snapshot():
|
||||
pv_data = kubectl_json(["get", "pv"], 30) or {}
|
||||
pvc_data = kubectl_json(["get", "pvc", "-A"], 30) or {}
|
||||
pod_data = kubectl_json(["get", "pod", "-A"], 30) or {}
|
||||
pvs = {}
|
||||
for pv in pv_data.get("items") or []:
|
||||
meta = pv.get("metadata") or {}
|
||||
name = meta.get("name")
|
||||
if name:
|
||||
pvs[name] = pv
|
||||
mounts = {}
|
||||
for pod in pod_data.get("items") or []:
|
||||
meta = pod.get("metadata") or {}
|
||||
ns = str(meta.get("namespace") or "")
|
||||
pod_name = str(meta.get("name") or "")
|
||||
phase = str(((pod.get("status") or {}).get("phase")) or "")
|
||||
if phase in set(["Succeeded", "Failed"]):
|
||||
continue
|
||||
spec = pod.get("spec") or {}
|
||||
for vol in spec.get("volumes") or []:
|
||||
claim = (vol.get("persistentVolumeClaim") or {}).get("claimName")
|
||||
if claim:
|
||||
mounts.setdefault((ns, claim), []).append(pod_name)
|
||||
rows = []
|
||||
for pvc in pvc_data.get("items") or []:
|
||||
meta = pvc.get("metadata") or {}
|
||||
spec = pvc.get("spec") or {}
|
||||
status = pvc.get("status") or {}
|
||||
ns = str(meta.get("namespace") or "")
|
||||
name = str(meta.get("name") or "")
|
||||
if ns not in set(["hwlab-ci", "agentrun-ci"]):
|
||||
continue
|
||||
volume = str(spec.get("volumeName") or "")
|
||||
pv = pvs.get(volume) or {}
|
||||
pv_spec = pv.get("spec") or {}
|
||||
owner_refs = meta.get("ownerReferences") or []
|
||||
owner_kind = None
|
||||
owner_name = None
|
||||
if owner_refs:
|
||||
owner_kind = owner_refs[0].get("kind")
|
||||
owner_name = owner_refs[0].get("name")
|
||||
host_path = pv_host_path(pv)
|
||||
active = sorted(mounts.get((ns, name), []))
|
||||
estimated = du_size(host_path, 8) if host_path else None
|
||||
rows.append({
|
||||
"namespace": ns,
|
||||
"pvc": name,
|
||||
"volume": volume or None,
|
||||
"phase": status.get("phase"),
|
||||
"ownerKind": owner_kind,
|
||||
"owner": owner_name,
|
||||
"ownerGroup": pvc_owner_group(ns, owner_name),
|
||||
"storageClass": spec.get("storageClassName") or pv_spec.get("storageClassName"),
|
||||
"reclaimPolicy": pv_spec.get("persistentVolumeReclaimPolicy"),
|
||||
"hostPath": host_path,
|
||||
"activeMountPods": active,
|
||||
"estimatedBytes": estimated,
|
||||
"estimatedHuman": fmt_bytes(estimated or 0),
|
||||
})
|
||||
rows.sort(key=lambda item: safe_int(item.get("estimatedBytes")), reverse=True)
|
||||
by_namespace = {}
|
||||
by_owner_group = {}
|
||||
for row in rows:
|
||||
for bucket, key in [(by_namespace, row.get("namespace") or "unknown"), (by_owner_group, row.get("ownerGroup") or "unknown")]:
|
||||
current = bucket.setdefault(key, {"count": 0, "estimatedBytes": 0, "activeMountCount": 0})
|
||||
current["count"] += 1
|
||||
current["estimatedBytes"] += safe_int(row.get("estimatedBytes"))
|
||||
current["activeMountCount"] += len(row.get("activeMountPods") or [])
|
||||
current["estimatedHuman"] = fmt_bytes(current["estimatedBytes"])
|
||||
return {
|
||||
"scope": "hwlab-ci and agentrun-ci PVCs only",
|
||||
"pvcCount": len(rows),
|
||||
"estimatedBytes": sum(safe_int(row.get("estimatedBytes")) for row in rows),
|
||||
"estimatedHuman": fmt_bytes(sum(safe_int(row.get("estimatedBytes")) for row in rows)),
|
||||
"byNamespace": by_namespace,
|
||||
"byOwnerGroup": by_owner_group,
|
||||
"topPvcs": rows[:int(OPTIONS.get("limit") or 50)],
|
||||
"handoff": {
|
||||
"hwlab": {
|
||||
"dryRun": "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 200 --dry-run",
|
||||
"releasedPvs": "bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 200 --dry-run",
|
||||
},
|
||||
"agentrun": {
|
||||
"dryRun": "bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
|
||||
"releasedPvs": "bun scripts/cli.ts agentrun v01 control-plane cleanup-released-pvs --limit 200 --dry-run",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
def registry_growth_snapshot():
|
||||
summary = {
|
||||
"path": REGISTRY_ROOT,
|
||||
"sizeBytes": du_size(REGISTRY_ROOT, 60) or 0,
|
||||
}
|
||||
summary["sizeHuman"] = fmt_bytes(summary["sizeBytes"])
|
||||
if OPTIONS.get("hwlabRegistry", False):
|
||||
plan = plan_registry_retention()
|
||||
retention = dict(plan.get("summary") or {})
|
||||
for key in ["registrySizeBytes", "estimatedReclaimBytes"]:
|
||||
if key in retention:
|
||||
retention[key.replace("Bytes", "Human")] = fmt_bytes(retention.get(key) or 0)
|
||||
summary["retentionPlan"] = retention
|
||||
else:
|
||||
summary["retentionPlan"] = {
|
||||
"skipped": True,
|
||||
"reason": "rerun snapshot with --include-hwlab-registry to compute tag/revision retention counters",
|
||||
}
|
||||
summary["cadence"] = {
|
||||
"dryRun": "daily or before/after every v0.2 CI/CD burst",
|
||||
"maintenanceRun": "weekly, or when root >=80%, or when registry growth exceeds the agreed daily threshold",
|
||||
"planCommand": "bun scripts/cli.ts gc remote %s plan --target-use-percent 70 --include-hwlab-registry --limit 50" % PROVIDER_ID,
|
||||
"snapshotCommand": "bun scripts/cli.ts gc remote %s snapshot --include-hwlab-registry --history-limit 12" % PROVIDER_ID,
|
||||
"runCommand": "bun scripts/cli.ts gc remote %s run --confirm --include-hwlab-registry --target-use-percent 70 --limit 50" % PROVIDER_ID,
|
||||
"defaultRetention": {
|
||||
"keepPerRepo": int(OPTIONS.get("registryKeepPerRepo") or 20),
|
||||
"minAgeHours": float(OPTIONS.get("registryMinAgeHours") or 48),
|
||||
"protects": ["current workload refs", "digest closure", "protected tags", "recent tags", "newest N tags per repo"],
|
||||
},
|
||||
}
|
||||
return summary
|
||||
|
||||
def growth_watermark_policy(root_disk):
|
||||
use_percent = root_disk.get("usePercent") if isinstance(root_disk, dict) else None
|
||||
if use_percent is None:
|
||||
state = "unknown"
|
||||
action = "collect-snapshot"
|
||||
elif use_percent < 75:
|
||||
state = "healthy"
|
||||
action = "observe-trend"
|
||||
elif use_percent < 80:
|
||||
state = "watch"
|
||||
action = "run-dry-run-plan"
|
||||
elif use_percent < 85:
|
||||
state = "maintenance"
|
||||
action = "schedule-owner-aware-retention"
|
||||
else:
|
||||
state = "emergency"
|
||||
action = "restore-runtime-then-file-evidence"
|
||||
return {
|
||||
"state": state,
|
||||
"recommendedAction": action,
|
||||
"watermarks": [
|
||||
{"range": "<75%", "action": "trend only"},
|
||||
{"range": "75%-80%", "action": "run dry-run plan and identify source"},
|
||||
{"range": "80%-85%", "action": "small owner-aware retention run"},
|
||||
{"range": ">=85%", "action": "runtime recovery first, then root-cause growth source"},
|
||||
],
|
||||
"growthThresholdPolicy": "If bytes/day remains high for consecutive snapshots, act before 80%; exact threshold should be set from the first week of saved snapshots.",
|
||||
}
|
||||
|
||||
def snapshot_metric_map(snapshot):
|
||||
metrics = {}
|
||||
root = snapshot.get("rootDisk") or {}
|
||||
if isinstance(root, dict) and root.get("usedBytes") is not None:
|
||||
metrics["root.usedBytes"] = {"value": safe_int(root.get("usedBytes")), "unit": "bytes", "label": "root used bytes"}
|
||||
for item in snapshot.get("sources") or []:
|
||||
if not isinstance(item, dict) or item.get("sizeBytes") is None:
|
||||
continue
|
||||
key = "source.%s.sizeBytes" % item.get("id")
|
||||
metrics[key] = {"value": safe_int(item.get("sizeBytes")), "unit": "bytes", "label": item.get("label") or item.get("id")}
|
||||
storage = ((snapshot.get("ciStorage") or {}).get("byOwnerGroup") or {})
|
||||
for owner, value in storage.items():
|
||||
metrics["ciStorage.%s.estimatedBytes" % owner] = {"value": safe_int((value or {}).get("estimatedBytes")), "unit": "bytes", "label": "CI storage %s" % owner}
|
||||
registry = snapshot.get("registry") or {}
|
||||
retention = registry.get("retentionPlan") or {}
|
||||
for key in ["totalTags", "totalRevisions", "deleteTags", "deleteRevisions", "estimatedReclaimBytes"]:
|
||||
if key in retention and retention.get(key) is not None:
|
||||
unit = "bytes" if key.endswith("Bytes") else "count"
|
||||
metrics["registry.%s" % key] = {"value": safe_int(retention.get(key)), "unit": unit, "label": "registry %s" % key}
|
||||
return metrics
|
||||
|
||||
def delta_metric_rows(before, after):
|
||||
before_metrics = snapshot_metric_map(before)
|
||||
after_metrics = snapshot_metric_map(after)
|
||||
before_ts = iso_to_epoch(before.get("observedAt"))
|
||||
after_ts = iso_to_epoch(after.get("observedAt"))
|
||||
seconds = (after_ts - before_ts) if before_ts is not None and after_ts is not None else None
|
||||
rows = []
|
||||
for key in sorted(set(before_metrics.keys()) | set(after_metrics.keys())):
|
||||
old = before_metrics.get(key, {"value": 0, "unit": (after_metrics.get(key) or {}).get("unit"), "label": key})
|
||||
new = after_metrics.get(key, {"value": 0, "unit": old.get("unit"), "label": old.get("label")})
|
||||
delta = safe_int(new.get("value")) - safe_int(old.get("value"))
|
||||
row = {
|
||||
"key": key,
|
||||
"label": new.get("label") or old.get("label") or key,
|
||||
"unit": new.get("unit") or old.get("unit"),
|
||||
"before": old.get("value"),
|
||||
"after": new.get("value"),
|
||||
"delta": delta,
|
||||
}
|
||||
if row["unit"] == "bytes":
|
||||
row["beforeHuman"] = fmt_bytes(row["before"] or 0)
|
||||
row["afterHuman"] = fmt_bytes(row["after"] or 0)
|
||||
row["deltaHuman"] = ("-" if delta < 0 else "") + fmt_bytes(abs(delta))
|
||||
if seconds and seconds > 0:
|
||||
per_day = int(delta * 86400 / seconds)
|
||||
row["perDayBytes"] = per_day
|
||||
row["perDayHuman"] = ("-" if per_day < 0 else "") + fmt_bytes(abs(per_day)) + "/day"
|
||||
rows.append(row)
|
||||
rows.sort(key=lambda item: safe_int(item.get("delta")), reverse=True)
|
||||
return {"durationSeconds": seconds, "metrics": rows}
|
||||
|
||||
def growth_trend_payload(points):
|
||||
points = [point for point in points if isinstance(point, dict)]
|
||||
if len(points) < 2:
|
||||
return {
|
||||
"pointCount": len(points),
|
||||
"state": "insufficient-history",
|
||||
"message": "Run snapshot at least twice to compute deltas.",
|
||||
}
|
||||
latest_delta = delta_metric_rows(points[-2], points[-1])
|
||||
window_delta = delta_metric_rows(points[0], points[-1])
|
||||
def rate_warning(delta):
|
||||
seconds = delta.get("durationSeconds")
|
||||
if seconds is not None and seconds < 3600:
|
||||
return {
|
||||
"code": "short-window-rate-noisy",
|
||||
"message": "Per-day rates from windows shorter than 1 hour are directional only; use daily snapshots for governance decisions.",
|
||||
"durationSeconds": seconds,
|
||||
}
|
||||
return None
|
||||
return {
|
||||
"pointCount": len(points),
|
||||
"oldestAt": points[0].get("observedAt"),
|
||||
"latestAt": points[-1].get("observedAt"),
|
||||
"latestDelta": {
|
||||
"durationSeconds": latest_delta.get("durationSeconds"),
|
||||
"rateWarning": rate_warning(latest_delta),
|
||||
"topGrowingBytes": [row for row in latest_delta.get("metrics", []) if row.get("unit") == "bytes" and safe_int(row.get("delta")) > 0][:10],
|
||||
"topShrinkingBytes": [row for row in reversed(latest_delta.get("metrics", [])) if row.get("unit") == "bytes" and safe_int(row.get("delta")) < 0][:10],
|
||||
"registryCounters": [row for row in latest_delta.get("metrics", []) if str(row.get("key", "")).startswith("registry.") and row.get("unit") == "count"],
|
||||
},
|
||||
"windowDelta": {
|
||||
"durationSeconds": window_delta.get("durationSeconds"),
|
||||
"rateWarning": rate_warning(window_delta),
|
||||
"topGrowingBytes": [row for row in window_delta.get("metrics", []) if row.get("unit") == "bytes" and safe_int(row.get("delta")) > 0][:10],
|
||||
"topShrinkingBytes": [row for row in reversed(window_delta.get("metrics", [])) if row.get("unit") == "bytes" and safe_int(row.get("delta")) < 0][:10],
|
||||
"registryCounters": [row for row in window_delta.get("metrics", []) if str(row.get("key", "")).startswith("registry.") and row.get("unit") == "count"],
|
||||
},
|
||||
}
|
||||
|
||||
def collect_growth_snapshot(observed_at, preflight):
|
||||
root_disk = df_snapshot()
|
||||
sources = disk_source_snapshot()
|
||||
ci_storage = ci_storage_snapshot()
|
||||
registry = registry_growth_snapshot()
|
||||
containerd = containerd_breakdown_snapshot()
|
||||
return {
|
||||
"ok": True,
|
||||
"action": "gc remote snapshot",
|
||||
"providerId": PROVIDER_ID,
|
||||
"dryRun": True,
|
||||
"mutation": False,
|
||||
"diagnosticStateMutation": bool(OPTIONS.get("saveSnapshot", True)),
|
||||
"observedAt": observed_at,
|
||||
"rootDisk": root_disk,
|
||||
"clusterPreflight": preflight,
|
||||
"sources": sources,
|
||||
"registry": registry,
|
||||
"ciStorage": ci_storage,
|
||||
"containerd": containerd,
|
||||
"policy": growth_watermark_policy(root_disk or {}),
|
||||
"commands": {
|
||||
"snapshot": "bun scripts/cli.ts gc remote %s snapshot --include-hwlab-registry --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
|
||||
"trend": "bun scripts/cli.ts gc remote %s trend --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
|
||||
"registryPlan": "bun scripts/cli.ts gc remote %s plan --target-use-percent 70 --include-hwlab-registry --limit 50" % PROVIDER_ID,
|
||||
"hwlabCiRetention": "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 200 --dry-run",
|
||||
"agentrunRetention": "bun scripts/cli.ts agentrun v01 control-plane cleanup-runs --min-age-minutes 30 --limit 200 --dry-run",
|
||||
},
|
||||
}
|
||||
|
||||
def allocated_file_size(path):
|
||||
try:
|
||||
stat = os.stat(path)
|
||||
@@ -1714,6 +2113,56 @@ def plan_payload(observed_at, preflight, protected, candidates, visible):
|
||||
def main():
|
||||
observed_at = now_iso()
|
||||
preflight = cluster_preflight()
|
||||
if ACTION == "trend":
|
||||
history_limit = int(OPTIONS.get("historyLimit") or 12)
|
||||
history = read_growth_snapshots(history_limit)
|
||||
emit_json({
|
||||
"ok": True,
|
||||
"action": "gc remote trend",
|
||||
"providerId": PROVIDER_ID,
|
||||
"dryRun": True,
|
||||
"mutation": False,
|
||||
"observedAt": observed_at,
|
||||
"statePath": growth_snapshot_path(),
|
||||
"historyLimit": history_limit,
|
||||
"trend": growth_trend_payload(history),
|
||||
"points": history if bool(OPTIONS.get("full")) else history[-min(len(history), 3):],
|
||||
"returnedPointCount": min(len(history), 3) if not bool(OPTIONS.get("full")) else len(history),
|
||||
"totalPointCount": len(history),
|
||||
"next": {
|
||||
"snapshot": "bun scripts/cli.ts gc remote %s snapshot --include-hwlab-registry --history-limit %s" % (PROVIDER_ID, history_limit),
|
||||
},
|
||||
}, persist_large=True)
|
||||
return 0
|
||||
if ACTION == "snapshot":
|
||||
history_limit = int(OPTIONS.get("historyLimit") or 12)
|
||||
snapshot = collect_growth_snapshot(observed_at, preflight)
|
||||
state_path = growth_snapshot_path()
|
||||
if bool(OPTIONS.get("saveSnapshot", True)):
|
||||
state_path = append_growth_snapshot(snapshot)
|
||||
history = read_growth_snapshots(history_limit)
|
||||
if not bool(OPTIONS.get("saveSnapshot", True)):
|
||||
history = history + [snapshot]
|
||||
snapshot.update({
|
||||
"statePath": state_path,
|
||||
"historyLimit": history_limit,
|
||||
"saved": bool(OPTIONS.get("saveSnapshot", True)),
|
||||
"trend": growth_trend_payload(history[-history_limit:]),
|
||||
"history": {
|
||||
"totalPointCount": len(read_growth_snapshots(1000000)) if bool(OPTIONS.get("saveSnapshot", True)) else len(history),
|
||||
"returnedPointCount": len(history[-min(len(history), 3):]),
|
||||
"recentPoints": history[-min(len(history), 3):] if bool(OPTIONS.get("full")) else [
|
||||
{
|
||||
"observedAt": item.get("observedAt"),
|
||||
"rootDisk": item.get("rootDisk"),
|
||||
"sourceCount": len(item.get("sources") or []),
|
||||
}
|
||||
for item in history[-min(len(history), 3):]
|
||||
],
|
||||
},
|
||||
})
|
||||
emit_json(snapshot, persist_large=True)
|
||||
return 0
|
||||
protected = collect_protected()
|
||||
candidates = collect_candidates(observed_at)
|
||||
visible = visible_items(candidates)
|
||||
|
||||
Reference in New Issue
Block a user