fix: add JD01 GC retention controls
This commit is contained in:
Codex
@@ -0,0 +1,292 @@
|
||||
def registry_growth_snapshot():
|
||||
summary = {
|
||||
"path": REGISTRY_ROOT,
|
||||
"sizeBytes": du_size(REGISTRY_ROOT, 60) or 0,
|
||||
}
|
||||
summary["sizeHuman"] = fmt_bytes(summary["sizeBytes"])
|
||||
if OPTIONS.get("hwlabRegistry", False):
|
||||
plan = plan_registry_retention()
|
||||
retention = dict(plan.get("summary") or {})
|
||||
for key in ["registrySizeBytes", "estimatedReclaimBytes"]:
|
||||
if key in retention:
|
||||
retention[key.replace("Bytes", "Human")] = fmt_bytes(retention.get(key) or 0)
|
||||
summary["retentionPlan"] = retention
|
||||
else:
|
||||
summary["retentionPlan"] = {
|
||||
"skipped": True,
|
||||
"reason": "rerun snapshot with --include-hwlab-registry to compute tag/revision retention counters",
|
||||
}
|
||||
summary["cadence"] = {
|
||||
"dryRun": "daily or before/after every v0.2 CI/CD burst",
|
||||
"maintenanceRun": "weekly, or when root >=80%, or when registry growth exceeds the agreed daily threshold",
|
||||
"planCommand": "bun scripts/cli.ts gc remote %s plan --target-use-percent 70 --include-hwlab-registry --limit 50" % PROVIDER_ID,
|
||||
"snapshotCommand": "bun scripts/cli.ts gc remote %s snapshot --include-hwlab-registry --history-limit 12" % PROVIDER_ID,
|
||||
"runCommand": "bun scripts/cli.ts gc remote %s run --confirm --include-hwlab-registry --target-use-percent 70 --limit 50" % PROVIDER_ID,
|
||||
"defaultRetention": {
|
||||
"keepPerRepo": int(OPTIONS.get("registryKeepPerRepo") or 20),
|
||||
"minAgeHours": float(OPTIONS.get("registryMinAgeHours") or 48),
|
||||
"protects": ["current workload refs", "digest closure", "protected tags", "recent tags", "newest N tags per repo"],
|
||||
},
|
||||
}
|
||||
return summary
|
||||
|
||||
def growth_watermark_policy(root_disk):
|
||||
use_percent = root_disk.get("usePercent") if isinstance(root_disk, dict) else None
|
||||
if use_percent is None:
|
||||
state = "unknown"
|
||||
action = "collect-snapshot"
|
||||
elif use_percent < 75:
|
||||
state = "healthy"
|
||||
action = "observe-trend"
|
||||
elif use_percent < 80:
|
||||
state = "watch"
|
||||
action = "run-dry-run-plan"
|
||||
elif use_percent < 85:
|
||||
state = "maintenance"
|
||||
action = "schedule-owner-aware-retention"
|
||||
else:
|
||||
state = "emergency"
|
||||
action = "restore-runtime-then-file-evidence"
|
||||
return {
|
||||
"state": state,
|
||||
"recommendedAction": action,
|
||||
"watermarks": [
|
||||
{"range": "<75%", "action": "trend only"},
|
||||
{"range": "75%-80%", "action": "run dry-run plan and identify source"},
|
||||
{"range": "80%-85%", "action": "small owner-aware retention run"},
|
||||
{"range": ">=85%", "action": "runtime recovery first, then root-cause growth source"},
|
||||
],
|
||||
"growthThresholdPolicy": "If bytes/day remains high for consecutive snapshots, act before 80%; exact threshold should be set from the first week of saved snapshots.",
|
||||
}
|
||||
|
||||
def snapshot_metric_map(snapshot):
|
||||
metrics = {}
|
||||
root = snapshot.get("rootDisk") or {}
|
||||
if isinstance(root, dict) and root.get("usedBytes") is not None:
|
||||
metrics["root.usedBytes"] = {"value": safe_int(root.get("usedBytes")), "unit": "bytes", "label": "root used bytes"}
|
||||
for item in snapshot.get("sources") or []:
|
||||
if not isinstance(item, dict) or item.get("sizeBytes") is None:
|
||||
continue
|
||||
key = "source.%s.sizeBytes" % item.get("id")
|
||||
metrics[key] = {"value": safe_int(item.get("sizeBytes")), "unit": "bytes", "label": item.get("label") or item.get("id")}
|
||||
storage = ((snapshot.get("ciStorage") or {}).get("byOwnerGroup") or {})
|
||||
if not storage:
|
||||
storage = ((snapshot.get("pvcAttribution") or {}).get("byOwnerGroup") or {})
|
||||
for owner, value in storage.items():
|
||||
metrics["ciStorage.%s.estimatedBytes" % owner] = {"value": safe_int((value or {}).get("estimatedBytes")), "unit": "bytes", "label": "CI storage %s" % owner}
|
||||
memory = snapshot.get("memoryPressure") or {}
|
||||
memory_summary = memory.get("summary") or {}
|
||||
if memory_summary.get("matchedRssBytes") is not None:
|
||||
metrics["memoryPressure.matchedRssBytes"] = {"value": safe_int(memory_summary.get("matchedRssBytes")), "unit": "bytes", "label": "matched observer/chrome RSS"}
|
||||
if memory_summary.get("observeStateBytes") is not None:
|
||||
metrics["memoryPressure.observeStateBytes"] = {"value": safe_int(memory_summary.get("observeStateBytes")), "unit": "bytes", "label": "web observe state bytes"}
|
||||
for key in ["matchedProcessCount", "activeObserverSignals", "staleObserverSignals"]:
|
||||
if memory_summary.get(key) is not None:
|
||||
metrics["memoryPressure.%s" % key] = {"value": safe_int(memory_summary.get(key)), "unit": "count", "label": "memory pressure %s" % key}
|
||||
registry = snapshot.get("registry") or {}
|
||||
retention = registry.get("retentionPlan") or {}
|
||||
for key in ["totalTags", "totalRevisions", "deleteTags", "deleteRevisions", "estimatedReclaimBytes"]:
|
||||
if key in retention and retention.get(key) is not None:
|
||||
unit = "bytes" if key.endswith("Bytes") else "count"
|
||||
metrics["registry.%s" % key] = {"value": safe_int(retention.get(key)), "unit": unit, "label": "registry %s" % key}
|
||||
return metrics
|
||||
|
||||
def delta_metric_rows(before, after):
|
||||
before_metrics = snapshot_metric_map(before)
|
||||
after_metrics = snapshot_metric_map(after)
|
||||
before_ts = iso_to_epoch(before.get("observedAt"))
|
||||
after_ts = iso_to_epoch(after.get("observedAt"))
|
||||
seconds = (after_ts - before_ts) if before_ts is not None and after_ts is not None else None
|
||||
rows = []
|
||||
for key in sorted(set(before_metrics.keys()) | set(after_metrics.keys())):
|
||||
old = before_metrics.get(key, {"value": 0, "unit": (after_metrics.get(key) or {}).get("unit"), "label": key})
|
||||
new = after_metrics.get(key, {"value": 0, "unit": old.get("unit"), "label": old.get("label")})
|
||||
delta = safe_int(new.get("value")) - safe_int(old.get("value"))
|
||||
row = {
|
||||
"key": key,
|
||||
"label": new.get("label") or old.get("label") or key,
|
||||
"unit": new.get("unit") or old.get("unit"),
|
||||
"before": old.get("value"),
|
||||
"after": new.get("value"),
|
||||
"delta": delta,
|
||||
}
|
||||
if row["unit"] == "bytes":
|
||||
row["beforeHuman"] = fmt_bytes(row["before"] or 0)
|
||||
row["afterHuman"] = fmt_bytes(row["after"] or 0)
|
||||
row["deltaHuman"] = ("-" if delta < 0 else "") + fmt_bytes(abs(delta))
|
||||
if seconds and seconds > 0:
|
||||
per_day = int(delta * 86400 / seconds)
|
||||
row["perDayBytes"] = per_day
|
||||
row["perDayHuman"] = ("-" if per_day < 0 else "") + fmt_bytes(abs(per_day)) + "/day"
|
||||
rows.append(row)
|
||||
rows.sort(key=lambda item: safe_int(item.get("delta")), reverse=True)
|
||||
return {"durationSeconds": seconds, "metrics": rows}
|
||||
|
||||
def growth_trend_payload(points):
|
||||
points = [point for point in points if isinstance(point, dict)]
|
||||
if len(points) < 2:
|
||||
return {
|
||||
"pointCount": len(points),
|
||||
"state": "insufficient-history",
|
||||
"message": "Run snapshot at least twice to compute deltas.",
|
||||
}
|
||||
latest_delta = delta_metric_rows(points[-2], points[-1])
|
||||
window_delta = delta_metric_rows(points[0], points[-1])
|
||||
def rate_warning(delta):
|
||||
seconds = delta.get("durationSeconds")
|
||||
if seconds is not None and seconds < 3600:
|
||||
return {
|
||||
"code": "short-window-rate-noisy",
|
||||
"message": "Per-day rates from windows shorter than 1 hour are directional only; use daily snapshots for governance decisions.",
|
||||
"durationSeconds": seconds,
|
||||
}
|
||||
return None
|
||||
return {
|
||||
"pointCount": len(points),
|
||||
"oldestAt": points[0].get("observedAt"),
|
||||
"latestAt": points[-1].get("observedAt"),
|
||||
"latestDelta": {
|
||||
"durationSeconds": latest_delta.get("durationSeconds"),
|
||||
"rateWarning": rate_warning(latest_delta),
|
||||
"topGrowingBytes": [row for row in latest_delta.get("metrics", []) if row.get("unit") == "bytes" and safe_int(row.get("delta")) > 0][:10],
|
||||
"topShrinkingBytes": [row for row in reversed(latest_delta.get("metrics", [])) if row.get("unit") == "bytes" and safe_int(row.get("delta")) < 0][:10],
|
||||
"registryCounters": [row for row in latest_delta.get("metrics", []) if str(row.get("key", "")).startswith("registry.") and row.get("unit") == "count"],
|
||||
},
|
||||
"windowDelta": {
|
||||
"durationSeconds": window_delta.get("durationSeconds"),
|
||||
"rateWarning": rate_warning(window_delta),
|
||||
"topGrowingBytes": [row for row in window_delta.get("metrics", []) if row.get("unit") == "bytes" and safe_int(row.get("delta")) > 0][:10],
|
||||
"topShrinkingBytes": [row for row in reversed(window_delta.get("metrics", [])) if row.get("unit") == "bytes" and safe_int(row.get("delta")) < 0][:10],
|
||||
"registryCounters": [row for row in window_delta.get("metrics", []) if str(row.get("key", "")).startswith("registry.") and row.get("unit") == "count"],
|
||||
},
|
||||
}
|
||||
|
||||
def compact_metric_rows(rows, limit=3):
|
||||
compact = []
|
||||
for row in (rows or [])[:limit]:
|
||||
compact.append({
|
||||
"key": row.get("key"),
|
||||
"label": row.get("label"),
|
||||
"unit": row.get("unit"),
|
||||
"delta": row.get("delta"),
|
||||
"deltaHuman": row.get("deltaHuman"),
|
||||
"perDayHuman": row.get("perDayHuman"),
|
||||
})
|
||||
return compact
|
||||
|
||||
def compact_trend_payload(payload):
|
||||
if payload.get("state") == "insufficient-history":
|
||||
return payload
|
||||
latest = payload.get("latestDelta") or {}
|
||||
window = payload.get("windowDelta") or {}
|
||||
return {
|
||||
"pointCount": payload.get("pointCount"),
|
||||
"oldestAt": payload.get("oldestAt"),
|
||||
"latestAt": payload.get("latestAt"),
|
||||
"latestDelta": {
|
||||
"durationSeconds": latest.get("durationSeconds"),
|
||||
"rateWarning": latest.get("rateWarning"),
|
||||
"topGrowingBytes": compact_metric_rows(latest.get("topGrowingBytes") or [], 1),
|
||||
"topShrinkingBytes": compact_metric_rows(latest.get("topShrinkingBytes") or [], 1),
|
||||
"registryCounters": compact_metric_rows(latest.get("registryCounters") or [], 1),
|
||||
},
|
||||
"windowDelta": {
|
||||
"durationSeconds": window.get("durationSeconds"),
|
||||
"rateWarning": window.get("rateWarning"),
|
||||
"topGrowingBytes": compact_metric_rows(window.get("topGrowingBytes") or [], 1),
|
||||
"topShrinkingBytes": compact_metric_rows(window.get("topShrinkingBytes") or [], 1),
|
||||
"registryCounters": compact_metric_rows(window.get("registryCounters") or [], 1),
|
||||
},
|
||||
"fullDisclosure": "rerun trend --full for all metric rows",
|
||||
}
|
||||
|
||||
def compact_growth_point(item):
|
||||
registry = item.get("registry") or {}
|
||||
retention = registry.get("retentionPlan") or {}
|
||||
ci_storage = item.get("ciStorage") or {}
|
||||
containerd = item.get("containerd") or {}
|
||||
memory = item.get("memoryPressure") or {}
|
||||
memory_summary = memory.get("summary") or {}
|
||||
observe = (memory.get("webObserve") or {})
|
||||
return {
|
||||
"observedAt": item.get("observedAt"),
|
||||
"rootDisk": item.get("rootDisk"),
|
||||
"sourceCount": len(item.get("sources") or []),
|
||||
"registry": {
|
||||
"sizeBytes": registry.get("sizeBytes"),
|
||||
"sizeHuman": registry.get("sizeHuman"),
|
||||
"totalTags": retention.get("totalTags"),
|
||||
"totalRevisions": retention.get("totalRevisions"),
|
||||
"deleteTags": retention.get("deleteTags"),
|
||||
"deleteRevisions": retention.get("deleteRevisions"),
|
||||
"estimatedReclaimBytes": retention.get("estimatedReclaimBytes"),
|
||||
"estimatedReclaimHuman": retention.get("estimatedReclaimHuman"),
|
||||
},
|
||||
"ciStorage": {
|
||||
"pvcCount": ci_storage.get("pvcCount"),
|
||||
"estimatedBytes": ci_storage.get("estimatedBytes"),
|
||||
"estimatedHuman": ci_storage.get("estimatedHuman"),
|
||||
"byOwnerGroup": ci_storage.get("byOwnerGroup"),
|
||||
},
|
||||
"containerd": {
|
||||
"state": containerd.get("state"),
|
||||
"cleanupSupported": containerd.get("cleanupSupported"),
|
||||
},
|
||||
"memoryPressure": {
|
||||
"matchedProcessCount": memory_summary.get("matchedProcessCount"),
|
||||
"matchedRssBytes": memory_summary.get("matchedRssBytes"),
|
||||
"matchedRssHuman": memory_summary.get("matchedRssHuman"),
|
||||
"activeObserverSignals": memory_summary.get("activeObserverSignals"),
|
||||
"staleObserverSignals": memory_summary.get("staleObserverSignals"),
|
||||
"observeStateBytes": memory_summary.get("observeStateBytes"),
|
||||
"observeStateHuman": memory_summary.get("observeStateHuman"),
|
||||
"webObserveRootCount": observe.get("rootCount"),
|
||||
},
|
||||
}
|
||||
|
||||
def collect_growth_snapshot(observed_at, preflight):
|
||||
root_disk = df_snapshot()
|
||||
sources = disk_source_snapshot()
|
||||
ci_storage = ci_storage_snapshot()
|
||||
memory_pressure = collect_memory_pressure()
|
||||
compact_pvc = compact_pvc_attribution(ci_storage)
|
||||
if bool(OPTIONS.get("full")):
|
||||
public_pvc = ci_storage
|
||||
public_memory = memory_pressure
|
||||
else:
|
||||
public_pvc = compact_ci_storage_summary(ci_storage)
|
||||
public_memory = compact_memory_summary(memory_pressure)
|
||||
registry = registry_growth_snapshot()
|
||||
containerd = containerd_breakdown_snapshot()
|
||||
commands = {
|
||||
"snapshot": "bun scripts/cli.ts gc remote %s snapshot --include-hwlab-registry --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
|
||||
"trend": "bun scripts/cli.ts gc remote %s trend --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
|
||||
"registryPlan": "bun scripts/cli.ts gc remote %s plan --target-use-percent 70 --include-hwlab-registry --limit 50" % PROVIDER_ID,
|
||||
"hwlabCiRetention": ((ci_storage.get("handoff") or {}).get("hwlab") or {}).get("dryRun"),
|
||||
"agentrunRetention": ((ci_storage.get("handoff") or {}).get("agentrun") or {}).get("dryRun"),
|
||||
"remotePolicy": "bun scripts/cli.ts gc remote %s policy plan" % PROVIDER_ID,
|
||||
}
|
||||
if not bool(OPTIONS.get("full")):
|
||||
commands = {
|
||||
"trend": "bun scripts/cli.ts gc remote %s trend --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
|
||||
"status": "bun scripts/cli.ts gc remote %s status --limit %s" % (PROVIDER_ID, int(OPTIONS.get("limit") or 50)),
|
||||
"full": "bun scripts/cli.ts gc remote %s snapshot --full --no-save" % PROVIDER_ID,
|
||||
}
|
||||
return {
|
||||
"ok": True,
|
||||
"action": "gc remote snapshot",
|
||||
"providerId": PROVIDER_ID,
|
||||
"dryRun": True,
|
||||
"mutation": False,
|
||||
"diagnosticStateMutation": bool(OPTIONS.get("saveSnapshot", True)),
|
||||
"observedAt": observed_at,
|
||||
"rootDisk": root_disk,
|
||||
"clusterPreflight": preflight,
|
||||
"sources": sources,
|
||||
"registry": registry,
|
||||
"pvcAttribution": public_pvc,
|
||||
"memoryPressure": public_memory,
|
||||
"containerd": containerd,
|
||||
"policy": growth_watermark_policy(root_disk or {}),
|
||||
"commands": commands,
|
||||
}
|
||||
Reference in New Issue
Block a user