pikasTech-unidesk/scripts/src/gc-remote-growth.py

def registry_growth_snapshot():
    summary = {
        "path": REGISTRY_ROOT,
        "sizeBytes": du_size(REGISTRY_ROOT, 60) or 0,
    }
    summary["sizeHuman"] = fmt_bytes(summary["sizeBytes"])
    if OPTIONS.get("hwlabRegistry", False):
        plan = plan_registry_retention()
        retention = dict(plan.get("summary") or {})
        for key in ["registrySizeBytes", "estimatedReclaimBytes"]:
            if key in retention:
                retention[key.replace("Bytes", "Human")] = fmt_bytes(retention.get(key) or 0)
        summary["retentionPlan"] = retention
    else:
        summary["retentionPlan"] = {
            "skipped": True,
            "reason": "rerun snapshot with --include-hwlab-registry to compute tag/revision retention counters",
        }
    summary["cadence"] = {
        "dryRun": "daily or before/after every v0.2 CI/CD burst",
        "maintenanceRun": "weekly, or when root >=80%, or when registry growth exceeds the agreed daily threshold",
        "planCommand": "bun scripts/cli.ts gc remote %s plan --target-use-percent 70 --include-hwlab-registry --limit 50" % PROVIDER_ID,
        "snapshotCommand": "bun scripts/cli.ts gc remote %s snapshot --include-hwlab-registry --history-limit 12" % PROVIDER_ID,
        "runCommand": "bun scripts/cli.ts gc remote %s run --confirm --include-hwlab-registry --target-use-percent 70 --limit 50" % PROVIDER_ID,
        "defaultRetention": {
            "keepPerRepo": int(OPTIONS.get("registryKeepPerRepo") or 20),
            "minAgeHours": float(OPTIONS.get("registryMinAgeHours") or 48),
            "protects": ["current workload refs", "digest closure", "protected tags", "recent tags", "newest N tags per repo"],
        },
    }
    return summary

def growth_watermark_policy(root_disk):
    use_percent = root_disk.get("usePercent") if isinstance(root_disk, dict) else None
    if use_percent is None:
        state = "unknown"
        action = "collect-snapshot"
    elif use_percent < 75:
        state = "healthy"
        action = "observe-trend"
    elif use_percent < 80:
        state = "watch"
        action = "run-dry-run-plan"
    elif use_percent < 85:
        state = "maintenance"
        action = "schedule-owner-aware-retention"
    else:
        state = "emergency"
        action = "restore-runtime-then-file-evidence"
    return {
        "state": state,
        "recommendedAction": action,
        "watermarks": [
            {"range": "<75%", "action": "trend only"},
            {"range": "75%-80%", "action": "run dry-run plan and identify source"},
            {"range": "80%-85%", "action": "small owner-aware retention run"},
            {"range": ">=85%", "action": "runtime recovery first, then root-cause growth source"},
        ],
        "growthThresholdPolicy": "If bytes/day remains high for consecutive snapshots, act before 80%; exact threshold should be set from the first week of saved snapshots.",
    }

def snapshot_metric_map(snapshot):
    metrics = {}
    root = snapshot.get("rootDisk") or {}
    if isinstance(root, dict) and root.get("usedBytes") is not None:
        metrics["root.usedBytes"] = {"value": safe_int(root.get("usedBytes")), "unit": "bytes", "label": "root used bytes"}
    for item in snapshot.get("sources") or []:
        if not isinstance(item, dict) or item.get("sizeBytes") is None:
            continue
        key = "source.%s.sizeBytes" % item.get("id")
        metrics[key] = {"value": safe_int(item.get("sizeBytes")), "unit": "bytes", "label": item.get("label") or item.get("id")}
    storage = ((snapshot.get("ciStorage") or {}).get("byOwnerGroup") or {})
    if not storage:
        storage = ((snapshot.get("pvcAttribution") or {}).get("byOwnerGroup") or {})
    for owner, value in storage.items():
        metrics["ciStorage.%s.estimatedBytes" % owner] = {"value": safe_int((value or {}).get("estimatedBytes")), "unit": "bytes", "label": "CI storage %s" % owner}
    memory = snapshot.get("memoryPressure") or {}
    memory_summary = memory.get("summary") or {}
    if memory_summary.get("matchedRssBytes") is not None:
        metrics["memoryPressure.matchedRssBytes"] = {"value": safe_int(memory_summary.get("matchedRssBytes")), "unit": "bytes", "label": "matched observer/chrome RSS"}
    if memory_summary.get("observeStateBytes") is not None:
        metrics["memoryPressure.observeStateBytes"] = {"value": safe_int(memory_summary.get("observeStateBytes")), "unit": "bytes", "label": "web observe state bytes"}
    for key in ["matchedProcessCount", "activeObserverSignals", "staleObserverSignals"]:
        if memory_summary.get(key) is not None:
            metrics["memoryPressure.%s" % key] = {"value": safe_int(memory_summary.get(key)), "unit": "count", "label": "memory pressure %s" % key}
    registry = snapshot.get("registry") or {}
    retention = registry.get("retentionPlan") or {}
    for key in ["totalTags", "totalRevisions", "deleteTags", "deleteRevisions", "estimatedReclaimBytes"]:
        if key in retention and retention.get(key) is not None:
            unit = "bytes" if key.endswith("Bytes") else "count"
            metrics["registry.%s" % key] = {"value": safe_int(retention.get(key)), "unit": unit, "label": "registry %s" % key}
    return metrics

def delta_metric_rows(before, after):
    before_metrics = snapshot_metric_map(before)
    after_metrics = snapshot_metric_map(after)
    before_ts = iso_to_epoch(before.get("observedAt"))
    after_ts = iso_to_epoch(after.get("observedAt"))
    seconds = (after_ts - before_ts) if before_ts is not None and after_ts is not None else None
    rows = []
    for key in sorted(set(before_metrics.keys()) | set(after_metrics.keys())):
        old = before_metrics.get(key, {"value": 0, "unit": (after_metrics.get(key) or {}).get("unit"), "label": key})
        new = after_metrics.get(key, {"value": 0, "unit": old.get("unit"), "label": old.get("label")})
        delta = safe_int(new.get("value")) - safe_int(old.get("value"))
        row = {
            "key": key,
            "label": new.get("label") or old.get("label") or key,
            "unit": new.get("unit") or old.get("unit"),
            "before": old.get("value"),
            "after": new.get("value"),
            "delta": delta,
        }
        if row["unit"] == "bytes":
            row["beforeHuman"] = fmt_bytes(row["before"] or 0)
            row["afterHuman"] = fmt_bytes(row["after"] or 0)
            row["deltaHuman"] = ("-" if delta < 0 else "") + fmt_bytes(abs(delta))
            if seconds and seconds > 0:
                per_day = int(delta * 86400 / seconds)
                row["perDayBytes"] = per_day
                row["perDayHuman"] = ("-" if per_day < 0 else "") + fmt_bytes(abs(per_day)) + "/day"
        rows.append(row)
    rows.sort(key=lambda item: safe_int(item.get("delta")), reverse=True)
    return {"durationSeconds": seconds, "metrics": rows}

def growth_trend_payload(points):
    points = [point for point in points if isinstance(point, dict)]
    if len(points) < 2:
        return {
            "pointCount": len(points),
            "state": "insufficient-history",
            "message": "Run snapshot at least twice to compute deltas.",
        }
    latest_delta = delta_metric_rows(points[-2], points[-1])
    window_delta = delta_metric_rows(points[0], points[-1])
    def rate_warning(delta):
        seconds = delta.get("durationSeconds")
        if seconds is not None and seconds < 3600:
            return {
                "code": "short-window-rate-noisy",
                "message": "Per-day rates from windows shorter than 1 hour are directional only; use daily snapshots for governance decisions.",
                "durationSeconds": seconds,
            }
        return None
    return {
        "pointCount": len(points),
        "oldestAt": points[0].get("observedAt"),
        "latestAt": points[-1].get("observedAt"),
        "latestDelta": {
            "durationSeconds": latest_delta.get("durationSeconds"),
            "rateWarning": rate_warning(latest_delta),
            "topGrowingBytes": [row for row in latest_delta.get("metrics", []) if row.get("unit") == "bytes" and safe_int(row.get("delta")) > 0][:10],
            "topShrinkingBytes": [row for row in reversed(latest_delta.get("metrics", [])) if row.get("unit") == "bytes" and safe_int(row.get("delta")) < 0][:10],
            "registryCounters": [row for row in latest_delta.get("metrics", []) if str(row.get("key", "")).startswith("registry.") and row.get("unit") == "count"],
        },
        "windowDelta": {
            "durationSeconds": window_delta.get("durationSeconds"),
            "rateWarning": rate_warning(window_delta),
            "topGrowingBytes": [row for row in window_delta.get("metrics", []) if row.get("unit") == "bytes" and safe_int(row.get("delta")) > 0][:10],
            "topShrinkingBytes": [row for row in reversed(window_delta.get("metrics", [])) if row.get("unit") == "bytes" and safe_int(row.get("delta")) < 0][:10],
            "registryCounters": [row for row in window_delta.get("metrics", []) if str(row.get("key", "")).startswith("registry.") and row.get("unit") == "count"],
        },
    }

def compact_metric_rows(rows, limit=3):
    compact = []
    for row in (rows or [])[:limit]:
        compact.append({
            "key": row.get("key"),
            "label": row.get("label"),
            "unit": row.get("unit"),
            "delta": row.get("delta"),
            "deltaHuman": row.get("deltaHuman"),
            "perDayHuman": row.get("perDayHuman"),
        })
    return compact

def compact_trend_payload(payload):
    if payload.get("state") == "insufficient-history":
        return payload
    latest = payload.get("latestDelta") or {}
    window = payload.get("windowDelta") or {}
    return {
        "pointCount": payload.get("pointCount"),
        "oldestAt": payload.get("oldestAt"),
        "latestAt": payload.get("latestAt"),
        "latestDelta": {
            "durationSeconds": latest.get("durationSeconds"),
            "rateWarning": latest.get("rateWarning"),
            "topGrowingBytes": compact_metric_rows(latest.get("topGrowingBytes") or [], 1),
            "topShrinkingBytes": compact_metric_rows(latest.get("topShrinkingBytes") or [], 1),
            "registryCounters": compact_metric_rows(latest.get("registryCounters") or [], 1),
        },
        "windowDelta": {
            "durationSeconds": window.get("durationSeconds"),
            "rateWarning": window.get("rateWarning"),
            "topGrowingBytes": compact_metric_rows(window.get("topGrowingBytes") or [], 1),
            "topShrinkingBytes": compact_metric_rows(window.get("topShrinkingBytes") or [], 1),
            "registryCounters": compact_metric_rows(window.get("registryCounters") or [], 1),
        },
        "fullDisclosure": "rerun trend --full for all metric rows",
    }

def compact_growth_point(item):
    registry = item.get("registry") or {}
    retention = registry.get("retentionPlan") or {}
    ci_storage = item.get("ciStorage") or {}
    containerd = item.get("containerd") or {}
    memory = item.get("memoryPressure") or {}
    memory_summary = memory.get("summary") or {}
    observe = (memory.get("webObserve") or {})
    return {
        "observedAt": item.get("observedAt"),
        "rootDisk": item.get("rootDisk"),
        "sourceCount": len(item.get("sources") or []),
        "registry": {
            "sizeBytes": registry.get("sizeBytes"),
            "sizeHuman": registry.get("sizeHuman"),
            "totalTags": retention.get("totalTags"),
            "totalRevisions": retention.get("totalRevisions"),
            "deleteTags": retention.get("deleteTags"),
            "deleteRevisions": retention.get("deleteRevisions"),
            "estimatedReclaimBytes": retention.get("estimatedReclaimBytes"),
            "estimatedReclaimHuman": retention.get("estimatedReclaimHuman"),
        },
        "ciStorage": {
            "pvcCount": ci_storage.get("pvcCount"),
            "estimatedBytes": ci_storage.get("estimatedBytes"),
            "estimatedHuman": ci_storage.get("estimatedHuman"),
            "byOwnerGroup": ci_storage.get("byOwnerGroup"),
        },
        "containerd": {
            "state": containerd.get("state"),
            "cleanupSupported": containerd.get("cleanupSupported"),
        },
        "memoryPressure": {
            "matchedProcessCount": memory_summary.get("matchedProcessCount"),
            "matchedRssBytes": memory_summary.get("matchedRssBytes"),
            "matchedRssHuman": memory_summary.get("matchedRssHuman"),
            "activeObserverSignals": memory_summary.get("activeObserverSignals"),
            "staleObserverSignals": memory_summary.get("staleObserverSignals"),
            "observeStateBytes": memory_summary.get("observeStateBytes"),
            "observeStateHuman": memory_summary.get("observeStateHuman"),
            "webObserveRootCount": observe.get("rootCount"),
        },
    }

def collect_growth_snapshot(observed_at, preflight):
    root_disk = df_snapshot()
    sources = disk_source_snapshot()
    ci_storage = ci_storage_snapshot()
    memory_pressure = collect_memory_pressure()
    compact_pvc = compact_pvc_attribution(ci_storage)
    if bool(OPTIONS.get("full")):
        public_pvc = ci_storage
        public_memory = memory_pressure
    else:
        public_pvc = compact_ci_storage_summary(ci_storage)
        public_memory = compact_memory_summary(memory_pressure)
    registry = registry_growth_snapshot()
    containerd = containerd_breakdown_snapshot()
    commands = {
        "snapshot": "bun scripts/cli.ts gc remote %s snapshot --include-hwlab-registry --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
        "trend": "bun scripts/cli.ts gc remote %s trend --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
        "registryPlan": "bun scripts/cli.ts gc remote %s plan --target-use-percent 70 --include-hwlab-registry --limit 50" % PROVIDER_ID,
        "hwlabCiRetention": ((ci_storage.get("handoff") or {}).get("hwlab") or {}).get("dryRun"),
        "agentrunRetention": ((ci_storage.get("handoff") or {}).get("agentrun") or {}).get("dryRun"),
        "remotePolicy": "bun scripts/cli.ts gc remote %s policy plan" % PROVIDER_ID,
    }
    if not bool(OPTIONS.get("full")):
        commands = {
            "trend": "bun scripts/cli.ts gc remote %s trend --history-limit %s" % (PROVIDER_ID, int(OPTIONS.get("historyLimit") or 12)),
            "status": "bun scripts/cli.ts gc remote %s status --limit %s" % (PROVIDER_ID, int(OPTIONS.get("limit") or 50)),
            "full": "bun scripts/cli.ts gc remote %s snapshot --full --no-save" % PROVIDER_ID,
        }
    return {
        "ok": True,
        "action": "gc remote snapshot",
        "providerId": PROVIDER_ID,
        "dryRun": True,
        "mutation": False,
        "diagnosticStateMutation": bool(OPTIONS.get("saveSnapshot", True)),
        "observedAt": observed_at,
        "rootDisk": root_disk,
        "clusterPreflight": preflight,
        "sources": sources,
        "registry": registry,
        "pvcAttribution": public_pvc,
        "memoryPressure": public_memory,
        "containerd": containerd,
        "policy": growth_watermark_policy(root_disk or {}),
        "commands": commands,
    }