From 8be605aac320eccaadd92a6fcba6cbef2999c54f Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 2 Jun 2026 00:26:28 +0000 Subject: [PATCH] feat: report remote gc target shortfall --- docs/reference/cli.md | 2 +- docs/reference/gc.md | 24 ++++----- scripts/src/gc-remote.ts | 110 +++++++++++++++++++++++++++++++++------ scripts/src/help.ts | 2 + 4 files changed, 110 insertions(+), 28 deletions(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 6c9f2b1c..badc9331 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -25,7 +25,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 DEV/PROD 滚动、P - `server swap status|ensure [--path /swapfile] [--size 2GiB] [--dry-run]` 是主 server swap 管理入口。`status` 仅读 `/proc/meminfo`、`/proc/swaps` 和 `/etc/fstab` 并返回 JSON；`ensure` 在已有任何 active swap 时只报告 no-op，在无 active swap 时创建固定 swapfile、`chmod 600`、`mkswap`、`swapon` 并尽量写入 `/etc/fstab`。输出必须包含 `before`、`after`、total memory、active swap、持久化状态、关键动作和错误详情；若 swap 已启用但 fstab 写入失败，状态为 `degraded`，调用者需按返回的 detail 修复持久化。 - `server logs` 返回 `logs/` 文件日志和 Docker 容器日志的尾部，默认限制输出大小，避免日志爆炸。实现必须只读取文件末尾字节，不得为了 tail 先把巨大日志完整读入 CLI 内存。 - `server cleanup plan [--min-age-hours N] [--limit N]` 只生成主 server Docker 镜像清理 dry-run 计划，不执行删除；默认 `--min-age-hours 24`，避免把刚发布或刚验证的镜像列为 stale。输出必须包含 `dryRun=true`、`mutation=false`、`policy.deletionExecuted=false`、active containers/images、受保护镜像、candidate stale images、估算释放空间、风险等级、`commandsToReview` 和人工审批清单。计划必须保守白名单：保留 running containers 使用的 image ID，保留 stopped containers 引用的 image ID 直到人工先复核容器，保留 `deploy.json`/`CI.json` 当前 commit-pinned artifact、Compose stable image、上游 digest pin 和 provider-gateway runner image；`protectedStorage` 必须显式列出 PostgreSQL named volume、Baidu Netdisk `.state`、D601 registry storage 和 Docker volumes/host data policy。该入口禁止生成或执行 `docker system prune`、`docker image prune`、`docker builder prune`、`docker volume rm`、`docker compose down -v`、数据库清理或 host data `rm` 命令；未来若增加真实删除，必须另设显式审批参数并先复核 dry-run 输出。 -- `gc plan|run --confirm|db-trace|policy|remote` 是主 server 和受控 provider 的磁盘高水位一次性缓解与长期防膨胀入口。`plan` 只读输出候选、风险、估算收益和保护对象；`run` 必须显式 `--confirm`；`gc remote ...` 通过 UniDesk SSH 透传执行远端 GC，G14/HWLAB registry retention、受限 core dump、保护对象、safe-stop 线和长期收益表的权威规则见 `docs/reference/gc.md`。 +- `gc plan|run --confirm|db-trace|policy|remote` 是主 server 和受控 provider 的磁盘高水位一次性缓解与长期防膨胀入口。`plan` 只读输出候选、风险、估算收益和保护对象；`run` 必须显式 `--confirm`；`gc remote ...` 通过 UniDesk SSH 透传执行远端 GC，`--target-use-percent N` 会在 `summary.target` 中报告目标水位所需释放量、候选估算、预计水位、缺口和 safe-stop 决策。G14/HWLAB registry retention、受限 core dump、保护对象、safe-stop 线和长期收益表的权威规则见 `docs/reference/gc.md`。 - `server rebuild ` 创建异步 job，先构建目标服务镜像，随后在 `.state/locks/server-compose.lock` 串行保护下用 `--no-deps --force-recreate` 替换目标 service 并等待容器 `healthy/running`；该命令用于替代手工删除容器的兜底流程，其中 `dev-frontend-proxy` 只更新主 server dev 入口薄代理，`todo-note`、`code-queue-mgr`、`project-manager`、`baidu-netdisk` 和 `oa-event-flow` 只重建主 server 承载的对应后端，不会重建或删除 database 命名卷。D601 Code Queue 执行面不由 `server rebuild` 管理，Rust backend-core 迭代不得用 `server rebuild backend-core` 在 master server 编译，规则见 `docs/reference/dev-environment.md`。 - `provider attach [--master-server URL] [--up] [--force]` 在新计算节点生成两项配置的 provider-gateway 挂载包：`.state/provider-.env` 默认只包含 `UNIDESK_MASTER_SERVER` 与 `PROVIDER_ID`，`provider-.yml` 固定 Docker socket、`pid: "host"`、`restart: always`、只读 `/workspace` 和 SSH 维护私钥挂载；`--up` 会立即执行生成的 `docker compose up -d --build`。`provider triage [--observed-error text] [--observed-scope scope] [--microservice id ...] [--full|--raw]` 是只读多信号健康裁决入口，会把单路径 `provider is not online`、SSH 超时、registry 失败和 service proxy 失败归类成 `runner-local-observation-gap`、`service-degraded`、`provider-degraded` 或 `global-blocker`。默认输出只返回裁决、scope、失败/降级/未知信号和有界 evidence 摘要，完整 evidence 必须显式加 `--full` 或 `--raw`；推荐交叉验证命令仍包含 `debug health`、`debug dispatch host.ssh --wait-ms 15000`、`ssh argv true`、`artifact-registry health --provider-id `、`microservice health k3sctl-adapter`、`microservice health code-queue` 和 `codex tasks --view supervisor --limit 20`。 - `ssh [operation args...]` / `tran [operation args...]` 通过 backend-core 内网 WebSocket broker 和 provider-gateway 的 Host SSH / WSL SSH 维护桥连接目标节点；`route` 基础形态是 provider id，例如 `D601` 或 `G14`，也可以扩展为纯定位路径 `provider:plane[:namespace:resource[:container]]`，例如 `D601:win`、`D601:win/c/test`、`G14:k3s`、`D601:k3s` 或 `G14:k3s::`。WSL provider 的 Windows cmd 入口固定写 `tran D601:win cmd `，需要 Windows cwd 时用 `tran D601:win/c/test cmd cd`，由 CLI 自动设置 `chcp 65001`、`PYTHONUTF8=1` 和 `PYTHONIOENCODING=utf-8`；命名只允许 `win`，不得使用 `win32`。非交互远端命令优先使用 `ssh argv ...`；需要 shell 脚本、管道、变量或循环时优先使用 quoted heredoc 单步传输，例如 `tran G14 script <<'SCRIPT'`、`tran G14:k3s script <<'SCRIPT'` 或 `tran G14:k3s:: script <<'SCRIPT'`，把脚本走 stdin。`script -- '<单个字符串>'` 是无需 stdin 的远端 shell one-liner，例如 `tran G14:/root/hwlab script -- 'cd /root/hwlab && git status --short --branch'`；`script -- <多个 argv>` 才是 direct argv，适合 `tran D601:/path script -- sed -n '1,20p' file` 这类带短横线的单进程命令。顶层 remote option parser 必须保留命令已经开始后的 `--`，不得把它吞成全局选项结束符。需要远端改文本文件时默认优先使用 ` apply-patch < patch.diff`；需要可靠传输非文本或整文件时使用 ` upload ` 和 ` download `，CLI 会按字节数与 SHA-256 自动校验并在 provider-gateway stdin/argv 限制下切换客户端分块策略；需要旧 helper 时显式使用 `:k3s:: apply-patch-v1` 或 ` apply-patch-v1`。ssh-like 命令遇到 timeout/kex/255 类失败时，CLI 会在 stderr 追加一行 `UNIDESK_SSH_HINT` JSON，提示 stdin script/argv 重试和 provider triage 交叉验证。 diff --git a/docs/reference/gc.md b/docs/reference/gc.md index bd4aaa56..f668e399 100644 --- a/docs/reference/gc.md +++ b/docs/reference/gc.md @@ -10,7 +10,7 @@ UniDesk 的磁盘治理入口是 `bun scripts/cli.ts gc ...`。该入口用于 - `gc db-trace plan|run --confirm --before-date YYYY-MM-DD --vacuum-full`：显式 trace 遥测留存入口；涉及数据库重写时按维护窗口处理。 - `gc remote plan|run --confirm|status --job-id `：通过 UniDesk SSH 透传在 provider host 上执行受控 GC。远端长任务必须使用异步 job 和 `status` 短查询，不应让单次 SSH 等待完整 registry GC 或其他长清理。 -所有成功和失败输出都必须是 JSON。`plan` 必须标记 `dryRun=true`、`mutation=false`；`run` 必须要求 `--confirm` 并报告 `diskBefore`、`diskAfter`、`summary`、`results` 和 `protected`。 +所有成功和失败输出都必须是 JSON。`plan` 必须标记 `dryRun=true`、`mutation=false`；`run` 必须要求 `--confirm` 并报告 `diskBefore`、`diskAfter`、`summary`、`results` 和 `protected`。远端 GC 可用 `--target-use-percent N` 显式表达目标根盘水位；`summary.target` 必须给出目标所需释放量、候选估算、预计水位、缺口和 `safeStop` 决策，避免靠人工心算判断是否应该继续扩大清理范围。 ## Protected Data @@ -74,14 +74,14 @@ Registry 执行必须以远端异步 job 完成，并具备以下维护保护： ## Safe Stop Line -磁盘目标可以设为 `<70%`，但安全边界优先级高于目标百分比。当执行以下命令后仍高于目标时，应停止自动清理并提交决策表： +磁盘目标可以设为 `<70%` 或维护窗口临时目标如 `<50%`，但安全边界优先级高于目标百分比。目标必须直接传给 CLI，使输出带有可机读缺口： ```bash -bun scripts/cli.ts gc remote G14 plan --limit 20 -bun scripts/cli.ts gc remote G14 plan --include-hwlab-registry --limit 20 +bun scripts/cli.ts gc remote G14 plan --target-use-percent 50 --limit 20 +bun scripts/cli.ts gc remote G14 plan --target-use-percent 50 --include-hwlab-registry --limit 20 ``` -若两个 plan 都没有有意义候选，剩余空间通常位于 registry 保留集、k3s runtime、containerd image cache 或 PVC 数据中。继续下降需要显式选择以下更高风险方案之一： +当 `summary.target.safeStop=true`、`state` 为 `shortfall` 或 `safe-stop-no-meaningful-candidates` 时，应停止自动清理并提交决策表。若默认 plan 和 registry plan 都无法覆盖目标缺口，剩余空间通常位于 registry 保留集、k3s runtime、containerd image cache 或 PVC 数据中。继续下降需要显式选择以下更高风险方案之一： | 方案 | 风险 | 需要的前置决策 | |---|---|---| @@ -100,16 +100,16 @@ G14 当前只有一个本机 k3s cluster；空间归因时不要把 `hwlab-dev` | k3s namespace/PVC | `kubectl get pv,pvc,pod -A -o json` 结合 local-path PV host path | 把可归属数据映射到 namespace/workload | | Registry repo/tag/revision | registry v2 repository/tag link、manifest revision 与 blob manifest 解析 | 判断镜像历史 tag、cache revision 与共享 layer 的贡献 | -当前 G14 高水位的长期基线分布如下，后续诊断出现同类量级时优先按同一顺序处理： +G14 高水位的长期基线分布如下，后续诊断出现同类量级时优先按同一顺序处理。registry retention 后，空间压力可能从 `/var/lib/hwlab/registry` 转移到 k3s containerd snapshot/content、local-path PVC 和 host containerd；这些都属于受保护运行面，不能为了达到百分比目标直接删除目录。 | 类别 | 路径 | 典型量级 | 归因说明 | |---|---|---:|---| -| HWLAB local registry | `/var/lib/hwlab/registry` | 约 60GiB | 最大头；按 repo/tag retention 管理 | -| k3s runtime | `/var/lib/rancher/k3s` | 约 24GiB | embedded containerd、local-path PVC 和 k3s server/db | -| k3s containerd snapshots | `/var/lib/rancher/k3s/agent/containerd/.../snapshots` | 约 14GiB | workload image layer/snapshot cache，共享占用，不能直接按单 pod 删除 | -| k3s containerd blobs | `/var/lib/rancher/k3s/agent/containerd/.../blobs` | 约 6GiB | k3s image content store，共享占用 | -| k3s local-path PVC | `/var/lib/rancher/k3s/storage` | 约 3GiB | 可按 namespace/PVC 归因 | -| host containerd | `/var/lib/containerd` | 约 9GiB | k3s 外的 host/containerd cache，默认不由 remote GC prune | +| HWLAB local registry | `/var/lib/hwlab/registry` | 高水位约 60GiB；retention 后约十几 GiB | 最大头；按 repo/tag/revision retention 管理，不能直接删 blob | +| k3s runtime | `/var/lib/rancher/k3s` | 约 45-55GiB | embedded containerd、local-path PVC 和 k3s server/db | +| k3s containerd snapshots | `/var/lib/rancher/k3s/agent/containerd/.../snapshots` | 约 30-40GiB | workload image layer/snapshot cache，共享占用，不能直接按单 pod 删除 | +| k3s containerd blobs | `/var/lib/rancher/k3s/agent/containerd/.../blobs` | 约 8-12GiB | k3s image content store，共享占用 | +| k3s local-path PVC | `/var/lib/rancher/k3s/storage` | 约 8-10GiB | 可按 namespace/PVC 归因，只能通过运行面 retention 入口清理 | +| host containerd | `/var/lib/containerd` | 约 9-12GiB | k3s 外的 host/containerd cache，默认不由 remote GC prune | | root workspaces/cache | `/root` | 约 2GiB | HWLAB/v0.2 worktree、npm/bun/cache 等 | | logs | `/var/log` | 约 1GiB | journald 约占一半，pod logs 很小 | diff --git a/scripts/src/gc-remote.ts b/scripts/src/gc-remote.ts index 04312b15..62ef70dd 100644 --- a/scripts/src/gc-remote.ts +++ b/scripts/src/gc-remote.ts @@ -20,6 +20,7 @@ interface RemoteGcOptions { registryGcOnly: boolean; registryKeepPerRepo: number; registryMinAgeHours: number; + targetUsePercent?: number; jobId?: string; limit: number; resultLimit: number; @@ -109,6 +110,10 @@ function parseRemoteGcOptions(args: string[]): RemoteGcOptions { } else if (arg === "--registry-min-age-hours") { const value = parseNonNegativeNumber(arg, args[++index]); options.registryMinAgeHours = value; + } else if (arg === "--target-use-percent") { + const value = parseNonNegativeNumber(arg, args[++index]); + if (!Number.isInteger(value) || value < 1 || value > 99) throw new Error("--target-use-percent must be an integer from 1 to 99"); + options.targetUsePercent = value; } else if (arg === "--job-id") { const value = args[++index]; if (!value || !/^[A-Za-z0-9._-]{1,128}$/u.test(value)) throw new Error("--job-id must be a safe job id"); @@ -529,6 +534,16 @@ def fmt_bytes(value): idx += 1 return ("%0.0f %s" if size >= 10 or idx == 0 else "%0.1f %s") % (size, units[idx]) +def disk_use_percent(size_bytes, used_bytes): + try: + size = int(size_bytes or 0) + used = int(used_bytes or 0) + except Exception: + return None + if size <= 0: + return None + return int((max(0, used) * 100 + size - 1) // size) + def parse_journal_usage(text): m = re.search(r"take up\s+([0-9.]+)\s*([KMGT]?)(?:i?B|B)?", text, re.I) if not m: @@ -1485,7 +1500,66 @@ def collect_candidates(observed_at): }) return sorted(candidates, key=lambda item: item.get("estimatedReclaimBytes") or 0, reverse=True) -def summarize(candidates, returned): +def target_assessment(disk, estimated_reclaim): + raw = OPTIONS.get("targetUsePercent") + if raw is None: + return None + if not disk: + return { + "targetUsePercent": raw, + "ok": False, + "state": "unavailable", + "reason": "disk-snapshot-unavailable", + } + try: + target = int(raw) + size = int(disk.get("sizeBytes") or 0) + used = int(disk.get("usedBytes") or 0) + reclaim = max(0, int(estimated_reclaim or 0)) + except Exception: + return { + "targetUsePercent": raw, + "ok": False, + "state": "unavailable", + "reason": "invalid-disk-snapshot", + } + target_used_bytes = (size * target) // 100 + required = max(0, used - target_used_bytes) + projected_used = max(0, used - reclaim) + projected_use_percent = disk_use_percent(size, projected_used) + enough = reclaim >= required + if required == 0: + state = "already-below-target" + elif enough: + state = "candidate-estimate-meets-target" + elif reclaim <= 0: + state = "safe-stop-no-meaningful-candidates" + else: + state = "shortfall" + shortfall = max(0, required - reclaim) + return { + "targetUsePercent": target, + "ok": required == 0 or enough, + "state": state, + "currentUsePercent": disk.get("usePercent"), + "currentUsedBytes": used, + "currentUsed": fmt_bytes(used), + "targetUsedBytes": target_used_bytes, + "targetUsed": fmt_bytes(target_used_bytes), + "requiredReclaimBytes": required, + "requiredReclaim": fmt_bytes(required), + "estimatedReclaimBytes": reclaim, + "estimatedReclaim": fmt_bytes(reclaim), + "shortfallBytes": shortfall, + "shortfall": fmt_bytes(shortfall), + "projectedUsedBytes": projected_used, + "projectedUsed": fmt_bytes(projected_used), + "projectedUsePercent": projected_use_percent, + "safeStop": required > 0 and not enough, + "decision": "stop-and-escalate-retention-or-capacity" if required > 0 and not enough else "target-covered-by-safe-candidates", + } + +def summarize(candidates, returned, disk=None): by_kind = {} total = 0 for item in candidates: @@ -1505,6 +1579,7 @@ def summarize(candidates, returned): "returnedEstimatedReclaimBytes": returned_total, "returnedEstimatedReclaim": fmt_bytes(returned_total), "byKind": by_kind, + "target": target_assessment(disk, total), } def assert_tmp_candidate(path): @@ -1604,6 +1679,7 @@ def returned_results(results): return (failed + started + succeeded)[:int(OPTIONS.get("resultLimit") or 50)] def plan_payload(observed_at, preflight, protected, candidates, visible): + disk = df_snapshot() return { "ok": True, "action": "gc remote plan", @@ -1612,9 +1688,9 @@ def plan_payload(observed_at, preflight, protected, candidates, visible): "mutation": False, "observedAt": observed_at, "options": OPTIONS, - "diskBefore": df_snapshot(), + "diskBefore": disk, "clusterPreflight": preflight, - "summary": summarize(candidates, visible), + "summary": summarize(candidates, visible, disk), "candidates": visible, "protected": protected, "policy": { @@ -1636,6 +1712,7 @@ def plan_payload(observed_at, preflight, protected, candidates, visible): "HWLAB DEV runtime and local-path PVC data are protected and require HWLAB-specific retention commands.", "Core dump cleanup only removes untracked /root/unidesk/core. regular files with no active fuser reference.", "HWLAB registry retention is opt-in: it keeps workload tag/digest refs, all tags newer than the retention age and the newest N tags per repo before official registry garbage-collect.", + "When summary.target.safeStop is true, do not broaden deletion scope; choose registry retention, k3s/containerd image cache maintenance, PVC/runtime retention or capacity expansion explicitly.", ], }, } @@ -1684,6 +1761,20 @@ def main(): disk_after = df_snapshot() failed = [item for item in results if item.get("status") == "failed"] returned = returned_results(results) + run_summary = summarize(visible, returned, disk_before) + run_summary.update({ + "plannedCandidateCount": len(visible), + "attemptedCount": len(results), + "startedCount": len([item for item in results if item.get("status") == "started"]), + "succeededCount": len([item for item in results if item.get("status") == "succeeded"]), + "failedCount": len(failed), + "actualDiskReclaimBytes": (disk_after["availableBytes"] - disk_before["availableBytes"]) if disk_before and disk_after else None, + "actualDiskReclaim": fmt_bytes(disk_after["availableBytes"] - disk_before["availableBytes"]) if disk_before and disk_after else None, + "targetAfter": target_assessment(disk_after, 0), + "resultCount": len(results), + "returnedResultCount": len(returned), + "omittedResultCount": max(0, len(results) - len(returned)), + }) payload = { "ok": len(failed) == 0, "action": "gc remote run", @@ -1696,18 +1787,7 @@ def main(): "diskAfter": disk_after, "clusterPreflight": preflight, "clusterAfter": cluster_preflight(), - "summary": { - "plannedCandidateCount": len(visible), - "attemptedCount": len(results), - "startedCount": len([item for item in results if item.get("status") == "started"]), - "succeededCount": len([item for item in results if item.get("status") == "succeeded"]), - "failedCount": len(failed), - "estimatedReclaimBytes": sum(int(item.get("estimatedReclaimBytes") or 0) for item in visible), - "actualDiskReclaimBytes": (disk_after["availableBytes"] - disk_before["availableBytes"]) if disk_before and disk_after else None, - "resultCount": len(results), - "returnedResultCount": len(returned), - "omittedResultCount": max(0, len(results) - len(returned)), - }, + "summary": run_summary, "results": returned, "protected": protected, } diff --git a/scripts/src/help.ts b/scripts/src/help.ts index d108ecf3..0cf8bf26 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -280,6 +280,7 @@ function gcHelp(): unknown { "bun scripts/cli.ts gc policy plan", "bun scripts/cli.ts gc policy install", "bun scripts/cli.ts gc remote G14 plan", + "bun scripts/cli.ts gc remote G14 plan --target-use-percent 50 --include-hwlab-registry", "bun scripts/cli.ts gc remote G14 run --confirm", "bun scripts/cli.ts gc remote G14 status --job-id ", "bun scripts/cli.ts gc plan --full", @@ -306,6 +307,7 @@ function gcHelp(): unknown { "--registry-gc-only": "remote G14 only: run official registry garbage-collect without deleting additional tags; intended for interrupted registry retention recovery", "--registry-keep-per-repo N": "remote registry only: keep at least N newest tags per service repo; default 20, minimum 1", "--registry-min-age-hours N": "remote registry only: keep all tags newer than N hours; default 48, minimum 0", + "--target-use-percent N": "remote only: evaluate whether planned candidates can reduce root filesystem use to N%; reports required reclaim, projected use, shortfall and safe-stop decision", "--job-id ID": "remote status only: inspect a long-running remote gc job", "--limit N": "number of candidates returned and executed by run when --full is not set; default 50", "--result-limit N": "number of per-candidate run results returned when --full is not set; default 50",