From 3b9c8d8ddb7988e35045c6610f21b737e04e7c72 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 29 May 2026 01:43:14 +0000 Subject: [PATCH] Add remote gc for provider hosts --- AGENTS.md | 2 +- docs/reference/cli.md | 2 +- scripts/src/gc-remote.ts | 1474 ++++++++++++++++++++++++++++++++++++++ scripts/src/gc.ts | 7 +- scripts/src/help.ts | 17 +- scripts/src/ssh.ts | 10 + 6 files changed, 1506 insertions(+), 6 deletions(-) create mode 100644 scripts/src/gc-remote.ts diff --git a/AGENTS.md b/AGENTS.md index 4947736c..1774739b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -146,7 +146,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - `bun scripts/cli.ts server swap status|ensure [--path /swapfile] [--size 2GiB] [--dry-run]`：以 JSON 查看或幂等创建主 server swapfile，`ensure` 输出 before/after、动作、持久化状态和 degraded/failed 详情，规则见 `docs/reference/deployment.md`。 - `bun scripts/cli.ts server logs [--tail-bytes N]`：分页返回文件日志与 Docker 日志尾部并带截断元数据，日志规则见 `docs/reference/observability.md`。 - `bun scripts/cli.ts server cleanup plan [--min-age-hours N] [--limit N]`：只读/干跑生成主 server Docker 镜像清理计划，默认只列出至少 24 小时前创建的非保护镜像，输出 active/protected images、stale candidates、预计释放空间、风险等级和必须人工确认的 `docker image rm` 命令；禁止默认删除、禁止 prune、禁止触碰 database volume、registry storage 或 Baidu Netdisk 状态。 -- `bun scripts/cli.ts gc plan|run|db-trace|policy`：主 server 磁盘高水位一次性缓解和低风险防膨胀入口，覆盖日志、journald、Docker BuildKit cache、allowlisted `/tmp` 诊断目录、显式 trace 遥测留存和 systemd 定时策略；`gc run` 和 `gc db-trace run` 需要显式确认，规则见 `docs/reference/cli.md`。 +- `bun scripts/cli.ts gc plan|run|db-trace|policy|remote`：主 server 或受控 provider 磁盘高水位一次性缓解和低风险防膨胀入口，覆盖日志、journald、Docker BuildKit cache、allowlisted `/tmp` 诊断目录、受限 core dump、显式 trace 遥测留存和 systemd 定时策略；`gc run`、`gc remote ... run` 和 `gc db-trace run` 需要显式确认，规则见 `docs/reference/cli.md`。 - `bun scripts/cli.ts server rebuild `：以 build-first、Compose lock、no-deps force-recreate 和 post-up validation 的异步 job 重建主 server Compose 内单个服务；对 database、File Browser、Code Queue 执行面、k3sctl-adapter 或未知对象返回结构化 `unsupported-server-rebuild`，规则见 `docs/reference/deployment.md` 与 `docs/reference/cicd-standardization.md`。 - `bun scripts/cli.ts provider attach [--master-server URL] [--up] [--force]` / `bun scripts/cli.ts provider triage [--observed-error text] [--observed-scope scope] [--microservice id ...] [--full|--raw]`：前者在新增计算节点上生成两项配置的 provider-gateway 挂载包；后者是只读多信号健康裁决入口，默认低噪声输出 `decision`、`healthyScopes`、`failedScopes`、`retryable` 和异常信号摘要，用来把单路径 `provider is not online`、SSH 超时、registry 失败或 proxy 失败归类为 `retryable-transient`、`service-degraded` 或 `global-offline`，完整 evidence 需显式 `--full|--raw`，规则见 `docs/reference/provider-gateway.md` 和 `docs/reference/code-queue-supervision.md`。 - `bun scripts/cli.ts ssh [operation args...]` / `tran [operation args...]`：通过 provider-gateway 的 Host SSH / WSL SSH 维护桥进入 provider、host workspace、Windows cmd route、k3s 控制面或 pod workspace，并提供带 SHA-256 校验的 `upload`/`download` 文件传输；主 server 人工/Codex 分布式操作必须用本机 `tran` wrapper，CLI 参考和可移植脚本可保留完整命令，细则见 `docs/reference/cli.md`、`docs/reference/windows-passthrough.md` 和 `docs/reference/provider-gateway.md`。 diff --git a/docs/reference/cli.md b/docs/reference/cli.md index f75538b2..13841a60 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -25,7 +25,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 DEV/PROD 滚动、P - `server swap status|ensure [--path /swapfile] [--size 2GiB] [--dry-run]` 是主 server swap 管理入口。`status` 仅读 `/proc/meminfo`、`/proc/swaps` 和 `/etc/fstab` 并返回 JSON；`ensure` 在已有任何 active swap 时只报告 no-op，在无 active swap 时创建固定 swapfile、`chmod 600`、`mkswap`、`swapon` 并尽量写入 `/etc/fstab`。输出必须包含 `before`、`after`、total memory、active swap、持久化状态、关键动作和错误详情；若 swap 已启用但 fstab 写入失败，状态为 `degraded`，调用者需按返回的 detail 修复持久化。 - `server logs` 返回 `logs/` 文件日志和 Docker 容器日志的尾部，默认限制输出大小，避免日志爆炸。实现必须只读取文件末尾字节，不得为了 tail 先把巨大日志完整读入 CLI 内存。 - `server cleanup plan [--min-age-hours N] [--limit N]` 只生成主 server Docker 镜像清理 dry-run 计划，不执行删除；默认 `--min-age-hours 24`，避免把刚发布或刚验证的镜像列为 stale。输出必须包含 `dryRun=true`、`mutation=false`、`policy.deletionExecuted=false`、active containers/images、受保护镜像、candidate stale images、估算释放空间、风险等级、`commandsToReview` 和人工审批清单。计划必须保守白名单：保留 running containers 使用的 image ID，保留 stopped containers 引用的 image ID 直到人工先复核容器，保留 `deploy.json`/`CI.json` 当前 commit-pinned artifact、Compose stable image、上游 digest pin 和 provider-gateway runner image；`protectedStorage` 必须显式列出 PostgreSQL named volume、Baidu Netdisk `.state`、D601 registry storage 和 Docker volumes/host data policy。该入口禁止生成或执行 `docker system prune`、`docker image prune`、`docker builder prune`、`docker volume rm`、`docker compose down -v`、数据库清理或 host data `rm` 命令；未来若增加真实删除，必须另设显式审批参数并先复核 dry-run 输出。 -- `gc plan|run --confirm` 是主 server 磁盘高水位的一次性短期缓解入口，覆盖 UniDesk 文件日志、Docker `json-file` 容器日志、systemd journal、Docker BuildKit cache 和 allowlisted `/tmp` 诊断目录。`gc plan` 默认只读并返回候选、风险、估算释放空间、受保护对象和数据库只读摘要；候选列表默认按释放空间排序限制为 50 条，`--limit N` 调整页大小，`--full|--raw` 才展开全部候选。输出中的 `estimatedReclaim` 表示全量候选估算，`returnedEstimatedReclaim` 表示当前输出页估算；`gc run` 默认只返回前 50 条结果，`--result-limit N` 调整结果页大小，避免 GC 自身输出膨胀。`gc run` 必须显式 `--confirm`，只执行当前输出候选中的短期清理动作，因此默认不会执行被分页隐藏的候选。`gc` 不删除 Docker image/container/volume，不触碰 PostgreSQL PGDATA，不删除 Baidu Netdisk staging/backups 或 D601 registry storage；默认路径中数据库只做诊断摘要。`gc policy plan|install` 渲染或安装低风险长期策略：journald 512MiB 上限和每日 `unidesk-gc.timer`，该 timer 只运行文件日志与 allowlisted `/tmp` 低风险 GC，不主动 vacuum journal，不触碰数据库、Docker image/volume 或 Baidu staging，输出写入 `.state/gc/`。`gc db-trace plan|run --confirm --before-date YYYY-MM-DD --vacuum-full` 是显式数据库 trace 遥测留存入口，只删除 `oa_events` 中默认 trace 高频事件类型在指定日期前的数据，并在 `--vacuum-full` 下重写 `public.oa_events` 让 `df` 真正回收磁盘；执行前必须确认近期 PostgreSQL basebackup，且应视为维护窗口操作。常用参数包括 `--logs-keep-days N`、`--file-log-max-bytes SIZE`、`--file-log-tail-bytes SIZE`、`--docker-log-max-bytes SIZE`、`--journal-target-size SIZE`、`--build-cache-until 24h`、显式清空全部 BuildKit cache 的 `--build-cache-all`、`--tmp-min-age-hours N` 和显式 `--include-browser-cache`。 +- `gc plan|run --confirm` 是主 server 磁盘高水位的一次性短期缓解入口，覆盖 UniDesk 文件日志、Docker `json-file` 容器日志、systemd journal、Docker BuildKit cache 和 allowlisted `/tmp` 诊断目录。`gc plan` 默认只读并返回候选、风险、估算释放空间、受保护对象和数据库只读摘要；候选列表默认按释放空间排序限制为 50 条，`--limit N` 调整页大小，`--full|--raw` 才展开全部候选。输出中的 `estimatedReclaim` 表示全量候选估算，`returnedEstimatedReclaim` 表示当前输出页估算；`gc run` 默认只返回前 50 条结果，`--result-limit N` 调整结果页大小，避免 GC 自身输出膨胀。`gc run` 必须显式 `--confirm`，只执行当前输出候选中的短期清理动作，因此默认不会执行被分页隐藏的候选。`gc` 不删除 Docker image/container/volume，不触碰 PostgreSQL PGDATA，不删除 Baidu Netdisk staging/backups 或 D601 registry storage；默认路径中数据库只做诊断摘要。`gc remote plan|run --confirm|status --job-id ` 通过 UniDesk SSH 透传在 provider host 上执行同类低风险 GC；G14 路径会先确认原生 k3s node 身份，并显式保护 `/var/lib/rancher/k3s`、`/var/lib/rancher/k3s/storage`、`/var/lib/kubelet`、`/var/lib/containerd`、HWLAB 固定 workspaces、Kubernetes workload/Secret/PVC/PV/Argo/Tekton 对象以及 Docker image/container/volume，默认只允许 journal vacuum、Docker json log truncate、Docker BuildKit cache prune、apt archive clean、allowlisted `/tmp` 临时目录删除，以及受限 core dump 清理；core dump 仅匹配 `/root/unidesk/core.` 这类普通文件，并在执行前重新校验路径 allowlist、Git 未跟踪和无 `fuser` 活跃引用。HWLAB registry 清理必须显式加 `--include-hwlab-registry` 才会进入，策略不是只留 latest：默认保留当前 workload 引用的 tag/digest、cache/latest、基础镜像 tag、48 小时内所有 tag，以及每个业务 repo 至少 20 个最新 tag；执行时会以远端异步 job 暂停 G14/v0.2 branch poller CronJob、等待 hwlab-ci PipelineRun/TaskRun/Job 空闲、通过 registry API 删除不再被保留 tag 引用的 manifest，再缩容 registry pod 后运行官方 `registry garbage-collect`，最后恢复 registry 和 CronJob；中断恢复可显式使用 `--registry-gc-only` 生成只跑官方 GC 的收尾候选，该候选不删除任何 tag。长任务状态通过 `gc remote status --job-id ` 查询。`gc policy plan|install` 渲染或安装低风险长期策略：journald 512MiB 上限和每日 `unidesk-gc.timer`，该 timer 只运行文件日志与 allowlisted `/tmp` 低风险 GC，不主动 vacuum journal，不触碰数据库、Docker image/volume 或 Baidu staging，输出写入 `.state/gc/`。`gc db-trace plan|run --confirm --before-date YYYY-MM-DD --vacuum-full` 是显式数据库 trace 遥测留存入口，只删除 `oa_events` 中默认 trace 高频事件类型在指定日期前的数据，并在 `--vacuum-full` 下重写 `public.oa_events` 让 `df` 真正回收磁盘；执行前必须确认近期 PostgreSQL basebackup，且应视为维护窗口操作。常用参数包括 `--logs-keep-days N`、`--file-log-max-bytes SIZE`、`--file-log-tail-bytes SIZE`、`--docker-log-max-bytes SIZE`、`--journal-target-size SIZE`、`--build-cache-until 24h`、显式清空全部 BuildKit cache 的 `--build-cache-all`、`--tmp-min-age-hours N`、显式 `--include-browser-cache`、远端 registry 的 `--registry-keep-per-repo N`、`--registry-min-age-hours N`、`--registry-gc-only` 与 `--job-id ID`。 - `server rebuild ` 创建异步 job，先构建目标服务镜像，随后在 `.state/locks/server-compose.lock` 串行保护下用 `--no-deps --force-recreate` 替换目标 service 并等待容器 `healthy/running`；该命令用于替代手工删除容器的兜底流程，其中 `dev-frontend-proxy` 只更新主 server dev 入口薄代理，`todo-note`、`code-queue-mgr`、`project-manager`、`baidu-netdisk` 和 `oa-event-flow` 只重建主 server 承载的对应后端，不会重建或删除 database 命名卷。D601 Code Queue 执行面不由 `server rebuild` 管理，Rust backend-core 迭代不得用 `server rebuild backend-core` 在 master server 编译，规则见 `docs/reference/dev-environment.md`。 - `provider attach [--master-server URL] [--up] [--force]` 在新计算节点生成两项配置的 provider-gateway 挂载包：`.state/provider-.env` 默认只包含 `UNIDESK_MASTER_SERVER` 与 `PROVIDER_ID`，`provider-.yml` 固定 Docker socket、`pid: "host"`、`restart: always`、只读 `/workspace` 和 SSH 维护私钥挂载；`--up` 会立即执行生成的 `docker compose up -d --build`。`provider triage [--observed-error text] [--observed-scope scope] [--microservice id ...] [--full|--raw]` 是只读多信号健康裁决入口，会把单路径 `provider is not online`、SSH 超时、registry 失败和 service proxy 失败归类成 `runner-local-observation-gap`、`service-degraded`、`provider-degraded` 或 `global-blocker`。默认输出只返回裁决、scope、失败/降级/未知信号和有界 evidence 摘要，完整 evidence 必须显式加 `--full` 或 `--raw`；推荐交叉验证命令仍包含 `debug health`、`debug dispatch host.ssh --wait-ms 15000`、`ssh argv true`、`artifact-registry health --provider-id `、`microservice health k3sctl-adapter`、`microservice health code-queue` 和 `codex tasks --view supervisor --limit 20`。 - `ssh [operation args...]` / `tran [operation args...]` 通过 backend-core 内网 WebSocket broker 和 provider-gateway 的 Host SSH / WSL SSH 维护桥连接目标节点；`route` 基础形态是 provider id，例如 `D601` 或 `G14`，也可以扩展为纯定位路径 `provider:plane[:namespace:resource[:container]]`，例如 `D601:win`、`D601:win/c/test`、`G14:k3s`、`D601:k3s` 或 `G14:k3s::`。WSL provider 的 Windows cmd 入口固定写 `tran D601:win cmd `，需要 Windows cwd 时用 `tran D601:win/c/test cmd cd`，由 CLI 自动设置 `chcp 65001`、`PYTHONUTF8=1` 和 `PYTHONIOENCODING=utf-8`；命名只允许 `win`，不得使用 `win32`。非交互远端命令优先使用 `ssh argv ...`；需要 shell 脚本、管道、变量或循环时优先使用 quoted heredoc 单步传输，例如 `tran G14 script <<'SCRIPT'`、`tran G14:k3s script <<'SCRIPT'` 或 `tran G14:k3s:: script <<'SCRIPT'`，把脚本走 stdin。`script -- '<单个字符串>'` 是无需 stdin 的远端 shell one-liner，例如 `tran G14:/root/hwlab script -- 'cd /root/hwlab && git status --short --branch'`；`script -- <多个 argv>` 才是 direct argv，适合 `tran D601:/path script -- sed -n '1,20p' file` 这类带短横线的单进程命令。顶层 remote option parser 必须保留命令已经开始后的 `--`，不得把它吞成全局选项结束符。需要远端改文本文件时默认优先使用 ` apply-patch < patch.diff`；需要可靠传输非文本或整文件时使用 ` upload ` 和 ` download `，CLI 会按字节数与 SHA-256 自动校验并在 provider-gateway stdin/argv 限制下切换客户端分块策略；需要旧 helper 时显式使用 `:k3s:: apply-patch-v1` 或 ` apply-patch-v1`。ssh-like 命令遇到 timeout/kex/255 类失败时，CLI 会在 stderr 追加一行 `UNIDESK_SSH_HINT` JSON，提示 stdin script/argv 重试和 provider triage 交叉验证。 diff --git a/scripts/src/gc-remote.ts b/scripts/src/gc-remote.ts new file mode 100644 index 00000000..f289ad71 --- /dev/null +++ b/scripts/src/gc-remote.ts @@ -0,0 +1,1474 @@ +import { Buffer } from "node:buffer"; + +import { type UniDeskConfig } from "./config"; +import { runSshCommandCapture } from "./ssh"; + +interface RemoteGcOptions { + confirm: boolean; + journal: boolean; + journalTargetBytes: number; + dockerLogs: boolean; + dockerLogMaxBytes: number; + buildCache: boolean; + buildCacheUntil: string; + tmp: boolean; + tmpMinAgeHours: number; + aptCache: boolean; + coreDumps: boolean; + coreDumpMinAgeHours: number; + hwlabRegistry: boolean; + registryGcOnly: boolean; + registryKeepPerRepo: number; + registryMinAgeHours: number; + jobId?: string; + limit: number; + resultLimit: number; + full: boolean; +} + +const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = { + confirm: false, + journal: true, + journalTargetBytes: 512 * 1024 * 1024, + dockerLogs: true, + dockerLogMaxBytes: 50 * 1024 * 1024, + buildCache: true, + buildCacheUntil: "24h", + tmp: true, + tmpMinAgeHours: 24, + aptCache: true, + coreDumps: true, + coreDumpMinAgeHours: 1, + hwlabRegistry: false, + registryGcOnly: false, + registryKeepPerRepo: 20, + registryMinAgeHours: 48, + limit: 50, + resultLimit: 50, + full: false, +}; + +export async function runRemoteGcCommand(config: UniDeskConfig, providerId: string | undefined, action: string | undefined, args: string[]): Promise { + if (providerId === undefined || providerId.length === 0) { + return { + ok: false, + error: "gc-remote-provider-required", + usage: "bun scripts/cli.ts gc remote plan|run|status [--confirm]", + }; + } + const subaction = action ?? "plan"; + const options = parseRemoteGcOptions(args); + if (subaction === "plan" || subaction === "dry-run") return await runRemoteGc(config, providerId, "plan", options); + if (subaction === "status") return await runRemoteGc(config, providerId, "status", options); + if (subaction === "run") { + if (!options.confirm) { + return { + ok: false, + error: "gc-remote-run-requires-confirm", + dryRun: true, + mutation: false, + requiredFlag: "--confirm", + planCommand: `bun scripts/cli.ts gc remote ${providerId} plan`, + runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm`, + }; + } + return await runRemoteGc(config, providerId, "run", options); + } + return { + ok: false, + error: "unsupported-gc-remote-action", + action: subaction, + supportedActions: ["plan", "run", "status"], + }; +} + +function parseRemoteGcOptions(args: string[]): RemoteGcOptions { + const options = { ...DEFAULT_REMOTE_OPTIONS }; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] ?? ""; + if (arg === "--confirm") { + options.confirm = true; + } else if (arg === "--dry-run") { + // accepted for plan compatibility + } else if (arg === "--journal-target-size") { + options.journalTargetBytes = parseSizeOption(arg, args[++index]); + } else if (arg === "--docker-log-max-bytes") { + options.dockerLogMaxBytes = parseSizeOption(arg, args[++index]); + } else if (arg === "--build-cache-until") { + const value = args[++index]; + if (!value || !/^\d+(s|m|h|d)$/u.test(value)) throw new Error(`${arg} must look like 24h, 7d, 30m or 60s`); + options.buildCacheUntil = value; + } else if (arg === "--tmp-min-age-hours") { + options.tmpMinAgeHours = parseNonNegativeNumber(arg, args[++index]); + } else if (arg === "--core-dump-min-age-hours") { + options.coreDumpMinAgeHours = parseNonNegativeNumber(arg, args[++index]); + } else if (arg === "--registry-keep-per-repo") { + const value = parseNonNegativeNumber(arg, args[++index]); + if (!Number.isInteger(value) || value < 10) throw new Error("--registry-keep-per-repo must be an integer >= 10"); + options.registryKeepPerRepo = Math.min(value, 50); + } else if (arg === "--registry-min-age-hours") { + const value = parseNonNegativeNumber(arg, args[++index]); + if (value < 24) throw new Error("--registry-min-age-hours must be >= 24"); + options.registryMinAgeHours = value; + } else if (arg === "--job-id") { + const value = args[++index]; + if (!value || !/^[A-Za-z0-9._-]{1,128}$/u.test(value)) throw new Error("--job-id must be a safe job id"); + options.jobId = value; + } else if (arg === "--limit") { + const value = parseNonNegativeNumber(arg, args[++index]); + if (!Number.isInteger(value) || value <= 0) throw new Error("--limit must be a positive integer"); + options.limit = Math.min(value, 5000); + } else if (arg === "--result-limit") { + const value = parseNonNegativeNumber(arg, args[++index]); + if (!Number.isInteger(value) || value <= 0) throw new Error("--result-limit must be a positive integer"); + options.resultLimit = Math.min(value, 5000); + } else if (arg === "--no-journal") { + options.journal = false; + } else if (arg === "--no-docker-logs") { + options.dockerLogs = false; + } else if (arg === "--no-build-cache") { + options.buildCache = false; + } else if (arg === "--no-tmp") { + options.tmp = false; + } else if (arg === "--no-apt-cache") { + options.aptCache = false; + } else if (arg === "--no-core-dumps") { + options.coreDumps = false; + } else if (arg === "--include-hwlab-registry") { + options.hwlabRegistry = true; + } else if (arg === "--registry-gc-only") { + options.hwlabRegistry = true; + options.registryGcOnly = true; + } else if (arg === "--full" || arg === "--raw") { + options.full = true; + } else { + throw new Error(`unknown gc remote option: ${arg}`); + } + } + return options; +} + +function parseNonNegativeNumber(name: string, raw: string | undefined): number { + const value = Number(raw); + if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a non-negative number`); + return value; +} + +function parseSizeOption(name: string, raw: string | undefined): number { + const value = parseSize(raw ?? ""); + if (value === null || value <= 0) throw new Error(`${name} must be a positive size such as 512M, 1GiB or 50000000`); + return value; +} + +function parseSize(raw: string): number | null { + const match = raw.trim().match(/^(\d+(?:\.\d+)?)\s*(b|k|kb|kib|m|mb|mib|g|gb|gib)?$/iu); + if (!match) return null; + const value = Number(match[1]); + const unit = (match[2] ?? "b").toLowerCase(); + const multiplier = + unit === "g" || unit === "gb" || unit === "gib" ? 1024 ** 3 + : unit === "m" || unit === "mb" || unit === "mib" ? 1024 ** 2 + : unit === "k" || unit === "kb" || unit === "kib" ? 1024 + : 1; + const bytes = Math.floor(value * multiplier); + return Number.isFinite(bytes) ? bytes : null; +} + +async function runRemoteGc(config: UniDeskConfig, providerId: string, action: "plan" | "run" | "status", options: RemoteGcOptions): Promise { + const scriptConfig = Buffer.from(JSON.stringify({ providerId, action, options }), "utf8").toString("base64"); + const result = await runSshCommandCapture(config, providerId, ["py"], remoteGcPython(scriptConfig)); + if (result.exitCode !== 0) { + return { + ok: false, + error: "gc-remote-command-failed", + providerId, + action: `gc remote ${action}`, + exitCode: result.exitCode, + stdoutTail: result.stdout.slice(-4000), + stderrTail: result.stderr.slice(-4000), + }; + } + try { + return JSON.parse(result.stdout) as unknown; + } catch (error) { + return { + ok: false, + error: "gc-remote-invalid-json", + providerId, + action: `gc remote ${action}`, + message: error instanceof Error ? error.message : String(error), + stdoutTail: result.stdout.slice(-4000), + stderrTail: result.stderr.slice(-4000), + }; + } +} + +function remoteGcPython(configBase64: string): string { + return String.raw` +import base64 +import json +import os +import re +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +CONFIG = json.loads(base64.b64decode("${configBase64}").decode("utf-8")) +PROVIDER_ID = str(CONFIG.get("providerId") or "") +ACTION = str(CONFIG.get("action") or "plan") +OPTIONS = CONFIG.get("options") or {} + +TMP_PREFIX_ALLOWLIST = [ + "hwlab-agent-", + "hwlab-cd-", + "hwlab-cli-cicd-", + "hwlab-codeagent-trace", + "hwlab-desired-state-", + "hwlab-g14-", + "hwlab-main-", + "hwlab-merge-", + "hwlab-pr", + "hwlab-refresh-", + "hwlab-remote-", + "hwlab-ts-check", + "hwlab-bun-runtime-check-", + "hwlab-v02-", + "playwright-artifacts-", + "playwright_chromiumdev_profile-", + "unidesk-apply-patch-v2-perf-", + "unidesk-clean-", + "unidesk-code-queue", + "unidesk-hwlab-cd-", + "unidesk-pr", + "unidesk-tran-runner", + "bunx-", + "codex-app-schema", + "codex-app-ts", + "marked-", + "node-compile-cache", +] + +TMP_EXACT_PROTECT = set([ + "/tmp/codex-apply-patch", + "/tmp/codex-ipc", + "/tmp/tmux-0", + "/tmp/snap-private-tmp", +]) + +CORE_DUMP_DIR_ALLOWLIST = set([ + "/root/unidesk", +]) + +REGISTRY_REPOSITORY_ROOT = "/var/lib/hwlab/registry/docker/registry/v2/repositories" +REGISTRY_ROOT = "/var/lib/hwlab/registry" +REGISTRY_PROTECTED_TAGS = set([ + "latest", + "16-alpine", + "20-bookworm-slim", + "node22-alpine-v1", + "node22-alpine-bun-v1", + "sidecar", + "1b99888d3dae", +]) + +EXPECTED_G14_NODE = "ubuntu-rog-zephyrus-g14-ga401iv-ga401iv" +REMOTE_GC_JOB_DIR = "/tmp/unidesk-gc-remote/jobs" + +def now_iso(): + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + +def command(cmd, timeout=10): + try: + p = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout) + return {"exitCode": p.returncode, "stdout": p.stdout, "stderr": p.stderr, "timedOut": False} + except subprocess.TimeoutExpired as exc: + return { + "exitCode": None, + "stdout": exc.stdout or "", + "stderr": exc.stderr or ("timed out after %ss" % timeout), + "timedOut": True, + } + except Exception as exc: + return {"exitCode": None, "stdout": "", "stderr": str(exc), "timedOut": False} + +def bounded(result): + return { + "exitCode": result.get("exitCode"), + "timedOut": bool(result.get("timedOut")), + "stdoutTail": str(result.get("stdout") or "")[-2000:], + "stderrTail": str(result.get("stderr") or "")[-2000:], + } + +def job_id_or_none(): + raw = str(OPTIONS.get("jobId") or "") + if raw and re.match(r"^[A-Za-z0-9._-]{1,128}$", raw): + return raw + return None + +def job_paths(job_id): + os.makedirs(REMOTE_GC_JOB_DIR, exist_ok=True) + return { + "state": os.path.join(REMOTE_GC_JOB_DIR, "%s.json" % job_id), + "log": os.path.join(REMOTE_GC_JOB_DIR, "%s.log" % job_id), + } + +def write_json_atomic(path, payload): + tmp = "%s.tmp.%s" % (path, os.getpid()) + with open(tmp, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False, indent=2) + handle.write("\n") + os.replace(tmp, path) + +def read_file_tail(path, limit=12000): + try: + size = os.path.getsize(path) + with open(path, "rb") as handle: + if size > limit: + handle.seek(size - limit) + data = handle.read() + return data.decode("utf-8", errors="replace") + except OSError: + return "" + +def remote_gc_job_status(): + job_id = job_id_or_none() + if not job_id: + return { + "ok": False, + "error": "gc-remote-status-requires-job-id", + "requiredFlag": "--job-id", + } + paths = job_paths(job_id) + if not os.path.isfile(paths["state"]): + return { + "ok": False, + "error": "gc-remote-job-not-found", + "jobId": job_id, + "statePath": paths["state"], + "logTail": read_file_tail(paths["log"]), + } + try: + with open(paths["state"], "r", encoding="utf-8") as handle: + payload = json.load(handle) + except Exception as exc: + return { + "ok": False, + "error": "gc-remote-job-state-invalid", + "jobId": job_id, + "message": str(exc), + "statePath": paths["state"], + "logTail": read_file_tail(paths["log"]), + } + payload["logTail"] = read_file_tail(paths["log"]) + return payload + +def path_size(path): + try: + if os.path.islink(path) or os.path.isfile(path): + return os.lstat(path).st_size + if not os.path.isdir(path): + return 0 + total = 0 + for root, dirs, files in os.walk(path): + for name in files: + child = os.path.join(root, name) + try: + total += os.lstat(child).st_size + except OSError: + pass + for name in dirs: + child = os.path.join(root, name) + try: + if os.path.islink(child): + total += os.lstat(child).st_size + except OSError: + pass + return total + except OSError: + return 0 + +def du_size(path, timeout=20): + if not os.path.exists(path): + return None + result = command(["du", "-sxB1", path], timeout) + if result["exitCode"] != 0: + return path_size(path) + text = result["stdout"].strip() + if not text: + return 0 + try: + return int(text.split()[0]) + except Exception: + return path_size(path) + +def allocated_file_size(path): + try: + stat = os.stat(path) + blocks = getattr(stat, "st_blocks", 0) + if blocks: + return int(blocks) * 512 + return int(stat.st_size) + except OSError: + return 0 + +def df_snapshot(): + result = command(["df", "-B1", "-P", "/"], 5) + if result["exitCode"] != 0: + return None + lines = result["stdout"].strip().splitlines() + if len(lines) < 2: + return None + parts = lines[1].split() + if len(parts) < 6: + return None + return { + "filesystem": parts[0], + "sizeBytes": int(parts[1]), + "usedBytes": int(parts[2]), + "availableBytes": int(parts[3]), + "usePercent": int(parts[4].replace("%", "")), + "mount": parts[5], + } + +def fmt_bytes(value): + units = ["B", "KiB", "MiB", "GiB", "TiB"] + size = float(max(0, int(value or 0))) + idx = 0 + while size >= 1024 and idx < len(units) - 1: + size /= 1024.0 + idx += 1 + return ("%0.0f %s" if size >= 10 or idx == 0 else "%0.1f %s") % (size, units[idx]) + +def parse_journal_usage(text): + m = re.search(r"take up\s+([0-9.]+)\s*([KMGT]?)(?:i?B|B)?", text, re.I) + if not m: + return None + mult = {"": 1, "K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}.get(m.group(2).upper(), 1) + return int(float(m.group(1)) * mult) + +def parse_docker_human_size(raw): + raw = str(raw).split("(")[0].strip() + m = re.match(r"^([0-9.]+)\s*([KMGT]?B)$", raw, re.I) + if not m: + return None + mult = {"B": 1, "KB": 1000, "MB": 1000**2, "GB": 1000**3, "TB": 1000**4}.get(m.group(2).upper(), 1) + return int(float(m.group(1)) * mult) + +def parse_docker_build_cache(text): + for line in text.splitlines(): + if not line.startswith("Build Cache"): + continue + match = re.match(r"^Build Cache\s+\S+\s+\S+\s+(\S+)\s+(\S+)", line.strip()) + if not match: + continue + size = parse_docker_human_size(match.group(1)) + reclaim = parse_docker_human_size(match.group(2)) + if size is None or reclaim is None: + return None + return {"sizeBytes": size, "reclaimableBytes": reclaim} + return None + +def docker_containers(): + ps = command(["docker", "ps", "-qa", "--no-trunc"], 5) + if ps["exitCode"] != 0 or not ps["stdout"].strip(): + return [] + ids = ps["stdout"].split() + inspect = command(["docker", "inspect"] + ids, 10) + if inspect["exitCode"] != 0 or not inspect["stdout"].strip(): + return [] + try: + data = json.loads(inspect["stdout"]) + except Exception: + return [] + rows = [] + for item in data: + cfg = item.get("Config") or {} + rows.append({ + "id": str(item.get("Id") or ""), + "name": str(item.get("Name") or "").lstrip("/"), + "image": str(cfg.get("Image") or item.get("Image") or ""), + "logPath": str(item.get("LogPath") or ""), + }) + return [row for row in rows if row["id"]] + +def cluster_preflight(): + node_cmd = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{\"\\n\"}{end}' 2>/dev/null"], 10) + pods_cmd = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pods -n hwlab-dev --no-headers 2>/dev/null | wc -l"], 10) + nodes = [line.strip() for line in node_cmd["stdout"].splitlines() if line.strip()] + expected = EXPECTED_G14_NODE if PROVIDER_ID.upper() == "G14" else None + ok = True + reason = "ok" + if expected is not None and expected not in nodes: + ok = False + reason = "expected-g14-node-missing" + return { + "ok": ok, + "reason": reason, + "providerId": PROVIDER_ID, + "hostname": command(["hostname"], 5)["stdout"].strip(), + "expectedNode": expected, + "nodes": nodes, + "nodeCommand": bounded(node_cmd), + "hwlabDevPodCount": int(pods_cmd["stdout"].strip() or "0") if pods_cmd["exitCode"] == 0 else None, + "hwlabDevPodCommand": bounded(pods_cmd), + } + +def active_hwlab_ci_writes(): + result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pipelinerun,taskrun -n hwlab-ci --no-headers 2>/dev/null | awk '$2 != \"True\" && $2 != \"False\" {print}' | head -40"], 15) + lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()] + return {"ok": result["exitCode"] == 0, "activeCount": len(lines), "activePreview": lines, "command": bounded(result)} + +def active_hwlab_ci_jobs(): + result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get jobs -n hwlab-ci --no-headers 2>/dev/null | awk '$2 != \"Complete\" && $2 != \"Failed\" {print}' | head -40"], 15) + lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()] + return {"ok": result["exitCode"] == 0, "activeCount": len(lines), "activePreview": lines, "command": bounded(result)} + +def wait_no_active_hwlab_ci(timeout=180): + deadline = time.time() + timeout + last = None + while time.time() < deadline: + writes = active_hwlab_ci_writes() + jobs = active_hwlab_ci_jobs() + last = {"writes": writes, "jobs": jobs} + if writes.get("ok") and jobs.get("ok") and int(writes.get("activeCount") or 0) == 0 and int(jobs.get("activeCount") or 0) == 0: + return {"ok": True, "last": last} + time.sleep(5) + return {"ok": False, "last": last} + +def kubectl_json(args, timeout=20): + result = command(["env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", "kubectl"] + args + ["-o", "json"], timeout) + if result["exitCode"] != 0: + return None + try: + return json.loads(result["stdout"] or "{}") + except Exception: + return None + +def kctl(args, timeout=30): + return command(["env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", "kubectl"] + args, timeout) + +def workload_image_refs(): + result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get deploy,sts,ds,pod -A -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.initContainers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null | sort -u"], 30) + refs = set() + digests = set() + for image in (result.get("stdout") or "").splitlines(): + image = image.strip() + if not image.startswith("127.0.0.1:5000/"): + continue + ref = image.split("127.0.0.1:5000/", 1)[1] + if "@sha256:" in ref: + repo, digest = ref.split("@", 1) + refs.add((repo, "@" + digest)) + digests.add("sha256:" + digest.split(":", 1)[1]) + elif ":" in ref: + repo, tag = ref.rsplit(":", 1) + refs.add((repo, tag)) + return refs, digests, bounded(result) + +def registry_request(method, path, headers=None, timeout=20): + url = "http://127.0.0.1:5000" + path + req = urllib.request.Request(url, method=method, headers=headers or {}) + with urllib.request.urlopen(req, timeout=timeout) as response: + body = response.read() + return {"status": response.status, "headers": dict(response.headers), "body": body.decode("utf-8", errors="replace")} + +def registry_tag_rows(): + rows = [] + root = REGISTRY_REPOSITORY_ROOT + if not os.path.isdir(root): + return rows + for repo_root, dirs, files in os.walk(root): + if os.path.basename(repo_root) != "tags": + continue + rel = os.path.relpath(repo_root, root) + suffix = "/_manifests/tags" + if not rel.endswith(suffix): + continue + repo = rel[:-len(suffix)] + try: + tags = os.listdir(repo_root) + except OSError: + continue + for tag in sorted(tags): + link = os.path.join(repo_root, tag, "current", "link") + if not os.path.isfile(link): + continue + try: + with open(link, "r", encoding="utf-8") as handle: + digest = handle.read().strip() + stat = os.stat(link) + except OSError: + continue + rows.append({ + "repo": repo, + "tag": tag, + "digest": digest, + "mtime": stat.st_mtime, + "mtimeIso": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(stat.st_mtime)), + "path": os.path.join(repo_root, tag), + }) + return rows + +def plan_registry_retention(): + keep_per_repo = int(OPTIONS.get("registryKeepPerRepo") or 5) + min_age_hours = float(OPTIONS.get("registryMinAgeHours") or 48) + cutoff = time.time() - min_age_hours * 3600 + refs, digests, refs_command = workload_image_refs() + rows = registry_tag_rows() + by_repo = {} + for row in rows: + by_repo.setdefault(row["repo"], []).append(row) + keep = set() + keep_reasons = {} + for repo, items in by_repo.items(): + items.sort(key=lambda item: item["mtime"], reverse=True) + for row in items[:keep_per_repo]: + key = (row["repo"], row["tag"]) + keep.add(key) + keep_reasons[key] = "latest-per-repo" + for row in items: + key = (row["repo"], row["tag"]) + if row["tag"] in REGISTRY_PROTECTED_TAGS: + keep.add(key) + keep_reasons[key] = "protected-tag" + if key in refs: + keep.add(key) + keep_reasons[key] = "workload-tag-ref" + if row["digest"] in digests: + keep.add(key) + keep_reasons[key] = "workload-digest-ref" + if row["repo"].startswith("hwlab/cache/"): + keep.add(key) + keep_reasons[key] = "cache-repo" + if row["mtime"] >= cutoff: + keep.add(key) + keep_reasons[key] = "recent-tag" + delete_rows = [] + kept_count = 0 + delete_by_repo = {} + keep_by_repo = {} + kept_digests = set() + for row in rows: + key = (row["repo"], row["tag"]) + should_delete = ( + key not in keep + and row["repo"].startswith("hwlab/hwlab-") + and re.match(r"^[0-9a-f]{7,40}$", row["tag"]) is not None + ) + if should_delete: + delete_rows.append(row) + delete_by_repo[row["repo"]] = delete_by_repo.get(row["repo"], 0) + 1 + else: + kept_count += 1 + kept_digests.add(row["digest"]) + keep_by_repo[row["repo"]] = keep_by_repo.get(row["repo"], 0) + 1 + deletable_manifests = {} + for row in delete_rows: + if row["digest"] in kept_digests: + continue + deletable_manifests.setdefault(row["repo"], set()).add(row["digest"]) + deletable_manifest_count = sum(len(items) for items in deletable_manifests.values()) + estimate = 0 + registry_size = du_size(REGISTRY_ROOT, 30) or 0 + if rows: + estimate = int(registry_size * min(0.75, deletable_manifest_count / float(len(rows)) * 0.55)) + return { + "tagRows": rows, + "deleteRows": delete_rows, + "summary": { + "totalTags": len(rows), + "repoCount": len(by_repo), + "keepPerRepo": keep_per_repo, + "minAgeHours": min_age_hours, + "protectedWorkloadRefs": len(refs), + "protectedDigestRefs": len(digests), + "keptTags": kept_count, + "deleteTags": len(delete_rows), + "deleteManifests": deletable_manifest_count, + "deleteByRepo": delete_by_repo, + "keepByRepo": keep_by_repo, + "registrySizeBytes": registry_size, + "estimatedReclaimBytes": estimate, + }, + "deleteManifestsByRepo": {repo: sorted(list(digests)) for repo, digests in deletable_manifests.items()}, + "refsCommand": refs_command, + } + +def registry_deployment_preflight(): + dep = kubectl_json(["-n", "hwlab-ci", "get", "deploy", "hwlab-registry"], 20) + if not dep: + return {"ok": False, "reason": "registry-deployment-missing"} + spec = ((dep.get("spec") or {}).get("template") or {}).get("spec") or {} + containers = spec.get("containers") or [] + volumes = spec.get("volumes") or [] + registry_container = next((item for item in containers if item.get("name") == "registry"), containers[0] if containers else {}) + mounts = registry_container.get("volumeMounts") or [] + has_host_path = any(((vol.get("hostPath") or {}).get("path") == REGISTRY_ROOT and vol.get("name") == "storage") for vol in volumes) + has_mount = any((mount.get("name") == "storage" and mount.get("mountPath") == "/var/lib/registry") for mount in mounts) + image = str(registry_container.get("image") or "") + ok = bool(has_host_path and has_mount and image.startswith("registry:") and spec.get("hostNetwork") is True) + return { + "ok": ok, + "reason": "ok" if ok else "unexpected-registry-deployment-shape", + "image": image, + "hostNetwork": spec.get("hostNetwork"), + "hasExpectedHostPath": has_host_path, + "hasExpectedMount": has_mount, + "replicas": (dep.get("spec") or {}).get("replicas"), + "readyReplicas": (dep.get("status") or {}).get("readyReplicas"), + } + +def cronjob_suspend_states(names): + states = {} + for name in names: + data = kubectl_json(["-n", "hwlab-ci", "get", "cronjob", name], 15) + if data: + states[name] = bool(((data.get("spec") or {}).get("suspend")) is True) + return states + +def patch_cronjob_suspend(name, suspend): + payload = json.dumps({"spec": {"suspend": bool(suspend)}}) + return kctl(["-n", "hwlab-ci", "patch", "cronjob", name, "--type=merge", "-p", payload], 30) + +def wait_registry_pod_count(target, timeout=90): + deadline = time.time() + timeout + last = None + while time.time() < deadline: + result = kctl(["-n", "hwlab-ci", "get", "pods", "-l", "app.kubernetes.io/name=hwlab-registry", "--no-headers"], 20) + last = bounded(result) + lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()] + active = [] + for line in lines: + parts = line.split() + status = parts[2] if len(parts) >= 3 else "" + if status in set(["Completed", "Error", "Failed", "Succeeded"]): + continue + active.append(line) + if len(active) == target: + return {"ok": True, "lines": active, "allLines": lines, "last": last} + time.sleep(2) + return {"ok": False, "lines": [], "last": last} + +def wait_pod_terminal(name, timeout=900): + deadline = time.time() + timeout + last = None + while time.time() < deadline: + data = kubectl_json(["-n", "hwlab-ci", "get", "pod", name], 20) + if data: + phase = ((data.get("status") or {}).get("phase")) or "" + last = {"phase": phase} + if phase == "Succeeded": + return {"ok": True, "phase": phase} + if phase == "Failed": + return {"ok": False, "phase": phase} + time.sleep(3) + return {"ok": False, "phase": "Timeout", "last": last} + +def execute_registry_retention(): + if PROVIDER_ID.upper() != "G14": + raise RuntimeError("HWLAB registry retention is only supported on G14") + deployment = registry_deployment_preflight() + if not deployment.get("ok"): + raise RuntimeError("registry deployment preflight failed: %s" % deployment.get("reason")) + active = active_hwlab_ci_writes() + if not active.get("ok") or int(active.get("activeCount") or 0) > 0: + raise RuntimeError("refusing registry maintenance while hwlab-ci PipelineRun/TaskRun is active") + plan = plan_registry_retention() + delete_rows = plan.get("deleteRows") or [] + delete_manifests = plan.get("deleteManifestsByRepo") or {} + if not delete_rows: + return {"reclaimedBytes": 0, "commandOutput": {"message": "no registry tags matched conservative retention", "registryPlan": plan.get("summary")}} + if not delete_manifests: + return {"reclaimedBytes": 0, "commandOutput": {"message": "matched tags are still referenced by retained tags; registry GC would not reclaim blobs", "registryPlan": plan.get("summary")}} + cronjobs = ["hwlab-g14-branch-poller", "hwlab-v02-branch-poller"] + original_crons = cronjob_suspend_states(cronjobs) + before = du_size(REGISTRY_ROOT, 60) or 0 + gc_name = "hwlab-registry-gc-%s" % int(time.time()) + steps = [] + try: + for name in original_crons: + result = patch_cronjob_suspend(name, True) + steps.append({"step": "suspend-cronjob", "name": name, "result": bounded(result)}) + if result["exitCode"] != 0: + raise RuntimeError("failed to suspend cronjob %s" % name) + idle_after_suspend = wait_no_active_hwlab_ci(180) + steps.append({"step": "idle-after-suspend", "result": idle_after_suspend}) + if not idle_after_suspend.get("ok"): + raise RuntimeError("refusing registry maintenance because hwlab-ci did not become idle after suspend") + + deleted_manifests = [] + for repo, digests in delete_manifests.items(): + encoded_repo = "/".join(urllib.parse.quote(part, safe="") for part in repo.split("/")) + for digest in digests: + try: + result = registry_request("DELETE", "/v2/%s/manifests/%s" % (encoded_repo, urllib.parse.quote(digest, safe=":")), {"Accept": "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json"}) + deleted_manifests.append({"repo": repo, "digest": digest, "status": result.get("status")}) + except urllib.error.HTTPError as exc: + if exc.code == 404: + deleted_manifests.append({"repo": repo, "digest": digest, "status": 404}) + else: + raise + steps.append({"step": "registry-api-delete-manifests", "count": len(deleted_manifests), "preview": deleted_manifests[:20]}) + + scale_down = kctl(["-n", "hwlab-ci", "scale", "deploy", "hwlab-registry", "--replicas=0"], 60) + steps.append({"step": "scale-registry-down", "result": bounded(scale_down)}) + if scale_down["exitCode"] != 0: + raise RuntimeError("failed to scale registry down") + waited_down = wait_registry_pod_count(0, 120) + steps.append({"step": "wait-registry-down", "result": waited_down}) + if not waited_down.get("ok"): + raise RuntimeError("registry pod did not scale down") + + deleted = [] + for row in delete_rows: + path = os.path.abspath(str(row.get("path") or "")) + if not path.startswith(REGISTRY_REPOSITORY_ROOT + "/") or "/_manifests/tags/" not in path: + raise RuntimeError("refusing unexpected registry tag path: %s" % path) + if not re.match(r"^[0-9a-f]{7,40}$", str(row.get("tag") or "")): + raise RuntimeError("refusing unexpected registry tag name: %s" % row.get("tag")) + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + deleted.append({"repo": row.get("repo"), "tag": row.get("tag"), "digest": row.get("digest")}) + steps.append({"step": "delete-tag-directories", "count": len(deleted)}) + + overrides = { + "apiVersion": "v1", + "spec": { + "restartPolicy": "Never", + "containers": [{ + "name": "registry-gc", + "image": "registry:2.8.3", + "command": ["registry", "garbage-collect", "/etc/docker/registry/config.yml"], + "volumeMounts": [{"name": "storage", "mountPath": "/var/lib/registry"}], + }], + "volumes": [{"name": "storage", "hostPath": {"path": REGISTRY_ROOT, "type": "DirectoryOrCreate"}}], + }, + } + run_gc = kctl(["-n", "hwlab-ci", "run", gc_name, "--restart=Never", "--image=registry:2.8.3", "--overrides=%s" % json.dumps(overrides)], 60) + steps.append({"step": "start-registry-gc-pod", "result": bounded(run_gc), "pod": gc_name}) + if run_gc["exitCode"] != 0: + raise RuntimeError("failed to start registry GC pod") + waited_gc = wait_pod_terminal(gc_name, 900) + steps.append({"step": "wait-registry-gc", "result": waited_gc}) + logs = kctl(["-n", "hwlab-ci", "logs", gc_name], 120) + steps.append({"step": "registry-gc-logs", "result": bounded(logs)}) + if not waited_gc.get("ok"): + raise RuntimeError("registry GC pod did not complete successfully") + finally: + cleanup_gc = kctl(["-n", "hwlab-ci", "delete", "pod", gc_name, "--ignore-not-found=true"], 60) + steps.append({"step": "delete-registry-gc-pod", "result": bounded(cleanup_gc)}) + scale_up = kctl(["-n", "hwlab-ci", "scale", "deploy", "hwlab-registry", "--replicas=%s" % int(deployment.get("replicas") or 1)], 60) + steps.append({"step": "scale-registry-up", "result": bounded(scale_up)}) + rollout = kctl(["-n", "hwlab-ci", "rollout", "status", "deploy/hwlab-registry", "--timeout=180s"], 200) + steps.append({"step": "wait-registry-rollout", "result": bounded(rollout)}) + for name, was_suspended in original_crons.items(): + restore = patch_cronjob_suspend(name, was_suspended) + steps.append({"step": "restore-cronjob", "name": name, "suspend": was_suspended, "result": bounded(restore)}) + after = du_size(REGISTRY_ROOT, 60) or 0 + return { + "reclaimedBytes": max(0, before - after), + "commandOutput": { + "registryPlan": plan.get("summary"), + "deletedTagCount": len(delete_rows), + "deletedManifestCount": sum(len(items) for items in delete_manifests.values()), + "diskBeforeBytes": before, + "diskAfterBytes": after, + "steps": steps[-12:], + }, + } + +def execute_registry_garbage_collect_only(): + if PROVIDER_ID.upper() != "G14": + raise RuntimeError("HWLAB registry garbage-collect is only supported on G14") + deployment = registry_deployment_preflight() + if not deployment.get("ok"): + raise RuntimeError("registry deployment preflight failed: %s" % deployment.get("reason")) + active = active_hwlab_ci_writes() + if not active.get("ok") or int(active.get("activeCount") or 0) > 0: + raise RuntimeError("refusing registry maintenance while hwlab-ci PipelineRun/TaskRun is active") + cronjobs = ["hwlab-g14-branch-poller", "hwlab-v02-branch-poller"] + original_crons = cronjob_suspend_states(cronjobs) + before = du_size(REGISTRY_ROOT, 60) or 0 + gc_name = "hwlab-registry-gc-%s" % int(time.time()) + steps = [] + try: + for name in original_crons: + result = patch_cronjob_suspend(name, True) + steps.append({"step": "suspend-cronjob", "name": name, "result": bounded(result)}) + if result["exitCode"] != 0: + raise RuntimeError("failed to suspend cronjob %s" % name) + idle_after_suspend = wait_no_active_hwlab_ci(180) + steps.append({"step": "idle-after-suspend", "result": idle_after_suspend}) + if not idle_after_suspend.get("ok"): + raise RuntimeError("refusing registry maintenance because hwlab-ci did not become idle after suspend") + + scale_down = kctl(["-n", "hwlab-ci", "scale", "deploy", "hwlab-registry", "--replicas=0"], 60) + steps.append({"step": "scale-registry-down", "result": bounded(scale_down)}) + if scale_down["exitCode"] != 0: + raise RuntimeError("failed to scale registry down") + waited_down = wait_registry_pod_count(0, 120) + steps.append({"step": "wait-registry-down", "result": waited_down}) + if not waited_down.get("ok"): + raise RuntimeError("registry pod did not scale down") + + overrides = { + "apiVersion": "v1", + "spec": { + "restartPolicy": "Never", + "containers": [{ + "name": "registry-gc", + "image": "registry:2.8.3", + "command": ["registry", "garbage-collect", "/etc/docker/registry/config.yml"], + "volumeMounts": [{"name": "storage", "mountPath": "/var/lib/registry"}], + }], + "volumes": [{"name": "storage", "hostPath": {"path": REGISTRY_ROOT, "type": "DirectoryOrCreate"}}], + }, + } + run_gc = kctl(["-n", "hwlab-ci", "run", gc_name, "--restart=Never", "--image=registry:2.8.3", "--overrides=%s" % json.dumps(overrides)], 60) + steps.append({"step": "start-registry-gc-pod", "result": bounded(run_gc), "pod": gc_name}) + if run_gc["exitCode"] != 0: + raise RuntimeError("failed to start registry GC pod") + waited_gc = wait_pod_terminal(gc_name, 900) + steps.append({"step": "wait-registry-gc", "result": waited_gc}) + logs = kctl(["-n", "hwlab-ci", "logs", gc_name], 120) + steps.append({"step": "registry-gc-logs", "result": bounded(logs)}) + if not waited_gc.get("ok"): + raise RuntimeError("registry GC pod did not complete successfully") + finally: + cleanup_gc = kctl(["-n", "hwlab-ci", "delete", "pod", gc_name, "--ignore-not-found=true"], 60) + steps.append({"step": "delete-registry-gc-pod", "result": bounded(cleanup_gc)}) + scale_up = kctl(["-n", "hwlab-ci", "scale", "deploy", "hwlab-registry", "--replicas=%s" % int(deployment.get("replicas") or 1)], 60) + steps.append({"step": "scale-registry-up", "result": bounded(scale_up)}) + rollout = kctl(["-n", "hwlab-ci", "rollout", "status", "deploy/hwlab-registry", "--timeout=180s"], 200) + steps.append({"step": "wait-registry-rollout", "result": bounded(rollout)}) + for name, was_suspended in original_crons.items(): + restore = patch_cronjob_suspend(name, was_suspended) + steps.append({"step": "restore-cronjob", "name": name, "suspend": was_suspended, "result": bounded(restore)}) + after = du_size(REGISTRY_ROOT, 60) or 0 + return { + "reclaimedBytes": max(0, before - after), + "commandOutput": { + "message": "official registry garbage-collect only; no additional tag deletion", + "diskBeforeBytes": before, + "diskAfterBytes": after, + "steps": steps[-12:], + }, + } + +def start_registry_retention_job(mode): + job_id = "g14-registry-%s-%s" % (int(time.time()), os.getpid()) + paths = job_paths(job_id) + started_at = now_iso() + initial = { + "ok": True, + "action": "gc remote status", + "providerId": PROVIDER_ID, + "jobId": job_id, + "status": "running", + "kind": "hwlab-registry-retention-gc" if mode == "retention" else "hwlab-registry-garbage-collect", + "mode": mode, + "startedAt": started_at, + "statePath": paths["state"], + "logPath": paths["log"], + "options": OPTIONS, + } + write_json_atomic(paths["state"], initial) + pid = os.fork() + if pid != 0: + return { + "status": "started", + "reclaimedBytes": None, + "commandOutput": { + "jobId": job_id, + "pid": pid, + "statePath": paths["state"], + "logPath": paths["log"], + "statusCommand": "bun scripts/cli.ts gc remote %s status --job-id %s" % (PROVIDER_ID, job_id), + "message": "registry retention GC is running as a detached remote job", + }, + } + + try: + os.setsid() + except Exception: + pass + try: + devnull = os.open(os.devnull, os.O_RDONLY) + os.dup2(devnull, 0) + os.close(devnull) + except Exception: + pass + try: + log_handle = open(paths["log"], "a", encoding="utf-8", buffering=1) + os.dup2(log_handle.fileno(), 1) + os.dup2(log_handle.fileno(), 2) + except Exception: + log_handle = None + try: + print("[%s] starting HWLAB registry %s job %s" % (now_iso(), mode, job_id), flush=True) + result = execute_registry_retention() if mode == "retention" else execute_registry_garbage_collect_only() + payload = dict(initial) + payload.update({ + "status": "succeeded", + "finishedAt": now_iso(), + "result": result, + "diskAfter": df_snapshot(), + "clusterAfter": cluster_preflight(), + }) + write_json_atomic(paths["state"], payload) + print("[%s] completed HWLAB registry %s job %s" % (now_iso(), mode, job_id), flush=True) + os._exit(0) + except Exception as exc: + payload = dict(initial) + payload.update({ + "ok": False, + "status": "failed", + "finishedAt": now_iso(), + "error": str(exc), + "diskAfter": df_snapshot(), + "clusterAfter": cluster_preflight(), + }) + try: + write_json_atomic(paths["state"], payload) + except Exception: + pass + print("[%s] failed HWLAB registry %s job %s: %s" % (now_iso(), mode, job_id, exc), flush=True) + os._exit(1) + finally: + try: + if log_handle: + log_handle.close() + except Exception: + pass + +def collect_protected(): + protected_paths = [ + ("hwlab-k3s-runtime", "/var/lib/rancher/k3s", "Native k3s runtime, containerd state, local-path storage and control-plane data are protected."), + ("hwlab-k3s-storage", "/var/lib/rancher/k3s/storage", "Local-path PVC data is protected; cleanup must go through HWLAB/Tekton retention commands."), + ("hwlab-kubelet", "/var/lib/kubelet", "Kubelet pod/runtime state is protected."), + ("host-containerd", "/var/lib/containerd", "Host containerd state is protected; generic gc does not prune containerd images."), + ("hwlab-host-data", "/var/lib/hwlab", "HWLAB host data/cache is protected from generic remote gc until a HWLAB-specific retention rule classifies it."), + ("hwlab-source", "/root/hwlab", "G14 HWLAB fixed source workspace is protected."), + ("hwlab-v02-source", "/root/hwlab-v02", "HWLAB v0.2 fixed source workspace is protected."), + ("agentrun-source", "/root/agentrun", "AgentRun fixed source workspace is protected."), + ("docker-images-and-volumes", "docker-images-volumes", "Remote gc does not remove Docker images, containers, volumes or Compose projects."), + ("k8s-api-objects", "deployments-statefulsets-secrets-pvcs", "Remote gc does not mutate Kubernetes workloads, Secrets, PVCs, PVs, Argo CD or Tekton objects."), + ] + result = [] + for kind, ref, reason in protected_paths: + item = {"kind": kind, "risk": "blocked", "ref": ref, "reason": reason} + if ref.startswith("/") and os.path.exists(ref): + item["sizeBytes"] = du_size(ref) + result.append(item) + return result + +def collect_candidates(observed_at): + candidates = [] + if OPTIONS.get("journal", True): + usage = command(["journalctl", "--disk-usage"], 5) + current = parse_journal_usage((usage["stdout"] or "") + (usage["stderr"] or "")) + target = int(OPTIONS.get("journalTargetBytes") or 536870912) + if current is not None and current > target: + candidates.append({ + "id": "journalctl:vacuum", + "kind": "journal-vacuum", + "risk": "medium", + "description": "Vacuum systemd journal to %s" % fmt_bytes(target), + "sizeBytes": current, + "estimatedReclaimBytes": max(0, current - target), + "action": {"command": ["journalctl", "--vacuum-size=%s" % target]}, + }) + + if OPTIONS.get("dockerLogs", True): + max_bytes = int(OPTIONS.get("dockerLogMaxBytes") or 52428800) + for container in docker_containers(): + path = container.get("logPath") or "" + if not path or not os.path.exists(path): + continue + try: + size = os.path.getsize(path) + except OSError: + continue + if size <= max_bytes: + continue + candidates.append({ + "id": "docker-json-log:%s" % container["id"], + "kind": "docker-json-log-truncate", + "risk": "medium", + "description": "Truncate Docker json-file log larger than %s" % fmt_bytes(max_bytes), + "path": path, + "container": {"id": container["id"][:12], "name": container["name"], "image": container["image"]}, + "sizeBytes": size, + "estimatedReclaimBytes": size, + "action": {"op": "truncate", "targetBytes": 0}, + }) + + if OPTIONS.get("buildCache", True): + system_df = command(["docker", "system", "df"], 8) + if system_df["exitCode"] == 0: + cache = parse_docker_build_cache(system_df["stdout"]) + if cache is not None and cache["reclaimableBytes"] > 0: + until = str(OPTIONS.get("buildCacheUntil") or "24h") + candidates.append({ + "id": "docker-builder:prune", + "kind": "docker-build-cache-prune", + "risk": "low", + "description": "Prune Docker BuildKit cache unused for %s" % until, + "sizeBytes": cache["sizeBytes"], + "estimatedReclaimBytes": cache["reclaimableBytes"], + "action": {"command": ["docker", "builder", "prune", "--all", "--force", "--filter", "until=%s" % until], "estimate": "docker-system-df-reclaimable-upper-bound"}, + }) + + if OPTIONS.get("aptCache", True): + apt_path = "/var/cache/apt/archives" + size = du_size(apt_path) or 0 + if size > 10 * 1024 * 1024: + candidates.append({ + "id": "apt-cache:clean", + "kind": "apt-cache-clean", + "risk": "low", + "description": "Clean downloaded apt package archives", + "path": apt_path, + "sizeBytes": size, + "estimatedReclaimBytes": size, + "action": {"command": ["apt-get", "clean"]}, + }) + + if OPTIONS.get("coreDumps", True): + cutoff = time.time() - float(OPTIONS.get("coreDumpMinAgeHours") or 1) * 3600 + for root in sorted(CORE_DUMP_DIR_ALLOWLIST): + if not os.path.isdir(root): + continue + for name in os.listdir(root): + if not re.match(r"^core\.\d+$", name): + continue + path = os.path.join(root, name) + try: + stat = os.lstat(path) + except OSError: + continue + if not os.path.isfile(path) or os.path.islink(path): + continue + if stat.st_mtime >= cutoff: + continue + disk_size = allocated_file_size(path) + candidates.append({ + "id": "core-dump:%s" % path, + "kind": "core-dump-delete", + "risk": "low", + "description": "Delete untracked process core dump older than %s hours" % OPTIONS.get("coreDumpMinAgeHours"), + "path": path, + "sizeBytes": int(disk_size), + "estimatedReclaimBytes": int(disk_size), + "apparentSizeBytes": int(stat.st_size), + "action": {"op": "unlink", "allowlist": "root-unidesk-core-dot-pid"}, + }) + + if OPTIONS.get("tmp", True) and os.path.isdir("/tmp"): + cutoff = time.time() - float(OPTIONS.get("tmpMinAgeHours") or 24) * 3600 + for name in os.listdir("/tmp"): + path = os.path.join("/tmp", name) + if path in TMP_EXACT_PROTECT: + continue + if not any(name.startswith(prefix) for prefix in TMP_PREFIX_ALLOWLIST): + continue + try: + stat = os.lstat(path) + except OSError: + continue + if stat.st_mtime >= cutoff: + continue + size = du_size(path, 8) or path_size(path) + if size <= 0: + continue + candidates.append({ + "id": "tmp:%s" % path, + "kind": "tmp-path-delete", + "risk": "low", + "description": "Delete allowlisted /tmp path older than %s hours" % OPTIONS.get("tmpMinAgeHours"), + "path": path, + "sizeBytes": size, + "estimatedReclaimBytes": size, + "action": {"op": "rm-recursive", "allowlist": "tmp-prefix"}, + }) + if OPTIONS.get("hwlabRegistry", False): + registry = plan_registry_retention() + summary = registry.get("summary") or {} + delete_rows = registry.get("deleteRows") or [] + estimate = int(summary.get("estimatedReclaimBytes") or 0) + if delete_rows: + candidates.append({ + "id": "hwlab-registry:retention-gc", + "kind": "hwlab-registry-retention-gc", + "risk": "medium", + "description": "Conservative HWLAB registry retention: keep current workload refs, recent tags and latest tags per repo, then run official registry garbage-collect", + "path": REGISTRY_ROOT, + "sizeBytes": int(summary.get("registrySizeBytes") or 0), + "estimatedReclaimBytes": estimate, + "action": { + "op": "registry-retention-gc", + "requiresMaintenanceWindow": True, + "keepPerRepo": summary.get("keepPerRepo"), + "minAgeHours": summary.get("minAgeHours"), + "deleteTags": len(delete_rows), + "deleteManifests": summary.get("deleteManifests"), + "deleteByRepo": summary.get("deleteByRepo"), + "protectedWorkloadRefs": summary.get("protectedWorkloadRefs"), + "protectedDigestRefs": summary.get("protectedDigestRefs"), + }, + }) + elif bool(OPTIONS.get("registryGcOnly")) and int(summary.get("totalTags") or 0) > 0 and int(summary.get("deleteTags") or 0) == 0: + candidates.append({ + "id": "hwlab-registry:garbage-collect-only", + "kind": "hwlab-registry-garbage-collect", + "risk": "medium", + "description": "Run official HWLAB registry garbage-collect without deleting additional tags; useful after a previously interrupted retention run", + "path": REGISTRY_ROOT, + "sizeBytes": int(summary.get("registrySizeBytes") or 0), + "estimatedReclaimBytes": 0, + "action": { + "op": "registry-garbage-collect-only", + "requiresMaintenanceWindow": True, + "deleteTags": 0, + "registryPlan": summary, + }, + }) + return sorted(candidates, key=lambda item: item.get("estimatedReclaimBytes") or 0, reverse=True) + +def summarize(candidates, returned): + by_kind = {} + total = 0 + for item in candidates: + size = int(item.get("estimatedReclaimBytes") or 0) + total += size + kind = item.get("kind") or "unknown" + current = by_kind.setdefault(kind, {"count": 0, "estimatedReclaimBytes": 0, "estimatedReclaim": "0 B"}) + current["count"] += 1 + current["estimatedReclaimBytes"] += size + current["estimatedReclaim"] = fmt_bytes(current["estimatedReclaimBytes"]) + returned_total = sum(int(item.get("estimatedReclaimBytes") or 0) for item in returned) + return { + "candidateCount": len(candidates), + "returnedCandidateCount": len(returned), + "estimatedReclaimBytes": total, + "estimatedReclaim": fmt_bytes(total), + "returnedEstimatedReclaimBytes": returned_total, + "returnedEstimatedReclaim": fmt_bytes(returned_total), + "byKind": by_kind, + } + +def assert_tmp_candidate(path): + resolved = os.path.abspath(path) + if not resolved.startswith("/tmp/"): + raise RuntimeError("refusing to remove non-/tmp path: %s" % path) + if resolved in TMP_EXACT_PROTECT: + raise RuntimeError("refusing to remove protected tmp path: %s" % path) + name = os.path.basename(resolved) + if not any(name.startswith(prefix) for prefix in TMP_PREFIX_ALLOWLIST): + raise RuntimeError("refusing to remove tmp path outside allowlist: %s" % path) + +def assert_core_dump_candidate(path): + resolved = os.path.abspath(path) + parent = os.path.dirname(resolved) + name = os.path.basename(resolved) + if parent not in CORE_DUMP_DIR_ALLOWLIST: + raise RuntimeError("refusing to remove core dump outside allowlisted directory: %s" % path) + if not re.match(r"^core\.\d+$", name): + raise RuntimeError("refusing to remove non core. file: %s" % path) + if not os.path.isfile(resolved) or os.path.islink(resolved): + raise RuntimeError("refusing to remove non-regular core dump: %s" % path) + git = command(["git", "-C", parent, "ls-files", "--error-unmatch", name], 10) + if git["exitCode"] == 0: + raise RuntimeError("refusing to remove git-tracked file: %s" % path) + if git["exitCode"] != 1: + raise RuntimeError("refusing to remove core dump because git tracking check failed: %s" % path) + fuser = command(["fuser", resolved], 5) + if fuser["exitCode"] is None: + raise RuntimeError("refusing to remove core dump because fuser check was unavailable: %s" % path) + if fuser["exitCode"] == 0: + raise RuntimeError("refusing to remove core dump with active process reference: %s" % path) + +def execute(candidate): + kind = candidate.get("kind") + if kind == "journal-vacuum": + result = command(["journalctl", "--vacuum-size=%s" % int(OPTIONS.get("journalTargetBytes") or 536870912)], 30) + if result["exitCode"] != 0: + raise RuntimeError((result["stderr"] or "journalctl vacuum failed").strip()) + return {"reclaimedBytes": None, "commandOutput": bounded(result)} + if kind == "docker-json-log-truncate": + path = candidate.get("path") or "" + if not path.startswith("/var/lib/docker/containers/"): + raise RuntimeError("refusing to truncate Docker log outside /var/lib/docker/containers") + before = os.path.getsize(path) if os.path.exists(path) else 0 + with open(path, "r+b") as handle: + handle.truncate(0) + return {"reclaimedBytes": before} + if kind == "docker-build-cache-prune": + until = str(OPTIONS.get("buildCacheUntil") or "24h") + result = command(["docker", "builder", "prune", "--all", "--force", "--filter", "until=%s" % until], 45) + if result["exitCode"] != 0: + raise RuntimeError((result["stderr"] or "docker builder prune failed").strip()) + return {"reclaimedBytes": None, "commandOutput": bounded(result)} + if kind == "apt-cache-clean": + before = du_size("/var/cache/apt/archives") or 0 + result = command(["apt-get", "clean"], 30) + if result["exitCode"] != 0: + raise RuntimeError((result["stderr"] or "apt-get clean failed").strip()) + after = du_size("/var/cache/apt/archives") or 0 + return {"reclaimedBytes": max(0, before - after), "commandOutput": bounded(result)} + if kind == "tmp-path-delete": + path = candidate.get("path") or "" + assert_tmp_candidate(path) + before = du_size(path, 8) or path_size(path) + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path, ignore_errors=True) + else: + try: + os.unlink(path) + except FileNotFoundError: + pass + return {"reclaimedBytes": before} + if kind == "core-dump-delete": + path = candidate.get("path") or "" + assert_core_dump_candidate(path) + before = allocated_file_size(path) + os.unlink(path) + return {"reclaimedBytes": before} + if kind == "hwlab-registry-retention-gc": + return start_registry_retention_job("retention") + if kind == "hwlab-registry-garbage-collect": + return start_registry_retention_job("garbage-collect") + raise RuntimeError("unsupported remote gc candidate kind: %s" % kind) + +def visible_items(items): + if bool(OPTIONS.get("full")): + return items + return items[:int(OPTIONS.get("limit") or 50)] + +def returned_results(results): + if bool(OPTIONS.get("full")): + return results + failed = [item for item in results if item.get("status") == "failed"] + started = [item for item in results if item.get("status") == "started"] + succeeded = [item for item in results if item.get("status") == "succeeded"] + return (failed + started + succeeded)[:int(OPTIONS.get("resultLimit") or 50)] + +def plan_payload(observed_at, preflight, protected, candidates, visible): + return { + "ok": True, + "action": "gc remote plan", + "providerId": PROVIDER_ID, + "dryRun": True, + "mutation": False, + "observedAt": observed_at, + "options": OPTIONS, + "diskBefore": df_snapshot(), + "clusterPreflight": preflight, + "summary": summarize(candidates, visible), + "candidates": visible, + "protected": protected, + "policy": { + "requiresRunConfirm": True, + "runCommand": "bun scripts/cli.ts gc remote %s run --confirm" % PROVIDER_ID, + "neverTouches": [ + "/var/lib/rancher/k3s", + "/var/lib/rancher/k3s/storage", + "/var/lib/kubelet", + "/var/lib/containerd", + "/var/lib/hwlab unless --include-hwlab-registry is explicitly supplied", + "Kubernetes Deployments/StatefulSets/Secrets/PVCs/PVs", + "HWLAB fixed source workspaces", + "Docker images, containers and volumes", + ], + "notes": [ + "Remote gc only executes the returned candidate page unless --full or a larger --limit is supplied.", + "G14 run requires the expected native k3s node preflight before mutation.", + "HWLAB DEV runtime and local-path PVC data are protected and require HWLAB-specific retention commands.", + "Core dump cleanup only removes untracked /root/unidesk/core. regular files with no active fuser reference.", + "HWLAB registry retention is opt-in: it keeps workload tag/digest refs, all tags newer than the retention age and the newest N tags per repo before official registry garbage-collect.", + ], + }, + } + +def main(): + observed_at = now_iso() + preflight = cluster_preflight() + protected = collect_protected() + candidates = collect_candidates(observed_at) + visible = visible_items(candidates) + if ACTION == "plan": + print(json.dumps(plan_payload(observed_at, preflight, protected, candidates, visible), ensure_ascii=False, indent=2)) + return 0 + if ACTION == "status": + print(json.dumps(remote_gc_job_status(), ensure_ascii=False, indent=2)) + return 0 + if ACTION != "run": + print(json.dumps({"ok": False, "error": "unsupported-remote-gc-action", "action": ACTION}, ensure_ascii=False, indent=2)) + return 0 + if PROVIDER_ID.upper() == "G14" and not preflight.get("ok"): + print(json.dumps({ + "ok": False, + "error": "gc-remote-g14-preflight-failed", + "action": "gc remote run", + "providerId": PROVIDER_ID, + "dryRun": True, + "mutation": False, + "clusterPreflight": preflight, + "plan": plan_payload(observed_at, preflight, protected, candidates, visible), + }, ensure_ascii=False, indent=2)) + return 0 + disk_before = df_snapshot() + results = [] + for candidate in visible: + try: + execution = execute(candidate) + item = dict(candidate) + item.update({"status": execution.get("status") or "succeeded", "reclaimedBytes": execution.get("reclaimedBytes")}) + if "commandOutput" in execution: + item["commandOutput"] = execution["commandOutput"] + results.append(item) + except Exception as exc: + item = dict(candidate) + item.update({"status": "failed", "reclaimedBytes": None, "error": str(exc)}) + results.append(item) + disk_after = df_snapshot() + failed = [item for item in results if item.get("status") == "failed"] + returned = returned_results(results) + payload = { + "ok": len(failed) == 0, + "action": "gc remote run", + "providerId": PROVIDER_ID, + "dryRun": False, + "mutation": True, + "observedAt": now_iso(), + "options": OPTIONS, + "diskBefore": disk_before, + "diskAfter": disk_after, + "clusterPreflight": preflight, + "clusterAfter": cluster_preflight(), + "summary": { + "plannedCandidateCount": len(visible), + "attemptedCount": len(results), + "startedCount": len([item for item in results if item.get("status") == "started"]), + "succeededCount": len([item for item in results if item.get("status") == "succeeded"]), + "failedCount": len(failed), + "estimatedReclaimBytes": sum(int(item.get("estimatedReclaimBytes") or 0) for item in visible), + "actualDiskReclaimBytes": (disk_after["availableBytes"] - disk_before["availableBytes"]) if disk_before and disk_after else None, + "resultCount": len(results), + "returnedResultCount": len(returned), + "omittedResultCount": max(0, len(results) - len(returned)), + }, + "results": returned, + "protected": protected, + } + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) +`; +} diff --git a/scripts/src/gc.ts b/scripts/src/gc.ts index a1c727cc..4db8f4e8 100644 --- a/scripts/src/gc.ts +++ b/scripts/src/gc.ts @@ -3,6 +3,7 @@ import { closeSync, existsSync, ftruncateSync, lstatSync, mkdirSync, openSync, r import { basename, join, resolve } from "node:path"; import { type UniDeskConfig, repoRoot, rootPath } from "./config"; +import { runRemoteGcCommand } from "./gc-remote"; type GcRisk = "low" | "medium" | "high" | "blocked"; type GcItemKind = @@ -183,6 +184,10 @@ const TMP_EXACT_PROTECT = new Set([ export async function runGcCommand(config: UniDeskConfig, args: string[]): Promise { const [action = "plan", ...rest] = args; + if (action === "remote") { + const [providerId, subaction = "plan", ...remoteArgs] = rest; + return await runRemoteGcCommand(config, providerId, subaction, remoteArgs); + } if (action === "policy") { const [subaction = "plan", ...policyArgs] = rest; const options = parseGcPolicyOptions(policyArgs); @@ -243,7 +248,7 @@ export async function runGcCommand(config: UniDeskConfig, args: string[]): Promi ok: false, error: "unsupported-gc-action", action, - supportedActions: ["plan", "run", "db-trace", "policy"], + supportedActions: ["plan", "run", "db-trace", "policy", "remote"], }; } diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 5ab2a1f9..cc56bcc4 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -18,7 +18,7 @@ export function rootHelp(): unknown { { command: "server swap status|ensure [--path /swapfile] [--size 2GiB] [--dry-run]", description: "Inspect or idempotently create host swap for low-memory main-server operation." }, { command: "server logs [--tail-bytes N]", description: "Return bounded tails from file logs and docker logs." }, { command: "server cleanup plan [--min-age-hours N] [--limit N]", description: "Dry-run Docker image cleanup plan only: list active/protected images, stale candidates older than the default 24h threshold, risk, estimated reclaim, and manual review commands without deleting anything." }, - { command: "gc plan|run|db-trace|policy [--confirm] [--logs-keep-days N] [--include-browser-cache]", description: "One-time main-server disk relief and low-risk anti-bloat policy for logs, journald, allowlisted /tmp artifacts and explicit trace telemetry retention; plan is read-only and run requires --confirm." }, + { command: "gc plan|run|db-trace|policy|remote [--confirm] [--logs-keep-days N] [--include-browser-cache]", description: "One-time main-server or remote provider disk relief and low-risk anti-bloat policy for logs, journald, allowlisted /tmp artifacts, scoped core dumps and explicit trace telemetry retention; plan is read-only and run requires --confirm." }, { command: "server rebuild ", description: "Maintenance-only local Compose rebuild for reviewed main-server services; frontend standard release must use CI artifact plus deploy apply dev/prod artifact consumers." }, { command: "provider attach [--master-server URL] [--up] [--force] | provider triage [--observed-error text] [--observed-scope scope] [--microservice id ...] [--full|--raw]", description: "Generate the minimal external provider-gateway env/compose bundle or run the low-noise read-only provider health triage contract." }, { command: "ssh [operation args...]", description: "Open a Host SSH / WSL SSH maintenance session through the provider-gateway bridge; route syntax such as `G14:k3s` or `D601:win/c/test` only locates distributed targets." }, @@ -263,7 +263,7 @@ function providerHelp(): unknown { function gcHelp(): unknown { return { - command: "gc plan|run|db-trace|policy", + command: "gc plan|run|db-trace|policy|remote", output: "json", usage: [ "bun scripts/cli.ts gc plan", @@ -275,9 +275,12 @@ function gcHelp(): unknown { "bun scripts/cli.ts gc db-trace run --confirm --before-date 2026-05-25 --vacuum-full", "bun scripts/cli.ts gc policy plan", "bun scripts/cli.ts gc policy install", + "bun scripts/cli.ts gc remote G14 plan", + "bun scripts/cli.ts gc remote G14 run --confirm", + "bun scripts/cli.ts gc remote G14 status --job-id ", "bun scripts/cli.ts gc plan --full", ], - description: "Plan or execute bounded one-time main-server disk relief for file logs, Docker json logs, systemd journal, Docker BuildKit cache, allowlisted /tmp artifacts and explicitly scoped database trace telemetry retention.", + description: "Plan or execute bounded one-time disk relief for file logs, Docker json logs, systemd journal, Docker BuildKit cache, allowlisted /tmp artifacts, scoped remote core dumps and explicitly scoped database trace telemetry retention.", safety: { default: "plan is read-only and mutation=false", runGuard: "run requires --confirm", @@ -293,6 +296,13 @@ function gcHelp(): unknown { "--build-cache-until DURATION": "prune Docker builder cache unused for duration; default 24h", "--build-cache-all": "prune all Docker builder cache without an until filter", "--tmp-min-age-hours N": "delete allowlisted /tmp artifacts older than N hours; default 24", + "--core-dump-min-age-hours N": "remote only: delete untracked allowlisted core. dumps older than N hours; default 1", + "--no-core-dumps": "remote only: do not include scoped core dump cleanup candidates", + "--include-hwlab-registry": "remote G14 only: opt in to conservative HWLAB registry tag retention plus official registry garbage-collect", + "--registry-gc-only": "remote G14 only: run official registry garbage-collect without deleting additional tags; intended for interrupted registry retention recovery", + "--registry-keep-per-repo N": "remote registry only: keep at least N newest tags per service repo; default 20, minimum 10", + "--registry-min-age-hours N": "remote registry only: keep all tags newer than N hours; default 48, minimum 24", + "--job-id ID": "remote status only: inspect a long-running remote gc job", "--limit N": "number of candidates returned and executed by run when --full is not set; default 50", "--result-limit N": "number of per-candidate run results returned when --full is not set; default 50", "--full|--raw": "return and run against all candidates rather than the default bounded page", @@ -300,6 +310,7 @@ function gcHelp(): unknown { "db-trace --before-date YYYY-MM-DD": "plan or delete default trace telemetry event types before the date", "db-trace run --vacuum-full": "rewrite public.oa_events after deletion so df can reclaim disk; requires maintenance window", "policy plan|install": "render or install journald caps and a daily file-log plus allowlisted /tmp low-risk gc systemd timer", + "remote plan|run": "run bounded GC through UniDesk SSH passthrough on a provider host; G14 protects HWLAB k3s/runtime/PVC/workspace paths, and HWLAB registry retention is explicit opt-in with workload-ref, recent-tag and per-repo tag protection", "--no-file-logs|--no-docker-logs|--no-journal|--no-build-cache|--no-tmp|--no-db-summary": "disable one collector", }, reference: "docs/reference/cli.md", diff --git a/scripts/src/ssh.ts b/scripts/src/ssh.ts index f50ae4e5..88e33246 100644 --- a/scripts/src/ssh.ts +++ b/scripts/src/ssh.ts @@ -2309,6 +2309,16 @@ async function runSshCaptureRemoteCommand(config: UniDeskConfig, invocation: Par }); } +export async function runSshCommandCapture(config: UniDeskConfig, target: string, args: string[], input?: string): Promise { + const invocation = parseSshInvocation(target, args); + const parsed = invocation.parsed; + if (parsed.remoteCommand === null) throw new Error(`ssh ${target} capture requires a non-interactive operation`); + const stdin = parsed.stdinPrefix !== undefined || parsed.stdinSuffix !== undefined + ? `${parsed.stdinPrefix ?? ""}${input ?? ""}${parsed.stdinSuffix ?? ""}` + : input; + return await runSshCaptureRemoteCommand(config, invocation, parsed.remoteCommand, stdin); +} + function writeChunkedStdin(stdin: NodeJS.WritableStream, input: string): void { const buffer = Buffer.from(input, "utf8"); const chunkSize = 32 * 1024;