fix: stabilize hwlab ci workspace cleanup
This commit is contained in:
Codex
@@ -43,7 +43,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 DEV/PROD 滚动、P
|
||||
- `artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`，由 systemd + Docker Compose 管理，位于 native k3s 故障域外；`deploy-service` 只拉取 CI 已发布的 commit-pinned 镜像、retag/recreate 或导入 native k3s，并做 live commit 验证，不构建 runtime source。`deploy-backend-core` 是 deprecated 兼容名，标准 backend-core prod CD 入口是 `deploy apply --env prod --service backend-core`。长期规则见 `docs/reference/artifact-registry.md`。
|
||||
- `commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr` 是 host Codex 指挥官直管微服务 skeleton 入口。当前命令返回 `phase=source-contract`、service/API/state/bridge/prompt/trace/#20/#46/ClaudeQQ 审批边界、.state/commander/ 状态模型、dev 无 daemon smoke contract、dry-run 计划和 GPT-5.5 PR prompt 边界辅助 lint，不接 live bridge、不注入 prompt、不发送 ClaudeQQ。`approval request --dry-run` 会生成 200 字以内中文纯文本 ClaudeQQ 审批草案、`notification-path-unavailable` blocker 和授权后唯一可用的 `bun scripts/cli.ts microservice proxy claudeqq /api/push/text --method POST --body-json '<payload>' --raw` 命令；不得提示使用本机 ClaudeQQ skill、powershell 或本地 server。`prompt-lint` 支持 `--prompt-file` 与 `--stdin`，输出 `ok`、`missingClauses`、`riskLevel`、`suggestedPatchSnippet` 且不回显完整 prompt；它是 commander 辅助检查，不是业务 PR 门禁，也不改变 `codex submit` 默认行为。`plan`、`smoke` 与 `approval request` 必须带 `--dry-run`；缺少时返回 `error=dry-run-required`。长期规则见 `docs/reference/host-codex-commander.md`。
|
||||
- `hwlab g14 monitor-prs [--once] [--dry-run] [--interval-seconds N] [--max-cycles N] [--timeout-seconds N]` 是当前 HWLAB G14 PR -> CI/CD -> DEV rollout 的一行式入口。普通调用创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand` 和 stdout/stderr 路径；后台 worker 每轮通过 UniDesk `gh pr list/preflight/merge` 监控 `pikasTech/HWLAB` base=`G14` 的 open PR，ready 时合并，然后通过 UniDesk `ssh G14:k3s` 观察 `hwlab-g14-ci-poll-<short>`、Argo `hwlab-g14-dev` 和 DEV `/health/live`，直到 DEV `Synced/Healthy` 且 Deployment/StatefulSet ready；历史 `Completed` smoke/debug pod 不作为 rollout blocker。每次成功 DEV rollout 后，worker 会定位或创建 #7“指挥简报索引”中的北京日期每日简报 issue，并追加 CI/CD 耗时、CI/CD 关键指标、语义化上线 changelog、自动 diff 摘要、PipelineRun、GitOps revision 和 DEV 验证摘要；关键指标来自 G14 Tekton TaskRun results，固定包含 `lazy build reused: x/y`、reused services、rebuild services 和每个 service 的独立耗时/状态/backend，用于观察 lazy build 机制效果。语义化 changelog 优先从 PR body 的 `## 修改`/`## 变更`/`## Changelog` 等段落提取，diff 摘要只作为文件和统计证据保留，不替代 changelog。也可用 `hwlab g14 record-rollout --pr <number> --source-commit <sha>` 手动补记，手动补记同样会按 PipelineRun 采集 TaskRun 指标。状态指针按用途分离：长期监控只写 `.state/hwlab-g14/latest-monitor-job.json`，`--once` 写 `latest-once-job.json`，`--dry-run` 写 `latest-dry-run-job.json`，`--once --dry-run` 写 `latest-once-dry-run-job.json`，避免一次性收口覆盖持续监控入口。`--once --dry-run` 只做单轮监控和 merge plan，不写 GitHub、不等待 rollout。该命令禁止使用原生 `gh` 或手拼 GitHub 请求；如果 UniDesk `gh` 子命令字段或行为不够，必须先改进 `scripts/src/gh.ts` 后再使用。
|
||||
- `hwlab g14 control-plane status|apply|rerun-current|cleanup-runs --lane v02 [--dry-run|--confirm]` 是 HWLAB `v0.2` 加法 lane 的受控 Tekton/Argo 控制面维护入口，只面向 G14 `/root/hwlab-v02`、branch `v0.2`、namespace `hwlab-ci` 和 Argo application `hwlab-g14-v02`；`status` 只读汇总 `hwlab-v02-ci-image-publish`、`hwlab-v02-branch-poller`、`hwlab-v02-control-plane-reconciler`、`hwlab-g14-v02` 和当前 commit PipelineRun。`apply` 先在 G14 `/root/hwlab-v02` 快进并执行 `scripts/g14-gitops-render.mjs --lane v02 --check`，再只通过 UniDesk `G14:k3s` route server-side apply `tekton-v02/rbac.yaml`、`pipeline.yaml`、`poller.yaml`、`control-plane-reconciler.yaml`、`argocd/project.yaml` 和 `argocd/application-v02.yaml`；默认 dry-run，真实写入必须加 `--confirm`，不会应用 runtime-v02 workload、Secret 或数据迁移。`rerun-current` 用于控制面更新后重跑当前 `origin/v0.2` commit：它只在同名 PipelineRun 非成功且非运行时删除旧失败 run 并从 `hwlab-v02-branch-poller` 创建一次性 Job；默认 dry-run，真实动作必须加 `--confirm`。`cleanup-runs --lane v02|g14|all [--min-age-minutes N] [--limit N]` 是完成态 PipelineRun 工作区 retention 入口，默认 dry-run 只列候选 PipelineRun 与 ownerReference 关联的 hwlab-ci 临时 PVC；真实清理必须加 `--confirm`，只删除已完成 PipelineRun，让 Tekton/local-path 按对象关系回收临时 PVC，不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。`hwlab g14 tools-image status|build --name ci-node-tools --tag <tag> [--dockerfile deploy/ci/hwlab-ci-node-tools.Dockerfile] [--dry-run|--confirm]` 是 G14 固定 HWLAB CI tools image 的受控 host build/push 入口：镜像内容必须来自 HWLAB repo 内 Dockerfile，`build` 默认 dry-run，真实写入必须加 `--confirm`，构建和 push 只发生在 G14 host 与本地 registry，不在 master server 构建，也不把 `apk add`/runtime install 塞进 Tekton PipelineRun。
|
||||
- `hwlab g14 control-plane status|apply|rerun-current --lane v02 [--dry-run|--confirm]`、`cleanup-runs --lane v02|g14|all [--dry-run|--confirm]` 和 `cleanup-released-pvs --lane all [--dry-run|--confirm]` 是 HWLAB `v0.2` 加法 lane 的受控 Tekton/Argo 控制面维护入口，只面向 G14 `/root/hwlab-v02`、branch `v0.2`、namespace `hwlab-ci` 和 Argo application `hwlab-g14-v02`；`status` 只读汇总 `hwlab-v02-ci-image-publish`、`hwlab-v02-branch-poller`、`hwlab-v02-control-plane-reconciler`、`hwlab-g14-v02` 和当前 commit PipelineRun。`apply` 先在 G14 `/root/hwlab-v02` 快进并执行 `scripts/g14-gitops-render.mjs --lane v02 --check`，再只通过 UniDesk `G14:k3s` route server-side apply `tekton-v02/rbac.yaml`、`pipeline.yaml`、`poller.yaml`、`control-plane-reconciler.yaml`、`argocd/project.yaml` 和 `argocd/application-v02.yaml`；默认 dry-run，真实写入必须加 `--confirm`，不会应用 runtime-v02 workload、Secret 或数据迁移。`rerun-current` 用于控制面更新后重跑当前 `origin/v0.2` commit：它只在同名 PipelineRun 非成功且非运行时删除旧失败 run 并从 `hwlab-v02-branch-poller` 创建一次性 Job；默认 dry-run，真实动作必须加 `--confirm`。`cleanup-runs --lane v02|g14|all [--min-age-minutes N] [--limit N]` 是完成态 PipelineRun 工作区 retention 入口，默认 dry-run 通过 `kubectl get -o jsonpath` 列候选 PipelineRun 与 ownerReference 关联的 hwlab-ci 临时 PVC；真实清理必须加 `--confirm`，只删除已完成 PipelineRun，让 Tekton/local-path 按对象关系回收临时 PVC，不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。`cleanup-released-pvs --lane all [--limit N]` 是 local-path 未自动回收后的补充 retention 入口，只列并删除 `Released`、`local-path`、`Delete`、`claimNamespace=hwlab-ci` 且 claim 名称形如 Tekton 临时 `pvc-*` 的 PV；真实清理必须加 `--confirm`，不触碰 registry storage、业务 namespace PVC、Secret、runtime workload 或 GitOps desired state。`hwlab g14 tools-image status|build --name ci-node-tools --tag <tag> [--dockerfile deploy/ci/hwlab-ci-node-tools.Dockerfile] [--dry-run|--confirm]` 是 G14 固定 HWLAB CI tools image 的受控 host build/push 入口：镜像内容必须来自 HWLAB repo 内 Dockerfile，`build` 默认 dry-run，真实写入必须加 `--confirm`，构建和 push 只发生在 G14 host 与本地 registry，不在 master server 构建，也不把 `apk add`/runtime install 塞进 Tekton PipelineRun。
|
||||
- `ssh gh:/owner/repo ...` 把 GitHub issue/PR 映射成只读/受控写入的虚拟文本目录，适合日报、PR 正文和 issue 正文的小补丁维护：`ssh gh:/pikasTech/HWLAB ls` 展示 `pr/` 与 `issue/`，`ssh gh:/pikasTech/HWLAB/pr ls [--limit N] [--full]` 和 `ssh gh:/pikasTech/HWLAB/issue ls [--limit N] [--full]` 展示条目状态、楼层数、正文长度和标题，`ssh gh:/pikasTech/HWLAB/pr/507 ls` 展示单个 PR 的一楼正文文件，`ssh gh:/pikasTech/HWLAB/505/1 cat|rg|patch-apply` 兼容旧式 issue/PR number route。`patch-apply` 使用 UniDesk 默认 apply-patch v2 的虚拟文件 executor，把正文一楼映射为 `body.md`，写回仍走 `bun scripts/cli.ts gh issue/pr update` 的 guard/concurrency 规则；`rm` 对正文一楼结构化拒绝，避免误删 issue/PR 正文。大正文读取必须展开 UniDesk gh dump 文件，否则 `cat/rg/patch-apply` 会误读为空，这是 `gh:` 虚拟文件接口的 P0 可见性契约。
|
||||
- `hwlab cd status|audit|preflight|apply --env dev [--dry-run]` 是旧 D601 HWLAB DEV CD 指挥侧 wrapper，仅用于显式 legacy 诊断和迁移对照。默认通过 UniDesk provider `host.ssh` 进入 D601，再调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs`，不内嵌发布 kubectl 逻辑：`status` 汇总固定 CD mirror、Git clean/main/origin-main、`deploy/deploy.json`/artifact catalog/report、D601 native k3s guard 和 CD Lease lock，并用 `scripts/dev-cd-apply.mjs --status --skip-live-verify` 取得 target/promotion 摘要；`audit` 在 k3s/CD 恢复后做只读健康审计，返回有界 JSON 的 blocker 分类、D601 guard/node、SecretRef 存在性、registry 可达性、Lease phase/holder/staleness、deploy.json 与 artifact/workload image 收敛、current Deployment image/revision/rollout、16666/16667 public health commit/readiness 和 DB/runtime durability 摘要；`preflight` 进一步检查必需 SecretRef 对象/键存在性并运行 HWLAB `scripts/dev-cd-apply.mjs --dry-run --skip-live-verify` 受控事务摘要。完整远端 stdout/stderr 写入 D601 `~/.state/unidesk-hwlab-cd/<run-id>/` 和本地 `.state/hwlab-cd/<run-id>/` task dump，stdout 只返回有界摘要。默认 HWLAB CD repo 是 `/home/ubuntu/hwlab_cd`，`/home/ubuntu/hwlab` runner 历史目录不得作为发布真相。wrapper 强制 `KUBECONFIG=/etc/rancher/k3s/k3s.yaml` 并只以这个显式目标作为 gate；显式目标出现 `docker-desktop`、`desktop-control-plane` 或 `127.0.0.1:11700` 信号会结构化拒绝，audit/preflight/apply --dry-run 都必须观察到 node `d601`。真实 apply 只暴露 `scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report` 命令形状并标注 host-commander-only，本 runner 不执行 live apply、rollout、Lease mutation 或 DEV deploy apply。长期规则见 `docs/reference/hwlab.md`。
|
||||
- `gh auth status [--repo owner/name]` 探测 GitHub 操作前置条件并输出脱敏 JSON：是否存在 `gh` binary、是否存在 `GH_TOKEN`/`GITHUB_TOKEN` 或可用 `gh auth token` fallback、REST API 是否可达、目标 repo 是否可见、issue 是否可读。degraded reason 必须归类为 `missing-binary`、`missing-token`、`auth-failed`、`github-transient`、`network-proxy-failed`、`permission-denied`、`repo-not-found`、`repo-forbidden`、`issue-not-found`、`pr-not-found`、`scope-insufficient`、`validation-failed`、`invalid-response` 或 `unsupported-command`，不得打印 token；失败对象必须包含 `runnerDisposition=infra-blocked|business-failed`，runner 应优先用该字段分流。`github-transient` 表示 GitHub DNS/API 连接在收到 HTTP 状态前失败，输出应带 `retryable=true` 或等价 commander action；这不是缺 token、认证失败、权限不足或 PR 语义失败。
|
||||
|
||||
+1
-1
@@ -57,7 +57,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
|
||||
{ command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." },
|
||||
{ command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." },
|
||||
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|rerun-current|cleanup-runs --lane v02 | hwlab g14 tools-image status|build", description: "Start the G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane and PipelineRun retention actions, or build/status fixed HWLAB CI tools images through UniDesk G14 routes." },
|
||||
{ command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|rerun-current|cleanup-runs|cleanup-released-pvs | hwlab g14 tools-image status|build", description: "Start the G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane and CI workspace retention actions, or build/status fixed HWLAB CI tools images through UniDesk G14 routes." },
|
||||
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." },
|
||||
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
|
||||
{ command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },
|
||||
|
||||
+151
-66
@@ -49,7 +49,7 @@ interface G14RecordRolloutOptions {
|
||||
}
|
||||
|
||||
interface G14ControlPlaneOptions {
|
||||
action: "status" | "apply" | "rerun-current" | "cleanup-runs";
|
||||
action: "status" | "apply" | "rerun-current" | "cleanup-runs" | "cleanup-released-pvs";
|
||||
lane: "v02" | "g14" | "all";
|
||||
dryRun: boolean;
|
||||
confirm: boolean;
|
||||
@@ -163,12 +163,14 @@ function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions {
|
||||
|
||||
function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions {
|
||||
const [actionRaw] = args;
|
||||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "rerun-current" && actionRaw !== "cleanup-runs") {
|
||||
throw new Error("control-plane usage: status|apply|rerun-current|cleanup-runs --lane v02 [--dry-run|--confirm]");
|
||||
if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "rerun-current" && actionRaw !== "cleanup-runs" && actionRaw !== "cleanup-released-pvs") {
|
||||
throw new Error("control-plane usage: status|apply|rerun-current --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]");
|
||||
}
|
||||
const lane = optionValue(args, "--lane") ?? "v02";
|
||||
const lane = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02");
|
||||
if (actionRaw === "cleanup-runs") {
|
||||
if (lane !== "v02" && lane !== "g14" && lane !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all");
|
||||
} else if (actionRaw === "cleanup-released-pvs") {
|
||||
if (lane !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane");
|
||||
} else if (lane !== "v02") {
|
||||
throw new Error("control-plane status/apply/rerun-current currently requires --lane v02");
|
||||
}
|
||||
@@ -382,65 +384,69 @@ function commandErrorSummary(result: CommandJsonResult): string {
|
||||
}
|
||||
|
||||
function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||||
const script = [
|
||||
"set -eu",
|
||||
'tmp="$(mktemp)"',
|
||||
'trap \'rm -f "$tmp"\' EXIT',
|
||||
`kubectl get pipelineruns -n ${shellQuote(CI_NAMESPACE)} -o json > "$tmp"`,
|
||||
`export HWLAB_CLEANUP_PREFIXES=${shellQuote(JSON.stringify(pipelinePrefixesForLane(options.lane)))}`,
|
||||
`export HWLAB_CLEANUP_MIN_AGE_MINUTES=${shellQuote(String(options.minAgeMinutes))}`,
|
||||
`export HWLAB_CLEANUP_LIMIT=${shellQuote(String(options.limit))}`,
|
||||
'export HWLAB_CLEANUP_PIPELINERUNS_JSON="$tmp"',
|
||||
"node <<'NODE'",
|
||||
"const fs = require('node:fs');",
|
||||
"const doc = JSON.parse(fs.readFileSync(process.env.HWLAB_CLEANUP_PIPELINERUNS_JSON, 'utf8'));",
|
||||
"const prefixes = JSON.parse(process.env.HWLAB_CLEANUP_PREFIXES || '[]');",
|
||||
"const minAgeMinutes = Number(process.env.HWLAB_CLEANUP_MIN_AGE_MINUTES || '60');",
|
||||
"const limit = Number(process.env.HWLAB_CLEANUP_LIMIT || '20');",
|
||||
"const now = Date.now();",
|
||||
"const out = (doc.items || [])",
|
||||
" .map((item) => {",
|
||||
" const metadata = item.metadata || {};",
|
||||
" const condition = ((item.status || {}).conditions || [])[0] || {};",
|
||||
" const name = String(metadata.name || '');",
|
||||
" const createdAt = String(metadata.creationTimestamp || '');",
|
||||
" const createdMs = Date.parse(createdAt);",
|
||||
" const ageMinutes = Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;",
|
||||
" return { name, createdAt, ageMinutes, status: condition.status || null, reason: condition.reason || null };",
|
||||
" })",
|
||||
" .filter((item) => item.name && prefixes.some((prefix) => item.name.startsWith(prefix)))",
|
||||
" .filter((item) => item.status === 'True' || item.status === 'False')",
|
||||
" .filter((item) => typeof item.ageMinutes === 'number' && item.ageMinutes >= minAgeMinutes)",
|
||||
" .sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))",
|
||||
" .slice(0, limit);",
|
||||
"console.log(JSON.stringify(out));",
|
||||
"NODE",
|
||||
].join("\n");
|
||||
const result = g14K3s(["script", "--", script], 60_000);
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pipelinerun",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.conditions[0].status}{"\\t"}{.status.conditions[0].reason}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRuns: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
const parsed = JSON.parse(statusText(result) || "[]") as unknown;
|
||||
return Array.isArray(parsed) ? parsed.map((item) => record(item)) : [];
|
||||
const prefixes = pipelinePrefixesForLane(options.lane);
|
||||
const now = Date.now();
|
||||
return statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", createdAt = "", status = "", reason = ""] = line.split("\t");
|
||||
const createdMs = Date.parse(createdAt);
|
||||
const ageMinutes = Number.isFinite(createdMs) ? Math.floor((now - createdMs) / 60000) : null;
|
||||
return { name, createdAt, ageMinutes, status: status || null, reason: reason || null };
|
||||
})
|
||||
.filter((item) => item.name.length > 0 && prefixes.some((prefix) => item.name.startsWith(prefix)))
|
||||
.filter((item) => item.status === "True" || item.status === "False")
|
||||
.filter((item) => typeof item.ageMinutes === "number" && item.ageMinutes >= options.minAgeMinutes)
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||||
.slice(0, options.limit);
|
||||
}
|
||||
|
||||
function listOwnedWorkspacePvcs(pipelineRunNames: string[]): CommandJsonResult {
|
||||
function listOwnedWorkspacePvcs(pipelineRunNames: string[]): Record<string, unknown>[] {
|
||||
if (pipelineRunNames.length === 0) {
|
||||
return {
|
||||
ok: true,
|
||||
command: [],
|
||||
exitCode: 0,
|
||||
stdout: "[]",
|
||||
stderr: "",
|
||||
parsed: [],
|
||||
};
|
||||
return [];
|
||||
}
|
||||
const script = [
|
||||
"set -eu",
|
||||
`export WANTED=${shellQuote(JSON.stringify(pipelineRunNames))}`,
|
||||
"kubectl get pvc -n hwlab-ci -o json | node -e 'const fs=require(\"fs\"); const doc=JSON.parse(fs.readFileSync(0,\"utf8\")); const wanted=new Set(JSON.parse(process.env.WANTED)); const out=(doc.items||[]).filter((pvc)=> (pvc.metadata?.ownerReferences||[]).some((ref)=>ref.kind===\"PipelineRun\"&&wanted.has(ref.name))).map((pvc)=>({name:pvc.metadata.name,volume:pvc.spec?.volumeName||null,phase:pvc.status?.phase||null,owner:(pvc.metadata.ownerReferences||[]).find((ref)=>ref.kind===\"PipelineRun\")?.name||null})); console.log(JSON.stringify(out));' ",
|
||||
].join("\n");
|
||||
return g14K3s(["script", "--", script], 60_000);
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pvc",
|
||||
"-n",
|
||||
CI_NAMESPACE,
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.spec.volumeName}{"\\t"}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{.metadata.ownerReferences[0].name}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list hwlab-ci PipelineRun PVCs: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
const wanted = new Set(pipelineRunNames);
|
||||
return statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", volume = "", phase = "", ownerKind = "", owner = ""] = line.split("\t");
|
||||
return {
|
||||
name,
|
||||
volume: volume || null,
|
||||
phase: phase || null,
|
||||
ownerKind: ownerKind || null,
|
||||
owner: owner || null,
|
||||
};
|
||||
})
|
||||
.filter((item) => item.ownerKind === "PipelineRun" && typeof item.owner === "string" && wanted.has(item.owner));
|
||||
}
|
||||
|
||||
function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResult {
|
||||
@@ -457,18 +463,94 @@ function deletePipelineRuns(names: string[], timeoutMs: number): CommandJsonResu
|
||||
return g14K3s(["kubectl", "delete", "pipelinerun", "-n", CI_NAMESPACE, ...names, "--ignore-not-found=true"], timeoutMs);
|
||||
}
|
||||
|
||||
function listReleasedCiWorkspacePvs(options: G14ControlPlaneOptions): Record<string, unknown>[] {
|
||||
const result = g14K3s([
|
||||
"kubectl",
|
||||
"get",
|
||||
"pv",
|
||||
"-o",
|
||||
'jsonpath={range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.phase}{"\\t"}{.spec.storageClassName}{"\\t"}{.spec.persistentVolumeReclaimPolicy}{"\\t"}{.spec.claimRef.namespace}{"\\t"}{.spec.claimRef.name}{"\\t"}{.spec.capacity.storage}{"\\n"}{end}',
|
||||
], 60_000);
|
||||
if (!isCommandSuccess(result)) {
|
||||
throw new Error(`failed to list released hwlab-ci PVs: ${commandErrorSummary(result)}`);
|
||||
}
|
||||
return statusText(result)
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const [name = "", createdAt = "", phase = "", storageClass = "", reclaimPolicy = "", claimNamespace = "", claimName = "", capacity = ""] = line.split("\t");
|
||||
return {
|
||||
name,
|
||||
createdAt,
|
||||
phase,
|
||||
storageClass,
|
||||
reclaimPolicy,
|
||||
claimNamespace,
|
||||
claimName,
|
||||
capacity,
|
||||
hostPath: claimNamespace === CI_NAMESPACE && claimName.length > 0 ? `/var/lib/rancher/k3s/storage/${name}_${claimNamespace}_${claimName}` : null,
|
||||
};
|
||||
})
|
||||
.filter((item) => item.phase === "Released")
|
||||
.filter((item) => item.storageClass === "local-path" && item.reclaimPolicy === "Delete")
|
||||
.filter((item) => item.claimNamespace === CI_NAMESPACE && typeof item.claimName === "string" && /^pvc-[a-z0-9]+$/u.test(item.claimName))
|
||||
.sort((a, b) => String(a.createdAt).localeCompare(String(b.createdAt)))
|
||||
.slice(0, options.limit);
|
||||
}
|
||||
|
||||
function deletePersistentVolumes(names: string[], timeoutMs: number): CommandJsonResult {
|
||||
if (names.length === 0) {
|
||||
return {
|
||||
ok: true,
|
||||
command: [],
|
||||
exitCode: 0,
|
||||
stdout: "no candidates",
|
||||
stderr: "",
|
||||
parsed: null,
|
||||
};
|
||||
}
|
||||
return g14K3s(["kubectl", "delete", "pv", ...names, "--ignore-not-found=true"], timeoutMs);
|
||||
}
|
||||
|
||||
function runControlPlaneReleasedPvCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const candidates = listReleasedCiWorkspacePvs(options);
|
||||
const candidateNames = candidates.map((item) => String(item.name));
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||||
mode: "dry-run",
|
||||
lane: options.lane,
|
||||
limit: options.limit,
|
||||
candidates,
|
||||
candidateCount: candidates.length,
|
||||
mutation: false,
|
||||
next: { confirm: `bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit ${options.limit} --confirm` },
|
||||
};
|
||||
}
|
||||
const deletion = deletePersistentVolumes(candidateNames, options.timeoutSeconds * 1000);
|
||||
return {
|
||||
ok: isCommandSuccess(deletion),
|
||||
command: "hwlab g14 control-plane cleanup-released-pvs",
|
||||
mode: "confirmed-cleanup",
|
||||
lane: options.lane,
|
||||
limit: options.limit,
|
||||
deletedPersistentVolumes: candidateNames,
|
||||
deletedPersistentVolumeCount: candidateNames.length,
|
||||
candidatesBefore: candidates,
|
||||
deletion,
|
||||
followUp: {
|
||||
diskPressure: "bun scripts/cli.ts ssh G14:k3s kubectl get node ubuntu-rog-zephyrus-g14-ga401iv-ga401iv -o jsonpath='{range .status.conditions[*]}{.type}{\"=\"}{.status}{\" \"}{.reason}{\"\\n\"}{end}'",
|
||||
storage: "bun scripts/cli.ts ssh G14 script -- 'df -h /; sudo du -xh -d 1 /var/lib/rancher/k3s/storage 2>/dev/null | sort -h | tail -20'",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
const candidates = listCleanupPipelineRuns(options);
|
||||
const candidateNames = candidates.map((item) => String(item.name));
|
||||
const pvcResult = listOwnedWorkspacePvcs(candidateNames);
|
||||
let ownedPvcs: unknown[] = [];
|
||||
if (isCommandSuccess(pvcResult)) {
|
||||
try {
|
||||
ownedPvcs = JSON.parse(statusText(pvcResult) || "[]") as unknown[];
|
||||
} catch {
|
||||
ownedPvcs = [];
|
||||
}
|
||||
}
|
||||
const ownedPvcs = listOwnedWorkspacePvcs(candidateNames);
|
||||
if (options.dryRun) {
|
||||
return {
|
||||
ok: true,
|
||||
@@ -642,6 +724,7 @@ function v02ControlPlaneStatus(sourceCommit: string | null = getV02Head()): Reco
|
||||
|
||||
function runV02ControlPlane(options: G14ControlPlaneOptions): Record<string, unknown> {
|
||||
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
|
||||
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
|
||||
const sourceCommit = getV02Head();
|
||||
if (sourceCommit === null) {
|
||||
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", workspace: V02_WORKSPACE };
|
||||
@@ -1578,11 +1661,13 @@ export function hwlabG14Help(): Record<string, unknown> {
|
||||
"bun scripts/cli.ts hwlab g14 control-plane rerun-current --lane v02 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --dry-run",
|
||||
"bun scripts/cli.ts hwlab g14 control-plane cleanup-released-pvs --lane all --limit 20 --confirm",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image status --name ci-node-tools --tag node22-alpine-bun-v1",
|
||||
"bun scripts/cli.ts hwlab g14 tools-image build --name ci-node-tools --tag node22-alpine-bun-v1 --confirm",
|
||||
"bun scripts/cli.ts job status <jobId> --tail-bytes 30000",
|
||||
],
|
||||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup helper, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; control-plane status/apply/rerun-current/cleanup-runs uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources and completed PipelineRun workspace retention only.",
|
||||
description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup helper, and controlled CI tools image build/status entry. The public monitor starts a fire-and-forget job; control-plane status/apply/rerun-current/cleanup-runs/cleanup-released-pvs uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources and completed CI workspace retention only.",
|
||||
defaults: {
|
||||
repo: HWLAB_REPO,
|
||||
base: G14_SOURCE_BRANCH,
|
||||
|
||||
Reference in New Issue
Block a user