From acc20b5c892e4ad9a68064a7fccdf5e2eee15f56 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 24 May 2026 02:27:15 +0000 Subject: [PATCH] feat(hwlab): add dev cd audit --- AGENTS.md | 2 +- docs/reference/cli.md | 2 +- docs/reference/hwlab.md | 2 + scripts/hwlab-cd-wrapper-contract-test.ts | 120 ++++- scripts/src/help.ts | 2 +- scripts/src/hwlab-cd-remote-runner.cjs | 610 +++++++++++++++++++++- scripts/src/hwlab-cd.ts | 10 +- 7 files changed, 731 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c08d1903..44d0200e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -65,7 +65,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - `bun scripts/cli.ts auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run`：查看 Auth Broker P0 Rust skeleton 与 CLI adapter contract，runner 无 `GH_TOKEN`/`GITHUB_TOKEN` 时返回结构化 `auth-missing`/`broker-needed`，不读取或打印 token 值，规则见 `docs/reference/auth-broker.md`。 - `bun scripts/cli.ts gh preflight|auth status|issue ...|pr list|files|diff --stat|read|view|preflight|closeout|create|edit|update|comment` / `bun scripts/code-queue-pr-preflight-example.ts`：通过 REST 执行安全 GitHub issue 读写、脱敏 auth/status 诊断、body-file Markdown 写入、当日滚动简报时间线 ClaudeQQ 通知、escape 扫描、只读 cleanup-plan 和 #20 board-audit、PR changed-file/stat summary、PR 创建/评论 dry-run、REST-only 低噪声 PR title/body 编辑、PR 收口元数据观察（含 merged/closed 区分与 merge commit）、低噪声 PR 收口 preflight 与 runner PR preflight；`gh issue/pr read|view` 支持 `owner/repo#number` shorthand，`--raw|--full` 是显式完整披露别名，`gh pr diff` 仅支持 `--stat` 紧凑 JSON，`gh pr merge` 当前仍结构化拒绝但普通 PR 可按任务边界用 repo-owned GitHub 路径收口，规则见 `docs/reference/cli.md` 和 `docs/reference/code-queue-supervision.md`。 - `bun scripts/cli.ts commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr`：查看 host Codex 指挥官直管微服务 skeleton 的 source/contract、无 daemon smoke 验证计划、.state/commander/ 状态模型、trace summary 聚合、ClaudeQQ 高风险请示草案和 GPT-5.5 PR prompt 边界辅助 lint；当前只返回 dry-run 计划和 backend-core `microservice proxy claudeqq` 授权后候选命令，不接 live bridge、不接管人工指挥官，不发送消息，`prompt-lint` 不作为业务 PR 门禁也不改变 `codex submit` 默认行为，规则见 `docs/reference/host-codex-commander.md`。 -- `bun scripts/cli.ts hwlab cd status --env dev` / `bun scripts/cli.ts hwlab cd apply --env dev --dry-run`：HWLAB DEV CD 指挥侧 wrapper，通过 D601 provider SSH 调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs` 并强制原生 k3s kubeconfig；真实 apply 只暴露 host-commander-only 命令形状，规则见 `docs/reference/hwlab.md`。 +- `bun scripts/cli.ts hwlab cd audit --env dev` / `status|preflight|apply --dry-run`：HWLAB DEV CD 指挥侧 wrapper，通过 D601 provider SSH 调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs` 并强制原生 k3s kubeconfig；`audit` 在 CD 恢复后只读分类 control-plane、SecretRef、registry、Lease、artifact/workload、16666/16667 和 DB/runtime durability 阻塞，真实 apply 只暴露 host-commander-only 命令形状，规则见 `docs/reference/hwlab.md`。 - `bun scripts/cli.ts ci install/status/run/publish-backend-core/publish-user-service/run-dev-e2e/logs`：在 D601 原生 k3s 上安装和运行 Tekton CI，支持每 commit 检查、Code Queue 只读性能门禁、`CI.json` catalog 驱动的 backend-core 与 user-service commit-pinned 镜像发布和手动触发的 `origin/master:deploy.json#environments.dev` 临时 namespace e2e；catalog/producer/consumer 分工见 `docs/reference/cicd-standardization.md`，`run-dev-e2e` 的 Git 控制 runner、短 launcher 和 no-CD 边界见 `docs/reference/dev-ci-runner.md`，Tekton 规则见 `docs/reference/ci.md`。 - `bun scripts/cli.ts codex deploy `：旧 Code Queue 兼容部署入口已禁用，原因是它会绕过受控部署边界直连 D601 部署 Code Queue；规则见 `docs/reference/codex-deploy.md`。 - `bun scripts/cli.ts codex prompt-lint [prompt|--prompt-file path|--prompt-stdin]` / `codex submit [prompt] [--prompt-file path|--prompt-stdin] [--queue ]` / `codex pr-preflight [--remote]`：`prompt-lint` 在派发/steer 前 dry-run 检查 runner prompt 的 DEV 测试授权分级（`read-only`/`live-read`/`live-mutating`）且不回显 prompt；`submit --dry-run` 同时给出 MiniMax/GPT/人工路由建议、该 lint 结果和 requested/effective execution mode；真实提交成功只返回写入确认、task id、服务级 runnerPermissions 和后续查看命令，不回显 prompt；`pr-preflight` 只读检查 D601 scheduler/runner 的 GitHub token、egress 和 PR 能力，PR 型派单前必须使用，规则见 `docs/reference/cli.md` 和 `docs/reference/code-queue-supervision.md`。 diff --git a/docs/reference/cli.md b/docs/reference/cli.md index c4813553..166c87d4 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -31,7 +31,7 @@ CLI 可以从 `master` 快速演进，但必须兼容 `deploy.json` 固定的 CI - `dev-env prewarm-images [--image image] [--provider-id D601] [--no-pull] [--proxy-url URL] [--pull-timeout-ms N] [--dry-run]` 创建异步 job，通过 UniDesk SSH 维护桥在 D601 上把开发底座依赖镜像从 Docker 缓存导入原生 k3s containerd。默认镜像是 `postgres:16-alpine` 和 `rancher/mirrored-library-busybox:1.36.1`，用于避免 `postgres-dev` 与 local-path helper pod 卡在外部 registry 拉取。该命令固定验证 `/etc/rancher/k3s/k3s.yaml` 指向的 native k3s 上下文，并输出 `dev_env_containerd_image_ready=...` 作为成功判据；它不 apply manifest、不修改生产 `unidesk` namespace。 - `artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`，由 systemd + Docker Compose 管理，位于 native k3s 故障域外；`deploy-service` 只拉取 CI 已发布的 commit-pinned 镜像、retag/recreate 或导入 native k3s，并做 live commit 验证，不构建 runtime source。`deploy-backend-core` 是 deprecated 兼容名，标准 backend-core prod CD 入口是 `deploy apply --env prod --service backend-core`。长期规则见 `docs/reference/artifact-registry.md`。 - `commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr` 是 host Codex 指挥官直管微服务 skeleton 入口。当前命令返回 `phase=source-contract`、service/API/state/bridge/prompt/trace/#20/#46/ClaudeQQ 审批边界、.state/commander/ 状态模型、dev 无 daemon smoke contract、dry-run 计划和 GPT-5.5 PR prompt 边界辅助 lint，不接 live bridge、不注入 prompt、不发送 ClaudeQQ。`approval request --dry-run` 会生成 200 字以内中文纯文本 ClaudeQQ 审批草案、`notification-path-unavailable` blocker 和授权后唯一可用的 `bun scripts/cli.ts microservice proxy claudeqq /api/push/text --method POST --body-json '' --raw` 命令；不得提示使用本机 ClaudeQQ skill、powershell 或本地 server。`prompt-lint` 支持 `--prompt-file` 与 `--stdin`，输出 `ok`、`missingClauses`、`riskLevel`、`suggestedPatchSnippet` 且不回显完整 prompt；它是 commander 辅助检查，不是业务 PR 门禁，也不改变 `codex submit` 默认行为。`plan`、`smoke` 与 `approval request` 必须带 `--dry-run`；缺少时返回 `error=dry-run-required`。长期规则见 `docs/reference/host-codex-commander.md`。 -- `hwlab cd status|preflight|apply --env dev [--dry-run]` 是 HWLAB DEV CD 指挥侧 wrapper。默认通过 UniDesk provider `host.ssh` 进入 D601，再调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs`，不内嵌发布 kubectl 逻辑：`status` 汇总固定 CD mirror、Git clean/main/origin-main、`deploy/deploy.json`/artifact catalog/report、D601 native k3s guard 和 CD Lease lock，并用 `scripts/dev-cd-apply.mjs --status --skip-live-verify` 取得 target/promotion 摘要；`preflight` 进一步检查必需 SecretRef 对象/键存在性并运行 HWLAB `scripts/dev-cd-apply.mjs --dry-run --skip-live-verify` 受控事务摘要。完整远端 stdout/stderr 写入 D601 `~/.state/unidesk-hwlab-cd//` 和本地 `.state/hwlab-cd//` task dump，stdout 只返回有界摘要。默认 HWLAB CD repo 是 `/home/ubuntu/hwlab_cd`，`/home/ubuntu/hwlab` runner 历史目录不得作为发布真相。wrapper 强制 `KUBECONFIG=/etc/rancher/k3s/k3s.yaml` 并只以这个显式目标作为 gate；显式目标出现 `docker-desktop`、`desktop-control-plane` 或 `127.0.0.1:11700` 信号会结构化拒绝，写操作计划还必须观察到 node `d601`。真实 apply 只暴露 `scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report` 命令形状并标注 host-commander-only，本 runner 不执行 live apply、rollout、Lease mutation 或 16666/16667 live verification。长期规则见 `docs/reference/hwlab.md`。 +- `hwlab cd status|audit|preflight|apply --env dev [--dry-run]` 是 HWLAB DEV CD 指挥侧 wrapper。默认通过 UniDesk provider `host.ssh` 进入 D601，再调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs`，不内嵌发布 kubectl 逻辑：`status` 汇总固定 CD mirror、Git clean/main/origin-main、`deploy/deploy.json`/artifact catalog/report、D601 native k3s guard 和 CD Lease lock，并用 `scripts/dev-cd-apply.mjs --status --skip-live-verify` 取得 target/promotion 摘要；`audit` 在 k3s/CD 恢复后做只读健康审计，返回有界 JSON 的 blocker 分类、D601 guard/node、SecretRef 存在性、registry 可达性、Lease phase/holder/staleness、deploy.json 与 artifact/workload image 收敛、current Deployment image/revision/rollout、16666/16667 public health commit/readiness 和 DB/runtime durability 摘要；`preflight` 进一步检查必需 SecretRef 对象/键存在性并运行 HWLAB `scripts/dev-cd-apply.mjs --dry-run --skip-live-verify` 受控事务摘要。完整远端 stdout/stderr 写入 D601 `~/.state/unidesk-hwlab-cd//` 和本地 `.state/hwlab-cd//` task dump，stdout 只返回有界摘要。默认 HWLAB CD repo 是 `/home/ubuntu/hwlab_cd`，`/home/ubuntu/hwlab` runner 历史目录不得作为发布真相。wrapper 强制 `KUBECONFIG=/etc/rancher/k3s/k3s.yaml` 并只以这个显式目标作为 gate；显式目标出现 `docker-desktop`、`desktop-control-plane` 或 `127.0.0.1:11700` 信号会结构化拒绝，audit/preflight/apply --dry-run 都必须观察到 node `d601`。真实 apply 只暴露 `scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report` 命令形状并标注 host-commander-only，本 runner 不执行 live apply、rollout、Lease mutation 或 DEV deploy apply。长期规则见 `docs/reference/hwlab.md`。 - `gh auth status [--repo owner/name]` 探测 GitHub 操作前置条件并输出脱敏 JSON：是否存在 `gh` binary、是否存在 `GH_TOKEN`/`GITHUB_TOKEN` 或可用 `gh auth token` fallback、REST API 是否可达、目标 repo 是否可见、issue 是否可读。degraded reason 必须归类为 `missing-binary`、`missing-token`、`auth-failed`、`github-transient`、`network-proxy-failed`、`permission-denied`、`repo-not-found`、`repo-forbidden`、`issue-not-found`、`pr-not-found`、`scope-insufficient`、`validation-failed`、`invalid-response` 或 `unsupported-command`，不得打印 token；失败对象必须包含 `runnerDisposition=infra-blocked|business-failed`，runner 应优先用该字段分流。`github-transient` 表示 GitHub DNS/API 连接在收到 HTTP 状态前失败，输出应带 `retryable=true` 或等价 commander action；这不是缺 token、认证失败、权限不足或 PR 语义失败。 - `codex prompt-lint [prompt|--prompt-file path|--prompt-stdin]` 是派发/steer 前的本地 dry-run prompt lint。它只读取 prompt 文本，返回 `dryRun=true`、`mutation=false`、`declaredClass`、`effectiveClass`、`requiredClass`、`dispatchDisposition`、缺失或矛盾项和有界 evidence，不访问 live service、不提交任务、不打印完整 prompt。分级固定为 `read-only`、`live-read`、`live-mutating`；未声明时按 `read-only` 处理。`codex submit --dry-run` 与 `codex steer --dry-run` 会嵌入同一 `promptLint` 结果，帮助指挥官在 dispatch/steer 前发现缺失或矛盾的 live mutation 授权。长期规则见 `docs/reference/code-queue-supervision.md` 的 DEV 测试授权分级。 - `gh issue list [owner/repo] [--state open|closed|all] [--limit N] [--repo owner/name] [--json number,title,state,url,updatedAt,createdAt,author,labels]` 通过 GitHub REST 列出 issue，默认 `state=open`、`limit=30`，输出稳定 JSON 且不依赖系统 `gh` binary。`owner/repo` 位置参数是 `--repo owner/repo` 的兼容别名；若位置 repo 与 `--repo` 冲突，或位置参数不是 `owner/repo`，必须结构化失败，禁止静默 fallback 到默认 repo。`--limit` 会映射到 GitHub `per_page` 并限制返回数量，避免一次拉爆上下文；未知 state 或未知 `--json` 字段必须结构化失败并带 `runnerDisposition=business-failed`。GitHub issues API 可能混入 PR，CLI 会从 `.data.issues` 中过滤 pull request。 diff --git a/docs/reference/hwlab.md b/docs/reference/hwlab.md index 44056c18..a3a8c769 100644 --- a/docs/reference/hwlab.md +++ b/docs/reference/hwlab.md @@ -46,6 +46,7 @@ UniDesk 指挥侧固定入口： ```sh bun scripts/cli.ts hwlab cd status --env dev +bun scripts/cli.ts hwlab cd audit --env dev bun scripts/cli.ts hwlab cd preflight --env dev bun scripts/cli.ts hwlab cd apply --env dev --dry-run ``` @@ -55,6 +56,7 @@ wrapper 的职责是把 host commander 常用的 HWLAB DEV rollout 查看/准备 - 默认 HWLAB CD repo 是 D601 固定干净 mirror `/home/ubuntu/hwlab_cd`，也可用 `--hwlab-repo` 显式指定同等干净 clone。wrapper 必须检查 `git status --short --branch`、origin remote、当前 branch `main`、本地 `origin/main`、`FETCH_HEAD` 和 worktree 权限；任何 dirty worktree、错误 remote、非 main、HEAD 未跟上本地 `origin/main` 或权限异常都返回结构化 blocker。`/home/ubuntu/hwlab` 是 runner 历史目录，不得作为发布真相。 - `deploy/deploy.json` 是唯一 desired-state。wrapper 只把 `deploy/artifact-catalog.dev.json`、`deploy/k8s/base/workloads.yaml` 和 `reports/dev-gate/dev-artifacts.json` 当作派生/证据读数；`status`/`preflight` 必须显示 target commit/ref、deploy.json、artifact catalog、workloads 和 live workload image 是否同源/收敛，不引入第二套 desired state。 - `status` 只读汇总 HWLAB repo path、Git clean/main/origin-main、desired-state 收敛、D601 native k3s guard 和 `Lease/hwlab-dev/hwlab-dev-cd-lock`；同时调用 HWLAB `scripts/dev-cd-apply.mjs --status --skip-live-verify` 取得 repo-owned target/promotion/deploy.json/artifact 摘要。16666/16667 live verification 不由本 runner 执行。 +- `audit` 是 DEV CD 恢复后的只读健康审计，不是验收 gate 或报告生成器。它在 `status` 受控路径上补充只读 `kubectl get`/HTTP health probes，输出有界 JSON summary，分类 `control-plane-unavailable`、`docker-desktop-context-risk`、`second-control-plane-risk`、`workspace-unavailable`、`dirty-worktree`、`secret-missing`、`registry-unavailable`、`lease-held`、`lease-stale-candidate`、`artifact-missing`、`artifact-mismatch`、`runtime-job-blocked`、`rollout-unhealthy`、`public-tunnel-unhealthy` 和 `db-runtime-durability-risk`。audit 只显示 Secret 对象/key 是否存在，不显示值；只读判断 Lease 是否 stale，不释放或 break；只读拉取 16666/16667 `/health/live` 的 commit/readiness 摘要，不把它当作 M3 DEV-LIVE 验收。 - `preflight` 在 `status` 的基础上检查 apply 前 SecretRef：`hwlab-cloud-api-dev-db/database-url`、`hwlab-cloud-api-dev-db-admin/admin-url`、`hwlab-code-agent-provider/openai-api-key`。只验证 Secret 对象和 key 元数据存在性，缺失时返回 blocker、影响范围和修复 runbook；禁止读取或打印 Secret value。 - `apply --dry-run` 调用 HWLAB `scripts/dev-cd-apply.mjs --dry-run --kubeconfig /etc/rancher/k3s/k3s.yaml --skip-live-verify`，只生成受控事务准备/阻塞摘要，不做真实 apply、rollout、Lease mutation 或 live verification。历史 `scripts/dev-deploy-apply.mjs` 可作为 HWLAB 内部支持脚本出现，但 UniDesk wrapper 不能把它当成平行 CD 入口。 - 完整下游 stdout/stderr 和 kubectl 读命令输出写入 D601 `~/.state/unidesk-hwlab-cd//`，UniDesk 本地仅保存 provider task stdout/stderr dump；CLI stdout 只显示有界摘要和 dump path。 diff --git a/scripts/hwlab-cd-wrapper-contract-test.ts b/scripts/hwlab-cd-wrapper-contract-test.ts index c67e6837..69c30be9 100644 --- a/scripts/hwlab-cd-wrapper-contract-test.ts +++ b/scripts/hwlab-cd-wrapper-contract-test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { buildHwlabCdRemoteCommandForTest } from "./src/hwlab-cd"; @@ -40,6 +40,7 @@ function makeFakeHwlabRepo(): string { mkdirSync(join(root, "reports/dev-gate"), { recursive: true }); writeFileSync(join(root, "scripts/dev-cd-apply.mjs"), [ "const kubeconfigIndex = process.argv.indexOf('--kubeconfig');", + "if (process.argv.includes('--apply') || process.argv.includes('--write-report') || process.argv.includes('--confirm-dev')) { throw new Error('mutation flag must not be used in wrapper tests'); }", "process.stdout.write(JSON.stringify({", " ok: true,", " status: 'pass',", @@ -85,6 +86,43 @@ function makeFakeHwlabRepo(): string { function makeFakeBin(mode: "native" | "desktop" | "stale-default" | "wrong-node" | "missing-secret" | "second-plane"): string { const bin = join(tmpdir(), `unidesk-hwlab-cd-bin-${process.pid}-${Date.now()}-${mode}-${Math.random().toString(16).slice(2)}`); mkdirSync(bin, { recursive: true }); + const deploymentJson = { + items: [ + { + metadata: { + name: "hwlab-cloud-api", + labels: { "app.kubernetes.io/name": "hwlab-cloud-api", "hwlab.pikastech.local/service-id": "hwlab-cloud-api" }, + annotations: { "deployment.kubernetes.io/revision": "7" }, + }, + spec: { + replicas: 1, + template: { spec: { containers: [{ name: "hwlab-cloud-api", image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", env: [{ name: "HWLAB_COMMIT_ID", value: "abc1234" }] }] } }, + }, + status: { availableReplicas: 1, updatedReplicas: 1, unavailableReplicas: 0, conditions: [{ type: "Available", status: "True" }, { type: "Progressing", status: "True", reason: "NewReplicaSetAvailable" }] }, + }, + { + metadata: { + name: "hwlab-cloud-web", + labels: { "app.kubernetes.io/name": "hwlab-cloud-web", "hwlab.pikastech.local/service-id": "hwlab-cloud-web" }, + annotations: { "deployment.kubernetes.io/revision": "8" }, + }, + spec: { + replicas: 1, + template: { spec: { containers: [{ name: "hwlab-cloud-web", image: "127.0.0.1:5000/hwlab/hwlab-cloud-web:abc1234", env: [{ name: "HWLAB_COMMIT_ID", value: "abc1234" }] }] } }, + }, + status: { availableReplicas: 1, updatedReplicas: 1, unavailableReplicas: 0, conditions: [{ type: "Available", status: "True" }, { type: "Progressing", status: "True" }] }, + }, + ], + }; + const podsJson = { + items: [ + { + metadata: { name: "hwlab-cloud-api-abc" }, + status: { containerStatuses: [{ name: "hwlab-cloud-api", image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", imageID: "docker-pullable://127.0.0.1:5000/hwlab/hwlab-cloud-api@sha256:" + "b".repeat(64), state: { running: {} } }] }, + }, + ], + }; + const jobsJson = { items: [{ metadata: { name: "hwlab-runtime-provision-old" }, status: { succeeded: 1, failed: 0, active: 0, completionTime: "2026-05-24T00:00:00Z" } }] }; const explicitContext = mode === "desktop" ? "docker-desktop" : "default"; const explicitServer = mode === "desktop" ? "https://127.0.0.1:11700" : "https://127.0.0.1:6443"; const explicitNodes = mode === "desktop" ? "desktop-control-plane" : mode === "wrong-node" ? "d602" : "d601"; @@ -112,6 +150,9 @@ function makeFakeBin(mode: "native" | "desktop" | "stale-default" | "wrong-node" " printf 'Error from server (NotFound): namespaces \"hwlab-dev\" not found\\n' >&2; exit 1", "fi", "if [[ \"$*\" =~ ^-n[[:space:]]+hwlab-dev[[:space:]]+get[[:space:]]+lease[[:space:]]+hwlab-dev-cd-lock[[:space:]]+-o[[:space:]]+json$ ]]; then printf 'Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found\\n' >&2; exit 1; fi", + "if [[ \"$*\" == '-n hwlab-dev get deployments -o json' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify(deploymentJson)) + "; exit 0; fi", + "if [[ \"$*\" == '-n hwlab-dev get pods -o json' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify(podsJson)) + "; exit 0; fi", + "if [[ \"$*\" == '-n hwlab-dev get jobs -o json' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify(jobsJson)) + "; exit 0; fi", "if [[ \"$*\" == '-n hwlab-dev get secret hwlab-code-agent-provider -o name' && " + JSON.stringify(mode) + " == 'missing-secret' ]]; then printf 'Error from server (NotFound): secrets \"hwlab-code-agent-provider\" not found\\n' >&2; exit 1; fi", "if [[ \"$*\" =~ ^-n\\ hwlab-dev\\ get\\ secret\\ ([^[:space:]]+)\\ -o\\ name$ ]]; then printf 'secret/%s\\n' \"${BASH_REMATCH[1]}\"; exit 0; fi", "if [[ \"$*\" == '-n hwlab-dev describe secret hwlab-cloud-api-dev-db' ]]; then printf 'Name: hwlab-cloud-api-dev-db\\nData\\n====\\ndatabase-url: 48 bytes\\n'; exit 0; fi", @@ -120,6 +161,34 @@ function makeFakeBin(mode: "native" | "desktop" | "stale-default" | "wrong-node" "printf 'unexpected kubectl args: %s\\n' \"$*\" >&2; exit 99", ].join("\n")); spawnSync("chmod", ["+x", join(bin, "kubectl")]); + writeFileSync(join(bin, "curl"), [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "url=\"${@: -1}\"", + "if [[ \"$*\" == *'http://127.0.0.1:5000/v2/'* ]]; then printf '{}\\n'; exit 0; fi", + "if [[ \"$url\" == 'http://74.48.78.17:16666/health/live' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify({ + serviceId: "hwlab-cloud-web", + environment: "dev", + status: "ready", + ready: true, + commit: { id: "abc1234" }, + image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-web:abc1234", tag: "abc1234" }, + blockerCodes: [], + })) + "; exit 0; fi", + "if [[ \"$url\" == 'http://74.48.78.17:16667/health/live' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify({ + serviceId: "hwlab-cloud-api", + environment: "dev", + status: "ready", + ready: true, + commit: { id: "abc1234" }, + image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", tag: "abc1234" }, + db: { ready: true, connected: true, liveDbEvidence: true, runtimeReadiness: { status: "ready", ready: true, blocker: null } }, + runtime: { status: "ready", ready: true, durable: true, durableRequested: true, liveRuntimeEvidence: true, blocker: null, adapter: "postgres" }, + blockerCodes: [], + })) + "; exit 0; fi", + "printf 'unexpected curl args: %s\\n' \"$*\" >&2; exit 22", + ].join("\n")); + spawnSync("chmod", ["+x", join(bin, "curl")]); return bin; } @@ -142,11 +211,14 @@ const secondPlaneBin = makeFakeBin("second-plane"); const help = runCli(["hwlab", "help"]); assert.equal(help.ok, true); assert.equal((help.data as JsonRecord).command, "hwlab cd"); +assert.equal(((help.data as JsonRecord).usage as string[]).includes("bun scripts/cli.ts hwlab cd audit --env dev"), true); const remoteCommand = buildHwlabCdRemoteCommandForTest(withLocalTransport(["cd", "apply", "--env", "dev", "--dry-run"])); assert.equal(remoteCommand.includes("scripts/dev-cd-apply.mjs"), true); assert.equal(remoteCommand.includes("/etc/rancher/k3s/k3s.yaml"), true); assert.equal(remoteCommand.includes("kubectl rollout"), false); +assert.equal(remoteCommand.includes("kubectl apply"), false); +assert.equal(remoteCommand.includes("break-stale-lock"), false); const realApply = runCli(["hwlab", "cd", "apply", "--env", "dev", "--hwlab-repo", fakeRepo], { PATH: `${nativeBin}:${process.env.PATH ?? ""}`, @@ -201,6 +273,46 @@ assert.equal(((dryRunData.target as JsonRecord).promotionCommit), "abc1234567890 assert.equal(((dryRunData.promotion as JsonRecord).source), "deploy/deploy.json"); assert.equal(JSON.stringify(dryRunData).includes("sk-secret"), false); +const reportsBefore = existsSync(join(fakeRepo, "reports")) ? JSON.stringify(readdirSync(join(fakeRepo, "reports"), { recursive: true })) : ""; +const audit = runCli(withLocalTransport([ + "hwlab", + "cd", + "audit", + "--env", + "dev", + "--hwlab-repo", + fakeRepo, +]), { + PATH: `${nativeBin}:${process.env.PATH ?? ""}`, +}); +assert.equal(audit.ok, true); +const auditData = dataOf(audit); +const auditSummary = auditData.audit as JsonRecord; +assert.equal(auditSummary.status, "pass"); +assert.equal((auditSummary.namespace), "hwlab-dev"); +assert.equal(((auditSummary.nodeGuard as JsonRecord).nodeNames as unknown[]).includes("d601"), true); +assert.equal(((auditSummary.secrets as JsonRecord).valuesRead), false); +assert.equal(((auditSummary.secrets as JsonRecord).valuesPrinted), false); +assert.equal(JSON.stringify(auditSummary).includes("48 bytes"), false); +assert.equal(((auditSummary.registry as JsonRecord).status), "pass"); +assert.equal(((auditSummary.lease as JsonRecord).staleClassification), "not-held"); +assert.equal((((auditSummary.desiredState as JsonRecord).imageConvergence as JsonRecord).status), "pass"); +assert.equal(((auditSummary.workload as JsonRecord).status), "healthy"); +assert.equal(((auditSummary.publicHealth as JsonRecord).status), "pass"); +assert.equal(((auditSummary.durability as JsonRecord).status), "pass"); +assert.deepEqual((auditSummary.blockerTypes as unknown[]), []); +assert.equal(((auditSummary.safety as JsonRecord).reportsWritten), false); +assert.equal(((auditSummary.safety as JsonRecord).cdLockMutated), false); +assert.equal((auditData.remote as JsonRecord).providerId, "D601"); +assert.equal(JSON.stringify(auditData).length < 80_000, true); +assert.equal(JSON.stringify(auditData).includes("kubectl apply"), false); +assert.equal(JSON.stringify(auditData).includes("kubectl rollout"), false); +assert.equal(JSON.stringify(auditData).includes("--apply"), false); +assert.equal(JSON.stringify(auditData).includes("--write-report"), false); +assert.equal(JSON.stringify(auditData).includes("sk-secret"), false); +const reportsAfter = existsSync(join(fakeRepo, "reports")) ? JSON.stringify(readdirSync(join(fakeRepo, "reports"), { recursive: true })) : ""; +assert.equal(reportsAfter, reportsBefore); + const status = runCli(withLocalTransport([ "hwlab", "cd", @@ -240,10 +352,9 @@ assert.equal((staleDefaultGuard.defaultKubectlDiagnostic as JsonRecord).status, const desktopRefusal = runCli(withLocalTransport([ "hwlab", "cd", - "apply", + "audit", "--env", "dev", - "--dry-run", "--hwlab-repo", fakeRepo, ]), { @@ -252,7 +363,8 @@ const desktopRefusal = runCli(withLocalTransport([ assert.equal(desktopRefusal.ok, false); const desktopData = dataOf(desktopRefusal); assert.equal(desktopData.status, "refused"); -assert.deepEqual((desktopData.nodeGuard as JsonRecord).refusalSignals, ["docker-desktop", "desktop-control-plane", "127.0.0.1:11700"]); +assert.deepEqual((((desktopData.audit as JsonRecord).nodeGuard as JsonRecord).refusalSignals), ["docker-desktop", "desktop-control-plane", "127.0.0.1:11700"]); +assert.equal(((desktopData.blockerTypes as unknown[]).includes("docker-desktop-context-risk")), true); const wrongNodeBlocked = runCli(withLocalTransport([ "hwlab", diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 795a861f..d6e85784 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -49,7 +49,7 @@ export function rootHelp(): unknown { { command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." }, { command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and merge blocked." }, { command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." }, - { command: "hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Bounded HWLAB DEV CD wrapper that calls HWLAB repo-owned scripts, forces D601 native k3s kubeconfig, refuses Docker Desktop control-plane signals, and writes full stdout/stderr dumps." }, + { command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Bounded HWLAB DEV CD wrapper that calls HWLAB repo-owned scripts, forces D601 native k3s kubeconfig, refuses Docker Desktop control-plane signals, and exposes read-only post-recovery blocker classification." }, { command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." }, { command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run supports --wait-ms N and retry-run reuses the failed run's schedule." }, { command: "schedule upsert-pgdata-backup [--time HH:MM] [--remote-base /SERVER_DATA/UNIDESK_PG_DATA]", description: "Create or update the daily PGDATA physical backup task that uploads monthly rotated archives to Baidu Netdisk." }, diff --git a/scripts/src/hwlab-cd-remote-runner.cjs b/scripts/src/hwlab-cd-remote-runner.cjs index 7ae7204a..4fb409f2 100644 --- a/scripts/src/hwlab-cd-remote-runner.cjs +++ b/scripts/src/hwlab-cd-remote-runner.cjs @@ -276,6 +276,168 @@ function readJsonSummary(repoPath, relativePath, fallback = {}) { } } +function readJsonFile(repoPath, relativePath) { + const absolutePath = path.join(repoPath, relativePath); + try { + const raw = fs.readFileSync(absolutePath, "utf8"); + return { ok: true, path: relativePath, exists: true, hash: sha256(raw), json: JSON.parse(raw), error: null }; + } catch (error) { + return { + ok: false, + path: relativePath, + exists: fs.existsSync(absolutePath), + hash: null, + json: null, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +function commitMatches(actual, expected) { + if (typeof actual !== "string" || typeof expected !== "string" || actual.length === 0 || expected.length === 0) return false; + return actual === expected || actual.startsWith(expected) || expected.startsWith(actual); +} + +function imageTag(image) { + const value = String(image || ""); + const slash = value.lastIndexOf("/"); + const colon = value.lastIndexOf(":"); + return colon > slash ? value.slice(colon + 1) : null; +} + +function servicesFromManifest(manifest) { + const services = Array.isArray(manifest?.services) + ? manifest.services + : manifest?.services && typeof manifest.services === "object" + ? Object.values(manifest.services) + : []; + return services + .filter((service) => service && typeof service === "object") + .map((service) => ({ + serviceId: stringValue(service.serviceId) || stringValue(service.id) || stringValue(service.name), + name: stringValue(service.name) || stringValue(service.serviceId) || stringValue(service.id), + image: stringValue(service.image), + imageTag: stringValue(service.imageTag) || imageTag(service.image), + commitId: stringValue(service.commitId) || stringValue(service.imageTag) || imageTag(service.image), + namespace: stringValue(service.namespace) || namespace, + healthPath: stringValue(service.healthPath) || "/health/live", + replicas: Number.isInteger(service.replicas) ? service.replicas : null, + env: asRecord(service.env) || {}, + digest: stringValue(service.digest), + publishState: stringValue(service.publishState) || stringValue(service.status), + artifactRequired: service.artifactRequired !== false, + })) + .filter((service) => service.serviceId); +} + +function summarizeDeployFile(file) { + const manifest = asRecord(file.json); + const endpoints = asRecord(manifest?.publicEndpoints) || {}; + return { + path: file.path, + exists: file.exists, + hash: file.hash, + commitId: stringValue(manifest?.commitId), + environment: stringValue(manifest?.environment), + namespace: stringValue(manifest?.namespace) || namespace, + endpoint: stringValue(manifest?.endpoint), + serviceCount: servicesFromManifest(manifest).length, + publicEndpoints: { + frontend: stringValue(asRecord(endpoints.frontend)?.url), + api: stringValue(asRecord(endpoints.api)?.url), + }, + unavailableReason: file.ok ? null : file.error, + }; +} + +function summarizeArtifactFile(file) { + const manifest = asRecord(file.json); + const services = servicesFromManifest(manifest); + const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 }; + for (const service of services) { + if (/^sha256:[a-f0-9]{64}$/u.test(service.digest || "")) digestCounts.sha256 += 1; + else if (service.digest === "not_published") digestCounts.notPublished += 1; + else digestCounts.invalid += 1; + } + return { + path: file.path, + exists: file.exists, + hash: file.hash, + commitId: stringValue(manifest?.commitId), + artifactState: stringValue(manifest?.artifactState), + ciPublished: manifest?.publish?.ciPublished ?? null, + registryVerified: manifest?.publish?.registryVerified ?? null, + serviceCount: services.length, + digestCounts, + unavailableReason: file.ok ? null : file.error, + }; +} + +function buildDesiredStateAudit(repoPath, target) { + const deployFile = readJsonFile(repoPath, "deploy/deploy.json"); + const catalogFile = readJsonFile(repoPath, "deploy/artifact-catalog.dev.json"); + const reportFile = readJsonFile(repoPath, "reports/dev-gate/dev-artifacts.json"); + const deployServices = servicesFromManifest(deployFile.json); + const catalogServices = servicesFromManifest(catalogFile.json); + const catalogById = new Map(catalogServices.map((service) => [service.serviceId, service])); + const mismatches = []; + const missing = []; + for (const service of deployServices) { + if (!service.image) continue; + const catalog = catalogById.get(service.serviceId); + if (!catalog && service.artifactRequired) { + missing.push({ serviceId: service.serviceId, reason: "missing-from-artifact-catalog" }); + continue; + } + if (catalog?.image && catalog.image !== service.image) { + mismatches.push({ serviceId: service.serviceId, deployImage: service.image, catalogImage: catalog.image }); + } + if (catalog && catalog.artifactRequired && catalog.publishState && catalog.publishState !== "published") { + missing.push({ serviceId: service.serviceId, reason: `artifact-${catalog.publishState}` }); + } + } + const deploySummary = summarizeDeployFile(deployFile); + const catalogSummary = summarizeArtifactFile(catalogFile); + const reportSummary = summarizeArtifactFile(reportFile); + const targetCommit = stringValue(target?.shortCommitId) || stringValue(target?.promotionCommit) || deploySummary.commitId; + const commitConverged = Boolean( + targetCommit && + commitMatches(deploySummary.commitId, targetCommit) && + (!catalogSummary.commitId || commitMatches(catalogSummary.commitId, targetCommit)) + ); + const status = !deployFile.ok || !catalogFile.ok || missing.length > 0 + ? "missing" + : mismatches.length > 0 || !commitConverged + ? "mismatch" + : "pass"; + return { + status, + targetCommit, + commitConverged, + deployJson: deploySummary, + artifactCatalog: catalogSummary, + artifactReport: reportSummary, + imageConvergence: { + status: missing.length === 0 && mismatches.length === 0 ? "pass" : missing.length > 0 ? "missing" : "mismatch", + serviceCount: deployServices.length, + catalogServiceCount: catalogServices.length, + missing: missing.slice(0, 12), + mismatches: mismatches.slice(0, 12), + }, + services: deployServices.map((service) => ({ + serviceId: service.serviceId, + image: service.image, + imageTag: service.imageTag, + catalogImage: catalogById.get(service.serviceId)?.image || null, + catalogDigest: catalogById.get(service.serviceId)?.digest || null, + replicas: service.replicas, + })).slice(0, 30), + serviceEnv: { + cloudApi: asRecord(deployServices.find((service) => service.serviceId === "hwlab-cloud-api")?.env) || {}, + }, + }; +} + function forbiddenKubeSignals(text) { const signals = []; if (/docker-desktop/iu.test(text)) signals.push("docker-desktop"); @@ -490,6 +652,409 @@ async function secretPreflight(kubeconfig, guard, dumpDir, timeoutMs) { }; } +async function registryAudit(kubeconfig, guard, dumpDir, timeoutMs) { + const result = await runCaptured(["curl", "-fsS", "--max-time", "5", "http://127.0.0.1:5000/v2/"], process.cwd(), dumpDir, "registry-v2", { timeoutMs: Math.min(timeoutMs, 8000) }); + const reachable = result.ok || /401|unauthorized/iu.test(`${result.stdoutText}\n${result.stderrText}`); + let k3sPullAccess = { status: "skipped", reason: "d601-native-k3s-guard-not-pass", pullFailureCount: null, pulledRegistryImageCount: null }; + if (guard.status === "pass") { + const env = { ...process.env, KUBECONFIG: kubeconfig }; + const pods = await runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "registry-k3s-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) }); + const parsed = asRecord(parseJson(pods.stdoutText)); + const items = Array.isArray(parsed?.items) ? parsed.items : []; + let pullFailureCount = 0; + let pulledRegistryImageCount = 0; + for (const item of items) { + const statuses = Array.isArray(item?.status?.containerStatuses) ? item.status.containerStatuses : []; + for (const status of statuses) { + const image = String(status?.image || ""); + const imageID = String(status?.imageID || ""); + const waiting = String(status?.state?.waiting?.reason || ""); + if (/ErrImagePull|ImagePullBackOff|InvalidImageName/u.test(waiting)) pullFailureCount += 1; + if (image.includes("127.0.0.1:5000/") && imageID.includes("sha256:")) pulledRegistryImageCount += 1; + } + } + k3sPullAccess = { + status: !pods.ok ? "unavailable" : pullFailureCount > 0 ? "blocked" : pulledRegistryImageCount > 0 ? "pass" : "degraded", + pullFailureCount, + pulledRegistryImageCount, + command: commandView(pods), + }; + } + return { + status: reachable && !["blocked", "unavailable"].includes(k3sPullAccess.status) ? "pass" : reachable ? "degraded" : "blocked", + endpoint: "http://127.0.0.1:5000/v2/", + processHttpAccess: { + status: reachable ? "pass" : "blocked", + reachable, + exitCode: result.exitCode, + command: commandView(result), + }, + k3sPullAccess, + }; +} + +function deploymentCondition(conditions, type) { + return (Array.isArray(conditions) ? conditions : []).find((condition) => condition?.type === type) || null; +} + +function envValue(container, name) { + const item = (Array.isArray(container?.env) ? container.env : []).find((entry) => entry?.name === name); + return item?.value ?? null; +} + +async function workloadAudit(kubeconfig, guard, desired, dumpDir, timeoutMs) { + if (guard.status !== "pass") { + return { status: "skipped", reason: "d601-native-k3s-guard-not-pass", namespace, deployments: [], currentImageConvergence: { status: "unavailable", unavailableReason: "d601-native-k3s-guard-not-pass" } }; + } + const env = { ...process.env, KUBECONFIG: kubeconfig }; + const [deployments, pods, jobs] = await Promise.all([ + runCaptured(["kubectl", "-n", namespace, "get", "deployments", "-o", "json"], process.cwd(), dumpDir, "workload-deployments", { env, timeoutMs: Math.min(timeoutMs, 10000) }), + runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "workload-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) }), + runCaptured(["kubectl", "-n", namespace, "get", "jobs", "-o", "json"], process.cwd(), dumpDir, "runtime-jobs", { env, timeoutMs: Math.min(timeoutMs, 10000) }), + ]); + const desiredById = new Map((desired.services || []).map((service) => [service.serviceId, service])); + const deploymentItems = Array.isArray(parseJson(deployments.stdoutText)?.items) ? parseJson(deployments.stdoutText).items : []; + const summaries = deploymentItems.map((deployment) => { + const metadata = asRecord(deployment.metadata) || {}; + const spec = asRecord(deployment.spec) || {}; + const status = asRecord(deployment.status) || {}; + const template = asRecord(spec.template) || {}; + const podSpec = asRecord(template.spec) || {}; + const containers = Array.isArray(podSpec.containers) ? podSpec.containers : []; + const container = containers[0] || {}; + const labels = asRecord(metadata.labels) || {}; + const serviceId = stringValue(labels["hwlab.pikastech.local/service-id"]) || stringValue(labels["app.kubernetes.io/name"]) || stringValue(metadata.name); + const desiredService = desiredById.get(serviceId); + const image = stringValue(container.image); + const desiredImage = desiredService?.image || null; + const available = Number(status.availableReplicas || 0); + const desiredReplicas = Number(spec.replicas || 0); + const updated = Number(status.updatedReplicas || 0); + const unavailable = Number(status.unavailableReplicas || 0); + const progressing = deploymentCondition(status.conditions, "Progressing"); + const availableCondition = deploymentCondition(status.conditions, "Available"); + return { + name: stringValue(metadata.name), + serviceId, + image, + imageTag: imageTag(image), + desiredImage, + imageMatchesDesired: desiredImage ? image === desiredImage : null, + revision: stringValue(asRecord(metadata.annotations)?.["deployment.kubernetes.io/revision"]), + replicas: desiredReplicas, + availableReplicas: available, + updatedReplicas: updated, + unavailableReplicas: unavailable, + rolloutStatus: desiredReplicas === 0 || (available >= desiredReplicas && updated >= desiredReplicas && unavailable === 0) ? "healthy" : "unhealthy", + conditions: { + available: availableCondition ? { status: availableCondition.status ?? null, reason: availableCondition.reason ?? null } : null, + progressing: progressing ? { status: progressing.status ?? null, reason: progressing.reason ?? null } : null, + }, + commitEnv: envValue(container, "HWLAB_COMMIT_ID"), + }; + }); + const podItems = Array.isArray(parseJson(pods.stdoutText)?.items) ? parseJson(pods.stdoutText).items : []; + const waiting = []; + for (const pod of podItems) { + const podName = stringValue(pod?.metadata?.name); + for (const status of Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : []) { + const reason = stringValue(status?.state?.waiting?.reason); + if (reason) waiting.push({ pod: podName, container: status.name ?? null, reason, image: status.image ?? null }); + } + } + const jobItems = Array.isArray(parseJson(jobs.stdoutText)?.items) ? parseJson(jobs.stdoutText).items : []; + const runtimeJobs = jobItems + .filter((job) => /hwlab-runtime|runtime-db|runtime/i.test(String(job?.metadata?.name || ""))) + .map((job) => ({ + name: stringValue(job?.metadata?.name), + succeeded: Number(job?.status?.succeeded || 0), + failed: Number(job?.status?.failed || 0), + active: Number(job?.status?.active || 0), + completionTime: stringValue(job?.status?.completionTime), + status: Number(job?.status?.failed || 0) > 0 ? "blocked" : Number(job?.status?.active || 0) > 0 ? "running" : Number(job?.status?.succeeded || 0) > 0 ? "pass" : "unknown", + })) + .slice(0, 12); + const imageMismatches = summaries.filter((deployment) => deployment.imageMatchesDesired === false); + const unhealthy = summaries.filter((deployment) => deployment.rolloutStatus === "unhealthy"); + return { + status: !deployments.ok ? "unavailable" : unhealthy.length > 0 ? "unhealthy" : imageMismatches.length > 0 ? "mismatch" : "healthy", + namespace, + deploymentCount: summaries.length, + deployments: summaries.slice(0, 30), + currentImageConvergence: { + status: imageMismatches.length === 0 ? "pass" : "mismatch", + mismatches: imageMismatches.map((deployment) => ({ serviceId: deployment.serviceId, image: deployment.image, desiredImage: deployment.desiredImage })).slice(0, 12), + }, + podWaiting: waiting.slice(0, 12), + runtimeJobs, + commands: [deployments, pods, jobs].map(commandView), + }; +} + +function compactHealth(body, expectedServiceId, expectedCommit) { + const json = asRecord(body); + const commit = stringValue(asRecord(json?.commit)?.id) || stringValue(json?.commitId) || stringValue(json?.revision) || stringValue(asRecord(json?.image)?.tag); + const runtime = asRecord(json?.runtime); + const db = asRecord(json?.db); + const blockerCodes = Array.isArray(json?.blockerCodes) ? json.blockerCodes.map(String).slice(0, 12) : []; + return { + serviceId: stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id), + environment: stringValue(json?.environment), + status: stringValue(json?.status), + ready: typeof json?.ready === "boolean" ? json.ready : null, + observedCommit: commit, + commitMatches: expectedCommit ? commitMatches(commit, expectedCommit) : null, + serviceMatches: expectedServiceId ? (stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id)) === expectedServiceId : null, + image: typeof json?.image === "string" ? json.image : stringValue(asRecord(json?.image)?.reference), + blockerCodes, + db: db === null ? null : { + ready: db.ready ?? null, + connected: db.connected ?? null, + liveDbEvidence: db.liveDbEvidence ?? null, + runtimeReadiness: asRecord(db.runtimeReadiness) ? { + status: stringValue(db.runtimeReadiness.status), + ready: db.runtimeReadiness.ready ?? null, + blocker: stringValue(db.runtimeReadiness.blocker), + } : null, + }, + runtime: runtime === null ? null : { + status: stringValue(runtime.status), + ready: runtime.ready ?? null, + durable: runtime.durable ?? null, + durableRequested: runtime.durableRequested ?? null, + liveRuntimeEvidence: runtime.liveRuntimeEvidence ?? null, + blocker: stringValue(runtime.blocker), + adapter: stringValue(runtime.adapter), + }, + }; +} + +async function httpHealthAudit(desired, dumpDir, timeoutMs) { + const endpoints = [ + { id: "cloud-web-16666", url: desired.deployJson.publicEndpoints.frontend || "http://74.48.78.17:16666", expectedServiceId: "hwlab-cloud-web" }, + { id: "cloud-api-16667", url: desired.deployJson.publicEndpoints.api || desired.deployJson.endpoint || "http://74.48.78.17:16667", expectedServiceId: "hwlab-cloud-api" }, + ]; + const expectedCommit = desired.targetCommit || desired.deployJson.commitId || null; + const observations = []; + for (const endpoint of endpoints) { + const result = await runCaptured(["curl", "-fsS", "--max-time", "8", `${endpoint.url.replace(/\/+$/u, "")}/health/live`], process.cwd(), dumpDir, `public-health-${endpoint.id}`, { timeoutMs: Math.min(timeoutMs, 10000) }); + const parsed = parseJson(result.stdoutText); + const health = compactHealth(parsed, endpoint.expectedServiceId, expectedCommit); + const reachable = result.ok && parsed !== null; + observations.push({ + id: endpoint.id, + url: endpoint.url, + status: reachable && health.serviceMatches !== false && health.commitMatches !== false ? "pass" : "blocked", + reachable, + expectedServiceId: endpoint.expectedServiceId, + expectedCommit, + health, + command: commandView(result), + }); + } + return { + status: observations.every((item) => item.status === "pass") ? "pass" : "blocked", + expectedCommit, + summary: { + checked: observations.length, + reachable: observations.filter((item) => item.reachable).length, + commitMatches: observations.filter((item) => item.health.commitMatches === true).length, + readinessPass: observations.filter((item) => item.health.ready !== false && !["blocked", "failed"].includes(String(item.health.status || ""))).length, + ports: [16666, 16667], + }, + endpoints: observations, + }; +} + +function runtimeDurabilityAudit(desired, publicHealth, secretRefs, workload) { + const api = publicHealth?.endpoints?.find((endpoint) => endpoint.id === "cloud-api-16667")?.health || null; + const cloudApiEnv = asRecord(desired.serviceEnv?.cloudApi) || {}; + const configuredDurable = cloudApiEnv.HWLAB_CLOUD_RUNTIME_DURABLE === "true"; + const configuredAdapter = stringValue(cloudApiEnv.HWLAB_CLOUD_RUNTIME_ADAPTER); + const dbSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db" && ref.secretKey === "database-url"); + const adminSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db-admin" && ref.secretKey === "admin-url"); + const runtime = asRecord(api?.runtime); + const dbRuntimeReadiness = asRecord(api?.db?.runtimeReadiness); + const liveReady = runtime?.durable === true && runtime?.ready !== false && runtime?.liveRuntimeEvidence === true && !["blocked", "degraded", "failed"].includes(String(runtime?.status || "")); + const status = liveReady + ? "pass" + : api + ? "risk" + : configuredDurable && configuredAdapter === "postgres" && dbSecret?.keyPresent === true && adminSecret?.keyPresent === true + ? "unavailable" + : "risk"; + return { + status, + configured: { + adapter: configuredAdapter, + durable: configuredDurable, + dbSecretPresent: dbSecret?.keyPresent === true, + adminSecretPresent: adminSecret?.keyPresent === true, + }, + live: api ? { + db: api.db, + runtime: api.runtime, + dbRuntimeReadiness, + } : null, + workloadEvidence: { + cloudApiDeployment: (workload?.deployments || []).find((deployment) => deployment.serviceId === "hwlab-cloud-api") || null, + }, + unavailableReason: api ? null : "public-health-cloud-api-unavailable", + risk: status === "pass" ? null : "db-runtime-durability-risk", + }; +} + +function blocker(scope, summary, details = {}) { + return { scope, summary, ...details }; +} + +function classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled }) { + const blockers = []; + if (guard?.status !== "pass") blockers.push(blocker("control-plane-unavailable", guard?.summary || "D601 native k3s guard did not pass.")); + if (guard?.refusalSignals?.length > 0) blockers.push(blocker("docker-desktop-context-risk", "Explicit D601 kubeconfig resolved to forbidden Docker Desktop control-plane signal.", { refusalSignals: guard.refusalSignals })); + if (guard?.secondControlPlaneRisk) blockers.push(blocker("second-control-plane-risk", "Bare kubectl can observe hwlab-dev through a different control plane; audit is read-only and will not release or mutate locks.")); + if (repo.status !== "selected") blockers.push(blocker("workspace-unavailable", "No eligible HWLAB CD repo was selected.")); + for (const item of git?.blockers || []) { + const scope = item.scope === "hwlab-git-clean" ? "dirty-worktree" : item.scope?.startsWith("hwlab-git") || item.scope?.includes("permission") ? "workspace-unavailable" : item.scope; + blockers.push(blocker(scope, item.summary)); + } + for (const item of secretRefs?.blockers || []) blockers.push(blocker("secret-missing", item.summary, { secretRef: item.scope })); + if (registry?.status === "blocked" || registry?.status === "degraded") blockers.push(blocker("registry-unavailable", "Registry reachability or k3s pull evidence is not healthy.", { status: registry.status })); + if (lock?.held === true) blockers.push(blocker("lease-held", `HWLAB DEV CD Lease is held by ${lock.ownerTaskId || lock.holderIdentity || "unknown"}.`, { phase: lock.phase, retryAfterSeconds: lock.retryAfterSeconds })); + if (lock?.stale === true) blockers.push(blocker("lease-stale-candidate", "HWLAB DEV CD Lease appears stale; audit is read-only and did not release or break it.", { phase: lock.phase })); + if (desired?.status === "missing") blockers.push(blocker("artifact-missing", "deploy/deploy.json or artifact catalog/report is missing or incomplete.", { imageConvergence: desired.imageConvergence })); + if (desired?.status === "mismatch" || workload?.currentImageConvergence?.status === "mismatch") blockers.push(blocker("artifact-mismatch", "deploy/deploy.json, artifact catalog, or current workload images are not converged.", { desired: desired?.status, currentImageConvergence: workload?.currentImageConvergence })); + if ((workload?.runtimeJobs || []).some((job) => job.status === "blocked" || job.status === "running")) blockers.push(blocker("runtime-job-blocked", "Runtime DB maintenance Job is blocked or still running.", { runtimeJobs: workload.runtimeJobs })); + if (workload?.status === "unhealthy") blockers.push(blocker("rollout-unhealthy", "One or more hwlab-dev Deployments are not fully available.", { deployments: (workload.deployments || []).filter((item) => item.rolloutStatus === "unhealthy").slice(0, 8) })); + if (publicHealth?.status === "blocked") blockers.push(blocker("public-tunnel-unhealthy", "16666/16667 public health did not pass read-only commit/readiness checks.", { summary: publicHealth.summary })); + if (durability?.status === "risk") blockers.push(blocker("db-runtime-durability-risk", "DB/runtime durability is not proven by live cloud-api health.", { live: durability.live, configured: durability.configured })); + if (controlled?.commandOk === false || controlled?.status === "blocked") blockers.push(blocker("runtime-job-blocked", "HWLAB repo-owned dev-cd-apply status reported blocked or failed.", { status: controlled?.status })); + return blockers; +} + +function auditSummary({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled, dumpDir }) { + const blockers = classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled }); + const blockerTypes = [...new Set(blockers.map((item) => item.scope))]; + return { + ok: blockers.length === 0, + status: blockers.length === 0 ? "pass" : guard?.refusal ? "refused" : "blocked", + env: "dev", + namespace, + nodeGuard: { + status: guard?.status ?? "unavailable", + nodeName: guard?.requiredNodeName ?? requiredNodeName, + nodeNames: guard?.nodeNames ?? [], + requiredNodePresent: guard?.requiredNodePresent ?? false, + currentContext: guard?.currentContext ?? null, + apiServer: guard?.apiServer ?? null, + kubeconfig: guard?.kubeconfig ?? nativeKubeconfig, + refusalSignals: guard?.refusalSignals ?? [], + secondControlPlaneRisk: guard?.secondControlPlaneRisk ?? false, + defaultKubectlDiagnostic: guard?.defaultKubectlDiagnostic ?? null, + }, + secrets: { + status: secretRefs?.status ?? "unavailable", + valuesRead: false, + valuesPrinted: false, + secretRefs: (secretRefs?.secretRefs || []).map((ref) => ({ + secretName: ref.secretName, + secretKey: ref.secretKey, + exists: ref.exists === true, + keyPresent: ref.keyPresent === true, + status: ref.status, + })), + unavailableReason: secretRefs?.reason ?? null, + }, + registry: { + status: registry?.status ?? "unavailable", + endpoint: registry?.endpoint ?? "http://127.0.0.1:5000/v2/", + processHttpAccess: registry?.processHttpAccess ? { status: registry.processHttpAccess.status, reachable: registry.processHttpAccess.reachable } : null, + k3sPullAccess: registry?.k3sPullAccess ? { + status: registry.k3sPullAccess.status, + pullFailureCount: registry.k3sPullAccess.pullFailureCount, + pulledRegistryImageCount: registry.k3sPullAccess.pulledRegistryImageCount, + } : null, + }, + lease: { + status: lock?.status ?? "unavailable", + phase: lock?.phase ?? null, + holder: lock?.ownerTaskId || lock?.holderIdentity || null, + ageSeconds: lock?.updatedAt && Number.isFinite(Date.parse(lock.updatedAt)) ? Math.max(0, Math.floor((Date.now() - Date.parse(lock.updatedAt)) / 1000)) : null, + retryAfterSeconds: lock?.retryAfterSeconds ?? null, + staleClassification: lock?.stale === true ? "lease-stale-candidate" : lock?.held === true ? "lease-held" : "not-held", + lockName: lock?.lockName ?? lockName, + }, + desiredState: { + status: desired?.status ?? "unavailable", + targetCommit: desired?.targetCommit ?? null, + deployJson: desired?.deployJson ?? null, + artifactCatalog: desired?.artifactCatalog ?? null, + artifactReport: desired?.artifactReport ?? null, + imageConvergence: desired?.imageConvergence ?? null, + }, + workload: { + status: workload?.status ?? "unavailable", + currentImageConvergence: workload?.currentImageConvergence ?? null, + deploymentCount: workload?.deploymentCount ?? 0, + deployments: (workload?.deployments || []).map((deployment) => ({ + name: deployment.name, + serviceId: deployment.serviceId, + image: deployment.image, + imageTag: deployment.imageTag, + desiredImage: deployment.desiredImage, + revision: deployment.revision, + replicas: deployment.replicas, + availableReplicas: deployment.availableReplicas, + rolloutStatus: deployment.rolloutStatus, + commitEnv: deployment.commitEnv, + })), + podWaiting: workload?.podWaiting ?? [], + runtimeJobs: workload?.runtimeJobs ?? [], + }, + publicHealth: { + status: publicHealth?.status ?? "unavailable", + expectedCommit: publicHealth?.expectedCommit ?? null, + summary: publicHealth?.summary ?? { checked: 0 }, + endpoints: (publicHealth?.endpoints || []).map((endpoint) => ({ + id: endpoint.id, + url: endpoint.url, + status: endpoint.status, + reachable: endpoint.reachable, + expectedServiceId: endpoint.expectedServiceId, + expectedCommit: endpoint.expectedCommit, + health: endpoint.health, + })), + }, + durability: { + status: durability?.status ?? "unavailable", + configured: durability?.configured ?? null, + live: durability?.live ?? null, + unavailableReason: durability?.unavailableReason ?? null, + }, + controlledDevCd: { + status: controlled?.status ?? "unavailable", + commandOk: controlled?.commandOk ?? null, + controlledEntrypoint: "scripts/dev-cd-apply.mjs", + target: controlled?.target ?? null, + blockers: controlled?.blockers ?? [], + }, + blockers, + blockerTypes, + dumpPath: dumpDir, + safety: { + mutation: false, + kubectlApplyExecuted: false, + rolloutExecuted: false, + liveVerifyExecuted: false, + cdLockMutated: false, + secretValuesRead: false, + secretValuesPrinted: false, + reportsWritten: false, + repoReportsWritten: false, + }, + }; +} + function summarizeControlled(parsed, command) { const record = asRecord(parsed); const target = asRecord(record?.target); @@ -552,7 +1117,7 @@ function collectBlockers({ repo, git, guard, lock, secretRefs, controlled, actio if (guard) { if (guard.refusal) blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary, refusal: true }); else if (guard.status !== "pass") blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary }); - if ((action === "preflight" || action === "apply") && guard.secondControlPlaneRisk) { + if ((action === "preflight" || action === "audit" || action === "apply") && guard.secondControlPlaneRisk) { blockers.push({ scope: "second-hwlab-dev-control-plane", summary: "Bare kubectl can observe hwlab-dev through a different control plane; refusing write-path planning." }); } } @@ -568,7 +1133,8 @@ function collectBlockers({ repo, git, guard, lock, secretRefs, controlled, actio } function nextSafeCommand(action, blockers) { - if (blockers.length > 0) return "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd status --env dev"; + if (blockers.length > 0) return action === "audit" ? "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd audit --env dev" : "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd status --env dev"; + if (action === "audit") return "audit is read-only; host commander may compare blockerTypes before deciding whether the unique CD runner should continue"; if (action === "status") return "bun scripts/cli.ts hwlab cd apply --env dev --dry-run"; return "host commander or the unique CD runner may decide whether to run node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report on D601; this wrapper did not apply or rollout"; } @@ -603,6 +1169,8 @@ async function main() { secretValuesPrinted: false, cdLockMutated: false, prodTouched: false, + reportsWritten: false, + repoReportsWritten: false, }, }; @@ -625,17 +1193,15 @@ async function main() { artifactReport: readJsonSummary(repoPath, "reports/dev-gate/dev-artifacts.json"), }; const lock = await lockStatus(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000)); - const needsSecretPreflight = action === "preflight" || action === "apply"; + const needsSecretPreflight = action === "preflight" || action === "audit" || action === "apply"; const secretRefs = needsSecretPreflight ? await secretPreflight(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000)) : { status: "skipped", reason: "status-does-not-run-secret-preflight", namespace, mutation: false, blockers: [] }; const preCommandBlockers = collectBlockers({ repo, git, guard, lock, secretRefs, action }); - const canRunControlled = preCommandBlockers.length === 0 || action === "status"; + const canRunControlled = preCommandBlockers.length === 0 || action === "status" || action === "audit"; const controlled = canRunControlled && git.status === "pass" && guard.status === "pass" ? await runDevCdApply(repoPath, options.kubeconfig || nativeKubeconfig, action === "status" ? "status" : "apply", dumpDir, Math.max(1000, timeoutMs - 5000)) : { status: "skipped", commandOk: true, reason: "preflight-blockers-before-controlled-dev-cd-apply" }; - const blockers = collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action }); - const ok = blockers.length === 0; const target = controlled.target || { ref: "deploy/deploy.json", promotionCommit: desiredState.deployJson.commitId || git.headCommit, @@ -643,6 +1209,38 @@ async function main() { promotionSource: "deploy/deploy.json", namespace, }; + if (action === "audit") { + const auditDesiredState = buildDesiredStateAudit(repoPath, target); + const [registry, workload, publicHealth] = await Promise.all([ + registryAudit(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000)), + workloadAudit(options.kubeconfig || nativeKubeconfig, guard, auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)), + httpHealthAudit(auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)), + ]); + const durability = runtimeDurabilityAudit(auditDesiredState, publicHealth, secretRefs, workload); + const summary = auditSummary({ repo, git, guard, secretRefs, registry, lock, desired: auditDesiredState, workload, publicHealth, durability, controlled, dumpDir }); + console.log(JSON.stringify({ + ...base, + ok: summary.ok, + status: summary.status, + audit: summary, + repo: { + status: repo.status, + path: repo.path, + source: repo.source, + rejected: repo.rejected, + }, + workspace: { path: repoPath, source: repo.source }, + remoteDumpPath: dumpDir, + reportDumpPath: controlled.command?.dump?.stdout || dumpDir, + blockers: summary.blockers, + blockerTypes: summary.blockerTypes, + nextSafeCommand: nextSafeCommand(action, summary.blockers), + }, null, 2)); + process.exitCode = summary.ok ? 0 : 1; + return; + } + const blockers = collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action }); + const ok = blockers.length === 0; console.log(JSON.stringify({ ...base, ok, diff --git a/scripts/src/hwlab-cd.ts b/scripts/src/hwlab-cd.ts index 585f5eb9..1decb369 100644 --- a/scripts/src/hwlab-cd.ts +++ b/scripts/src/hwlab-cd.ts @@ -6,7 +6,7 @@ import { join } from "node:path"; import { readConfig, repoRoot, rootPath, type UniDeskConfig } from "./config"; import { d601NativeKubeconfig } from "./d601-k3s-guard"; -type HwlabCdAction = "status" | "preflight" | "apply"; +type HwlabCdAction = "status" | "preflight" | "audit" | "apply"; type HwlabCdEnvironment = "dev"; type HwlabCdTransport = "frontend" | "local"; @@ -89,8 +89,8 @@ function envTransport(): HwlabCdTransport { function parseOptions(args: string[]): HwlabCdOptions { const [scope, actionArg] = args; - if (scope !== "cd") throw new Error("hwlab usage: bun scripts/cli.ts hwlab cd status|preflight|apply --env dev"); - if (actionArg !== "status" && actionArg !== "preflight" && actionArg !== "apply") throw new Error("hwlab cd usage: status|preflight|apply"); + if (scope !== "cd") throw new Error("hwlab usage: bun scripts/cli.ts hwlab cd status|preflight|audit|apply --env dev"); + if (actionArg !== "status" && actionArg !== "preflight" && actionArg !== "audit" && actionArg !== "apply") throw new Error("hwlab cd usage: status|preflight|audit|apply"); const options: HwlabCdOptions = { action: actionArg, @@ -498,6 +498,7 @@ export function hwlabHelp(): Record { output: "json", usage: [ "bun scripts/cli.ts hwlab cd status --env dev", + "bun scripts/cli.ts hwlab cd audit --env dev", "bun scripts/cli.ts hwlab cd preflight --env dev", "bun scripts/cli.ts hwlab cd apply --env dev --dry-run", ], @@ -508,7 +509,8 @@ export function hwlabHelp(): Record { `default HWLAB CD repo is ${defaultHwlabCdRepoPath}; ${rejectedRunnerHistoryRepoPath} is rejected as runner history`, "deploy/deploy.json remains the authoritative desired-state source", "preflight/apply --dry-run check required SecretRef object/key metadata without reading or printing Secret values", - "status/preflight/apply --dry-run call only HWLAB scripts/dev-cd-apply.mjs with --skip-live-verify; no apply, rollout, lock mutation, live verification, DB write, or secret read is executed", + "audit/status/preflight/apply --dry-run call only HWLAB scripts/dev-cd-apply.mjs read-only/status paths with --skip-live-verify; no apply, rollout, lock mutation, DEV acceptance live verification, DB write, or secret read is executed", + "audit adds bounded read-only kubectl get/curl health probes for blocker classification; full stdout/stderr stays in temp dump paths, not reports/", "real apply is structured refused and must remain with the host commander or unique CD runner", ], };