feat(hwlab): add dev cd audit

This commit is contained in:
Codex
2026-05-24 02:27:15 +00:00
parent 6d59684d67
commit acc20b5c89
7 changed files with 731 additions and 17 deletions
+1 -1
View File
@@ -65,7 +65,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台;本文
- `bun scripts/cli.ts auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run`:查看 Auth Broker P0 Rust skeleton 与 CLI adapter contractrunner 无 `GH_TOKEN`/`GITHUB_TOKEN` 时返回结构化 `auth-missing`/`broker-needed`,不读取或打印 token 值,规则见 `docs/reference/auth-broker.md`
- `bun scripts/cli.ts gh preflight|auth status|issue ...|pr list|files|diff --stat|read|view|preflight|closeout|create|edit|update|comment` / `bun scripts/code-queue-pr-preflight-example.ts`:通过 REST 执行安全 GitHub issue 读写、脱敏 auth/status 诊断、body-file Markdown 写入、当日滚动简报时间线 ClaudeQQ 通知、escape 扫描、只读 cleanup-plan 和 #20 board-audit、PR changed-file/stat summary、PR 创建/评论 dry-run、REST-only 低噪声 PR title/body 编辑、PR 收口元数据观察(含 merged/closed 区分与 merge commit)、低噪声 PR 收口 preflight 与 runner PR preflight`gh issue/pr read|view` 支持 `owner/repo#number` shorthand`--raw|--full` 是显式完整披露别名,`gh pr diff` 仅支持 `--stat` 紧凑 JSON`gh pr merge` 当前仍结构化拒绝但普通 PR 可按任务边界用 repo-owned GitHub 路径收口,规则见 `docs/reference/cli.md``docs/reference/code-queue-supervision.md`
- `bun scripts/cli.ts commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr`:查看 host Codex 指挥官直管微服务 skeleton 的 source/contract、无 daemon smoke 验证计划、.state/commander/ 状态模型、trace summary 聚合、ClaudeQQ 高风险请示草案和 GPT-5.5 PR prompt 边界辅助 lint;当前只返回 dry-run 计划和 backend-core `microservice proxy claudeqq` 授权后候选命令,不接 live bridge、不接管人工指挥官,不发送消息,`prompt-lint` 不作为业务 PR 门禁也不改变 `codex submit` 默认行为,规则见 `docs/reference/host-codex-commander.md`
- `bun scripts/cli.ts hwlab cd status --env dev` / `bun scripts/cli.ts hwlab cd apply --env dev --dry-run`HWLAB DEV CD 指挥侧 wrapper,通过 D601 provider SSH 调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs` 并强制原生 k3s kubeconfig;真实 apply 只暴露 host-commander-only 命令形状,规则见 `docs/reference/hwlab.md`
- `bun scripts/cli.ts hwlab cd audit --env dev` / `status|preflight|apply --dry-run`HWLAB DEV CD 指挥侧 wrapper,通过 D601 provider SSH 调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs` 并强制原生 k3s kubeconfig`audit` 在 CD 恢复后只读分类 control-plane、SecretRef、registry、Lease、artifact/workload、16666/16667 和 DB/runtime durability 阻塞,真实 apply 只暴露 host-commander-only 命令形状,规则见 `docs/reference/hwlab.md`
- `bun scripts/cli.ts ci install/status/run/publish-backend-core/publish-user-service/run-dev-e2e/logs`:在 D601 原生 k3s 上安装和运行 Tekton CI,支持每 commit 检查、Code Queue 只读性能门禁、`CI.json` catalog 驱动的 backend-core 与 user-service commit-pinned 镜像发布和手动触发的 `origin/master:deploy.json#environments.dev` 临时 namespace e2ecatalog/producer/consumer 分工见 `docs/reference/cicd-standardization.md``run-dev-e2e` 的 Git 控制 runner、短 launcher 和 no-CD 边界见 `docs/reference/dev-ci-runner.md`Tekton 规则见 `docs/reference/ci.md`
- `bun scripts/cli.ts codex deploy <commitId>`:旧 Code Queue 兼容部署入口已禁用,原因是它会绕过受控部署边界直连 D601 部署 Code Queue;规则见 `docs/reference/codex-deploy.md`
- `bun scripts/cli.ts codex prompt-lint [prompt|--prompt-file path|--prompt-stdin]` / `codex submit [prompt] [--prompt-file path|--prompt-stdin] [--queue <id>]` / `codex pr-preflight [--remote]``prompt-lint` 在派发/steer 前 dry-run 检查 runner prompt 的 DEV 测试授权分级(`read-only`/`live-read`/`live-mutating`)且不回显 prompt`submit --dry-run` 同时给出 MiniMax/GPT/人工路由建议、该 lint 结果和 requested/effective execution mode;真实提交成功只返回写入确认、task id、服务级 runnerPermissions 和后续查看命令,不回显 prompt;`pr-preflight` 只读检查 D601 scheduler/runner 的 GitHub token、egress 和 PR 能力,PR 型派单前必须使用,规则见 `docs/reference/cli.md``docs/reference/code-queue-supervision.md`
+1 -1
View File
@@ -31,7 +31,7 @@ CLI 可以从 `master` 快速演进,但必须兼容 `deploy.json` 固定的 CI
- `dev-env prewarm-images [--image image] [--provider-id D601] [--no-pull] [--proxy-url URL] [--pull-timeout-ms N] [--dry-run]` 创建异步 job,通过 UniDesk SSH 维护桥在 D601 上把开发底座依赖镜像从 Docker 缓存导入原生 k3s containerd。默认镜像是 `postgres:16-alpine``rancher/mirrored-library-busybox:1.36.1`,用于避免 `postgres-dev` 与 local-path helper pod 卡在外部 registry 拉取。该命令固定验证 `/etc/rancher/k3s/k3s.yaml` 指向的 native k3s 上下文,并输出 `dev_env_containerd_image_ready=...` 作为成功判据;它不 apply manifest、不修改生产 `unidesk` namespace。
- `artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`,由 systemd + Docker Compose 管理,位于 native k3s 故障域外;`deploy-service` 只拉取 CI 已发布的 commit-pinned 镜像、retag/recreate 或导入 native k3s,并做 live commit 验证,不构建 runtime source。`deploy-backend-core` 是 deprecated 兼容名,标准 backend-core prod CD 入口是 `deploy apply --env prod --service backend-core`。长期规则见 `docs/reference/artifact-registry.md`
- `commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr` 是 host Codex 指挥官直管微服务 skeleton 入口。当前命令返回 `phase=source-contract`、service/API/state/bridge/prompt/trace/#20/#46/ClaudeQQ 审批边界、.state/commander/ 状态模型、dev 无 daemon smoke contract、dry-run 计划和 GPT-5.5 PR prompt 边界辅助 lint,不接 live bridge、不注入 prompt、不发送 ClaudeQQ。`approval request --dry-run` 会生成 200 字以内中文纯文本 ClaudeQQ 审批草案、`notification-path-unavailable` blocker 和授权后唯一可用的 `bun scripts/cli.ts microservice proxy claudeqq /api/push/text --method POST --body-json '<payload>' --raw` 命令;不得提示使用本机 ClaudeQQ skill、powershell 或本地 server。`prompt-lint` 支持 `--prompt-file``--stdin`,输出 `ok``missingClauses``riskLevel``suggestedPatchSnippet` 且不回显完整 prompt;它是 commander 辅助检查,不是业务 PR 门禁,也不改变 `codex submit` 默认行为。`plan``smoke``approval request` 必须带 `--dry-run`;缺少时返回 `error=dry-run-required`。长期规则见 `docs/reference/host-codex-commander.md`
- `hwlab cd status|preflight|apply --env dev [--dry-run]` 是 HWLAB DEV CD 指挥侧 wrapper。默认通过 UniDesk provider `host.ssh` 进入 D601,再调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs`,不内嵌发布 kubectl 逻辑:`status` 汇总固定 CD mirror、Git clean/main/origin-main、`deploy/deploy.json`/artifact catalog/report、D601 native k3s guard 和 CD Lease lock,并用 `scripts/dev-cd-apply.mjs --status --skip-live-verify` 取得 target/promotion 摘要;`preflight` 进一步检查必需 SecretRef 对象/键存在性并运行 HWLAB `scripts/dev-cd-apply.mjs --dry-run --skip-live-verify` 受控事务摘要。完整远端 stdout/stderr 写入 D601 `~/.state/unidesk-hwlab-cd/<run-id>/` 和本地 `.state/hwlab-cd/<run-id>/` task dumpstdout 只返回有界摘要。默认 HWLAB CD repo 是 `/home/ubuntu/hwlab_cd``/home/ubuntu/hwlab` runner 历史目录不得作为发布真相。wrapper 强制 `KUBECONFIG=/etc/rancher/k3s/k3s.yaml` 并只以这个显式目标作为 gate;显式目标出现 `docker-desktop``desktop-control-plane``127.0.0.1:11700` 信号会结构化拒绝,写操作计划还必须观察到 node `d601`。真实 apply 只暴露 `scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report` 命令形状并标注 host-commander-only,本 runner 不执行 live apply、rollout、Lease mutation 或 16666/16667 live verification。长期规则见 `docs/reference/hwlab.md`
- `hwlab cd status|audit|preflight|apply --env dev [--dry-run]` 是 HWLAB DEV CD 指挥侧 wrapper。默认通过 UniDesk provider `host.ssh` 进入 D601,再调用 HWLAB repo-owned `scripts/dev-cd-apply.mjs`,不内嵌发布 kubectl 逻辑:`status` 汇总固定 CD mirror、Git clean/main/origin-main、`deploy/deploy.json`/artifact catalog/report、D601 native k3s guard 和 CD Lease lock,并用 `scripts/dev-cd-apply.mjs --status --skip-live-verify` 取得 target/promotion 摘要;`audit` 在 k3s/CD 恢复后做只读健康审计,返回有界 JSON 的 blocker 分类、D601 guard/node、SecretRef 存在性、registry 可达性、Lease phase/holder/staleness、deploy.json 与 artifact/workload image 收敛、current Deployment image/revision/rollout、16666/16667 public health commit/readiness 和 DB/runtime durability 摘要;`preflight` 进一步检查必需 SecretRef 对象/键存在性并运行 HWLAB `scripts/dev-cd-apply.mjs --dry-run --skip-live-verify` 受控事务摘要。完整远端 stdout/stderr 写入 D601 `~/.state/unidesk-hwlab-cd/<run-id>/` 和本地 `.state/hwlab-cd/<run-id>/` task dumpstdout 只返回有界摘要。默认 HWLAB CD repo 是 `/home/ubuntu/hwlab_cd``/home/ubuntu/hwlab` runner 历史目录不得作为发布真相。wrapper 强制 `KUBECONFIG=/etc/rancher/k3s/k3s.yaml` 并只以这个显式目标作为 gate;显式目标出现 `docker-desktop``desktop-control-plane``127.0.0.1:11700` 信号会结构化拒绝,audit/preflight/apply --dry-run 都必须观察到 node `d601`。真实 apply 只暴露 `scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report` 命令形状并标注 host-commander-only,本 runner 不执行 live apply、rollout、Lease mutation 或 DEV deploy apply。长期规则见 `docs/reference/hwlab.md`
- `gh auth status [--repo owner/name]` 探测 GitHub 操作前置条件并输出脱敏 JSON:是否存在 `gh` binary、是否存在 `GH_TOKEN`/`GITHUB_TOKEN` 或可用 `gh auth token` fallback、REST API 是否可达、目标 repo 是否可见、issue 是否可读。degraded reason 必须归类为 `missing-binary``missing-token``auth-failed``github-transient``network-proxy-failed``permission-denied``repo-not-found``repo-forbidden``issue-not-found``pr-not-found``scope-insufficient``validation-failed``invalid-response``unsupported-command`,不得打印 token;失败对象必须包含 `runnerDisposition=infra-blocked|business-failed`runner 应优先用该字段分流。`github-transient` 表示 GitHub DNS/API 连接在收到 HTTP 状态前失败,输出应带 `retryable=true` 或等价 commander action;这不是缺 token、认证失败、权限不足或 PR 语义失败。
- `codex prompt-lint [prompt|--prompt-file path|--prompt-stdin]` 是派发/steer 前的本地 dry-run prompt lint。它只读取 prompt 文本,返回 `dryRun=true``mutation=false``declaredClass``effectiveClass``requiredClass``dispatchDisposition`、缺失或矛盾项和有界 evidence,不访问 live service、不提交任务、不打印完整 prompt。分级固定为 `read-only``live-read``live-mutating`;未声明时按 `read-only` 处理。`codex submit --dry-run``codex steer --dry-run` 会嵌入同一 `promptLint` 结果,帮助指挥官在 dispatch/steer 前发现缺失或矛盾的 live mutation 授权。长期规则见 `docs/reference/code-queue-supervision.md` 的 DEV 测试授权分级。
- `gh issue list [owner/repo] [--state open|closed|all] [--limit N] [--repo owner/name] [--json number,title,state,url,updatedAt,createdAt,author,labels]` 通过 GitHub REST 列出 issue,默认 `state=open``limit=30`,输出稳定 JSON 且不依赖系统 `gh` binary。`owner/repo` 位置参数是 `--repo owner/repo` 的兼容别名;若位置 repo 与 `--repo` 冲突,或位置参数不是 `owner/repo`,必须结构化失败,禁止静默 fallback 到默认 repo。`--limit` 会映射到 GitHub `per_page` 并限制返回数量,避免一次拉爆上下文;未知 state 或未知 `--json` 字段必须结构化失败并带 `runnerDisposition=business-failed`。GitHub issues API 可能混入 PRCLI 会从 `.data.issues` 中过滤 pull request。
+2
View File
@@ -46,6 +46,7 @@ UniDesk 指挥侧固定入口:
```sh
bun scripts/cli.ts hwlab cd status --env dev
bun scripts/cli.ts hwlab cd audit --env dev
bun scripts/cli.ts hwlab cd preflight --env dev
bun scripts/cli.ts hwlab cd apply --env dev --dry-run
```
@@ -55,6 +56,7 @@ wrapper 的职责是把 host commander 常用的 HWLAB DEV rollout 查看/准备
- 默认 HWLAB CD repo 是 D601 固定干净 mirror `/home/ubuntu/hwlab_cd`,也可用 `--hwlab-repo` 显式指定同等干净 clone。wrapper 必须检查 `git status --short --branch`、origin remote、当前 branch `main`、本地 `origin/main``FETCH_HEAD` 和 worktree 权限;任何 dirty worktree、错误 remote、非 main、HEAD 未跟上本地 `origin/main` 或权限异常都返回结构化 blocker。`/home/ubuntu/hwlab` 是 runner 历史目录,不得作为发布真相。
- `deploy/deploy.json` 是唯一 desired-state。wrapper 只把 `deploy/artifact-catalog.dev.json``deploy/k8s/base/workloads.yaml``reports/dev-gate/dev-artifacts.json` 当作派生/证据读数;`status`/`preflight` 必须显示 target commit/ref、deploy.json、artifact catalog、workloads 和 live workload image 是否同源/收敛,不引入第二套 desired state。
- `status` 只读汇总 HWLAB repo path、Git clean/main/origin-main、desired-state 收敛、D601 native k3s guard 和 `Lease/hwlab-dev/hwlab-dev-cd-lock`;同时调用 HWLAB `scripts/dev-cd-apply.mjs --status --skip-live-verify` 取得 repo-owned target/promotion/deploy.json/artifact 摘要。16666/16667 live verification 不由本 runner 执行。
- `audit` 是 DEV CD 恢复后的只读健康审计,不是验收 gate 或报告生成器。它在 `status` 受控路径上补充只读 `kubectl get`/HTTP health probes,输出有界 JSON summary,分类 `control-plane-unavailable``docker-desktop-context-risk``second-control-plane-risk``workspace-unavailable``dirty-worktree``secret-missing``registry-unavailable``lease-held``lease-stale-candidate``artifact-missing``artifact-mismatch``runtime-job-blocked``rollout-unhealthy``public-tunnel-unhealthy``db-runtime-durability-risk`。audit 只显示 Secret 对象/key 是否存在,不显示值;只读判断 Lease 是否 stale,不释放或 break;只读拉取 16666/16667 `/health/live` 的 commit/readiness 摘要,不把它当作 M3 DEV-LIVE 验收。
- `preflight``status` 的基础上检查 apply 前 SecretRef`hwlab-cloud-api-dev-db/database-url``hwlab-cloud-api-dev-db-admin/admin-url``hwlab-code-agent-provider/openai-api-key`。只验证 Secret 对象和 key 元数据存在性,缺失时返回 blocker、影响范围和修复 runbook;禁止读取或打印 Secret value。
- `apply --dry-run` 调用 HWLAB `scripts/dev-cd-apply.mjs --dry-run --kubeconfig /etc/rancher/k3s/k3s.yaml --skip-live-verify`,只生成受控事务准备/阻塞摘要,不做真实 apply、rollout、Lease mutation 或 live verification。历史 `scripts/dev-deploy-apply.mjs` 可作为 HWLAB 内部支持脚本出现,但 UniDesk wrapper 不能把它当成平行 CD 入口。
- 完整下游 stdout/stderr 和 kubectl 读命令输出写入 D601 `~/.state/unidesk-hwlab-cd/<run-id>/`UniDesk 本地仅保存 provider task stdout/stderr dumpCLI stdout 只显示有界摘要和 dump path。
+116 -4
View File
@@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdirSync, writeFileSync } from "node:fs";
import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { buildHwlabCdRemoteCommandForTest } from "./src/hwlab-cd";
@@ -40,6 +40,7 @@ function makeFakeHwlabRepo(): string {
mkdirSync(join(root, "reports/dev-gate"), { recursive: true });
writeFileSync(join(root, "scripts/dev-cd-apply.mjs"), [
"const kubeconfigIndex = process.argv.indexOf('--kubeconfig');",
"if (process.argv.includes('--apply') || process.argv.includes('--write-report') || process.argv.includes('--confirm-dev')) { throw new Error('mutation flag must not be used in wrapper tests'); }",
"process.stdout.write(JSON.stringify({",
" ok: true,",
" status: 'pass',",
@@ -85,6 +86,43 @@ function makeFakeHwlabRepo(): string {
function makeFakeBin(mode: "native" | "desktop" | "stale-default" | "wrong-node" | "missing-secret" | "second-plane"): string {
const bin = join(tmpdir(), `unidesk-hwlab-cd-bin-${process.pid}-${Date.now()}-${mode}-${Math.random().toString(16).slice(2)}`);
mkdirSync(bin, { recursive: true });
const deploymentJson = {
items: [
{
metadata: {
name: "hwlab-cloud-api",
labels: { "app.kubernetes.io/name": "hwlab-cloud-api", "hwlab.pikastech.local/service-id": "hwlab-cloud-api" },
annotations: { "deployment.kubernetes.io/revision": "7" },
},
spec: {
replicas: 1,
template: { spec: { containers: [{ name: "hwlab-cloud-api", image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", env: [{ name: "HWLAB_COMMIT_ID", value: "abc1234" }] }] } },
},
status: { availableReplicas: 1, updatedReplicas: 1, unavailableReplicas: 0, conditions: [{ type: "Available", status: "True" }, { type: "Progressing", status: "True", reason: "NewReplicaSetAvailable" }] },
},
{
metadata: {
name: "hwlab-cloud-web",
labels: { "app.kubernetes.io/name": "hwlab-cloud-web", "hwlab.pikastech.local/service-id": "hwlab-cloud-web" },
annotations: { "deployment.kubernetes.io/revision": "8" },
},
spec: {
replicas: 1,
template: { spec: { containers: [{ name: "hwlab-cloud-web", image: "127.0.0.1:5000/hwlab/hwlab-cloud-web:abc1234", env: [{ name: "HWLAB_COMMIT_ID", value: "abc1234" }] }] } },
},
status: { availableReplicas: 1, updatedReplicas: 1, unavailableReplicas: 0, conditions: [{ type: "Available", status: "True" }, { type: "Progressing", status: "True" }] },
},
],
};
const podsJson = {
items: [
{
metadata: { name: "hwlab-cloud-api-abc" },
status: { containerStatuses: [{ name: "hwlab-cloud-api", image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", imageID: "docker-pullable://127.0.0.1:5000/hwlab/hwlab-cloud-api@sha256:" + "b".repeat(64), state: { running: {} } }] },
},
],
};
const jobsJson = { items: [{ metadata: { name: "hwlab-runtime-provision-old" }, status: { succeeded: 1, failed: 0, active: 0, completionTime: "2026-05-24T00:00:00Z" } }] };
const explicitContext = mode === "desktop" ? "docker-desktop" : "default";
const explicitServer = mode === "desktop" ? "https://127.0.0.1:11700" : "https://127.0.0.1:6443";
const explicitNodes = mode === "desktop" ? "desktop-control-plane" : mode === "wrong-node" ? "d602" : "d601";
@@ -112,6 +150,9 @@ function makeFakeBin(mode: "native" | "desktop" | "stale-default" | "wrong-node"
" printf 'Error from server (NotFound): namespaces \"hwlab-dev\" not found\\n' >&2; exit 1",
"fi",
"if [[ \"$*\" =~ ^-n[[:space:]]+hwlab-dev[[:space:]]+get[[:space:]]+lease[[:space:]]+hwlab-dev-cd-lock[[:space:]]+-o[[:space:]]+json$ ]]; then printf 'Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found\\n' >&2; exit 1; fi",
"if [[ \"$*\" == '-n hwlab-dev get deployments -o json' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify(deploymentJson)) + "; exit 0; fi",
"if [[ \"$*\" == '-n hwlab-dev get pods -o json' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify(podsJson)) + "; exit 0; fi",
"if [[ \"$*\" == '-n hwlab-dev get jobs -o json' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify(jobsJson)) + "; exit 0; fi",
"if [[ \"$*\" == '-n hwlab-dev get secret hwlab-code-agent-provider -o name' && " + JSON.stringify(mode) + " == 'missing-secret' ]]; then printf 'Error from server (NotFound): secrets \"hwlab-code-agent-provider\" not found\\n' >&2; exit 1; fi",
"if [[ \"$*\" =~ ^-n\\ hwlab-dev\\ get\\ secret\\ ([^[:space:]]+)\\ -o\\ name$ ]]; then printf 'secret/%s\\n' \"${BASH_REMATCH[1]}\"; exit 0; fi",
"if [[ \"$*\" == '-n hwlab-dev describe secret hwlab-cloud-api-dev-db' ]]; then printf 'Name: hwlab-cloud-api-dev-db\\nData\\n====\\ndatabase-url: 48 bytes\\n'; exit 0; fi",
@@ -120,6 +161,34 @@ function makeFakeBin(mode: "native" | "desktop" | "stale-default" | "wrong-node"
"printf 'unexpected kubectl args: %s\\n' \"$*\" >&2; exit 99",
].join("\n"));
spawnSync("chmod", ["+x", join(bin, "kubectl")]);
writeFileSync(join(bin, "curl"), [
"#!/usr/bin/env bash",
"set -euo pipefail",
"url=\"${@: -1}\"",
"if [[ \"$*\" == *'http://127.0.0.1:5000/v2/'* ]]; then printf '{}\\n'; exit 0; fi",
"if [[ \"$url\" == 'http://74.48.78.17:16666/health/live' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify({
serviceId: "hwlab-cloud-web",
environment: "dev",
status: "ready",
ready: true,
commit: { id: "abc1234" },
image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-web:abc1234", tag: "abc1234" },
blockerCodes: [],
})) + "; exit 0; fi",
"if [[ \"$url\" == 'http://74.48.78.17:16667/health/live' ]]; then printf '%s\\n' " + JSON.stringify(JSON.stringify({
serviceId: "hwlab-cloud-api",
environment: "dev",
status: "ready",
ready: true,
commit: { id: "abc1234" },
image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", tag: "abc1234" },
db: { ready: true, connected: true, liveDbEvidence: true, runtimeReadiness: { status: "ready", ready: true, blocker: null } },
runtime: { status: "ready", ready: true, durable: true, durableRequested: true, liveRuntimeEvidence: true, blocker: null, adapter: "postgres" },
blockerCodes: [],
})) + "; exit 0; fi",
"printf 'unexpected curl args: %s\\n' \"$*\" >&2; exit 22",
].join("\n"));
spawnSync("chmod", ["+x", join(bin, "curl")]);
return bin;
}
@@ -142,11 +211,14 @@ const secondPlaneBin = makeFakeBin("second-plane");
const help = runCli(["hwlab", "help"]);
assert.equal(help.ok, true);
assert.equal((help.data as JsonRecord).command, "hwlab cd");
assert.equal(((help.data as JsonRecord).usage as string[]).includes("bun scripts/cli.ts hwlab cd audit --env dev"), true);
const remoteCommand = buildHwlabCdRemoteCommandForTest(withLocalTransport(["cd", "apply", "--env", "dev", "--dry-run"]));
assert.equal(remoteCommand.includes("scripts/dev-cd-apply.mjs"), true);
assert.equal(remoteCommand.includes("/etc/rancher/k3s/k3s.yaml"), true);
assert.equal(remoteCommand.includes("kubectl rollout"), false);
assert.equal(remoteCommand.includes("kubectl apply"), false);
assert.equal(remoteCommand.includes("break-stale-lock"), false);
const realApply = runCli(["hwlab", "cd", "apply", "--env", "dev", "--hwlab-repo", fakeRepo], {
PATH: `${nativeBin}:${process.env.PATH ?? ""}`,
@@ -201,6 +273,46 @@ assert.equal(((dryRunData.target as JsonRecord).promotionCommit), "abc1234567890
assert.equal(((dryRunData.promotion as JsonRecord).source), "deploy/deploy.json");
assert.equal(JSON.stringify(dryRunData).includes("sk-secret"), false);
const reportsBefore = existsSync(join(fakeRepo, "reports")) ? JSON.stringify(readdirSync(join(fakeRepo, "reports"), { recursive: true })) : "";
const audit = runCli(withLocalTransport([
"hwlab",
"cd",
"audit",
"--env",
"dev",
"--hwlab-repo",
fakeRepo,
]), {
PATH: `${nativeBin}:${process.env.PATH ?? ""}`,
});
assert.equal(audit.ok, true);
const auditData = dataOf(audit);
const auditSummary = auditData.audit as JsonRecord;
assert.equal(auditSummary.status, "pass");
assert.equal((auditSummary.namespace), "hwlab-dev");
assert.equal(((auditSummary.nodeGuard as JsonRecord).nodeNames as unknown[]).includes("d601"), true);
assert.equal(((auditSummary.secrets as JsonRecord).valuesRead), false);
assert.equal(((auditSummary.secrets as JsonRecord).valuesPrinted), false);
assert.equal(JSON.stringify(auditSummary).includes("48 bytes"), false);
assert.equal(((auditSummary.registry as JsonRecord).status), "pass");
assert.equal(((auditSummary.lease as JsonRecord).staleClassification), "not-held");
assert.equal((((auditSummary.desiredState as JsonRecord).imageConvergence as JsonRecord).status), "pass");
assert.equal(((auditSummary.workload as JsonRecord).status), "healthy");
assert.equal(((auditSummary.publicHealth as JsonRecord).status), "pass");
assert.equal(((auditSummary.durability as JsonRecord).status), "pass");
assert.deepEqual((auditSummary.blockerTypes as unknown[]), []);
assert.equal(((auditSummary.safety as JsonRecord).reportsWritten), false);
assert.equal(((auditSummary.safety as JsonRecord).cdLockMutated), false);
assert.equal((auditData.remote as JsonRecord).providerId, "D601");
assert.equal(JSON.stringify(auditData).length < 80_000, true);
assert.equal(JSON.stringify(auditData).includes("kubectl apply"), false);
assert.equal(JSON.stringify(auditData).includes("kubectl rollout"), false);
assert.equal(JSON.stringify(auditData).includes("--apply"), false);
assert.equal(JSON.stringify(auditData).includes("--write-report"), false);
assert.equal(JSON.stringify(auditData).includes("sk-secret"), false);
const reportsAfter = existsSync(join(fakeRepo, "reports")) ? JSON.stringify(readdirSync(join(fakeRepo, "reports"), { recursive: true })) : "";
assert.equal(reportsAfter, reportsBefore);
const status = runCli(withLocalTransport([
"hwlab",
"cd",
@@ -240,10 +352,9 @@ assert.equal((staleDefaultGuard.defaultKubectlDiagnostic as JsonRecord).status,
const desktopRefusal = runCli(withLocalTransport([
"hwlab",
"cd",
"apply",
"audit",
"--env",
"dev",
"--dry-run",
"--hwlab-repo",
fakeRepo,
]), {
@@ -252,7 +363,8 @@ const desktopRefusal = runCli(withLocalTransport([
assert.equal(desktopRefusal.ok, false);
const desktopData = dataOf(desktopRefusal);
assert.equal(desktopData.status, "refused");
assert.deepEqual((desktopData.nodeGuard as JsonRecord).refusalSignals, ["docker-desktop", "desktop-control-plane", "127.0.0.1:11700"]);
assert.deepEqual((((desktopData.audit as JsonRecord).nodeGuard as JsonRecord).refusalSignals), ["docker-desktop", "desktop-control-plane", "127.0.0.1:11700"]);
assert.equal(((desktopData.blockerTypes as unknown[]).includes("docker-desktop-context-risk")), true);
const wrongNodeBlocked = runCli(withLocalTransport([
"hwlab",
+1 -1
View File
@@ -49,7 +49,7 @@ export function rootHelp(): unknown {
{ command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." },
{ command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and merge blocked." },
{ command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." },
{ command: "hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Bounded HWLAB DEV CD wrapper that calls HWLAB repo-owned scripts, forces D601 native k3s kubeconfig, refuses Docker Desktop control-plane signals, and writes full stdout/stderr dumps." },
{ command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Bounded HWLAB DEV CD wrapper that calls HWLAB repo-owned scripts, forces D601 native k3s kubeconfig, refuses Docker Desktop control-plane signals, and exposes read-only post-recovery blocker classification." },
{ command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." },
{ command: "schedule list|get|runs|run|retry-run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run <id> supports --wait-ms N and retry-run reuses the failed run's schedule." },
{ command: "schedule upsert-pgdata-backup [--time HH:MM] [--remote-base /SERVER_DATA/UNIDESK_PG_DATA]", description: "Create or update the daily PGDATA physical backup task that uploads monthly rotated archives to Baidu Netdisk." },
+604 -6
View File
@@ -276,6 +276,168 @@ function readJsonSummary(repoPath, relativePath, fallback = {}) {
}
}
function readJsonFile(repoPath, relativePath) {
const absolutePath = path.join(repoPath, relativePath);
try {
const raw = fs.readFileSync(absolutePath, "utf8");
return { ok: true, path: relativePath, exists: true, hash: sha256(raw), json: JSON.parse(raw), error: null };
} catch (error) {
return {
ok: false,
path: relativePath,
exists: fs.existsSync(absolutePath),
hash: null,
json: null,
error: error instanceof Error ? error.message : String(error),
};
}
}
function commitMatches(actual, expected) {
if (typeof actual !== "string" || typeof expected !== "string" || actual.length === 0 || expected.length === 0) return false;
return actual === expected || actual.startsWith(expected) || expected.startsWith(actual);
}
function imageTag(image) {
const value = String(image || "");
const slash = value.lastIndexOf("/");
const colon = value.lastIndexOf(":");
return colon > slash ? value.slice(colon + 1) : null;
}
function servicesFromManifest(manifest) {
const services = Array.isArray(manifest?.services)
? manifest.services
: manifest?.services && typeof manifest.services === "object"
? Object.values(manifest.services)
: [];
return services
.filter((service) => service && typeof service === "object")
.map((service) => ({
serviceId: stringValue(service.serviceId) || stringValue(service.id) || stringValue(service.name),
name: stringValue(service.name) || stringValue(service.serviceId) || stringValue(service.id),
image: stringValue(service.image),
imageTag: stringValue(service.imageTag) || imageTag(service.image),
commitId: stringValue(service.commitId) || stringValue(service.imageTag) || imageTag(service.image),
namespace: stringValue(service.namespace) || namespace,
healthPath: stringValue(service.healthPath) || "/health/live",
replicas: Number.isInteger(service.replicas) ? service.replicas : null,
env: asRecord(service.env) || {},
digest: stringValue(service.digest),
publishState: stringValue(service.publishState) || stringValue(service.status),
artifactRequired: service.artifactRequired !== false,
}))
.filter((service) => service.serviceId);
}
function summarizeDeployFile(file) {
const manifest = asRecord(file.json);
const endpoints = asRecord(manifest?.publicEndpoints) || {};
return {
path: file.path,
exists: file.exists,
hash: file.hash,
commitId: stringValue(manifest?.commitId),
environment: stringValue(manifest?.environment),
namespace: stringValue(manifest?.namespace) || namespace,
endpoint: stringValue(manifest?.endpoint),
serviceCount: servicesFromManifest(manifest).length,
publicEndpoints: {
frontend: stringValue(asRecord(endpoints.frontend)?.url),
api: stringValue(asRecord(endpoints.api)?.url),
},
unavailableReason: file.ok ? null : file.error,
};
}
function summarizeArtifactFile(file) {
const manifest = asRecord(file.json);
const services = servicesFromManifest(manifest);
const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 };
for (const service of services) {
if (/^sha256:[a-f0-9]{64}$/u.test(service.digest || "")) digestCounts.sha256 += 1;
else if (service.digest === "not_published") digestCounts.notPublished += 1;
else digestCounts.invalid += 1;
}
return {
path: file.path,
exists: file.exists,
hash: file.hash,
commitId: stringValue(manifest?.commitId),
artifactState: stringValue(manifest?.artifactState),
ciPublished: manifest?.publish?.ciPublished ?? null,
registryVerified: manifest?.publish?.registryVerified ?? null,
serviceCount: services.length,
digestCounts,
unavailableReason: file.ok ? null : file.error,
};
}
function buildDesiredStateAudit(repoPath, target) {
const deployFile = readJsonFile(repoPath, "deploy/deploy.json");
const catalogFile = readJsonFile(repoPath, "deploy/artifact-catalog.dev.json");
const reportFile = readJsonFile(repoPath, "reports/dev-gate/dev-artifacts.json");
const deployServices = servicesFromManifest(deployFile.json);
const catalogServices = servicesFromManifest(catalogFile.json);
const catalogById = new Map(catalogServices.map((service) => [service.serviceId, service]));
const mismatches = [];
const missing = [];
for (const service of deployServices) {
if (!service.image) continue;
const catalog = catalogById.get(service.serviceId);
if (!catalog && service.artifactRequired) {
missing.push({ serviceId: service.serviceId, reason: "missing-from-artifact-catalog" });
continue;
}
if (catalog?.image && catalog.image !== service.image) {
mismatches.push({ serviceId: service.serviceId, deployImage: service.image, catalogImage: catalog.image });
}
if (catalog && catalog.artifactRequired && catalog.publishState && catalog.publishState !== "published") {
missing.push({ serviceId: service.serviceId, reason: `artifact-${catalog.publishState}` });
}
}
const deploySummary = summarizeDeployFile(deployFile);
const catalogSummary = summarizeArtifactFile(catalogFile);
const reportSummary = summarizeArtifactFile(reportFile);
const targetCommit = stringValue(target?.shortCommitId) || stringValue(target?.promotionCommit) || deploySummary.commitId;
const commitConverged = Boolean(
targetCommit &&
commitMatches(deploySummary.commitId, targetCommit) &&
(!catalogSummary.commitId || commitMatches(catalogSummary.commitId, targetCommit))
);
const status = !deployFile.ok || !catalogFile.ok || missing.length > 0
? "missing"
: mismatches.length > 0 || !commitConverged
? "mismatch"
: "pass";
return {
status,
targetCommit,
commitConverged,
deployJson: deploySummary,
artifactCatalog: catalogSummary,
artifactReport: reportSummary,
imageConvergence: {
status: missing.length === 0 && mismatches.length === 0 ? "pass" : missing.length > 0 ? "missing" : "mismatch",
serviceCount: deployServices.length,
catalogServiceCount: catalogServices.length,
missing: missing.slice(0, 12),
mismatches: mismatches.slice(0, 12),
},
services: deployServices.map((service) => ({
serviceId: service.serviceId,
image: service.image,
imageTag: service.imageTag,
catalogImage: catalogById.get(service.serviceId)?.image || null,
catalogDigest: catalogById.get(service.serviceId)?.digest || null,
replicas: service.replicas,
})).slice(0, 30),
serviceEnv: {
cloudApi: asRecord(deployServices.find((service) => service.serviceId === "hwlab-cloud-api")?.env) || {},
},
};
}
function forbiddenKubeSignals(text) {
const signals = [];
if (/docker-desktop/iu.test(text)) signals.push("docker-desktop");
@@ -490,6 +652,409 @@ async function secretPreflight(kubeconfig, guard, dumpDir, timeoutMs) {
};
}
async function registryAudit(kubeconfig, guard, dumpDir, timeoutMs) {
const result = await runCaptured(["curl", "-fsS", "--max-time", "5", "http://127.0.0.1:5000/v2/"], process.cwd(), dumpDir, "registry-v2", { timeoutMs: Math.min(timeoutMs, 8000) });
const reachable = result.ok || /401|unauthorized/iu.test(`${result.stdoutText}\n${result.stderrText}`);
let k3sPullAccess = { status: "skipped", reason: "d601-native-k3s-guard-not-pass", pullFailureCount: null, pulledRegistryImageCount: null };
if (guard.status === "pass") {
const env = { ...process.env, KUBECONFIG: kubeconfig };
const pods = await runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "registry-k3s-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) });
const parsed = asRecord(parseJson(pods.stdoutText));
const items = Array.isArray(parsed?.items) ? parsed.items : [];
let pullFailureCount = 0;
let pulledRegistryImageCount = 0;
for (const item of items) {
const statuses = Array.isArray(item?.status?.containerStatuses) ? item.status.containerStatuses : [];
for (const status of statuses) {
const image = String(status?.image || "");
const imageID = String(status?.imageID || "");
const waiting = String(status?.state?.waiting?.reason || "");
if (/ErrImagePull|ImagePullBackOff|InvalidImageName/u.test(waiting)) pullFailureCount += 1;
if (image.includes("127.0.0.1:5000/") && imageID.includes("sha256:")) pulledRegistryImageCount += 1;
}
}
k3sPullAccess = {
status: !pods.ok ? "unavailable" : pullFailureCount > 0 ? "blocked" : pulledRegistryImageCount > 0 ? "pass" : "degraded",
pullFailureCount,
pulledRegistryImageCount,
command: commandView(pods),
};
}
return {
status: reachable && !["blocked", "unavailable"].includes(k3sPullAccess.status) ? "pass" : reachable ? "degraded" : "blocked",
endpoint: "http://127.0.0.1:5000/v2/",
processHttpAccess: {
status: reachable ? "pass" : "blocked",
reachable,
exitCode: result.exitCode,
command: commandView(result),
},
k3sPullAccess,
};
}
function deploymentCondition(conditions, type) {
return (Array.isArray(conditions) ? conditions : []).find((condition) => condition?.type === type) || null;
}
function envValue(container, name) {
const item = (Array.isArray(container?.env) ? container.env : []).find((entry) => entry?.name === name);
return item?.value ?? null;
}
async function workloadAudit(kubeconfig, guard, desired, dumpDir, timeoutMs) {
if (guard.status !== "pass") {
return { status: "skipped", reason: "d601-native-k3s-guard-not-pass", namespace, deployments: [], currentImageConvergence: { status: "unavailable", unavailableReason: "d601-native-k3s-guard-not-pass" } };
}
const env = { ...process.env, KUBECONFIG: kubeconfig };
const [deployments, pods, jobs] = await Promise.all([
runCaptured(["kubectl", "-n", namespace, "get", "deployments", "-o", "json"], process.cwd(), dumpDir, "workload-deployments", { env, timeoutMs: Math.min(timeoutMs, 10000) }),
runCaptured(["kubectl", "-n", namespace, "get", "pods", "-o", "json"], process.cwd(), dumpDir, "workload-pods", { env, timeoutMs: Math.min(timeoutMs, 10000) }),
runCaptured(["kubectl", "-n", namespace, "get", "jobs", "-o", "json"], process.cwd(), dumpDir, "runtime-jobs", { env, timeoutMs: Math.min(timeoutMs, 10000) }),
]);
const desiredById = new Map((desired.services || []).map((service) => [service.serviceId, service]));
const deploymentItems = Array.isArray(parseJson(deployments.stdoutText)?.items) ? parseJson(deployments.stdoutText).items : [];
const summaries = deploymentItems.map((deployment) => {
const metadata = asRecord(deployment.metadata) || {};
const spec = asRecord(deployment.spec) || {};
const status = asRecord(deployment.status) || {};
const template = asRecord(spec.template) || {};
const podSpec = asRecord(template.spec) || {};
const containers = Array.isArray(podSpec.containers) ? podSpec.containers : [];
const container = containers[0] || {};
const labels = asRecord(metadata.labels) || {};
const serviceId = stringValue(labels["hwlab.pikastech.local/service-id"]) || stringValue(labels["app.kubernetes.io/name"]) || stringValue(metadata.name);
const desiredService = desiredById.get(serviceId);
const image = stringValue(container.image);
const desiredImage = desiredService?.image || null;
const available = Number(status.availableReplicas || 0);
const desiredReplicas = Number(spec.replicas || 0);
const updated = Number(status.updatedReplicas || 0);
const unavailable = Number(status.unavailableReplicas || 0);
const progressing = deploymentCondition(status.conditions, "Progressing");
const availableCondition = deploymentCondition(status.conditions, "Available");
return {
name: stringValue(metadata.name),
serviceId,
image,
imageTag: imageTag(image),
desiredImage,
imageMatchesDesired: desiredImage ? image === desiredImage : null,
revision: stringValue(asRecord(metadata.annotations)?.["deployment.kubernetes.io/revision"]),
replicas: desiredReplicas,
availableReplicas: available,
updatedReplicas: updated,
unavailableReplicas: unavailable,
rolloutStatus: desiredReplicas === 0 || (available >= desiredReplicas && updated >= desiredReplicas && unavailable === 0) ? "healthy" : "unhealthy",
conditions: {
available: availableCondition ? { status: availableCondition.status ?? null, reason: availableCondition.reason ?? null } : null,
progressing: progressing ? { status: progressing.status ?? null, reason: progressing.reason ?? null } : null,
},
commitEnv: envValue(container, "HWLAB_COMMIT_ID"),
};
});
const podItems = Array.isArray(parseJson(pods.stdoutText)?.items) ? parseJson(pods.stdoutText).items : [];
const waiting = [];
for (const pod of podItems) {
const podName = stringValue(pod?.metadata?.name);
for (const status of Array.isArray(pod?.status?.containerStatuses) ? pod.status.containerStatuses : []) {
const reason = stringValue(status?.state?.waiting?.reason);
if (reason) waiting.push({ pod: podName, container: status.name ?? null, reason, image: status.image ?? null });
}
}
const jobItems = Array.isArray(parseJson(jobs.stdoutText)?.items) ? parseJson(jobs.stdoutText).items : [];
const runtimeJobs = jobItems
.filter((job) => /hwlab-runtime|runtime-db|runtime/i.test(String(job?.metadata?.name || "")))
.map((job) => ({
name: stringValue(job?.metadata?.name),
succeeded: Number(job?.status?.succeeded || 0),
failed: Number(job?.status?.failed || 0),
active: Number(job?.status?.active || 0),
completionTime: stringValue(job?.status?.completionTime),
status: Number(job?.status?.failed || 0) > 0 ? "blocked" : Number(job?.status?.active || 0) > 0 ? "running" : Number(job?.status?.succeeded || 0) > 0 ? "pass" : "unknown",
}))
.slice(0, 12);
const imageMismatches = summaries.filter((deployment) => deployment.imageMatchesDesired === false);
const unhealthy = summaries.filter((deployment) => deployment.rolloutStatus === "unhealthy");
return {
status: !deployments.ok ? "unavailable" : unhealthy.length > 0 ? "unhealthy" : imageMismatches.length > 0 ? "mismatch" : "healthy",
namespace,
deploymentCount: summaries.length,
deployments: summaries.slice(0, 30),
currentImageConvergence: {
status: imageMismatches.length === 0 ? "pass" : "mismatch",
mismatches: imageMismatches.map((deployment) => ({ serviceId: deployment.serviceId, image: deployment.image, desiredImage: deployment.desiredImage })).slice(0, 12),
},
podWaiting: waiting.slice(0, 12),
runtimeJobs,
commands: [deployments, pods, jobs].map(commandView),
};
}
function compactHealth(body, expectedServiceId, expectedCommit) {
const json = asRecord(body);
const commit = stringValue(asRecord(json?.commit)?.id) || stringValue(json?.commitId) || stringValue(json?.revision) || stringValue(asRecord(json?.image)?.tag);
const runtime = asRecord(json?.runtime);
const db = asRecord(json?.db);
const blockerCodes = Array.isArray(json?.blockerCodes) ? json.blockerCodes.map(String).slice(0, 12) : [];
return {
serviceId: stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id),
environment: stringValue(json?.environment),
status: stringValue(json?.status),
ready: typeof json?.ready === "boolean" ? json.ready : null,
observedCommit: commit,
commitMatches: expectedCommit ? commitMatches(commit, expectedCommit) : null,
serviceMatches: expectedServiceId ? (stringValue(json?.serviceId) || stringValue(asRecord(json?.service)?.id)) === expectedServiceId : null,
image: typeof json?.image === "string" ? json.image : stringValue(asRecord(json?.image)?.reference),
blockerCodes,
db: db === null ? null : {
ready: db.ready ?? null,
connected: db.connected ?? null,
liveDbEvidence: db.liveDbEvidence ?? null,
runtimeReadiness: asRecord(db.runtimeReadiness) ? {
status: stringValue(db.runtimeReadiness.status),
ready: db.runtimeReadiness.ready ?? null,
blocker: stringValue(db.runtimeReadiness.blocker),
} : null,
},
runtime: runtime === null ? null : {
status: stringValue(runtime.status),
ready: runtime.ready ?? null,
durable: runtime.durable ?? null,
durableRequested: runtime.durableRequested ?? null,
liveRuntimeEvidence: runtime.liveRuntimeEvidence ?? null,
blocker: stringValue(runtime.blocker),
adapter: stringValue(runtime.adapter),
},
};
}
async function httpHealthAudit(desired, dumpDir, timeoutMs) {
const endpoints = [
{ id: "cloud-web-16666", url: desired.deployJson.publicEndpoints.frontend || "http://74.48.78.17:16666", expectedServiceId: "hwlab-cloud-web" },
{ id: "cloud-api-16667", url: desired.deployJson.publicEndpoints.api || desired.deployJson.endpoint || "http://74.48.78.17:16667", expectedServiceId: "hwlab-cloud-api" },
];
const expectedCommit = desired.targetCommit || desired.deployJson.commitId || null;
const observations = [];
for (const endpoint of endpoints) {
const result = await runCaptured(["curl", "-fsS", "--max-time", "8", `${endpoint.url.replace(/\/+$/u, "")}/health/live`], process.cwd(), dumpDir, `public-health-${endpoint.id}`, { timeoutMs: Math.min(timeoutMs, 10000) });
const parsed = parseJson(result.stdoutText);
const health = compactHealth(parsed, endpoint.expectedServiceId, expectedCommit);
const reachable = result.ok && parsed !== null;
observations.push({
id: endpoint.id,
url: endpoint.url,
status: reachable && health.serviceMatches !== false && health.commitMatches !== false ? "pass" : "blocked",
reachable,
expectedServiceId: endpoint.expectedServiceId,
expectedCommit,
health,
command: commandView(result),
});
}
return {
status: observations.every((item) => item.status === "pass") ? "pass" : "blocked",
expectedCommit,
summary: {
checked: observations.length,
reachable: observations.filter((item) => item.reachable).length,
commitMatches: observations.filter((item) => item.health.commitMatches === true).length,
readinessPass: observations.filter((item) => item.health.ready !== false && !["blocked", "failed"].includes(String(item.health.status || ""))).length,
ports: [16666, 16667],
},
endpoints: observations,
};
}
function runtimeDurabilityAudit(desired, publicHealth, secretRefs, workload) {
const api = publicHealth?.endpoints?.find((endpoint) => endpoint.id === "cloud-api-16667")?.health || null;
const cloudApiEnv = asRecord(desired.serviceEnv?.cloudApi) || {};
const configuredDurable = cloudApiEnv.HWLAB_CLOUD_RUNTIME_DURABLE === "true";
const configuredAdapter = stringValue(cloudApiEnv.HWLAB_CLOUD_RUNTIME_ADAPTER);
const dbSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db" && ref.secretKey === "database-url");
const adminSecret = (secretRefs?.secretRefs || []).find((ref) => ref.secretName === "hwlab-cloud-api-dev-db-admin" && ref.secretKey === "admin-url");
const runtime = asRecord(api?.runtime);
const dbRuntimeReadiness = asRecord(api?.db?.runtimeReadiness);
const liveReady = runtime?.durable === true && runtime?.ready !== false && runtime?.liveRuntimeEvidence === true && !["blocked", "degraded", "failed"].includes(String(runtime?.status || ""));
const status = liveReady
? "pass"
: api
? "risk"
: configuredDurable && configuredAdapter === "postgres" && dbSecret?.keyPresent === true && adminSecret?.keyPresent === true
? "unavailable"
: "risk";
return {
status,
configured: {
adapter: configuredAdapter,
durable: configuredDurable,
dbSecretPresent: dbSecret?.keyPresent === true,
adminSecretPresent: adminSecret?.keyPresent === true,
},
live: api ? {
db: api.db,
runtime: api.runtime,
dbRuntimeReadiness,
} : null,
workloadEvidence: {
cloudApiDeployment: (workload?.deployments || []).find((deployment) => deployment.serviceId === "hwlab-cloud-api") || null,
},
unavailableReason: api ? null : "public-health-cloud-api-unavailable",
risk: status === "pass" ? null : "db-runtime-durability-risk",
};
}
function blocker(scope, summary, details = {}) {
return { scope, summary, ...details };
}
function classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled }) {
const blockers = [];
if (guard?.status !== "pass") blockers.push(blocker("control-plane-unavailable", guard?.summary || "D601 native k3s guard did not pass."));
if (guard?.refusalSignals?.length > 0) blockers.push(blocker("docker-desktop-context-risk", "Explicit D601 kubeconfig resolved to forbidden Docker Desktop control-plane signal.", { refusalSignals: guard.refusalSignals }));
if (guard?.secondControlPlaneRisk) blockers.push(blocker("second-control-plane-risk", "Bare kubectl can observe hwlab-dev through a different control plane; audit is read-only and will not release or mutate locks."));
if (repo.status !== "selected") blockers.push(blocker("workspace-unavailable", "No eligible HWLAB CD repo was selected."));
for (const item of git?.blockers || []) {
const scope = item.scope === "hwlab-git-clean" ? "dirty-worktree" : item.scope?.startsWith("hwlab-git") || item.scope?.includes("permission") ? "workspace-unavailable" : item.scope;
blockers.push(blocker(scope, item.summary));
}
for (const item of secretRefs?.blockers || []) blockers.push(blocker("secret-missing", item.summary, { secretRef: item.scope }));
if (registry?.status === "blocked" || registry?.status === "degraded") blockers.push(blocker("registry-unavailable", "Registry reachability or k3s pull evidence is not healthy.", { status: registry.status }));
if (lock?.held === true) blockers.push(blocker("lease-held", `HWLAB DEV CD Lease is held by ${lock.ownerTaskId || lock.holderIdentity || "unknown"}.`, { phase: lock.phase, retryAfterSeconds: lock.retryAfterSeconds }));
if (lock?.stale === true) blockers.push(blocker("lease-stale-candidate", "HWLAB DEV CD Lease appears stale; audit is read-only and did not release or break it.", { phase: lock.phase }));
if (desired?.status === "missing") blockers.push(blocker("artifact-missing", "deploy/deploy.json or artifact catalog/report is missing or incomplete.", { imageConvergence: desired.imageConvergence }));
if (desired?.status === "mismatch" || workload?.currentImageConvergence?.status === "mismatch") blockers.push(blocker("artifact-mismatch", "deploy/deploy.json, artifact catalog, or current workload images are not converged.", { desired: desired?.status, currentImageConvergence: workload?.currentImageConvergence }));
if ((workload?.runtimeJobs || []).some((job) => job.status === "blocked" || job.status === "running")) blockers.push(blocker("runtime-job-blocked", "Runtime DB maintenance Job is blocked or still running.", { runtimeJobs: workload.runtimeJobs }));
if (workload?.status === "unhealthy") blockers.push(blocker("rollout-unhealthy", "One or more hwlab-dev Deployments are not fully available.", { deployments: (workload.deployments || []).filter((item) => item.rolloutStatus === "unhealthy").slice(0, 8) }));
if (publicHealth?.status === "blocked") blockers.push(blocker("public-tunnel-unhealthy", "16666/16667 public health did not pass read-only commit/readiness checks.", { summary: publicHealth.summary }));
if (durability?.status === "risk") blockers.push(blocker("db-runtime-durability-risk", "DB/runtime durability is not proven by live cloud-api health.", { live: durability.live, configured: durability.configured }));
if (controlled?.commandOk === false || controlled?.status === "blocked") blockers.push(blocker("runtime-job-blocked", "HWLAB repo-owned dev-cd-apply status reported blocked or failed.", { status: controlled?.status }));
return blockers;
}
function auditSummary({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled, dumpDir }) {
const blockers = classifyAuditBlockers({ repo, git, guard, secretRefs, registry, lock, desired, workload, publicHealth, durability, controlled });
const blockerTypes = [...new Set(blockers.map((item) => item.scope))];
return {
ok: blockers.length === 0,
status: blockers.length === 0 ? "pass" : guard?.refusal ? "refused" : "blocked",
env: "dev",
namespace,
nodeGuard: {
status: guard?.status ?? "unavailable",
nodeName: guard?.requiredNodeName ?? requiredNodeName,
nodeNames: guard?.nodeNames ?? [],
requiredNodePresent: guard?.requiredNodePresent ?? false,
currentContext: guard?.currentContext ?? null,
apiServer: guard?.apiServer ?? null,
kubeconfig: guard?.kubeconfig ?? nativeKubeconfig,
refusalSignals: guard?.refusalSignals ?? [],
secondControlPlaneRisk: guard?.secondControlPlaneRisk ?? false,
defaultKubectlDiagnostic: guard?.defaultKubectlDiagnostic ?? null,
},
secrets: {
status: secretRefs?.status ?? "unavailable",
valuesRead: false,
valuesPrinted: false,
secretRefs: (secretRefs?.secretRefs || []).map((ref) => ({
secretName: ref.secretName,
secretKey: ref.secretKey,
exists: ref.exists === true,
keyPresent: ref.keyPresent === true,
status: ref.status,
})),
unavailableReason: secretRefs?.reason ?? null,
},
registry: {
status: registry?.status ?? "unavailable",
endpoint: registry?.endpoint ?? "http://127.0.0.1:5000/v2/",
processHttpAccess: registry?.processHttpAccess ? { status: registry.processHttpAccess.status, reachable: registry.processHttpAccess.reachable } : null,
k3sPullAccess: registry?.k3sPullAccess ? {
status: registry.k3sPullAccess.status,
pullFailureCount: registry.k3sPullAccess.pullFailureCount,
pulledRegistryImageCount: registry.k3sPullAccess.pulledRegistryImageCount,
} : null,
},
lease: {
status: lock?.status ?? "unavailable",
phase: lock?.phase ?? null,
holder: lock?.ownerTaskId || lock?.holderIdentity || null,
ageSeconds: lock?.updatedAt && Number.isFinite(Date.parse(lock.updatedAt)) ? Math.max(0, Math.floor((Date.now() - Date.parse(lock.updatedAt)) / 1000)) : null,
retryAfterSeconds: lock?.retryAfterSeconds ?? null,
staleClassification: lock?.stale === true ? "lease-stale-candidate" : lock?.held === true ? "lease-held" : "not-held",
lockName: lock?.lockName ?? lockName,
},
desiredState: {
status: desired?.status ?? "unavailable",
targetCommit: desired?.targetCommit ?? null,
deployJson: desired?.deployJson ?? null,
artifactCatalog: desired?.artifactCatalog ?? null,
artifactReport: desired?.artifactReport ?? null,
imageConvergence: desired?.imageConvergence ?? null,
},
workload: {
status: workload?.status ?? "unavailable",
currentImageConvergence: workload?.currentImageConvergence ?? null,
deploymentCount: workload?.deploymentCount ?? 0,
deployments: (workload?.deployments || []).map((deployment) => ({
name: deployment.name,
serviceId: deployment.serviceId,
image: deployment.image,
imageTag: deployment.imageTag,
desiredImage: deployment.desiredImage,
revision: deployment.revision,
replicas: deployment.replicas,
availableReplicas: deployment.availableReplicas,
rolloutStatus: deployment.rolloutStatus,
commitEnv: deployment.commitEnv,
})),
podWaiting: workload?.podWaiting ?? [],
runtimeJobs: workload?.runtimeJobs ?? [],
},
publicHealth: {
status: publicHealth?.status ?? "unavailable",
expectedCommit: publicHealth?.expectedCommit ?? null,
summary: publicHealth?.summary ?? { checked: 0 },
endpoints: (publicHealth?.endpoints || []).map((endpoint) => ({
id: endpoint.id,
url: endpoint.url,
status: endpoint.status,
reachable: endpoint.reachable,
expectedServiceId: endpoint.expectedServiceId,
expectedCommit: endpoint.expectedCommit,
health: endpoint.health,
})),
},
durability: {
status: durability?.status ?? "unavailable",
configured: durability?.configured ?? null,
live: durability?.live ?? null,
unavailableReason: durability?.unavailableReason ?? null,
},
controlledDevCd: {
status: controlled?.status ?? "unavailable",
commandOk: controlled?.commandOk ?? null,
controlledEntrypoint: "scripts/dev-cd-apply.mjs",
target: controlled?.target ?? null,
blockers: controlled?.blockers ?? [],
},
blockers,
blockerTypes,
dumpPath: dumpDir,
safety: {
mutation: false,
kubectlApplyExecuted: false,
rolloutExecuted: false,
liveVerifyExecuted: false,
cdLockMutated: false,
secretValuesRead: false,
secretValuesPrinted: false,
reportsWritten: false,
repoReportsWritten: false,
},
};
}
function summarizeControlled(parsed, command) {
const record = asRecord(parsed);
const target = asRecord(record?.target);
@@ -552,7 +1117,7 @@ function collectBlockers({ repo, git, guard, lock, secretRefs, controlled, actio
if (guard) {
if (guard.refusal) blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary, refusal: true });
else if (guard.status !== "pass") blockers.push({ scope: "d601-native-k3s-guard", summary: guard.summary });
if ((action === "preflight" || action === "apply") && guard.secondControlPlaneRisk) {
if ((action === "preflight" || action === "audit" || action === "apply") && guard.secondControlPlaneRisk) {
blockers.push({ scope: "second-hwlab-dev-control-plane", summary: "Bare kubectl can observe hwlab-dev through a different control plane; refusing write-path planning." });
}
}
@@ -568,7 +1133,8 @@ function collectBlockers({ repo, git, guard, lock, secretRefs, controlled, actio
}
function nextSafeCommand(action, blockers) {
if (blockers.length > 0) return "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd status --env dev";
if (blockers.length > 0) return action === "audit" ? "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd audit --env dev" : "resolve the structured blockers, then rerun bun scripts/cli.ts hwlab cd status --env dev";
if (action === "audit") return "audit is read-only; host commander may compare blockerTypes before deciding whether the unique CD runner should continue";
if (action === "status") return "bun scripts/cli.ts hwlab cd apply --env dev --dry-run";
return "host commander or the unique CD runner may decide whether to run node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report on D601; this wrapper did not apply or rollout";
}
@@ -603,6 +1169,8 @@ async function main() {
secretValuesPrinted: false,
cdLockMutated: false,
prodTouched: false,
reportsWritten: false,
repoReportsWritten: false,
},
};
@@ -625,17 +1193,15 @@ async function main() {
artifactReport: readJsonSummary(repoPath, "reports/dev-gate/dev-artifacts.json"),
};
const lock = await lockStatus(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000));
const needsSecretPreflight = action === "preflight" || action === "apply";
const needsSecretPreflight = action === "preflight" || action === "audit" || action === "apply";
const secretRefs = needsSecretPreflight
? await secretPreflight(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000))
: { status: "skipped", reason: "status-does-not-run-secret-preflight", namespace, mutation: false, blockers: [] };
const preCommandBlockers = collectBlockers({ repo, git, guard, lock, secretRefs, action });
const canRunControlled = preCommandBlockers.length === 0 || action === "status";
const canRunControlled = preCommandBlockers.length === 0 || action === "status" || action === "audit";
const controlled = canRunControlled && git.status === "pass" && guard.status === "pass"
? await runDevCdApply(repoPath, options.kubeconfig || nativeKubeconfig, action === "status" ? "status" : "apply", dumpDir, Math.max(1000, timeoutMs - 5000))
: { status: "skipped", commandOk: true, reason: "preflight-blockers-before-controlled-dev-cd-apply" };
const blockers = collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action });
const ok = blockers.length === 0;
const target = controlled.target || {
ref: "deploy/deploy.json",
promotionCommit: desiredState.deployJson.commitId || git.headCommit,
@@ -643,6 +1209,38 @@ async function main() {
promotionSource: "deploy/deploy.json",
namespace,
};
if (action === "audit") {
const auditDesiredState = buildDesiredStateAudit(repoPath, target);
const [registry, workload, publicHealth] = await Promise.all([
registryAudit(options.kubeconfig || nativeKubeconfig, guard, dumpDir, Math.min(timeoutMs, 15000)),
workloadAudit(options.kubeconfig || nativeKubeconfig, guard, auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)),
httpHealthAudit(auditDesiredState, dumpDir, Math.min(timeoutMs, 15000)),
]);
const durability = runtimeDurabilityAudit(auditDesiredState, publicHealth, secretRefs, workload);
const summary = auditSummary({ repo, git, guard, secretRefs, registry, lock, desired: auditDesiredState, workload, publicHealth, durability, controlled, dumpDir });
console.log(JSON.stringify({
...base,
ok: summary.ok,
status: summary.status,
audit: summary,
repo: {
status: repo.status,
path: repo.path,
source: repo.source,
rejected: repo.rejected,
},
workspace: { path: repoPath, source: repo.source },
remoteDumpPath: dumpDir,
reportDumpPath: controlled.command?.dump?.stdout || dumpDir,
blockers: summary.blockers,
blockerTypes: summary.blockerTypes,
nextSafeCommand: nextSafeCommand(action, summary.blockers),
}, null, 2));
process.exitCode = summary.ok ? 0 : 1;
return;
}
const blockers = collectBlockers({ repo, git, guard, lock, secretRefs, controlled, action });
const ok = blockers.length === 0;
console.log(JSON.stringify({
...base,
ok,
+6 -4
View File
@@ -6,7 +6,7 @@ import { join } from "node:path";
import { readConfig, repoRoot, rootPath, type UniDeskConfig } from "./config";
import { d601NativeKubeconfig } from "./d601-k3s-guard";
type HwlabCdAction = "status" | "preflight" | "apply";
type HwlabCdAction = "status" | "preflight" | "audit" | "apply";
type HwlabCdEnvironment = "dev";
type HwlabCdTransport = "frontend" | "local";
@@ -89,8 +89,8 @@ function envTransport(): HwlabCdTransport {
function parseOptions(args: string[]): HwlabCdOptions {
const [scope, actionArg] = args;
if (scope !== "cd") throw new Error("hwlab usage: bun scripts/cli.ts hwlab cd status|preflight|apply --env dev");
if (actionArg !== "status" && actionArg !== "preflight" && actionArg !== "apply") throw new Error("hwlab cd usage: status|preflight|apply");
if (scope !== "cd") throw new Error("hwlab usage: bun scripts/cli.ts hwlab cd status|preflight|audit|apply --env dev");
if (actionArg !== "status" && actionArg !== "preflight" && actionArg !== "audit" && actionArg !== "apply") throw new Error("hwlab cd usage: status|preflight|audit|apply");
const options: HwlabCdOptions = {
action: actionArg,
@@ -498,6 +498,7 @@ export function hwlabHelp(): Record<string, unknown> {
output: "json",
usage: [
"bun scripts/cli.ts hwlab cd status --env dev",
"bun scripts/cli.ts hwlab cd audit --env dev",
"bun scripts/cli.ts hwlab cd preflight --env dev",
"bun scripts/cli.ts hwlab cd apply --env dev --dry-run",
],
@@ -508,7 +509,8 @@ export function hwlabHelp(): Record<string, unknown> {
`default HWLAB CD repo is ${defaultHwlabCdRepoPath}; ${rejectedRunnerHistoryRepoPath} is rejected as runner history`,
"deploy/deploy.json remains the authoritative desired-state source",
"preflight/apply --dry-run check required SecretRef object/key metadata without reading or printing Secret values",
"status/preflight/apply --dry-run call only HWLAB scripts/dev-cd-apply.mjs with --skip-live-verify; no apply, rollout, lock mutation, live verification, DB write, or secret read is executed",
"audit/status/preflight/apply --dry-run call only HWLAB scripts/dev-cd-apply.mjs read-only/status paths with --skip-live-verify; no apply, rollout, lock mutation, DEV acceptance live verification, DB write, or secret read is executed",
"audit adds bounded read-only kubectl get/curl health probes for blocker classification; full stdout/stderr stays in temp dump paths, not reports/",
"real apply is structured refused and must remain with the host commander or unique CD runner",
],
};