diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 0975a875..88c10f3c 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -16,7 +16,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 runtime lane 滚动 当现有 CLI 对某个 CI/CD 操作缺字段、缺动作、缺状态或缺权限时,处理顺序是先补 CLI,再执行发布或治理动作。临时低层 route 写操作只允许用于一次性止血,并且必须随后把稳定能力补进 CLI 与本参考文档;不能把手工 `kubectl apply/delete/annotate`、原生 GitHub CLI、手写 REST 请求或 registry shell 脚本沉淀成长期流程。长时观察仍遵守 60 秒短查询和 submit-and-poll 语义,不用单个 `trans`/`tran` 等待完整 PipelineRun 或 Argo rollout 结束。 -`hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 [--dry-run|--confirm]` 是 v03 platform PostgreSQL bridge 的受控收敛入口,用于让 Argo 跟踪固定 `EndpointSlice/g14-platform-postgres-host`,并清理旧 `Endpoints/g14-platform-postgres` 及其派生的随机 EndpointSlice。`hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 [--dry-run|--confirm]` 是 v03 迁移到 G14 platform PostgreSQL 后清理旧 repo-owned Postgres StatefulSet/Service/ConfigMap/Secret/PVC 的受控入口。迁移后的 `hwlab-cloud-api-v03-db`、`hwlab-v03-openfga` 和 `hwpod-v03-db` SecretRef 来自 G14 host platform DB 凭据文件,不再从 lane-local Postgres Secret 派生;v03+ 的 `hwlab nodes secret ensure --name hwlab-cloud-api-v03-db|hwlab-v03-openfga|hwpod-v03-db` 旧路径已删除,status 只做 redacted SecretRef 与 `g14-platform-postgres` Service、固定 EndpointSlice 和旧 Endpoints 缺席观测。平台 DB 运行、SecretRef 轮换边界和 health 验证见 `docs/reference/g14-platform-db.md`。 +`hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 [--dry-run|--confirm]` 是 v03 platform PostgreSQL bridge 的受控收敛入口,用于让 Argo 跟踪固定 `EndpointSlice/g14-platform-postgres-host`,并清理旧 `Endpoints/g14-platform-postgres` 及其派生的随机 EndpointSlice。`hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 [--dry-run|--confirm]` 是 v03 迁移到 G14 platform PostgreSQL 后清理旧 repo-owned Postgres StatefulSet/Service/ConfigMap/Secret/PVC 的受控入口。迁移后的 `hwlab-cloud-api-v03-db` 和 `hwlab-v03-openfga` SecretRef 来自 G14 host platform DB 凭据文件,不再从 lane-local Postgres Secret 派生;`hwpod-v03-db`、`hwpod_v03`、`hwpod_v03_app` 是废弃残留,不是完成态 SecretRef。v03+ 的 `hwlab nodes secret ensure --name hwlab-cloud-api-v03-db|hwlab-v03-openfga` 旧路径已删除,status 只做 redacted SecretRef 与 `g14-platform-postgres` Service、固定 EndpointSlice 和旧 Endpoints 缺席观测;发现 HWPOD DB 残留时使用 `hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db [--dry-run|--confirm]` 受控清理。平台 DB 运行、SecretRef 轮换边界和 health 验证见 `docs/reference/g14-platform-db.md`。 `hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider` 是 v03 Code Agent / MoonBridge provider SecretRef 的受控 bootstrap 入口;`ensure` 只从集群内既有 `hwlab-v02/hwlab-v02-code-agent-provider` 复制 `openai-api-key`、`opencode-api-key` 到 lane-local Secret,输出仅披露 source/target Secret 名、key presence、decoded byte count、mutation 和后续命令,禁止打印 base64、解码值、完整 API key 或可复用凭据。OpenFGA 和 master admin API key 继续使用同一命名空间下的 `hwlab nodes secret ... --name hwlab-v03-openfga|hwlab-v03-master-server-admin-api-key`。 @@ -62,7 +62,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 runtime lane 滚动 创建 PipelineRun 前会读取 `devops-infra` mirror refs,若 `localV02` 未等于当前 source commit,则自动执行一次受控 manual `git-mirror sync` Job 并复核 ref,复核失败时停止触发,避免 Tekton `prepare-source` 已知失败;services 参数只包含 v02 runtime service matrix,`hwlab-cli` 是固定 repo 短连接源码工具,不进入 PipelineRun service build。 `--dry-run` 只报告是否会 pre-sync,不创建 Job;confirmed trigger 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand`、stdout/stderr 路径,避免 git mirror pre-sync 或 PipelineRun 创建期间长时间阻塞;`--wait` 路径也必须向 stderr 输出 `hwlab.v02.trigger.progress` JSON 事件,覆盖 `control-plane-refresh`、`git-mirror-pre-sync` 和 `create-pipelinerun`,避免异步 job 长时间只有启动命令而无法判断卡点;默认 JSON 必须对 `manifest_b64`、长脚本和远端 stdout/stderr 做有界摘要,保留长度与 hash,最终 trigger 结果只返回阶段摘要和关键 tail,完整内容通过 job stdout/stderr 文件渐进披露;只有现场同步调试才显式加 `--wait`;旧 `rerun-current` 只作为输入别名保留。PipelineRun `Completed`、Argo `Synced/Healthy` 和 `webAssets.ok=true` 只证明 G14 runtime 已更新;交付收口还必须用 `hwlab g14 git-mirror status` 查看 `cache.summary.pendingFlush`,若为 true,继续执行受控 `hwlab g14 git-mirror flush --confirm` 并用 job status 轮询到 `pendingFlush=false`。 - `hwlab g14 control-plane runtime-migration --lane v02 [--dry-run|--allow-live-db-read --dry-run|--confirm]` 只通过 `hwlab-v02` namespace 当前 `deployment/hwlab-cloud-api -c hwlab-cloud-api` 内 repo-owned migration CLI 执行;不读取或打印 Secret 值、不触碰 PROD、不绕到手工 `psql`。 -- `hwlab g14 secret status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm]` 和 `hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key [--dry-run|--confirm]` 是 HWLAB runtime lane SecretRef bootstrap 的保留入口。v03+ Cloud API/OpenFGA/HWPOD datastore SecretRef 已迁移到 G14 platform PostgreSQL,`hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db|hwlab-v03-openfga|hwpod-v03-db` 只做 redacted SecretRef 与 `g14-platform-postgres` bridge 观测;旧 `ensure` 路径已删除,不再从 `hwlab-v03-postgres` Secret 或 StatefulSet 派生。平台库凭据、桥接 Service 和 SecretRef 轮换边界见 `docs/reference/g14-platform-db.md`。master server admin API key preset 确保本机 `/root/.config/hwlab-v0x/master-server-admin-api-key.env` 以 0600 保存 `HWLAB_API_KEY`,并同步到对应 lane 的 `*-master-server-admin-api-key/api-key`。`status` 只返回 key 是否存在、解码后字节数、key prefix、bridge 存在性和 runtime health 相关结果,永远不读取、不打印、不回传 secret 明文。`hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 [--dry-run|--confirm]` 是 v0.3+ 迁移到 G14 平台 Postgres 后的受控残留清理入口,精确删除旧 repo-owned `hwlab-v03-postgres` StatefulSet/Service/ConfigMap/Secret 和 `data-hwlab-v03-postgres-0` PVC;它要求 `g14-platform-postgres` Service 已存在,默认 dry-run,不触碰平台数据库、OpenFGA/Cloud API/HWPOD 当前 SecretRef 或 GitOps desired state。`hwlab g14 secret delete --lane v02 --name [--dry-run|--confirm]` 只用于删除确认已不被 workload 引用的 v0.2 废弃 Secret,默认 dry-run,拒绝删除 OpenFGA/Postgres/master admin API key 等必需 Secret;共享 device-pod API key 已退出当前授权路径,不再提供 ensure/bootstrap 入口。 +- `hwlab g14 secret status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm]` 和 `hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key [--dry-run|--confirm]` 是 HWLAB runtime lane SecretRef bootstrap 的保留入口。v03+ Cloud API/OpenFGA datastore SecretRef 已迁移到 G14 platform PostgreSQL,`hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db|hwlab-v03-openfga` 只做 redacted SecretRef 与 `g14-platform-postgres` bridge 观测;旧 `ensure` 路径已删除,不再从 `hwlab-v03-postgres` Secret 或 StatefulSet 派生。`hwpod-v03-db`、`hwpod_v03`、`hwpod_v03_app` 是废弃残留,不能作为 status 完成态保留,发现后用 `hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db [--dry-run|--confirm]` 清理。平台库凭据、桥接 Service 和 SecretRef 轮换边界见 `docs/reference/g14-platform-db.md`。master server admin API key preset 确保本机 `/root/.config/hwlab-v0x/master-server-admin-api-key.env` 以 0600 保存 `HWLAB_API_KEY`,并同步到对应 lane 的 `*-master-server-admin-api-key/api-key`。`status` 只返回 key 是否存在、解码后字节数、key prefix、bridge 存在性和 runtime health 相关结果,永远不读取、不打印、不回传 secret 明文。`hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 [--dry-run|--confirm]` 是 v0.3+ 迁移到 G14 平台 Postgres 后的受控残留清理入口,精确删除旧 repo-owned `hwlab-v03-postgres` StatefulSet/Service/ConfigMap/Secret 和 `data-hwlab-v03-postgres-0` PVC;它要求 `g14-platform-postgres` Service 已存在,默认 dry-run,不触碰平台数据库、OpenFGA/Cloud API 当前 SecretRef 或 GitOps desired state。`hwlab g14 secret delete --lane v02 --name [--dry-run|--confirm]` 只用于删除确认已不被 workload 引用的 v0.2 废弃 Secret,默认 dry-run,拒绝删除 OpenFGA/Postgres/master admin API key 等必需 Secret;共享 device-pod API key 已退出当前授权路径,不再提供 ensure/bootstrap 入口。 - `hwlab g14 control-plane cleanup-runs --lane v02|v03|g14|all [--min-age-minutes N] [--limit N] [--dry-run|--confirm]` 是完成态 PipelineRun 工作区 retention 入口;真实清理只删除已完成 PipelineRun,让 Tekton/local-path 回收临时 PVC,不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。带 `--pipeline-run ` 或 `--source-commit ` 的定点清理必须先直接查询目标 PipelineRun,而不是只从全量列表过滤;不存在的目标返回 `target-pipelinerun-not-found`,未完成目标返回 `target-pipelinerun-not-terminal`,空查询和读取失败分别返回 `target-pipelinerun-query-empty` / `target-pipelinerun-query-failed`,年龄保护仍返回 `below-min-age`。`hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run ` 是 v0.3 failed run 受控重试前的清理入口。 - `hwlab g14 control-plane cleanup-released-pvs --lane all [--limit N] [--dry-run|--confirm]` 是 local-path 未自动回收后的补充 retention 入口;只列并删除 `Released`、`local-path`、`Delete`、`claimNamespace=hwlab-ci` 且 claim 名称形如 Tekton 临时 `pvc-*` 的 PV。 - `hwlab g14 git-mirror status|apply|sync|flush [--dry-run|--confirm]` 是 `devops-infra` git mirror/relay 的受控维护入口:`apply` 渲染并 server-side apply `devops-infra/git-mirror.yaml`,同时删除遗留 `git-mirror-hwlab-sync` CronJob;`sync` 创建一次性 manual Job,把 GitHub allowlist refs 拉入本地 mirror;`flush` 创建一次性 manual Job,把本地 `v0.2-gitops` 快进推回 GitHub。 diff --git a/docs/reference/g14-platform-db.md b/docs/reference/g14-platform-db.md index a7b3d2a9..b82d771a 100644 --- a/docs/reference/g14-platform-db.md +++ b/docs/reference/g14-platform-db.md @@ -4,7 +4,7 @@ ## 定位 -G14 platform DB 是 G14 host OS 上的原生 PostgreSQL,不是 k3s workload,也不属于 `devops-infra` namespace。它位于 HWLAB runtime lane、AgentRun、HWPOD 和未来 G14 平台控制能力之下,作为节点级基础设施承载 durable runtime、OpenFGA datastore、平台控制元数据、租约、审计和状态观测。 +G14 platform DB 是 G14 host OS 上的原生 PostgreSQL,不是 k3s workload,也不属于 `devops-infra` namespace。它位于 HWLAB runtime lane、AgentRun 和未来 G14 平台控制能力之下,作为节点级基础设施承载 durable runtime、OpenFGA datastore、平台控制元数据、租约、审计和状态观测。 数据库可以保存一部分 k3s 管理所需的状态,例如节点 inventory、runtime lane registry、operation lease、k3s state observation 和 audit event;它不直接执行 `kubectl`、Argo、Tekton 或 GitHub 写操作。k3s 写操作仍必须通过 UniDesk 受控 CLI 或对应控制器执行,数据库只作为状态与协调底座。 @@ -39,10 +39,11 @@ PostgreSQL 只监听 G14 host loopback 与 k3s pod 可达的 node gateway 地址 - `g14_platform_control` / `g14_platform_app`:G14 节点平台控制元数据、租约、观测和审计。 - `hwlab_v03` / `hwlab_v03_app`:HWLAB v0.3 cloud-api durable runtime。 - `openfga_v03` / `openfga_v03_app`:HWLAB v0.3 OpenFGA datastore。 -- `hwpod_v03` / `hwpod_v03_app`:HWPOD v0.3 迁移后的平台库目标;HWPOD 不应继续拥有自带 DB。 `postgres` 只作为管理数据库保留。新增 runtime lane 时按同一模式分配独立 database/role,不复用业务应用账号访问 `g14_platform_control`。 +HWPOD v0.3 不再有当前 platform DB 目标;`hwpod-v03-db`、`hwpod_v03` 和 `hwpod_v03_app` 出现时只能视为废弃残留,不能作为 SecretRef 完成态或平台库目标保留。 + ## k3s 桥接 业务 Pod 不直接访问 host IP 字面量,而是访问所在 namespace 内的 `g14-platform-postgres` Service。HWLAB v0.3 的桥接对象在 `hwlab-v03` namespace 中: @@ -56,9 +57,8 @@ PostgreSQL 只监听 G14 host loopback 与 k3s pod 可达的 node gateway 地址 - `hwlab-cloud-api-v03-db/database-url` 指向 `hwlab_v03`。 - `hwlab-v03-openfga/datastore-uri` 指向 `openfga_v03`。 -- `hwpod-v03-db/database-url` 指向 `hwpod_v03`。 -这些 SecretRef 的源凭据来自 G14 host 的 platform DB credentials file,不再从旧 lane-local Postgres Secret 派生。迁移完成后,旧的 repo-owned PostgreSQL bootstrap/ensure 路径已经删除,不能用于轮换 `hwlab-cloud-api-v03-db` 或 `hwlab-v03-openfga`。如需重复轮换,必须先补齐平台 DB 专用 UniDesk CLI,让它从 host 凭据文件读取、写入 k3s Secret,并输出 redacted 状态;不得把手写 `kubectl create secret` 或旧 StatefulSet ensure 路径沉淀成长期流程。 +这些 SecretRef 的源凭据来自 G14 host 的 platform DB credentials file,不再从旧 lane-local Postgres Secret 派生。`hwpod-v03-db/database-url` 是废弃残留,应该缺席;发现它时用受控清理入口删除,不能为它补 status 可见性或轮换路径。迁移完成后,旧的 repo-owned PostgreSQL bootstrap/ensure 路径已经删除,不能用于轮换 `hwlab-cloud-api-v03-db` 或 `hwlab-v03-openfga`。如需重复轮换,必须先补齐平台 DB 专用 UniDesk CLI,让它从 host 凭据文件读取、写入 k3s Secret,并输出 redacted 状态;不得把手写 `kubectl create secret` 或旧 StatefulSet ensure 路径沉淀成长期流程。 当前运行面验证使用 control-plane status 和 live health: @@ -118,6 +118,15 @@ curl -fsS http://74.48.78.17:20667/health/live | jq '{status, ready, environment 其中 `endpoints/g14-platform-postgres` 应为空,`endpointslice/g14-platform-postgres-host` 应存在且是唯一带 `kubernetes.io/service-name=g14-platform-postgres` label 的 EndpointSlice。 +HWPOD v0.3 DB 残留不属于旧自有 HWLAB Postgres StatefulSet 清理目标,必须用单独的废弃对象清理入口处理: + +```bash +bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run +bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm +``` + +该入口只允许清理已确认不被 workload 引用的 `hwpod-v03-db` k3s Secret,并同步清理 G14 host PostgreSQL 上同名废弃目标对应的 `hwpod_v03` database 与 `hwpod_v03_app` role;输出只披露对象是否存在、是否删除和退出码,不打印 URI、密码或可复用凭据。清理后 `hwpod-v03-db`、`hwpod_v03`、`hwpod_v03_app` 都应缺席。 + ## 备份与恢复 备份脚本固定在 G14 host: diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index 5f9d92da..ef188d46 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -164,17 +164,14 @@ assertCondition( hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider")) && hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm")) && hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db")) - && hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret status --node G14 --lane v03 --name hwpod-v03-db")) && hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run")) && hwlabNodeHelpJson.includes("hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider") && hwlabNodeHelpJson.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm") && hwlabNodeHelpJson.includes("hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db") - && hwlabNodeHelpJson.includes("hwlab nodes secret status --node G14 --lane v03 --name hwpod-v03-db") && hwlabNodeHelpJson.includes("hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run") && !hwlabNodeHelpJson.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-cloud-api-v03-db --confirm") - && !hwlabNodeHelpJson.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwpod-v03-db --confirm") && !hwlabNodeHelpJson.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --confirm"), - "v0.3 node-scoped secret help must expose provider bootstrap plus platform DB status/cleanup without old DB ensure paths", + "v0.3 node-scoped secret help must expose provider bootstrap plus Cloud API platform DB status and old DB cleanup without old DB ensure paths", { hwlabHelpUsage, hwlabNodeHelp: hwlabNodeHelp() }, ); const cloudApiDbStatus = nodeSecretStatusFromTextForTest([ @@ -222,48 +219,6 @@ assertCondition( "cloud-api DB Secret status must be redacted while proving native platform DB SecretRef and bridge are present", cloudApiDbStatus, ); -const hwpodDbStatus = nodeSecretStatusFromTextForTest([ - "namespace\thwlab-v03", - "secret\thwpod-v03-db", - "key\tdatabase-url", - "preset\thwpod-db", - "action\tobserved", - "dryRun\ttrue", - "mutation\tfalse", - "platformDbMode\ttrue", - "afterExists\tyes", - "afterDatabaseUrlPresent\tyes", - "afterDatabaseUrlBytes\t172", - "legacyPostgresSecret\thwlab-v03-postgres", - "legacyPostgresSecretExists\tno", - "platformService\tg14-platform-postgres", - "platformServiceExists\tyes", - "platformEndpointsExists\tno", - "platformEndpointSlice\tg14-platform-postgres-host", - "platformEndpointSliceExists\tyes", - "dbName\thwpod_v03", - "dbUser\thwpod_v03_app", - "dbHost\tg14-platform-postgres.hwlab-v03.svc.cluster.local", - "dbHostMatchesPlatform\tyes", - "dbNameMatchesExpected\tyes", - "dbUserMatchesExpected\tyes", -].join("\n"), true, 0, ""); -assertCondition( - record(hwpodDbStatus).ok === true - && record(hwpodDbStatus).valuesRedacted === true - && record(hwpodDbStatus).platformDbMode === true - && record(record(hwpodDbStatus).legacyPostgresSecret).exists === false - && record(record(hwpodDbStatus).platformService).endpointsExist === false - && record(record(hwpodDbStatus).platformService).legacyEndpointsAbsent === true - && record(record(hwpodDbStatus).platformService).endpointSlice === "g14-platform-postgres-host" - && record(hwpodDbStatus).dbName === "hwpod_v03" - && record(hwpodDbStatus).dbUser === "hwpod_v03_app" - && record(hwpodDbStatus).dbHost === "g14-platform-postgres.hwlab-v03.svc.cluster.local" - && !JSON.stringify(hwpodDbStatus).includes("postgres://") - && !JSON.stringify(hwpodDbStatus).includes("password"), - "HWPOD DB Secret status must prove the v0.3 platform DB SecretRef without exposing database-url material", - hwpodDbStatus, -); const openFgaPlatformStatus = nodeSecretStatusFromTextForTest([ "namespace\thwlab-v03", "secret\thwlab-v03-openfga", @@ -316,13 +271,6 @@ assertCondition( "v0.3 cloud-api DB ensure must reject the removed lane-local Postgres path", removedCloudApiDbEnsure, ); -const removedHwpodDbEnsure = runCommand(["bun", "scripts/cli.ts", "hwlab", "nodes", "secret", "ensure", "--node", "G14", "--lane", "v03", "--name", "hwpod-v03-db", "--dry-run"], process.cwd(), { timeoutMs: 30_000 }); -assertCondition( - removedHwpodDbEnsure.exitCode !== 0 - && `${removedHwpodDbEnsure.stdout}\n${removedHwpodDbEnsure.stderr}`.includes("was removed after native platform DB migration"), - "v0.3 HWPOD DB ensure must reject the removed lane-local Postgres path", - removedHwpodDbEnsure, -); const removedOpenFgaEnsure = runCommand(["bun", "scripts/cli.ts", "hwlab", "nodes", "secret", "ensure", "--node", "G14", "--lane", "v03", "--name", "hwlab-v03-openfga", "--dry-run"], process.cwd(), { timeoutMs: 30_000 }); assertCondition( removedOpenFgaEnsure.exitCode !== 0 diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 5a79ec81..46a54c67 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -10221,9 +10221,10 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db", - "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwpod-v03-db", "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run", + "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm", "bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name --dry-run", diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index 421d52ca..ec8001e8 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -5,8 +5,8 @@ import { startJob } from "./jobs"; import { runHwlabG14Command } from "./hwlab-g14"; import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane } from "./hwlab-node-lanes"; -type SecretAction = "status" | "ensure" | "cleanup-owned-postgres"; -type SecretPreset = "openfga" | "master-server-admin-api-key" | "code-agent-provider" | "cloud-api-db" | "hwpod-db" | "owned-postgres-cleanup"; +type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete"; +type SecretPreset = "openfga" | "master-server-admin-api-key" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup"; type DelegatedNodeDomain = "control-plane" | "git-mirror"; interface NodeSecretOptions { @@ -41,11 +41,9 @@ interface RuntimeSecretSpec { cloudApiDbUser: string; cloudApiDbHost: string; cloudApiDeployment: string; - hwpodDbSecret: string; - hwpodDbKey: string; - hwpodDbName: string; - hwpodDbUser: string; - hwpodDbHost: string; + obsoleteHwpodDbSecret: string; + obsoleteHwpodDbName: string; + obsoleteHwpodDbUser: string; codeAgentProviderSecret: string; codeAgentProviderSourceNamespace: string; codeAgentProviderSourceSecret: string; @@ -92,9 +90,10 @@ export function hwlabNodeHelp(): Record { "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db", - "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwpod-v03-db", "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run", "bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run", + "bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm", ], @@ -215,8 +214,8 @@ function rewriteDelegatedNodeString(value: string, scoped: ReturnType { const spec = runtimeSecretSpec(options); + if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec); const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun ? readMasterAdminApiKey().key : ""; @@ -377,7 +372,7 @@ function runNodeSecret(options: NodeSecretOptions): Record { ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec) : options.preset === "master-server-admin-api-key" ? masterAdminApiKeySecretScript(options, spec) - : options.preset === "cloud-api-db" || options.preset === "hwpod-db" + : options.preset === "cloud-api-db" ? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec) : options.preset === "owned-postgres-cleanup" ? ownedPostgresCleanupScript(options, spec) @@ -386,7 +381,8 @@ function runNodeSecret(options: NodeSecretOptions): Record { const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec); const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0; const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0; - const ok = dryRunOk || cleanupDryRunOk ? true : status.ok === true; + const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true; + const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true; return { ok, command: `hwlab nodes secret ${options.action}`, @@ -396,7 +392,7 @@ function runNodeSecret(options: NodeSecretOptions): Record { secret: options.name, key: options.key ?? null, preset: options.preset, - mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" ? "confirmed-delete" : "confirmed-ensure", + mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure", status, mutation: status.mutation === true, result: compactCommandResult(result), @@ -409,7 +405,10 @@ function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec): if (options.action === "cleanup-owned-postgres") { return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` }; } - if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga" || options.preset === "hwpod-db")) { + if (options.action === "cleanup-obsolete") { + return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` }; + } + if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) { return { status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`, controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`, @@ -422,6 +421,50 @@ function runTransScript(node: string, script: string, input: string, timeoutSeco return runCommand(["/root/.local/bin/trans", `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); } +function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult { + return runCommand(["/root/.local/bin/trans", node, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 }); +} + +function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record { + const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds); + const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec); + const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true }; + const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds); + const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec); + const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false); + const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true; + return { + ok, + command: `hwlab nodes secret ${options.action}`, + node: options.node, + lane: options.lane, + namespace: spec.namespace, + secret: options.name, + key: null, + preset: options.preset, + mode: options.dryRun ? "dry-run" : "confirmed-delete", + status: { + ok, + preset: "obsolete-hwpod-db-cleanup", + dryRun: options.dryRun, + mutation, + kubernetesSecret: kubernetesStatus, + platformDatabase: platformDbStatus, + hostMutationSkipped: !options.dryRun && hostOptions.dryRun, + summary: ok + ? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove` + : `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`, + }, + mutation, + result: { + kubernetesSecret: compactCommandResult(kubernetesResult), + platformDatabase: compactCommandResult(platformDbResult), + }, + valuesRedacted: true, + next: ok ? undefined : nextSecretCommand(options, spec), + }; +} + function runNodeEndpointBridge(options: ReturnType): Record { if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm"); const dryRun = options.dryRun || !options.confirm; @@ -796,15 +839,156 @@ function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSec ].join("\n"); } +function obsoleteSecretCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `namespace=${shellQuote(spec.namespace)}`, + `secret=${shellQuote(options.name)}`, + `expected_secret=${shellQuote(spec.obsoleteHwpodDbSecret)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + "preset=obsolete-secret-cleanup", + "exists_flag() { kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && printf yes || printf no; }", + "workload_refs=$(kubectl -n \"$namespace\" get deploy,statefulset,daemonset,job,cronjob -o yaml 2>/dev/null | grep -n -C 2 \"$secret\" || true)", + "workload_refs_present=$([ -n \"$workload_refs\" ] && printf yes || printf no)", + "workload_refs_preview=$(printf '%s' \"$workload_refs\" | sed -n '1,20p' | tr '\\n' ';' | cut -c1-1000)", + "before_exists=$(exists_flag)", + "action=observed", + "mutation=false", + "delete_secret_exit=", + "if [ \"$secret\" != \"$expected_secret\" ]; then action=unsupported-secret; fi", + "if [ \"$action\" = observed ]; then", + " if [ \"$workload_refs_present\" = yes ]; then", + " action=blocked-referenced", + " elif [ \"$dry_run\" = true ]; then", + " if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi", + " else", + " kubectl -n \"$namespace\" delete secret \"$secret\" --ignore-not-found=true >/tmp/hwlab-obsolete-secret-delete.out 2>/tmp/hwlab-obsolete-secret-delete.err", + " delete_secret_exit=$?", + " for _ in $(seq 1 15); do", + " current_exists=$(exists_flag)", + " if [ \"$current_exists\" != yes ]; then break; fi", + " sleep 1", + " done", + " if [ \"$delete_secret_exit\" -eq 0 ]; then", + " if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi", + " else", + " action=delete-failed", + " fi", + " fi", + "fi", + "after_exists=$(exists_flag)", + "printf 'namespace\\t%s\\n' \"$namespace\"", + "printf 'secret\\t%s\\n' \"$secret\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeSecretExists\\t%s\\n' \"$before_exists\"", + "printf 'afterSecretExists\\t%s\\n' \"$after_exists\"", + "printf 'workloadRefsPresent\\t%s\\n' \"$workload_refs_present\"", + "printf 'workloadRefsPreview\\t%s\\n' \"$workload_refs_preview\"", + "printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"", + "if [ \"$action\" = unsupported-secret ]; then exit 43; fi", + "if [ \"$workload_refs_present\" = yes ]; then exit 46; fi", + "if [ \"$dry_run\" != true ] && [ \"$after_exists\" = yes ]; then exit 47; fi", + "if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi", + ].join("\n"); +} + +function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { + return [ + "set +e", + `db_name=${shellQuote(spec.obsoleteHwpodDbName)}`, + `db_user=${shellQuote(spec.obsoleteHwpodDbUser)}`, + `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, + "preset=obsolete-platform-db-cleanup", + "database_exists_flag() {", + " output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_database where datname='$db_name');\" 2>/tmp/hwlab-obsolete-platform-db-probe.err)", + " code=$?", + " if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi", + " [ \"$output\" = t ] && printf yes || printf no", + "}", + "role_exists_flag() {", + " output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_roles where rolname='$db_user');\" 2>/tmp/hwlab-obsolete-platform-role-probe.err)", + " code=$?", + " if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi", + " [ \"$output\" = t ] && printf yes || printf no", + "}", + "before_database_exists=$(database_exists_flag)", + "before_database_probe_exit=$?", + "before_role_exists=$(role_exists_flag)", + "before_role_probe_exit=$?", + "action=observed", + "mutation=false", + "drop_database_exit=", + "drop_role_exit=", + "before_any=false", + "if [ \"$before_database_exists\" = yes ] || [ \"$before_role_exists\" = yes ]; then before_any=true; fi", + "if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ]; then", + " action=probe-failed", + "elif [ \"$dry_run\" = true ]; then", + " if [ \"$before_any\" = true ]; then action=would-drop; else action=already-absent; fi", + "else", + " if [ \"$before_database_exists\" = yes ]; then", + " sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_name=\"$db_name\" >/tmp/hwlab-obsolete-platform-db-drop.out 2>/tmp/hwlab-obsolete-platform-db-drop.err <<'SQL'", + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db_name' AND pid <> pg_backend_pid();", + "DROP DATABASE IF EXISTS :\"db_name\";", + "SQL", + " drop_database_exit=$?", + " else", + " drop_database_exit=0", + " fi", + " if [ \"$drop_database_exit\" -eq 0 ] && [ \"$before_role_exists\" = yes ]; then", + " sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_user=\"$db_user\" >/tmp/hwlab-obsolete-platform-role-drop.out 2>/tmp/hwlab-obsolete-platform-role-drop.err <<'SQL'", + "DROP ROLE IF EXISTS :\"db_user\";", + "SQL", + " drop_role_exit=$?", + " elif [ \"$drop_database_exit\" -eq 0 ]; then", + " drop_role_exit=0", + " else", + " drop_role_exit=", + " fi", + " if [ \"$drop_database_exit\" = 0 ] && [ \"$drop_role_exit\" = 0 ]; then", + " if [ \"$before_any\" = true ]; then action=dropped; mutation=true; else action=already-absent; fi", + " else", + " action=drop-failed", + " fi", + "fi", + "after_database_exists=$(database_exists_flag)", + "after_database_probe_exit=$?", + "after_role_exists=$(role_exists_flag)", + "after_role_probe_exit=$?", + "printf 'database\\t%s\\n' \"$db_name\"", + "printf 'role\\t%s\\n' \"$db_user\"", + "printf 'preset\\t%s\\n' \"$preset\"", + "printf 'action\\t%s\\n' \"$action\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'beforeDatabaseExists\\t%s\\n' \"$before_database_exists\"", + "printf 'beforeRoleExists\\t%s\\n' \"$before_role_exists\"", + "printf 'afterDatabaseExists\\t%s\\n' \"$after_database_exists\"", + "printf 'afterRoleExists\\t%s\\n' \"$after_role_exists\"", + "printf 'beforeDatabaseProbeExitCode\\t%s\\n' \"$before_database_probe_exit\"", + "printf 'beforeRoleProbeExitCode\\t%s\\n' \"$before_role_probe_exit\"", + "printf 'afterDatabaseProbeExitCode\\t%s\\n' \"$after_database_probe_exit\"", + "printf 'afterRoleProbeExitCode\\t%s\\n' \"$after_role_probe_exit\"", + "printf 'dropDatabaseExitCode\\t%s\\n' \"$drop_database_exit\"", + "printf 'dropRoleExitCode\\t%s\\n' \"$drop_role_exit\"", + "if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ] || [ \"$after_database_exists\" = unknown ] || [ \"$after_role_exists\" = unknown ]; then exit 49; fi", + "if [ \"$dry_run\" != true ] && { [ \"$after_database_exists\" = yes ] || [ \"$after_role_exists\" = yes ]; }; then exit 50; fi", + "if [ -n \"$drop_database_exit\" ] && [ \"$drop_database_exit\" != 0 ]; then exit \"$drop_database_exit\"; fi", + "if [ -n \"$drop_role_exit\" ] && [ \"$drop_role_exit\" != 0 ]; then exit \"$drop_role_exit\"; fi", + ].join("\n"); +} + function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string { const isOpenFga = options.preset === "openfga"; - const isHwpodDb = options.preset === "hwpod-db"; const platformEndpointSlice = `${spec.platformPostgresService}-host`; return [ "set +e", `namespace=${shellQuote(spec.namespace)}`, - `name=${shellQuote(isOpenFga ? spec.openFgaSecret : isHwpodDb ? spec.hwpodDbSecret : spec.cloudApiDbSecret)}`, - `database_url_key=${shellQuote(isOpenFga ? OPENFGA_DATASTORE_URI_KEY : isHwpodDb ? spec.hwpodDbKey : spec.cloudApiDbKey)}`, + `name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`, + `database_url_key=${shellQuote(isOpenFga ? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey)}`, `authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`, `postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`, `legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`, @@ -812,9 +996,9 @@ function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeS `platform_endpointslice=${shellQuote(platformEndpointSlice)}`, `platform_host=${shellQuote(spec.platformPostgresService)}`, `platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`, - `db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : isHwpodDb ? spec.hwpodDbName : spec.cloudApiDbName)}`, - `db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : isHwpodDb ? spec.hwpodDbUser : spec.cloudApiDbUser)}`, - `db_host=${shellQuote(isOpenFga ? spec.openFgaDbHost : isHwpodDb ? spec.hwpodDbHost : spec.cloudApiDbHost)}`, + `db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`, + `db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`, + `db_host=${shellQuote(isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost)}`, `selected_key=${shellQuote(options.key ?? "")}`, `preset=${shellQuote(options.preset)}`, "dry_run=true", @@ -1399,6 +1583,37 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number : `owned Postgres resources still exist in ${fields.namespace || spec.namespace}`, }; } + if (fields.preset === "obsolete-secret-cleanup") { + const absent = fields.afterSecretExists !== "yes"; + const refsAbsent = fields.workloadRefsPresent !== "yes"; + const dryRun = fields.dryRun === "true"; + return { + ok: commandOk && refsAbsent && (dryRun || absent), + namespace: fields.namespace || spec.namespace, + secret: fields.secret || spec.obsoleteHwpodDbSecret, + preset: "obsolete-secret-cleanup", + action: fields.action || null, + dryRun, + mutation: fields.mutation === "true", + before: { + secretExists: fields.beforeSecretExists === "yes", + }, + after: { + secretExists: fields.afterSecretExists === "yes", + }, + workloadRefs: { + present: fields.workloadRefsPresent === "yes", + preview: fields.workloadRefsPreview || "", + }, + deleteSecretExitCode: numericField(fields.deleteSecretExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: refsAbsent && (dryRun || absent) + ? `${fields.secret || spec.obsoleteHwpodDbSecret} is unreferenced${dryRun ? "" : " and absent"}` + : `${fields.secret || spec.obsoleteHwpodDbSecret} still present or referenced`, + }; + } if (fields.preset === "master-server-admin-api-key") { const afterBytes = numericField(fields.afterApiKeyBytes); const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0; @@ -1462,13 +1677,7 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number summary: healthy ? `${fields.secret || spec.codeAgentProviderSecret} has a usable provider key` : `${fields.secret || spec.codeAgentProviderSecret} missing provider keys`, }; } - if (fields.preset === "cloud-api-db" || fields.preset === "hwpod-db") { - const hwpodDb = fields.preset === "hwpod-db"; - const expectedSecret = hwpodDb ? spec.hwpodDbSecret : spec.cloudApiDbSecret; - const expectedKey = hwpodDb ? spec.hwpodDbKey : spec.cloudApiDbKey; - const expectedDbName = hwpodDb ? spec.hwpodDbName : spec.cloudApiDbName; - const expectedDbUser = hwpodDb ? spec.hwpodDbUser : spec.cloudApiDbUser; - const expectedDbHost = hwpodDb ? spec.hwpodDbHost : spec.cloudApiDbHost; + if (fields.preset === "cloud-api-db") { const beforeUrlBytes = numericField(fields.beforeDatabaseUrlBytes); const afterUrlBytes = numericField(fields.afterDatabaseUrlBytes); if (fields.platformDbMode === "true") { @@ -1485,9 +1694,9 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, - secret: fields.secret || expectedSecret, - key: fields.key || expectedKey, - preset: hwpodDb ? "hwpod-db" : "cloud-api-db", + secret: fields.secret || spec.cloudApiDbSecret, + key: fields.key || spec.cloudApiDbKey, + preset: "cloud-api-db", action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", @@ -1508,9 +1717,9 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`, endpointSliceExists: fields.platformEndpointSliceExists === "yes", }, - dbName: fields.dbName || expectedDbName, - dbUser: fields.dbUser || expectedDbUser, - dbHost: fields.dbHost || expectedDbHost, + dbName: fields.dbName || spec.cloudApiDbName, + dbUser: fields.dbUser || spec.cloudApiDbUser, + dbHost: fields.dbHost || spec.cloudApiDbHost, dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes", dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes", dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes", @@ -1518,24 +1727,8 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, summary: healthy - ? `${fields.secret || expectedSecret}/${fields.key || expectedKey} points to ${fields.platformService || spec.platformPostgresService}` - : `${fields.secret || expectedSecret}/${fields.key || expectedKey} is not aligned to platform DB`, - }; - } - if (hwpodDb) { - return { - ok: false, - namespace: fields.namespace || spec.namespace, - secret: fields.secret || expectedSecret, - key: fields.key || expectedKey, - preset: "hwpod-db", - action: fields.action || null, - dryRun: fields.dryRun === "true", - mutation: false, - exitCode, - stderr: commandOk ? "" : stderr.trim().slice(0, 2000), - valuesRedacted: true, - summary: "HWPOD DB Secret status is only supported for native platform DB lanes", + ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.platformService || spec.platformPostgresService}` + : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} is not aligned to platform DB`, }; } const keysHealthy = fields.afterExists === "yes" && @@ -1546,8 +1739,8 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number return { ok: commandOk && healthy, namespace: fields.namespace || spec.namespace, - secret: fields.secret || expectedSecret, - key: fields.key || expectedKey, + secret: fields.secret || spec.cloudApiDbSecret, + key: fields.key || spec.cloudApiDbKey, preset: "cloud-api-db", action: fields.action || null, dryRun: fields.dryRun === "true", @@ -1573,9 +1766,9 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number }, postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes", postgresSecret: fields.postgresSecret || spec.postgresSecret, - dbName: fields.dbName || expectedDbName, - dbUser: fields.dbUser || expectedDbUser, - dbHost: fields.dbHost || expectedDbHost, + dbName: fields.dbName || spec.cloudApiDbName, + dbUser: fields.dbUser || spec.cloudApiDbUser, + dbHost: fields.dbHost || spec.cloudApiDbHost, cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment, applyExitCode: numericField(fields.applyExitCode), dbEnsureExitCode: numericField(fields.dbEnsureExitCode), @@ -1584,7 +1777,7 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number exitCode, stderr: commandOk ? "" : stderr.trim().slice(0, 2000), valuesRedacted: true, - summary: healthy ? `${fields.secret || expectedSecret}/${fields.key || expectedKey} exists and runtime database is present` : `${fields.secret || expectedSecret}/${fields.key || expectedKey} or runtime database missing`, + summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} exists and runtime database is present` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`, }; } const afterAuthnBytes = numericField(fields.afterAuthnBytes); @@ -1681,6 +1874,50 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number }; } +function obsoletePlatformDbStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record { + const fields = keyValueLinesFromText(text); + const dryRun = fields.dryRun === "true"; + const databaseAbsent = fields.afterDatabaseExists !== "yes" && fields.afterDatabaseExists !== "unknown"; + const roleAbsent = fields.afterRoleExists !== "yes" && fields.afterRoleExists !== "unknown"; + const probesOk = fields.beforeDatabaseExists !== "unknown" && + fields.beforeRoleExists !== "unknown" && + fields.afterDatabaseExists !== "unknown" && + fields.afterRoleExists !== "unknown"; + return { + ok: commandOk && probesOk && (dryRun || (databaseAbsent && roleAbsent)), + database: fields.database || spec.obsoleteHwpodDbName, + role: fields.role || spec.obsoleteHwpodDbUser, + preset: "obsolete-platform-db-cleanup", + action: fields.action || null, + dryRun, + mutation: fields.mutation === "true", + before: { + databaseExists: fields.beforeDatabaseExists === "yes", + roleExists: fields.beforeRoleExists === "yes", + }, + after: { + databaseExists: fields.afterDatabaseExists === "yes", + roleExists: fields.afterRoleExists === "yes", + }, + beforeProbeExitCode: { + database: numericField(fields.beforeDatabaseProbeExitCode), + role: numericField(fields.beforeRoleProbeExitCode), + }, + afterProbeExitCode: { + database: numericField(fields.afterDatabaseProbeExitCode), + role: numericField(fields.afterRoleProbeExitCode), + }, + dropDatabaseExitCode: numericField(fields.dropDatabaseExitCode), + dropRoleExitCode: numericField(fields.dropRoleExitCode), + exitCode, + stderr: commandOk ? "" : stderr.trim().slice(0, 2000), + valuesRedacted: true, + summary: probesOk && (dryRun || (databaseAbsent && roleAbsent)) + ? `${fields.database || spec.obsoleteHwpodDbName} and ${fields.role || spec.obsoleteHwpodDbUser} are ${dryRun ? "observable" : "absent"}` + : `${fields.database || spec.obsoleteHwpodDbName} or ${fields.role || spec.obsoleteHwpodDbUser} still present or unobservable`, + }; +} + export function nodeSecretStatusFromTextForTest(text: string, commandOk: boolean, exitCode: number | null, stderr: string, node = "G14", lane = "v03"): Record { return secretStatusFromText(text, commandOk, exitCode, stderr, runtimeSecretSpec({ node, lane })); }