fix: remove obsolete hwpod db path

This commit is contained in:
Codex
2026-06-09 07:44:48 +00:00
parent b7a44ee302
commit dab7db6ea7
5 changed files with 336 additions and 141 deletions
+2 -2
View File
@@ -16,7 +16,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 runtime lane 滚动
当现有 CLI 对某个 CI/CD 操作缺字段、缺动作、缺状态或缺权限时,处理顺序是先补 CLI,再执行发布或治理动作。临时低层 route 写操作只允许用于一次性止血,并且必须随后把稳定能力补进 CLI 与本参考文档;不能把手工 `kubectl apply/delete/annotate`、原生 GitHub CLI、手写 REST 请求或 registry shell 脚本沉淀成长期流程。长时观察仍遵守 60 秒短查询和 submit-and-poll 语义,不用单个 `trans`/`tran` 等待完整 PipelineRun 或 Argo rollout 结束。
`hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 [--dry-run|--confirm]` 是 v03 platform PostgreSQL bridge 的受控收敛入口,用于让 Argo 跟踪固定 `EndpointSlice/g14-platform-postgres-host`,并清理旧 `Endpoints/g14-platform-postgres` 及其派生的随机 EndpointSlice。`hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 [--dry-run|--confirm]` 是 v03 迁移到 G14 platform PostgreSQL 后清理旧 repo-owned Postgres StatefulSet/Service/ConfigMap/Secret/PVC 的受控入口。迁移后的 `hwlab-cloud-api-v03-db``hwlab-v03-openfga``hwpod-v03-db` SecretRef 来自 G14 host platform DB 凭据文件,不再从 lane-local Postgres Secret 派生;v03+ 的 `hwlab nodes secret ensure --name hwlab-cloud-api-v03-db|hwlab-v03-openfga|hwpod-v03-db` 旧路径已删除,status 只做 redacted SecretRef 与 `g14-platform-postgres` Service、固定 EndpointSlice 和旧 Endpoints 缺席观测。平台 DB 运行、SecretRef 轮换边界和 health 验证见 `docs/reference/g14-platform-db.md`
`hwlab nodes control-plane allow-endpoint-bridge --node G14 --lane v03 [--dry-run|--confirm]` 是 v03 platform PostgreSQL bridge 的受控收敛入口,用于让 Argo 跟踪固定 `EndpointSlice/g14-platform-postgres-host`,并清理旧 `Endpoints/g14-platform-postgres` 及其派生的随机 EndpointSlice。`hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 [--dry-run|--confirm]` 是 v03 迁移到 G14 platform PostgreSQL 后清理旧 repo-owned Postgres StatefulSet/Service/ConfigMap/Secret/PVC 的受控入口。迁移后的 `hwlab-cloud-api-v03-db``hwlab-v03-openfga` SecretRef 来自 G14 host platform DB 凭据文件,不再从 lane-local Postgres Secret 派生;`hwpod-v03-db``hwpod_v03``hwpod_v03_app` 是废弃残留,不是完成态 SecretRef。v03+ 的 `hwlab nodes secret ensure --name hwlab-cloud-api-v03-db|hwlab-v03-openfga` 旧路径已删除,status 只做 redacted SecretRef 与 `g14-platform-postgres` Service、固定 EndpointSlice 和旧 Endpoints 缺席观测;发现 HWPOD DB 残留时使用 `hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db [--dry-run|--confirm]` 受控清理。平台 DB 运行、SecretRef 轮换边界和 health 验证见 `docs/reference/g14-platform-db.md`
`hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider` 是 v03 Code Agent / MoonBridge provider SecretRef 的受控 bootstrap 入口;`ensure` 只从集群内既有 `hwlab-v02/hwlab-v02-code-agent-provider` 复制 `openai-api-key``opencode-api-key` 到 lane-local Secret,输出仅披露 source/target Secret 名、key presence、decoded byte count、mutation 和后续命令,禁止打印 base64、解码值、完整 API key 或可复用凭据。OpenFGA 和 master admin API key 继续使用同一命名空间下的 `hwlab nodes secret ... --name hwlab-v03-openfga|hwlab-v03-master-server-admin-api-key`
@@ -62,7 +62,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 runtime lane 滚动
创建 PipelineRun 前会读取 `devops-infra` mirror refs,若 `localV02` 未等于当前 source commit,则自动执行一次受控 manual `git-mirror sync` Job 并复核 ref,复核失败时停止触发,避免 Tekton `prepare-source` 已知失败;services 参数只包含 v02 runtime service matrix`hwlab-cli` 是固定 repo 短连接源码工具,不进入 PipelineRun service build。
`--dry-run` 只报告是否会 pre-sync,不创建 Jobconfirmed trigger 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id``statusCommand`、stdout/stderr 路径,避免 git mirror pre-sync 或 PipelineRun 创建期间长时间阻塞;`--wait` 路径也必须向 stderr 输出 `hwlab.v02.trigger.progress` JSON 事件,覆盖 `control-plane-refresh``git-mirror-pre-sync``create-pipelinerun`,避免异步 job 长时间只有启动命令而无法判断卡点;默认 JSON 必须对 `manifest_b64`、长脚本和远端 stdout/stderr 做有界摘要,保留长度与 hash,最终 trigger 结果只返回阶段摘要和关键 tail,完整内容通过 job stdout/stderr 文件渐进披露;只有现场同步调试才显式加 `--wait`;旧 `rerun-current` 只作为输入别名保留。PipelineRun `Completed`、Argo `Synced/Healthy``webAssets.ok=true` 只证明 G14 runtime 已更新;交付收口还必须用 `hwlab g14 git-mirror status` 查看 `cache.summary.pendingFlush`,若为 true,继续执行受控 `hwlab g14 git-mirror flush --confirm` 并用 job status 轮询到 `pendingFlush=false`
- `hwlab g14 control-plane runtime-migration --lane v02 [--dry-run|--allow-live-db-read --dry-run|--confirm]` 只通过 `hwlab-v02` namespace 当前 `deployment/hwlab-cloud-api -c hwlab-cloud-api` 内 repo-owned migration CLI 执行;不读取或打印 Secret 值、不触碰 PROD、不绕到手工 `psql`
- `hwlab g14 secret status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm]``hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key [--dry-run|--confirm]` 是 HWLAB runtime lane SecretRef bootstrap 的保留入口。v03+ Cloud API/OpenFGA/HWPOD datastore SecretRef 已迁移到 G14 platform PostgreSQL`hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db|hwlab-v03-openfga|hwpod-v03-db` 只做 redacted SecretRef 与 `g14-platform-postgres` bridge 观测;旧 `ensure` 路径已删除,不再从 `hwlab-v03-postgres` Secret 或 StatefulSet 派生。平台库凭据、桥接 Service 和 SecretRef 轮换边界见 `docs/reference/g14-platform-db.md`。master server admin API key preset 确保本机 `/root/.config/hwlab-v0x/master-server-admin-api-key.env` 以 0600 保存 `HWLAB_API_KEY`,并同步到对应 lane 的 `*-master-server-admin-api-key/api-key``status` 只返回 key 是否存在、解码后字节数、key prefix、bridge 存在性和 runtime health 相关结果,永远不读取、不打印、不回传 secret 明文。`hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 [--dry-run|--confirm]` 是 v0.3+ 迁移到 G14 平台 Postgres 后的受控残留清理入口,精确删除旧 repo-owned `hwlab-v03-postgres` StatefulSet/Service/ConfigMap/Secret 和 `data-hwlab-v03-postgres-0` PVC;它要求 `g14-platform-postgres` Service 已存在,默认 dry-run,不触碰平台数据库、OpenFGA/Cloud API/HWPOD 当前 SecretRef 或 GitOps desired state。`hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> [--dry-run|--confirm]` 只用于删除确认已不被 workload 引用的 v0.2 废弃 Secret,默认 dry-run,拒绝删除 OpenFGA/Postgres/master admin API key 等必需 Secret;共享 device-pod API key 已退出当前授权路径,不再提供 ensure/bootstrap 入口。
- `hwlab g14 secret status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm]``hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key [--dry-run|--confirm]` 是 HWLAB runtime lane SecretRef bootstrap 的保留入口。v03+ Cloud API/OpenFGA datastore SecretRef 已迁移到 G14 platform PostgreSQL`hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db|hwlab-v03-openfga` 只做 redacted SecretRef 与 `g14-platform-postgres` bridge 观测;旧 `ensure` 路径已删除,不再从 `hwlab-v03-postgres` Secret 或 StatefulSet 派生。`hwpod-v03-db``hwpod_v03``hwpod_v03_app` 是废弃残留,不能作为 status 完成态保留,发现后用 `hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db [--dry-run|--confirm]` 清理。平台库凭据、桥接 Service 和 SecretRef 轮换边界见 `docs/reference/g14-platform-db.md`。master server admin API key preset 确保本机 `/root/.config/hwlab-v0x/master-server-admin-api-key.env` 以 0600 保存 `HWLAB_API_KEY`,并同步到对应 lane 的 `*-master-server-admin-api-key/api-key``status` 只返回 key 是否存在、解码后字节数、key prefix、bridge 存在性和 runtime health 相关结果,永远不读取、不打印、不回传 secret 明文。`hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 [--dry-run|--confirm]` 是 v0.3+ 迁移到 G14 平台 Postgres 后的受控残留清理入口,精确删除旧 repo-owned `hwlab-v03-postgres` StatefulSet/Service/ConfigMap/Secret 和 `data-hwlab-v03-postgres-0` PVC;它要求 `g14-platform-postgres` Service 已存在,默认 dry-run,不触碰平台数据库、OpenFGA/Cloud API 当前 SecretRef 或 GitOps desired state。`hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> [--dry-run|--confirm]` 只用于删除确认已不被 workload 引用的 v0.2 废弃 Secret,默认 dry-run,拒绝删除 OpenFGA/Postgres/master admin API key 等必需 Secret;共享 device-pod API key 已退出当前授权路径,不再提供 ensure/bootstrap 入口。
- `hwlab g14 control-plane cleanup-runs --lane v02|v03|g14|all [--min-age-minutes N] [--limit N] [--dry-run|--confirm]` 是完成态 PipelineRun 工作区 retention 入口;真实清理只删除已完成 PipelineRun,让 Tekton/local-path 回收临时 PVC,不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。带 `--pipeline-run <name>``--source-commit <full-sha>` 的定点清理必须先直接查询目标 PipelineRun,而不是只从全量列表过滤;不存在的目标返回 `target-pipelinerun-not-found`,未完成目标返回 `target-pipelinerun-not-terminal`,空查询和读取失败分别返回 `target-pipelinerun-query-empty` / `target-pipelinerun-query-failed`,年龄保护仍返回 `below-min-age``hwlab nodes control-plane cleanup-runs --node G14 --lane v03 --pipeline-run <name>` 是 v0.3 failed run 受控重试前的清理入口。
- `hwlab g14 control-plane cleanup-released-pvs --lane all [--limit N] [--dry-run|--confirm]` 是 local-path 未自动回收后的补充 retention 入口;只列并删除 `Released``local-path``Delete``claimNamespace=hwlab-ci` 且 claim 名称形如 Tekton 临时 `pvc-*` 的 PV。
- `hwlab g14 git-mirror status|apply|sync|flush [--dry-run|--confirm]``devops-infra` git mirror/relay 的受控维护入口:`apply` 渲染并 server-side apply `devops-infra/git-mirror.yaml`,同时删除遗留 `git-mirror-hwlab-sync` CronJob`sync` 创建一次性 manual Job,把 GitHub allowlist refs 拉入本地 mirror`flush` 创建一次性 manual Job,把本地 `v0.2-gitops` 快进推回 GitHub。
+13 -4
View File
@@ -4,7 +4,7 @@
## 定位
G14 platform DB 是 G14 host OS 上的原生 PostgreSQL,不是 k3s workload,也不属于 `devops-infra` namespace。它位于 HWLAB runtime lane、AgentRun、HWPOD 和未来 G14 平台控制能力之下,作为节点级基础设施承载 durable runtime、OpenFGA datastore、平台控制元数据、租约、审计和状态观测。
G14 platform DB 是 G14 host OS 上的原生 PostgreSQL,不是 k3s workload,也不属于 `devops-infra` namespace。它位于 HWLAB runtime lane、AgentRun 和未来 G14 平台控制能力之下,作为节点级基础设施承载 durable runtime、OpenFGA datastore、平台控制元数据、租约、审计和状态观测。
数据库可以保存一部分 k3s 管理所需的状态,例如节点 inventory、runtime lane registry、operation lease、k3s state observation 和 audit event;它不直接执行 `kubectl`、Argo、Tekton 或 GitHub 写操作。k3s 写操作仍必须通过 UniDesk 受控 CLI 或对应控制器执行,数据库只作为状态与协调底座。
@@ -39,10 +39,11 @@ PostgreSQL 只监听 G14 host loopback 与 k3s pod 可达的 node gateway 地址
- `g14_platform_control` / `g14_platform_app`:G14 节点平台控制元数据、租约、观测和审计。
- `hwlab_v03` / `hwlab_v03_app`HWLAB v0.3 cloud-api durable runtime。
- `openfga_v03` / `openfga_v03_app`HWLAB v0.3 OpenFGA datastore。
- `hwpod_v03` / `hwpod_v03_app`:HWPOD v0.3 迁移后的平台库目标;HWPOD 不应继续拥有自带 DB。
`postgres` 只作为管理数据库保留。新增 runtime lane 时按同一模式分配独立 database/role,不复用业务应用账号访问 `g14_platform_control`
HWPOD v0.3 不再有当前 platform DB 目标;`hwpod-v03-db``hwpod_v03``hwpod_v03_app` 出现时只能视为废弃残留,不能作为 SecretRef 完成态或平台库目标保留。
## k3s 桥接
业务 Pod 不直接访问 host IP 字面量,而是访问所在 namespace 内的 `g14-platform-postgres` Service。HWLAB v0.3 的桥接对象在 `hwlab-v03` namespace 中:
@@ -56,9 +57,8 @@ PostgreSQL 只监听 G14 host loopback 与 k3s pod 可达的 node gateway 地址
- `hwlab-cloud-api-v03-db/database-url` 指向 `hwlab_v03`
- `hwlab-v03-openfga/datastore-uri` 指向 `openfga_v03`
- `hwpod-v03-db/database-url` 指向 `hwpod_v03`
这些 SecretRef 的源凭据来自 G14 host 的 platform DB credentials file,不再从旧 lane-local Postgres Secret 派生。迁移完成后,旧的 repo-owned PostgreSQL bootstrap/ensure 路径已经删除,不能用于轮换 `hwlab-cloud-api-v03-db``hwlab-v03-openfga`。如需重复轮换,必须先补齐平台 DB 专用 UniDesk CLI,让它从 host 凭据文件读取、写入 k3s Secret,并输出 redacted 状态;不得把手写 `kubectl create secret` 或旧 StatefulSet ensure 路径沉淀成长期流程。
这些 SecretRef 的源凭据来自 G14 host 的 platform DB credentials file,不再从旧 lane-local Postgres Secret 派生。`hwpod-v03-db/database-url` 是废弃残留,应该缺席;发现它时用受控清理入口删除,不能为它补 status 可见性或轮换路径。迁移完成后,旧的 repo-owned PostgreSQL bootstrap/ensure 路径已经删除,不能用于轮换 `hwlab-cloud-api-v03-db``hwlab-v03-openfga`。如需重复轮换,必须先补齐平台 DB 专用 UniDesk CLI,让它从 host 凭据文件读取、写入 k3s Secret,并输出 redacted 状态;不得把手写 `kubectl create secret` 或旧 StatefulSet ensure 路径沉淀成长期流程。
当前运行面验证使用 control-plane status 和 live health
@@ -118,6 +118,15 @@ curl -fsS http://74.48.78.17:20667/health/live | jq '{status, ready, environment
其中 `endpoints/g14-platform-postgres` 应为空,`endpointslice/g14-platform-postgres-host` 应存在且是唯一带 `kubernetes.io/service-name=g14-platform-postgres` label 的 EndpointSlice。
HWPOD v0.3 DB 残留不属于旧自有 HWLAB Postgres StatefulSet 清理目标,必须用单独的废弃对象清理入口处理:
```bash
bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run
bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm
```
该入口只允许清理已确认不被 workload 引用的 `hwpod-v03-db` k3s Secret,并同步清理 G14 host PostgreSQL 上同名废弃目标对应的 `hwpod_v03` database 与 `hwpod_v03_app` role;输出只披露对象是否存在、是否删除和退出码,不打印 URI、密码或可复用凭据。清理后 `hwpod-v03-db``hwpod_v03``hwpod_v03_app` 都应缺席。
## 备份与恢复
备份脚本固定在 G14 host
+1 -53
View File
@@ -164,17 +164,14 @@ assertCondition(
hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider"))
&& hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm"))
&& hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db"))
&& hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret status --node G14 --lane v03 --name hwpod-v03-db"))
&& hwlabHelpUsage.some((line) => line.includes("hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run"))
&& hwlabNodeHelpJson.includes("hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider")
&& hwlabNodeHelpJson.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm")
&& hwlabNodeHelpJson.includes("hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db")
&& hwlabNodeHelpJson.includes("hwlab nodes secret status --node G14 --lane v03 --name hwpod-v03-db")
&& hwlabNodeHelpJson.includes("hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run")
&& !hwlabNodeHelpJson.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-cloud-api-v03-db --confirm")
&& !hwlabNodeHelpJson.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwpod-v03-db --confirm")
&& !hwlabNodeHelpJson.includes("hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --confirm"),
"v0.3 node-scoped secret help must expose provider bootstrap plus platform DB status/cleanup without old DB ensure paths",
"v0.3 node-scoped secret help must expose provider bootstrap plus Cloud API platform DB status and old DB cleanup without old DB ensure paths",
{ hwlabHelpUsage, hwlabNodeHelp: hwlabNodeHelp() },
);
const cloudApiDbStatus = nodeSecretStatusFromTextForTest([
@@ -222,48 +219,6 @@ assertCondition(
"cloud-api DB Secret status must be redacted while proving native platform DB SecretRef and bridge are present",
cloudApiDbStatus,
);
const hwpodDbStatus = nodeSecretStatusFromTextForTest([
"namespace\thwlab-v03",
"secret\thwpod-v03-db",
"key\tdatabase-url",
"preset\thwpod-db",
"action\tobserved",
"dryRun\ttrue",
"mutation\tfalse",
"platformDbMode\ttrue",
"afterExists\tyes",
"afterDatabaseUrlPresent\tyes",
"afterDatabaseUrlBytes\t172",
"legacyPostgresSecret\thwlab-v03-postgres",
"legacyPostgresSecretExists\tno",
"platformService\tg14-platform-postgres",
"platformServiceExists\tyes",
"platformEndpointsExists\tno",
"platformEndpointSlice\tg14-platform-postgres-host",
"platformEndpointSliceExists\tyes",
"dbName\thwpod_v03",
"dbUser\thwpod_v03_app",
"dbHost\tg14-platform-postgres.hwlab-v03.svc.cluster.local",
"dbHostMatchesPlatform\tyes",
"dbNameMatchesExpected\tyes",
"dbUserMatchesExpected\tyes",
].join("\n"), true, 0, "");
assertCondition(
record(hwpodDbStatus).ok === true
&& record(hwpodDbStatus).valuesRedacted === true
&& record(hwpodDbStatus).platformDbMode === true
&& record(record(hwpodDbStatus).legacyPostgresSecret).exists === false
&& record(record(hwpodDbStatus).platformService).endpointsExist === false
&& record(record(hwpodDbStatus).platformService).legacyEndpointsAbsent === true
&& record(record(hwpodDbStatus).platformService).endpointSlice === "g14-platform-postgres-host"
&& record(hwpodDbStatus).dbName === "hwpod_v03"
&& record(hwpodDbStatus).dbUser === "hwpod_v03_app"
&& record(hwpodDbStatus).dbHost === "g14-platform-postgres.hwlab-v03.svc.cluster.local"
&& !JSON.stringify(hwpodDbStatus).includes("postgres://")
&& !JSON.stringify(hwpodDbStatus).includes("password"),
"HWPOD DB Secret status must prove the v0.3 platform DB SecretRef without exposing database-url material",
hwpodDbStatus,
);
const openFgaPlatformStatus = nodeSecretStatusFromTextForTest([
"namespace\thwlab-v03",
"secret\thwlab-v03-openfga",
@@ -316,13 +271,6 @@ assertCondition(
"v0.3 cloud-api DB ensure must reject the removed lane-local Postgres path",
removedCloudApiDbEnsure,
);
const removedHwpodDbEnsure = runCommand(["bun", "scripts/cli.ts", "hwlab", "nodes", "secret", "ensure", "--node", "G14", "--lane", "v03", "--name", "hwpod-v03-db", "--dry-run"], process.cwd(), { timeoutMs: 30_000 });
assertCondition(
removedHwpodDbEnsure.exitCode !== 0
&& `${removedHwpodDbEnsure.stdout}\n${removedHwpodDbEnsure.stderr}`.includes("was removed after native platform DB migration"),
"v0.3 HWPOD DB ensure must reject the removed lane-local Postgres path",
removedHwpodDbEnsure,
);
const removedOpenFgaEnsure = runCommand(["bun", "scripts/cli.ts", "hwlab", "nodes", "secret", "ensure", "--node", "G14", "--lane", "v03", "--name", "hwlab-v03-openfga", "--dry-run"], process.cwd(), { timeoutMs: 30_000 });
assertCondition(
removedOpenFgaEnsure.exitCode !== 0
+2 -1
View File
@@ -10221,9 +10221,10 @@ export function hwlabG14Help(): Record<string, unknown> {
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwpod-v03-db",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run",
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm",
"bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name <obsolete-hwlab-v02-secret> --dry-run",
+318 -81
View File
@@ -5,8 +5,8 @@ import { startJob } from "./jobs";
import { runHwlabG14Command } from "./hwlab-g14";
import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane } from "./hwlab-node-lanes";
type SecretAction = "status" | "ensure" | "cleanup-owned-postgres";
type SecretPreset = "openfga" | "master-server-admin-api-key" | "code-agent-provider" | "cloud-api-db" | "hwpod-db" | "owned-postgres-cleanup";
type SecretAction = "status" | "ensure" | "cleanup-owned-postgres" | "cleanup-obsolete";
type SecretPreset = "openfga" | "master-server-admin-api-key" | "code-agent-provider" | "cloud-api-db" | "owned-postgres-cleanup" | "obsolete-secret-cleanup";
type DelegatedNodeDomain = "control-plane" | "git-mirror";
interface NodeSecretOptions {
@@ -41,11 +41,9 @@ interface RuntimeSecretSpec {
cloudApiDbUser: string;
cloudApiDbHost: string;
cloudApiDeployment: string;
hwpodDbSecret: string;
hwpodDbKey: string;
hwpodDbName: string;
hwpodDbUser: string;
hwpodDbHost: string;
obsoleteHwpodDbSecret: string;
obsoleteHwpodDbName: string;
obsoleteHwpodDbUser: string;
codeAgentProviderSecret: string;
codeAgentProviderSourceNamespace: string;
codeAgentProviderSourceSecret: string;
@@ -92,9 +90,10 @@ export function hwlabNodeHelp(): Record<string, unknown> {
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-cloud-api-v03-db",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwpod-v03-db",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --dry-run",
"bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node G14 --lane v03 --confirm",
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --dry-run",
"bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node G14 --lane v03 --name hwpod-v03-db --confirm",
"bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-code-agent-provider",
"bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-code-agent-provider --confirm",
],
@@ -215,8 +214,8 @@ function rewriteDelegatedNodeString(value: string, scoped: ReturnType<typeof par
function parseSecretOptions(args: string[]): NodeSecretOptions {
const [actionRaw] = args;
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres") {
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-cloud-api-vNN-db|hwpod-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm]");
if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "cleanup-owned-postgres" && actionRaw !== "cleanup-obsolete") {
throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key|hwlab-cloud-api-vNN-db|hwlab-vNN-code-agent-provider [--dry-run|--confirm] | cleanup-owned-postgres --node NODE --lane vNN [--dry-run|--confirm] | cleanup-obsolete --node NODE --lane vNN --name hwpod-vNN-db [--dry-run|--confirm]");
}
const node = requiredOption(args, "--node");
assertNodeId(node);
@@ -243,6 +242,21 @@ function parseSecretOptions(args: string[]): NodeSecretOptions {
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
};
}
if (actionRaw === "cleanup-obsolete") {
if (lane === "v02") throw new Error("secret cleanup-obsolete is only for v0.3+ lanes after deprecated v0.3 HWPOD DB SecretRef cleanup");
if (key !== undefined) throw new Error("secret cleanup-obsolete does not accept --key");
if (name !== spec.obsoleteHwpodDbSecret) throw new Error(`secret cleanup-obsolete for --lane ${lane} currently only targets obsolete ${spec.obsoleteHwpodDbSecret}`);
return {
action: actionRaw,
node,
lane,
name,
preset: "obsolete-secret-cleanup",
confirm,
dryRun: explicitDryRun || !confirm,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 180, 900),
};
}
if (name === spec.masterAdminApiKeySecret) {
if (key !== undefined && key !== MASTER_ADMIN_API_KEY_KEY) throw new Error(`secret ${name} supports only key ${MASTER_ADMIN_API_KEY_KEY}`);
return {
@@ -290,26 +304,8 @@ function parseSecretOptions(args: string[]): NodeSecretOptions {
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900),
};
}
if (name === spec.hwpodDbSecret) {
if (!spec.platformDb) throw new Error(`secret ${name} is only supported for v0.3+ platform DB lanes`);
if (key !== undefined && key !== spec.hwpodDbKey) throw new Error(`secret ${name} supports only key ${spec.hwpodDbKey}`);
if (actionRaw === "ensure") {
throw new Error(`secret ensure for ${name} on --lane ${lane} was removed after native platform DB migration; use status plus platform DB SecretRef rotation CLI when it exists`);
}
return {
action: actionRaw,
node,
lane,
name,
key: key ?? spec.hwpodDbKey,
preset: "hwpod-db",
confirm,
dryRun: true,
timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 240, 900),
};
}
if (name !== spec.openFgaSecret) {
throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.cloudApiDbSecret}, ${spec.hwpodDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}`);
throw new Error(`secret status/ensure supports --name ${spec.openFgaSecret}, ${spec.masterAdminApiKeySecret}, ${spec.cloudApiDbSecret}, or ${spec.codeAgentProviderSecret} for --lane ${lane}; obsolete ${spec.obsoleteHwpodDbSecret} must use cleanup-obsolete`);
}
if (key !== undefined && key !== OPENFGA_AUTHN_KEY && key !== OPENFGA_DATASTORE_URI_KEY && key !== OPENFGA_POSTGRES_PASSWORD_KEY) {
throw new Error(`secret ${name} supports keys ${OPENFGA_AUTHN_KEY}, ${OPENFGA_DATASTORE_URI_KEY}, and ${OPENFGA_POSTGRES_PASSWORD_KEY}`);
@@ -356,11 +352,9 @@ function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecret
cloudApiDbUser: platformDb ? `hwlab_${input.lane}_app` : `hwlab_${input.lane}`,
cloudApiDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
cloudApiDeployment: "hwlab-cloud-api",
hwpodDbSecret: `hwpod-${input.lane}-db`,
hwpodDbKey: CLOUD_API_DB_KEY,
hwpodDbName: `hwpod_${input.lane}`,
hwpodDbUser: platformDb ? `hwpod_${input.lane}_app` : `hwpod_${input.lane}`,
hwpodDbHost: platformDb ? platformPostgresHost : legacyPostgresHost,
obsoleteHwpodDbSecret: `hwpod-${input.lane}-db`,
obsoleteHwpodDbName: `hwpod_${input.lane}`,
obsoleteHwpodDbUser: `hwpod_${input.lane}_app`,
codeAgentProviderSecret: `${namespace}-code-agent-provider`,
codeAgentProviderSourceNamespace: CODE_AGENT_PROVIDER_SOURCE_NAMESPACE,
codeAgentProviderSourceSecret: CODE_AGENT_PROVIDER_SOURCE_SECRET,
@@ -370,6 +364,7 @@ function runtimeSecretSpec(input: { node: string; lane: string }): RuntimeSecret
function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
const spec = runtimeSecretSpec(options);
if (options.preset === "obsolete-secret-cleanup") return runObsoleteSecretCleanup(options, spec);
const input = options.preset === "master-server-admin-api-key" && options.action === "ensure" && !options.dryRun
? readMasterAdminApiKey().key
: "";
@@ -377,7 +372,7 @@ function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : openFgaSecretScript(options, spec)
: options.preset === "master-server-admin-api-key"
? masterAdminApiKeySecretScript(options, spec)
: options.preset === "cloud-api-db" || options.preset === "hwpod-db"
: options.preset === "cloud-api-db"
? spec.platformDb ? platformDbSecretStatusScript(options, spec) : cloudApiDbSecretScript(options, spec)
: options.preset === "owned-postgres-cleanup"
? ownedPostgresCleanupScript(options, spec)
@@ -386,7 +381,8 @@ function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
const status = secretStatusFromText(statusText(result), result.exitCode === 0, result.exitCode, result.stderr, spec);
const dryRunOk = options.action === "ensure" && options.dryRun && result.exitCode === 0;
const cleanupDryRunOk = options.action === "cleanup-owned-postgres" && options.dryRun && result.exitCode === 0;
const ok = dryRunOk || cleanupDryRunOk ? true : status.ok === true;
const obsoleteCleanupDryRunOk = options.action === "cleanup-obsolete" && options.dryRun && status.ok === true;
const ok = dryRunOk || cleanupDryRunOk || obsoleteCleanupDryRunOk ? true : status.ok === true;
return {
ok,
command: `hwlab nodes secret ${options.action}`,
@@ -396,7 +392,7 @@ function runNodeSecret(options: NodeSecretOptions): Record<string, unknown> {
secret: options.name,
key: options.key ?? null,
preset: options.preset,
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" ? "confirmed-delete" : "confirmed-ensure",
mode: options.action === "status" ? "status" : options.dryRun ? "dry-run" : options.action === "cleanup-owned-postgres" || options.action === "cleanup-obsolete" ? "confirmed-delete" : "confirmed-ensure",
status,
mutation: status.mutation === true,
result: compactCommandResult(result),
@@ -409,7 +405,10 @@ function nextSecretCommand(options: NodeSecretOptions, spec: RuntimeSecretSpec):
if (options.action === "cleanup-owned-postgres") {
return { ensure: `bun scripts/cli.ts hwlab nodes secret cleanup-owned-postgres --node ${options.node} --lane ${options.lane} --confirm` };
}
if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga" || options.preset === "hwpod-db")) {
if (options.action === "cleanup-obsolete") {
return { cleanup: `bun scripts/cli.ts hwlab nodes secret cleanup-obsolete --node ${options.node} --lane ${options.lane} --name ${options.name} --confirm` };
}
if (spec.platformDb && (options.preset === "cloud-api-db" || options.preset === "openfga")) {
return {
status: `bun scripts/cli.ts hwlab nodes secret status --node ${options.node} --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""}`,
controlPlaneStatus: `bun scripts/cli.ts hwlab nodes control-plane status --node ${options.node} --lane ${options.lane}`,
@@ -422,6 +421,50 @@ function runTransScript(node: string, script: string, input: string, timeoutSeco
return runCommand(["/root/.local/bin/trans", `${node}:k3s`, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
}
function runTransHostScript(node: string, script: string, input: string, timeoutSeconds: number): CommandResult {
return runCommand(["/root/.local/bin/trans", node, "script", "--", script], repoRoot, { input, timeoutMs: timeoutSeconds * 1000 });
}
function runObsoleteSecretCleanup(options: NodeSecretOptions, spec: RuntimeSecretSpec): Record<string, unknown> {
const kubernetesResult = runTransScript(options.node, obsoleteSecretCleanupScript(options, spec), "", options.timeoutSeconds);
const kubernetesStatus = secretStatusFromText(statusText(kubernetesResult), kubernetesResult.exitCode === 0, kubernetesResult.exitCode, kubernetesResult.stderr, spec);
const hostOptions = { ...options, dryRun: options.dryRun || kubernetesStatus.ok !== true };
const platformDbResult = runTransHostScript(options.node, obsoletePlatformDbCleanupScript(hostOptions, spec), "", options.timeoutSeconds);
const platformDbStatus = obsoletePlatformDbStatusFromText(statusText(platformDbResult), platformDbResult.exitCode === 0, platformDbResult.exitCode, platformDbResult.stderr, spec);
const ok = kubernetesStatus.ok === true && platformDbStatus.ok === true && (options.dryRun || hostOptions.dryRun === false);
const mutation = kubernetesStatus.mutation === true || platformDbStatus.mutation === true;
return {
ok,
command: `hwlab nodes secret ${options.action}`,
node: options.node,
lane: options.lane,
namespace: spec.namespace,
secret: options.name,
key: null,
preset: options.preset,
mode: options.dryRun ? "dry-run" : "confirmed-delete",
status: {
ok,
preset: "obsolete-hwpod-db-cleanup",
dryRun: options.dryRun,
mutation,
kubernetesSecret: kubernetesStatus,
platformDatabase: platformDbStatus,
hostMutationSkipped: !options.dryRun && hostOptions.dryRun,
summary: ok
? `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, and ${spec.obsoleteHwpodDbUser} are absent or ready to remove`
: `${spec.obsoleteHwpodDbSecret}, ${spec.obsoleteHwpodDbName}, or ${spec.obsoleteHwpodDbUser} still needs cleanup`,
},
mutation,
result: {
kubernetesSecret: compactCommandResult(kubernetesResult),
platformDatabase: compactCommandResult(platformDbResult),
},
valuesRedacted: true,
next: ok ? undefined : nextSecretCommand(options, spec),
};
}
function runNodeEndpointBridge(options: ReturnType<typeof parseNodeScopedDelegatedOptions>): Record<string, unknown> {
if (options.dryRun && options.confirm) throw new Error("control-plane allow-endpoint-bridge accepts only one of --dry-run or --confirm");
const dryRun = options.dryRun || !options.confirm;
@@ -796,15 +839,156 @@ function ownedPostgresCleanupScript(options: NodeSecretOptions, spec: RuntimeSec
].join("\n");
}
function obsoleteSecretCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`secret=${shellQuote(options.name)}`,
`expected_secret=${shellQuote(spec.obsoleteHwpodDbSecret)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
"preset=obsolete-secret-cleanup",
"exists_flag() { kubectl -n \"$namespace\" get secret \"$secret\" >/dev/null 2>&1 && printf yes || printf no; }",
"workload_refs=$(kubectl -n \"$namespace\" get deploy,statefulset,daemonset,job,cronjob -o yaml 2>/dev/null | grep -n -C 2 \"$secret\" || true)",
"workload_refs_present=$([ -n \"$workload_refs\" ] && printf yes || printf no)",
"workload_refs_preview=$(printf '%s' \"$workload_refs\" | sed -n '1,20p' | tr '\\n' ';' | cut -c1-1000)",
"before_exists=$(exists_flag)",
"action=observed",
"mutation=false",
"delete_secret_exit=",
"if [ \"$secret\" != \"$expected_secret\" ]; then action=unsupported-secret; fi",
"if [ \"$action\" = observed ]; then",
" if [ \"$workload_refs_present\" = yes ]; then",
" action=blocked-referenced",
" elif [ \"$dry_run\" = true ]; then",
" if [ \"$before_exists\" = yes ]; then action=would-delete; else action=already-absent; fi",
" else",
" kubectl -n \"$namespace\" delete secret \"$secret\" --ignore-not-found=true >/tmp/hwlab-obsolete-secret-delete.out 2>/tmp/hwlab-obsolete-secret-delete.err",
" delete_secret_exit=$?",
" for _ in $(seq 1 15); do",
" current_exists=$(exists_flag)",
" if [ \"$current_exists\" != yes ]; then break; fi",
" sleep 1",
" done",
" if [ \"$delete_secret_exit\" -eq 0 ]; then",
" if [ \"$before_exists\" = yes ]; then action=deleted; mutation=true; else action=already-absent; fi",
" else",
" action=delete-failed",
" fi",
" fi",
"fi",
"after_exists=$(exists_flag)",
"printf 'namespace\\t%s\\n' \"$namespace\"",
"printf 'secret\\t%s\\n' \"$secret\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeSecretExists\\t%s\\n' \"$before_exists\"",
"printf 'afterSecretExists\\t%s\\n' \"$after_exists\"",
"printf 'workloadRefsPresent\\t%s\\n' \"$workload_refs_present\"",
"printf 'workloadRefsPreview\\t%s\\n' \"$workload_refs_preview\"",
"printf 'deleteSecretExitCode\\t%s\\n' \"$delete_secret_exit\"",
"if [ \"$action\" = unsupported-secret ]; then exit 43; fi",
"if [ \"$workload_refs_present\" = yes ]; then exit 46; fi",
"if [ \"$dry_run\" != true ] && [ \"$after_exists\" = yes ]; then exit 47; fi",
"if [ -n \"$delete_secret_exit\" ] && [ \"$delete_secret_exit\" != 0 ]; then exit \"$delete_secret_exit\"; fi",
].join("\n");
}
function obsoletePlatformDbCleanupScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
return [
"set +e",
`db_name=${shellQuote(spec.obsoleteHwpodDbName)}`,
`db_user=${shellQuote(spec.obsoleteHwpodDbUser)}`,
`dry_run=${shellQuote(options.dryRun ? "true" : "false")}`,
"preset=obsolete-platform-db-cleanup",
"database_exists_flag() {",
" output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_database where datname='$db_name');\" 2>/tmp/hwlab-obsolete-platform-db-probe.err)",
" code=$?",
" if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi",
" [ \"$output\" = t ] && printf yes || printf no",
"}",
"role_exists_flag() {",
" output=$(sudo -u postgres psql -d postgres -Atqc \"select exists(select 1 from pg_roles where rolname='$db_user');\" 2>/tmp/hwlab-obsolete-platform-role-probe.err)",
" code=$?",
" if [ \"$code\" -ne 0 ]; then printf unknown; return \"$code\"; fi",
" [ \"$output\" = t ] && printf yes || printf no",
"}",
"before_database_exists=$(database_exists_flag)",
"before_database_probe_exit=$?",
"before_role_exists=$(role_exists_flag)",
"before_role_probe_exit=$?",
"action=observed",
"mutation=false",
"drop_database_exit=",
"drop_role_exit=",
"before_any=false",
"if [ \"$before_database_exists\" = yes ] || [ \"$before_role_exists\" = yes ]; then before_any=true; fi",
"if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ]; then",
" action=probe-failed",
"elif [ \"$dry_run\" = true ]; then",
" if [ \"$before_any\" = true ]; then action=would-drop; else action=already-absent; fi",
"else",
" if [ \"$before_database_exists\" = yes ]; then",
" sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_name=\"$db_name\" >/tmp/hwlab-obsolete-platform-db-drop.out 2>/tmp/hwlab-obsolete-platform-db-drop.err <<'SQL'",
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = :'db_name' AND pid <> pg_backend_pid();",
"DROP DATABASE IF EXISTS :\"db_name\";",
"SQL",
" drop_database_exit=$?",
" else",
" drop_database_exit=0",
" fi",
" if [ \"$drop_database_exit\" -eq 0 ] && [ \"$before_role_exists\" = yes ]; then",
" sudo -u postgres psql -v ON_ERROR_STOP=1 -d postgres -v db_user=\"$db_user\" >/tmp/hwlab-obsolete-platform-role-drop.out 2>/tmp/hwlab-obsolete-platform-role-drop.err <<'SQL'",
"DROP ROLE IF EXISTS :\"db_user\";",
"SQL",
" drop_role_exit=$?",
" elif [ \"$drop_database_exit\" -eq 0 ]; then",
" drop_role_exit=0",
" else",
" drop_role_exit=",
" fi",
" if [ \"$drop_database_exit\" = 0 ] && [ \"$drop_role_exit\" = 0 ]; then",
" if [ \"$before_any\" = true ]; then action=dropped; mutation=true; else action=already-absent; fi",
" else",
" action=drop-failed",
" fi",
"fi",
"after_database_exists=$(database_exists_flag)",
"after_database_probe_exit=$?",
"after_role_exists=$(role_exists_flag)",
"after_role_probe_exit=$?",
"printf 'database\\t%s\\n' \"$db_name\"",
"printf 'role\\t%s\\n' \"$db_user\"",
"printf 'preset\\t%s\\n' \"$preset\"",
"printf 'action\\t%s\\n' \"$action\"",
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
"printf 'mutation\\t%s\\n' \"$mutation\"",
"printf 'beforeDatabaseExists\\t%s\\n' \"$before_database_exists\"",
"printf 'beforeRoleExists\\t%s\\n' \"$before_role_exists\"",
"printf 'afterDatabaseExists\\t%s\\n' \"$after_database_exists\"",
"printf 'afterRoleExists\\t%s\\n' \"$after_role_exists\"",
"printf 'beforeDatabaseProbeExitCode\\t%s\\n' \"$before_database_probe_exit\"",
"printf 'beforeRoleProbeExitCode\\t%s\\n' \"$before_role_probe_exit\"",
"printf 'afterDatabaseProbeExitCode\\t%s\\n' \"$after_database_probe_exit\"",
"printf 'afterRoleProbeExitCode\\t%s\\n' \"$after_role_probe_exit\"",
"printf 'dropDatabaseExitCode\\t%s\\n' \"$drop_database_exit\"",
"printf 'dropRoleExitCode\\t%s\\n' \"$drop_role_exit\"",
"if [ \"$before_database_exists\" = unknown ] || [ \"$before_role_exists\" = unknown ] || [ \"$after_database_exists\" = unknown ] || [ \"$after_role_exists\" = unknown ]; then exit 49; fi",
"if [ \"$dry_run\" != true ] && { [ \"$after_database_exists\" = yes ] || [ \"$after_role_exists\" = yes ]; }; then exit 50; fi",
"if [ -n \"$drop_database_exit\" ] && [ \"$drop_database_exit\" != 0 ]; then exit \"$drop_database_exit\"; fi",
"if [ -n \"$drop_role_exit\" ] && [ \"$drop_role_exit\" != 0 ]; then exit \"$drop_role_exit\"; fi",
].join("\n");
}
function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeSecretSpec): string {
const isOpenFga = options.preset === "openfga";
const isHwpodDb = options.preset === "hwpod-db";
const platformEndpointSlice = `${spec.platformPostgresService}-host`;
return [
"set +e",
`namespace=${shellQuote(spec.namespace)}`,
`name=${shellQuote(isOpenFga ? spec.openFgaSecret : isHwpodDb ? spec.hwpodDbSecret : spec.cloudApiDbSecret)}`,
`database_url_key=${shellQuote(isOpenFga ? OPENFGA_DATASTORE_URI_KEY : isHwpodDb ? spec.hwpodDbKey : spec.cloudApiDbKey)}`,
`name=${shellQuote(isOpenFga ? spec.openFgaSecret : spec.cloudApiDbSecret)}`,
`database_url_key=${shellQuote(isOpenFga ? OPENFGA_DATASTORE_URI_KEY : spec.cloudApiDbKey)}`,
`authn_key=${shellQuote(OPENFGA_AUTHN_KEY)}`,
`postgres_password_key=${shellQuote(OPENFGA_POSTGRES_PASSWORD_KEY)}`,
`legacy_postgres_secret=${shellQuote(spec.postgresSecret)}`,
@@ -812,9 +996,9 @@ function platformDbSecretStatusScript(options: NodeSecretOptions, spec: RuntimeS
`platform_endpointslice=${shellQuote(platformEndpointSlice)}`,
`platform_host=${shellQuote(spec.platformPostgresService)}`,
`platform_host_fqdn=${shellQuote(spec.openFgaDbHost)}`,
`db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : isHwpodDb ? spec.hwpodDbName : spec.cloudApiDbName)}`,
`db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : isHwpodDb ? spec.hwpodDbUser : spec.cloudApiDbUser)}`,
`db_host=${shellQuote(isOpenFga ? spec.openFgaDbHost : isHwpodDb ? spec.hwpodDbHost : spec.cloudApiDbHost)}`,
`db_name=${shellQuote(isOpenFga ? spec.openFgaDbName : spec.cloudApiDbName)}`,
`db_user=${shellQuote(isOpenFga ? spec.openFgaDbUser : spec.cloudApiDbUser)}`,
`db_host=${shellQuote(isOpenFga ? spec.openFgaDbHost : spec.cloudApiDbHost)}`,
`selected_key=${shellQuote(options.key ?? "")}`,
`preset=${shellQuote(options.preset)}`,
"dry_run=true",
@@ -1399,6 +1583,37 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
: `owned Postgres resources still exist in ${fields.namespace || spec.namespace}`,
};
}
if (fields.preset === "obsolete-secret-cleanup") {
const absent = fields.afterSecretExists !== "yes";
const refsAbsent = fields.workloadRefsPresent !== "yes";
const dryRun = fields.dryRun === "true";
return {
ok: commandOk && refsAbsent && (dryRun || absent),
namespace: fields.namespace || spec.namespace,
secret: fields.secret || spec.obsoleteHwpodDbSecret,
preset: "obsolete-secret-cleanup",
action: fields.action || null,
dryRun,
mutation: fields.mutation === "true",
before: {
secretExists: fields.beforeSecretExists === "yes",
},
after: {
secretExists: fields.afterSecretExists === "yes",
},
workloadRefs: {
present: fields.workloadRefsPresent === "yes",
preview: fields.workloadRefsPreview || "",
},
deleteSecretExitCode: numericField(fields.deleteSecretExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: refsAbsent && (dryRun || absent)
? `${fields.secret || spec.obsoleteHwpodDbSecret} is unreferenced${dryRun ? "" : " and absent"}`
: `${fields.secret || spec.obsoleteHwpodDbSecret} still present or referenced`,
};
}
if (fields.preset === "master-server-admin-api-key") {
const afterBytes = numericField(fields.afterApiKeyBytes);
const healthy = fields.afterExists === "yes" && fields.afterApiKeyPresent === "yes" && typeof afterBytes === "number" && afterBytes > 0;
@@ -1462,13 +1677,7 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
summary: healthy ? `${fields.secret || spec.codeAgentProviderSecret} has a usable provider key` : `${fields.secret || spec.codeAgentProviderSecret} missing provider keys`,
};
}
if (fields.preset === "cloud-api-db" || fields.preset === "hwpod-db") {
const hwpodDb = fields.preset === "hwpod-db";
const expectedSecret = hwpodDb ? spec.hwpodDbSecret : spec.cloudApiDbSecret;
const expectedKey = hwpodDb ? spec.hwpodDbKey : spec.cloudApiDbKey;
const expectedDbName = hwpodDb ? spec.hwpodDbName : spec.cloudApiDbName;
const expectedDbUser = hwpodDb ? spec.hwpodDbUser : spec.cloudApiDbUser;
const expectedDbHost = hwpodDb ? spec.hwpodDbHost : spec.cloudApiDbHost;
if (fields.preset === "cloud-api-db") {
const beforeUrlBytes = numericField(fields.beforeDatabaseUrlBytes);
const afterUrlBytes = numericField(fields.afterDatabaseUrlBytes);
if (fields.platformDbMode === "true") {
@@ -1485,9 +1694,9 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || expectedSecret,
key: fields.key || expectedKey,
preset: hwpodDb ? "hwpod-db" : "cloud-api-db",
secret: fields.secret || spec.cloudApiDbSecret,
key: fields.key || spec.cloudApiDbKey,
preset: "cloud-api-db",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: fields.mutation === "true",
@@ -1508,9 +1717,9 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
endpointSlice: fields.platformEndpointSlice || `${spec.platformPostgresService}-host`,
endpointSliceExists: fields.platformEndpointSliceExists === "yes",
},
dbName: fields.dbName || expectedDbName,
dbUser: fields.dbUser || expectedDbUser,
dbHost: fields.dbHost || expectedDbHost,
dbName: fields.dbName || spec.cloudApiDbName,
dbUser: fields.dbUser || spec.cloudApiDbUser,
dbHost: fields.dbHost || spec.cloudApiDbHost,
dbHostMatchesPlatform: fields.dbHostMatchesPlatform === "yes",
dbNameMatchesExpected: fields.dbNameMatchesExpected === "yes",
dbUserMatchesExpected: fields.dbUserMatchesExpected === "yes",
@@ -1518,24 +1727,8 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy
? `${fields.secret || expectedSecret}/${fields.key || expectedKey} points to ${fields.platformService || spec.platformPostgresService}`
: `${fields.secret || expectedSecret}/${fields.key || expectedKey} is not aligned to platform DB`,
};
}
if (hwpodDb) {
return {
ok: false,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || expectedSecret,
key: fields.key || expectedKey,
preset: "hwpod-db",
action: fields.action || null,
dryRun: fields.dryRun === "true",
mutation: false,
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: "HWPOD DB Secret status is only supported for native platform DB lanes",
? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} points to ${fields.platformService || spec.platformPostgresService}`
: `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} is not aligned to platform DB`,
};
}
const keysHealthy = fields.afterExists === "yes" &&
@@ -1546,8 +1739,8 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
return {
ok: commandOk && healthy,
namespace: fields.namespace || spec.namespace,
secret: fields.secret || expectedSecret,
key: fields.key || expectedKey,
secret: fields.secret || spec.cloudApiDbSecret,
key: fields.key || spec.cloudApiDbKey,
preset: "cloud-api-db",
action: fields.action || null,
dryRun: fields.dryRun === "true",
@@ -1573,9 +1766,9 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
},
postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes",
postgresSecret: fields.postgresSecret || spec.postgresSecret,
dbName: fields.dbName || expectedDbName,
dbUser: fields.dbUser || expectedDbUser,
dbHost: fields.dbHost || expectedDbHost,
dbName: fields.dbName || spec.cloudApiDbName,
dbUser: fields.dbUser || spec.cloudApiDbUser,
dbHost: fields.dbHost || spec.cloudApiDbHost,
cloudApiDeployment: fields.cloudApiDeployment || spec.cloudApiDeployment,
applyExitCode: numericField(fields.applyExitCode),
dbEnsureExitCode: numericField(fields.dbEnsureExitCode),
@@ -1584,7 +1777,7 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: healthy ? `${fields.secret || expectedSecret}/${fields.key || expectedKey} exists and runtime database is present` : `${fields.secret || expectedSecret}/${fields.key || expectedKey} or runtime database missing`,
summary: healthy ? `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} exists and runtime database is present` : `${fields.secret || spec.cloudApiDbSecret}/${fields.key || spec.cloudApiDbKey} or runtime database missing`,
};
}
const afterAuthnBytes = numericField(fields.afterAuthnBytes);
@@ -1681,6 +1874,50 @@ function secretStatusFromText(text: string, commandOk: boolean, exitCode: number
};
}
function obsoletePlatformDbStatusFromText(text: string, commandOk: boolean, exitCode: number | null, stderr: string, spec: RuntimeSecretSpec): Record<string, unknown> {
const fields = keyValueLinesFromText(text);
const dryRun = fields.dryRun === "true";
const databaseAbsent = fields.afterDatabaseExists !== "yes" && fields.afterDatabaseExists !== "unknown";
const roleAbsent = fields.afterRoleExists !== "yes" && fields.afterRoleExists !== "unknown";
const probesOk = fields.beforeDatabaseExists !== "unknown" &&
fields.beforeRoleExists !== "unknown" &&
fields.afterDatabaseExists !== "unknown" &&
fields.afterRoleExists !== "unknown";
return {
ok: commandOk && probesOk && (dryRun || (databaseAbsent && roleAbsent)),
database: fields.database || spec.obsoleteHwpodDbName,
role: fields.role || spec.obsoleteHwpodDbUser,
preset: "obsolete-platform-db-cleanup",
action: fields.action || null,
dryRun,
mutation: fields.mutation === "true",
before: {
databaseExists: fields.beforeDatabaseExists === "yes",
roleExists: fields.beforeRoleExists === "yes",
},
after: {
databaseExists: fields.afterDatabaseExists === "yes",
roleExists: fields.afterRoleExists === "yes",
},
beforeProbeExitCode: {
database: numericField(fields.beforeDatabaseProbeExitCode),
role: numericField(fields.beforeRoleProbeExitCode),
},
afterProbeExitCode: {
database: numericField(fields.afterDatabaseProbeExitCode),
role: numericField(fields.afterRoleProbeExitCode),
},
dropDatabaseExitCode: numericField(fields.dropDatabaseExitCode),
dropRoleExitCode: numericField(fields.dropRoleExitCode),
exitCode,
stderr: commandOk ? "" : stderr.trim().slice(0, 2000),
valuesRedacted: true,
summary: probesOk && (dryRun || (databaseAbsent && roleAbsent))
? `${fields.database || spec.obsoleteHwpodDbName} and ${fields.role || spec.obsoleteHwpodDbUser} are ${dryRun ? "observable" : "absent"}`
: `${fields.database || spec.obsoleteHwpodDbName} or ${fields.role || spec.obsoleteHwpodDbUser} still present or unobservable`,
};
}
export function nodeSecretStatusFromTextForTest(text: string, commandOk: boolean, exitCode: number | null, stderr: string, node = "G14", lane = "v03"): Record<string, unknown> {
return secretStatusFromText(text, commandOk, exitCode, stderr, runtimeSecretSpec({ node, lane }));
}