From 8f51bbca910d7b155ecc0d38b79ea11e95e8679d Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 8 Jun 2026 14:18:58 +0000 Subject: [PATCH] refactor: add node-scoped HWLAB lane CLI --- ...b-g14-lanes.yaml => hwlab-node-lanes.yaml} | 16 +- docs/reference/cli.md | 4 +- docs/reference/g14.md | 2 +- scripts/hwlab-g14-contract-test.ts | 37 +- scripts/src/help.ts | 3 +- scripts/src/hwlab-g14.ts | 380 ++++++++++++++---- ...hwlab-g14-lanes.ts => hwlab-node-lanes.ts} | 30 +- scripts/src/hwlab-node.ts | 152 ++++++- 8 files changed, 508 insertions(+), 116 deletions(-) rename config/{hwlab-g14-lanes.yaml => hwlab-node-lanes.yaml} (95%) rename scripts/src/{hwlab-g14-lanes.ts => hwlab-node-lanes.ts} (94%) diff --git a/config/hwlab-g14-lanes.yaml b/config/hwlab-node-lanes.yaml similarity index 95% rename from config/hwlab-g14-lanes.yaml rename to config/hwlab-node-lanes.yaml index 7029e570..9900c872 100644 --- a/config/hwlab-g14-lanes.yaml +++ b/config/hwlab-node-lanes.yaml @@ -7,9 +7,9 @@ nodes: route: G14 kubeRoute: G14:k3s sourceWorkspace: /root/hwlab - gitopsRoot: deploy/gitops/g14 - networkProfile: g14-ci-egress - downloadProfile: g14-default + gitopsRoot: deploy/gitops/node + networkProfile: node-ci-egress + downloadProfile: node-default lanes: v02: @@ -56,7 +56,7 @@ lanes: workspace: /root/hwlab-v03 cicdRepo: /root/hwlab-v03-cicd.git cicdRepoLock: /tmp/hwlab-v03-cicd-repo.lock - app: hwlab-g14-v03 + app: hwlab-node-v03 pipeline: hwlab-v03-ci-image-publish pipelineRunPrefix: hwlab-v03-ci-poll serviceAccountName: hwlab-v03-tekton-runner @@ -68,7 +68,7 @@ lanes: gitopsBranch: v0.3-gitops catalogPath: deploy/artifact-catalog.v03.json runtime: - path: deploy/gitops/g14/runtime-v03 + path: deploy/gitops/node/runtime-v03 namespace: hwlab-v03 renderDir: runtime-v03 tektonDir: tekton-v03 @@ -86,7 +86,7 @@ lanes: apiUrl: http://74.48.78.17:19767 networkProfiles: - g14-ci-egress: + node-ci-egress: proxy: http: http://g14-provider-egress-proxy.unidesk.svc.cluster.local:18789 https: http://g14-provider-egress-proxy.unidesk.svc.cluster.local:18789 @@ -128,7 +128,7 @@ networkProfiles: - host.docker.internal - registry.npmjs.org - .registry.npmjs.org - g14-node-local-vpn: + node-local-vpn: proxy: http: http://127.0.0.1:10808 https: http://127.0.0.1:10808 @@ -155,7 +155,7 @@ networkProfiles: - 127.0.0.1:5000 downloadProfiles: - g14-default: + node-default: git: proxyMode: inherit retries: 3 diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 327330ef..d0a42700 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -52,12 +52,12 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 DEV/PROD 滚动、P - `agentrun v01 control-plane status|trigger-current|refresh|cleanup-runs|cleanup-released-pvs [--dry-run|--confirm]` 是 AgentRun `v0.1` 在 G14 k3s 的受控 Tekton/Argo 入口。`status` 只读汇总固定 source worktree commit、对应 commit-pinned PipelineRun、GitOps latest、Argo Application、`agentrun-v01` manager source commit、`planArtifacts.summary`、env image result 和 git mirror 摘要，并报告 manager/Argo/GitOps 是否对齐当前 source commit。默认输出是 compact commander 视图：`summary` 给出 source、PipelineRun、Argo、manager image、git mirror 和 `aligned` 结论；`timings` 给出 `sourceMs`、`runtimeMs`、`gitMirrorMs` 和 `totalMs`；远端 stdout/stderr tail 默认省略，失败时仍展开必要 tail，完整 tail 用 `--full`，原始 git mirror cache 用 `--raw`。`status` 聚合 source 后会并行读取 runtime 和 git mirror，并向 stderr 输出 `agentrun.control-plane.status.progress` JSON 事件，覆盖 `source`、`runtime`、`git-mirror` 的 started/succeeded/failed 和 elapsedMs，避免 10s 以上状态聚合期间无可见进展；`trigger-current` 先快进 `G14:/root/agentrun-v01` 到 `origin/v0.1`，检查 `devops-infra` mirror 的 `localV01` 是否等于目标 source commit，必要时先执行受控 mirror sync，再创建 `agentrun-v01-ci-` PipelineRun。confirmed trigger 只提交 CI/CD 工作并返回后续 `status` 命令，不等待完整 PipelineRun；同名 PipelineRun 运行中或已成功时拒绝重复触发，只允许失败态重建或首次创建。`refresh` 只对 `argocd/agentrun-g14-v01` 执行 hard refresh，用于 GitOps promotion 已完成但 Argo 仍停留旧 revision 时的受控同步入口；它不直接 patch runtime workload。`cleanup-runs` 只清理 `agentrun-ci` 中已完成且超过 `--min-age-minutes` 的 `agentrun-v01-ci-*` PipelineRun，通过 Tekton ownerRef 回收临时 workspace PVC；dry-run 必须输出候选 PipelineRun、owned PVC、active mount 保护、local-path 实际估算 bytes 和 confirm 命令。`cleanup-released-pvs` 只处理 `agentrun-ci`、`local-path`、`Delete` reclaim policy 的 `Released` PV，用于 PipelineRun 删除后残留 PV 的二次回收；它不触碰 runtime namespace、业务 PVC、Secret、registry storage 或 GitOps desired state。AgentRun 运行时和 SPEC 事实来源仍在 AgentRun 仓库，UniDesk 只维护受控运维入口。 - `agentrun v01 git-mirror status|sync|flush [--dry-run|--confirm]` 是 AgentRun `v0.1` 使用 `devops-infra` git mirror/relay 的受控维护入口。`status` 默认返回 read/write URL、`localV01`、`githubV01`、`localGitops`、`githubGitops`、`pendingFlush`、`githubInSync` 和 exact full-SHA shallow fetch 摘要，不默认展开完整 cache stdout；需要探测 tail 时用 `--full`，需要原始 cache 输出时用 `--raw`。`sync` 创建 manual Job，把 GitHub `v0.1` 和 `v0.1-gitops` refs 拉入 `/cache/pikasTech/agentrun.git`；`flush` 把本地 `v0.1-gitops` 快进推回 GitHub。confirmed `sync`/`flush` 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand` 和日志路径；只有现场同步调试才显式加 `--wait`。该入口与 HWLAB v0.2 mirror 共用 `devops-infra` 服务和 cache PVC，但 repo path、refs、status 文件和 CLI 命令彼此独立。 - `hwlab g14 control-plane status|apply --lane v02 [--dry-run|--confirm]` 是 HWLAB `v0.2` 加法 lane 的受控 Tekton/Argo 控制面维护入口，source commit 只来自 G14 专用 bare repo `/root/hwlab-v02-cicd.git` 的 `refs/remotes/origin/v0.2`；`/root/hwlab-v02` 只作为人工开发和短连接源码工具 workspace 被观测，dirty/stale 状态必须输出为 isolated warning 而不能阻塞 CI/CD。该入口面向 branch `v0.2`、namespace `hwlab-ci` 和 Argo application `hwlab-g14-v02`；默认 `status` 只读汇总最新 source head 的 pipeline、RBAC/ServiceAccount、Argo、当前 commit PipelineRun、当前 PipelineRun 的 TaskRun 条件摘要、最近 PipelineRun 摘要、活跃 PipelineRun、遗留 v02 CronJob 清理状态、commit alignment，以及 19666/19667 的 Cloud Web 静态资源和 API live 探针。分支被后续提交推进后，要复查已完成 run 时使用 `status --lane v02 --pipeline-run hwlab-v02-ci-poll-`；已知完整 source SHA 但不想依赖最新 head 时使用 `status --lane v02 --source-commit `。定点 `status` 输出 `statusTarget.mode` 和 `targetValidation`，只检查指定 PipelineRun/source commit 的证据；`targetValidation.state=passed` 表示该目标已满足 PipelineRun succeeded、Argo `Synced/Healthy`、19666/19667 探针、Git mirror flushed，并且该 run 的 `planArtifacts.rolloutServices` 运行时 source commit 对齐；`planArtifacts.reusedServices` 作为 runtime/provenance 证据呈现，但不能被强制要求等于目标 source commit。`targetValidation.state=superseded` 表示该目标已成功且 runtime 已被同一分支后续成功 PipelineRun 取代，`falseGreenGuard` 在该状态下应标为 superseded/not-applicable。两种状态都不得因为 `origin/v0.2` 后续推进而把历史 run 判为失败；默认不带定点参数时仍严格判定最新 source head alignment。TaskRun 摘要的 `performance` 字段会把超过 120s 的 build TaskRun 标为慢任务、超过 180s 标为 critical warning，用于暴露 env reuse/git mirror 命中率回归，但不作为阻断门禁；CI/CD 性能验收应同时看 `planArtifacts.summary`、`taskRuns.performance.warningCount` 和 PipelineRun duration，纯 CLI/文档或无 runtime 重建需求的后续提交应稳定表现为 `build=0 reuse=` 且无 build TaskRun warning，首次引入或切换 env image 时允许只构建必要 env image 一次。`webAssets` 必须直接给出 `readonly-rpc` 删除、sidebar/workspace/event panel 关键 CSS、`/app.js` 是否可读取和字节数、`/health/live` 与 API revision；`apiRevision` 是 cloud-api 服务自身 revision，Cloud Web 静态资源变更时允许它与 source commit 不同，不能把这种差异误判成 Cloud Web 未发布。默认只读取必要字段，禁止把完整 PipelineRun spec、Tekton 内联脚本、历史大对象或整份 CSS/HTML/JS 展开到默认输出；`apply` 先自动 fetch `/root/hwlab-v02-cicd.git` 并从 commit-pinned detached worktree 执行 render check，再经 `G14:k3s` server-side apply `tekton-v02/rbac.yaml`、`pipeline.yaml`、`argocd/project.yaml` 和 `argocd/application-v02.yaml`，confirmed apply 会删除遗留 v02 CronJob，但不会应用 runtime-v02 workload、Secret 或数据迁移。 -- `hwlab g14 control-plane status|apply|trigger-current --lane v03` 使用 `/root/unidesk/config/hwlab-g14-lanes.yaml` 生成 runtime lane spec。该 YAML 同时声明 `nodes.G14`、`lanes.v02/v03`、`networkProfiles` 和 `downloadProfiles`；status 默认输出 `expected.configPath`、node、network/download profile、有效 `NO_PROXY` 和 Docker build proxy 摘要，confirmed trigger 创建的 PipelineRun annotation 记录 node/profile id。新增 `v0.4+` lane 时应先加 YAML，再让 GitOps render、CI prepare-source/catalog fetch、runtime env 注入和 Secret/DB/bootstrap helper 消费同一 spec；不要在脚本、CI manifest 或 helper 默认值里新增散落的 G14 代理、下载源或端口硬编码。`hyueapi.com` / `.hyueapi.com` 是强制保留的 `NO_PROXY` 条目，lane 覆盖不得移除。 +- `hwlab nodes control-plane status|apply|refresh|trigger-current --node G14 --lane v03` 使用 `/root/unidesk/config/hwlab-node-lanes.yaml` 生成 runtime lane spec。该 YAML 同时声明 `nodes.G14`、`lanes.v02/v03`、`networkProfiles` 和 `downloadProfiles`；status 默认输出 `expected.configPath`、node、network/download profile、有效 `NO_PROXY`、Docker build proxy、Argo Application 和 runtime workload 摘要，confirmed trigger 创建的 PipelineRun annotation 记录 node/profile id。`refresh` 是受控 Argo 恢复入口，只终止 stale Application operation 并执行 hard refresh，不直接 patch runtime workload；用于 GitOps revision 已更新但 Argo 仍卡在旧 hook/operation 时让 Argo 重新从当前 desired state 自愈。新增 `v0.4+` lane 时应先加 YAML，再让 GitOps render、CI prepare-source/catalog fetch、runtime env 注入和 Secret/DB/bootstrap helper 消费同一 spec；不要在脚本、CI manifest 或 helper 默认值里新增散落的 G14 代理、下载源或端口硬编码。`hyueapi.com` / `.hyueapi.com` 是强制保留的 `NO_PROXY` 条目，lane 覆盖不得移除。 - `hwlab g14 control-plane trigger-current --lane v02 [--dry-run|--confirm]` 是 v02 标准手动触发入口：先自动 fetch `/root/hwlab-v02-cicd.git`，解析当前 `origin/v0.2` full SHA，创建 commit-pinned `hwlab-v02-ci-poll-` PipelineRun；读 Git 走 `git-mirror-http.devops-infra.svc.cluster.local`，GitOps promotion 写 `git-mirror-write.devops-infra.svc.cluster.local`；confirmed trigger 在创建 PipelineRun 前会先按当前 source commit 在 G14 临时 detached worktree 中 render，再 server-side apply v02 Tekton RBAC、Pipeline 与 Argo Application，避免 CI/CD 脚本或 runtime-ready 逻辑已合并但集群仍执行旧 Pipeline 定义；该 render 不要求固定 `/root/hwlab-v02` 工作树 clean，也不得因 `.worktree/` 或其他并行未提交修改阻塞；同名 PipelineRun 存在时默认复用现有状态，不删除重建，失败 run 的重试策略必须显式设计，不能恢复默认 delete/create。 创建 PipelineRun 前会读取 `devops-infra` mirror refs，若 `localV02` 未等于当前 source commit，则自动执行一次受控 manual `git-mirror sync` Job 并复核 ref，复核失败时停止触发，避免 Tekton `prepare-source` 已知失败；services 参数只包含 v02 runtime service matrix，`hwlab-cli` 是固定 repo 短连接源码工具，不进入 PipelineRun service build。 `--dry-run` 只报告是否会 pre-sync，不创建 Job；confirmed trigger 默认创建 `.state/jobs/` 异步 job 并立刻返回 `job.id`、`statusCommand`、stdout/stderr 路径，避免 git mirror pre-sync 或 PipelineRun 创建期间长时间阻塞；`--wait` 路径也必须向 stderr 输出 `hwlab.v02.trigger.progress` JSON 事件，覆盖 `control-plane-refresh`、`git-mirror-pre-sync` 和 `create-pipelinerun`，避免异步 job 长时间只有启动命令而无法判断卡点；默认 JSON 必须对 `manifest_b64`、长脚本和远端 stdout/stderr 做有界摘要，保留长度与 hash，最终 trigger 结果只返回阶段摘要和关键 tail，完整内容通过 job stdout/stderr 文件渐进披露；只有现场同步调试才显式加 `--wait`；旧 `rerun-current` 只作为输入别名保留。PipelineRun `Completed`、Argo `Synced/Healthy` 和 `webAssets.ok=true` 只证明 G14 runtime 已更新；交付收口还必须用 `hwlab g14 git-mirror status` 查看 `cache.summary.pendingFlush`，若为 true，继续执行受控 `hwlab g14 git-mirror flush --confirm` 并用 job status 轮询到 `pendingFlush=false`。 - `hwlab g14 control-plane runtime-migration --lane v02 [--dry-run|--allow-live-db-read --dry-run|--confirm]` 只通过 `hwlab-v02` namespace 当前 `deployment/hwlab-cloud-api -c hwlab-cloud-api` 内 repo-owned migration CLI 执行；不读取或打印 Secret 值、不触碰 PROD、不绕到手工 `psql`。 -- `hwlab g14 secret status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm]` 是 HWLAB v0.2 runtime SecretRef bootstrap 的标准入口：OpenFGA preset 确保 preshared token、datastore URI、Postgres password 和对应数据库/角色存在；master server admin API key preset 确保本机 `/root/.config/hwlab-v02/master-server-admin-api-key.env` 以 0600 保存 `HWLAB_API_KEY`，并同步到 `hwlab-v02-master-server-admin-api-key/api-key`。`status` 只返回 key 是否存在、解码后字节数、key prefix 和 DB 对象存在性，永远不读取、不打印、不回传 secret 明文。`hwlab g14 secret delete --lane v02 --name [--dry-run|--confirm]` 只用于删除确认已不被 workload 引用的 v0.2 废弃 Secret，默认 dry-run，拒绝删除 OpenFGA/Postgres/master admin API key 等必需 Secret；共享 device-pod API key 已退出当前授权路径，不再提供 ensure/bootstrap 入口。 +- `hwlab g14 secret status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm]` 和 `hwlab nodes secret status|ensure --node G14 --lane v03 --name hwlab-v03-openfga|hwlab-v03-master-server-admin-api-key [--dry-run|--confirm]` 是 HWLAB runtime lane SecretRef bootstrap 的标准入口：OpenFGA preset 确保 preshared token、datastore URI 和 Postgres password 存在；v02 继续确保独立 `hwlab_openfga` 数据库/角色存在，v03+ 默认从 lane spec 派生 namespace、Postgres Secret 和主 DB 用户，避免复制 v02 专用硬编码。master server admin API key preset 确保本机 `/root/.config/hwlab-v0x/master-server-admin-api-key.env` 以 0600 保存 `HWLAB_API_KEY`，并同步到对应 lane 的 `*-master-server-admin-api-key/api-key`。`status` 只返回 key 是否存在、解码后字节数、key prefix 和 DB 对象存在性，永远不读取、不打印、不回传 secret 明文。`hwlab g14 secret delete --lane v02 --name [--dry-run|--confirm]` 只用于删除确认已不被 workload 引用的 v0.2 废弃 Secret，默认 dry-run，拒绝删除 OpenFGA/Postgres/master admin API key 等必需 Secret；共享 device-pod API key 已退出当前授权路径，不再提供 ensure/bootstrap 入口。 - `hwlab g14 control-plane cleanup-runs --lane v02|g14|all [--min-age-minutes N] [--limit N] [--dry-run|--confirm]` 是完成态 PipelineRun 工作区 retention 入口；真实清理只删除已完成 PipelineRun，让 Tekton/local-path 回收临时 PVC，不触碰 registry storage、业务 PVC、Secret、runtime workload 或 GitOps desired state。 - `hwlab g14 control-plane cleanup-released-pvs --lane all [--limit N] [--dry-run|--confirm]` 是 local-path 未自动回收后的补充 retention 入口；只列并删除 `Released`、`local-path`、`Delete`、`claimNamespace=hwlab-ci` 且 claim 名称形如 Tekton 临时 `pvc-*` 的 PV。 - `hwlab g14 git-mirror status|apply|sync|flush [--dry-run|--confirm]` 是 `devops-infra` git mirror/relay 的受控维护入口：`apply` 渲染并 server-side apply `devops-infra/git-mirror.yaml`，同时删除遗留 `git-mirror-hwlab-sync` CronJob；`sync` 创建一次性 manual Job，把 GitHub allowlist refs 拉入本地 mirror；`flush` 创建一次性 manual Job，把本地 `v0.2-gitops` 快进推回 GitHub。 diff --git a/docs/reference/g14.md b/docs/reference/g14.md index 29326b0e..0201677f 100644 --- a/docs/reference/g14.md +++ b/docs/reference/g14.md @@ -69,7 +69,7 @@ The `v0.2` CI/CD integration must be additive: add a manual UniDesk trigger, ded For current G14/v0.2, `deploy/deploy.yaml` is the single human-authored deploy/runtime config source. `deploy/deploy.json` is not a v0.2 compatibility source and must not be recreated for this lane. YAML and JSON parsing are centralized in the HWLAB repo's format-agnostic config layer: `scripts/src/structured-config.mjs` handles file format parsing/writing, while `scripts/src/deploy-config.mjs` owns deploy-config defaults and shape. Renderers, planners, smoke scripts and CLIs should consume that layer through `readStructuredFile` / `writeStructuredFile` / `readDeployConfig` or equivalent helpers; do not scatter direct YAML parser imports or ad hoc `readFile` + `YAML.parse` calls. Legacy D601 HWLAB CD still has its own `deploy/deploy.json` desired-state documented in `docs/reference/hwlab.md`; that legacy path is not the G14/v0.2 authority. -On the UniDesk control-plane side, HWLAB G14 runtime lane expansion is sourced from `/root/unidesk/config/hwlab-g14-lanes.yaml`. That YAML owns `nodes`, `lanes`, `networkProfiles` and `downloadProfiles` for the UniDesk trigger/status/apply layer: `lanes.v03.node` points at `nodes.G14`, while proxy URLs, `NO_PROXY`, git/npm/pip/docker/curl retry/download defaults and Docker build proxy settings live under profile objects. G14 must remain a node config value, not a hardcoded code path such as `g14Proxy` or `g14GitOps`; future `v0.4+` lanes should be added by adding YAML entries and then consuming the generated spec. `hyueapi.com` and `.hyueapi.com` are required `NO_PROXY` entries and must remain present in effective runtime and Docker build proxy environments. +On the UniDesk control-plane side, HWLAB G14 runtime lane expansion is sourced from `/root/unidesk/config/hwlab-node-lanes.yaml`. That YAML owns `nodes`, `lanes`, `networkProfiles` and `downloadProfiles` for the UniDesk trigger/status/apply layer: `lanes.v03.node` points at `nodes.G14`, while proxy URLs, `NO_PROXY`, git/npm/pip/docker/curl retry/download defaults and Docker build proxy settings live under profile objects. G14 must remain a node config value, not a hardcoded code path such as `g14Proxy` or `g14GitOps`; future `v0.4+` lanes should be added by adding YAML entries and then consuming the generated spec. `hyueapi.com` and `.hyueapi.com` are required `NO_PROXY` entries and must remain present in effective runtime and Docker build proxy environments. The `devops-infra` git mirror/relay remains manual and CLI-controlled, not CronJob-driven. The standard `v0.2` delivery trigger is `bun scripts/cli.ts hwlab g14 monitor-prs --lane v02`: it watches base=`v0.2` PRs, waits for GitHub preflight/CI readiness, auto-merges only ready and non-conflicting PRs, then drives the same controlled CD path and comments pending/blocked/succeeded/failed/timeout state back to the PR. The lower-level `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm` remains the manual recovery or diagnosis entry; it must fetch `/root/hwlab-v02-cicd.git`, resolve the current `origin/v0.2` source commit, check the mirror's `localV02` ref before creating the PipelineRun, run one bounded manual `git-mirror sync` Job when the mirror is stale, and only continue after the mirror ref matches the current source commit. Use `hwlab g14 git-mirror sync --confirm` directly only for explicit mirror maintenance or diagnosis. diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index 9d990617..d92b9ac8 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -1,5 +1,6 @@ import { activeV02PipelineRuns, g14ObservabilityQueryAssertion, gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parseK8sCpuMillicores, parseK8sMemoryMiB, parsePipelineTaskRunMetrics, parseV02TriggerSnapshot, rolloutRecordBody, runtimeLanePipelineRunManifest, semanticChangelogBullets, summarizeV02CdStatus, v02CloseoutVerdict, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02ExistingPipelineRunReuseDecision, v02FalseGreenGuard, v02GitMirrorPreSyncWaitMs, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableGitMirrorPreSyncMarker, v02ReusableRefreshMarker, v02StatusHistoryPolicy, v02TaskRunPerformanceSummary } from "./src/hwlab-g14"; -import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec } from "./src/hwlab-g14-lanes"; +import { hwlabNodeHelp } from "./src/hwlab-node"; +import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec } from "./src/hwlab-node-lanes"; import { runCommand } from "./src/command"; function assertCondition(condition: unknown, message: string, detail: unknown = {}): void { @@ -36,6 +37,7 @@ assertCondition( const hwlabHelp = hwlabG14Help(); const hwlabHelpUsage = Array.isArray(hwlabHelp.usage) ? hwlabHelp.usage.map((line: unknown) => String(line)) : []; const hwlabHelpJson = JSON.stringify(hwlabHelp); +const hwlabNodeHelpJson = JSON.stringify(hwlabNodeHelp()); assertCondition( hwlabHelpUsage.some((line) => line.includes("control-plane status --lane v02 --pipeline-run")) && hwlabHelpUsage.some((line) => line.includes("control-plane status --lane v02 --source-commit")) @@ -48,17 +50,18 @@ assertCondition( hwlabHelpUsage, ); assertCondition( - hwlabHelpUsage.some((line) => line.includes("control-plane status --lane v03")) - && hwlabHelpUsage.some((line) => line.includes("control-plane apply --lane v03 --dry-run")) - && hwlabHelpUsage.some((line) => line.includes("control-plane trigger-current --lane v03 --dry-run")) - && hwlabHelpJson.includes("runtime lane v02/v03"), - "v0.3 control-plane help must expose the runtime lane bootstrap status/apply/trigger entrypoints", - hwlabHelpUsage, + hwlabHelpUsage.some((line) => line.includes("hwlab nodes control-plane status --node G14 --lane v03")) + && hwlabHelpUsage.some((line) => line.includes("hwlab nodes control-plane apply --node G14 --lane v03 --dry-run")) + && hwlabHelpUsage.some((line) => line.includes("hwlab nodes control-plane trigger-current --node G14 --lane v03 --dry-run")) + && hwlabNodeHelpJson.includes("hwlab nodes control-plane status --node G14 --lane v03") + && hwlabHelpJson.includes("config/hwlab-node-lanes.yaml"), + "v0.3 control-plane help must expose node-scoped runtime lane bootstrap status/apply/trigger entrypoints", + { hwlabHelpUsage, hwlabNodeHelp: hwlabNodeHelp() }, ); const v03LaneSpec = hwlabRuntimeLaneSpec("v03"); assertCondition( JSON.stringify(hwlabRuntimeLaneIds()) === JSON.stringify(["v02", "v03"]) - && hwlabRuntimeLaneConfigPath() === "config/hwlab-g14-lanes.yaml" + && hwlabRuntimeLaneConfigPath() === "config/hwlab-node-lanes.yaml" && v03LaneSpec.nodeId === "G14" && v03LaneSpec.nodeRoute === "G14" && v03LaneSpec.sourceBranch === "v0.3" @@ -70,8 +73,8 @@ assertCondition( && v03LaneSpec.pipeline === "hwlab-v03-ci-image-publish" && v03LaneSpec.publicWebUrl.endsWith(":19766") && v03LaneSpec.publicApiUrl.endsWith(":19767") - && v03LaneSpec.networkProfileId === "g14-ci-egress" - && v03LaneSpec.downloadProfileId === "g14-default" + && v03LaneSpec.networkProfileId === "node-ci-egress" + && v03LaneSpec.downloadProfileId === "node-default" && v03LaneSpec.networkProfile.proxy.noProxy.includes("hyueapi.com") && v03LaneSpec.networkProfile.proxy.noProxy.includes(".hyueapi.com") && v03LaneSpec.networkProfile.dockerBuildProxy.noProxy.includes("hyueapi.com") @@ -82,9 +85,9 @@ assertCondition( v03LaneSpec, ); assertCondition( - hwlabHelpJson.includes("config/hwlab-g14-lanes.yaml") - && hwlabHelpJson.includes("g14-ci-egress") - && hwlabHelpJson.includes("g14-default") + hwlabHelpJson.includes("config/hwlab-node-lanes.yaml") + && hwlabHelpJson.includes("node-ci-egress") + && hwlabHelpJson.includes("node-default") && hwlabHelpJson.includes("hyueapi.com"), "G14 HWLAB help must expose the runtime lane YAML and required network/download profile identifiers", hwlabHelp, @@ -93,8 +96,8 @@ const v03PipelineRunManifest = runtimeLanePipelineRunManifest(v03LaneSpec, "aaaa const v03PipelineRunAnnotations = record(record(v03PipelineRunManifest.metadata).annotations); assertCondition( v03PipelineRunAnnotations["hwlab.pikastech.local/node"] === "G14" - && v03PipelineRunAnnotations["hwlab.pikastech.local/network-profile"] === "g14-ci-egress" - && v03PipelineRunAnnotations["hwlab.pikastech.local/download-profile"] === "g14-default" + && v03PipelineRunAnnotations["hwlab.pikastech.local/network-profile"] === "node-ci-egress" + && v03PipelineRunAnnotations["hwlab.pikastech.local/download-profile"] === "node-default" && String(v03PipelineRunAnnotations["hwlab.pikastech.local/no-proxy-required"]).includes("hyueapi.com"), "runtime lane PipelineRun manifest must preserve node/network/download profile provenance", v03PipelineRunAnnotations, @@ -109,8 +112,8 @@ assertCondition( ); assertCondition( hwlabHelpUsage.some((line) => line.includes("git-mirror apply --lane v02 --confirm")) - && hwlabHelpUsage.some((line) => line.includes("git-mirror apply --lane v03 --confirm")), - "git mirror help must expose lane-selected apply so v0.3 mirror config is not rendered through v0.2", + && hwlabHelpUsage.some((line) => line.includes("hwlab nodes git-mirror apply --node G14 --lane v03 --confirm")), + "git mirror help must expose node-scoped v0.3 apply so v0.3 mirror config is not rendered through v0.2", hwlabHelpUsage, ); assertCondition( diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 6842a8ef..87014453 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -59,7 +59,8 @@ export function rootHelp(): unknown { { command: "auth-broker contract|health --dry-run|credential-request --dry-run|pr-preflight --dry-run", description: "Inspect the P0 Rust auth broker and CLI adapter contract without reading token values, writing GitHub, or starting services." }, { command: "gh preflight|auth|issue|pr", description: "Run safe GitHub issue and PR CRUD/lifecycle operations through REST with body-file update replace/append, comment delete, token diagnostics, PR closeout preflight, hard delete unsupported, and guarded PR merge." }, { command: "commander contract|plan --dry-run|smoke --dry-run|approval request --dry-run|prompt-lint --kind gpt55-pr", description: "Host Codex commander skeleton contract, no-daemon smoke plan, dry-run approval preview, and advisory GPT-5.5 PR prompt boundary lint without live bridges, message sends, or submit gating." }, - { command: "hwlab nodes secret status|ensure --node G14 --lane v03 | hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Manage HWLAB node/lane runtime prerequisites, start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." }, + { command: "hwlab nodes control-plane|git-mirror|secret --node G14 --lane v03", description: "Manage HWLAB node/lane runtime prerequisites for v0.3+ with the node identity passed as data instead of a command family." }, + { command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." }, { command: "agentrun v01 control-plane status|trigger-current|refresh|cleanup-runs|cleanup-released-pvs", description: "Run bounded AgentRun v0.1 Tekton/Argo status, manual PipelineRun trigger, Argo refresh, and completed CI workspace retention through UniDesk G14 routes." }, { command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." }, { command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." }, diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 72771134..c09d13de 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -4,7 +4,7 @@ import { createHash, randomBytes } from "node:crypto"; import { repoRoot, rootPath, type Config } from "./config"; import { runCommand } from "./command"; import { readJob, startJob } from "./jobs"; -import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-g14-lanes"; +import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; const HWLAB_REPO = "pikasTech/HWLAB"; const G14_SOURCE_BRANCH = "G14"; @@ -24,7 +24,6 @@ const V02_POLLER = "hwlab-v02-branch-poller"; const V02_RECONCILER = "hwlab-v02-control-plane-reconciler"; const V02_PIPELINERUN_PREFIX = V02_LANE_SPEC.pipelineRunPrefix; const V02_CONTROL_PLANE_FIELD_MANAGER = V02_LANE_SPEC.controlPlaneFieldManager; -const V02_SECRET_FIELD_MANAGER = "unidesk-hwlab-v02-secret"; const V02_GIT_URL = V02_LANE_SPEC.gitUrl; const V02_GIT_READ_URL = V02_LANE_SPEC.gitReadUrl; const V02_GIT_WRITE_URL = V02_LANE_SPEC.gitWriteUrl; @@ -123,7 +122,7 @@ interface G14RecordRolloutOptions { } interface G14ControlPlaneOptions { - action: "status" | "closeout" | "apply" | "trigger-current" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration"; + action: "status" | "closeout" | "apply" | "trigger-current" | "refresh" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration"; lane: HwlabRuntimeLane | "g14" | "all"; dryRun: boolean; confirm: boolean; @@ -188,7 +187,7 @@ interface G14ObservabilityOptions { interface G14SecretOptions { action: "status" | "ensure" | "delete"; - lane: "v02"; + lane: HwlabRuntimeLane; dryRun: boolean; confirm: boolean; name: string; @@ -357,11 +356,12 @@ function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions { actionRaw !== "closeout" && actionRaw !== "apply" && actionRaw !== "trigger-current" && + actionRaw !== "refresh" && actionRaw !== "cleanup-runs" && actionRaw !== "cleanup-released-pvs" && actionRaw !== "runtime-migration" ) { - throw new Error("control-plane usage: status|apply|trigger-current --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]"); + throw new Error("control-plane usage: status|apply|trigger-current|refresh --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]"); } const laneRaw = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02"); let lane: G14ControlPlaneOptions["lane"]; @@ -538,16 +538,22 @@ function parseObservabilityOptions(args: string[]): G14ObservabilityOptions { function parseSecretOptions(args: string[]): G14SecretOptions { const [actionRaw] = args; if (actionRaw !== "status" && actionRaw !== "ensure" && actionRaw !== "delete") { - throw new Error("secret usage: status|ensure --lane v02 --name hwlab-v02-openfga|hwlab-v02-master-server-admin-api-key [--dry-run|--confirm] | delete --lane v02 --name [--dry-run|--confirm]"); + throw new Error("secret usage: status|ensure --lane v02|v03 --name hwlab-v0x-openfga|hwlab-v0x-master-server-admin-api-key [--dry-run|--confirm] | delete --lane v02 --name [--dry-run|--confirm]"); } - const lane = optionValue(args, "--lane") ?? "v02"; - if (lane !== "v02") throw new Error("secret currently supports --lane v02"); - const name = optionValue(args, "--name") ?? V02_OPENFGA_SECRET; + const laneRaw = optionValue(args, "--lane") ?? "v02"; + if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`secret --lane must be one of ${hwlabRuntimeLaneIds().join(", ")}`); + const lane = laneRaw; + const spec = hwlabRuntimeLaneSpec(lane); + const openFgaSecret = runtimeLaneOpenFgaSecretName(spec); + const masterAdminSecret = runtimeLaneMasterAdminApiKeySecretName(spec); + const postgresSecret = runtimeLanePostgresSecretName(spec); + const name = optionValue(args, "--name") ?? openFgaSecret; const key = optionValue(args, "--key"); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("secret accepts only one of --confirm or --dry-run"); if (actionRaw === "delete") { + if (lane !== "v02") throw new Error("secret delete is currently limited to --lane v02 obsolete cleanup"); if (key !== undefined) throw new Error("secret delete does not accept --key; it deletes a whole obsolete Secret object"); if (!/^hwlab-v02-[a-z0-9-]+$/u.test(name)) throw new Error("secret delete requires a hwlab-v02-* Secret name"); if (name === V02_OPENFGA_SECRET || name === V02_MASTER_ADMIN_API_KEY_SECRET || name === "hwlab-v02-postgres") throw new Error(`secret delete refuses required v0.2 Secret ${name}`); @@ -561,8 +567,8 @@ function parseSecretOptions(args: string[]): G14SecretOptions { timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600), }; } - if (name === V02_MASTER_ADMIN_API_KEY_SECRET) { - if (key !== undefined && key !== V02_MASTER_ADMIN_API_KEY_SECRET_KEY) throw new Error(`secret ${V02_MASTER_ADMIN_API_KEY_SECRET} supports only key ${V02_MASTER_ADMIN_API_KEY_SECRET_KEY}`); + if (name === masterAdminSecret) { + if (key !== undefined && key !== V02_MASTER_ADMIN_API_KEY_SECRET_KEY) throw new Error(`secret ${masterAdminSecret} supports only key ${V02_MASTER_ADMIN_API_KEY_SECRET_KEY}`); return { action: actionRaw, lane, @@ -574,12 +580,13 @@ function parseSecretOptions(args: string[]): G14SecretOptions { timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 120, 600), }; } - if (name !== V02_OPENFGA_SECRET) { - throw new Error(`secret status/ensure currently supports only --name ${V02_OPENFGA_SECRET} or ${V02_MASTER_ADMIN_API_KEY_SECRET}; use secret delete for obsolete Secret objects`); + if (name !== openFgaSecret) { + throw new Error(`secret status/ensure currently supports only --name ${openFgaSecret} or ${masterAdminSecret} for --lane ${lane}; use secret delete for obsolete v02 Secret objects`); } if (key !== undefined && key !== V02_OPENFGA_AUTHN_SECRET_KEY && key !== V02_OPENFGA_DATASTORE_URI_SECRET_KEY && key !== V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY) { - throw new Error(`secret ${V02_OPENFGA_SECRET} supports keys ${V02_OPENFGA_AUTHN_SECRET_KEY}, ${V02_OPENFGA_DATASTORE_URI_SECRET_KEY}, and ${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY}`); + throw new Error(`secret ${openFgaSecret} supports keys ${V02_OPENFGA_AUTHN_SECRET_KEY}, ${V02_OPENFGA_DATASTORE_URI_SECRET_KEY}, and ${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY}`); } + if (lane !== "v02" && key === V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY) throw new Error(`${openFgaSecret}/${V02_OPENFGA_POSTGRES_PASSWORD_SECRET_KEY} is derived from ${postgresSecret}/POSTGRES_PASSWORD`); return { action: actionRaw, lane, @@ -717,17 +724,43 @@ function hwlabApiKeyPrefix(value: string): string { return `hwl_live_${segment}`.slice(0, 24); } -function localMasterAdminApiKeyStatus(): Record { - if (!existsSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV)) { - return { exists: false, path: V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, mode: null, valueBytes: 0, keyPrefix: null }; +function runtimeLaneSecretFieldManager(spec: HwlabRuntimeLaneSpec): string { + return `unidesk-hwlab-${spec.lane}-secret`; +} + +function runtimeLanePostgresSecretName(spec: HwlabRuntimeLaneSpec): string { + return `${spec.runtimeNamespace}-postgres`; +} + +function runtimeLanePrimaryDbName(spec: HwlabRuntimeLaneSpec): string { + return `hwlab_${spec.lane}`; +} + +function runtimeLaneOpenFgaSecretName(spec: HwlabRuntimeLaneSpec): string { + return `hwlab-${spec.lane}-openfga`; +} + +function runtimeLaneMasterAdminApiKeySecretName(spec: HwlabRuntimeLaneSpec): string { + return `hwlab-${spec.lane}-master-server-admin-api-key`; +} + +function runtimeLaneMasterAdminApiKeyLocalEnv(spec: HwlabRuntimeLaneSpec): string { + if (spec.lane === "v02") return V02_MASTER_ADMIN_API_KEY_LOCAL_ENV; + return `/root/.config/hwlab-${spec.lane}/master-server-admin-api-key.env`; +} + +function localMasterAdminApiKeyStatus(spec: HwlabRuntimeLaneSpec): Record { + const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec); + if (!existsSync(envPath)) { + return { exists: false, path: envPath, mode: null, valueBytes: 0, keyPrefix: null }; } - const stat = statSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV); - const content = readFileSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, "utf8"); + const stat = statSync(envPath); + const content = readFileSync(envPath, "utf8"); const match = content.match(/^HWLAB_API_KEY=(.+)$/mu); const value = match?.[1]?.trim() ?? ""; return { exists: true, - path: V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, + path: envPath, mode: `0${(stat.mode & 0o777).toString(8)}`, valueBytes: Buffer.byteLength(value, "utf8"), keyPrefix: value ? hwlabApiKeyPrefix(value) : null, @@ -735,21 +768,22 @@ function localMasterAdminApiKeyStatus(): Record { }; } -function readOrCreateLocalMasterAdminApiKey(dryRun: boolean): { key: string | null; created: boolean; status: Record } { - const existing = localMasterAdminApiKeyStatus(); +function readOrCreateLocalMasterAdminApiKey(spec: HwlabRuntimeLaneSpec, dryRun: boolean): { key: string | null; created: boolean; status: Record } { + const envPath = runtimeLaneMasterAdminApiKeyLocalEnv(spec); + const existing = localMasterAdminApiKeyStatus(spec); if (existing.exists === true) { - const content = readFileSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, "utf8"); + const content = readFileSync(envPath, "utf8"); const match = content.match(/^HWLAB_API_KEY=(.+)$/mu); const key = match?.[1]?.trim() ?? ""; - if (!key.startsWith("hwl_live_")) throw new Error(`${V02_MASTER_ADMIN_API_KEY_LOCAL_ENV} exists but does not contain a hwl_live_ HWLAB_API_KEY`); - if (!dryRun && existing.mode !== "0600") chmodSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, 0o600); - return { key, created: false, status: localMasterAdminApiKeyStatus() }; + if (!key.startsWith("hwl_live_")) throw new Error(`${envPath} exists but does not contain a hwl_live_ HWLAB_API_KEY`); + if (!dryRun && existing.mode !== "0600") chmodSync(envPath, 0o600); + return { key, created: false, status: localMasterAdminApiKeyStatus(spec) }; } if (dryRun) return { key: null, created: false, status: existing }; const key = generateHwlabApiKey(); - mkdirSync(dirname(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV), { recursive: true, mode: 0o700 }); - writeFileSync(V02_MASTER_ADMIN_API_KEY_LOCAL_ENV, `# HWLAB v0.2 master server admin API key; do not commit or print.\nHWLAB_API_KEY=${key}\n`, { mode: 0o600 }); - return { key, created: true, status: localMasterAdminApiKeyStatus() }; + mkdirSync(dirname(envPath), { recursive: true, mode: 0o700 }); + writeFileSync(envPath, `# HWLAB ${spec.version} master server admin API key; do not commit or print.\nHWLAB_API_KEY=${key}\n`, { mode: 0o600 }); + return { key, created: true, status: localMasterAdminApiKeyStatus(spec) }; } function redactLargePayloads(value: string): string { @@ -2877,10 +2911,18 @@ function runtimeLaneControlPlaneRenderScript(spec: HwlabRuntimeLaneSpec, sourceC " exit 42", " fi", "fi", - "if [ -f scripts/run-bun.mjs ]; then", - ` node scripts/run-bun.mjs scripts/g14-gitops-render.mjs --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, + "if [ -f scripts/gitops-render.mjs ]; then", + " render_script=scripts/gitops-render.mjs", + "elif [ \"$render_lane\" = v02 ] && [ -f scripts/g14-gitops-render.mjs ]; then", + " render_script=scripts/g14-gitops-render.mjs", "else", - ` node scripts/g14-gitops-render.mjs --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, + ` echo "${spec.version} control-plane render script missing: scripts/gitops-render.mjs" >&2`, + " exit 43", + "fi", + "if [ -f scripts/run-bun.mjs ]; then", + ` node scripts/run-bun.mjs "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, + "else", + ` node "$render_script" --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, "fi", ].join("\n"); } @@ -3664,11 +3706,158 @@ function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: V02Co }, next: { apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm`, + refresh: `bun scripts/cli.ts hwlab g14 control-plane refresh --lane ${spec.lane} --confirm`, triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm`, }, }; } +function runtimeLaneArgoRefreshScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string { + const operationPatch = JSON.stringify({ operation: null }); + return [ + "set +e", + `app=${shellQuote(spec.app)}`, + `argo_namespace=${shellQuote(ARGO_NAMESPACE)}`, + `runtime_namespace=${shellQuote(spec.runtimeNamespace)}`, + `lane=${shellQuote(spec.lane)}`, + `dry_run=${shellQuote(dryRun ? "true" : "false")}`, + `operation_patch=${shellQuote(operationPatch)}`, + "app_jsonpath() { kubectl -n \"$argo_namespace\" get application \"$app\" -o \"jsonpath=$1\" 2>/dev/null || true; }", + "job_jsonpath() { kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o \"jsonpath=$1\" 2>/dev/null || true; }", + "before_operation_revision=$(app_jsonpath '{.operation.sync.revision}')", + "before_operation_phase=$(app_jsonpath '{.status.operationState.phase}')", + "before_operation_message=$(app_jsonpath '{.status.operationState.message}')", + "before_sync_revision=$(app_jsonpath '{.status.sync.revision}')", + "before_sync_status=$(app_jsonpath '{.status.sync.status}')", + "before_health=$(app_jsonpath '{.status.health.status}')", + "before_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')", + "before_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)", + "before_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')", + "before_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')", + "before_hook_active=$(job_jsonpath '{.status.active}')", + "before_operation_present=$([ -n \"$before_operation_revision$before_operation_phase$before_operation_message\" ] && printf yes || printf no)", + "patch_exit=", + "patch_output=skipped-no-live-operation", + "annotate_exit=", + "annotate_output=", + "if [ \"$before_operation_present\" = yes ]; then", + " if [ \"$dry_run\" = true ]; then", + " patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" --dry-run=server -o name 2>&1)", + " else", + " patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" -o name 2>&1)", + " fi", + " patch_exit=$?", + "fi", + "if [ \"$dry_run\" = true ]; then", + " annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)", + "else", + " annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)", + "fi", + "annotate_exit=$?", + "if [ \"$dry_run\" != true ]; then sleep 2; fi", + "after_operation_revision=$(app_jsonpath '{.operation.sync.revision}')", + "after_operation_phase=$(app_jsonpath '{.status.operationState.phase}')", + "after_operation_message=$(app_jsonpath '{.status.operationState.message}')", + "after_sync_revision=$(app_jsonpath '{.status.sync.revision}')", + "after_sync_status=$(app_jsonpath '{.status.sync.status}')", + "after_health=$(app_jsonpath '{.status.health.status}')", + "after_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')", + "after_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)", + "after_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')", + "after_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')", + "after_hook_active=$(job_jsonpath '{.status.active}')", + "after_operation_present=$([ -n \"$after_operation_revision$after_operation_phase$after_operation_message\" ] && printf yes || printf no)", + "printf 'app\\t%s\\n' \"$app\"", + "printf 'lane\\t%s\\n' \"$lane\"", + "printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"", + "printf 'dryRun\\t%s\\n' \"$dry_run\"", + "printf 'beforeOperationPresent\\t%s\\n' \"$before_operation_present\"", + "printf 'beforeOperationRevision\\t%s\\n' \"$before_operation_revision\"", + "printf 'beforeOperationPhase\\t%s\\n' \"$before_operation_phase\"", + "printf 'beforeOperationMessage\\t%s\\n' \"$before_operation_message\"", + "printf 'beforeSyncRevision\\t%s\\n' \"$before_sync_revision\"", + "printf 'beforeSyncStatus\\t%s\\n' \"$before_sync_status\"", + "printf 'beforeHealth\\t%s\\n' \"$before_health\"", + "printf 'beforeResourceVersion\\t%s\\n' \"$before_resource_version\"", + "printf 'beforeHookJob\\t%s\\n' \"$before_hook_job\"", + "printf 'beforeHookSourceCommit\\t%s\\n' \"$before_hook_source\"", + "printf 'beforeHookPhase\\t%s\\n' \"$before_hook_phase\"", + "printf 'beforeHookActive\\t%s\\n' \"$before_hook_active\"", + "printf 'patchExitCode\\t%s\\n' \"$patch_exit\"", + "printf 'patchOutput\\t%s\\n' \"$patch_output\"", + "printf 'annotateExitCode\\t%s\\n' \"$annotate_exit\"", + "printf 'annotateOutput\\t%s\\n' \"$annotate_output\"", + "printf 'afterOperationPresent\\t%s\\n' \"$after_operation_present\"", + "printf 'afterOperationRevision\\t%s\\n' \"$after_operation_revision\"", + "printf 'afterOperationPhase\\t%s\\n' \"$after_operation_phase\"", + "printf 'afterOperationMessage\\t%s\\n' \"$after_operation_message\"", + "printf 'afterSyncRevision\\t%s\\n' \"$after_sync_revision\"", + "printf 'afterSyncStatus\\t%s\\n' \"$after_sync_status\"", + "printf 'afterHealth\\t%s\\n' \"$after_health\"", + "printf 'afterResourceVersion\\t%s\\n' \"$after_resource_version\"", + "printf 'afterHookJob\\t%s\\n' \"$after_hook_job\"", + "printf 'afterHookSourceCommit\\t%s\\n' \"$after_hook_source\"", + "printf 'afterHookPhase\\t%s\\n' \"$after_hook_phase\"", + "printf 'afterHookActive\\t%s\\n' \"$after_hook_active\"", + "if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi", + "if [ -n \"$annotate_exit\" ] && [ \"$annotate_exit\" != 0 ]; then exit \"$annotate_exit\"; fi", + ].join("\n"); +} + +function refreshRuntimeLaneArgoApplication(spec: HwlabRuntimeLaneSpec, options: G14ControlPlaneOptions): Record { + const result = g14K3s(["script", "--", runtimeLaneArgoRefreshScript(spec, options.dryRun)], options.timeoutSeconds * 1000); + const fields = keyValueLinesFromText(statusText(result)); + const ok = isCommandSuccess(result); + return { + ok, + command: `hwlab g14 control-plane refresh --lane ${spec.lane}`, + lane: spec.lane, + mode: options.dryRun ? "dry-run" : "confirmed-refresh", + argoApplication: spec.app, + runtimeNamespace: spec.runtimeNamespace, + dryRun: options.dryRun, + action: { + terminatedOperation: fields.beforeOperationPresent === "yes", + hardRefreshRequested: numericField(fields.annotateExitCode) === 0, + mutation: !options.dryRun && ok, + }, + before: { + operationPresent: fields.beforeOperationPresent === "yes", + operationRevision: fields.beforeOperationRevision || null, + operationPhase: fields.beforeOperationPhase || null, + operationMessage: fields.beforeOperationMessage || null, + syncRevision: fields.beforeSyncRevision || null, + syncStatus: fields.beforeSyncStatus || null, + health: fields.beforeHealth || null, + resourceVersion: fields.beforeResourceVersion || null, + hookJob: fields.beforeHookJob || null, + hookSourceCommit: fields.beforeHookSourceCommit || null, + hookPhase: fields.beforeHookPhase || null, + hookActive: fields.beforeHookActive || null, + }, + after: { + operationPresent: fields.afterOperationPresent === "yes", + operationRevision: fields.afterOperationRevision || null, + operationPhase: fields.afterOperationPhase || null, + operationMessage: fields.afterOperationMessage || null, + syncRevision: fields.afterSyncRevision || null, + syncStatus: fields.afterSyncStatus || null, + health: fields.afterHealth || null, + resourceVersion: fields.afterResourceVersion || null, + hookJob: fields.afterHookJob || null, + hookSourceCommit: fields.afterHookSourceCommit || null, + hookPhase: fields.afterHookPhase || null, + hookActive: fields.afterHookActive || null, + }, + patchExitCode: numericField(fields.patchExitCode), + annotateExitCode: numericField(fields.annotateExitCode), + result: compactCommandResult(result), + next: { + status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane}`, + }, + }; +} + function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14ControlPlaneOptions): Record { if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-runs" || options.action === "cleanup-released-pvs") { return { @@ -3676,7 +3865,7 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`, lane: spec.lane, degradedReason: "unsupported-runtime-lane-action", - message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current`, + message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current/refresh`, }; } if (options.action === "status" && options.pipelineRun !== undefined) { @@ -3722,6 +3911,9 @@ function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14Cont if (options.action === "status") { return runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }); } + if (options.action === "refresh") { + return refreshRuntimeLaneArgoApplication(spec, options); + } if (options.action === "apply") { const render = runRuntimeLaneRenderToTemp(spec, sourceCommit); if (!isCommandSuccess(render.result)) { @@ -3937,6 +4129,7 @@ function runV02ControlPlane(options: G14ControlPlaneOptions): Record/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", "decoded_value() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null || true; fi; }", "decoded_length() { if [ -n \"$1\" ]; then printf '%s' \"$1\" | base64 -d 2>/dev/null | wc -c | tr -d ' '; else printf '0'; fi; }", - "psql_scalar() { kubectl -n \"$namespace\" exec statefulset/hwlab-v02-postgres -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U hwlab_v02 -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", + "psql_scalar() { kubectl -n \"$namespace\" exec \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -U \"$postgres_admin_user\" -d postgres -tAc \"$1\" 2>/dev/null | tr -d '[:space:]'; }", "probe_db() {", + " if [ \"$db_mode\" = primary ]; then", + " role_result=primary", + " database_result=primary", + " probe_exit=not-required", + " return", + " fi", " role_result=unknown", " database_result=unknown", " probe_exit=missing-postgres-admin-secret", " if [ -n \"$postgres_admin_password\" ]; then", - ` role_result=$(psql_scalar "select exists(select 1 from pg_roles where rolname='${V02_OPENFGA_DB_USER}');")`, + " role_result=$(psql_scalar \"select exists(select 1 from pg_roles where rolname='$db_user');\")", " role_exit=$?", - ` database_result=$(psql_scalar "select exists(select 1 from pg_database where datname='${V02_OPENFGA_DB_NAME}');")`, + " database_result=$(psql_scalar \"select exists(select 1 from pg_database where datname='$db_name');\")", " database_exit=$?", " if [ \"$role_exit\" -eq 0 ] && [ \"$database_exit\" -eq 0 ]; then probe_exit=0; else probe_exit=$role_exit/$database_exit; fi", " fi", @@ -4263,7 +4477,7 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string { "authn_value=$(decoded_value \"$before_authn_b64\")", "datastore_uri=$(decoded_value \"$before_uri_b64\")", "pg_password=$(decoded_value \"$before_pg_password_b64\")", - "postgres_admin_b64=$(kubectl -n \"$namespace\" get secret hwlab-v02-postgres -o 'go-template={{ index .data \"POSTGRES_PASSWORD\" }}' 2>/dev/null || true)", + "postgres_admin_b64=$(kubectl -n \"$namespace\" get secret \"$postgres_secret\" -o 'go-template={{ index .data \"POSTGRES_PASSWORD\" }}' 2>/dev/null || true)", "postgres_admin_present=$([ -n \"$postgres_admin_b64\" ] && printf yes || printf no)", "postgres_admin_password=$(decoded_value \"$postgres_admin_b64\")", "probe_db", @@ -4280,8 +4494,10 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string { " [ \"$before_uri_present\" = yes ] && [ \"$before_uri_bytes\" -gt 0 ] || missing_secret=true", " [ \"$before_pg_password_present\" = yes ] && [ \"$before_pg_password_bytes\" -gt 0 ] || missing_secret=true", " missing_db=false", - " [ \"$db_role_exists_before\" = t ] || missing_db=true", - " [ \"$db_database_exists_before\" = t ] || missing_db=true", + " if [ \"$db_mode\" = dedicated ]; then", + " [ \"$db_role_exists_before\" = t ] || missing_db=true", + " [ \"$db_database_exists_before\" = t ] || missing_db=true", + " fi", " if [ \"$dry_run\" = true ]; then", " if [ \"$missing_secret\" = true ] || [ \"$missing_db\" = true ]; then action=would-ensure; else action=kept; fi", " elif ! command -v openssl >/dev/null 2>&1; then", @@ -4292,12 +4508,20 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string { " apply_exit=44", " else", " [ -n \"$authn_value\" ] || authn_value=$(openssl rand -base64 48)", - " [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)", + " if [ \"$db_mode\" = primary ]; then", + " [ -n \"$pg_password\" ] || pg_password=\"$postgres_admin_password\"", + " else", + " [ -n \"$pg_password\" ] || pg_password=$(openssl rand -hex 24)", + " fi", " [ -n \"$datastore_uri\" ] || datastore_uri=\"postgres://$db_user:$pg_password@$db_host:5432/$db_name?sslmode=disable\"", " kubectl -n \"$namespace\" create secret generic \"$name\" --from-literal=\"$authn_key=$authn_value\" --from-literal=\"$datastore_uri_key=$datastore_uri\" --from-literal=\"$postgres_password_key=$pg_password\" --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=\"$field_manager\" -f -", " apply_exit=$?", " if [ \"$apply_exit\" -eq 0 ]; then", - " kubectl -n \"$namespace\" exec -i statefulset/hwlab-v02-postgres -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U hwlab_v02 -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'", + " if [ \"$db_mode\" = primary ]; then", + " action=ensured", + " mutation=true", + " else", + " kubectl -n \"$namespace\" exec -i \"statefulset/$postgres_statefulset\" -c postgres -- env PGPASSWORD=\"$postgres_admin_password\" psql -v ON_ERROR_STOP=1 -U \"$postgres_admin_user\" -d postgres -v db_name=\"$db_name\" -v db_user=\"$db_user\" -v db_pass=\"$pg_password\" >/tmp/hwlab-openfga-psql.out 2>/tmp/hwlab-openfga-psql.err <<'SQL'", "SELECT format('CREATE ROLE %I LOGIN PASSWORD %L', :'db_user', :'db_pass')", "WHERE NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = :'db_user')", "\\gexec", @@ -4307,8 +4531,9 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string { "\\gexec", "ALTER DATABASE :\"db_name\" OWNER TO :\"db_user\";", "SQL", - " db_ensure_exit=$?", - " if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi", + " db_ensure_exit=$?", + " if [ \"$db_ensure_exit\" -eq 0 ]; then action=ensured; mutation=true; else action=db-ensure-failed; fi", + " fi", " else", " action=apply-failed", " fi", @@ -4335,6 +4560,7 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string { "printf 'action\\t%s\\n' \"$action\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'mutation\\t%s\\n' \"$mutation\"", + "printf 'dbMode\\t%s\\n' \"$db_mode\"", "printf 'beforeExists\\t%s\\n' \"$before_exists\"", "printf 'beforeAuthnPresent\\t%s\\n' \"$before_authn_present\"", "printf 'beforeAuthnBytes\\t%s\\n' \"$before_authn_bytes\"", @@ -4350,6 +4576,7 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string { "printf 'afterPostgresPasswordPresent\\t%s\\n' \"$after_pg_password_present\"", "printf 'afterPostgresPasswordBytes\\t%s\\n' \"$after_pg_password_bytes\"", "printf 'postgresAdminSecretPresent\\t%s\\n' \"$postgres_admin_present\"", + "printf 'postgresSecret\\t%s\\n' \"$postgres_secret\"", "printf 'dbName\\t%s\\n' \"$db_name\"", "printf 'dbUser\\t%s\\n' \"$db_user\"", "printf 'dbRoleExistsBefore\\t%s\\n' \"$db_role_exists_before\"", @@ -4369,15 +4596,18 @@ function v02OpenFgaSecretScript(options: G14SecretOptions): string { ].join("\n"); } -function v02MasterAdminApiKeySecretScript(options: G14SecretOptions): string { +function v02MasterAdminApiKeySecretScript(options: G14SecretOptions, spec: HwlabRuntimeLaneSpec): string { + const namespace = spec.runtimeNamespace; + const name = runtimeLaneMasterAdminApiKeySecretName(spec); + const fieldManager = runtimeLaneSecretFieldManager(spec); return [ "set +e", - `namespace=${shellQuote(V02_RUNTIME_NAMESPACE)}`, - `name=${shellQuote(V02_MASTER_ADMIN_API_KEY_SECRET)}`, + `namespace=${shellQuote(namespace)}`, + `name=${shellQuote(name)}`, `api_key_name=${shellQuote(V02_MASTER_ADMIN_API_KEY_SECRET_KEY)}`, `action_request=${shellQuote(options.action)}`, `dry_run=${shellQuote(options.dryRun ? "true" : "false")}`, - `field_manager=${shellQuote(V02_SECRET_FIELD_MANAGER)}`, + `field_manager=${shellQuote(fieldManager)}`, "preset=master-server-admin-api-key", "secret_exists_flag() { kubectl -n \"$namespace\" get secret \"$name\" >/dev/null 2>&1 && printf yes || printf no; }", "secret_b64_key() { kubectl -n \"$namespace\" get secret \"$name\" -o \"go-template={{ index .data \\\"$1\\\" }}\" 2>/dev/null || true; }", @@ -4504,7 +4734,7 @@ function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: num typeof afterAuthnBytes === "number" && afterAuthnBytes > 0 && typeof afterUriBytes === "number" && afterUriBytes > 0 && typeof afterPasswordBytes === "number" && afterPasswordBytes > 0; - const databaseHealthy = fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"; + const databaseHealthy = fields.dbMode === "primary" || (fields.dbRoleExistsAfter === "t" && fields.dbDatabaseExistsAfter === "t"); const healthy = keysHealthy && databaseHealthy; return { ok: commandOk && healthy, @@ -4515,6 +4745,7 @@ function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: num action: fields.action || null, dryRun: fields.dryRun === "true", mutation: fields.mutation === "true", + dbMode: fields.dbMode || "dedicated", before: { exists: fields.beforeExists === "yes", authnPresharedKey: { keyPresent: fields.beforeAuthnPresent === "yes", valueBytes: numericField(fields.beforeAuthnBytes) }, @@ -4538,6 +4769,7 @@ function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: num }, }, postgresAdminSecretPresent: fields.postgresAdminSecretPresent === "yes", + postgresSecret: fields.postgresSecret || null, dbName: fields.dbName || V02_OPENFGA_DB_NAME, dbUser: fields.dbUser || V02_OPENFGA_DB_USER, applyExitCode: numericField(fields.applyExitCode), @@ -4566,9 +4798,10 @@ function v02SecretStatusFromText(text: string, commandOk: boolean, exitCode: num } function runG14Secret(options: G14SecretOptions): Record { + const spec = hwlabRuntimeLaneSpec(options.lane); const script = v02SecretScript(options); const localAdminApiKey = options.preset === "master-server-admin-api-key" - ? readOrCreateLocalMasterAdminApiKey(options.action !== "ensure" || options.dryRun) + ? readOrCreateLocalMasterAdminApiKey(spec, options.action !== "ensure" || options.dryRun) : null; const result = options.preset === "master-server-admin-api-key" ? g14K3sInlineScriptWithInput(script, localAdminApiKey?.key ?? "", options.timeoutSeconds * 1000 + 2000) @@ -4579,9 +4812,9 @@ function runG14Secret(options: G14SecretOptions): Record { const ok = dryRunOk || deleteDryRunOk ? true : status.ok === true; return { ok, - command: `hwlab g14 secret ${options.action} --lane v02`, + command: `hwlab g14 secret ${options.action} --lane ${options.lane}`, lane: options.lane, - namespace: V02_RUNTIME_NAMESPACE, + namespace: spec.runtimeNamespace, secret: options.name, key: options.key ?? null, preset: options.preset, @@ -4597,7 +4830,7 @@ function runG14Secret(options: G14SecretOptions): Record { ? undefined : options.action === "delete" ? undefined - : { ensure: `bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }, + : { ensure: `bun scripts/cli.ts hwlab g14 secret ensure --lane ${options.lane} --name ${options.name}${options.key ? ` --key ${options.key}` : ""} --confirm` }, }; } @@ -8433,14 +8666,18 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab g14 control-plane closeout --lane v02 --source-commit ", "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --dry-run", "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm", + "bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --dry-run", + "bun scripts/cli.ts hwlab g14 control-plane refresh --lane v02 --confirm", "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm", "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait", - "bun scripts/cli.ts hwlab g14 control-plane status --lane v03", - "bun scripts/cli.ts hwlab g14 control-plane apply --lane v03 --dry-run", - "bun scripts/cli.ts hwlab g14 control-plane apply --lane v03 --confirm", - "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v03 --dry-run", - "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v03 --confirm", - "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v03 --confirm --wait", + "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm --wait", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --pipeline-run hwlab-v02-ci-poll- --dry-run", @@ -8453,13 +8690,18 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-openfga", "bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --dry-run", "bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-openfga --confirm", + "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga", + "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --dry-run", + "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --confirm", "bun scripts/cli.ts hwlab g14 secret status --lane v02 --name hwlab-v02-master-server-admin-api-key", "bun scripts/cli.ts hwlab g14 secret ensure --lane v02 --name hwlab-v02-master-server-admin-api-key --confirm", + "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key", + "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm", "bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name --dry-run", "bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name --confirm", "bun scripts/cli.ts hwlab g14 git-mirror status", "bun scripts/cli.ts hwlab g14 git-mirror apply --lane v02 --confirm", - "bun scripts/cli.ts hwlab g14 git-mirror apply --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes git-mirror apply --node G14 --lane v03 --confirm", "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm", "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm", "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait", @@ -8480,7 +8722,7 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab g14 upstream-image ensure --name openfga --tag v1.17.0 --confirm", "bun scripts/cli.ts job status --tail-bytes 30000", ], - description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, runtime lane v02/v03 control-plane apply/status/trigger entry, v0.2 runtime SecretRef bootstrap, devops-infra git mirror and observability maintenance, controlled CI tools image build/status entry, and allowlisted upstream image mirroring. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane v02 keeps the full closeout/cleanup/runtime-migration verdict path; v03+ uses the runtime lane spec for Tekton/Argo apply, status visibility, and commit-pinned PipelineRun trigger. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. upstream-image status/ensure only mirrors allowlisted upstream runtime images into the G14 local registry. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob. observability status/apply/query/targets/boundary/closeout owns the shared Prometheus Operator and Prometheus instance in devops-infra, adds bounded PromQL assertions and semantic closeout summaries, while HWLAB lane manifests own only ServiceMonitor and PrometheusRule objects.", + description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, node-scoped runtime lane v03 control-plane apply/status/refresh/trigger entry, runtime lane SecretRef bootstrap, devops-infra git mirror and observability maintenance, controlled CI tools image build/status entry, and allowlisted upstream image mirroring. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane v02 keeps the full closeout/cleanup/runtime-migration verdict path; v03+ is advertised through `hwlab nodes ... --node --lane vNN` so node identity remains configuration data instead of a command family. secret status/ensure is the standard runtime lane SecretRef bootstrap path for OpenFGA and master admin API key; it never reads or prints secret values. upstream-image status/ensure only mirrors allowlisted upstream runtime images into the G14 local registry. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob. observability status/apply/query/targets/boundary/closeout owns the shared Prometheus Operator and Prometheus instance in devops-infra, adds bounded PromQL assertions and semantic closeout summaries, while HWLAB lane manifests own only ServiceMonitor and PrometheusRule objects.", defaults: { repo: HWLAB_REPO, base: G14_SOURCE_BRANCH, diff --git a/scripts/src/hwlab-g14-lanes.ts b/scripts/src/hwlab-node-lanes.ts similarity index 94% rename from scripts/src/hwlab-g14-lanes.ts rename to scripts/src/hwlab-node-lanes.ts index e840ca0b..f8d3a875 100644 --- a/scripts/src/hwlab-g14-lanes.ts +++ b/scripts/src/hwlab-node-lanes.ts @@ -92,7 +92,7 @@ export interface HwlabRuntimeLaneSpec { readonly downloadProfile: HwlabDownloadProfileSpec; } -export const HWLAB_G14_LANE_CONFIG_PATH = "config/hwlab-g14-lanes.yaml"; +export const HWLAB_NODE_LANE_CONFIG_PATH = "config/hwlab-node-lanes.yaml"; interface HwlabLaneConfig { readonly id: HwlabRuntimeLane; @@ -120,7 +120,7 @@ interface HwlabLaneConfig { readonly public: { readonly webUrl: string; readonly apiUrl: string }; } -interface HwlabG14LaneConfig { +interface HwlabNodeLaneConfig { readonly requiredNoProxy: readonly string[]; readonly nodes: Record; readonly lanes: Record; @@ -283,13 +283,13 @@ function laneConfig(id: HwlabRuntimeLane, raw: Record): HwlabLa }; } -function readHwlabG14LaneConfig(): HwlabG14LaneConfig { - const path = rootPath(HWLAB_G14_LANE_CONFIG_PATH); +function readHwlabNodeLaneConfig(): HwlabNodeLaneConfig { + const path = rootPath(HWLAB_NODE_LANE_CONFIG_PATH); const raw = readFileSync(path, "utf8"); - const parsed = asRecord(Bun.YAML.parse(raw) as unknown, HWLAB_G14_LANE_CONFIG_PATH); - const requiredNoProxy = stringArrayField(parsed, "requiredNoProxy", HWLAB_G14_LANE_CONFIG_PATH); + const parsed = asRecord(Bun.YAML.parse(raw) as unknown, HWLAB_NODE_LANE_CONFIG_PATH); + const requiredNoProxy = stringArrayField(parsed, "requiredNoProxy", HWLAB_NODE_LANE_CONFIG_PATH); for (const required of ["hyueapi.com", ".hyueapi.com"]) { - if (!requiredNoProxy.includes(required)) throw new Error(`${HWLAB_G14_LANE_CONFIG_PATH}.requiredNoProxy must include ${required}`); + if (!requiredNoProxy.includes(required)) throw new Error(`${HWLAB_NODE_LANE_CONFIG_PATH}.requiredNoProxy must include ${required}`); } const nodes = Object.fromEntries(sortedRecordEntries(parsed.nodes, "nodes").map(([id, item]) => [id, nodeConfig(id, item)])); const networkProfiles = Object.fromEntries( @@ -312,12 +312,12 @@ function readHwlabG14LaneConfig(): HwlabG14LaneConfig { return { requiredNoProxy, nodes, lanes, networkProfiles, downloadProfiles }; } -const HWLAB_G14_LANE_CONFIG = readHwlabG14LaneConfig(); +const HWLAB_NODE_LANE_CONFIG = readHwlabNodeLaneConfig(); function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec { - const node = HWLAB_G14_LANE_CONFIG.nodes[config.node]; - const networkProfile = HWLAB_G14_LANE_CONFIG.networkProfiles[node.networkProfileId]; - const downloadProfile = HWLAB_G14_LANE_CONFIG.downloadProfiles[node.downloadProfileId]; + const node = HWLAB_NODE_LANE_CONFIG.nodes[config.node]; + const networkProfile = HWLAB_NODE_LANE_CONFIG.networkProfiles[node.networkProfileId]; + const downloadProfile = HWLAB_NODE_LANE_CONFIG.downloadProfiles[node.downloadProfileId]; return { lane: config.id, nodeId: node.id, @@ -358,7 +358,7 @@ function buildRuntimeLaneSpec(config: HwlabLaneConfig): HwlabRuntimeLaneSpec { } const RUNTIME_LANE_SPECS = Object.fromEntries( - Object.values(HWLAB_G14_LANE_CONFIG.lanes).map((config) => [config.id, buildRuntimeLaneSpec(config)]), + Object.values(HWLAB_NODE_LANE_CONFIG.lanes).map((config) => [config.id, buildRuntimeLaneSpec(config)]), ) as Record; export function isHwlabRuntimeLane(value: string): value is HwlabRuntimeLane { @@ -374,13 +374,13 @@ export function hwlabRuntimeLaneIds(): HwlabRuntimeLane[] { } export function hwlabRuntimeNodeIds(): string[] { - return Object.keys(HWLAB_G14_LANE_CONFIG.nodes); + return Object.keys(HWLAB_NODE_LANE_CONFIG.nodes); } export function hwlabRuntimeLaneConfigPath(): string { - return HWLAB_G14_LANE_CONFIG_PATH; + return HWLAB_NODE_LANE_CONFIG_PATH; } export function hwlabRequiredNoProxyEntries(): string[] { - return [...HWLAB_G14_LANE_CONFIG.requiredNoProxy]; + return [...HWLAB_NODE_LANE_CONFIG.requiredNoProxy]; } diff --git a/scripts/src/hwlab-node.ts b/scripts/src/hwlab-node.ts index afa3513a..7119292d 100644 --- a/scripts/src/hwlab-node.ts +++ b/scripts/src/hwlab-node.ts @@ -1,9 +1,13 @@ import { existsSync, readFileSync } from "node:fs"; import { repoRoot, type Config } from "./config"; import { runCommand, type CommandResult } from "./command"; +import { startJob } from "./jobs"; +import { runHwlabG14Command } from "./hwlab-g14"; +import { hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane } from "./hwlab-node-lanes"; type SecretAction = "status" | "ensure"; type SecretPreset = "openfga" | "master-server-admin-api-key"; +type DelegatedNodeDomain = "control-plane" | "git-mirror"; interface NodeSecretOptions { action: SecretAction; @@ -41,8 +45,11 @@ const OPENFGA_POSTGRES_PASSWORD_KEY = "postgres-password"; export async function runHwlabNodeCommand(_config: Config, args: string[]): Promise> { if (args.length === 0 || args.includes("--help") || args.includes("-h")) return hwlabNodeHelp(); const [domain] = args; + if (domain === "control-plane" || domain === "git-mirror") { + return runNodeDelegatedDomain(_config, domain, args.slice(1)); + } if (domain !== "secret") { - return { ok: false, command: `hwlab node ${domain ?? ""}`.trim(), message: "supported commands: hwlab node secret status|ensure" }; + return { ok: false, command: `hwlab nodes ${domain ?? ""}`.trim(), message: "supported commands: hwlab nodes control-plane, hwlab nodes git-mirror, hwlab nodes secret" }; } const options = parseSecretOptions(args.slice(1)); return runNodeSecret(options); @@ -53,7 +60,13 @@ export function hwlabNodeHelp(): Record { ok: true, command: "hwlab nodes", description: "Node/lane oriented HWLAB operations. G14 is a node id value passed by --node, not a command family.", + configPath: hwlabRuntimeLaneConfigPath(), examples: [ + "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane apply --node G14 --lane v03 --dry-run", + "bun scripts/cli.ts hwlab nodes control-plane refresh --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes control-plane trigger-current --node G14 --lane v03 --confirm", + "bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes secret status --node G14 --lane v03 --name hwlab-v03-openfga", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-openfga --confirm", "bun scripts/cli.ts hwlab nodes secret ensure --node G14 --lane v03 --name hwlab-v03-master-server-admin-api-key --confirm", @@ -61,14 +74,123 @@ export function hwlabNodeHelp(): Record { }; } +async function runNodeDelegatedDomain(config: Config, domain: DelegatedNodeDomain, args: string[]): Promise> { + const scoped = parseNodeScopedDelegatedOptions(domain, args); + if (domain === "control-plane" && scoped.action === "trigger-current" && scoped.confirm && !scoped.dryRun && !scoped.wait) { + return startNodeDelegatedJob(scoped); + } + if (domain === "git-mirror" && (scoped.action === "sync" || scoped.action === "flush") && scoped.confirm && !scoped.dryRun && !scoped.wait) { + return startNodeDelegatedJob(scoped); + } + const delegatedArgs = stripOption(args, "--node"); + const result = await runHwlabG14Command(config, [domain, ...delegatedArgs]); + return rewriteDelegatedNodeResult(result, scoped); +} + +function parseNodeScopedDelegatedOptions(domain: DelegatedNodeDomain, args: string[]): { + domain: DelegatedNodeDomain; + action: string; + node: string; + lane: HwlabRuntimeLane; + confirm: boolean; + dryRun: boolean; + wait: boolean; + timeoutSeconds: number; + originalArgs: string[]; +} { + const [actionRaw] = args; + if (typeof actionRaw !== "string" || actionRaw.startsWith("--")) throw new Error(`${domain} usage: ${domain} ACTION --node NODE --lane vNN [--dry-run|--confirm]`); + const node = requiredOption(args, "--node"); + assertNodeId(node); + const laneRaw = requiredOption(args, "--lane"); + if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`--lane must be one of v02, v03; got ${laneRaw}`); + const spec = hwlabRuntimeLaneSpec(laneRaw); + if (spec.nodeId !== node) throw new Error(`lane ${laneRaw} is configured for node ${spec.nodeId}; got --node ${node}`); + const confirm = args.includes("--confirm"); + const dryRun = args.includes("--dry-run"); + if (confirm && dryRun) throw new Error(`${domain} accepts only one of --confirm or --dry-run`); + return { + domain, + action: actionRaw, + node, + lane: laneRaw, + confirm, + dryRun, + wait: args.includes("--wait"), + timeoutSeconds: positiveIntegerOption(args, "--timeout-seconds", 1800, 3600), + originalArgs: [...args], + }; +} + +function startNodeDelegatedJob(options: ReturnType): Record { + const commandArgs = [ + "hwlab", + "nodes", + options.domain, + ...stripOptions(options.originalArgs, ["--node", "--lane", "--confirm", "--dry-run", "--wait", "--timeout-seconds"]), + "--node", + options.node, + "--lane", + options.lane, + "--confirm", + "--timeout-seconds", + String(options.timeoutSeconds), + "--wait", + ]; + const command = ["bun", "scripts/cli.ts", ...commandArgs]; + const job = startJob( + `hwlab_nodes_${options.lane}_${options.domain}_${options.action}`, + command, + `Run HWLAB ${options.lane} ${options.domain} ${options.action} for node ${options.node}`, + ); + return { + ok: true, + command: `hwlab nodes ${options.domain} ${options.action} --node ${options.node} --lane ${options.lane}`, + node: options.node, + lane: options.lane, + mode: "async-job", + reason: "confirmed control-plane/mirror actions can spend tens of seconds on remote work; default is fire-and-forget to avoid silent blocking", + job, + statusCommand: `bun scripts/cli.ts job status ${job.id}`, + waitCommand: command.join(" "), + }; +} + +function rewriteDelegatedNodeResult(value: unknown, scoped: ReturnType): Record { + const rewritten = rewriteDelegatedNodeValue(value, scoped); + const result = typeof rewritten === "object" && rewritten !== null && !Array.isArray(rewritten) ? rewritten as Record : { value: rewritten }; + return { + ...result, + node: scoped.node, + commandFamily: "hwlab nodes", + }; +} + +function rewriteDelegatedNodeValue(value: unknown, scoped: ReturnType): unknown { + if (typeof value === "string") return rewriteDelegatedNodeString(value, scoped); + if (Array.isArray(value)) return value.map((item) => rewriteDelegatedNodeValue(item, scoped)); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewriteDelegatedNodeValue(item, scoped)])); +} + +function rewriteDelegatedNodeString(value: string, scoped: ReturnType): string { + const replaceCommand = (text: string, domain: DelegatedNodeDomain) => { + const escapedDomain = domain.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + return text + .replace(new RegExp(`bun scripts/cli\\.ts hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `bun scripts/cli.ts hwlab nodes ${domain} $1 --node ${scoped.node}`) + .replace(new RegExp(`hwlab g14 ${escapedDomain} ([a-z-]+)`, "gu"), `hwlab nodes ${domain} $1 --node ${scoped.node}`); + }; + return replaceCommand(replaceCommand(value, "control-plane"), "git-mirror"); +} + function parseSecretOptions(args: string[]): NodeSecretOptions { const [actionRaw] = args; if (actionRaw !== "status" && actionRaw !== "ensure") { throw new Error("secret usage: status|ensure --node NODE --lane vNN --name hwlab-vNN-openfga|hwlab-vNN-master-server-admin-api-key [--dry-run|--confirm]"); } - const node = optionValue(args, "--node") ?? "G14"; + const node = requiredOption(args, "--node"); assertNodeId(node); - const lane = optionValue(args, "--lane") ?? "v03"; + const lane = requiredOption(args, "--lane"); assertLane(lane); const spec = runtimeSecretSpec({ node, lane }); const name = optionValue(args, "--name") ?? spec.openFgaSecret; @@ -448,6 +570,30 @@ function optionValue(args: string[], name: string): string | undefined { return value; } +function requiredOption(args: string[], name: string): string { + const value = optionValue(args, name); + if (value === undefined) throw new Error(`${name} is required`); + return value; +} + +function stripOption(args: string[], name: string): string[] { + return stripOptions(args, [name]); +} + +function stripOptions(args: string[], names: readonly string[]): string[] { + const remove = new Set(names); + const without: string[] = []; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] ?? ""; + if (remove.has(arg)) { + if (arg !== "--confirm" && arg !== "--dry-run" && arg !== "--wait") index += 1; + continue; + } + without.push(arg); + } + return without; +} + function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const raw = optionValue(args, name); if (raw === undefined) return defaultValue;