diff --git a/AGENTS.md b/AGENTS.md index fa75281f..d1b12072 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -227,6 +227,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台;本文 - `bun scripts/cli.ts server rebuild `:以 build-first、Compose lock、no-deps force-recreate 和 post-up validation 的异步 job 重建主 server Compose 内单个服务;对 database、File Browser、Code Queue 执行面、k3sctl-adapter 或未知对象返回结构化 `unsupported-server-rebuild`,规则见 `docs/reference/deployment.md` 与 `docs/reference/cicd-standardization.md`。 - `bun scripts/cli.ts provider attach [--master-server URL] [--up] [--force]` / `bun scripts/cli.ts provider triage [--observed-error text] [--observed-scope scope] [--microservice id ...] [--full|--raw]`:前者在新增计算节点上生成两项配置的 provider-gateway 挂载包;后者是只读多信号健康裁决入口,默认低噪声输出 `decision`、`healthyScopes`、`failedScopes`、`retryable` 和异常信号摘要,用来把单路径 `provider is not online`、SSH 超时、registry 失败或 proxy 失败归类为 `retryable-transient`、`service-degraded` 或 `global-offline`,完整 evidence 需显式 `--full|--raw`,规则见 `docs/reference/provider-gateway.md` 和 `docs/reference/code-queue-supervision.md`。 - `bun scripts/cli.ts platform-db postgres plan|status|apply --config config/platform-db/postgres-pk01.yaml`:管理 PK01 host-native PostgreSQL 16 外置平台库、TLS、Secret 导出和备份;跨节点消费者直连 YAML 声明的 PK01 公网 endpoint,不经 master server 中转,规则见 `docs/reference/pk01.md`。 +- `bun scripts/cli.ts secrets plan|sync|status --config config/secrets-distribution.yaml --scope platform-infra`:按 YAML 声明把本地 Secret sourceRef 下发到 G14 `platform-infra` Kubernetes Secret key,禁止从运行面反推密码/API key,规则见 `docs/reference/platform-infra.md#secret-distribution-boundary`。 - `trans [operation args...]` / `tran [operation args...]`:通过 provider-gateway 的 Host SSH / WSL SSH 维护桥进入 provider、host workspace、Windows cmd route、k3s 控制面或 pod workspace,并提供带 SHA-256 校验的 `upload`/`download` 文件传输;主 server 人工/Codex 分布式操作必须优先用本机 `trans` wrapper,`tran` 只作为兼容入口,细则见 `docs/reference/cli.md`、`docs/reference/windows-passthrough.md` 和 `docs/reference/provider-gateway.md`。 - `bun scripts/cli.ts microservice list/status/health/diagnostics/tunnel-self-test/proxy`:管理和验证挂载在主 server、计算节点 Docker 或 k3s 控制面上的用户服务,`status/health/diagnostics` 默认 compact summary 并用 `--full|--raw` 展开完整 body,`proxy` 支持受控 JSON body,OA Event Flow/Todo Note/Baidu Netdisk/Code Queue Manager on main-server、k3s Control/Code Queue 执行面/MDTODO/Decision Center/FindJob/Pipeline/MET Nonlinear on D601 的规则见 `docs/reference/microservices.md`。 - `bun scripts/cli.ts microservice health/diagnostics/proxy code-agent-sandbox`:验证独立 Code Agent Sandbox 的 health、只读 diagnostics、trace 和 adapter/mode/credential boundary 契约,规则见 `docs/reference/code-agent-sandbox.md`。 @@ -278,7 +279,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台;本文 - `docs/reference/hwlab.md`:HWLAB 指挥侧固定 workspace、G14 主运行面、D601 legacy/硬件桥接边界、最小 device-agent/gateway 桥接模型和受控发布边界。 - `docs/reference/g14.md`:G14 provider 节点、k3s 控制桥、legacy DEV/PROD 退役边界、当前 HWLAB runtime lane、device-agent 手动实验边界、Code Queue/CI 候选目标和节点本地 VPN proxy bootstrap 边界。 - `docs/reference/pk01.md`:PK01 腾讯云 provider-gateway、pikanode/MET Docker workload、SSH 透传、磁盘 GC 和 pikanode temp 长效 retention 边界。 -- `docs/reference/platform-infra.md`:G14 `platform-infra` namespace、YAML-first shared service 配置、Sub2API/Codex pool、FRP 暴露和 on-demand availability probe 开发边界;Sub2API 日常操作统一见 `$unidesk-sub2api`(`.agents/skills/unidesk-sub2api/SKILL.md`)。 +- `docs/reference/platform-infra.md`:G14 `platform-infra` namespace、YAML-first shared service 配置、Secret distribution、Sub2API/Codex pool、FRP 暴露和 on-demand availability probe 开发边界;Sub2API 日常操作统一见 `$unidesk-sub2api`(`.agents/skills/unidesk-sub2api/SKILL.md`)。 - `docs/reference/master-server-ops.md`:主 server 本机 Codex profile wrapper、ACX/GOCX/Moon Bridge 路由边界、默认模型、真实调用验收和 MiniMax session recovery 规则。 - `docs/reference/g14-observability-infra.md`:G14 原生 k3s 上 Prometheus Operator、`devops-infra` 监控基础设施、跨 namespace scrape 声明和安全边界。 - `docs/reference/gc.md`:UniDesk 主 server 和 provider 磁盘 GC、G14/HWLAB registry retention、safe-stop 线和长期防膨胀收益规则。 diff --git a/config/secrets-distribution.yaml b/config/secrets-distribution.yaml new file mode 100644 index 00000000..3c574fbb --- /dev/null +++ b/config/secrets-distribution.yaml @@ -0,0 +1,95 @@ +version: 1 +kind: unidesk-secret-distribution + +metadata: + id: platform-infra-runtime-secrets + owner: unidesk + relatedIssues: + - 297 + - 300 + - 313 + +sources: + root: /root/unidesk/.state/secrets + files: + - sourceRef: platform-db/langbot-db.env + type: env + requiredKeys: + - LANGBOT_DB_USER + - LANGBOT_DB_PASSWORD + - LANGBOT_DB_NAME + createIfMissing: + enabled: false + - sourceRef: platform-db/n8n-db.env + type: env + requiredKeys: + - N8N_DB_USER + - N8N_DB_PASSWORD + - N8N_DB_NAME + createIfMissing: + enabled: false + - sourceRef: platform-infra/langbot.env + type: env + requiredKeys: + - DATABASE_URL + - LANGBOT_JWT_SECRET + - LANGBOT_API_KEY + createIfMissing: + enabled: true + randomHex: + LANGBOT_JWT_SECRET: 32 + randomBase64Url: + LANGBOT_API_KEY: + bytes: 32 + prefix: lbk_ + - sourceRef: platform-infra/n8n.env + type: env + requiredKeys: + - DATABASE_URL + - N8N_ENCRYPTION_KEY + createIfMissing: + enabled: true + randomBase64Url: + N8N_ENCRYPTION_KEY: + bytes: 32 + prefix: "" + +targets: + - id: platform-infra-g14 + route: G14:k3s + namespace: platform-infra + scope: platform-infra + enabled: true + +kubernetesSecrets: + - name: langbot-runtime + targetId: platform-infra-g14 + secretName: langbot-secrets + type: Opaque + data: + - sourceRef: platform-db/langbot-db.env + sourceKey: LANGBOT_DB_PASSWORD + targetKey: DATABASE_PASSWORD + - sourceRef: platform-infra/langbot.env + sourceKey: LANGBOT_JWT_SECRET + targetKey: SYSTEM_JWT_SECRET + - sourceRef: platform-infra/langbot.env + sourceKey: LANGBOT_API_KEY + targetKey: LANGBOT_API_KEY + - sourceRef: platform-infra/langbot.env + sourceKey: DATABASE_URL + targetKey: DATABASE_URL + - name: n8n-runtime + targetId: platform-infra-g14 + secretName: n8n-secrets + type: Opaque + data: + - sourceRef: platform-db/n8n-db.env + sourceKey: N8N_DB_PASSWORD + targetKey: DB_POSTGRESDB_PASSWORD + - sourceRef: platform-infra/n8n.env + sourceKey: N8N_ENCRYPTION_KEY + targetKey: N8N_ENCRYPTION_KEY + - sourceRef: platform-infra/n8n.env + sourceKey: DATABASE_URL + targetKey: DATABASE_URL diff --git a/docs/reference/cli.md b/docs/reference/cli.md index c9a2fd87..9da7e732 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -76,6 +76,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 runtime lane 滚动 `status` 返回 read/write URL、last sync/write/flush、本地 ref、GitHub staging ref 和 pending flush 状态,并在 `cache.summary` 给出 `localV02`、`localGitops`、`githubGitops`、`pendingFlush`、`flushNeeded`、`githubInSync` 和下一条受控 `flushCommand`。confirmed `sync` 和 `flush` 默认创建 `.state/jobs/` 异步 job 并立刻返回可查询状态,只有现场同步调试才显式加 `--wait`;mirror 不设置 CronJob。 如果 PipelineRun 的 `gitops-promote` 阶段报 git mirror 控制面漂移或 refs 不一致,先执行 `hwlab g14 git-mirror apply --confirm` 重新应用当前 `devops-infra/git-mirror.yaml` hook/ConfigMap,再执行 `hwlab g14 git-mirror sync --confirm --wait` 复核 refs;失败的同名 PipelineRun 只能通过 `hwlab g14 control-plane cleanup-runs --lane --pipeline-run --confirm` 受控清理后重试,不要用原生 `kubectl delete` 或手工改 mirror hook。修复后仍必须用 `control-plane status --pipeline-run ` 和 `git-mirror status` 分别确认 runtime closeout 与 GitHub flush。 - `platform-infra sub2api|langbot|n8n|wechat-archive ...` 是 `platform-infra` namespace 内平台公共服务和公共工作流的受控入口;`sub2api` 支持 `plan|apply|status|validate|codex-pool`,`langbot` 和 `n8n` 支持各自 YAML-controlled `plan|apply|status|logs|validate` 等公共服务操作,`wechat-archive` 支持 `plan|apply|status|validate|pull`,用 `config/platform-infra/wechat-archive.yaml` 声明 LangBot/n8n/Baidu Netdisk 归档链路。`--target` 选择运行目标,默认 `G14` 为 active runtime,`D601` 为同一 YAML 控制的 standby predeploy target。镜像版本和 target 边界由 `config/platform-infra/*.yaml` 控制,Codex 上游池、统一 API key Secret、FRP 公网端口和 master `~/.codex` 消费端由 `config/platform-infra/sub2api-codex-pool.yaml` 控制;完整 Sub2API 日常部署、上游增删、FRP 暴露、local Codex 配置、验收和排障步骤统一见 `$unidesk-sub2api`(`.agents/skills/unidesk-sub2api/SKILL.md`)。`docs/reference/platform-infra.md` 保留 namespace、YAML-first、路由、Secret 脱敏、PK01 Caddy+FRP 和探针开发边界。 +- `secrets plan|sync|status --config config/secrets-distribution.yaml --scope platform-infra` 是平台服务本地 Secret sourceRef 到 Kubernetes Secret key 的受控下发入口。`plan` 只读展示 sourceRef、必需 key、目标 Secret/key、missingKeys 和 fingerprint;`sync --confirm` 只按 YAML 声明创建允许生成的本地 key 并下发到声明的目标 Secret;`status` 只验证 live Secret key presence。该入口禁止从 live pod 或 Kubernetes Secret 反推密码、API key、JWT secret、n8n encryption key 或 `DATABASE_URL`,输出也不得打印 base64、解码值或远端 raw transcript;即使显式 `--raw` 也只返回脱敏 summary 和 raw omission 标记。LangBot/n8n Secret 轮换和缺 key 修复规则见 `docs/reference/platform-infra.md#secret-distribution-boundary`。 - `hwlab g14 observability status|apply|query|targets|boundary|closeout [--lane v02] [--promql ] [--expect-count N] [--expect-value V] [--dry-run|--confirm]` 是 G14 `devops-infra` 共享监控基础设施和 HWLAB v0.2 监控 closeout 的受控入口。`apply` 固定安装 Prometheus Operator `v0.91.0`、Prometheus `v3.12.0`、Prometheus 发现 RBAC、`devops-infra` 内 Prometheus 实例和 ClusterIP query Service,并给被允许发现的 workload namespace 打低风险 label;它不把 Prometheus、Grafana 或 Alertmanager 部署到 `hwlab-v02`,也不接管 HWLAB runtime Deployment/Service。`status` 只读汇总 CRD、operator Deployment、Prometheus CR/pod/service、`hwlab-v02` ServiceMonitor/PrometheusRule 和 bounded `up` 查询;`query` 只通过 Kubernetes service proxy 查询 Prometheus,支持 `--expect-count` / `--expect-value` 输出 `assertion`、bad values 和 missing/extra series;`targets` 汇总 ServiceMonitor/PrometheusRule、metrics sidecar readiness/restart、三层指标值和 `metrics.k8s.io` 当前 CPU/内存资源快照;`boundary` 验证 workload namespace 没有 Prometheus/Alertmanager,并对 `19666/19667` 公网 `/metrics` 做负向验证;`closeout` 聚合平台 ready、scrape reachable、sidecar serving、business health probe、resource snapshot、namespace boundary 和 public metrics exposure 语义结论。长期边界见 `docs/reference/g14-observability-infra.md`。 - `hwlab g14 tools-image status|build --name ci-node-tools --tag [--dockerfile deploy/ci/hwlab-ci-node-tools.Dockerfile] [--dry-run|--confirm]` 是 G14 固定 HWLAB CI tools image 的受控 host build/push 入口;构建和 push 只发生在 G14 host 与本地 registry,不在 master server 构建,也不把 `apk add`/runtime install 塞进 Tekton PipelineRun。 - `trans gh:/owner/repo ...` 把 GitHub issue/PR 映射成只读/受控写入的虚拟文本目录,适合日报、PR 正文和 issue 正文的小补丁维护:`trans gh:/pikasTech/HWLAB ls` 展示 `pr/` 与 `issue/`,`trans gh:/pikasTech/HWLAB/pr ls [--limit N] [--full]` 和 `trans gh:/pikasTech/HWLAB/issue ls [--limit N] [--full]` 展示条目状态、楼层数、正文长度和标题,`trans gh:/pikasTech/HWLAB/pr/507 ls` 展示单个 PR 的一楼正文文件,`trans gh:/pikasTech/HWLAB/505/1 cat|rg|patch-apply` 兼容旧式 issue/PR number route。`patch-apply` 使用 UniDesk 默认 apply-patch v2 的虚拟文件 executor,把正文一楼映射为 `body.md`,写回仍走 `bun scripts/cli.ts gh issue/pr update` 的 guard/concurrency 规则;`rm` 对正文一楼结构化拒绝,避免误删 issue/PR 正文。大正文读取必须展开 UniDesk gh dump 文件,否则 `cat/rg/patch-apply` 会误读为空,这是 `gh:` 虚拟文件接口的 P0 可见性契约。 diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index cfb336ef..1eeae042 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -9,6 +9,14 @@ - Code that reads platform YAML must validate object shape, field types, required fields, Kubernetes names, image strings, and ports before mutating G14 k3s or local consumer files. - Do not hide image versions, namespace names, endpoint URLs, FRP ports, or profile lists in Python/TOML/JSON helper constants when they are UniDesk-owned choices. External tools may still require their own TOML/JSON/env file formats at the edge. +## Secret Distribution Boundary + +- Platform service credential distribution is YAML-controlled by `config/secrets-distribution.yaml` and the canonical entrypoint is `bun scripts/cli.ts secrets plan|sync|status --config config/secrets-distribution.yaml --scope platform-infra`. +- The YAML maps local secret source files under the declared `sources.root` to target Kubernetes Secret names and keys. It is the source of authority for LangBot/n8n runtime Secret handoff; do not reverse-engineer passwords, API keys, encryption keys or `DATABASE_URL` values from live pods or existing Kubernetes Secrets. +- `secrets plan` is read-only and may show sourceRef paths, required key names, generated-key intent, target Secret names, target keys, presence, missing keys and fingerprints. `secrets sync --confirm` may create missing local generated keys only when YAML explicitly allows `createIfMissing`; database passwords exported by `platform-db postgres` are not regenerated here. `secrets status` verifies live Secret key presence without decoding values. +- CLI output for Secret distribution may disclose key names, object names, byte/count-style metadata and fingerprints only. It must not print base64 payloads, decoded values, full `DATABASE_URL`, API keys, JWT secrets, encryption keys, database passwords or copy-pastable credential mutation commands. +- Service-specific `platform-infra langbot|n8n apply` commands may read the declared local sourceRef files to render/apply runtime Secrets, but they must not infer missing values from the current runtime. If required local source keys are absent, the durable fix is `secrets sync` or the owning YAML/Secret source path, not a runtime reverse lookup. + ## Sub2API Deployment Boundary - Sub2API is a platform service operated by UniDesk in namespace `platform-infra`. It is not a HWLAB lane workload, AgentRun workload, D601 user service, or master server daemon. @@ -26,6 +34,7 @@ - LangBot is a UniDesk-operated public platform service in namespace `platform-infra`. The canonical entrypoint is `bun scripts/cli.ts platform-infra langbot plan|apply|status|logs|validate|bootstrap-api-key|query`; G14 is the default runtime target. - LangBot configuration is YAML-first in `config/platform-infra/langbot.yaml`. Image tag, target namespace, PVCs, PK01 Caddy/FRP exposure, API key seed source, and official WeChat adapter metadata must stay in YAML rather than helper constants or manual runtime patches. +- LangBot runtime Secret handoff uses `config/secrets-distribution.yaml` and `bun scripts/cli.ts secrets ... --scope platform-infra`. `platform-infra langbot apply` must not create hidden passwords or reverse-read live Kubernetes Secret values to fill missing local source keys. - LangBot uses the existing PK01 host-native PostgreSQL instance through `config/platform-db/postgres-pk01.yaml` and `platform-db postgres`. Adding LangBot state means adding a dedicated database and role inside that existing instance; do not deploy a second PostgreSQL StatefulSet, container, or external DB instance for LangBot. - Public exposure uses PK01 Caddy plus FRP to the G14 ClusterIP service. Do not add Kubernetes Ingress, NodePort, LoadBalancer, host networking, or host ports for LangBot unless a later YAML-controlled platform decision changes the exposure model. - LangBot's built-in Web frontend and API share the same public HTTPS origin. CLI queries must use the YAML-declared API key source and must report key names/fingerprints only, never the API key value. @@ -40,6 +49,7 @@ - n8n is the UniDesk-operated workflow/automation layer for LangBot and platform service integration. It is a workflow bridge for webhook orchestration, service calls, manual approval flows and external integrations; it does not replace LangBot or become the chat runtime. - The canonical entrypoint is `bun scripts/cli.ts platform-infra n8n plan|apply|status|logs|validate`; G14 is the default runtime target and `config/platform-infra/n8n.yaml` is the YAML source of truth. +- n8n runtime Secret handoff uses `config/secrets-distribution.yaml` and `bun scripts/cli.ts secrets ... --scope platform-infra`. `platform-infra n8n apply` must not create hidden encryption keys or reverse-read live Kubernetes Secret values to fill missing local source keys. - n8n uses the existing Pika01/PK01 host-native PostgreSQL instance through `config/platform-db/postgres-pk01.yaml` and `platform-db postgres`. Adding n8n state means adding a dedicated `n8n` database and role inside that single external PostgreSQL instance; do not deploy an in-cluster PostgreSQL StatefulSet, a second PostgreSQL instance, or long-term SQLite state for n8n. - Public exposure uses PK01 Caddy plus FRP to the G14 ClusterIP service at `https://n8n.pikapython.com`. Do not add Kubernetes Ingress, NodePort, LoadBalancer, host networking, or host ports for n8n unless a later YAML-controlled platform decision changes the exposure model. - n8n reverse-proxy and webhook settings such as public base URL, `WEBHOOK_URL`, proxy hop trust and PostgreSQL connection fields must be rendered from YAML. Secret output may show key names, presence and fingerprints only; it must not print the database password, `N8N_ENCRYPTION_KEY`, or full `DATABASE_URL`. diff --git a/scripts/cli.ts b/scripts/cli.ts index 128fec6c..3ec1ba5e 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -25,6 +25,7 @@ import { isHelpToken, rootHelp, serverHelp, sshHelp, staticNamespaceHelp } from import { runServerCleanupCommand } from "./src/server-cleanup"; import { runGcCommand } from "./src/gc"; import { runPlatformDbCommand } from "./src/platform-db"; +import { runSecretsCommand } from "./src/secrets"; const remoteOptions = extractRemoteCliOptions(process.argv.slice(2)); const args = remoteOptions.args; @@ -348,6 +349,14 @@ async function main(): Promise { return; } + if (top === "secrets") { + const result = await runSecretsCommand(readConfig(), args.slice(1)); + const ok = (result as { ok?: unknown }).ok !== false; + emitJson(commandName, result, ok); + if (!ok) process.exitCode = 1; + return; + } + const config = readConfig(); const autoRemoteCiPublishPlan = autoRemoteCiPublishUserServiceDryRunPlan(config, args); if (autoRemoteCiPublishPlan.enabled && autoRemoteCiPublishPlan.host !== null) { diff --git a/scripts/src/help.ts b/scripts/src/help.ts index a2d04e54..a4a8faaf 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -1,6 +1,7 @@ import { ghHelp } from "./gh"; import { authBrokerHelp } from "./auth-broker"; import { platformDbHelp } from "./platform-db"; +import { secretsHelp } from "./secrets"; export function rootHelp(): unknown { return { @@ -61,6 +62,7 @@ export function rootHelp(): unknown { { command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." }, { command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send|control-plane|git-mirror", description: "Use AgentRun v0.1 resource primitives with low-noise human output by default; session follow-up uses send only and the server decides internal steer vs turn." }, { command: "platform-infra sub2api|langbot|n8n|wechat-archive ...", description: "Deploy platform-infra services such as Sub2API, LangBot and n8n, manage YAML-controlled public FRP/Caddy exposure and WeChat archive workflows, and inspect status/logs without printing secrets." }, + { command: "secrets plan|sync|status", description: "Plan, push and inspect YAML-declared local secret source keys to Kubernetes Secrets without printing or reverse-engineering values." }, { command: "platform-db postgres plan|status|apply", description: "Manage YAML-declared host-native PostgreSQL 16 on PK01 for Sub2API/platform state, with PostgreSQL native TLS and redacted secret exports." }, { command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." }, { command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." }, @@ -715,6 +717,7 @@ export async function staticNamespaceHelp(args: string[]): Promise (await import("./agentrun")).agentRunHelp(), agentRunHelpSummary()); if (top === "platform-infra") return loadHelp(async () => (await import("./platform-infra")).platformInfraHelp(), platformInfraHelpSummary()); if (top === "platform-db") return platformDbHelp(); + if (top === "secrets") return secretsHelp(); if (top === "hwlab" && (sub === "node" || sub === "nodes") && args[2] === "control-plane" && args[3] === "infra") { return loadHelp(async () => (await import("./hwlab-node-control-plane")).hwlabNodeControlPlaneInfraHelp(), hwlabNodeHelpSummary()); } diff --git a/scripts/src/platform-infra-langbot.ts b/scripts/src/platform-infra-langbot.ts index 86c55b5e..c6a2be3d 100644 --- a/scripts/src/platform-infra-langbot.ts +++ b/scripts/src/platform-infra-langbot.ts @@ -1,6 +1,6 @@ -import { createHash, randomBytes } from "node:crypto"; -import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, isAbsolute, join } from "node:path"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { isAbsolute, join } from "node:path"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; import { startJob } from "./jobs"; @@ -116,7 +116,7 @@ interface SecretMaterial { dbSourcePath: string; appSourceRef: string; appSourcePath: string; - action: "create" | "update" | "none"; + action: "read"; values: { dbUser: string; dbPassword: string; @@ -423,6 +423,7 @@ function plan(options: CommonOptions): Record { resourcePolicy: "No Kubernetes CPU/memory requests or limits are rendered.", }, next: { + secrets: "bun scripts/cli.ts secrets sync --config config/secrets-distribution.yaml --scope platform-infra --confirm", dryRun: `bun scripts/cli.ts platform-infra langbot apply --target ${target.id} --dry-run`, apply: `bun scripts/cli.ts platform-infra langbot apply --target ${target.id} --confirm`, status: `bun scripts/cli.ts platform-infra langbot status --target ${target.id}`, @@ -488,6 +489,7 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise existing[key] !== next[key]) ? "update" : "none"; - if (action !== "none") writeEnvFile(appSourcePath, next); + if (!existsSync(appSourcePath)) throw new Error(`LangBot app secret source ${redactRepoPath(appSourcePath)} is missing; run bun scripts/cli.ts secrets sync --config config/secrets-distribution.yaml --scope platform-infra --confirm first`); + const appValues = parseEnvFile(readFileSync(appSourcePath, "utf8")); + const jwtSecret = requiredEnvValue(appValues, "LANGBOT_JWT_SECRET", langbot.runtime.secrets.appSourceRef); + const apiKey = requiredEnvValue(appValues, langbot.apiKey.key, langbot.apiKey.sourceRef); const values = { dbUser, dbPassword, dbName, - jwtSecret: next.LANGBOT_JWT_SECRET, - apiKey: next[langbot.apiKey.key], + jwtSecret, + apiKey, }; return { dbSourceRef: langbot.runtime.database.sourceRef, dbSourcePath: redactRepoPath(dbSourcePath), appSourceRef: langbot.runtime.secrets.appSourceRef, appSourcePath: redactRepoPath(appSourcePath), - action, + action: "read", values, fingerprint: fingerprintValues({ dbPassword, jwtSecret: values.jwtSecret, apiKey: values.apiKey }, ["dbPassword", "jwtSecret", "apiKey"]), valuesPrinted: false, @@ -1663,18 +1661,6 @@ function unquoteEnvValue(value: string): string { return value; } -function writeEnvFile(path: string, values: Record): void { - mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - const lines = Object.keys(values).sort().map((key) => `${key}=${quoteEnv(values[key])}`); - writeFileSync(path, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); - chmodSync(path, 0o600); -} - -function quoteEnv(value: string): string { - if (/^[A-Za-z0-9_./:@%?&=+-]+$/u.test(value)) return value; - return `'${value.replaceAll("'", "'\"'\"'")}'`; -} - function sqlLiteral(value: string): string { return `'${value.replaceAll("'", "''")}'`; } diff --git a/scripts/src/platform-infra-n8n.ts b/scripts/src/platform-infra-n8n.ts index fc7e6f0e..a25b4e13 100644 --- a/scripts/src/platform-infra-n8n.ts +++ b/scripts/src/platform-infra-n8n.ts @@ -1,6 +1,6 @@ -import { createHash, randomBytes } from "node:crypto"; -import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, isAbsolute, join } from "node:path"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { isAbsolute, join } from "node:path"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; import { startJob } from "./jobs"; @@ -97,7 +97,7 @@ interface SecretMaterial { dbSourcePath: string; appSourceRef: string; appSourcePath: string; - action: "create" | "update" | "none"; + action: "read"; values: { dbUser: string; dbPassword: string; @@ -231,6 +231,7 @@ function plan(options: CommonOptions): Record { }, next: { postgres: "bun scripts/cli.ts platform-db postgres apply --config config/platform-db/postgres-pk01.yaml --confirm", + secrets: "bun scripts/cli.ts secrets sync --config config/secrets-distribution.yaml --scope platform-infra --confirm", dryRun: `bun scripts/cli.ts platform-infra n8n apply --target ${target.id} --dry-run`, apply: `bun scripts/cli.ts platform-infra n8n apply --target ${target.id} --confirm`, status: `bun scripts/cli.ts platform-infra n8n status --target ${target.id}`, @@ -303,6 +304,7 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise): void { - mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - const lines = Object.keys(values).sort().map((key) => `${key}=${quoteEnv(values[key])}`); - writeFileSync(path, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); - chmodSync(path, 0o600); -} - -function quoteEnv(value: string): string { - if (/^[A-Za-z0-9_./:@%?&=+-]+$/u.test(value)) return value; - return `'${value.replaceAll("'", "'\"'\"'")}'`; -} - function requiredEnvValue(values: Record, key: string, sourceRef: string): string { const value = values[key]; if (value === undefined || value.length === 0) throw new Error(`${sourceRef} is missing required key ${key}`); diff --git a/scripts/src/secrets.ts b/scripts/src/secrets.ts new file mode 100644 index 00000000..16087921 --- /dev/null +++ b/scripts/src/secrets.ts @@ -0,0 +1,826 @@ +import { randomBytes } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join } from "node:path"; +import type { UniDeskConfig } from "./config"; +import { rootPath } from "./config"; +import { startJob } from "./jobs"; +import type { SshCaptureResult } from "./ssh"; +import { capture, fingerprintValues, parseJsonOutput } from "./platform-infra-public-service"; + +const defaultConfigPath = "config/secrets-distribution.yaml"; +const fieldManager = "unidesk-secret-distribution"; + +interface SecretsOptions { + configPath: string; + scope: string | null; + targetId: string | null; + confirm: boolean; + dryRun: boolean; + wait: boolean; + full: boolean; + raw: boolean; +} + +interface SecretDistributionConfig { + configPath: string; + version: number; + kind: "unidesk-secret-distribution"; + metadata: { id: string; owner: string; relatedIssues: number[] }; + sources: { root: string; files: SourceFileConfig[] }; + targets: DistributionTarget[]; + kubernetesSecrets: KubernetesSecretConfig[]; +} + +interface SourceFileConfig { + sourceRef: string; + type: "env"; + requiredKeys: string[]; + createIfMissing: { + enabled: boolean; + values: Record; + randomHex: Record; + randomBase64Url: Record; + }; +} + +interface DistributionTarget { + id: string; + route: string; + namespace: string; + scope: string; + enabled: boolean; +} + +interface KubernetesSecretConfig { + name: string; + targetId: string; + secretName: string; + type: "Opaque"; + data: SecretDataMapping[]; +} + +interface SecretDataMapping { + sourceRef: string; + sourceKey: string; + targetKey: string; +} + +interface SourceMaterial { + sourceRef: string; + sourcePath: string; + exists: boolean; + requiredKeys: string[]; + presentKeys: string[]; + missingKeys: string[]; + action: "none" | "create" | "update" | "blocked"; + generatedKeys: string[]; + unmaterializedGeneratedKeys: string[]; + values: Record; + fingerprint: string | null; +} + +interface SourceInspection { + ok: boolean; + root: string; + entries: Array>; + materials: Map; +} + +interface DesiredSecret { + name: string; + target: DistributionTarget; + secretName: string; + type: "Opaque"; + data: Record; + keySources: Array<{ sourceRef: string; sourceKey: string; targetKey: string }>; + missingKeys: Array<{ sourceRef: string; sourceKey: string; targetKey: string }>; + pendingGeneratedKeys: string[]; + fingerprint: string | null; +} + +export function secretsHelp(): Record { + return { + command: "secrets plan|sync|status", + output: "json", + usage: [ + "bun scripts/cli.ts secrets plan --config config/secrets-distribution.yaml --scope platform-infra", + "bun scripts/cli.ts secrets sync --config config/secrets-distribution.yaml --scope platform-infra --confirm", + "bun scripts/cli.ts secrets status --config config/secrets-distribution.yaml --scope platform-infra", + ], + configTruth: defaultConfigPath, + secretPolicy: "Secret values are never printed or reverse-engineered from runtime. YAML sourceRef files are the authority; sync only pushes declared keys to declared Kubernetes Secret keys.", + }; +} + +export async function runSecretsCommand(config: UniDeskConfig, args: string[]): Promise> { + const [action = "plan"] = args; + if (action === "help" || action === "--help" || action === "-h") return secretsHelp(); + if (action === "plan") return plan(parseOptions(args.slice(1))); + if (action === "sync") return await sync(config, parseOptions(args.slice(1))); + if (action === "status") return await status(config, parseOptions(args.slice(1))); + return { ok: false, error: "unsupported-secrets-command", args, help: secretsHelp() }; +} + +function parseOptions(args: string[]): SecretsOptions { + let configPath = defaultConfigPath; + let scope: string | null = null; + let targetId: string | null = null; + let confirm = false; + let dryRun = false; + let wait = false; + let full = false; + let raw = false; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--config") { + configPath = readOptionValue(args, index, "--config"); + index += 1; + } else if (arg.startsWith("--config=")) { + configPath = arg.slice("--config=".length); + } else if (arg === "--scope") { + scope = simpleId(readOptionValue(args, index, "--scope"), "--scope"); + index += 1; + } else if (arg.startsWith("--scope=")) { + scope = simpleId(arg.slice("--scope=".length), "--scope"); + } else if (arg === "--target") { + targetId = simpleId(readOptionValue(args, index, "--target"), "--target"); + index += 1; + } else if (arg.startsWith("--target=")) { + targetId = simpleId(arg.slice("--target=".length), "--target"); + } else if (arg === "--confirm") { + confirm = true; + } else if (arg === "--dry-run") { + dryRun = true; + } else if (arg === "--wait") { + wait = true; + } else if (arg === "--full") { + full = true; + } else if (arg === "--raw") { + raw = true; + full = true; + } else { + throw new Error(`unsupported secrets option: ${arg}`); + } + } + if (confirm && dryRun) throw new Error("secrets sync accepts only one of --confirm or --dry-run"); + return { configPath, scope, targetId, confirm, dryRun, wait, full, raw }; +} + +function plan(options: SecretsOptions): Record { + const distribution = readSecretDistributionConfig(options.configPath); + const sources = inspectSources(distribution, false); + const desired = desiredSecrets(distribution, options, sources); + return { + ok: sources.ok && desired.every((secret) => secret.missingKeys.length === 0), + action: "secrets-plan", + mutation: false, + config: configSummary(distribution, options), + localSources: sourceSummary(sources), + desiredSecrets: desired.map(desiredSecretSummary), + policy: { + sourceAuthority: "local YAML-declared sourceRef files under sources.root", + runtimeReverseEngineering: false, + valuesPrinted: false, + }, + next: { + sync: `bun scripts/cli.ts secrets sync --config ${distribution.configPath}${options.scope === null ? "" : ` --scope ${options.scope}`}${options.targetId === null ? "" : ` --target ${options.targetId}`} --confirm`, + status: `bun scripts/cli.ts secrets status --config ${distribution.configPath}${options.scope === null ? "" : ` --scope ${options.scope}`}${options.targetId === null ? "" : ` --target ${options.targetId}`}`, + }, + }; +} + +async function sync(config: UniDeskConfig, options: SecretsOptions): Promise> { + const distribution = readSecretDistributionConfig(options.configPath); + if (!options.confirm || options.dryRun) { + const planned = plan(options); + return { ...planned, action: "secrets-sync", mode: "dry-run", mutation: false }; + } + if (!options.wait) { + const jobArgs = ["bun", "scripts/cli.ts", "secrets", "sync", "--config", distribution.configPath, "--confirm", "--wait"]; + if (options.scope !== null) jobArgs.push("--scope", options.scope); + if (options.targetId !== null) jobArgs.push("--target", options.targetId); + const job = startJob("secrets_sync", jobArgs, "Sync YAML-declared local secret source keys into declared Kubernetes Secrets without printing values"); + return { + ok: true, + action: "secrets-sync", + mode: "async-job", + mutation: true, + config: configSummary(distribution, options), + job, + statusCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, + }; + } + const sources = inspectSources(distribution, true); + if (!sources.ok) return { ok: false, action: "secrets-sync", mode: "blocked-local-sources", mutation: true, config: configSummary(distribution, options), localSources: sourceSummary(sources) }; + const desired = desiredSecrets(distribution, options, sources); + const missing = desired.flatMap((secret) => secret.missingKeys); + if (missing.length > 0) { + return { ok: false, action: "secrets-sync", mode: "blocked-missing-secret-data", mutation: true, config: configSummary(distribution, options), localSources: sourceSummary(sources), missing, valuesPrinted: false }; + } + const perTarget = await Promise.all(groupDesiredSecretsByTarget(desired).map(async (group) => await applyTargetSecrets(config, group.target, group.secrets, options))); + return { + ok: perTarget.every((item) => item.ok === true), + action: "secrets-sync", + mode: "confirmed", + mutation: true, + config: configSummary(distribution, options), + localSources: sourceSummary(sources), + desiredSecrets: desired.map(desiredSecretSummary), + targets: perTarget, + valuesPrinted: false, + }; +} + +async function status(config: UniDeskConfig, options: SecretsOptions): Promise> { + const distribution = readSecretDistributionConfig(options.configPath); + const sources = inspectSources(distribution, false); + const desired = desiredSecrets(distribution, options, sources); + const perTarget = await Promise.all(groupDesiredSecretsByTarget(desired).map(async (group) => await statusTargetSecrets(config, group.target, group.secrets, options))); + return { + ok: perTarget.every((item) => item.ok === true), + action: "secrets-status", + mutation: false, + config: configSummary(distribution, options), + localSources: sourceSummary(sources), + desiredSecrets: desired.map(desiredSecretSummary), + targets: perTarget, + valuesPrinted: false, + }; +} + +function readSecretDistributionConfig(pathArg: string): SecretDistributionConfig { + const configPath = resolveConfigPath(pathArg); + const label = displayConfigPath(pathArg); + const root = asRecord(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown, label); + const version = integerField(root, "version", label); + const kind = stringField(root, "kind", label); + if (kind !== "unidesk-secret-distribution") throw new Error(`${label}.kind must be unidesk-secret-distribution`); + const metadata = objectField(root, "metadata", label); + const sources = objectField(root, "sources", label); + const config: SecretDistributionConfig = { + configPath: label, + version, + kind, + metadata: { + id: stringField(metadata, "id", `${label}.metadata`), + owner: stringField(metadata, "owner", `${label}.metadata`), + relatedIssues: numberArrayField(metadata, "relatedIssues", `${label}.metadata`), + }, + sources: { + root: stringField(sources, "root", `${label}.sources`), + files: arrayOfRecords(sources.files, `${label}.sources.files`).map((item, index) => parseSourceFile(item, `${label}.sources.files[${index}]`)), + }, + targets: arrayOfRecords(root.targets, `${label}.targets`).map((item, index) => parseTarget(item, `${label}.targets[${index}]`)), + kubernetesSecrets: arrayOfRecords(root.kubernetesSecrets, `${label}.kubernetesSecrets`).map((item, index) => parseKubernetesSecret(item, `${label}.kubernetesSecrets[${index}]`)), + }; + validateDistributionConfig(config); + return config; +} + +function parseSourceFile(record: Record, path: string): SourceFileConfig { + const type = stringField(record, "type", path); + if (type !== "env") throw new Error(`${path}.type must be env`); + const createRaw = record.createIfMissing === undefined ? {} : objectField(record, "createIfMissing", path); + const randomBase64UrlRaw = createRaw.randomBase64Url === undefined ? {} : objectField(createRaw, "randomBase64Url", `${path}.createIfMissing`); + return { + sourceRef: sourceRefField(record, "sourceRef", path), + type, + requiredKeys: stringArrayField(record, "requiredKeys", path).map((key, index) => envKeyValue(key, `${path}.requiredKeys[${index}]`)), + createIfMissing: { + enabled: createRaw.enabled === undefined ? false : booleanField(createRaw, "enabled", `${path}.createIfMissing`), + values: createRaw.values === undefined ? {} : stringMapField(createRaw, "values", `${path}.createIfMissing`), + randomHex: createRaw.randomHex === undefined ? {} : numberMapField(createRaw, "randomHex", `${path}.createIfMissing`), + randomBase64Url: Object.fromEntries(Object.entries(randomBase64UrlRaw).map(([key, value]) => [envKeyValue(key, `${path}.createIfMissing.randomBase64Url`), randomBase64UrlSpec(value, `${path}.createIfMissing.randomBase64Url.${key}`)])), + }, + }; +} + +function parseTarget(record: Record, path: string): DistributionTarget { + return { + id: simpleId(stringField(record, "id", path), `${path}.id`), + route: stringField(record, "route", path), + namespace: kubernetesNameField(record, "namespace", path), + scope: simpleId(stringField(record, "scope", path), `${path}.scope`), + enabled: booleanField(record, "enabled", path), + }; +} + +function parseKubernetesSecret(record: Record, path: string): KubernetesSecretConfig { + const type = stringField(record, "type", path); + if (type !== "Opaque") throw new Error(`${path}.type must be Opaque`); + return { + name: simpleId(stringField(record, "name", path), `${path}.name`), + targetId: simpleId(stringField(record, "targetId", path), `${path}.targetId`), + secretName: kubernetesNameField(record, "secretName", path), + type, + data: arrayOfRecords(record.data, `${path}.data`).map((item, index) => ({ + sourceRef: sourceRefField(item, "sourceRef", `${path}.data[${index}]`), + sourceKey: envKeyField(item, "sourceKey", `${path}.data[${index}]`), + targetKey: kubernetesSecretKeyField(item, "targetKey", `${path}.data[${index}]`), + })), + }; +} + +function validateDistributionConfig(config: SecretDistributionConfig): void { + if (config.sources.files.length === 0) throw new Error(`${config.configPath}.sources.files must not be empty`); + if (config.targets.length === 0) throw new Error(`${config.configPath}.targets must not be empty`); + const sources = new Map(config.sources.files.map((item) => [item.sourceRef, item])); + const targets = new Set(config.targets.map((item) => item.id)); + const targetSecrets = new Set(); + for (const secret of config.kubernetesSecrets) { + if (!targets.has(secret.targetId)) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name}.targetId is not declared in targets`); + const secretIdentity = `${secret.targetId}/${secret.secretName}`; + if (targetSecrets.has(secretIdentity)) throw new Error(`${config.configPath} declares duplicate target Secret ${secretIdentity}`); + targetSecrets.add(secretIdentity); + const targetKeys = new Set(); + for (const item of secret.data) { + const source = sources.get(item.sourceRef); + if (source === undefined) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} references undeclared sourceRef ${item.sourceRef}`); + if (!source.requiredKeys.includes(item.sourceKey)) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} maps ${item.sourceRef}.${item.sourceKey}, but that key is not listed in sources.files.requiredKeys`); + if (targetKeys.has(item.targetKey)) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} maps duplicate target key ${item.targetKey}`); + targetKeys.add(item.targetKey); + } + } +} + +function inspectSources(config: SecretDistributionConfig, materialize: boolean): SourceInspection { + const root = secretRoot(config); + const materials = new Map(); + const entries = config.sources.files.map((source) => { + const sourcePath = join(root, source.sourceRef); + const exists = existsSync(sourcePath); + const existing = exists ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {}; + const next = { ...existing }; + const generatedKeys: string[] = []; + const unmaterializedGeneratedKeys: string[] = []; + if (source.createIfMissing.enabled) { + for (const [key, value] of Object.entries(source.createIfMissing.values)) { + if (next[key] === undefined || next[key].length === 0) { + next[key] = value; + generatedKeys.push(key); + } + } + for (const [key, bytes] of Object.entries(source.createIfMissing.randomHex)) { + if (next[key] === undefined || next[key].length === 0) { + next[key] = materialize ? randomBytes(bytes).toString("hex") : ``; + generatedKeys.push(key); + if (!materialize) unmaterializedGeneratedKeys.push(key); + } + } + for (const [key, spec] of Object.entries(source.createIfMissing.randomBase64Url)) { + if (next[key] === undefined || next[key].length === 0) { + next[key] = materialize ? `${spec.prefix}${randomBytes(spec.bytes).toString("base64url")}` : ``; + generatedKeys.push(key); + if (!materialize) unmaterializedGeneratedKeys.push(key); + } + } + } + const missingBefore = source.requiredKeys.filter((key) => existing[key] === undefined || existing[key].length === 0); + const missingKeys = source.requiredKeys.filter((key) => next[key] === undefined || next[key].length === 0); + const action = missingKeys.length > 0 ? "blocked" : !exists ? "create" : missingBefore.length > 0 ? "update" : "none"; + if (materialize && action !== "blocked" && (action === "create" || action === "update")) writeEnvFile(sourcePath, next); + const material: SourceMaterial = { + sourceRef: source.sourceRef, + sourcePath, + exists, + requiredKeys: source.requiredKeys, + presentKeys: source.requiredKeys.filter((key) => next[key] !== undefined && next[key].length > 0), + missingKeys, + action, + generatedKeys, + unmaterializedGeneratedKeys, + values: next, + fingerprint: missingKeys.length === 0 && unmaterializedGeneratedKeys.length === 0 ? fingerprintValues(next, source.requiredKeys) : null, + }; + materials.set(source.sourceRef, material); + return { + sourceRef: source.sourceRef, + sourcePath: redactRepoPath(sourcePath), + exists, + requiredKeys: source.requiredKeys, + presentKeys: material.presentKeys, + missingKeys, + action, + generatedKeys, + unmaterializedGeneratedKeys, + fingerprint: material.fingerprint, + valuesPrinted: false, + }; + }); + return { + ok: entries.every((entry) => (entry.missingKeys as string[]).length === 0), + root: redactRepoPath(root), + entries, + materials, + }; +} + +function desiredSecrets(config: SecretDistributionConfig, options: SecretsOptions, sources: SourceInspection): DesiredSecret[] { + const targets = new Map(config.targets.map((target) => [target.id, target])); + return config.kubernetesSecrets + .map((secret) => ({ secret, target: targets.get(secret.targetId) })) + .filter((item): item is { secret: KubernetesSecretConfig; target: DistributionTarget } => item.target !== undefined && item.target.enabled) + .filter(({ target }) => options.scope === null || target.scope === options.scope) + .filter(({ target }) => options.targetId === null || target.id === options.targetId) + .map(({ secret, target }) => { + const data: Record = {}; + const missingKeys: DesiredSecret["missingKeys"] = []; + const pendingGeneratedKeys: string[] = []; + for (const item of secret.data) { + const source = sources.materials.get(item.sourceRef); + const value = source?.values[item.sourceKey]; + if (value === undefined || value.length === 0) { + missingKeys.push({ sourceRef: item.sourceRef, sourceKey: item.sourceKey, targetKey: item.targetKey }); + } else { + data[item.targetKey] = value; + } + if (source?.unmaterializedGeneratedKeys.includes(item.sourceKey) === true) pendingGeneratedKeys.push(item.targetKey); + } + const keys = Object.keys(data); + return { + name: secret.name, + target, + secretName: secret.secretName, + type: secret.type, + data, + keySources: secret.data, + missingKeys, + pendingGeneratedKeys, + fingerprint: missingKeys.length === 0 && pendingGeneratedKeys.length === 0 ? fingerprintValues(data, keys) : null, + }; + }); +} + +async function applyTargetSecrets(config: UniDeskConfig, target: DistributionTarget, secrets: DesiredSecret[], options: SecretsOptions): Promise> { + if (secrets.length === 0) return { ok: true, target: targetSummary(target), mode: "skipped-no-secrets" }; + const yaml = renderSecretManifest(target, secrets); + const result = await capture(config, target.route, ["script"], applySecretScript(target, secrets, yaml)); + const parsed = parseJsonOutput(result.stdout); + return { + ok: result.exitCode === 0 && boolField(parsed, "ok", false), + target: targetSummary(target), + summary: parsed, + remote: secretCaptureSummary(result), + ...(options.raw ? { rawCaptureOmitted: true, rawPolicy: "Secret distribution never returns raw SSH capture because remote output can contain credential-bearing diagnostics." } : {}), + }; +} + +async function statusTargetSecrets(config: UniDeskConfig, target: DistributionTarget, secrets: DesiredSecret[], options: SecretsOptions): Promise> { + const result = await capture(config, target.route, ["script"], statusSecretScript(target, secrets)); + const parsed = parseJsonOutput(result.stdout); + return { + ok: result.exitCode === 0 && boolField(parsed, "ok", false), + target: targetSummary(target), + summary: parsed, + remote: secretCaptureSummary(result), + ...(options.raw ? { rawCaptureOmitted: true, rawPolicy: "Secret distribution never returns raw SSH capture because remote output can contain credential-bearing diagnostics." } : {}), + }; +} + +function secretCaptureSummary(result: SshCaptureResult): Record { + return { + exitCode: result.exitCode, + stdoutBytes: Buffer.byteLength(result.stdout, "utf8"), + stderrBytes: Buffer.byteLength(result.stderr, "utf8"), + stdoutTailOmitted: true, + stderrTailOmitted: true, + valuesPrinted: false, + }; +} + +function renderSecretManifest(target: DistributionTarget, secrets: DesiredSecret[]): string { + return secrets.map((secret) => `apiVersion: v1 +kind: Secret +metadata: + name: ${secret.secretName} + namespace: ${target.namespace} + labels: + app.kubernetes.io/managed-by: unidesk + app.kubernetes.io/part-of: ${target.scope} +type: ${secret.type} +data: +${Object.entries(secret.data).sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => ` ${key}: ${Buffer.from(value, "utf8").toString("base64")}`).join("\n")} +`).join("---\n"); +} + +function applySecretScript(target: DistributionTarget, secrets: DesiredSecret[], yaml: string): string { + const manifestB64 = Buffer.from(yaml, "utf8").toString("base64"); + const summaryB64 = Buffer.from(JSON.stringify(secrets.map(remoteSecretSummary)), "utf8").toString("base64"); + return ` +set -u +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +manifest="$tmp/secrets.yaml" +printf '%s' '${manifestB64}' | base64 -d >"$manifest" +kubectl create namespace ${target.namespace} --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$tmp/ns.out" 2>"$tmp/ns.err" +ns_rc=$? +if [ "$ns_rc" -eq 0 ]; then + kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f "$manifest" >"$tmp/apply.out" 2>"$tmp/apply.err" + apply_rc=$? +else + : >"$tmp/apply.out" + printf '%s\\n' 'skipped because namespace sync failed' >"$tmp/apply.err" + apply_rc=1 +fi +python3 - "$ns_rc" "$apply_rc" "$tmp/ns.out" "$tmp/ns.err" "$tmp/apply.out" "$tmp/apply.err" <<'PY' +import base64, json, sys +ns_rc, apply_rc = int(sys.argv[1]), int(sys.argv[2]) +def text(path, limit=5000): + try: + return open(path, encoding="utf-8", errors="replace").read()[-limit:] + except FileNotFoundError: + return "" +payload = { + "ok": ns_rc == 0 and apply_rc == 0, + "namespace": "${target.namespace}", + "secrets": json.loads(base64.b64decode("${summaryB64}").decode("utf-8")), + "valuesPrinted": False, + "steps": { + "namespace": {"exitCode": ns_rc, "stdout": text(sys.argv[3]), "stderr": text(sys.argv[4])}, + "apply": {"exitCode": apply_rc, "stdout": text(sys.argv[5]), "stderr": text(sys.argv[6])}, + }, +} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if payload["ok"] else 1) +PY +`; +} + +function statusSecretScript(target: DistributionTarget, secrets: DesiredSecret[]): string { + const summaryB64 = Buffer.from(JSON.stringify(secrets.map(remoteSecretSummary)), "utf8").toString("base64"); + const commands = secrets.map((secret, index) => [ + `kubectl -n ${target.namespace} get secret ${secret.secretName} -o json >"$tmp/secret.${index}.json" 2>"$tmp/secret.${index}.err"`, + `printf '%s' "$?" >"$tmp/secret.${index}.rc"`, + ].join("\n")).join("\n"); + return ` +set -u +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +${commands} +python3 - "$tmp" <<'PY' +import base64, json, os, sys +tmp = sys.argv[1] +expected = json.loads(base64.b64decode("${summaryB64}").decode("utf-8")) +items = [] +ok = True +for index, item in enumerate(expected): + try: + rc = int(open(os.path.join(tmp, f"secret.{index}.rc"), encoding="utf-8").read() or "1") + except FileNotFoundError: + rc = 1 + try: + observed = json.load(open(os.path.join(tmp, f"secret.{index}.json"), encoding="utf-8")) + except Exception: + observed = None + data = (observed or {}).get("data") or {} + observed_keys = sorted(data.keys()) + expected_keys = item.get("keys") or [] + missing = [key for key in expected_keys if key not in observed_keys] + exists = rc == 0 + item_ok = exists and len(missing) == 0 + ok = ok and item_ok + items.append({ + "name": item.get("name"), + "secretName": item.get("secretName"), + "exists": exists, + "keys": observed_keys, + "expectedKeys": expected_keys, + "missingKeys": missing, + "ok": item_ok, + "valuesPrinted": False, + }) +payload = {"ok": ok, "namespace": "${target.namespace}", "secrets": items, "valuesPrinted": False} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if ok else 1) +PY +`; +} + +function groupDesiredSecretsByTarget(secrets: DesiredSecret[]): Array<{ target: DistributionTarget; secrets: DesiredSecret[] }> { + const groups = new Map(); + for (const secret of secrets) { + const existing = groups.get(secret.target.id); + if (existing === undefined) groups.set(secret.target.id, { target: secret.target, secrets: [secret] }); + else existing.secrets.push(secret); + } + return Array.from(groups.values()); +} + +function configSummary(config: SecretDistributionConfig, options: SecretsOptions): Record { + return { + path: config.configPath, + metadata: config.metadata, + root: redactRepoPath(secretRoot(config)), + scope: options.scope, + targetId: options.targetId, + sources: config.sources.files.map((item) => ({ sourceRef: item.sourceRef, requiredKeys: item.requiredKeys, createIfMissing: item.createIfMissing.enabled })), + targets: config.targets.filter((target) => (options.scope === null || target.scope === options.scope) && (options.targetId === null || target.id === options.targetId)).map(targetSummary), + valuesPrinted: false, + }; +} + +function sourceSummary(sources: SourceInspection): Record { + return { ok: sources.ok, root: sources.root, entries: sources.entries, valuesPrinted: false }; +} + +function desiredSecretSummary(secret: DesiredSecret): Record { + return { + name: secret.name, + target: targetSummary(secret.target), + secretName: secret.secretName, + keys: Object.keys(secret.data).sort(), + keySources: secret.keySources, + missingKeys: secret.missingKeys, + pendingGeneratedKeys: secret.pendingGeneratedKeys, + fingerprint: secret.fingerprint, + valuesPrinted: false, + }; +} + +function remoteSecretSummary(secret: DesiredSecret): Record { + return { + name: secret.name, + secretName: secret.secretName, + keys: Object.keys(secret.data).sort(), + fingerprint: secret.fingerprint, + valuesPrinted: false, + }; +} + +function targetSummary(target: DistributionTarget): Record { + return { id: target.id, route: target.route, namespace: target.namespace, scope: target.scope }; +} + +function readOptionValue(args: string[], index: number, option: string): string { + const value = args[index + 1]; + if (value === undefined || value.length === 0 || value.startsWith("--")) throw new Error(`${option} requires a value`); + return value; +} + +function resolveConfigPath(pathArg: string): string { + if (pathArg.startsWith("/") || pathArg.includes("\0")) throw new Error("--config must be a repo-relative YAML path"); + if (pathArg.includes("..")) throw new Error("--config must not contain .."); + return rootPath(pathArg); +} + +function displayConfigPath(pathArg: string): string { + if (pathArg.startsWith("/") || pathArg.includes("..")) throw new Error("--config must be a repo-relative YAML path without .."); + return pathArg; +} + +function secretRoot(config: SecretDistributionConfig): string { + const root = config.sources.root; + return isAbsolute(root) ? root : rootPath(root); +} + +function parseEnvFile(text: string): Record { + const result: Record = {}; + for (const rawLine of text.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq <= 0) continue; + const key = line.slice(0, eq).trim(); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) continue; + result[key] = unquoteEnvValue(line.slice(eq + 1).trim()); + } + return result; +} + +function unquoteEnvValue(value: string): string { + if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith("\"") && value.endsWith("\""))) return value.slice(1, -1); + return value; +} + +function writeEnvFile(path: string, values: Record): void { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const lines = Object.keys(values).sort().map((key) => `${key}=${quoteEnv(values[key])}`); + writeFileSync(path, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); +} + +function quoteEnv(value: string): string { + if (/^[A-Za-z0-9_./:@%?&=+-]+$/u.test(value)) return value; + return `'${value.replaceAll("'", "'\"'\"'")}'`; +} + +function redactRepoPath(path: string): string { + const root = rootPath(); + return path.startsWith(`${root}/`) ? path.slice(root.length + 1) : path; +} + +function boolField(value: Record | null, key: string, fallback: boolean): boolean { + return typeof value?.[key] === "boolean" ? value[key] : fallback; +} + +function asRecord(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be a YAML object`); + return value as Record; +} + +function objectField(obj: Record, key: string, path: string): Record { + return asRecord(obj[key], `${path}.${key}`); +} + +function arrayOfRecords(value: unknown, path: string): Record[] { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`); + return value.map((item, index) => asRecord(item, `${path}[${index}]`)); +} + +function stringField(obj: Record, key: string, path: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value.trim(); +} + +function integerField(obj: Record, key: string, path: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isInteger(value)) throw new Error(`${path}.${key} must be an integer`); + return value; +} + +function booleanField(obj: Record, key: string, path: string): boolean { + const value = obj[key]; + if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); + return value; +} + +function stringArrayField(obj: Record, key: string, path: string): string[] { + const value = obj[key]; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.trim().length === 0)) throw new Error(`${path}.${key} must be a string array`); + return value.map((item) => (item as string).trim()); +} + +function numberArrayField(obj: Record, key: string, path: string): number[] { + const value = obj[key]; + if (!Array.isArray(value) || value.some((item) => typeof item !== "number" || !Number.isInteger(item))) throw new Error(`${path}.${key} must be an integer array`); + return value as number[]; +} + +function stringMapField(obj: Record, key: string, path: string): Record { + const record = objectField(obj, key, path); + const result: Record = {}; + for (const [itemKey, itemValue] of Object.entries(record)) { + result[envKeyValue(itemKey, `${path}.${key}`)] = stringValue(itemValue, `${path}.${key}.${itemKey}`); + } + return result; +} + +function numberMapField(obj: Record, key: string, path: string): Record { + const record = objectField(obj, key, path); + const result: Record = {}; + for (const [itemKey, itemValue] of Object.entries(record)) { + result[envKeyValue(itemKey, `${path}.${key}`)] = randomByteCount(itemValue, `${path}.${key}.${itemKey}`); + } + return result; +} + +function randomBase64UrlSpec(value: unknown, path: string): { bytes: number; prefix: string } { + const record = asRecord(value, path); + return { + bytes: randomByteCount(record.bytes, `${path}.bytes`), + prefix: record.prefix === undefined ? "" : stringValue(record.prefix, `${path}.prefix`), + }; +} + +function stringValue(value: unknown, path: string): string { + if (typeof value !== "string") throw new Error(`${path} must be a string`); + return value; +} + +function randomByteCount(value: unknown, path: string): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < 16 || value > 128) throw new Error(`${path} must be an integer in 16..128`); + return value; +} + +function sourceRefField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (value.startsWith("/") || value.includes("..") || value.includes("\0") || !/^[A-Za-z0-9_./-]+$/u.test(value)) throw new Error(`${path}.${key} must be a relative source ref without ..`); + return value; +} + +function envKeyField(obj: Record, key: string, path: string): string { + return envKeyValue(stringField(obj, key, path), `${path}.${key}`); +} + +function envKeyValue(value: string, path: string): string { + if (!/^[A-Z_][A-Z0-9_]*$/u.test(value)) throw new Error(`${path} must be an uppercase env key`); + return value; +} + +function kubernetesNameField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value)) throw new Error(`${path}.${key} must be a Kubernetes name`); + return value; +} + +function kubernetesSecretKeyField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path}.${key} must be a Kubernetes Secret key`); + return value; +} + +function simpleId(value: string, path: string): string { + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a simple id`); + return value; +}