From c23d9b5ea06b82b96960f9bf039f18f82e0ee8f2 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 12 Jun 2026 07:56:46 +0000 Subject: [PATCH] feat: activate D601 sub2api external edge --- .agents/skills/unidesk-sub2api/SKILL.md | 24 +- config/platform-db/postgres-pk01.yaml | 17 +- config/platform-infra/sub2api.yaml | 91 +- docs/reference/pk01.md | 10 + docs/reference/platform-infra.md | 26 +- .../platform-infra-sub2api-codex-sentinel.ts | 24 + scripts/src/platform-infra-sub2api-codex.ts | 302 ++- scripts/src/platform-infra.ts | 2045 ++++++++++++++++- 8 files changed, 2375 insertions(+), 164 deletions(-) diff --git a/.agents/skills/unidesk-sub2api/SKILL.md b/.agents/skills/unidesk-sub2api/SKILL.md index 5550e0ab..e0169d9a 100644 --- a/.agents/skills/unidesk-sub2api/SKILL.md +++ b/.agents/skills/unidesk-sub2api/SKILL.md @@ -5,7 +5,7 @@ description: UniDesk Sub2API 平台运维技能。用户提到 Sub2API、sub2api # UniDesk Sub2API -UniDesk 在 k3s `platform-infra` namespace 运维 Sub2API。G14 是默认 active runtime；D601 只作为同一 YAML/CLI 控制下的 standby predeploy target，外置 DB 未就绪时应用和本地 Redis cache 都保持 replicas=0。日常操作统一使用 UniDesk CLI，不直接写 Kubernetes 资源或手工调用 Sub2API 管理 API。 +UniDesk 在 k3s `platform-infra` namespace 运维 Sub2API。G14 是默认 active runtime；D601 由同一 YAML/CLI 控制，可保持 standby predeploy，也可在外置 DB、镜像、FRP 和 egress proxy 条件就绪后作为 external-active target 运行。日常操作统一使用 UniDesk CLI，不直接写 Kubernetes 资源或手工调用 Sub2API 管理 API。 **固定入口**: `cd /root/unidesk && bun scripts/cli.ts platform-infra sub2api ...` @@ -33,8 +33,8 @@ bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id ` / `auth.json.` 并在 YAML 里声明。 - 输出只能包含 Secret 路径、长度、preview/fingerprint；禁止打印完整 API key、admin password、JWT secret、TOTP key。 @@ -57,7 +57,7 @@ bun scripts/cli.ts platform-infra sub2api validate --target D601 - `plan` 读取 `config/platform-infra/sub2api.yaml`，渲染 `src/components/platform-infra/sub2api/sub2api.k8s.yaml`，检查 no Ingress/NodePort/LoadBalancer/hostPort/hostNetwork/resource limits，并要求 `NetworkPolicy/allow-all` 随 manifest 受控创建。 - `apply --confirm` 默认创建异步 job；按返回的 `job status` 命令轮询，再跑 `status` 和 `validate`。 - `status --full|--raw` 只在需要展开远端 stdout/stderr 或原始 JSON 时使用。 -- `validate` 是按需验收，不是连续可用性探针。对 D601 standby，`validate --target D601` 验证预部署形态，不要求外置 DB 当前可连接。 +- `validate` 是按需验收，不是连续可用性探针。对 D601 standby，`validate --target D601` 验证预部署形态，不要求外置 DB 当前可连接；对 D601 external-active，必须验证外置 DB、ephemeral Redis、Sub2API service、YAML egress proxy 和目标级 public exposure。 ## 镜像升级 @@ -73,14 +73,22 @@ bun scripts/cli.ts platform-infra sub2api validate --target D601 ```bash bun scripts/cli.ts platform-infra sub2api codex-pool plan +bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm +bun scripts/cli.ts platform-infra sub2api codex-pool sync --target D601 --confirm bun scripts/cli.ts platform-infra sub2api codex-pool validate +bun scripts/cli.ts platform-infra sub2api codex-pool validate --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status +bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm +bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --target D601 --confirm bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm +bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --target D601 --account unidesk-codex-hy --confirm bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report +bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report --target D601 bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --confirm +bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --target D601 --confirm ``` `config/platform-infra/sub2api-codex-pool.yaml` 控制： @@ -113,7 +121,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --confirm `sync --confirm` 会登录 Sub2API admin、创建/更新 group、创建/更新 YAML 中的 `unidesk-codex-*` accounts、创建/复用统一 API key Secret，并把未处于哨兵 active quarantine 的 managed account 的 `schedulable=true` 恢复为过程控制基线；它默认不删除 YAML 中缺席的 managed account。只有明确退役上游时才使用 `sync --confirm --prune-removed` 删除缺席且 `extra.unidesk_managed=true` 的 `unidesk-codex-*` account。 -`sentinel-image status|build` 管理哨兵 Python 运行环境镜像。镜像由 YAML 的 `sentinel.image` 基础镜像和 `sentinel.sdk.openaiPythonVersion` 派生，发布到 G14 本地 registry `127.0.0.1:5000/platform-infra/sub2api-account-sentinel:`；`build --confirm` 会先检查 registry tag，存在则快速复用，不存在才在 G14 host 构建并 push。CronJob 启动时只校验 SDK 版本，不在运行时 `pip install`。 +`sentinel-image status|build` 管理哨兵 Python 运行环境镜像。镜像由 YAML 的 `sentinel.image` 基础镜像和 `sentinel.sdk.openaiPythonVersion` 派生，发布到目标 runtime 的本地 registry；`build --confirm` 会先检查 registry tag，存在则快速复用，不存在才在目标 host 构建并 push。CronJob 启动时只校验 SDK 版本，不在运行时 `pip install`。 `sync --confirm` 同时会按 YAML 渲染账号级哨兵资源，并在 monitor 开启时先确保可复用哨兵镜像存在。当前目标是 `sentinel.monitor.enabled=true` + `sentinel.actions.enabled=true` 的 marker-only 自动冻结/恢复；不要手工 patch CronJob、Secret 或 Sub2API account。若 YAML 新增账号或修改 profile/base URL/API key fingerprint/upstream User-Agent/Responses WebSocket mode，sync 会从变更前 runtime state 写入 pending probe 记录并立即安排 sentinel probe，但默认仍保持该 account 可调度；只有实际 marker probe 非命中或已有 active quarantine 才会冻结账号。sentinel 冻结/恢复只改 `schedulable=false|true`，不得顺手调用 Sub2API `recover-state` 清除请求路径临时不可调度或其他 runtime backoff。无关账号的既有成功/失败退避不能被重置。若 YAML 下调失败冻结最大窗口，sync 会把仍 active 的旧冻结状态迁移到当前最大窗口内并立即安排 recovery probe，但不会直接解冻。若怀疑某个账号被误判，先用 `codex-pool sentinel-probe --account --confirm` 立即触发该账号测量；该命令从现有 CronJob 模板派生一次性 Job，复用同一份 Secret、ConfigMap、OpenAI SDK probe、token/cost 账本和冻结/恢复状态机。 @@ -160,13 +168,14 @@ bun scripts/cli.ts platform-infra sub2api codex-pool expose bun scripts/cli.ts platform-infra sub2api codex-pool expose --confirm ``` -- 由 `publicExposure` YAML 控制。默认公共端是 `publicBaseUrl`，master 本地消费端是 `masterBaseUrl`。 +- 由 YAML `publicExposure` 控制。Codex pool 默认公共端是 `publicBaseUrl`，master 本地消费端是 `masterBaseUrl`；D601 external-active 的目标级 public exposure 在 `config/platform-infra/sub2api.yaml` 中声明。 - `expose --confirm` 只为 YAML 指定的 `remotePort` 补 master `frps` allow port，并在 G14 创建/更新 `sub2api-frpc`。 - master Caddy site 也由 `publicExposure.masterCaddy` 渲染；`responseHeaderTimeoutSeconds` 必须足够覆盖 Codex `/responses/compact` 长请求，避免 Caddy 先返回 504 而 Sub2API 后台实际稍后成功。具体数值只改 `config/platform-infra/sub2api-codex-pool.yaml`，修改后跑 `codex-pool expose --confirm`，再核对 Caddyfile 中渲染出的 `response_header_timeout`。 - master Caddy 的短窗口边缘重试由 `publicExposure.masterCaddy.edgeRetry` 渲染；用于吸收 FRP remotePort 短暂关闭、`connect: connection refused`、EOF 或 connection reset 这类请求尚未稳定到达 Sub2API 的 502。具体 retry 时长、间隔和 `retryMatch` 范围只写 YAML，修改后跑 `codex-pool expose --confirm`，再核对 Caddyfile 中渲染出的 `lb_try_duration`、`lb_try_interval` 和 `lb_retry_match`。不要手工 patch `/etc/caddy/Caddyfile`。 - 非幂等 POST 的 round-trip retry 必须收窄到 YAML `retryMatch` 声明的安全路径；普通 `/responses` 上游账号错误仍归 Sub2API failover / temp-unschedulable / sentinel 处理，不用 Caddy 重放整段推理请求来掩盖账号池问题。 - 同一个 FRP TCP 入口同时暴露 OpenAI-compatible API 和 Sub2API 管理 UI `/login`。不要另开第二个管理端口，除非 YAML 明确声明新的暴露决策。 - Sub2API Kubernetes Service 继续保持 ClusterIP。 +- D601 external-active 的公开路径是 `client -> PK01 Caddy -> PK01 frps remotePort -> D601 frpc -> Sub2API`，不经过 pikanode，也不经过 master server 反代。PK01 Caddy 下载必须使用 YAML `publicExposure.pk01.caddyDownloadProxyUrl` 指定的 proxy；如果 Caddy 下载慢，先确认 apply 输出里是 `downloadProxy.mode=curl-proxy`。`api.pikapython.com` 必须先解析到 YAML 声明的 PK01 公网地址，HTTPS 才能作为最终验证。 ## 配置 master Codex 消费端 @@ -193,6 +202,7 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm - `sub2api validate`：app、PostgreSQL、Redis、service proxy、`NetworkPolicy/allow-all` 和临时跨 Pod PostgreSQL/Redis 连通性检查通过。 - `codex-pool validate`：统一 key 的 `GET /v1/models` 成功，并用 `localCodex.responsesSmokeModel` 跑一次小的 `POST /v1/responses` smoke；owner balance / owner concurrency 已满足 YAML 最小值，capacity、WebSocket v2、Sub2API 内置 temporary-unschedulable 开关/规则和 sentinel runtime 状态与 YAML 对齐；`validation.gatewayResponsesRecent` 汇总最近 6 小时普通 `/responses` 和 `/v1/responses` 的 failover、forward failure、最终 4xx/5xx、慢 final error 与 `context canceled` 证据，`validation.gatewayCompactRecent` 单独汇总 `/responses/compact` 证据。若当前 Responses smoke `ok=true` 但 recent 字段 `degraded=true`，先区分是历史窗口残留还是新的 request id 正在失败；长期判定见 `docs/reference/platform-infra.md`。 - 若 `publicExposure.enabled=true`，确认 FRP path 可用；`expose --confirm` 会用未带 key 的 public `/v1/models` 401 作为网关可达性探针。 +- 若目标声明了 `egressProxy.enabled=true`，确认 proxy Deployment/Service ready，Sub2API 和 sentinel env 与 YAML 对齐，并通过 YAML 声明的 health URL 完成代理出站探针。 如果要证明真实模型请求可用，使用最小 `/v1/responses` 或等价 Codex smoke。不要把 group-level `/v1/models` 成功解释成每个上游 account 都健康。 @@ -205,6 +215,8 @@ bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm - pool key 401：跑 `codex-pool sync --confirm` 重建 Sub2API key 与 k3s Secret 绑定，再跑 `codex-pool validate`。 - 运行中过去的验证探针残留：只用 `codex-pool cleanup-probes --confirm` 清理 `unidesk-probe-*` 临时资源；不要把真实 managed account 删除当作探针清理或可用性恢复。 - FRP 不通：先看 `codex-pool expose --confirm` 输出的 `masterFrps`、`masterCaddy`、`sub2api-frpc` 和 public 401 probe；需要低层证据时只用 `trans G14:k3s` 做 bounded 查询。 +- D601 external-active 的 `api.pikapython.com` 不通：先区分 DNS/TLS/Caddy/FRP/Sub2API。DNS 未解析到 YAML 声明的 PK01 地址时，Caddy ACME 会失败，`https://api.pikapython.com` 不能算完成；可用 PK01 loopback FRP 端口和 PK01 公网 remotePort 证明 D601 FRP 数据路径，但最终仍要等 DNS 生效后重跑 HTTPS health、`/v1/models` 和 `/v1/responses`。 +- Caddy 下载慢或失败：先确认 `config/platform-infra/sub2api.yaml` 已设置 `publicExposure.pk01.caddyDownloadProxyUrl`，并重跑 `sub2api apply --target D601 --confirm` 看 PK01 apply summary 中的 `downloadProxy.mode=curl-proxy`。不要反复裸连 GitHub release。 - `/responses/compact` 在接近 master Caddy `response_header_timeout` 的固定时长后返回 504，或 Sub2API 日志稍后记录 `codex.remote_compact.succeeded` 时，优先检查 master Caddy `response_header_timeout` 是否由 YAML `publicExposure.masterCaddy.responseHeaderTimeoutSeconds` 渲染，修正后跑 `codex-pool expose --confirm`；这类边缘代理超时不会触发 Sub2API 账号级临时下线。reload 前已经在途的 compact 请求仍可能按旧 timeout 结束，判断修复是否生效时只看 reload 之后新发起的请求。 - `/responses/compact` 或普通 public URL 在几秒窗口内出现 502，Caddy 日志显示 `dial tcp 127.0.0.1:: connect: connection refused`、`EOF` 或 `connection reset by peer`，同时 frps 日志出现 `platform-infra-sub2api proxy closing` / `listener is closed` / `new proxy ... success`，说明失败在 master Caddy 与 FRP remotePort 边缘层，Sub2API 和 sentinel 可能完全看不到。先确认 `publicExposure.masterCaddy.edgeRetry` 已按 YAML 渲染并 `codex-pool expose --confirm` 生效；若仍频繁发生，再继续查 G14 `sub2api-frpc` 到 master `frps` 的控制连接稳定性。不要把这类边缘 502 误判成账号池上游错误，也不要通过禁用账号恢复。 - default profile 递归：检查 YAML default entry 是否使用 `*.pre-sub2api` 备份文件；必要时恢复备份后重新 `configure-local --confirm`。 diff --git a/config/platform-db/postgres-pk01.yaml b/config/platform-db/postgres-pk01.yaml index 08495fed..b6711e4b 100644 --- a/config/platform-db/postgres-pk01.yaml +++ b/config/platform-db/postgres-pk01.yaml @@ -110,20 +110,35 @@ postgres: user: sub2api address: 10.0.8.0/22 method: scram-sha-256 + - type: hostssl + database: postgres + user: sub2api + address: 10.0.8.0/22 + method: scram-sha-256 - type: hostssl database: sub2api user: sub2api address: 74.48.78.17/32 method: scram-sha-256 + - type: hostssl + database: postgres + user: sub2api + address: 74.48.78.17/32 + method: scram-sha-256 - type: hostssl database: sub2api user: sub2api address: 36.49.29.73/32 method: scram-sha-256 + - type: hostssl + database: postgres + user: sub2api + address: 36.49.29.73/32 + method: scram-sha-256 secrets: source: master-local - root: .state/secrets + root: /root/unidesk/.state/secrets entries: - name: sub2api-db-credentials sourceRef: platform-db/sub2api-db.env diff --git a/config/platform-infra/sub2api.yaml b/config/platform-infra/sub2api.yaml index 361c6aa8..9742976d 100644 --- a/config/platform-infra/sub2api.yaml +++ b/config/platform-infra/sub2api.yaml @@ -15,24 +15,101 @@ targets: - id: D601 route: D601:k3s namespace: platform-infra - role: standby + role: active-standby enabled: true - databaseMode: external-pending + databaseMode: external-active redisMode: local-ephemeral - appReplicas: 0 - redisReplicas: 0 + appReplicas: 1 + redisReplicas: 1 + image: + repository: 127.0.0.1:5000/platform-infra/sub2api + tag: 0.1.136 + pullPolicy: IfNotPresent + dependencyImages: + postgres: docker.m.daocloud.io/library/postgres:18-alpine + redis: docker.m.daocloud.io/library/redis:8-alpine + publicExposure: + enabled: true + publicBaseUrl: https://api.pikapython.com + dns: + hostname: api.pikapython.com + expectedA: 82.156.23.220 + resolvers: [1.1.1.1, 8.8.8.8, 223.5.5.5, 114.114.114.114] + frpc: + deploymentName: sub2api-frpc + secretName: sub2api-frpc-secrets + secretKey: frpc.toml + image: 127.0.0.1:5000/hwlab/frpc:v0.68.1 + serverAddr: 82.156.23.220 + serverPort: 22000 + proxyName: platform-infra-sub2api-d601-api + remotePort: 22098 + localIP: sub2api.platform-infra.svc.cluster.local + localPort: 8080 + tokenSourceRef: platform-infra/pk01-frp.env + tokenSourceKey: FRP_TOKEN + pk01: + route: PK01 + caddyBinaryPath: /usr/local/bin/caddy + caddyDownloadUrl: https://caddyserver.com/api/download?os=linux&arch=amd64 + caddyDownloadProxyUrl: http://127.0.0.1:18789 + caddyConfigPath: /etc/caddy/Caddyfile + caddyServiceName: caddy + caddyStorageDir: /var/lib/caddy + caddyEmail: ops@pikapython.com + pikanodeRoot: /home/ubuntu/pikanode + pikanodeContainerName: pikanode + pikanodeImage: pikanode + pikanodeHttpHostPort: 18888 + responseHeaderTimeoutSeconds: 600 + egressProxy: + enabled: true + deploymentName: sub2api-egress-proxy + serviceName: sub2api-egress-proxy + secretName: sub2api-egress-proxy-config + secretKey: config.json + image: 127.0.0.1:5000/platform-infra/sing-box:latest + imagePullPolicy: IfNotPresent + listenPort: 10808 + sourceRef: platform-infra/master-vpn-subscription.env + sourceKey: MASTER_VPN_SUBSCRIPTION_URL + sourceType: subscription-url + preferredOutbound: vless-reality + applyToSub2Api: true + applyToSentinel: true + healthProbeUrl: https://www.gstatic.com/generate_204 + noProxy: + - localhost + - 127.0.0.1 + - ::1 + - .svc + - .cluster.local + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + - 82.156.23.220 + - 74.48.78.17 + - hyueapi.com + - .hyueapi.com runtime: database: mode: external - sourceRef: platform-db/postgres-active.env + sourceRef: platform-db/sub2api-db.env + sourceKeys: + user: SUB2API_DB_USER + password: SUB2API_DB_PASSWORD + dbName: SUB2API_DB_NAME secretName: sub2api-secrets passwordKey: POSTGRES_PASSWORD - host: pika01-postgres.pending.local + host: 82.156.23.220 port: 5432 user: sub2api dbName: sub2api - sslMode: prefer + sslMode: require pendingAllowed: true + secrets: + root: /root/unidesk/.state/secrets + appSourceRef: platform-infra/sub2api.env redis: serviceName: sub2api-redis persistence: false diff --git a/docs/reference/pk01.md b/docs/reference/pk01.md index 12777650..72319ac0 100644 --- a/docs/reference/pk01.md +++ b/docs/reference/pk01.md @@ -58,6 +58,16 @@ PK01 currently hosts existing Docker workloads: `pikanode` mounts `/home/ubuntu/pikanode` read-write into the container. Static/generated download artifacts under `html/download/` and repository data under `files/` may be user-visible or needed by the service. They are not generic GC candidates. +## Sub2API Caddy Edge + +PK01 may act as the public Caddy edge for a YAML-declared D601 Sub2API target. The durable source of truth is `config/platform-infra/sub2api.yaml`; do not hand-edit PK01 Caddy or FRP state as a separate routing truth. + +In this mode, public ports `80` and `443` belong to Caddy. The existing `pikanode` container must be bound to a loopback HTTP port and used only as the apex PikaPython/PikaNode upstream. The `api.pikapython.com` site must reverse proxy directly to the YAML-declared FRP remote port, so API traffic follows `client -> PK01 Caddy -> PK01 frps remote port -> D601 frpc -> D601 Sub2API`. It must not pass through pikanode or a master-server reverse proxy. + +Caddy binary installation is also YAML-controlled. If `publicExposure.pk01.caddyDownloadProxyUrl` is set, PK01 Caddy downloads must use that proxy URL; the PK01 loopback provider egress proxy is the preferred source. A slow or failing Caddy download should first be treated as missing proxy use, not as a reason to keep retrying a naked GitHub release download. + +The public certificate depends on DNS. The `api.pikapython.com` record must resolve to the YAML-declared PK01 public address before Caddy can complete ACME issuance. If the DNS record is absent or stale, local probes such as PK01 `127.0.0.1:` and public-IP remote-port checks can prove the FRP data path, but final `https://api.pikapython.com` validation remains blocked until DNS is corrected. + ## Host PostgreSQL PK01 host-native PostgreSQL is declared by `config/platform-db/postgres-pk01.yaml` and managed through `bun scripts/cli.ts platform-db postgres plan|status|apply`; daily operation commands live in `$unidesk-ops` at `.agents/skills/unidesk-ops/SKILL.md`. It is a host systemd service, not a Docker container or k3s workload. The YAML is the source of truth for PostgreSQL version, TLS mode, listening addresses, `pg_hba` source CIDRs, generated Secret source files, exported `DATABASE_URL`, and backup timer settings. diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 81af195c..18cacbd1 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -1,6 +1,6 @@ # Platform Infra -`platform-infra` is the k3s namespace for UniDesk-operated shared platform services. G14 is the active default runtime for this namespace; D601 may host explicitly declared standby platform targets when the service needs node-local preparation or cutover capacity. It is separate from HWLAB runtime lanes, AgentRun lanes, D601 user services, and legacy `devops-infra` control-plane helpers. New shared infra should land here first; old `devops-infra` resources migrate gradually only when a concrete owner and validation path exist. +`platform-infra` is the k3s namespace for UniDesk-operated shared platform services. G14 is the default active runtime for this namespace; D601 may host explicitly declared standby or externally backed active targets when the service needs node-local preparation, cutover capacity, or a direct public edge. It is separate from HWLAB runtime lanes, AgentRun lanes, D601 user services, and legacy `devops-infra` control-plane helpers. New shared infra should land here first; old `devops-infra` resources migrate gradually only when a concrete owner and validation path exists. ## Source Of Truth @@ -12,15 +12,15 @@ ## Sub2API Deployment Boundary - Sub2API is a platform service operated by UniDesk in namespace `platform-infra`. It is not a HWLAB lane workload, AgentRun workload, D601 user service, or master server daemon. -- The canonical deployment entrypoint is `bun scripts/cli.ts platform-infra sub2api plan|apply|status|validate|codex-pool`. Runtime targets are selected with `--target`; `G14` is the active default target and `D601` is a standby target controlled by the same YAML. Daily operation procedures live in `$unidesk-sub2api` at `.agents/skills/unidesk-sub2api/SKILL.md`. This reference keeps only development boundaries and project-specific source-of-truth rules. +- The canonical deployment entrypoint is `bun scripts/cli.ts platform-infra sub2api plan|apply|status|validate|codex-pool`. Runtime targets are selected with `--target`; `G14` is the default active target and `D601` is controlled by the same YAML as either standby predeploy or externally backed active runtime. Daily operation procedures live in `$unidesk-sub2api` at `.agents/skills/unidesk-sub2api/SKILL.md`. This reference keeps only development boundaries and project-specific source-of-truth rules. - Raw `kubectl` through `trans :k3s` is only for bounded diagnosis and evidence, not a formal mutate path. - The image version is controlled by `config/platform-infra/sub2api.yaml`. Image update procedures are daily operations owned by `$unidesk-sub2api`; the development boundary is that image choices remain YAML-controlled. - Sub2API should stay ClusterIP-only by default. Do not add Ingress, NodePort, LoadBalancer, or broad FRP exposure unless a YAML-controlled public exposure decision exists. - Sub2API currently has no resource limits by design. Do not add CPU or memory limits unless a later explicit decision changes that policy and stores the new policy in YAML. - Master server is a consumer/control host, not the runtime location. Do not deploy Sub2API, PostgreSQL, Redis, or heavy validation loops on master server. -- D601 Sub2API is a predeployment target, not a second active singleton. While the platform database handoff is pending, it must render without a local PostgreSQL StatefulSet, keep the Sub2API app and local Redis cache scaled to zero, and use only ephemeral Redis storage when Redis is later activated. After the external platform DB endpoint, Secret, and runtime images are ready, activation must be expressed by YAML and applied through the same `platform-infra sub2api --target D601` CLI path. +- D601 Sub2API is selected by YAML, not by ad hoc runtime patches. In standby mode it must render without a local PostgreSQL StatefulSet, keep the Sub2API app and local Redis cache scaled to zero, and use only ephemeral Redis storage when Redis is later activated. In externally backed active mode it connects directly to the YAML-declared external PostgreSQL endpoint with `sslmode=require`, keeps durable app state outside the D601 k3s node, and uses local Redis only as ephemeral cache. Activation must be applied through the same `platform-infra sub2api --target D601` CLI path. - External platform PostgreSQL endpoints for Sub2API are produced by the platform DB YAML and its `platform-db postgres` CLI. Cross-node Sub2API consumers connect directly to that endpoint; the master server is not a PostgreSQL data-plane relay. DNS aliases are optional when the exported `DATABASE_URL` uses a reachable IP with `sslmode=require`; current PK01-specific rules live in `docs/reference/pk01.md`. -- Sub2API account sentinel and public FRP exposure remain singleton concerns. Do not create a second sentinel or public management surface for D601 unless a later YAML-controlled decision explicitly moves or splits that responsibility. +- Sub2API account sentinel and public exposure are target-scoped YAML decisions. Do not create a second sentinel, FRP client, public management surface, or edge proxy by hand; enable or move those resources only through the target YAML and the `platform-infra sub2api` / `codex-pool --target` CLI paths. ## Codex Pool Routing @@ -45,7 +45,7 @@ - Codex account-state, quota prompts, model-routing failures, encrypted-content affinity failures, gateway wrappers, and timeout-like upstream errors must be handled by the generic temporary-unschedulable/failover path plus the external marker sentinel. Do not change membership, priority, capacity, load factor, WebSocket mode, `pool_mode`, or a specific provider's status merely to work around those errors. If a matching upstream failure still logs `openai.forward_failed` without `openai.upstream_failover_switching`, the missing fix is in Sub2API's HTTP `/responses` failover classification/error propagation, not in account pinning. - `profiles.entries[].openaiResponsesWebSocketsV2Mode` is the account-level Responses WebSocket v2 switch for OpenAI-compatible upstreams that require WebSocket transport. Allowed values are `off`, `ctx_pool`, and `passthrough`; omit the field unless that upstream needs it. - `profiles.entries[].upstreamUserAgent` is an optional account-level upstream request User-Agent override. Use it only for upstreams that require a Codex CLI compatible User-Agent; keep the value YAML-controlled and newline-free. -- `publicExposure` controls the optional FRP bridge from master server to the G14 ClusterIP service. +- `publicExposure` in `config/platform-infra/sub2api-codex-pool.yaml` controls the default Codex-pool public bridge from master server to the G14 ClusterIP service. Target-level `publicExposure` in `config/platform-infra/sub2api.yaml` controls non-master exposure such as a D601-to-PK01 edge. - `publicExposure.masterCaddy.responseHeaderTimeoutSeconds` controls the master Caddy `response_header_timeout` for the public Sub2API site. It must be long enough for Codex `/responses/compact` requests; otherwise Caddy can return a client-visible 504 before Sub2API finishes the upstream compact request, and that edge timeout is not an account-level upstream failure that Sub2API can use for temporary-unschedulable failover. The numeric value belongs only in `config/platform-infra/sub2api-codex-pool.yaml`; after changing it, use `codex-pool expose --confirm` to reload Caddy and verify the rendered `response_header_timeout`. Requests that were already in flight before the reload may still finish with the previous timeout, so post-change evidence should check only requests that started after the reload. - `publicExposure.masterCaddy.edgeRetry` controls the master Caddy reverse-proxy retry window for the public Sub2API site. This belongs at the edge because FRP remotePort listener loss, `connection refused`, EOF, or connection reset can happen before a request reaches Sub2API, so Sub2API account failover and sentinel logic cannot observe or recover that request. Keep retry scope narrow, especially for non-idempotent POST traffic: connection-attempt failures may be retried by the reverse proxy, while round-trip retry after an upstream connection was established should be limited by YAML `retryMatch` to paths that are safe to repeat, such as compact. Retry durations and intervals belong only in YAML; after changing them, run `codex-pool expose --confirm` and verify the rendered Caddyfile contains the expected `lb_try_duration`, `lb_try_interval`, and `lb_retry_match`. - `localCodex` controls how the master server's current `~/.codex` consumer files are backed up and rewritten. Keep `supportsWebSockets` and `responsesWebSocketsV2` in the same state, and enable them only when at least one YAML-managed account has a current direct Codex WSv2 smoke that passes. If no upstream profile can sustain Responses WSv2, the honest long-term state is `false/false` so Codex uses HTTP Responses directly instead of repeatedly reconnecting before `response.completed`. `localCodex.responsesSmokeModel` is the YAML-declared model used by `codex-pool validate` for the lightweight `POST /v1/responses` smoke. @@ -92,7 +92,7 @@ If the YAML success cadence maximum is lowered or an account changes trust class Operational observation for this sentinel should use the read-only `codex-pool sentinel-report` table or its `--raw` form. It is the canonical low-noise view for per-account probe count, trust class, marker result, HTTP/error diagnostics, freeze TTL, success cadence, success cadence maximum, next probe time, and recent CronJob runs; raw ConfigMap dumps and ad hoc log scraping are fallback diagnostics, not the primary state surface. -The request path is: +The default G14 Codex-pool request path is: 1. A client sends an OpenAI-compatible request to the configured consumer base URL, normally `https://sub2api.74-48-78-17.nip.io/v1/...`, with the unified API key. 2. master `frps` forwards the TCP connection to `platform-infra/sub2api-frpc` when `publicExposure.enabled` is true. @@ -100,6 +100,10 @@ The request path is: 4. Sub2API validates the unified key and resolves its `group_id`. 5. Accounts listed in `profiles.entries` are bound to the same group via `group_ids`, so Sub2API dispatches through that group using its own account selection semantics. +The D601 externally backed request path is different when target-level `publicExposure.enabled=true` in `config/platform-infra/sub2api.yaml`: client traffic reaches PK01 Caddy, PK01 forwards to the YAML-declared FRP remote port, D601 `sub2api-frpc` connects directly to PK01 `frps`, and FRP forwards to `sub2api.platform-infra.svc.cluster.local:8080` on D601. This path does not pass through the master server or the pikanode reverse proxy. `api.pikapython.com` must resolve to the YAML-declared PK01 public address before Caddy can obtain or renew the public certificate; when DNS is missing, PK01 local FRP probes and public-IP remote-port probes may prove the edge path, but they are not a substitute for final `https://api.pikapython.com` validation. + +When target-level `egressProxy.enabled=true`, the D601 target renders an in-cluster HTTP(S) proxy client from the master VPN subscription source declared in YAML. The CLI injects the resulting proxy URL and `NO_PROXY` into Sub2API and, when requested by YAML, the Codex account sentinel. `platform-infra sub2api validate --target D601 --full` must prove the proxy Deployment/Service is ready and that an app pod can complete the YAML-declared health probe through the proxy. Subscription contents and generated proxy configs are Secret material and must not be printed. + Adding, removing, exposing, validating, and configuring local Codex consumers are daily operations covered by `$unidesk-sub2api`. The development rule is that ordinary pool membership changes stay YAML-only and do not add code or CI/CD. Code changes are only appropriate when UniDesk needs to render or validate a Sub2API capability that already exists upstream, such as account-level WebSocket mode or per-account upstream User-Agent. If Sub2API itself does not support a desired behavior, do not magic-patch it through UniDesk scripts, Kubernetes hotfixes, local forks, or hidden compatibility paths; either leave the behavior unsupported or pursue it upstream as an explicit Sub2API feature. `codex-pool sync --confirm` and `codex-pool validate` are runtime operations that may need more than one SSH short-connection window because they log in to Sub2API, reconcile accounts, inspect recent logs, and run gateway smoke requests. The formal entry remains the UniDesk CLI, which must use a submit-and-short-poll control shape or an equivalent remote job wrapper instead of one long `trans G14:k3s script` call. If these commands fail with `UNIDESK_SSH_RUNTIME_TIMEOUT` while the remote operation may still be running, treat it as a control-plane visibility gap first: improve or use the CLI's job/poll path, then rerun `sync` or `validate`. Do not replace it with raw `kubectl`, manual Sub2API admin API patches, repeated blind full loops, or Sub2API source modifications. @@ -112,18 +116,18 @@ When `publicExposure.enabled` is true, the same FRP TCP bridge exposes both Open The public management UI is an operations endpoint. Keep Sub2API itself in `platform-infra`, keep the Kubernetes Service as ClusterIP, and treat FRP as the only public bridge unless a later decision explicitly changes the exposure model. -The public bridge has two separate failure classes. Sub2API upstream/account failures are visible in Sub2API logs and currently belong to sentinel quarantine plus normal Sub2API routing among schedulable accounts. Edge failures between master Caddy and the FRP remotePort are not visible to Sub2API; symptoms include Caddy `connect: connection refused`, EOF, connection reset, or short 502 bursts while frps closes and reopens the configured remotePort. Those failures must be diagnosed from Caddy and frps/frpc evidence and mitigated through YAML-controlled Caddy edge retry or FRP stability fixes, not by disabling accounts or changing pool membership. +The public bridge has two separate failure classes. Sub2API upstream/account failures are visible in Sub2API logs and currently belong to sentinel quarantine plus normal Sub2API routing among schedulable accounts. Edge failures between Caddy and the FRP remote port are not visible to Sub2API; symptoms include Caddy `connect: connection refused`, EOF, connection reset, TLS/certificate failures, DNS NXDOMAIN, or short 502 bursts while frps closes and reopens the configured remote port. Those failures must be diagnosed from DNS, Caddy, and frps/frpc evidence and mitigated through YAML-controlled Caddy edge retry, DNS correction, or FRP stability fixes, not by disabling accounts or changing pool membership. ## Availability And Probes Kubernetes readiness is not the same as pool availability: - The Sub2API app, PostgreSQL, and Redis manifests include container-level health probes. These only prove the pods and local dependencies are healthy enough for Kubernetes scheduling. -- The FRP client deployment is currently a simple connector deployment and does not itself prove that master-local traffic reaches Sub2API. +- The FRP client deployment is a connector deployment and does not itself prove that edge traffic reaches Sub2API. - No scheduled `CronJob`, `ServiceMonitor`, or `PodMonitor` currently proves the full unified Codex API path. - `platform-infra sub2api validate` and `platform-infra sub2api codex-pool validate` are on-demand checks. Operational usage is documented in `$unidesk-sub2api`; they are acceptable for deployment closeout, but they are not continuous monitoring. `codex-pool validate` must test both `GET /v1/models` and a small `POST /v1/responses` request, and the Responses smoke should report request id, selected/final account evidence, upstream failover count, and whether the validation succeeded only after failover. It should also summarize recent `/responses` and `/responses/compact` gateway failures separately so ordinary long streaming failures are not hidden behind compact-only evidence. - `codex-pool validate` must not create mock upstreams or temporary failover-probe accounts as its default proof of Sub2API behavior. When a suspected failover path is in question, validate should surface the relevant source-path expectation and real runtime evidence: request ids, selected/final account ids, `openai.upstream_failover_switching`, `openai.forward_failed`, `openai.account_select_failed`, and final status. If runtime evidence contradicts the source-path expectation, fix Sub2API or the UniDesk integration path rather than converting the mismatch into a mock-only success. -- Public exposure closeout must include the edge layer when the user-facing URL is involved. A Sub2API-side compact success summary does not rule out Caddy/FRP 502s that happened before Sub2API received the request; inspect the edge Caddy/frps/frpc evidence or use a CLI report that summarizes it before declaring public compact stable. +- Public exposure closeout must include the edge layer when the user-facing URL is involved. A Sub2API-side compact success summary does not rule out DNS, Caddy, TLS, or FRP failures that happened before Sub2API received the request; inspect the edge evidence or use a CLI report that summarizes it before declaring the public URL stable. - Because `codex-pool validate` includes account alignment, recent-log inspection, and gateway smoke, timeout of the CLI transport is not valid negative evidence about Sub2API scheduling by itself. Closeout evidence must come from the final structured validation result or from an explicitly reported remote job failure with stdout/stderr tail, not from a single low-level `trans` timeout. When an automatic availability probe is added, it should be YAML-controlled and cover these layers without printing secrets: @@ -133,6 +137,8 @@ When an automatic availability probe is added, it should be YAML-controlled and 3. A tiny `POST /v1/responses` call through the same consumer URL for true OpenAI-compatible request validation. 4. Optional per-upstream account probes if Sub2API exposes a safe account selection or admin-health mechanism; otherwise document that group-level success does not prove every upstream account is healthy. +For D601 public exposure, the equivalent probe set must use the target URL from `config/platform-infra/sub2api.yaml`, include the PK01 Caddy/FRP edge, and require `api.pikapython.com` DNS to resolve to the YAML-declared address before treating HTTPS as validated. + Until continuous probing exists, closeout comments must state that validation was on-demand and include the exact CLI/API entrypoints used. ## k3s Network Policy Requirements @@ -169,4 +175,4 @@ spec: This policy must be included in the `sub2api plan` / `apply` manifest rendering so that it is created as part of the normal deployment flow, not maintained as a manual one-off. -`platform-infra sub2api status` must report whether `NetworkPolicy/allow-all` exists and still has `podSelector: {}`, `policyTypes: [Ingress, Egress]`, `ingress: [{}]`, and `egress: [{}]`. For active bundled targets, `platform-infra sub2api validate` must also run temporary in-namespace probe pods that connect to `sub2api-postgres:5432` and `sub2api-redis:6379`; local `pg_isready` inside the PostgreSQL pod alone is insufficient because it does not exercise kube-router cross-pod policy evaluation. For external-DB pending standby targets, `validate --target` checks the predeployment shape instead: no local PostgreSQL, app replicas zero, ClusterIP services, allow-all NetworkPolicy, and local Redis declared as ephemeral cache with readiness required only when Redis replicas are above zero. +`platform-infra sub2api status` must report whether `NetworkPolicy/allow-all` exists and still has `podSelector: {}`, `policyTypes: [Ingress, Egress]`, `ingress: [{}]`, and `egress: [{}]`. For active bundled targets, `platform-infra sub2api validate` must also run temporary in-namespace probe pods that connect to `sub2api-postgres:5432` and `sub2api-redis:6379`; local `pg_isready` inside the PostgreSQL pod alone is insufficient because it does not exercise kube-router cross-pod policy evaluation. For external-DB standby targets, `validate --target` checks the predeployment shape: no local PostgreSQL, app replicas zero, ClusterIP services, allow-all NetworkPolicy, and local Redis declared as ephemeral cache with readiness required only when Redis replicas are above zero. For external-DB active targets, `validate --target` checks that the app uses the external database endpoint, local Redis is ephemeral, no local PostgreSQL StatefulSet exists, and any YAML-declared egress proxy and public exposure resources are present and probed through their configured paths. diff --git a/scripts/src/platform-infra-sub2api-codex-sentinel.ts b/scripts/src/platform-infra-sub2api-codex-sentinel.ts index 6b22e400..93323619 100644 --- a/scripts/src/platform-infra-sub2api-codex-sentinel.ts +++ b/scripts/src/platform-infra-sub2api-codex-sentinel.ts @@ -80,6 +80,10 @@ export interface CodexPoolSentinelManifestOptions { serviceName: string; serviceDns: string; appSecretName: string; + proxy?: { + httpProxy: string; + noProxy: string; + } | null; } export function defaultCodexPoolSentinelConfig(): CodexPoolSentinelConfig { @@ -317,6 +321,25 @@ export function renderCodexPoolSentinelManifest( const activeDeadlineSeconds = Math.max(300, Math.min(3600, config.probe.timeoutSeconds + 240)); const command = sentinelContainerShellCommand(config); const runtimeImage = codexPoolSentinelRuntimeImage(config).runtimeImage; + const proxyEnv = options.proxy?.httpProxy + ? ` - name: HTTP_PROXY + value: ${JSON.stringify(options.proxy.httpProxy)} + - name: HTTPS_PROXY + value: ${JSON.stringify(options.proxy.httpProxy)} + - name: ALL_PROXY + value: ${JSON.stringify(options.proxy.httpProxy)} + - name: http_proxy + value: ${JSON.stringify(options.proxy.httpProxy)} + - name: https_proxy + value: ${JSON.stringify(options.proxy.httpProxy)} + - name: all_proxy + value: ${JSON.stringify(options.proxy.httpProxy)} + - name: NO_PROXY + value: ${JSON.stringify(options.proxy.noProxy)} + - name: no_proxy + value: ${JSON.stringify(options.proxy.noProxy)} +` + : ""; return `apiVersion: v1 kind: Secret metadata: @@ -446,6 +469,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace +${proxyEnv} volumeMounts: - name: sentinel-config mountPath: /opt/sentinel diff --git a/scripts/src/platform-infra-sub2api-codex.ts b/scripts/src/platform-infra-sub2api-codex.ts index 22d163bb..7565609a 100644 --- a/scripts/src/platform-infra-sub2api-codex.ts +++ b/scripts/src/platform-infra-sub2api-codex.ts @@ -17,11 +17,13 @@ import { import { runSshCommandCapture, type SshCaptureResult } from "./ssh"; const g14K3sRoute = "G14:k3s"; +const defaultTargetId = "G14"; const namespace = "platform-infra"; const serviceName = "sub2api"; const serviceDns = `${serviceName}.${namespace}.svc.cluster.local:8080`; const fieldManager = "unidesk-platform-infra"; const appSecretName = "sub2api-secrets"; +const sub2apiConfigPath = rootPath("config", "platform-infra", "sub2api.yaml"); const codexPoolConfigPath = rootPath("config", "platform-infra", "sub2api-codex-pool.yaml"); const sentinelImageDockerfilePath = rootPath("src", "components", "platform-infra", "sub2api", "sentinel.Dockerfile"); const defaultPoolGroupName = "unidesk-codex-pool"; @@ -37,6 +39,7 @@ const remoteJobPollMs = 5_000; interface DisclosureOptions { full: boolean; raw: boolean; + targetId: string; } interface SyncOptions extends DisclosureOptions { @@ -70,6 +73,21 @@ interface SentinelImageOptions extends DisclosureOptions { dryRun: boolean; } +interface CodexPoolRuntimeTarget { + id: string; + route: string; + namespace: string; + serviceName: string; + serviceDns: string; + appSecretName: string; + egressProxy: { + enabled: boolean; + applyToSentinel: boolean; + httpProxy: string; + noProxy: string; + } | null; +} + interface CodexProfile { profile: string; accountName: string; @@ -204,14 +222,15 @@ export function codexPoolHelp(): unknown { output: "json, except trace and sentinel-report default to low-noise text tables", usage: [ "bun scripts/cli.ts platform-infra sub2api codex-pool plan", - "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm [--prune-removed]", - "bun scripts/cli.ts platform-infra sub2api codex-pool validate [--full|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool trace --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status", - "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm", - "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm", - "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report [--events 20|--full|--raw]", - "bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --confirm", + "bun scripts/cli.ts platform-infra sub2api codex-pool plan --target D601", + "bun scripts/cli.ts platform-infra sub2api codex-pool sync [--target D601] --confirm [--prune-removed]", + "bun scripts/cli.ts platform-infra sub2api codex-pool validate [--target D601] [--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool trace [--target D601] --request-id [--since 24h|--tail 20000|--context-seconds 300|--show-lines|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status [--target D601]", + "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build [--target D601] --confirm", + "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe [--target D601] --account unidesk-codex-hy --confirm", + "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-report [--target D601] [--events 20|--full|--raw]", + "bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes [--target D601] --confirm", "bun scripts/cli.ts platform-infra sub2api codex-pool expose --confirm", "bun scripts/cli.ts platform-infra sub2api codex-pool configure-local --confirm", ], @@ -254,14 +273,14 @@ export async function runCodexPoolCommand(config: UniDeskConfig, args: string[]) } function parseSyncOptions(args: string[]): SyncOptions { - validateOptions(args, new Set(["--confirm", "--prune-removed", "--full", "--raw"])); - const disclosure = parseDisclosureOptions(args.filter((arg) => arg !== "--confirm" && arg !== "--prune-removed")); + validateOptions(args, new Set(["--confirm", "--prune-removed", "--full", "--raw", "--target"])); + const disclosure = parseDisclosureOptions(stripBooleanOptions(args, new Set(["--confirm", "--prune-removed"]))); return { ...disclosure, confirm: args.includes("--confirm"), pruneRemoved: args.includes("--prune-removed") }; } function parseConfirmOptions(args: string[]): ConfirmOptions { - validateOptions(args, new Set(["--confirm", "--full", "--raw"])); - const disclosure = parseDisclosureOptions(args.filter((arg) => arg !== "--confirm")); + validateOptions(args, new Set(["--confirm", "--full", "--raw", "--target"])); + const disclosure = parseDisclosureOptions(stripBooleanOptions(args, new Set(["--confirm"]))); return { ...disclosure, confirm: args.includes("--confirm") }; } @@ -271,7 +290,8 @@ function parseSentinelImageOptions(args: string[]): SentinelImageOptions { let confirm = false; let explicitDryRun = false; const disclosureArgs: string[] = []; - for (const arg of rest) { + for (let index = 0; index < rest.length; index += 1) { + const arg = rest[index]!; if (arg === "--confirm") { confirm = true; continue; @@ -280,8 +300,14 @@ function parseSentinelImageOptions(args: string[]): SentinelImageOptions { explicitDryRun = true; continue; } - if (arg === "--full" || arg === "--raw") { + if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) { disclosureArgs.push(arg); + if (arg === "--target") { + const value = rest[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); + disclosureArgs.push(value); + index += 1; + } continue; } throw new Error(`unsupported option: ${arg}`); @@ -306,8 +332,14 @@ function parseSentinelProbeOptions(args: string[]): SentinelProbeOptions { confirm = true; continue; } - if (arg === "--full" || arg === "--raw") { + if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) { disclosureArgs.push(arg); + if (arg === "--target") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); + disclosureArgs.push(value); + index += 1; + } continue; } if (arg === "--account") { @@ -336,8 +368,14 @@ function parseSentinelReportOptions(args: string[]): SentinelReportOptions { const disclosureArgs: string[] = []; for (let index = 0; index < args.length; index += 1) { const arg = args[index]!; - if (arg === "--full" || arg === "--raw") { + if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) { disclosureArgs.push(arg); + if (arg === "--target") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); + disclosureArgs.push(value); + index += 1; + } continue; } if (arg === "--events") { @@ -369,8 +407,14 @@ function parseTraceOptions(args: string[]): TraceOptions { const disclosureArgs: string[] = []; for (let index = 0; index < args.length; index += 1) { const arg = args[index]!; - if (arg === "--full" || arg === "--raw") { + if (arg === "--full" || arg === "--raw" || arg === "--target" || arg.startsWith("--target=")) { disclosureArgs.push(arg); + if (arg === "--target") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); + disclosureArgs.push(value); + index += 1; + } continue; } if (arg === "--show-lines") { @@ -452,9 +496,26 @@ function readReportEventLimit(raw: string, option: string): number { } function parseDisclosureOptions(args: string[]): DisclosureOptions { - validateOptions(args, new Set(["--full", "--raw"])); + validateOptions(args, new Set(["--full", "--raw", "--target"])); const raw = args.includes("--raw"); - return { full: raw || args.includes("--full"), raw }; + return { full: raw || args.includes("--full"), raw, targetId: parseTargetId(args) }; +} + +function parseTargetId(args: string[]): string { + let targetId = defaultTargetId; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]!; + if (arg === "--target") { + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) throw new Error("--target requires a value"); + targetId = value; + index += 1; + continue; + } + if (arg.startsWith("--target=")) targetId = arg.slice("--target=".length); + } + if (!/^[A-Za-z0-9._-]+$/u.test(targetId)) throw new Error("--target must be a simple target id"); + return targetId; } function splitAccountNames(value: string): string[] { @@ -462,14 +523,68 @@ function splitAccountNames(value: string): string[] { } function validateOptions(args: string[], booleanOptions: Set): void { - for (const arg of args) { + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]!; + if (arg === "--target") { + index += 1; + continue; + } + if (arg.startsWith("--target=") && booleanOptions.has("--target")) continue; if (booleanOptions.has(arg)) continue; throw new Error(`unsupported option: ${arg}`); } } -function codexPoolPlan(options: DisclosureOptions = { full: false, raw: false }): Record { +function stripBooleanOptions(args: string[], stripped: Set): string[] { + return args.filter((arg) => !stripped.has(arg)); +} + +function codexPoolRuntimeTarget(targetId: string = defaultTargetId): CodexPoolRuntimeTarget { + const parsed = Bun.YAML.parse(readFileSync(sub2apiConfigPath, "utf8")) as unknown; + if (!isRecord(parsed) || !Array.isArray(parsed.targets)) throw new Error(`${sub2apiConfigPath}.targets must be a list`); + const raw = parsed.targets.find((item) => isRecord(item) && String(item.id ?? "").toLowerCase() === targetId.toLowerCase()); + if (!isRecord(raw)) throw new Error(`${sub2apiConfigPath}.targets does not contain target ${targetId}`); + const id = stringValue(raw.id) ?? targetId; + const route = stringValue(raw.route) ?? (id === defaultTargetId ? g14K3sRoute : ""); + const targetNamespace = stringValue(raw.namespace) ?? namespace; + if (route.length === 0) throw new Error(`${sub2apiConfigPath}.targets[${id}].route is required`); + validateKubernetesName(targetNamespace, `${sub2apiConfigPath}.targets[${id}].namespace`, true); + + let egressProxy: CodexPoolRuntimeTarget["egressProxy"] = null; + if (isRecord(raw.egressProxy) && raw.egressProxy.enabled === true) { + const proxyServiceName = stringValue(raw.egressProxy.serviceName); + const listenPort = numberValue(raw.egressProxy.listenPort); + if (proxyServiceName === null || listenPort === null) throw new Error(`${sub2apiConfigPath}.targets[${id}].egressProxy.serviceName/listenPort are required`); + validateKubernetesName(proxyServiceName, `${sub2apiConfigPath}.targets[${id}].egressProxy.serviceName`, true); + if (!Number.isInteger(listenPort) || listenPort < 1 || listenPort > 65535) throw new Error(`${sub2apiConfigPath}.targets[${id}].egressProxy.listenPort must be a TCP port`); + const noProxyRaw = Array.isArray(raw.egressProxy.noProxy) ? raw.egressProxy.noProxy : []; + const noProxy = noProxyRaw.map((entry) => stringValue(entry)).filter((entry): entry is string => entry !== null && entry.length > 0).join(","); + egressProxy = { + enabled: true, + applyToSentinel: raw.egressProxy.applyToSentinel === undefined ? true : raw.egressProxy.applyToSentinel === true, + httpProxy: `http://${proxyServiceName}.${targetNamespace}.svc.cluster.local:${listenPort}`, + noProxy, + }; + } + + return { + id, + route, + namespace: targetNamespace, + serviceName, + serviceDns: `${serviceName}.${targetNamespace}.svc.cluster.local:8080`, + appSecretName, + egressProxy, + }; +} + +function targetFlag(target: CodexPoolRuntimeTarget): string { + return target.id === defaultTargetId ? "" : ` --target ${target.id}`; +} + +function codexPoolPlan(options: DisclosureOptions = { full: false, raw: false, targetId: defaultTargetId }): Record { const pool = readCodexPoolConfig(); + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const profiles = collectCodexProfiles(); const ok = profiles.length > 0 && profiles.every((profile) => profile.ok); return { @@ -481,7 +596,7 @@ function codexPoolPlan(options: DisclosureOptions = { full: false, raw: false }) authPattern: "YAML-selected auth files under ~/.codex", valuesPrinted: false, }, - target: poolTarget(), + target: poolTarget(pool, runtimeTarget), config: { path: codexPoolConfigPath, pool: options.full ? pool : codexPoolConfigSummary(pool), @@ -490,9 +605,9 @@ function codexPoolPlan(options: DisclosureOptions = { full: false, raw: false }) decision: { accountType: "openai/apikey", grouping: `All discovered Codex profiles are bound to one Sub2API group named ${pool.groupName}.`, - unifiedApiKey: `The client-facing API_KEY is controlled by k3s Secret ${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}.`, + unifiedApiKey: `The client-facing API_KEY is controlled by k3s Secret ${runtimeTarget.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}.`, sentinel: pool.sentinel.monitor.enabled - ? `Account sentinel is enabled as k8s CronJob ${namespace}/${pool.sentinel.cronJobName}; actions.enabled=${pool.sentinel.actions.enabled}.` + ? `Account sentinel is enabled as k8s CronJob ${runtimeTarget.namespace}/${pool.sentinel.cronJobName}; actions.enabled=${pool.sentinel.actions.enabled}.` : "Account sentinel monitoring is disabled by YAML.", publicExposure: pool.publicExposure.enabled ? `Default Codex consumers use ${codexConsumerBaseUrl(pool)}; bounded master-local probes may use ${pool.publicExposure.masterBaseUrl}. FRP proxy ${pool.publicExposure.proxyName} maps public ${pool.publicExposure.publicBaseUrl} to ${pool.publicExposure.localIP}:${pool.publicExposure.localPort}.` @@ -501,13 +616,14 @@ function codexPoolPlan(options: DisclosureOptions = { full: false, raw: false }) configPolicy: "UniDesk-owned durable configuration remains YAML-first; local ~/.codex files and runtime Secrets are not committed.", }, next: ok - ? { sync: "bun scripts/cli.ts platform-infra sub2api codex-pool sync --confirm" } + ? { sync: `bun scripts/cli.ts platform-infra sub2api codex-pool sync${targetFlag(runtimeTarget)} --confirm` } : { fix: "Ensure every discovered config.toml profile has a base_url and either auth.json OPENAI_API_KEY or the configured env_key present in this shell." }, }; } async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promise> { const pool = readCodexPoolConfig(); + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const profiles = collectCodexProfiles(); const planOk = profiles.length > 0 && profiles.every((profile) => profile.ok); if (!options.confirm || !planOk) { @@ -522,7 +638,7 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi } const sentinelImage = pool.sentinel.monitor.enabled - ? await runCodexPoolSentinelImage(config, pool, { action: "build", confirm: true, dryRun: false, full: options.full, raw: false }) + ? await runCodexPoolSentinelImage(config, pool, { action: "build", confirm: true, dryRun: false, full: options.full, raw: false, targetId: options.targetId }) : { ok: true, mode: "skipped-monitor-disabled" }; if (sentinelImage.ok !== true) { return { @@ -531,7 +647,7 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi mode: "blocked-sentinel-image", sentinelImage, next: { - image: "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm", + image: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build${targetFlag(runtimeTarget)} --confirm`, }, }; } @@ -540,10 +656,14 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi pruneRemoved: options.pruneRemoved, sentinel: { manifest: renderCodexPoolSentinelManifest(pool.sentinel, sentinelProfileSecrets(profiles), { - namespace, - serviceName, - serviceDns, - appSecretName, + namespace: runtimeTarget.namespace, + serviceName: runtimeTarget.serviceName, + serviceDns: runtimeTarget.serviceDns, + appSecretName: runtimeTarget.appSecretName, + proxy: runtimeTarget.egressProxy?.applyToSentinel ? { + httpProxy: runtimeTarget.egressProxy.httpProxy, + noProxy: runtimeTarget.egressProxy.noProxy, + } : null, }), summary: codexPoolSentinelSummary(pool.sentinel), }, @@ -589,7 +709,7 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi tempUnschedulableCredentials: renderSub2ApiTempUnschedulableCredentials(profile.tempUnschedulable), })), }; - const result = await runRemoteCodexPoolScript(config, "sync", syncScript(payload, pool)); + const result = await runRemoteCodexPoolScript(config, "sync", syncScript(payload, pool, runtimeTarget), runtimeTarget); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { @@ -614,7 +734,7 @@ async function codexPoolSync(config: UniDeskConfig, options: SyncOptions): Promi ? compactCapture(result, { full: options.full || result.exitCode !== 0 }) : options.full ? parsed : codexPoolSyncSummary(parsed), next: { - validate: "bun scripts/cli.ts platform-infra sub2api codex-pool validate", + validate: `bun scripts/cli.ts platform-infra sub2api codex-pool validate${targetFlag(runtimeTarget)}`, }, }; } @@ -625,6 +745,7 @@ async function codexPoolSentinelImage(config: UniDeskConfig, options: SentinelIm } async function runCodexPoolSentinelImage(config: UniDeskConfig, pool: CodexPoolConfig, options: SentinelImageOptions): Promise> { + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const target = codexPoolSentinelRuntimeImage(pool.sentinel); if (options.action === "build" && options.dryRun) { return { @@ -635,13 +756,13 @@ async function runCodexPoolSentinelImage(config: UniDeskConfig, pool: CodexPoolC dockerfile: sentinelImageDockerfilePath, mutation: false, next: { - confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build --confirm", + confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image build${targetFlag(runtimeTarget)} --confirm`, }, }; } const mode: RemoteCodexPoolMode = options.action === "status" ? "sentinel-image-status" : "sentinel-image-build"; - const script = options.action === "status" ? sentinelImageStatusScript(pool) : sentinelImageBuildScript(pool); - const result = await runRemoteCodexPoolScript(config, mode, script); + const script = options.action === "status" ? sentinelImageStatusScript(pool, runtimeTarget) : sentinelImageBuildScript(pool, runtimeTarget); + const result = await runRemoteCodexPoolScript(config, mode, script, runtimeTarget); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { @@ -665,7 +786,8 @@ async function runCodexPoolSentinelImage(config: UniDeskConfig, pool: CodexPoolC async function codexPoolValidate(config: UniDeskConfig, options: DisclosureOptions): Promise> { const pool = readCodexPoolConfig(); - const result = await runRemoteCodexPoolScript(config, "validate", validateScript(pool)); + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); + const result = await runRemoteCodexPoolScript(config, "validate", validateScript(pool, runtimeTarget), runtimeTarget); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { @@ -685,7 +807,8 @@ async function codexPoolValidate(config: UniDeskConfig, options: DisclosureOptio async function codexPoolTrace(config: UniDeskConfig, options: TraceOptions): Promise | RenderedCliResult> { const pool = readCodexPoolConfig(); - const result = await capture(config, g14K3sRoute, ["script"], traceScript(pool, options)); + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); + const result = await capture(config, runtimeTarget.route, ["script"], traceScript(pool, options, runtimeTarget)); const parsed = parseJsonOutput(result.stdout); const ok = result.exitCode === 0 && boolField(parsed, "ok", false); if (options.raw) { @@ -707,7 +830,8 @@ async function codexPoolTrace(config: UniDeskConfig, options: TraceOptions): Pro async function codexPoolSentinelReport(config: UniDeskConfig, options: SentinelReportOptions): Promise | RenderedCliResult> { const pool = readCodexPoolConfig(); - const result = await capture(config, g14K3sRoute, ["script"], sentinelReportScript(pool, options.events)); + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); + const result = await capture(config, runtimeTarget.route, ["script"], sentinelReportScript(pool, options.events, runtimeTarget)); const parsed = parseJsonOutput(result.stdout); const ok = result.exitCode === 0 && boolField(parsed, "ok", false); if (options.raw) { @@ -729,6 +853,7 @@ async function codexPoolSentinelReport(config: UniDeskConfig, options: SentinelR async function codexPoolSentinelProbe(config: UniDeskConfig, options: SentinelProbeOptions): Promise> { const pool = readCodexPoolConfig(); + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); const configuredAccounts = desiredAccountNames(pool); const missing = options.accounts.filter((account) => !configuredAccounts.includes(account)); if (missing.length > 0) { @@ -746,11 +871,11 @@ async function codexPoolSentinelProbe(config: UniDeskConfig, options: SentinelPr ok: true, action: "platform-infra-sub2api-codex-pool-sentinel-probe", mode: "dry-run", - target: poolTarget(pool), + target: poolTarget(pool, runtimeTarget), accounts: options.accounts, effect: "Would create one Kubernetes Job from the managed sentinel CronJob and force an immediate marker probe for the requested account(s).", next: { - confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account ${options.accounts.join(",")} --confirm`, + confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe${targetFlag(runtimeTarget)} --account ${options.accounts.join(",")} --confirm`, }, valuesPrinted: false, }; @@ -758,7 +883,7 @@ async function codexPoolSentinelProbe(config: UniDeskConfig, options: SentinelPr const payload = { accounts: options.accounts, }; - const result = await runRemoteCodexPoolScript(config, "sentinel-probe", sentinelProbeScript(payload, pool)); + const result = await runRemoteCodexPoolScript(config, "sentinel-probe", sentinelProbeScript(payload, pool, runtimeTarget), runtimeTarget); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { @@ -777,19 +902,20 @@ async function codexPoolSentinelProbe(config: UniDeskConfig, options: SentinelPr } async function codexPoolCleanupProbes(config: UniDeskConfig, options: ConfirmOptions): Promise> { + const runtimeTarget = codexPoolRuntimeTarget(options.targetId); if (!options.confirm) { return { ok: true, action: "platform-infra-sub2api-codex-pool-cleanup-probes", mode: "dry-run", - target: poolTarget(), + target: poolTarget(readCodexPoolConfig(), runtimeTarget), scope: "Only deletes temporary resources whose names start with unidesk-probe-.", - next: { confirm: "bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes --confirm" }, + next: { confirm: `bun scripts/cli.ts platform-infra sub2api codex-pool cleanup-probes${targetFlag(runtimeTarget)} --confirm` }, valuesPrinted: false, }; } const pool = readCodexPoolConfig(); - const result = await capture(config, g14K3sRoute, ["script"], cleanupProbesScript(pool)); + const result = await capture(config, runtimeTarget.route, ["script"], cleanupProbesScript(pool, runtimeTarget)); const parsed = parseJsonOutput(result.stdout); if (options.raw) { return { @@ -2485,16 +2611,17 @@ function codexPoolSyncSummary(parsed: Record | null): Record { +function poolTarget(pool = readCodexPoolConfig(), target = codexPoolRuntimeTarget(defaultTargetId)): Record { return { - route: g14K3sRoute, - namespace, + id: target.id, + route: target.route, + namespace: target.namespace, service: serviceName, - serviceDns, + serviceDns: target.serviceDns, configPath: codexPoolConfigPath, groupName: pool.groupName, apiKeyName: pool.apiKeyName, - apiKeySecret: `${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`, + apiKeySecret: `${target.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey}`, minOwnerConcurrency: pool.minOwnerConcurrency, minOwnerConcurrencySource: pool.minOwnerConcurrencySource, accountCapacityTotal: desiredAccountCapacityTotal(pool), @@ -2507,6 +2634,12 @@ function poolTarget(pool = readCodexPoolConfig()): Record { cronJobName: pool.sentinel.cronJobName, stateConfigMapName: pool.sentinel.stateConfigMapName, }, + egressProxy: target.egressProxy === null ? null : { + enabled: target.egressProxy.enabled, + applyToSentinel: target.egressProxy.applyToSentinel, + httpProxy: target.egressProxy.httpProxy, + noProxy: target.egressProxy.noProxy, + }, valuesPrinted: false, }; } @@ -2902,10 +3035,10 @@ spec: `; } -async function fetchPoolApiKey(config: UniDeskConfig, pool: CodexPoolConfig): Promise<{ apiKey: string | null; error: string | null }> { - const result = await capture(config, g14K3sRoute, ["script"], ` +async function fetchPoolApiKey(config: UniDeskConfig, pool: CodexPoolConfig, target = codexPoolRuntimeTarget(defaultTargetId)): Promise<{ apiKey: string | null; error: string | null }> { + const result = await capture(config, target.route, ["script"], ` set -u -kubectl -n ${namespace} get secret ${pool.apiKeySecretName} -o json +kubectl -n ${target.namespace} get secret ${pool.apiKeySecretName} -o json `); if (result.exitCode !== 0) { return { apiKey: null, error: `read pool API key secret failed: ${result.stderr.slice(-1000)}` }; @@ -2913,7 +3046,7 @@ kubectl -n ${namespace} get secret ${pool.apiKeySecretName} -o json const parsed = parseJsonOutput(result.stdout); const data = isRecord(parsed?.data) ? parsed.data : null; const encoded = typeof data?.[pool.apiKeySecretKey] === "string" ? data[pool.apiKeySecretKey] : null; - if (encoded === null) return { apiKey: null, error: `${namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey} missing` }; + if (encoded === null) return { apiKey: null, error: `${target.namespace}/${pool.apiKeySecretName}.${pool.apiKeySecretKey} missing` }; try { const apiKey = Buffer.from(encoded, "base64").toString("utf8"); return apiKey.length > 0 ? { apiKey, error: null } : { apiKey: null, error: "decoded API key is empty" }; @@ -3208,16 +3341,16 @@ export function codexPoolSentinelProbeConfigFingerprint(input: { })); } -function syncScript(payload: unknown, pool: CodexPoolConfig): string { +function syncScript(payload: unknown, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); - return remotePythonScript("sync", encoded, pool); + return remotePythonScript("sync", encoded, pool, target); } -function validateScript(pool: CodexPoolConfig): string { - return remotePythonScript("validate", "", pool); +function validateScript(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { + return remotePythonScript("validate", "", pool, target); } -function traceScript(pool: CodexPoolConfig, options: TraceOptions): string { +function traceScript(pool: CodexPoolConfig, options: TraceOptions, target: CodexPoolRuntimeTarget): string { const encoded = Buffer.from(JSON.stringify({ requestId: options.requestId, since: options.since, @@ -3225,15 +3358,15 @@ function traceScript(pool: CodexPoolConfig, options: TraceOptions): string { contextSeconds: options.contextSeconds, showLines: options.showLines, }), "utf8").toString("base64"); - return remotePythonScript("trace", encoded, pool); + return remotePythonScript("trace", encoded, pool, target); } -function sentinelProbeScript(payload: unknown, pool: CodexPoolConfig): string { +function sentinelProbeScript(payload: unknown, pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { const encoded = Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); - return remotePythonScript("sentinel-probe", encoded, pool); + return remotePythonScript("sentinel-probe", encoded, pool, target); } -function sentinelReportScript(pool: CodexPoolConfig, events: number): string { +function sentinelReportScript(pool: CodexPoolConfig, events: number, target: CodexPoolRuntimeTarget): string { const stateName = pool.sentinel.stateConfigMapName; const cronJobName = pool.sentinel.cronJobName; return ` @@ -3243,7 +3376,7 @@ import json import subprocess from datetime import datetime, timezone, timedelta -NAMESPACE = ${JSON.stringify(namespace)} +NAMESPACE = ${JSON.stringify(target.namespace)} STATE_NAME = ${JSON.stringify(stateName)} CRONJOB_NAME = ${JSON.stringify(cronJobName)} EVENT_LIMIT = ${JSON.stringify(events)} @@ -3445,22 +3578,22 @@ PY `; } -function cleanupProbesScript(pool: CodexPoolConfig): string { - return remotePythonScript("cleanup-probes", "", pool); +function cleanupProbesScript(pool: CodexPoolConfig, target: CodexPoolRuntimeTarget): string { + return remotePythonScript("cleanup-probes", "", pool, target); } -function sentinelImageStatusScript(pool: CodexPoolConfig): string { +function sentinelImageStatusScript(pool: CodexPoolConfig, targetRuntime: CodexPoolRuntimeTarget): string { const target = codexPoolSentinelRuntimeImage(pool.sentinel); - return remoteSentinelImageScript("status", target, pool.sentinel, null); + return remoteSentinelImageScript("status", target, pool.sentinel, null, targetRuntime); } -function sentinelImageBuildScript(pool: CodexPoolConfig): string { +function sentinelImageBuildScript(pool: CodexPoolConfig, targetRuntime: CodexPoolRuntimeTarget): string { const target = codexPoolSentinelRuntimeImage(pool.sentinel); const dockerfile = readFileSync(sentinelImageDockerfilePath, "utf8"); - return remoteSentinelImageScript("build", target, pool.sentinel, dockerfile); + return remoteSentinelImageScript("build", target, pool.sentinel, dockerfile, targetRuntime); } -function remoteSentinelImageScript(mode: "status" | "build", target: ReturnType, sentinel: CodexPoolSentinelConfig, dockerfile: string | null): string { +function remoteSentinelImageScript(mode: "status" | "build", target: ReturnType, sentinel: CodexPoolSentinelConfig, dockerfile: string | null, targetRuntime: CodexPoolRuntimeTarget): string { const dockerfileB64 = dockerfile === null ? "" : Buffer.from(dockerfile, "utf8").toString("base64"); return ` set -eu @@ -3470,6 +3603,7 @@ repo=${shQuote("platform-infra/sub2api-account-sentinel")} tag=${shQuote(target.tag)} base_image=${shQuote(target.baseImage)} openai_version=${shQuote(sentinel.sdk.openaiPythonVersion)} +runtime_target=${shQuote(targetRuntime.id)} work=/tmp/unidesk-sub2api-sentinel-image mkdir -p "$work" dockerfile_path="$work/sentinel.Dockerfile" @@ -3527,7 +3661,13 @@ ${dockerfileB64} UNIDESK_SENTINEL_DOCKERFILE_B64 export NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal,74.48.78.17,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12,10.42.0.0/16,10.43.0.0/16,.svc,.svc.cluster.local,.cluster.local,kubernetes,kubernetes.default,kubernetes.default.svc,127.0.0.1:5000,localhost:5000 export no_proxy=$NO_PROXY -docker build --pull \\ +set -- --pull +base_image_source="registry" +if [ "$runtime_target" != "G14" ] && docker image inspect "$base_image" >/dev/null 2>&1; then + set -- + base_image_source="local-cache" +fi +docker build "$@" \\ --build-arg BASE_IMAGE="$base_image" \\ --build-arg OPENAI_PYTHON_VERSION="$openai_version" \\ --build-arg HTTP_PROXY= --build-arg HTTPS_PROXY= --build-arg http_proxy= --build-arg https_proxy= \\ @@ -3546,6 +3686,7 @@ print(json.dumps({ "image": "${target.runtimeImage}", "baseImage": "${target.baseImage}", "tag": "${target.tag}", + "baseImageSource": "${"${base_image_source}"}", "digest": "${"${digest}"}" or None, }, ensure_ascii=False, indent=2)) PY @@ -3620,7 +3761,7 @@ function desiredAccountTempUnschedulableMap(pool: CodexPoolConfig): Record { +async function runRemoteCodexPoolScript(config: UniDeskConfig, mode: RemoteCodexPoolMode, script: string, target = codexPoolRuntimeTarget(defaultTargetId)): Promise { const jobName = `codex-pool-${mode}-${Date.now().toString(36)}`.slice(0, 63); const startedAtMs = Date.now(); - const start = await capture(config, g14K3sRoute, ["script"], remoteJobStartScript(jobName, script)); + const start = await capture(config, target.route, ["script"], remoteJobStartScript(jobName, script)); const started = parseJsonOutput(start.stdout); if (start.exitCode !== 0 || boolField(started, "ok", false) !== true) return start; let latest: RemoteCodexPoolJobStatus | null = null; while (Date.now() - startedAtMs <= remoteJobTimeoutMs) { await sleep(remoteJobPollMs); - const probe = await capture(config, g14K3sRoute, ["script"], remoteJobStatusScript(jobName)); + const probe = await capture(config, target.route, ["script"], remoteJobStatusScript(jobName)); const parsed = parseJsonOutput(probe.stdout); latest = normalizeRemoteJobStatus(parsed); process.stderr.write(`${JSON.stringify({ diff --git a/scripts/src/platform-infra.ts b/scripts/src/platform-infra.ts index 31ef3f5b..b894dc70 100644 --- a/scripts/src/platform-infra.ts +++ b/scripts/src/platform-infra.ts @@ -1,5 +1,6 @@ -import { createHash } from "node:crypto"; -import { readFileSync } from "node:fs"; +import { createHash, randomBytes } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join } from "node:path"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; import { startJob } from "./jobs"; @@ -12,10 +13,11 @@ const serviceName = "sub2api"; const fieldManager = "unidesk-platform-infra"; const manifestPath = rootPath("src", "components", "platform-infra", "sub2api", "sub2api.k8s.yaml"); const configPath = rootPath("config", "platform-infra", "sub2api.yaml"); +const repoRoot = rootPath(); const secretName = "sub2api-secrets"; const requiredSecretKeys = ["POSTGRES_PASSWORD", "ADMIN_PASSWORD", "JWT_SECRET", "TOTP_ENCRYPTION_KEY"] as const; -type DatabaseMode = "bundled" | "external-pending"; +type DatabaseMode = "bundled" | "external-pending" | "external-active"; type RedisMode = "bundled-persistent" | "local-ephemeral"; interface Sub2ApiConfig { @@ -24,6 +26,10 @@ interface Sub2ApiConfig { tag: string; pullPolicy: "Always" | "IfNotPresent" | "Never"; }; + dependencyImages: { + postgres: string; + redis: string; + }; security: { urlAllowlist: { enabled: boolean; @@ -35,6 +41,7 @@ interface Sub2ApiConfig { targets: Sub2ApiTargetConfig[]; runtime: { database: ExternalDatabaseConfig; + secrets: RuntimeSecretsConfig; redis: RuntimeRedisConfig; appData: { mode: "persistent-pvc" | "empty-dir"; @@ -56,11 +63,78 @@ interface Sub2ApiTargetConfig { redisMode: RedisMode; appReplicas: number; redisReplicas: number; + image: Partial; + dependencyImages: Partial; + publicExposure: Sub2ApiPublicExposureConfig | null; + egressProxy: Sub2ApiEgressProxyConfig | null; +} + +interface Sub2ApiPublicExposureConfig { + enabled: boolean; + publicBaseUrl: string; + dns: { + hostname: string; + expectedA: string; + resolvers: string[]; + }; + frpc: { + deploymentName: string; + secretName: string; + secretKey: string; + image: string; + serverAddr: string; + serverPort: number; + proxyName: string; + remotePort: number; + localIP: string; + localPort: number; + tokenSourceRef: string; + tokenSourceKey: string; + }; + pk01: { + route: string; + caddyBinaryPath: string; + caddyDownloadUrl: string; + caddyDownloadProxyUrl: string | null; + caddyConfigPath: string; + caddyServiceName: string; + caddyStorageDir: string; + caddyEmail: string; + pikanodeRoot: string; + pikanodeContainerName: string; + pikanodeImage: string; + pikanodeHttpHostPort: number; + responseHeaderTimeoutSeconds: number; + }; +} + +interface Sub2ApiEgressProxyConfig { + enabled: boolean; + deploymentName: string; + serviceName: string; + secretName: string; + secretKey: string; + image: string; + imagePullPolicy: "Always" | "IfNotPresent" | "Never"; + listenPort: number; + sourceRef: string; + sourceKey: string; + sourceType: "subscription-url"; + preferredOutbound: "vless-reality" | "hysteria2"; + noProxy: string[]; + applyToSub2Api: boolean; + applyToSentinel: boolean; + healthProbeUrl: string; } interface ExternalDatabaseConfig { mode: "external"; sourceRef: string; + sourceKeys: { + user: string; + password: string; + dbName: string; + }; secretName: string; passwordKey: string; host: string; @@ -71,11 +145,50 @@ interface ExternalDatabaseConfig { pendingAllowed: boolean; } +interface RuntimeSecretsConfig { + root: string; + appSourceRef: string; +} + interface RuntimeRedisConfig { serviceName: string; persistence: boolean; } +interface ExternalActiveSecretMaterial { + sourceRef: string; + sourcePath: string; + appSourceRef: string; + appSourcePath: string; + action: "create" | "update" | "none"; + fingerprint: string; + values: Record; +} + +interface PublicExposureSecretMaterial { + sourceRef: string; + sourcePath: string; + secretName: string; + secretKey: string; + fingerprint: string; + frpcToml: string; + valuesPrinted: false; +} + +interface EgressProxySecretMaterial { + sourceRef: string; + sourcePath: string; + secretName: string; + secretKey: string; + fingerprint: string; + subscriptionBytes: number; + selectedOutbound: string; + configJson: string; + proxyUrl: string; + noProxy: string; + valuesPrinted: false; +} + export function platformInfraHelp(): unknown { return { command: "platform-infra sub2api plan|apply|status|validate|codex-pool", @@ -93,7 +206,7 @@ export function platformInfraHelp(): unknown { "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-image status", "bun scripts/cli.ts platform-infra sub2api codex-pool sentinel-probe --account unidesk-codex-hy --confirm", ], - description: "Operate YAML-controlled Sub2API platform-infra targets. G14 remains the active bundled deployment; D601 is a standby predeployment target with external DB pending and no local PostgreSQL.", + description: "Operate YAML-controlled Sub2API platform-infra targets. G14 remains the active bundled deployment; D601 can be held at external-pending predeployment or activated against the external PK01 PostgreSQL runtime.", target: { default: defaultTargetId, namespace, @@ -222,8 +335,9 @@ function readSub2ApiConfig(): Sub2ApiConfig { const tag = stringField(record, "tag", "image"); const pullPolicy = stringField(record, "pullPolicy", "image"); if (pullPolicy !== "Always" && pullPolicy !== "IfNotPresent" && pullPolicy !== "Never") throw new Error(`${configPath}.image.pullPolicy must be Always, IfNotPresent, or Never`); - if (!/^[a-z0-9._/-]+(?::[0-9]+)?$/u.test(repository)) throw new Error(`${configPath}.image.repository has an unsupported format`); + if (!isImageRepository(repository)) throw new Error(`${configPath}.image.repository has an unsupported format`); if (!/^[A-Za-z0-9._-]+$/u.test(tag)) throw new Error(`${configPath}.image.tag has an unsupported format`); + const dependencyImages = parseDependencyImages(root); const security = objectField(parsed as Record, "security", ""); const urlAllowlist = objectField(security, "urlAllowlist", "security"); const enabled = booleanField(urlAllowlist, "enabled", "security.urlAllowlist"); @@ -234,6 +348,7 @@ function readSub2ApiConfig(): Sub2ApiConfig { const runtime = parseRuntime(root); return { image: { repository, tag, pullPolicy }, + dependencyImages, security: { urlAllowlist: { enabled, @@ -247,6 +362,18 @@ function readSub2ApiConfig(): Sub2ApiConfig { }; } +function parseDependencyImages(root: Record): Sub2ApiConfig["dependencyImages"] { + const value = root.dependencyImages; + if (value === undefined) return { postgres: "postgres:18-alpine", redis: "redis:8-alpine" }; + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${configPath}.dependencyImages must be an object`); + const record = value as Record; + const postgres = stringField(record, "postgres", "dependencyImages"); + const redis = stringField(record, "redis", "dependencyImages"); + if (!isImageReference(postgres)) throw new Error(`${configPath}.dependencyImages.postgres has an unsupported format`); + if (!isImageReference(redis)) throw new Error(`${configPath}.dependencyImages.redis has an unsupported format`); + return { postgres, redis }; +} + function parseTargets(root: Record): Sub2ApiTargetConfig[] { const value = root.targets; if (value === undefined) return defaultTargets(); @@ -260,18 +387,22 @@ function parseTargets(root: Record): Sub2ApiTargetConfig[] { const targetNamespace = stringField(record, "namespace", path); const role = stringField(record, "role", path); const enabled = booleanField(record, "enabled", path); - const databaseMode = enumField(record, "databaseMode", path, ["bundled", "external-pending"] as const); + const databaseMode = enumField(record, "databaseMode", path, ["bundled", "external-pending", "external-active"] as const); const redisMode = enumField(record, "redisMode", path, ["bundled-persistent", "local-ephemeral"] as const); const appReplicas = integerField(record, "appReplicas", path); const redisReplicas = record.redisReplicas === undefined ? (databaseMode === "external-pending" && appReplicas === 0 ? 0 : 1) : integerField(record, "redisReplicas", path); + const image = targetImageOverride(record, path); + const dependencyImages = targetDependencyImageOverride(record, path); + const publicExposure = parsePublicExposureConfig(record.publicExposure, path); + const egressProxy = parseEgressProxyConfig(record.egressProxy, path); if (!/^[A-Za-z0-9._-]+$/u.test(id)) throw new Error(`${configPath}.${path}.id must be a simple target id`); if (!/^[A-Za-z0-9:_./-]+$/u.test(route)) throw new Error(`${configPath}.${path}.route has an unsupported format`); if (!isKubernetesName(targetNamespace)) throw new Error(`${configPath}.${path}.namespace must be a Kubernetes namespace name`); if (appReplicas < 0 || appReplicas > 1) throw new Error(`${configPath}.${path}.appReplicas must be 0 or 1`); if (redisReplicas < 0 || redisReplicas > 1) throw new Error(`${configPath}.${path}.redisReplicas must be 0 or 1`); - return { id, route, namespace: targetNamespace, role, enabled, databaseMode, redisMode, appReplicas, redisReplicas }; + return { id, route, namespace: targetNamespace, role, enabled, databaseMode, redisMode, appReplicas, redisReplicas, image, dependencyImages, publicExposure, egressProxy }; }); const ids = new Set(); for (const target of targets) { @@ -294,10 +425,154 @@ function defaultTargets(): Sub2ApiTargetConfig[] { redisMode: "bundled-persistent", appReplicas: 1, redisReplicas: 1, + image: {}, + dependencyImages: {}, + publicExposure: null, + egressProxy: null, }, ]; } +function targetImageOverride(record: Record, path: string): Partial { + if (record.image === undefined) return {}; + const image = objectField(record, "image", path); + const override: Partial = {}; + if (image.repository !== undefined) { + const repository = stringField(image, "repository", `${path}.image`); + if (!isImageRepository(repository)) throw new Error(`${configPath}.${path}.image.repository has an unsupported format`); + override.repository = repository; + } + if (image.tag !== undefined) { + const tag = stringField(image, "tag", `${path}.image`); + if (!/^[A-Za-z0-9._-]+$/u.test(tag)) throw new Error(`${configPath}.${path}.image.tag has an unsupported format`); + override.tag = tag; + } + if (image.pullPolicy !== undefined) { + override.pullPolicy = enumField(image, "pullPolicy", `${path}.image`, ["Always", "IfNotPresent", "Never"] as const); + } + return override; +} + +function targetDependencyImageOverride(record: Record, path: string): Partial { + if (record.dependencyImages === undefined) return {}; + const dependencyImages = objectField(record, "dependencyImages", path); + const override: Partial = {}; + if (dependencyImages.postgres !== undefined) { + const postgres = stringField(dependencyImages, "postgres", `${path}.dependencyImages`); + if (!isImageReference(postgres)) throw new Error(`${configPath}.${path}.dependencyImages.postgres has an unsupported format`); + override.postgres = postgres; + } + if (dependencyImages.redis !== undefined) { + const redis = stringField(dependencyImages, "redis", `${path}.dependencyImages`); + if (!isImageReference(redis)) throw new Error(`${configPath}.${path}.dependencyImages.redis has an unsupported format`); + override.redis = redis; + } + return override; +} + +function parsePublicExposureConfig(value: unknown, path: string): Sub2ApiPublicExposureConfig | null { + if (value === undefined || value === null) return null; + if (typeof value !== "object" || Array.isArray(value)) throw new Error(`${configPath}.${path}.publicExposure must be an object`); + const record = value as Record; + const enabled = booleanField(record, "enabled", `${path}.publicExposure`); + const publicBaseUrl = stringField(record, "publicBaseUrl", `${path}.publicExposure`); + const dnsRaw = objectField(record, "dns", `${path}.publicExposure`); + const frpcRaw = objectField(record, "frpc", `${path}.publicExposure`); + const pk01Raw = objectField(record, "pk01", `${path}.publicExposure`); + const hostname = stringField(dnsRaw, "hostname", `${path}.publicExposure.dns`); + const expectedA = stringField(dnsRaw, "expectedA", `${path}.publicExposure.dns`); + const resolvers = dnsRaw.resolvers === undefined + ? ["1.1.1.1", "8.8.8.8", "223.5.5.5", "114.114.114.114"] + : stringArrayField(dnsRaw, "resolvers", `${path}.publicExposure.dns`); + const exposure: Sub2ApiPublicExposureConfig = { + enabled, + publicBaseUrl: normalizePublicBaseUrl(publicBaseUrl, `${path}.publicExposure.publicBaseUrl`), + dns: { + hostname, + expectedA, + resolvers, + }, + frpc: { + deploymentName: stringField(frpcRaw, "deploymentName", `${path}.publicExposure.frpc`), + secretName: stringField(frpcRaw, "secretName", `${path}.publicExposure.frpc`), + secretKey: stringField(frpcRaw, "secretKey", `${path}.publicExposure.frpc`), + image: stringField(frpcRaw, "image", `${path}.publicExposure.frpc`), + serverAddr: stringField(frpcRaw, "serverAddr", `${path}.publicExposure.frpc`), + serverPort: integerField(frpcRaw, "serverPort", `${path}.publicExposure.frpc`), + proxyName: stringField(frpcRaw, "proxyName", `${path}.publicExposure.frpc`), + remotePort: integerField(frpcRaw, "remotePort", `${path}.publicExposure.frpc`), + localIP: stringField(frpcRaw, "localIP", `${path}.publicExposure.frpc`), + localPort: integerField(frpcRaw, "localPort", `${path}.publicExposure.frpc`), + tokenSourceRef: stringField(frpcRaw, "tokenSourceRef", `${path}.publicExposure.frpc`), + tokenSourceKey: stringField(frpcRaw, "tokenSourceKey", `${path}.publicExposure.frpc`), + }, + pk01: { + route: stringField(pk01Raw, "route", `${path}.publicExposure.pk01`), + caddyBinaryPath: stringField(pk01Raw, "caddyBinaryPath", `${path}.publicExposure.pk01`), + caddyDownloadUrl: stringField(pk01Raw, "caddyDownloadUrl", `${path}.publicExposure.pk01`), + caddyDownloadProxyUrl: pk01Raw.caddyDownloadProxyUrl === undefined ? null : stringField(pk01Raw, "caddyDownloadProxyUrl", `${path}.publicExposure.pk01`), + caddyConfigPath: stringField(pk01Raw, "caddyConfigPath", `${path}.publicExposure.pk01`), + caddyServiceName: stringField(pk01Raw, "caddyServiceName", `${path}.publicExposure.pk01`), + caddyStorageDir: stringField(pk01Raw, "caddyStorageDir", `${path}.publicExposure.pk01`), + caddyEmail: stringField(pk01Raw, "caddyEmail", `${path}.publicExposure.pk01`), + pikanodeRoot: stringField(pk01Raw, "pikanodeRoot", `${path}.publicExposure.pk01`), + pikanodeContainerName: stringField(pk01Raw, "pikanodeContainerName", `${path}.publicExposure.pk01`), + pikanodeImage: stringField(pk01Raw, "pikanodeImage", `${path}.publicExposure.pk01`), + pikanodeHttpHostPort: integerField(pk01Raw, "pikanodeHttpHostPort", `${path}.publicExposure.pk01`), + responseHeaderTimeoutSeconds: integerField(pk01Raw, "responseHeaderTimeoutSeconds", `${path}.publicExposure.pk01`), + }, + }; + validatePublicExposureConfig(exposure, path); + return exposure; +} + +function parseEgressProxyConfig(value: unknown, path: string): Sub2ApiEgressProxyConfig | null { + if (value === undefined || value === null) return null; + if (typeof value !== "object" || Array.isArray(value)) throw new Error(`${configPath}.${path}.egressProxy must be an object`); + const record = value as Record; + const sourceType = enumField(record, "sourceType", `${path}.egressProxy`, ["subscription-url"] as const); + const preferredOutbound = enumField(record, "preferredOutbound", `${path}.egressProxy`, ["vless-reality", "hysteria2"] as const); + const imagePullPolicy = record.imagePullPolicy === undefined + ? "IfNotPresent" + : enumField(record, "imagePullPolicy", `${path}.egressProxy`, ["Always", "IfNotPresent", "Never"] as const); + const noProxy = record.noProxy === undefined + ? [ + "localhost", + "127.0.0.1", + "::1", + ".svc", + ".cluster.local", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "82.156.23.220", + "74.48.78.17", + "hyueapi.com", + ".hyueapi.com", + ] + : stringArrayField(record, "noProxy", `${path}.egressProxy`); + const proxy: Sub2ApiEgressProxyConfig = { + enabled: booleanField(record, "enabled", `${path}.egressProxy`), + deploymentName: stringField(record, "deploymentName", `${path}.egressProxy`), + serviceName: stringField(record, "serviceName", `${path}.egressProxy`), + secretName: stringField(record, "secretName", `${path}.egressProxy`), + secretKey: stringField(record, "secretKey", `${path}.egressProxy`), + image: stringField(record, "image", `${path}.egressProxy`), + imagePullPolicy, + listenPort: integerField(record, "listenPort", `${path}.egressProxy`), + sourceRef: stringField(record, "sourceRef", `${path}.egressProxy`), + sourceKey: stringField(record, "sourceKey", `${path}.egressProxy`), + sourceType, + preferredOutbound, + noProxy, + applyToSub2Api: record.applyToSub2Api === undefined ? true : booleanField(record, "applyToSub2Api", `${path}.egressProxy`), + applyToSentinel: record.applyToSentinel === undefined ? true : booleanField(record, "applyToSentinel", `${path}.egressProxy`), + healthProbeUrl: record.healthProbeUrl === undefined ? "https://www.gstatic.com/generate_204" : stringField(record, "healthProbeUrl", `${path}.egressProxy`), + }; + validateEgressProxyConfig(proxy, path); + return proxy; +} + function parseRuntime(root: Record): Sub2ApiConfig["runtime"] { const value = root.runtime; if (value === undefined) { @@ -305,6 +580,11 @@ function parseRuntime(root: Record): Sub2ApiConfig["runtime"] { database: { mode: "external", sourceRef: "platform-db/postgres-active.env", + sourceKeys: { + user: "SUB2API_DB_USER", + password: "SUB2API_DB_PASSWORD", + dbName: "SUB2API_DB_NAME", + }, secretName, passwordKey: "POSTGRES_PASSWORD", host: "pika01-postgres.pending.local", @@ -314,6 +594,10 @@ function parseRuntime(root: Record): Sub2ApiConfig["runtime"] { sslMode: "prefer", pendingAllowed: true, }, + secrets: { + root: ".state/secrets", + appSourceRef: "platform-infra/sub2api.env", + }, redis: { serviceName: "sub2api-redis", persistence: false }, appData: { mode: "persistent-pvc" }, sentinel: { mode: "singleton", enabledOnTargets: ["G14"] }, @@ -324,6 +608,12 @@ function parseRuntime(root: Record): Sub2ApiConfig["runtime"] { const database = objectField(runtime, "database", "runtime"); const databaseMode = enumField(database, "mode", "runtime.database", ["external"] as const); const sourceRef = stringField(database, "sourceRef", "runtime.database"); + const sourceKeysRaw = database.sourceKeys === undefined ? {} : objectField(database, "sourceKeys", "runtime.database"); + const sourceKeys = { + user: sourceKeysRaw.user === undefined ? "SUB2API_DB_USER" : stringField(sourceKeysRaw, "user", "runtime.database.sourceKeys"), + password: sourceKeysRaw.password === undefined ? "SUB2API_DB_PASSWORD" : stringField(sourceKeysRaw, "password", "runtime.database.sourceKeys"), + dbName: sourceKeysRaw.dbName === undefined ? "SUB2API_DB_NAME" : stringField(sourceKeysRaw, "dbName", "runtime.database.sourceKeys"), + }; const secret = stringField(database, "secretName", "runtime.database"); const passwordKey = stringField(database, "passwordKey", "runtime.database"); const host = stringField(database, "host", "runtime.database"); @@ -333,6 +623,10 @@ function parseRuntime(root: Record): Sub2ApiConfig["runtime"] { const sslMode = stringField(database, "sslMode", "runtime.database"); const pendingAllowed = booleanField(database, "pendingAllowed", "runtime.database"); if (!isKubernetesName(secret)) throw new Error(`${configPath}.runtime.database.secretName must be a Kubernetes Secret name`); + if (!/^[A-Za-z0-9_./-]+$/u.test(sourceRef)) throw new Error(`${configPath}.runtime.database.sourceRef has an unsupported format`); + for (const [key, value] of Object.entries(sourceKeys)) { + if (!/^[A-Z0-9_]+$/u.test(value)) throw new Error(`${configPath}.runtime.database.sourceKeys.${key} must be an env key`); + } if (!/^[A-Z0-9_]+$/u.test(passwordKey)) throw new Error(`${configPath}.runtime.database.passwordKey must be an env key`); if (!/^[A-Za-z0-9._-]+$/u.test(host)) throw new Error(`${configPath}.runtime.database.host has an unsupported format`); if (!/^[A-Za-z0-9_][-A-Za-z0-9_]*$/u.test(user)) throw new Error(`${configPath}.runtime.database.user has an unsupported format`); @@ -344,6 +638,11 @@ function parseRuntime(root: Record): Sub2ApiConfig["runtime"] { const redisServiceName = stringField(redis, "serviceName", "runtime.redis"); const redisPersistence = booleanField(redis, "persistence", "runtime.redis"); if (!isKubernetesName(redisServiceName)) throw new Error(`${configPath}.runtime.redis.serviceName must be a Kubernetes Service name`); + const secretsRaw = runtime.secrets === undefined ? {} : objectField(runtime, "secrets", "runtime"); + const secretRoot = secretsRaw.root === undefined ? ".state/secrets" : stringField(secretsRaw, "root", "runtime.secrets"); + const appSourceRef = secretsRaw.appSourceRef === undefined ? "platform-infra/sub2api.env" : stringField(secretsRaw, "appSourceRef", "runtime.secrets"); + if (!/^[A-Za-z0-9_./-]+$/u.test(secretRoot)) throw new Error(`${configPath}.runtime.secrets.root has an unsupported format`); + if (!/^[A-Za-z0-9_./-]+$/u.test(appSourceRef)) throw new Error(`${configPath}.runtime.secrets.appSourceRef has an unsupported format`); const appData = objectField(runtime, "appData", "runtime"); const appDataMode = enumField(appData, "mode", "runtime.appData", ["persistent-pvc", "empty-dir"] as const); const sentinel = objectField(runtime, "sentinel", "runtime"); @@ -351,7 +650,8 @@ function parseRuntime(root: Record): Sub2ApiConfig["runtime"] { const enabledOnTargets = stringArrayField(sentinel, "enabledOnTargets", "runtime.sentinel"); if (sentinelMode !== "singleton") throw new Error(`${configPath}.runtime.sentinel.mode must be singleton`); return { - database: { mode: "external", sourceRef, secretName: secret, passwordKey, host, port, user, dbName, sslMode, pendingAllowed }, + database: { mode: "external", sourceRef, sourceKeys, secretName: secret, passwordKey, host, port, user, dbName, sslMode, pendingAllowed }, + secrets: { root: secretRoot, appSourceRef }, redis: { serviceName: redisServiceName, persistence: redisPersistence }, appData: { mode: appDataMode }, sentinel: { mode: "singleton", enabledOnTargets }, @@ -401,6 +701,109 @@ function isKubernetesName(value: string): boolean { return /^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value); } +function isImageRepository(value: string): boolean { + return /^[A-Za-z0-9._:/-]+$/u.test(value) && !value.includes("..") && !value.includes("@") && !value.endsWith(":"); +} + +function isImageReference(value: string): boolean { + return /^[A-Za-z0-9._:/-]+$/u.test(value) && value.includes(":") && !value.includes(".."); +} + +function validatePublicExposureConfig(config: Sub2ApiPublicExposureConfig, path: string): void { + const url = new URL(config.publicBaseUrl); + if (url.protocol !== "https:") throw new Error(`${configPath}.${path}.publicExposure.publicBaseUrl must use https://`); + if (url.hostname !== config.dns.hostname) throw new Error(`${configPath}.${path}.publicExposure.publicBaseUrl hostname must match publicExposure.dns.hostname`); + validateHostname(config.dns.hostname, `${path}.publicExposure.dns.hostname`); + if (!/^[0-9.]+$/u.test(config.dns.expectedA)) throw new Error(`${configPath}.${path}.publicExposure.dns.expectedA must be an IPv4 address`); + for (const resolver of config.dns.resolvers) { + if (!/^[0-9.]+$/u.test(resolver)) throw new Error(`${configPath}.${path}.publicExposure.dns.resolvers entries must be IPv4 resolvers`); + } + validateKubernetesResourceName(config.frpc.deploymentName, `${path}.publicExposure.frpc.deploymentName`); + validateKubernetesResourceName(config.frpc.secretName, `${path}.publicExposure.frpc.secretName`); + if (config.frpc.secretKey !== "frpc.toml") throw new Error(`${configPath}.${path}.publicExposure.frpc.secretKey must be frpc.toml`); + if (!isImageReference(config.frpc.image)) throw new Error(`${configPath}.${path}.publicExposure.frpc.image has an unsupported format`); + validateHostOrIp(config.frpc.serverAddr, `${path}.publicExposure.frpc.serverAddr`); + validatePort(config.frpc.serverPort, `${path}.publicExposure.frpc.serverPort`); + validateProxyName(config.frpc.proxyName, `${path}.publicExposure.frpc.proxyName`); + validatePort(config.frpc.remotePort, `${path}.publicExposure.frpc.remotePort`); + validateHostOrIp(config.frpc.localIP, `${path}.publicExposure.frpc.localIP`); + validatePort(config.frpc.localPort, `${path}.publicExposure.frpc.localPort`); + if (!/^[A-Za-z0-9_./-]+$/u.test(config.frpc.tokenSourceRef)) throw new Error(`${configPath}.${path}.publicExposure.frpc.tokenSourceRef has an unsupported format`); + if (!/^[A-Z0-9_]+$/u.test(config.frpc.tokenSourceKey)) throw new Error(`${configPath}.${path}.publicExposure.frpc.tokenSourceKey must be an env key`); + if (!/^[A-Za-z0-9:_./-]+$/u.test(config.pk01.route)) throw new Error(`${configPath}.${path}.publicExposure.pk01.route has an unsupported format`); + if (!config.pk01.caddyBinaryPath.startsWith("/")) throw new Error(`${configPath}.${path}.publicExposure.pk01.caddyBinaryPath must be absolute`); + if (!/^https:\/\//u.test(config.pk01.caddyDownloadUrl)) throw new Error(`${configPath}.${path}.publicExposure.pk01.caddyDownloadUrl must use https://`); + if (config.pk01.caddyDownloadProxyUrl !== null) validateProxyUrl(config.pk01.caddyDownloadProxyUrl, `${path}.publicExposure.pk01.caddyDownloadProxyUrl`); + if (!config.pk01.caddyConfigPath.startsWith("/")) throw new Error(`${configPath}.${path}.publicExposure.pk01.caddyConfigPath must be absolute`); + validateProxyName(config.pk01.caddyServiceName, `${path}.publicExposure.pk01.caddyServiceName`); + if (!config.pk01.caddyStorageDir.startsWith("/")) throw new Error(`${configPath}.${path}.publicExposure.pk01.caddyStorageDir must be absolute`); + if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/u.test(config.pk01.caddyEmail)) throw new Error(`${configPath}.${path}.publicExposure.pk01.caddyEmail must be an email address`); + if (!config.pk01.pikanodeRoot.startsWith("/")) throw new Error(`${configPath}.${path}.publicExposure.pk01.pikanodeRoot must be absolute`); + validateProxyName(config.pk01.pikanodeContainerName, `${path}.publicExposure.pk01.pikanodeContainerName`); + if (!isImageRepository(config.pk01.pikanodeImage) && !isImageReference(config.pk01.pikanodeImage)) throw new Error(`${configPath}.${path}.publicExposure.pk01.pikanodeImage has an unsupported format`); + validatePort(config.pk01.pikanodeHttpHostPort, `${path}.publicExposure.pk01.pikanodeHttpHostPort`); + if (config.pk01.responseHeaderTimeoutSeconds < 1 || config.pk01.responseHeaderTimeoutSeconds > 3600) throw new Error(`${configPath}.${path}.publicExposure.pk01.responseHeaderTimeoutSeconds must be 1..3600`); +} + +function validateEgressProxyConfig(config: Sub2ApiEgressProxyConfig, path: string): void { + validateKubernetesResourceName(config.deploymentName, `${path}.egressProxy.deploymentName`); + validateKubernetesResourceName(config.serviceName, `${path}.egressProxy.serviceName`); + validateKubernetesResourceName(config.secretName, `${path}.egressProxy.secretName`); + if (config.secretKey !== "config.json") throw new Error(`${configPath}.${path}.egressProxy.secretKey must be config.json`); + if (!isImageReference(config.image)) throw new Error(`${configPath}.${path}.egressProxy.image has an unsupported format`); + validatePort(config.listenPort, `${path}.egressProxy.listenPort`); + if (!/^[A-Za-z0-9_./-]+$/u.test(config.sourceRef)) throw new Error(`${configPath}.${path}.egressProxy.sourceRef has an unsupported format`); + if (!/^[A-Z0-9_]+$/u.test(config.sourceKey)) throw new Error(`${configPath}.${path}.egressProxy.sourceKey must be an env key`); + for (const entry of config.noProxy) { + if (!/^[A-Za-z0-9.:_/*-]+$/u.test(entry)) throw new Error(`${configPath}.${path}.egressProxy.noProxy contains an unsupported entry`); + } + const url = new URL(config.healthProbeUrl); + if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`${configPath}.${path}.egressProxy.healthProbeUrl must use http:// or https://`); +} + +function normalizePublicBaseUrl(value: string, path: string): string { + let parsed: URL; + try { + parsed = new URL(value); + } catch { + throw new Error(`${configPath}.${path} must be a valid URL`); + } + parsed.pathname = parsed.pathname.replace(/\/+$/u, ""); + if (parsed.search || parsed.hash) throw new Error(`${configPath}.${path} must not include query or hash`); + return parsed.toString().replace(/\/+$/u, ""); +} + +function validateHostname(value: string, path: string): void { + if (!/^[A-Za-z0-9.-]+$/u.test(value) || !value.includes(".")) throw new Error(`${configPath}.${path} must be a hostname`); +} + +function validateKubernetesResourceName(value: string, path: string): void { + if (!isKubernetesName(value)) throw new Error(`${configPath}.${path} must be a Kubernetes resource name`); +} + +function validateHostOrIp(value: string, path: string): void { + if (!/^[A-Za-z0-9._:-]+$/u.test(value)) throw new Error(`${configPath}.${path} has an unsupported format`); +} + +function validatePort(value: number, path: string): void { + if (value < 1 || value > 65535) throw new Error(`${configPath}.${path} must be a TCP port`); +} + +function validateProxyName(value: string, path: string): void { + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${configPath}.${path} has an unsupported format`); +} + +function validateProxyUrl(value: string, path: string): void { + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error(`${configPath}.${path} must be a valid proxy URL`); + } + if (url.protocol !== "http:" && url.protocol !== "https:") throw new Error(`${configPath}.${path} must use http:// or https://`); + if (url.username || url.password || url.search || url.hash) throw new Error(`${configPath}.${path} must not include credentials, query, or hash`); +} + function resolveTarget(sub2api: Sub2ApiConfig, targetId: string): Sub2ApiTargetConfig { const target = sub2api.targets.find((item) => item.id.toLowerCase() === targetId.toLowerCase()); if (target === undefined) { @@ -411,18 +814,42 @@ function resolveTarget(sub2api: Sub2ApiConfig, targetId: string): Sub2ApiTargetC return target; } -function imageRef(sub2api: Sub2ApiConfig): string { - return `${sub2api.image.repository}:${sub2api.image.tag}`; +function targetImage(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): Sub2ApiConfig["image"] { + return { + repository: target.image.repository ?? sub2api.image.repository, + tag: target.image.tag ?? sub2api.image.tag, + pullPolicy: target.image.pullPolicy ?? sub2api.image.pullPolicy, + }; +} + +function targetDependencyImages(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): Sub2ApiConfig["dependencyImages"] { + return { + postgres: target.dependencyImages.postgres ?? sub2api.dependencyImages.postgres, + redis: target.dependencyImages.redis ?? sub2api.dependencyImages.redis, + }; +} + +function imageRef(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { + const image = targetImage(sub2api, target); + return `${image.repository}:${image.tag}`; +} + +function isExternalTarget(target: Sub2ApiTargetConfig): boolean { + return target.databaseMode === "external-pending" || target.databaseMode === "external-active"; } function manifest(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { - if (target.databaseMode === "external-pending") return externalPendingManifest(sub2api, target); + if (isExternalTarget(target)) return externalPendingManifest(sub2api, target); const template = readFileSync(manifestPath, "utf8"); const urlAllowlist = sub2api.security.urlAllowlist; const configHash = configHashFor(sub2api, target); + const image = targetImage(sub2api, target); + const dependencyImages = targetDependencyImages(sub2api, target); return template - .replaceAll("__SUB2API_IMAGE__", imageRef(sub2api)) - .replaceAll("__SUB2API_IMAGE_PULL_POLICY__", sub2api.image.pullPolicy) + .replaceAll("__SUB2API_IMAGE__", imageRef(sub2api, target)) + .replaceAll("__SUB2API_IMAGE_PULL_POLICY__", image.pullPolicy) + .replaceAll("postgres:18-alpine", dependencyImages.postgres) + .replaceAll("redis:8-alpine", dependencyImages.redis) .replaceAll("__SUB2API_CONFIG_HASH__", configHash) .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_ENABLED__", String(urlAllowlist.enabled)) .replaceAll("__SUB2API_SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP__", String(urlAllowlist.allowInsecureHttp)) @@ -433,7 +860,8 @@ function manifest(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { function configHashFor(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { return createHash("sha256") .update(JSON.stringify({ - image: sub2api.image, + image: targetImage(sub2api, target), + dependencyImages: targetDependencyImages(sub2api, target), security: sub2api.security, target, runtime: sub2api.runtime, @@ -445,9 +873,15 @@ function configHashFor(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): str function externalPendingManifest(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { const urlAllowlist = sub2api.security.urlAllowlist; const database = sub2api.runtime.database; + const databaseStatus = target.databaseMode === "external-active" ? "external-db-active" : "pending-external-db"; const redisService = sub2api.runtime.redis.serviceName; const appReplicas = target.appReplicas; const redisReplicas = target.redisReplicas; + const image = targetImage(sub2api, target); + const dependencyImages = targetDependencyImages(sub2api, target); + const publicExposure = renderPublicExposureManifest(target); + const egressProxy = renderEgressProxyManifest(target); + const proxyEnv = sub2ApiProxyEnv(target); return `apiVersion: v1 kind: Namespace metadata: @@ -515,7 +949,15 @@ data: SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP: "${urlAllowlist.allowInsecureHttp}" SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS: "${urlAllowlist.allowPrivateHosts}" SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS: "${urlAllowlist.upstreamHosts.join(",")}" - UPDATE_PROXY_URL: "" + UPDATE_PROXY_URL: "${proxyEnv.httpProxy}" + HTTP_PROXY: "${proxyEnv.httpProxy}" + HTTPS_PROXY: "${proxyEnv.httpProxy}" + ALL_PROXY: "${proxyEnv.httpProxy}" + http_proxy: "${proxyEnv.httpProxy}" + https_proxy: "${proxyEnv.httpProxy}" + all_proxy: "${proxyEnv.httpProxy}" + NO_PROXY: "${proxyEnv.noProxy}" + no_proxy: "${proxyEnv.noProxy}" GATEWAY_OPENAI_RESPONSE_HEADER_TIMEOUT: "0" GATEWAY_OPENAI_HTTP2_ENABLED: "true" GATEWAY_OPENAI_HTTP2_ALLOW_PROXY_FALLBACK_TO_HTTP1: "true" @@ -602,7 +1044,7 @@ spec: fsGroup: 999 containers: - name: redis - image: redis:8-alpine + image: ${dependencyImages.redis} imagePullPolicy: IfNotPresent command: - sh @@ -651,7 +1093,7 @@ metadata: app.kubernetes.io/part-of: platform-infra app.kubernetes.io/managed-by: unidesk unidesk.ai/runtime-node: ${target.id} - unidesk.ai/database-status: pending-external-db + unidesk.ai/database-status: ${databaseStatus} spec: replicas: ${appReplicas} strategy: @@ -665,7 +1107,7 @@ spec: annotations: unidesk.ai/sub2api-config-hash: "${configHashFor(sub2api, target)}" unidesk.ai/database-source-ref: "${database.sourceRef}" - unidesk.ai/database-status: "pending-external-db" + unidesk.ai/database-status: "${databaseStatus}" labels: app.kubernetes.io/name: sub2api app.kubernetes.io/component: app @@ -675,14 +1117,14 @@ spec: fsGroup: 1000 initContainers: - name: wait-postgres - image: postgres:18-alpine + image: ${dependencyImages.postgres} imagePullPolicy: IfNotPresent command: - sh - -c - until pg_isready -h ${database.host} -p ${database.port} -U ${database.user} -d ${database.dbName}; do sleep 2; done - name: wait-redis - image: redis:8-alpine + image: ${dependencyImages.redis} imagePullPolicy: IfNotPresent command: - sh @@ -690,8 +1132,8 @@ spec: - until redis-cli -h ${redisService} ping | grep -q PONG; do sleep 2; done containers: - name: sub2api - image: ${imageRef(sub2api)} - imagePullPolicy: ${sub2api.image.pullPolicy} + image: ${imageRef(sub2api, target)} + imagePullPolicy: ${image.pullPolicy} ports: - name: http containerPort: 8080 @@ -748,9 +1190,213 @@ spec: volumes: - name: sub2api-data emptyDir: {} +${publicExposure} +${egressProxy} `; } +function sub2ApiProxyEnv(target: Sub2ApiTargetConfig): { httpProxy: string; noProxy: string } { + const proxy = target.egressProxy; + if (proxy === null || !proxy.enabled || !proxy.applyToSub2Api) return { httpProxy: "", noProxy: "" }; + return { + httpProxy: `http://${proxy.serviceName}.${target.namespace}.svc.cluster.local:${proxy.listenPort}`, + noProxy: proxy.noProxy.join(","), + }; +} + +function renderPublicExposureManifest(target: Sub2ApiTargetConfig): string { + const exposure = target.publicExposure; + if (exposure === null || !exposure.enabled) return ""; + return `--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${exposure.frpc.deploymentName} + namespace: ${target.namespace} + labels: + app.kubernetes.io/name: ${exposure.frpc.deploymentName} + app.kubernetes.io/component: tunnel + app.kubernetes.io/part-of: platform-infra + app.kubernetes.io/managed-by: unidesk + unidesk.ai/runtime-node: ${target.id} + unidesk.ai/public-hostname: ${exposure.dns.hostname} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ${exposure.frpc.deploymentName} + app.kubernetes.io/component: tunnel + template: + metadata: + labels: + app.kubernetes.io/name: ${exposure.frpc.deploymentName} + app.kubernetes.io/component: tunnel + app.kubernetes.io/part-of: platform-infra + annotations: + unidesk.ai/public-base-url: "${exposure.publicBaseUrl}" + unidesk.ai/frp-server: "${exposure.frpc.serverAddr}:${exposure.frpc.serverPort}" + unidesk.ai/frp-remote-port: "${exposure.frpc.remotePort}" + spec: + containers: + - name: frpc + image: ${exposure.frpc.image} + imagePullPolicy: IfNotPresent + args: + - -c + - /etc/frp/frpc.toml + volumeMounts: + - name: frpc-config + mountPath: /etc/frp/frpc.toml + subPath: ${exposure.frpc.secretKey} + readOnly: true + volumes: + - name: frpc-config + secret: + secretName: ${exposure.frpc.secretName} +`; +} + +function renderEgressProxyManifest(target: Sub2ApiTargetConfig): string { + const proxy = target.egressProxy; + if (proxy === null || !proxy.enabled) return ""; + return `--- +apiVersion: v1 +kind: Service +metadata: + name: ${proxy.serviceName} + namespace: ${target.namespace} + labels: + app.kubernetes.io/name: ${proxy.deploymentName} + app.kubernetes.io/component: egress-proxy + app.kubernetes.io/part-of: platform-infra + app.kubernetes.io/managed-by: unidesk + unidesk.ai/runtime-node: ${target.id} + unidesk.ai/proxy-source: master-vpn-subscription +spec: + selector: + app.kubernetes.io/name: ${proxy.deploymentName} + app.kubernetes.io/component: egress-proxy + ports: + - name: http + port: ${proxy.listenPort} + targetPort: proxy +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${proxy.deploymentName} + namespace: ${target.namespace} + labels: + app.kubernetes.io/name: ${proxy.deploymentName} + app.kubernetes.io/component: egress-proxy + app.kubernetes.io/part-of: platform-infra + app.kubernetes.io/managed-by: unidesk + unidesk.ai/runtime-node: ${target.id} + unidesk.ai/proxy-source: master-vpn-subscription +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: ${proxy.deploymentName} + app.kubernetes.io/component: egress-proxy + template: + metadata: + labels: + app.kubernetes.io/name: ${proxy.deploymentName} + app.kubernetes.io/component: egress-proxy + app.kubernetes.io/part-of: platform-infra + annotations: + unidesk.ai/proxy-source-ref: "${proxy.sourceRef}" + unidesk.ai/proxy-preferred-outbound: "${proxy.preferredOutbound}" + spec: + containers: + - name: proxy + image: ${proxy.image} + imagePullPolicy: ${proxy.imagePullPolicy} + args: + - run + - -c + - /etc/sing-box/${proxy.secretKey} + ports: + - name: proxy + containerPort: ${proxy.listenPort} + readinessProbe: + tcpSocket: + port: proxy + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + livenessProbe: + tcpSocket: + port: proxy + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 5 + failureThreshold: 6 + volumeMounts: + - name: proxy-config + mountPath: /etc/sing-box/${proxy.secretKey} + subPath: ${proxy.secretKey} + readOnly: true + volumes: + - name: proxy-config + secret: + secretName: ${proxy.secretName} +`; +} + +function publicExposureSummary(exposure: Sub2ApiPublicExposureConfig): Record { + return { + enabled: exposure.enabled, + publicBaseUrl: exposure.publicBaseUrl, + dns: exposure.dns, + frpc: { + deploymentName: exposure.frpc.deploymentName, + secretName: exposure.frpc.secretName, + image: exposure.frpc.image, + serverAddr: exposure.frpc.serverAddr, + serverPort: exposure.frpc.serverPort, + proxyName: exposure.frpc.proxyName, + remotePort: exposure.frpc.remotePort, + localIP: exposure.frpc.localIP, + localPort: exposure.frpc.localPort, + tokenSourceRef: exposure.frpc.tokenSourceRef, + valuesPrinted: false, + }, + pk01: { + route: exposure.pk01.route, + mode: "caddy-edge", + caddyBinaryPath: exposure.pk01.caddyBinaryPath, + caddyDownloadProxyUrl: exposure.pk01.caddyDownloadProxyUrl, + caddyConfigPath: exposure.pk01.caddyConfigPath, + caddyServiceName: exposure.pk01.caddyServiceName, + pikanodeHttpHostPort: exposure.pk01.pikanodeHttpHostPort, + }, + }; +} + +function egressProxySummary(proxy: Sub2ApiEgressProxyConfig): Record { + return { + enabled: proxy.enabled, + mode: "master-vpn-subscription-http-proxy", + deploymentName: proxy.deploymentName, + serviceName: proxy.serviceName, + secretName: proxy.secretName, + image: proxy.image, + imagePullPolicy: proxy.imagePullPolicy, + listenPort: proxy.listenPort, + sourceRef: proxy.sourceRef, + sourceType: proxy.sourceType, + preferredOutbound: proxy.preferredOutbound, + applyToSub2Api: proxy.applyToSub2Api, + applyToSentinel: proxy.applyToSentinel, + healthProbeUrl: proxy.healthProbeUrl, + noProxy: proxy.noProxy, + valuesPrinted: false, + }; +} + function plan(options: TargetOptions): Record { const sub2api = readSub2ApiConfig(); const target = resolveTarget(sub2api, options.targetId); @@ -770,18 +1416,22 @@ function plan(options: TargetOptions): Record { serviceDns: `${serviceName}.${target.namespace}.svc.cluster.local:8080`, }, config: { - image: imageRef(sub2api), - pullPolicy: sub2api.image.pullPolicy, + image: imageRef(sub2api, target), + pullPolicy: targetImage(sub2api, target).pullPolicy, + dependencyImages: targetDependencyImages(sub2api, target), security: sub2api.security, target: { databaseMode: target.databaseMode, redisMode: target.redisMode, appReplicas: target.appReplicas, redisReplicas: target.redisReplicas, + publicExposure: target.publicExposure === null ? null : publicExposureSummary(target.publicExposure), + egressProxy: target.egressProxy === null ? null : egressProxySummary(target.egressProxy), }, - externalDatabase: target.databaseMode === "external-pending" ? { - status: "pending-external-db", + externalDatabase: isExternalTarget(target) ? { + status: target.databaseMode === "external-active" ? "external-db-active" : "pending-external-db", sourceRef: sub2api.runtime.database.sourceRef, + sourceKeys: sub2api.runtime.database.sourceKeys, secretName: sub2api.runtime.database.secretName, passwordKey: sub2api.runtime.database.passwordKey, host: sub2api.runtime.database.host, @@ -797,14 +1447,41 @@ function plan(options: TargetOptions): Record { namespace: target.namespace, reason: target.id === "G14" ? "Sub2API remains the active G14 platform-infra deployment." - : "D601 is a standby Sub2API platform-infra target prepared through YAML; it does not run until the external Pika01/PK01 database secret is ready.", - exposure: "ClusterIP only; no public ingress or node-level exposure.", + : target.databaseMode === "external-active" + ? "D601 is activated as a platform-infra Sub2API target against the external PK01 PostgreSQL runtime while keeping app state outside the k3s node." + : "D601 is a standby Sub2API platform-infra target prepared through YAML; it does not run until the external Pika01/PK01 database secret is ready.", + exposure: target.publicExposure?.enabled + ? `Public HTTPS ${target.publicExposure.publicBaseUrl} through PK01 Caddy and D601 frpc; no master server forwarding and no Kubernetes Ingress/NodePort/LoadBalancer.` + : "ClusterIP only; no public ingress or node-level exposure.", resourcePolicy: "No Kubernetes CPU/memory requests or limits, matching issue #220.", imageVersionControl: "Sub2API image repository/tag/pullPolicy are controlled by config/platform-infra/sub2api.yaml in the UniDesk repository.", urlAllowlistControl: "Sub2API upstream URL validation options are controlled by config/platform-infra/sub2api.yaml and rendered to SECURITY_URL_ALLOWLIST_* env vars.", networkPolicy: "NetworkPolicy/allow-all is rendered with the deployment so kube-router cannot silently default-deny Sub2API cross-pod traffic.", - dataStores: target.databaseMode === "external-pending" - ? ["External PostgreSQL pending from platform-db/Pika01", target.redisReplicas === 0 ? "D601 local Redis 8 ephemeral cache, scaled to zero until activation" : "D601 local Redis 8 ephemeral cache"] + publicExposure: target.publicExposure?.enabled + ? { + mode: "pk01-caddy-frp-direct", + dataPath: "client -> PK01 Caddy -> PK01 frps remotePort -> D601 frpc -> Sub2API", + pikanodeRole: "pikapython.com upstream only; api.pikapython.com does not pass through pikanode Express", + publicBaseUrl: target.publicExposure.publicBaseUrl, + hostname: target.publicExposure.dns.hostname, + expectedA: target.publicExposure.dns.expectedA, + } + : null, + egressProxy: target.egressProxy?.enabled + ? { + mode: "D601 in-cluster HTTP proxy client to the master VPN subscription", + service: `${target.egressProxy.serviceName}.${target.namespace}.svc.cluster.local:${target.egressProxy.listenPort}`, + sourceRef: target.egressProxy.sourceRef, + applyToSub2Api: target.egressProxy.applyToSub2Api, + applyToSentinel: target.egressProxy.applyToSentinel, + valuesPrinted: false, + } + : null, + dataStores: isExternalTarget(target) + ? [ + target.databaseMode === "external-active" ? "External PostgreSQL active from platform-db/PK01" : "External PostgreSQL pending from platform-db/Pika01", + target.redisReplicas === 0 ? "D601 local Redis 8 ephemeral cache, scaled to zero until activation" : "D601 local Redis 8 ephemeral cache", + ] : ["PostgreSQL 18", "Redis 8"], appPoolCaps: { databaseMaxOpenConns: 10, @@ -879,10 +1556,14 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise = { + POSTGRES_PASSWORD: next.POSTGRES_PASSWORD, + ADMIN_PASSWORD: next.ADMIN_PASSWORD, + JWT_SECRET: next.JWT_SECRET, + TOTP_ENCRYPTION_KEY: next.TOTP_ENCRYPTION_KEY, + }; + const missing = requiredSecretKeys.filter((key) => values[key].length === 0); + if (missing.length > 0) throw new Error(`external-active app secret source is missing ${missing.join(", ")}`); + const action = !existedBefore + ? "create" + : requiredSecretKeys.some((key) => existing[key] !== values[key]) + ? "update" + : "none"; + if (action !== "none") writeEnvFile(appSourcePath, { ...existing, ...values }); + return { + sourceRef: database.sourceRef, + sourcePath: redactRepoPath(dbSourcePath), + appSourceRef: sub2api.runtime.secrets.appSourceRef, + appSourcePath: redactRepoPath(appSourcePath), + action, + fingerprint: fingerprintValues(values, [...requiredSecretKeys]), + values, + }; +} + +function preparePublicExposureSecret(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): PublicExposureSecretMaterial | null { + const exposure = target.publicExposure; + if (exposure === null || !exposure.enabled) return null; + const sourcePath = join(secretRoot(sub2api), exposure.frpc.tokenSourceRef); + if (!existsSync(sourcePath)) throw new Error(`publicExposure requires ${redactRepoPath(sourcePath)} with ${exposure.frpc.tokenSourceKey}; copy the PK01 frps token into the local secret source first`); + const values = parseEnvFile(readFileSync(sourcePath, "utf8")); + const token = requiredEnvValue(values, exposure.frpc.tokenSourceKey, exposure.frpc.tokenSourceRef); + const frpcToml = [ + `serverAddr = "${exposure.frpc.serverAddr}"`, + `serverPort = ${exposure.frpc.serverPort}`, + `loginFailExit = true`, + `auth.token = "${escapeTomlString(token)}"`, + "", + "[[proxies]]", + `name = "${exposure.frpc.proxyName}"`, + `type = "tcp"`, + `localIP = "${exposure.frpc.localIP}"`, + `localPort = ${exposure.frpc.localPort}`, + `remotePort = ${exposure.frpc.remotePort}`, + "", + ].join("\n"); + return { + sourceRef: exposure.frpc.tokenSourceRef, + sourcePath: redactRepoPath(sourcePath), + secretName: exposure.frpc.secretName, + secretKey: exposure.frpc.secretKey, + fingerprint: fingerprintValues({ token, frpcToml }, ["token", "frpcToml"]), + frpcToml, + valuesPrinted: false, + }; +} + +function prepareEgressProxySecret(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): EgressProxySecretMaterial | null { + const proxy = target.egressProxy; + if (proxy === null || !proxy.enabled) return null; + const sourcePath = join(secretRoot(sub2api), proxy.sourceRef); + if (!existsSync(sourcePath)) throw new Error(`egressProxy requires ${redactRepoPath(sourcePath)} with ${proxy.sourceKey}; materialize the master VPN subscription source first`); + const values = parseEnvFile(readFileSync(sourcePath, "utf8")); + const subscriptionUrl = requiredEnvValue(values, proxy.sourceKey, proxy.sourceRef); + const subscription = fetchSubscription(subscriptionUrl); + const decoded = decodeSubscription(subscription.body); + const nodes = decoded.split(/\r?\n/gu).map((line) => line.trim()).filter(Boolean); + const selected = selectSubscriptionNode(nodes, proxy.preferredOutbound); + const configJson = renderSingBoxConfig(selected, proxy); + const proxyUrl = `http://${proxy.serviceName}.${target.namespace}.svc.cluster.local:${proxy.listenPort}`; + const noProxy = proxy.noProxy.join(","); + return { + sourceRef: proxy.sourceRef, + sourcePath: redactRepoPath(sourcePath), + secretName: proxy.secretName, + secretKey: proxy.secretKey, + fingerprint: fingerprintValues({ subscription: subscription.body, configJson }, ["subscription", "configJson"]), + subscriptionBytes: Buffer.byteLength(subscription.body, "utf8"), + selectedOutbound: selected.kind, + configJson, + proxyUrl, + noProxy, + valuesPrinted: false, + }; +} + +function fetchSubscription(subscriptionUrl: string): { body: string } { + const shellUrl = shQuote(subscriptionUrl); + const command = `curl -fsSL --connect-timeout 10 --max-time 30 ${shellUrl}`; + const proc = Bun.spawnSync(["bash", "-lc", command], { stdout: "pipe", stderr: "pipe" }); + if (proc.exitCode !== 0) { + const stderr = new TextDecoder().decode(proc.stderr).slice(-500); + throw new Error(`failed to fetch master VPN subscription: exit=${proc.exitCode} stderr=${stderr}`); + } + const body = new TextDecoder().decode(proc.stdout).trim(); + if (body.length === 0) throw new Error("master VPN subscription is empty"); + return { body }; +} + +function decodeSubscription(raw: string): string { + if (raw.includes("://")) return raw; + const normalized = raw.replace(/\s+/gu, ""); + const padding = "=".repeat((4 - (normalized.length % 4)) % 4); + try { + return Buffer.from(`${normalized}${padding}`, "base64").toString("utf8"); + } catch (error) { + throw new Error(`master VPN subscription is neither URI list nor base64 URI list: ${String(error)}`); + } +} + +type SubscriptionNode = + | { kind: "vless-reality"; uri: string; uuid: string; host: string; port: number; sni: string; flow: string | null; fingerprint: string | null; publicKey: string; shortId: string | null } + | { kind: "hysteria2"; uri: string; password: string; host: string; port: number; sni: string | null; obfsPassword: string | null; insecure: boolean }; + +function selectSubscriptionNode(nodes: string[], preferred: Sub2ApiEgressProxyConfig["preferredOutbound"]): SubscriptionNode { + const parsed = nodes.map(parseSubscriptionNode).filter((node): node is SubscriptionNode => node !== null); + const preferredNode = parsed.find((node) => node.kind === preferred); + if (preferredNode !== undefined) return preferredNode; + const fallback = parsed.find((node) => node.kind === "vless-reality") ?? parsed.find((node) => node.kind === "hysteria2"); + if (fallback !== undefined) return fallback; + throw new Error("master VPN subscription does not contain a supported vless-reality or hysteria2 node"); +} + +function parseSubscriptionNode(uri: string): SubscriptionNode | null { + let url: URL; + try { + url = new URL(uri); + } catch { + return null; + } + if (url.protocol === "vless:") { + const params = url.searchParams; + if (params.get("security") !== "reality") return null; + const publicKey = params.get("pbk"); + if (publicKey === null || publicKey.length === 0) return null; + return { + kind: "vless-reality", + uri, + uuid: decodeURIComponent(url.username), + host: requiredUrlHost(url, uri), + port: requiredUrlPort(url, uri), + sni: params.get("sni") ?? "", + flow: params.get("flow"), + fingerprint: params.get("fp"), + publicKey, + shortId: params.get("sid"), + }; + } + if (url.protocol === "hysteria2:" || url.protocol === "hy2:") { + const params = url.searchParams; + return { + kind: "hysteria2", + uri, + password: decodeURIComponent(url.username), + host: requiredUrlHost(url, uri), + port: requiredUrlPort(url, uri), + sni: params.get("sni"), + obfsPassword: params.get("obfs-password"), + insecure: params.get("insecure") === "1" || params.get("insecure") === "true", + }; + } + return null; +} + +function requiredUrlHost(url: URL, uri: string): string { + if (url.hostname.length === 0) throw new Error(`VPN subscription node is missing host: ${redactSubscriptionUri(uri)}`); + return url.hostname; +} + +function requiredUrlPort(url: URL, uri: string): number { + const port = Number(url.port); + if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`VPN subscription node is missing a valid port: ${redactSubscriptionUri(uri)}`); + return port; +} + +function renderSingBoxConfig(node: SubscriptionNode, proxy: Sub2ApiEgressProxyConfig): string { + const outbound = node.kind === "vless-reality" + ? { + type: "vless", + tag: "master-vpn", + server: node.host, + server_port: node.port, + uuid: node.uuid, + flow: node.flow ?? undefined, + tls: { + enabled: true, + server_name: node.sni, + utls: { enabled: true, fingerprint: node.fingerprint ?? "chrome" }, + reality: { enabled: true, public_key: node.publicKey, short_id: node.shortId ?? "" }, + }, + } + : { + type: "hysteria2", + tag: "master-vpn", + server: node.host, + server_port: node.port, + password: node.password, + obfs: node.obfsPassword === null ? undefined : { type: "salamander", password: node.obfsPassword }, + tls: { enabled: true, server_name: node.sni ?? node.host, insecure: node.insecure }, + }; + const config = stripUndefined({ + log: { level: "info", timestamp: true }, + inbounds: [ + { + type: "mixed", + tag: "mixed-in", + listen: "0.0.0.0", + listen_port: proxy.listenPort, + }, + ], + outbounds: [ + outbound, + { type: "direct", tag: "direct" }, + { type: "block", tag: "block" }, + ], + route: { + rules: [ + { ip_is_private: true, outbound: "direct" }, + { domain_suffix: ["cluster.local", "svc"], outbound: "direct" }, + ], + final: "master-vpn", + }, + }); + return `${JSON.stringify(config, null, 2)}\n`; +} + +function stripUndefined(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stripUndefined); + if (value !== null && typeof value === "object") { + const output: Record = {}; + for (const [key, child] of Object.entries(value as Record)) { + if (child !== undefined) output[key] = stripUndefined(child); + } + return output; + } + return value; +} + +function redactSubscriptionUri(uri: string): string { + return uri.replace(/\/\/[^@]+@/u, "//@").replace(/password=[^&#]+/giu, "password=").replace(/pbk=[^&#]+/giu, "pbk="); +} + +function escapeTomlString(value: string): string { + return value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\""); +} + +async function applyPk01PublicExposure(config: UniDeskConfig, target: Sub2ApiTargetConfig): Promise> { + const exposure = target.publicExposure; + if (exposure === null || !exposure.enabled) return { ok: true, action: "not-enabled" }; + const start = await startPk01PublicExposureJob(config, target, exposure); + if (!start.ok || typeof start.remoteJobId !== "string") { + return { + ok: false, + action: "platform-infra-sub2api-pk01-public-exposure", + route: exposure.pk01.route, + mode: "remote-job-start-failed", + remoteJob: start, + }; + } + const waited = await waitPk01PublicExposureJob(config, exposure, start.remoteJobId); + const terminal = asRecordOrNull(waited.terminal); + const summary = asRecordOrNull(terminal?.summary); + return { + ok: waited.ok === true && boolField(summary, "ok", false), + action: "platform-infra-sub2api-pk01-public-exposure", + route: exposure.pk01.route, + mode: "remote-job", + summary, + remoteJob: { + start, + waited, + }, + }; +} + +async function startPk01PublicExposureJob(config: UniDeskConfig, target: Sub2ApiTargetConfig, exposure: Sub2ApiPublicExposureConfig): Promise> { + const jobId = `sub2api-pk01-exposure-${new Date().toISOString().replace(/[^0-9A-Za-z]/gu, "")}-${randomBytes(3).toString("hex")}`; + const payload = pk01PublicExposureScript(target, exposure); + const payloadB64 = Buffer.from(payload, "utf8").toString("base64"); + const script = ` +set -eu +job_id=${shQuote(jobId)} +base="$HOME/.unidesk/platform-infra/sub2api-pk01-exposure/jobs" +job_dir="$base/$job_id" +mkdir -p "$job_dir" +payload="$job_dir/payload.sh" +runner="$job_dir/runner.sh" +stdout="$job_dir/stdout.log" +stderr="$job_dir/stderr.log" +state="$job_dir/state.json" +printf '%s' '${payloadB64}' | base64 -d >"$payload" +chmod 0700 "$payload" +cat >"$runner" <<'RUNNER' +#!/bin/sh +set +e +job_dir="$1" +payload="$job_dir/payload.sh" +stdout="$job_dir/stdout.log" +stderr="$job_dir/stderr.log" +state="$job_dir/state.json" +write_state() { + status="$1" + stage="$2" + exit_code="$3" + python3 - "$state" "$status" "$stage" "$exit_code" <<'PY' +import datetime +import json +import sys + +path, status, stage, exit_code = sys.argv[1:5] +payload = { + "ok": status == "succeeded", + "status": status, + "stage": stage, + "updatedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "exitCode": None if exit_code == "" else int(exit_code), +} +if status in ("succeeded", "failed"): + payload["finishedAt"] = payload["updatedAt"] +with open(path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False) +PY +} +write_state running start "" +sh "$payload" >"$stdout" 2>"$stderr" +rc=$? +if [ "$rc" -eq 0 ]; then + write_state succeeded complete "$rc" +else + write_state failed complete "$rc" +fi +exit "$rc" +RUNNER +chmod 0700 "$runner" +python3 - "$state" <<'PY' +import datetime +import json +import sys + +payload = { + "ok": False, + "status": "queued", + "stage": "queued", + "updatedAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "exitCode": None, +} +with open(sys.argv[1], "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False) +PY +nohup "$runner" "$job_dir" >/dev/null 2>&1 & +printf '%s' "$!" >"$job_dir/pid" +python3 - "$job_id" "$job_dir" <<'PY' +import json +import os +import sys + +job_id, job_dir = sys.argv[1:3] +payload = { + "ok": True, + "remoteJobId": job_id, + "statePath": os.path.join(job_dir, "state.json"), + "stdoutPath": os.path.join(job_dir, "stdout.log"), + "stderrPath": os.path.join(job_dir, "stderr.log"), + "valuesPrinted": False, +} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY +`; + const result = await capture(config, exposure.pk01.route, ["script"], script); + const parsed = parseJsonOutput(result.stdout); + return { + ok: result.exitCode === 0 && boolField(parsed, "ok", false), + remoteJobId: typeof parsed?.remoteJobId === "string" ? parsed.remoteJobId : null, + parsed, + capture: compactCapture(result, { full: result.exitCode !== 0 }), + }; +} + +async function waitPk01PublicExposureJob(config: UniDeskConfig, exposure: Sub2ApiPublicExposureConfig, remoteJobId: string): Promise> { + const observations: Array> = []; + const maxPolls = 120; + let lastProgressKey: string | null = null; + for (let attempt = 0; attempt < maxPolls; attempt += 1) { + const observation = await readPk01PublicExposureJob(config, exposure, remoteJobId); + observations.push(observation.summary); + const statusValue = observation.status; + const stage = typeof observation.summary.stage === "string" ? observation.summary.stage : null; + const progressKey = `${statusValue ?? "unknown"}:${stage ?? "unknown"}`; + if (attempt === 0 || progressKey !== lastProgressKey || attempt % 6 === 0 || statusValue === "succeeded" || statusValue === "failed") { + lastProgressKey = progressKey; + console.error(JSON.stringify({ + event: "platform-infra.sub2api.pk01-exposure.progress", + at: new Date().toISOString(), + remoteJobId, + attempt: attempt + 1, + status: statusValue, + stage, + exitCode: observation.summary.exitCode ?? null, + })); + } + if (statusValue === "succeeded" || statusValue === "failed") { + return { + ok: statusValue === "succeeded" && boolField(asRecordOrNull(observation.summary.summary), "ok", false), + attempts: attempt + 1, + terminal: observation.summary, + recent: observations.slice(-10), + }; + } + await Bun.sleep(5000); + } + return { + ok: false, + attempts: maxPolls, + terminal: null, + recent: observations.slice(-10), + error: "PK01 public exposure job did not finish before local polling budget", + }; +} + +async function readPk01PublicExposureJob(config: UniDeskConfig, exposure: Sub2ApiPublicExposureConfig, remoteJobId: string): Promise<{ status: string | null; summary: Record }> { + const safeId = remoteJobId.replace(/[^A-Za-z0-9_.-]/gu, ""); + const script = ` +set +e +job_dir="$HOME/.unidesk/platform-infra/sub2api-pk01-exposure/jobs/${safeId}" +state="$job_dir/state.json" +stdout="$job_dir/stdout.log" +stderr="$job_dir/stderr.log" +if [ -f "$state" ]; then cat "$state"; else printf '{"status":"missing","stage":"missing","exitCode":null}'; fi +printf '\\n---STDOUT---\\n' +if [ -f "$stdout" ]; then tail -200 "$stdout"; fi +printf '\\n---STDERR---\\n' +if [ -f "$stderr" ]; then tail -120 "$stderr"; fi +printf '\\n---DOWNLOAD---\\n' +download_part="$HOME/.cache/unidesk/platform-infra/caddy-linux-amd64.part" +download_cache="$HOME/.cache/unidesk/platform-infra/caddy-linux-amd64" +if [ -f "$download_part" ]; then stat -c 'part_bytes=%s part_mtime=%y' "$download_part"; fi +if [ -f "$download_cache" ]; then stat -c 'cache_bytes=%s cache_mtime=%y' "$download_cache"; fi +`; + const result = await capture(config, exposure.pk01.route, ["script"], script); + const [stateText, rest = ""] = result.stdout.split("\n---STDOUT---\n"); + const [stdoutAndMaybeStderr = "", downloadProgress = ""] = rest.split("\n---DOWNLOAD---\n"); + const [stdoutTail = "", stderrTail = ""] = stdoutAndMaybeStderr.split("\n---STDERR---\n"); + const parsed = parseJsonOutput(stateText) ?? {}; + const statusValue = typeof parsed.status === "string" ? parsed.status : null; + const summary = parseJsonOutput(stdoutTail); + return { + status: statusValue, + summary: { + ok: result.exitCode === 0, + remoteJobId, + status: statusValue, + stage: parsed.stage ?? null, + updatedAt: parsed.updatedAt ?? null, + finishedAt: parsed.finishedAt ?? null, + exitCode: parsed.exitCode ?? null, + summary, + stdoutTail: stdoutTail.slice(-8000), + stderrTail: stderrTail.slice(-4000), + downloadProgress: downloadProgress.slice(-1000), + capture: compactCapture(result, { full: result.exitCode !== 0 }), + }, + }; +} + +function pk01PublicExposureScript(target: Sub2ApiTargetConfig, exposure: Sub2ApiPublicExposureConfig): string { + const caddyfile = renderPk01Caddyfile(exposure); + const serviceUnit = renderPk01CaddyService(exposure); + const caddyfileB64 = Buffer.from(caddyfile, "utf8").toString("base64"); + const serviceUnitB64 = Buffer.from(serviceUnit, "utf8").toString("base64"); + const caddyDownloadProxyArgs = exposure.pk01.caddyDownloadProxyUrl === null ? "" : `--proxy ${shQuote(exposure.pk01.caddyDownloadProxyUrl)}`; + const caddyDownloadProxyUrl = exposure.pk01.caddyDownloadProxyUrl ?? ""; + return ` +set -u +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +caddyfile="$tmp/Caddyfile" +service_unit="$tmp/${exposure.pk01.caddyServiceName}.service" +printf '%s' '${caddyfileB64}' | base64 -d >"$caddyfile" +printf '%s' '${serviceUnitB64}' | base64 -d >"$service_unit" +download_out="$tmp/download.out" +download_err="$tmp/download.err" +install_out="$tmp/install.out" +install_err="$tmp/install.err" +validate_out="$tmp/validate.out" +validate_err="$tmp/validate.err" +pikanode_out="$tmp/pikanode.out" +pikanode_err="$tmp/pikanode.err" +caddy_out="$tmp/caddy.out" +caddy_err="$tmp/caddy.err" +probe_out="$tmp/probe.out" +probe_err="$tmp/probe.err" +download_action="kept-existing" +download_proxy_url=${shQuote(caddyDownloadProxyUrl)} +download_proxy_mode="direct" +if [ -n "$download_proxy_url" ]; then download_proxy_mode="curl-proxy"; fi +download_rc=0 +if ! [ -x ${shQuote(exposure.pk01.caddyBinaryPath)} ]; then + cache_dir="$HOME/.cache/unidesk/platform-infra" + cache_path="$cache_dir/caddy-linux-amd64" + cache_part="$cache_path.part" + mkdir -p "$cache_dir" + if [ -x "$cache_path" ]; then + download_action="used-cache" + "$cache_path" version >"$download_out" 2>"$download_err" || true + else + download_action="downloaded" + resume_args= + if [ -s "$cache_part" ]; then resume_args="-C -"; fi + curl -fL --connect-timeout 20 --retry 5 --retry-delay 3 --max-time 600 ${caddyDownloadProxyArgs} $resume_args ${shQuote(exposure.pk01.caddyDownloadUrl)} -o "$cache_part" >"$download_out" 2>"$download_err" + download_rc=$? + if [ "$download_rc" -eq 33 ]; then + rm -f "$cache_part" + curl -fL --connect-timeout 20 --retry 5 --retry-delay 3 --max-time 600 ${caddyDownloadProxyArgs} ${shQuote(exposure.pk01.caddyDownloadUrl)} -o "$cache_part" >>"$download_out" 2>>"$download_err" + download_rc=$? + fi + if [ "$download_rc" -eq 0 ]; then + mv "$cache_part" "$cache_path" + chmod 0755 "$cache_path" + fi + fi + if [ "$download_rc" -eq 0 ]; then + sudo install -m 0755 "$cache_path" ${shQuote(exposure.pk01.caddyBinaryPath)} >>"$download_out" 2>>"$download_err" + download_rc=$? + fi +else + rm -f "$HOME/.cache/unidesk/platform-infra/caddy-linux-amd64.part" + ${shQuote(exposure.pk01.caddyBinaryPath)} version >"$download_out" 2>"$download_err" || true +fi +install_rc=1 +if [ "$download_rc" -eq 0 ]; then + sudo mkdir -p "$(dirname ${shQuote(exposure.pk01.caddyConfigPath)})" ${shQuote(exposure.pk01.caddyStorageDir)} /etc/systemd/system + sudo install -m 0644 "$caddyfile" ${shQuote(exposure.pk01.caddyConfigPath)} >"$install_out" 2>"$install_err" + install_rc=$? + if [ "$install_rc" -eq 0 ]; then + sudo install -m 0644 "$service_unit" /etc/systemd/system/${exposure.pk01.caddyServiceName}.service >>"$install_out" 2>>"$install_err" + install_rc=$? + fi +fi +validate_rc=1 +if [ "$install_rc" -eq 0 ]; then + sudo ${shQuote(exposure.pk01.caddyBinaryPath)} validate --config ${shQuote(exposure.pk01.caddyConfigPath)} >"$validate_out" 2>"$validate_err" + validate_rc=$? +else + : >"$validate_out" + : >"$validate_err" +fi +pikanode_rc=1 +if [ "$validate_rc" -eq 0 ]; then + if docker inspect ${shQuote(exposure.pk01.pikanodeContainerName)} >/dev/null 2>&1; then + docker rm -f ${shQuote(exposure.pk01.pikanodeContainerName)} >"$pikanode_out" 2>"$pikanode_err" + pikanode_rc=$? + else + : >"$pikanode_out" + : >"$pikanode_err" + pikanode_rc=0 + fi + if [ "$pikanode_rc" -eq 0 ]; then + docker run -d --name ${shQuote(exposure.pk01.pikanodeContainerName)} \\ + --restart=always \\ + -p 127.0.0.1:${exposure.pk01.pikanodeHttpHostPort}:8888 \\ + -p 22000-22099:22000-22099 \\ + -p 7500:7500 \\ + -v ${shQuote(exposure.pk01.pikanodeRoot)}:/usr/src/app \\ + -v /etc/letsencrypt:/etc/letsencrypt:ro \\ + -w /usr/src/app \\ + ${shQuote(exposure.pk01.pikanodeImage)} \\ + sh init.sh >>"$pikanode_out" 2>>"$pikanode_err" + pikanode_rc=$? + if [ "$pikanode_rc" -ne 0 ]; then + printf '%s\\n' '[rollback] restarting pikanode with previous public 80/443 mapping' >>"$pikanode_err" + docker rm -f ${shQuote(exposure.pk01.pikanodeContainerName)} >>"$pikanode_out" 2>>"$pikanode_err" || true + docker run -d --name ${shQuote(exposure.pk01.pikanodeContainerName)} \\ + --restart=always \\ + -p 80:8888 \\ + -p 22000-22099:22000-22099 \\ + -p 443:443 \\ + -p 7500:7500 \\ + -v ${shQuote(exposure.pk01.pikanodeRoot)}:/usr/src/app \\ + -v /etc/letsencrypt:/etc/letsencrypt:ro \\ + -w /usr/src/app \\ + ${shQuote(exposure.pk01.pikanodeImage)} \\ + sh init.sh >>"$pikanode_out" 2>>"$pikanode_err" || true + fi + fi +else + : >"$pikanode_out" + : >"$pikanode_err" +fi +caddy_rc=1 +if [ "$pikanode_rc" -eq 0 ]; then + sudo systemctl daemon-reload >"$caddy_out" 2>"$caddy_err" + sudo systemctl enable --now ${shQuote(exposure.pk01.caddyServiceName)} >>"$caddy_out" 2>>"$caddy_err" + caddy_rc=$? + if [ "$caddy_rc" -eq 0 ]; then + sudo systemctl reload ${shQuote(exposure.pk01.caddyServiceName)} >>"$caddy_out" 2>>"$caddy_err" || sudo systemctl restart ${shQuote(exposure.pk01.caddyServiceName)} >>"$caddy_out" 2>>"$caddy_err" + caddy_rc=$? + fi +fi +probe_rc=1 +if [ "$caddy_rc" -eq 0 ]; then + { + printf '[pikanode]\\n' + curl -kfsS --max-time 10 --resolve pikapython.com:443:127.0.0.1 https://pikapython.com/ -o /dev/null -w 'status=%{http_code}\\n' + printf '[api-health]\\n' + curl -kfsS --max-time 10 --resolve ${exposure.dns.hostname}:443:127.0.0.1 ${exposure.publicBaseUrl}/health -o /dev/null -w 'status=%{http_code}\\n' || true + } >"$probe_out" 2>"$probe_err" + probe_rc=0 +else + : >"$probe_out" + : >"$probe_err" +fi +python3 - "$download_rc" "$install_rc" "$validate_rc" "$pikanode_rc" "$caddy_rc" "$probe_rc" "$download_action" "$download_proxy_mode" "$download_proxy_url" "$download_out" "$download_err" "$install_out" "$install_err" "$validate_out" "$validate_err" "$pikanode_out" "$pikanode_err" "$caddy_out" "$caddy_err" "$probe_out" "$probe_err" <<'PY' +import json +import sys + +download_rc, install_rc, validate_rc, pikanode_rc, caddy_rc, probe_rc = [int(value) for value in sys.argv[1:7]] +download_action = sys.argv[7] +download_proxy_mode = sys.argv[8] +download_proxy_url = sys.argv[9] or None +paths = sys.argv[10:] + +def text(path, limit=4000): + try: + return open(path, encoding="utf-8", errors="replace").read()[-limit:] + except FileNotFoundError: + return "" + +payload = { + "ok": download_rc == 0 and install_rc == 0 and validate_rc == 0 and pikanode_rc == 0 and caddy_rc == 0, + "target": "${target.id}", + "mode": "pk01-caddy-frp-direct", + "publicBaseUrl": "${exposure.publicBaseUrl}", + "hostname": "${exposure.dns.hostname}", + "expectedA": "${exposure.dns.expectedA}", + "dataPath": "client -> PK01 Caddy -> PK01 frps remotePort -> D601 frpc -> Sub2API", + "pikanodeRole": "pikapython.com upstream only; api.pikapython.com does not pass through pikanode Express", + "caddy": { + "binaryPath": "${exposure.pk01.caddyBinaryPath}", + "configPath": "${exposure.pk01.caddyConfigPath}", + "serviceName": "${exposure.pk01.caddyServiceName}", + "downloadProxy": {"mode": download_proxy_mode, "url": download_proxy_url or None}, + }, + "frp": { + "server": "${exposure.frpc.serverAddr}:${exposure.frpc.serverPort}", + "remotePort": ${exposure.frpc.remotePort}, + "proxyName": "${exposure.frpc.proxyName}", + }, + "steps": { + "downloadCaddy": {"exitCode": download_rc, "action": download_action, "stdout": text(paths[0]), "stderr": text(paths[1])}, + "installConfig": {"exitCode": install_rc, "stdout": text(paths[2]), "stderr": text(paths[3])}, + "validateCaddyConfig": {"exitCode": validate_rc, "stdout": text(paths[4]), "stderr": text(paths[5])}, + "restartPikanodeLoopback": {"exitCode": pikanode_rc, "stdout": text(paths[6]), "stderr": text(paths[7])}, + "startCaddy": {"exitCode": caddy_rc, "stdout": text(paths[8]), "stderr": text(paths[9])}, + "localProbe": {"exitCode": probe_rc, "stdout": text(paths[10]), "stderr": text(paths[11])}, + }, + "valuesPrinted": False, +} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if payload["ok"] else 1) +PY +`; +} + +function renderPk01Caddyfile(exposure: Sub2ApiPublicExposureConfig): string { + const apexHost = baseDomain(exposure.dns.hostname); + return `{ + email ${exposure.pk01.caddyEmail} + storage file_system { + root ${exposure.pk01.caddyStorageDir} + } +} + +${apexHost}, www.${apexHost} { + reverse_proxy 127.0.0.1:${exposure.pk01.pikanodeHttpHostPort} +} + +${exposure.dns.hostname} { + reverse_proxy 127.0.0.1:${exposure.frpc.remotePort} { + transport http { + response_header_timeout ${exposure.pk01.responseHeaderTimeoutSeconds}s + } + } +} +`; +} + +function renderPk01CaddyService(exposure: Sub2ApiPublicExposureConfig): string { + return `[Unit] +Description=UniDesk PK01 Caddy edge for PikaPython and Sub2API +After=network-online.target docker.service +Wants=network-online.target + +[Service] +Type=simple +ExecStart=${exposure.pk01.caddyBinaryPath} run --environ --config ${exposure.pk01.caddyConfigPath} +ExecReload=${exposure.pk01.caddyBinaryPath} reload --config ${exposure.pk01.caddyConfigPath} --force +Restart=always +RestartSec=3 +LimitNOFILE=1048576 +AmbientCapabilities=CAP_NET_BIND_SERVICE +NoNewPrivileges=true +Environment=XDG_DATA_HOME=${exposure.pk01.caddyStorageDir} +Environment=XDG_CONFIG_HOME=/etc/caddy + +[Install] +WantedBy=multi-user.target +`; +} + +function baseDomain(hostname: string): string { + const parts = hostname.split("."); + return parts.length <= 2 ? hostname : parts.slice(-2).join("."); +} + +function shQuote(value: string): string { + return `'${value.replaceAll("'", "'\"'\"'")}'`; +} + +function secretRoot(sub2api: Sub2ApiConfig): string { + const root = sub2api.runtime.secrets.root; + return isAbsolute(root) ? root : rootPath(root); +} + +function requiredEnvValue(values: Record, key: string, sourceRef: string): string { + const value = values[key]; + if (value === undefined || value.length === 0) throw new Error(`${sourceRef} is missing required key ${key}`); + return value; +} + +function parseEnvFile(text: string): Record { + const result: Record = {}; + for (const rawLine of text.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq <= 0) continue; + const key = line.slice(0, eq).trim(); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) continue; + result[key] = unquoteEnvValue(line.slice(eq + 1).trim()); + } + return result; +} + +function unquoteEnvValue(value: string): string { + if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith("\"") && value.endsWith("\""))) { + return value.slice(1, -1); + } + return value; +} + +function writeEnvFile(path: string, values: Record): void { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const lines = Object.keys(values) + .sort() + .map((key) => `${key}=${quoteEnv(values[key])}`); + writeFileSync(path, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); +} + +function quoteEnv(value: string): string { + if (/^[A-Za-z0-9_./:@%?&=+-]+$/u.test(value)) return value; + return `'${value.replaceAll("'", "'\"'\"'")}'`; +} + +function fingerprintValues(values: Record, keys: string[]): string { + const hash = createHash("sha256"); + for (const key of keys) { + hash.update(key); + hash.update("\0"); + hash.update(values[key] ?? ""); + hash.update("\0"); + } + return `sha256:${hash.digest("hex")}`; +} + +function redactRepoPath(path: string): string { + return path.startsWith(`${repoRoot}/`) ? path.slice(repoRoot.length + 1) : path; +} + function dryRunScript(yaml: string, target: Sub2ApiTargetConfig): string { const encoded = Buffer.from(yaml, "utf8").toString("base64"); return ` @@ -1124,9 +2627,62 @@ PY `; } -function applyScript(yaml: string, target: Sub2ApiTargetConfig): string { +function applyScript( + yaml: string, + target: Sub2ApiTargetConfig, + secretMaterial: ExternalActiveSecretMaterial | null, + publicExposureSecretMaterial: PublicExposureSecretMaterial | null, + egressProxySecretMaterial: EgressProxySecretMaterial | null, +): string { const encoded = Buffer.from(yaml, "utf8").toString("base64"); - const managesSecret = target.databaseMode !== "external-pending"; + if (target.databaseMode === "external-active" && secretMaterial === null) throw new Error("external-active apply requires secret material"); + const externalSecretFiles = secretMaterial === null + ? "" + : requiredSecretKeys.map((key) => { + const value = Buffer.from(secretMaterial.values[key], "utf8").toString("base64"); + return ` printf '%s' '${value}' | base64 -d >"$tmp/secret.${key}"`; + }).join("\n"); + const secretSourceSummary = secretMaterial === null + ? "None" + : `json.loads(${JSON.stringify(JSON.stringify({ + sourceRef: secretMaterial.sourceRef, + sourcePath: secretMaterial.sourcePath, + appSourceRef: secretMaterial.appSourceRef, + appSourcePath: secretMaterial.appSourcePath, + sourceAction: secretMaterial.action, + fingerprint: secretMaterial.fingerprint, + valuesPrinted: false, + }))})`; + const exposureSecretFile = publicExposureSecretMaterial === null + ? "" + : ` printf '%s' '${Buffer.from(publicExposureSecretMaterial.frpcToml, "utf8").toString("base64")}' | base64 -d >"$tmp/frpc.toml"`; + const exposureSecretSummary = publicExposureSecretMaterial === null + ? "None" + : `json.loads(${JSON.stringify(JSON.stringify({ + sourceRef: publicExposureSecretMaterial.sourceRef, + sourcePath: publicExposureSecretMaterial.sourcePath, + secretName: publicExposureSecretMaterial.secretName, + secretKey: publicExposureSecretMaterial.secretKey, + fingerprint: publicExposureSecretMaterial.fingerprint, + valuesPrinted: false, + }))})`; + const egressProxySecretFile = egressProxySecretMaterial === null + ? "" + : ` printf '%s' '${Buffer.from(egressProxySecretMaterial.configJson, "utf8").toString("base64")}' | base64 -d >"$tmp/egress-proxy-config.json"`; + const egressProxySecretSummary = egressProxySecretMaterial === null + ? "None" + : `json.loads(${JSON.stringify(JSON.stringify({ + sourceRef: egressProxySecretMaterial.sourceRef, + sourcePath: egressProxySecretMaterial.sourcePath, + secretName: egressProxySecretMaterial.secretName, + secretKey: egressProxySecretMaterial.secretKey, + fingerprint: egressProxySecretMaterial.fingerprint, + subscriptionBytes: egressProxySecretMaterial.subscriptionBytes, + selectedOutbound: egressProxySecretMaterial.selectedOutbound, + proxyUrl: egressProxySecretMaterial.proxyUrl, + noProxy: egressProxySecretMaterial.noProxy, + valuesPrinted: false, + }))})`; return ` set -u tmp="$(mktemp -d)" @@ -1137,17 +2693,35 @@ ns_out="$tmp/ns.out" ns_err="$tmp/ns.err" secret_out="$tmp/secret.out" secret_err="$tmp/secret.err" +exposure_secret_out="$tmp/exposure-secret.out" +exposure_secret_err="$tmp/exposure-secret.err" +egress_secret_out="$tmp/egress-secret.out" +egress_secret_err="$tmp/egress-secret.err" apply_out="$tmp/apply.out" apply_err="$tmp/apply.err" kubectl create namespace ${target.namespace} --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$ns_out" 2>"$ns_err" ns_rc=$? secret_action="unknown" secret_rc=0 +exposure_secret_action="not-enabled" +exposure_secret_rc=0 +egress_secret_action="not-enabled" +egress_secret_rc=0 if [ "$ns_rc" -eq 0 ]; then - if [ "${managesSecret ? "true" : "false"}" != "true" ]; then + if [ "${target.databaseMode}" = "external-pending" ]; then secret_action="external-pending-not-managed" : >"$secret_out" printf '%s\\n' 'external DB target expects its DB credential Secret from the platform DB handoff; predeploy does not create placeholder secrets' >"$secret_err" + elif [ "${target.databaseMode}" = "external-active" ]; then +${externalSecretFiles} + kubectl -n ${target.namespace} create secret generic ${secretName} \\ + --from-file=POSTGRES_PASSWORD="$tmp/secret.POSTGRES_PASSWORD" \\ + --from-file=ADMIN_PASSWORD="$tmp/secret.ADMIN_PASSWORD" \\ + --from-file=JWT_SECRET="$tmp/secret.JWT_SECRET" \\ + --from-file=TOTP_ENCRYPTION_KEY="$tmp/secret.TOTP_ENCRYPTION_KEY" \\ + --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$secret_out" 2>"$secret_err" + secret_rc=$? + secret_action="external-active-synced" elif kubectl -n ${target.namespace} get secret ${secretName} >/dev/null 2>&1; then secret_action="kept-existing" : >"$secret_out" @@ -1170,30 +2744,56 @@ if [ "$ns_rc" -eq 0 ]; then secret_rc=$? secret_action="created" fi + if [ "${publicExposureSecretMaterial === null ? "false" : "true"}" = "true" ]; then +${exposureSecretFile} + kubectl -n ${target.namespace} create secret generic ${publicExposureSecretMaterial?.secretName ?? "sub2api-frpc-secrets"} \\ + --from-file=${publicExposureSecretMaterial?.secretKey ?? "frpc.toml"}="$tmp/frpc.toml" \\ + --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$exposure_secret_out" 2>"$exposure_secret_err" + exposure_secret_rc=$? + exposure_secret_action="synced" + else + : >"$exposure_secret_out" + : >"$exposure_secret_err" + fi + if [ "${egressProxySecretMaterial === null ? "false" : "true"}" = "true" ]; then +${egressProxySecretFile} + kubectl -n ${target.namespace} create secret generic ${egressProxySecretMaterial?.secretName ?? "sub2api-egress-proxy-config"} \\ + --from-file=${egressProxySecretMaterial?.secretKey ?? "config.json"}="$tmp/egress-proxy-config.json" \\ + --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$egress_secret_out" 2>"$egress_secret_err" + egress_secret_rc=$? + egress_secret_action="synced" + else + : >"$egress_secret_out" + : >"$egress_secret_err" + fi fi apply_rc=1 -if [ "$ns_rc" -eq 0 ] && [ "$secret_rc" -eq 0 ]; then +if [ "$ns_rc" -eq 0 ] && [ "$secret_rc" -eq 0 ] && [ "$exposure_secret_rc" -eq 0 ] && [ "$egress_secret_rc" -eq 0 ]; then kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f "$manifest" >"$apply_out" 2>"$apply_err" apply_rc=$? else : >"$apply_out" - printf '%s\\n' 'skipped because namespace or secret step failed' >"$apply_err" + printf '%s\\n' 'skipped because namespace, app secret, public exposure secret, or egress proxy secret step failed' >"$apply_err" fi -python3 - "$ns_rc" "$secret_rc" "$apply_rc" "$secret_action" "$ns_out" "$ns_err" "$secret_out" "$secret_err" "$apply_out" "$apply_err" <<'PY' +python3 - "$ns_rc" "$secret_rc" "$exposure_secret_rc" "$egress_secret_rc" "$apply_rc" "$secret_action" "$exposure_secret_action" "$egress_secret_action" "$ns_out" "$ns_err" "$secret_out" "$secret_err" "$exposure_secret_out" "$exposure_secret_err" "$egress_secret_out" "$egress_secret_err" "$apply_out" "$apply_err" <<'PY' import json import sys ns_rc = int(sys.argv[1]) secret_rc = int(sys.argv[2]) -apply_rc = int(sys.argv[3]) -secret_action = sys.argv[4] -paths = sys.argv[5:] +exposure_secret_rc = int(sys.argv[3]) +egress_secret_rc = int(sys.argv[4]) +apply_rc = int(sys.argv[5]) +secret_action = sys.argv[6] +exposure_secret_action = sys.argv[7] +egress_secret_action = sys.argv[8] +paths = sys.argv[9:] def text(path): try: return open(path, encoding="utf-8").read() except FileNotFoundError: return "" payload = { - "ok": ns_rc == 0 and secret_rc == 0 and apply_rc == 0, + "ok": ns_rc == 0 and secret_rc == 0 and exposure_secret_rc == 0 and egress_secret_rc == 0 and apply_rc == 0, "target": "${target.id}", "namespace": "${target.namespace}", "databaseMode": "${target.databaseMode}", @@ -1203,13 +2803,18 @@ payload = { "name": "${secretName}", "action": secret_action, "requiredKeys": ${JSON.stringify(requiredSecretKeys)}, - "managedByThisApply": ${managesSecret ? "True" : "False"}, + "managedByThisApply": ${target.databaseMode === "external-pending" ? "False" : "True"}, + "externalActiveSource": ${secretSourceSummary}, "valuesPrinted": False, }, + "publicExposure": ${exposureSecretSummary}, + "egressProxy": ${egressProxySecretSummary}, "steps": { "namespace": {"exitCode": ns_rc, "stdout": text(paths[0])[-4000:], "stderr": text(paths[1])[-4000:]}, "secret": {"exitCode": secret_rc, "stdout": text(paths[2])[-4000:], "stderr": text(paths[3])[-4000:]}, - "apply": {"exitCode": apply_rc, "stdout": text(paths[4])[-8000:], "stderr": text(paths[5])[-4000:]}, + "publicExposureSecret": {"exitCode": exposure_secret_rc, "action": exposure_secret_action, "stdout": text(paths[4])[-4000:], "stderr": text(paths[5])[-4000:]}, + "egressProxySecret": {"exitCode": egress_secret_rc, "action": egress_secret_action, "stdout": text(paths[6])[-4000:], "stderr": text(paths[7])[-4000:]}, + "apply": {"exitCode": apply_rc, "stdout": text(paths[8])[-8000:], "stderr": text(paths[9])[-4000:]}, }, } print(json.dumps(payload, ensure_ascii=False, indent=2)) @@ -1219,9 +2824,12 @@ PY } function statusScript(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { - const expectedImage = imageRef(sub2api); + const expectedImage = imageRef(sub2api, target); const expectedUrlAllowlist = sub2api.security.urlAllowlist; const externalPending = target.databaseMode === "external-pending"; + const externalActive = target.databaseMode === "external-active"; + const proxy = target.egressProxy?.enabled ? target.egressProxy : null; + const expectedProxyEnv = sub2ApiProxyEnv(target); return ` set -u tmp="$(mktemp -d)" @@ -1248,7 +2856,7 @@ capture_json limitranges kubectl -n ${target.namespace} get limitrange pod_name="$(kubectl -n ${target.namespace} get pod -l app.kubernetes.io/name=${serviceName},app.kubernetes.io/component=app -o jsonpath='{.items[0].metadata.name}' 2>"$tmp/pod-name.err" || true)" printf '%s' "$pod_name" >"$tmp/pod-name.txt" if [ -n "$pod_name" ]; then - kubectl -n ${target.namespace} exec "$pod_name" -- sh -c 'printf "SECURITY_URL_ALLOWLIST_ENABLED=%s\\n" "$SECURITY_URL_ALLOWLIST_ENABLED"; printf "SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP=%s\\n" "$SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP"; printf "SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS=%s\\n" "$SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS"; printf "SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS=%s\\n" "$SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS"' >"$tmp/pod-env.out" 2>"$tmp/pod-env.err" + kubectl -n ${target.namespace} exec "$pod_name" -- sh -c 'printf "SECURITY_URL_ALLOWLIST_ENABLED=%s\\n" "$SECURITY_URL_ALLOWLIST_ENABLED"; printf "SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP=%s\\n" "$SECURITY_URL_ALLOWLIST_ALLOW_INSECURE_HTTP"; printf "SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS=%s\\n" "$SECURITY_URL_ALLOWLIST_ALLOW_PRIVATE_HOSTS"; printf "SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS=%s\\n" "$SECURITY_URL_ALLOWLIST_UPSTREAM_HOSTS"; printf "HTTP_PROXY=%s\\n" "$HTTP_PROXY"; printf "HTTPS_PROXY=%s\\n" "$HTTPS_PROXY"; printf "ALL_PROXY=%s\\n" "$ALL_PROXY"; printf "NO_PROXY=%s\\n" "$NO_PROXY"' >"$tmp/pod-env.out" 2>"$tmp/pod-env.err" printf '%s' "$?" >"$tmp/pod-env.rc" else : >"$tmp/pod-env.out" @@ -1444,6 +3052,7 @@ configmap_data = (configmap or {}).get("data") or {} secret_keys = sorted(((secret or {}).get("data") or {}).keys()) missing_secret_keys = [key for key in ${JSON.stringify(requiredSecretKeys)} if key not in secret_keys] external_pending = ${externalPending ? "True" : "False"} +external_active = ${externalActive ? "True" : "False"} expected_app_replicas = ${target.appReplicas} expected_redis_replicas = ${target.redisReplicas} service_violations = [] @@ -1486,9 +3095,32 @@ expected_url_allowlist_strings = { "allowPrivateHosts": str(expected_url_allowlist.get("allowPrivateHosts")).lower(), "upstreamHosts": ",".join(expected_url_allowlist.get("upstreamHosts") or []), } +expected_proxy_env = json.loads(${JSON.stringify(JSON.stringify({ + enabled: proxy !== null && proxy.applyToSub2Api, + httpProxy: expectedProxyEnv.httpProxy, + noProxy: expectedProxyEnv.noProxy, + deploymentName: proxy?.deploymentName ?? null, + serviceName: proxy?.serviceName ?? null, + }))}) url_allowlist_configmap_aligned = url_allowlist_runtime == expected_url_allowlist_strings url_allowlist_pod_env_aligned = rc("pod-env") == 0 and url_allowlist_pod_env == expected_url_allowlist_strings url_allowlist_aligned = url_allowlist_configmap_aligned and (url_allowlist_pod_env_aligned or (external_pending and expected_app_replicas == 0)) +proxy_env_runtime = { + "HTTP_PROXY": pod_env.get("HTTP_PROXY"), + "HTTPS_PROXY": pod_env.get("HTTPS_PROXY"), + "ALL_PROXY": pod_env.get("ALL_PROXY"), + "NO_PROXY": pod_env.get("NO_PROXY"), +} +if expected_proxy_env.get("enabled"): + proxy_env_aligned = ( + rc("pod-env") == 0 + and proxy_env_runtime.get("HTTP_PROXY") == expected_proxy_env.get("httpProxy") + and proxy_env_runtime.get("HTTPS_PROXY") == expected_proxy_env.get("httpProxy") + and proxy_env_runtime.get("ALL_PROXY") == expected_proxy_env.get("httpProxy") + and proxy_env_runtime.get("NO_PROXY") == expected_proxy_env.get("noProxy") + ) +else: + proxy_env_aligned = True allow_all_network_policy = next((item for item in networkpolicies if item.get("metadata", {}).get("name") == "allow-all"), None) network_policy = { "requiredName": "allow-all", @@ -1511,17 +3143,29 @@ local_postgres_present = any(item.get("metadata", {}).get("name") == "sub2api-po redis_pvc_present = any(item.get("metadata", {}).get("name") == "sub2api-redis-data" for item in pvcs) secret_ready = len(missing_secret_keys) == 0 secret_ok = secret_ready or external_pending -state_model_ok = (not external_pending) or ( - not local_postgres_present - and not redis_pvc_present - and sub2api_desired_aligned - and redis_desired_aligned - and expected_app_replicas == 0 - and expected_redis_replicas == 0 -) -status_label = "pending-external-db" if external_pending else "active" +if external_pending: + state_model_ok = ( + not local_postgres_present + and not redis_pvc_present + and sub2api_desired_aligned + and redis_desired_aligned + and expected_app_replicas == 0 + and expected_redis_replicas == 0 + ) +elif external_active: + state_model_ok = ( + not local_postgres_present + and not redis_pvc_present + and sub2api_desired_aligned + and redis_desired_aligned + and expected_app_replicas == 1 + and expected_redis_replicas == 1 + ) +else: + state_model_ok = local_postgres_present and sub2api_desired_aligned and redis_desired_aligned +status_label = "pending-external-db" if external_pending else "external-db-active" if external_active else "active" payload = { - "ok": rc("ns") == 0 and workload_ready and image_aligned and url_allowlist_aligned and network_policy["ok"] and boundary["internalOnly"] and len(resource_violations) == 0 and boundary["resourceQuotaCount"] == 0 and boundary["limitRangeCount"] == 0 and secret_ok and state_model_ok, + "ok": rc("ns") == 0 and workload_ready and image_aligned and url_allowlist_aligned and proxy_env_aligned and network_policy["ok"] and boundary["internalOnly"] and len(resource_violations) == 0 and boundary["resourceQuotaCount"] == 0 and boundary["limitRangeCount"] == 0 and secret_ok and state_model_ok, "target": "${target.id}", "route": "${target.route}", "namespace": "${target.namespace}", @@ -1577,6 +3221,16 @@ payload = { }, } }, + "egressProxy": { + "enabled": expected_proxy_env.get("enabled"), + "deploymentName": expected_proxy_env.get("deploymentName"), + "serviceName": expected_proxy_env.get("serviceName"), + "expectedHttpProxy": expected_proxy_env.get("httpProxy"), + "expectedNoProxy": expected_proxy_env.get("noProxy"), + "podEnv": proxy_env_runtime, + "aligned": proxy_env_aligned, + "valuesPrinted": False, + }, "boundary": boundary, "serviceDns": "${serviceName}.${target.namespace}.svc.cluster.local:8080", "next": { @@ -1735,8 +3389,275 @@ PY `; } +function validateExternalActiveScript(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { + const database = sub2api.runtime.database; + const redisService = sub2api.runtime.redis.serviceName; + const exposure = target.publicExposure?.enabled ? target.publicExposure : null; + const proxy = target.egressProxy?.enabled ? target.egressProxy : null; + const proxyUrl = proxy === null ? "" : `http://${proxy.serviceName}.${target.namespace}.svc.cluster.local:${proxy.listenPort}`; + const publicProbeBlock = exposure === null ? ` +public_dns_rc=0 +public_probe_rc=0 +: >"$tmp/public-dns.out" +: >"$tmp/public-dns.err" +: >"$tmp/public-probe.out" +: >"$tmp/public-probe.err" +` : ` +public_dns_rc=1 +{ + for resolver in ${exposure.dns.resolvers.map((resolver) => shQuote(resolver)).join(" ")}; do + printf 'resolver=%s\\n' "$resolver" + dig +short "@$resolver" ${shQuote(exposure.dns.hostname)} A || true + done +} >"$tmp/public-dns.out" 2>"$tmp/public-dns.err" +if grep -Fx ${shQuote(exposure.dns.expectedA)} "$tmp/public-dns.out" >/dev/null 2>&1; then public_dns_rc=0; fi +curl -fsS --max-time 20 ${shQuote(`${exposure.publicBaseUrl}/health`)} >"$tmp/public-probe.out" 2>"$tmp/public-probe.err" +public_probe_rc=$? +`; + const egressProbeBlock = proxy === null ? ` +egress_deploy_rc=0 +egress_svc_rc=0 +egress_probe_rc=0 +: >"$tmp/egress-deploy.json" +: >"$tmp/egress-deploy.err" +: >"$tmp/egress-svc.json" +: >"$tmp/egress-svc.err" +: >"$tmp/egress-probe.out" +: >"$tmp/egress-probe.err" +` : ` +capture_json egress-deploy kubectl -n ${target.namespace} get deployment ${proxy.deploymentName} +egress_deploy_rc=$(cat "$tmp/egress-deploy.rc") +capture_json egress-svc kubectl -n ${target.namespace} get service ${proxy.serviceName} +egress_svc_rc=$(cat "$tmp/egress-svc.rc") +if [ -n "$app_pod" ]; then + kubectl -n ${target.namespace} exec "$app_pod" -- sh -c 'HTTP_PROXY="$1" HTTPS_PROXY="$1" ALL_PROXY="$1" NO_PROXY="$2" curl -fsS --max-time 20 -o /dev/null -w "status=%{http_code}\\n" "$3"' sh ${shQuote(proxyUrl)} ${shQuote(proxy.noProxy.join(","))} ${shQuote(proxy.healthProbeUrl)} >"$tmp/egress-probe.out" 2>"$tmp/egress-probe.err" + egress_probe_rc=$? +else + egress_probe_rc=1 + printf '%s\\n' 'sub2api app pod not found' >"$tmp/egress-probe.err" +fi +`; + const egressProbeJson = proxy === null ? "None" : `{ + "enabled": True, + "mode": "master-vpn-subscription-http-proxy", + "deploymentName": "${proxy.deploymentName}", + "serviceName": "${proxy.serviceName}", + "proxyUrl": "${proxyUrl}", + "healthProbeUrl": "${proxy.healthProbeUrl}", + "applyToSub2Api": ${proxy.applyToSub2Api ? "True" : "False"}, + "applyToSentinel": ${proxy.applyToSentinel ? "True" : "False"}, + "deployment": { + "exitCode": egress_deploy_rc, + "ready": deployment_ready(load("egress-deploy")), + "stderr": text("egress-deploy.err", 2000), + }, + "service": { + "exitCode": egress_svc_rc, + "type": ((load("egress-svc") or {}).get("spec") or {}).get("type"), + "clusterIP": ((load("egress-svc") or {}).get("spec") or {}).get("clusterIP"), + "stderr": text("egress-svc.err", 2000), + }, + "probe": { + "exitCode": egress_probe_rc, + "stdout": text("egress-probe.out", 2000), + "stderr": text("egress-probe.err", 2000), + }, + "valuesPrinted": False, + }`; + const egressProbeOkExpr = proxy === null ? "True" : "(egress_deploy_rc == 0 and egress_svc_rc == 0 and deployment_ready(load('egress-deploy')) and egress_probe_rc == 0)"; + const publicProbeJson = exposure === null ? "None" : `{ + "enabled": True, + "publicBaseUrl": "${exposure.publicBaseUrl}", + "hostname": "${exposure.dns.hostname}", + "expectedA": "${exposure.dns.expectedA}", + "mode": "pk01-caddy-frp-direct", + "dataPath": "client -> PK01 Caddy -> PK01 frps remotePort -> D601 frpc -> Sub2API", + "dns": { + "exitCode": public_dns_rc, + "resolvers": ${JSON.stringify(exposure.dns.resolvers)}, + "stdout": text("public-dns.out", 2000), + "stderr": text("public-dns.err", 2000), + }, + "health": { + "exitCode": public_probe_rc, + "url": "${exposure.publicBaseUrl}/health", + "stdout": text("public-probe.out", 2000), + "stderr": text("public-probe.err", 2000), + }, + }`; + const publicProbeOkExpr = exposure === null ? "True" : "(public_dns_rc == 0 and public_probe_rc == 0)"; + return ` +set -u +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +capture_json() { + name="$1" + shift + "$@" -o json >"$tmp/$name.json" 2>"$tmp/$name.err" + rc=$? + printf '%s' "$rc" >"$tmp/$name.rc" +} +capture_json networkpolicy kubectl -n ${target.namespace} get networkpolicy allow-all +capture_json secret kubectl -n ${target.namespace} get secret ${secretName} +capture_json statefulsets kubectl -n ${target.namespace} get statefulsets +capture_json pvc kubectl -n ${target.namespace} get pvc +app_pod="$(kubectl -n ${target.namespace} get pod -l app.kubernetes.io/name=${serviceName},app.kubernetes.io/component=app -o jsonpath='{.items[0].metadata.name}' 2>"$tmp/app-pod.err" || true)" +redis_pod="$(kubectl -n ${target.namespace} get pod -l app.kubernetes.io/name=${redisService},app.kubernetes.io/component=redis -o jsonpath='{.items[0].metadata.name}' 2>"$tmp/redis-pod.err" || true)" +printf '%s' "$app_pod" >"$tmp/app-pod.txt" +printf '%s' "$redis_pod" >"$tmp/redis-pod.txt" +kubectl get --raw /api/v1/namespaces/${target.namespace}/services/${serviceName}:8080/proxy/health >"$tmp/health.body" 2>"$tmp/health.err" +health_rc=$? +kubectl get --raw /api/v1/namespaces/${target.namespace}/services/${serviceName}:8080/proxy/ >"$tmp/root.body" 2>"$tmp/root.err" +root_rc=$? +if [ -n "$app_pod" ]; then + kubectl -n ${target.namespace} exec "$app_pod" -- sh -c 'printf "DATABASE_HOST=%s\\n" "$DATABASE_HOST"; printf "DATABASE_PORT=%s\\n" "$DATABASE_PORT"; printf "DATABASE_USER=%s\\n" "$DATABASE_USER"; printf "DATABASE_DBNAME=%s\\n" "$DATABASE_DBNAME"; printf "DATABASE_SSLMODE=%s\\n" "$DATABASE_SSLMODE"; printf "REDIS_HOST=%s\\n" "$REDIS_HOST"; if [ -n "$DATABASE_PASSWORD" ]; then printf "DATABASE_PASSWORD_PRESENT=true\\n"; else printf "DATABASE_PASSWORD_PRESENT=false\\n"; fi' >"$tmp/app-env.out" 2>"$tmp/app-env.err" + printf '%s' "$?" >"$tmp/app-env.rc" +else + : >"$tmp/app-env.out" + printf '%s\\n' 'sub2api app pod not found' >"$tmp/app-env.err" + printf '%s' "1" >"$tmp/app-env.rc" +fi +if [ -n "$redis_pod" ]; then + kubectl -n ${target.namespace} exec "$redis_pod" -- redis-cli ping >"$tmp/redis.out" 2>"$tmp/redis.err" + redis_rc=$? +else + redis_rc=1 + printf '%s\\n' 'sub2api redis pod not found' >"$tmp/redis.err" +fi +${publicProbeBlock} +${egressProbeBlock} +python3 - "$tmp" "$health_rc" "$root_rc" "$redis_rc" "$public_dns_rc" "$public_probe_rc" "$egress_deploy_rc" "$egress_svc_rc" "$egress_probe_rc" <<'PY' +import json +import os +import sys + +tmp = sys.argv[1] +health_rc, root_rc, redis_rc, public_dns_rc, public_probe_rc, egress_deploy_rc, egress_svc_rc, egress_probe_rc = [int(value) for value in sys.argv[2:]] + +def rc(name): + try: + return int(open(os.path.join(tmp, f"{name}.rc"), encoding="utf-8").read() or "1") + except FileNotFoundError: + return 1 + +def load(name): + path = os.path.join(tmp, f"{name}.json") + if not os.path.exists(path): + return None + try: + return json.load(open(path, encoding="utf-8")) + except json.JSONDecodeError: + return None + +def items(name): + data = load(name) + if not isinstance(data, dict): + return [] + return data.get("items") or [] + +def text(name, limit=4000): + path = os.path.join(tmp, name) + try: + return open(path, encoding="utf-8", errors="replace").read()[-limit:] + except FileNotFoundError: + return "" + +def is_allow_all_network_policy(item): + if not isinstance(item, dict): + return False + spec = item.get("spec") or {} + return ( + item.get("metadata", {}).get("name") == "allow-all" + and spec.get("podSelector") == {} + and set(spec.get("policyTypes") or []) == {"Ingress", "Egress"} + and spec.get("ingress") == [{}] + and spec.get("egress") == [{}] + ) + +def deployment_ready(item): + if not isinstance(item, dict): + return False + spec = item.get("spec") or {} + status = item.get("status") or {} + desired = spec.get("replicas", 1) + return status.get("availableReplicas", 0) >= desired + +env = {} +for line in text("app-env.out").splitlines(): + if "=" not in line: + continue + key, value = line.split("=", 1) + env[key] = value + +network_policy_obj = load("networkpolicy") +network_policy_ok = rc("networkpolicy") == 0 and is_allow_all_network_policy(network_policy_obj) +secret = load("secret") +secret_keys = sorted(((secret or {}).get("data") or {}).keys()) +missing_secret_keys = [key for key in ${JSON.stringify(requiredSecretKeys)} if key not in secret_keys] +local_postgres_present = any(item.get("metadata", {}).get("name") == "sub2api-postgres" for item in items("statefulsets") + items("pvc")) +redis_pvc_present = any(item.get("metadata", {}).get("name") == "sub2api-redis-data" for item in items("pvc")) +env_aligned = ( + rc("app-env") == 0 + and env.get("DATABASE_HOST") == "${database.host}" + and env.get("DATABASE_PORT") == "${database.port}" + and env.get("DATABASE_USER") == "${database.user}" + and env.get("DATABASE_DBNAME") == "${database.dbName}" + and env.get("DATABASE_SSLMODE") == "${database.sslMode}" + and env.get("REDIS_HOST") == "${redisService}" + and env.get("DATABASE_PASSWORD_PRESENT") == "true" +) +payload = { + "ok": health_rc == 0 and root_rc == 0 and redis_rc == 0 and network_policy_ok and len(missing_secret_keys) == 0 and env_aligned and not local_postgres_present and not redis_pvc_present and ${publicProbeOkExpr} and ${egressProbeOkExpr}, + "target": "${target.id}", + "namespace": "${target.namespace}", + "status": "external-db-active", + "databaseMode": "${target.databaseMode}", + "externalDatabase": { + "host": "${database.host}", + "port": ${database.port}, + "user": "${database.user}", + "dbName": "${database.dbName}", + "sslMode": "${database.sslMode}", + "secretName": "${database.secretName}", + "passwordKey": "${database.passwordKey}", + "passwordPrinted": False, + }, + "publicExposure": ${publicProbeJson}, + "egressProxy": ${egressProbeJson}, + "checks": { + "allowAllNetworkPolicy": {"exitCode": rc("networkpolicy"), "ok": network_policy_ok, "stderr": text("networkpolicy.err", 2000)}, + "secretReady": {"ok": len(missing_secret_keys) == 0, "missingKeys": missing_secret_keys, "valuesPrinted": False}, + "sub2apiHealthViaKubernetesServiceProxy": { + "exitCode": health_rc, + "method": "kubectl get --raw /api/v1/namespaces/${target.namespace}/services/${serviceName}:8080/proxy/health", + "bodyPreview": text("health.body", 2000), + "stderr": text("health.err", 2000), + }, + "sub2apiRootViaKubernetesServiceProxy": { + "exitCode": root_rc, + "method": "kubectl get --raw /api/v1/namespaces/${target.namespace}/services/${serviceName}:8080/proxy/", + "bodyBytes": len(text("root.body").encode("utf-8")), + "bodyPreview": text("root.body", 400), + "stderr": text("root.err", 2000), + }, + "appEnvAligned": { + "ok": env_aligned, + "podName": text("app-pod.txt"), + "env": {key: env.get(key) for key in ["DATABASE_HOST", "DATABASE_PORT", "DATABASE_USER", "DATABASE_DBNAME", "DATABASE_SSLMODE", "REDIS_HOST", "DATABASE_PASSWORD_PRESENT"]}, + "stderr": text("app-env.err", 2000), + }, + "redisPing": {"exitCode": redis_rc, "podName": text("redis-pod.txt"), "stdout": text("redis.out", 2000), "stderr": text("redis.err", 2000)}, + "externalStateOnly": {"ok": not local_postgres_present and not redis_pvc_present, "localPostgresPresent": local_postgres_present, "redisPvcPresent": redis_pvc_present}, + }, +} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if payload["ok"] else 1) +PY +`; +} + function validateExternalPendingScript(sub2api: Sub2ApiConfig, target: Sub2ApiTargetConfig): string { - const expectedImage = imageRef(sub2api); + const expectedImage = imageRef(sub2api, target); const database = sub2api.runtime.database; const redisService = sub2api.runtime.redis.serviceName; return ` @@ -1913,6 +3834,10 @@ function boolField(value: Record | null, key: string, defaultVa return typeof field === "boolean" ? field : defaultValue; } +function asRecordOrNull(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; +} + function compactCapture(result: SshCaptureResult, options: { full?: boolean } = {}): Record { const full = options.full ?? false; return {