diff --git a/AGENTS.md b/AGENTS.md index 4ebda137..b981e467 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,8 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - P0: UniDesk 自有配置一律优先使用 YAML（`.yaml`/`.yml`），包括 `config/` 下的运行面、平台基础设施、节点/lane、部署参数和可调版本配置；除非外部工具硬性要求 JSON/TOML/ENV 等格式，禁止新增 JSON 作为 UniDesk 自有配置真相。 - P0: 需要代码读取的 YAML 配置必须显式校验格式、字段类型和必填项；配置校验只保证“能被正确读取和渲染”，不得把业务策略、调度策略或数值选择写成代码硬编码、schema 硬范围、合同测试或隐藏默认值。后续版本、镜像、namespace、endpoint、容量、冷却时间、退避窗口等可调项必须从 YAML 配置进入受控 CLI，具体数值以 YAML 为准。 +- P0: UniDesk 自有平台服务的密钥、密码、API key、JWT/encryption key 和 `DATABASE_URL` 等凭据绑定，必须采用 YAML 声明 sourceRef/targetKey 并通过受控 CLI 下发；运行面 Kubernetes Secret、pod env、日志或数据库状态只能作为 presence/fingerprint/health 观测对象，禁止作为 source of truth 反推、解码、回填或再生成本地凭据。 +- P0: 受控密钥下发 CLI 的输出只能披露对象名、key 名、sourceRef、缺失项、fingerprint、字节数和执行摘要；禁止打印 base64 payload、解码值、完整 DSN、可复制凭据或远端 raw transcript。缺少密钥时修 YAML/sourceRef/上游 Secret 生成入口，再执行受控 sync/apply，不得从运行面倒推补值。 - P0: YAML-first 异构分布式运维架构、现有 YAML 归属优先、禁止硬编码 node/service、公共 ops 层抽取和薄 domain CLI 规则见 `docs/reference/yaml-first-ops.md`。 ## P0 最高优先级：G14 platform-infra 规则 @@ -228,6 +230,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - `bun scripts/cli.ts server rebuild `：以 build-first、Compose lock、no-deps force-recreate 和 post-up validation 的异步 job 重建主 server Compose 内单个服务；对 database、File Browser、Code Queue 执行面、k3sctl-adapter 或未知对象返回结构化 `unsupported-server-rebuild`，规则见 `docs/reference/deployment.md` 与 `docs/reference/cicd-standardization.md`。 - `bun scripts/cli.ts provider attach [--master-server URL] [--up] [--force]` / `bun scripts/cli.ts provider triage [--observed-error text] [--observed-scope scope] [--microservice id ...] [--full|--raw]`：前者在新增计算节点上生成两项配置的 provider-gateway 挂载包；后者是只读多信号健康裁决入口，默认低噪声输出 `decision`、`healthyScopes`、`failedScopes`、`retryable` 和异常信号摘要，用来把单路径 `provider is not online`、SSH 超时、registry 失败或 proxy 失败归类为 `retryable-transient`、`service-degraded` 或 `global-offline`，完整 evidence 需显式 `--full|--raw`，规则见 `docs/reference/provider-gateway.md` 和 `docs/reference/code-queue-supervision.md`。 - `bun scripts/cli.ts platform-db postgres plan|status|apply --config config/platform-db/postgres-pk01.yaml`：管理 PK01 host-native PostgreSQL 16 外置平台库、TLS、Secret 导出和备份；跨节点消费者直连 YAML 声明的 PK01 公网 endpoint，不经 master server 中转，规则见 `docs/reference/pk01.md`。 +- `bun scripts/cli.ts secrets plan|sync|status --config config/secrets-distribution.yaml --scope platform-infra`：按 YAML 声明把本地 Secret sourceRef 下发到 G14 `platform-infra` Kubernetes Secret key，禁止从运行面反推密码/API key，规则见 `docs/reference/platform-infra.md#secret-distribution-boundary`。 - `trans [operation args...]` / `tran [operation args...]`：通过 provider-gateway 的 Host SSH / WSL SSH 维护桥进入 provider、host workspace、Windows cmd route、k3s 控制面或 pod workspace，并提供带 SHA-256 校验的 `upload`/`download` 文件传输；主 server 人工/Codex 分布式操作必须优先用本机 `trans` wrapper，`tran` 只作为兼容入口，细则见 `docs/reference/cli.md`、`docs/reference/windows-passthrough.md` 和 `docs/reference/provider-gateway.md`。 - `bun scripts/cli.ts microservice list/status/health/diagnostics/tunnel-self-test/proxy`：管理和验证挂载在主 server、计算节点 Docker 或 k3s 控制面上的用户服务，`status/health/diagnostics` 默认 compact summary 并用 `--full|--raw` 展开完整 body，`proxy` 支持受控 JSON body，OA Event Flow/Todo Note/Baidu Netdisk/Code Queue Manager on main-server、k3s Control/Code Queue 执行面/MDTODO/Decision Center/FindJob/Pipeline/MET Nonlinear on D601 的规则见 `docs/reference/microservices.md`。 - `bun scripts/cli.ts microservice health/diagnostics/proxy code-agent-sandbox`：验证独立 Code Agent Sandbox 的 health、只读 diagnostics、trace 和 adapter/mode/credential boundary 契约，规则见 `docs/reference/code-agent-sandbox.md`。 @@ -280,7 +283,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - `docs/reference/g14.md`：G14 provider 节点、k3s 控制桥、legacy DEV/PROD 退役边界、当前 HWLAB runtime lane、device-agent 手动实验边界、Code Queue/CI 候选目标和节点本地 VPN proxy bootstrap 边界。 - `docs/reference/pk01.md`：PK01 腾讯云 provider-gateway、pikanode/MET Docker workload、SSH 透传、磁盘 GC 和 pikanode temp 长效 retention 边界。 - `docs/reference/yaml-first-ops.md`：YAML-first 异构分布式运维架构、现有 YAML 归属优先、公共 ops 层抽取、禁止硬编码 node/service 和薄 domain CLI 规则。 -- `docs/reference/platform-infra.md`：G14 `platform-infra` namespace、YAML-first shared service 配置、Sub2API/Codex pool、FRP 暴露和 on-demand availability probe 开发边界；Sub2API 日常操作统一见 `$unidesk-sub2api`（`.agents/skills/unidesk-sub2api/SKILL.md`）。 +- `docs/reference/platform-infra.md`：G14 `platform-infra` namespace、YAML-first shared service 配置、YAML-controlled Secret distribution/no runtime reverse inference、Sub2API/Codex pool、FRP 暴露和 on-demand availability probe 开发边界；Sub2API 日常操作统一见 `$unidesk-sub2api`（`.agents/skills/unidesk-sub2api/SKILL.md`）。 - `docs/reference/master-server-ops.md`：主 server 本机 Codex profile wrapper、ACX/GOCX/Moon Bridge 路由边界、默认模型、真实调用验收和 MiniMax session recovery 规则。 - `docs/reference/g14-observability-infra.md`：G14 原生 k3s 上 Prometheus Operator、`devops-infra` 监控基础设施、跨 namespace scrape 声明和安全边界。 - `docs/reference/gc.md`：UniDesk 主 server 和 provider 磁盘 GC、G14/HWLAB registry retention、safe-stop 线和长期防膨胀收益规则。 diff --git a/config/platform-infra/wechat-archive.yaml b/config/platform-infra/wechat-archive.yaml index d49b6b94..9dffd921 100644 --- a/config/platform-infra/wechat-archive.yaml +++ b/config/platform-infra/wechat-archive.yaml @@ -17,6 +17,18 @@ langbot: publicBaseUrl: https://langbot.pikapython.com expectedAdapter: openclaw-weixin callbackPath: /callback/command + readOnlyRecord: + enabled: true + mode: webhook-skip-pipeline + webhookName: UniDesk WeChat Archive Read Only + webhookDescription: Archive LangBot/OpenClaw inbound messages and return skip_pipeline=true so personal WeChat never receives automated replies. + discardPipelineUuid: __discard__ + pipeline: + name: UniDesk WeChat Baidu Archive + description: Forward WeChat/OpenClaw messages to the UniDesk n8n archive workflow. + runner: n8n-service-api + outputKey: response + timeoutSeconds: 120 notes: - LangBot receives WeChat/OpenClaw messages and forwards archive events to the n8n workflow webhook. - Credentials for real WeChat channels are bound in LangBot runtime and are not stored in this YAML. @@ -31,6 +43,13 @@ n8n: active: true timezone: Asia/Shanghai +archiveCallback: + publicUrl: http://74.48.78.17:18081/webhooks/wechat-archive + secretRoot: /root/unidesk/.state/secrets + tokenSourceRef: platform-infra/wechat-archive.env + tokenKey: UNIDESK_WECHAT_ARCHIVE_TOKEN + timeoutMs: 90000 + baiduNetdisk: serviceId: baidu-netdisk proxyMode: backend-core-microservice-proxy diff --git a/config/secrets-distribution.yaml b/config/secrets-distribution.yaml new file mode 100644 index 00000000..bdd4527f --- /dev/null +++ b/config/secrets-distribution.yaml @@ -0,0 +1,105 @@ +version: 1 +kind: unidesk-secret-distribution + +metadata: + id: platform-infra-runtime-secrets + owner: unidesk + relatedIssues: + - 297 + - 300 + - 313 + +sources: + root: /root/unidesk/.state/secrets + files: + - sourceRef: platform-db/langbot-db.env + type: env + requiredKeys: + - LANGBOT_DB_USER + - LANGBOT_DB_PASSWORD + - LANGBOT_DB_NAME + createIfMissing: + enabled: false + - sourceRef: platform-db/n8n-db.env + type: env + requiredKeys: + - N8N_DB_USER + - N8N_DB_PASSWORD + - N8N_DB_NAME + createIfMissing: + enabled: false + - sourceRef: platform-infra/langbot.env + type: env + requiredKeys: + - DATABASE_URL + - LANGBOT_JWT_SECRET + - LANGBOT_API_KEY + createIfMissing: + enabled: true + randomHex: + LANGBOT_JWT_SECRET: 32 + randomBase64Url: + LANGBOT_API_KEY: + bytes: 32 + prefix: lbk_ + - sourceRef: platform-infra/n8n.env + type: env + requiredKeys: + - DATABASE_URL + - N8N_ENCRYPTION_KEY + createIfMissing: + enabled: true + randomBase64Url: + N8N_ENCRYPTION_KEY: + bytes: 32 + prefix: "" + - sourceRef: platform-infra/wechat-archive.env + type: env + requiredKeys: + - UNIDESK_WECHAT_ARCHIVE_TOKEN + createIfMissing: + enabled: true + randomBase64Url: + UNIDESK_WECHAT_ARCHIVE_TOKEN: + bytes: 32 + prefix: uwa_ + +targets: + - id: platform-infra-g14 + route: G14:k3s + namespace: platform-infra + scope: platform-infra + enabled: true + +kubernetesSecrets: + - name: langbot-runtime + targetId: platform-infra-g14 + secretName: langbot-secrets + type: Opaque + data: + - sourceRef: platform-db/langbot-db.env + sourceKey: LANGBOT_DB_PASSWORD + targetKey: DATABASE_PASSWORD + - sourceRef: platform-infra/langbot.env + sourceKey: LANGBOT_JWT_SECRET + targetKey: SYSTEM_JWT_SECRET + - sourceRef: platform-infra/langbot.env + sourceKey: LANGBOT_API_KEY + targetKey: LANGBOT_API_KEY + - sourceRef: platform-infra/langbot.env + sourceKey: DATABASE_URL + targetKey: DATABASE_URL + - name: n8n-runtime + targetId: platform-infra-g14 + secretName: n8n-secrets + type: Opaque + data: + - sourceRef: platform-db/n8n-db.env + sourceKey: N8N_DB_PASSWORD + targetKey: DB_POSTGRESDB_PASSWORD + - sourceRef: platform-infra/n8n.env + sourceKey: N8N_ENCRYPTION_KEY + targetKey: N8N_ENCRYPTION_KEY + - sourceRef: platform-infra/n8n.env + sourceKey: DATABASE_URL + targetKey: DATABASE_URL diff --git a/docker-compose.yml b/docker-compose.yml index 4479514e..6ae21825 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -390,6 +390,7 @@ services: PROVIDER_TOKEN: "${UNIDESK_PROVIDER_TOKEN}" UNIDESK_SSH_CLIENT_TOKEN: "${UNIDESK_SSH_CLIENT_TOKEN:-}" UNIDESK_SSH_CLIENT_ROUTE_ALLOWLIST: "${UNIDESK_SSH_CLIENT_ROUTE_ALLOWLIST:-G14,G14:*,D601,D601:*}" + UNIDESK_WECHAT_ARCHIVE_TOKEN: "${UNIDESK_WECHAT_ARCHIVE_TOKEN:-}" SESSION_SECRET: "${UNIDESK_SESSION_SECRET}" SESSION_TTL_SECONDS: "${UNIDESK_SESSION_TTL_SECONDS}" UNIDESK_DEPLOY_REF: "${UNIDESK_FRONTEND_DEPLOY_REF:-deploy.json#environments.prod.services.frontend}" diff --git a/docs/reference/cli.md b/docs/reference/cli.md index d7cf9675..3fde2132 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -76,6 +76,7 @@ CI/CD、GitOps、rollout、artifact 发布、PR 合并后的 runtime lane 滚动 `status` 返回 read/write URL、last sync/write/flush、本地 ref、GitHub staging ref 和 pending flush 状态，并在 `cache.summary` 给出 `localV02`、`localGitops`、`githubGitops`、`pendingFlush`、`flushNeeded`、`githubInSync` 和下一条受控 `flushCommand`。confirmed `sync` 和 `flush` 默认创建 `.state/jobs/` 异步 job 并立刻返回可查询状态，只有现场同步调试才显式加 `--wait`；mirror 不设置 CronJob。 如果 PipelineRun 的 `gitops-promote` 阶段报 git mirror 控制面漂移或 refs 不一致，先执行 `hwlab g14 git-mirror apply --confirm` 重新应用当前 `devops-infra/git-mirror.yaml` hook/ConfigMap，再执行 `hwlab g14 git-mirror sync --confirm --wait` 复核 refs；失败的同名 PipelineRun 只能通过 `hwlab g14 control-plane cleanup-runs --lane --pipeline-run --confirm` 受控清理后重试，不要用原生 `kubectl delete` 或手工改 mirror hook。修复后仍必须用 `control-plane status --pipeline-run ` 和 `git-mirror status` 分别确认 runtime closeout 与 GitHub flush。 - `platform-infra sub2api|langbot|n8n|wechat-archive ...` 是 `platform-infra` namespace 内平台公共服务和公共工作流的受控入口；`sub2api` 支持 `plan|apply|status|validate|codex-pool`，`langbot` 和 `n8n` 支持各自 YAML-controlled `plan|apply|status|logs|validate` 等公共服务操作，`wechat-archive` 支持 `plan|apply|status|validate|pull`，用 `config/platform-infra/wechat-archive.yaml` 声明 LangBot/n8n/Baidu Netdisk 归档链路。`--target` 选择运行目标，默认 `G14` 为 active runtime，`D601` 为同一 YAML 控制的 standby predeploy target。镜像版本和 target 边界由 `config/platform-infra/*.yaml` 控制，Codex 上游池、统一 API key Secret、FRP 公网端口和 master `~/.codex` 消费端由 `config/platform-infra/sub2api-codex-pool.yaml` 控制；完整 Sub2API 日常部署、上游增删、FRP 暴露、local Codex 配置、验收和排障步骤统一见 `$unidesk-sub2api`（`.agents/skills/unidesk-sub2api/SKILL.md`）。`docs/reference/platform-infra.md` 保留 namespace、YAML-first、路由、Secret 脱敏、PK01 Caddy+FRP 和探针开发边界。 +- `secrets plan|sync|status --config config/secrets-distribution.yaml --scope platform-infra` 是平台服务本地 Secret sourceRef 到 Kubernetes Secret key 的受控下发入口。`plan` 只读展示 sourceRef、必需 key、目标 Secret/key、missingKeys 和 fingerprint；`sync --confirm` 只按 YAML 声明创建允许生成的本地 key 并下发到声明的目标 Secret；`status` 只验证 live Secret key presence。该入口禁止从 live pod 或 Kubernetes Secret 反推密码、API key、JWT secret、n8n encryption key 或 `DATABASE_URL`，输出也不得打印 base64、解码值或远端 raw transcript；即使显式 `--raw` 也只返回脱敏 summary 和 raw omission 标记。LangBot/n8n Secret 轮换和缺 key 修复规则见 `docs/reference/platform-infra.md#secret-distribution-boundary`。 - `hwlab g14 observability status|apply|query|targets|boundary|closeout [--lane v02] [--promql ] [--expect-count N] [--expect-value V] [--dry-run|--confirm]` 是 G14 `devops-infra` 共享监控基础设施和 HWLAB v0.2 监控 closeout 的受控入口。`apply` 固定安装 Prometheus Operator `v0.91.0`、Prometheus `v3.12.0`、Prometheus 发现 RBAC、`devops-infra` 内 Prometheus 实例和 ClusterIP query Service，并给被允许发现的 workload namespace 打低风险 label；它不把 Prometheus、Grafana 或 Alertmanager 部署到 `hwlab-v02`，也不接管 HWLAB runtime Deployment/Service。`status` 只读汇总 CRD、operator Deployment、Prometheus CR/pod/service、`hwlab-v02` ServiceMonitor/PrometheusRule 和 bounded `up` 查询；`query` 只通过 Kubernetes service proxy 查询 Prometheus，支持 `--expect-count` / `--expect-value` 输出 `assertion`、bad values 和 missing/extra series；`targets` 汇总 ServiceMonitor/PrometheusRule、metrics sidecar readiness/restart、三层指标值和 `metrics.k8s.io` 当前 CPU/内存资源快照；`boundary` 验证 workload namespace 没有 Prometheus/Alertmanager，并对 `19666/19667` 公网 `/metrics` 做负向验证；`closeout` 聚合平台 ready、scrape reachable、sidecar serving、business health probe、resource snapshot、namespace boundary 和 public metrics exposure 语义结论。长期边界见 `docs/reference/g14-observability-infra.md`。 - `hwlab g14 tools-image status|build --name ci-node-tools --tag [--dockerfile deploy/ci/hwlab-ci-node-tools.Dockerfile] [--dry-run|--confirm]` 是 G14 固定 HWLAB CI tools image 的受控 host build/push 入口；构建和 push 只发生在 G14 host 与本地 registry，不在 master server 构建，也不把 `apk add`/runtime install 塞进 Tekton PipelineRun。 - `trans gh:/owner/repo ...` 把 GitHub issue/PR 映射成只读/受控写入的虚拟文本目录，适合日报、PR 正文和 issue 正文的小补丁维护：`trans gh:/pikasTech/HWLAB ls` 展示 `pr/` 与 `issue/`，`trans gh:/pikasTech/HWLAB/pr ls [--limit N] [--full]` 和 `trans gh:/pikasTech/HWLAB/issue ls [--limit N] [--full]` 展示条目状态、楼层数、正文长度和标题，`trans gh:/pikasTech/HWLAB/pr/507 ls` 展示单个 PR 的一楼正文文件，`trans gh:/pikasTech/HWLAB/505/1 cat|rg|patch-apply` 兼容旧式 issue/PR number route。`patch-apply` 使用 UniDesk 默认 apply-patch v2 的虚拟文件 executor，把正文一楼映射为 `body.md`，写回仍走 `bun scripts/cli.ts gh issue/pr update` 的 guard/concurrency 规则；`rm` 对正文一楼结构化拒绝，避免误删 issue/PR 正文。大正文读取必须展开 UniDesk gh dump 文件，否则 `cat/rg/patch-apply` 会误读为空，这是 `gh:` 虚拟文件接口的 P0 可见性契约。 diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 91109ec9..77deb2dd 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -9,6 +9,17 @@ - Code that reads platform YAML must validate object shape, field types, required fields, Kubernetes names, image strings, and ports before mutating G14 k3s or local consumer files. - Do not hide image versions, namespace names, endpoint URLs, FRP ports, or profile lists in Python/TOML/JSON helper constants when they are UniDesk-owned choices. External tools may still require their own TOML/JSON/env file formats at the edge. +## Secret Distribution Boundary + +- UniDesk-owned platform service credential distribution must be YAML-controlled: declare the sourceRef, source key, target Secret, and target key first, then use the controlled CLI to sync/apply it. Runtime Kubernetes Secrets, pod env, logs, and database state are observation surfaces, not credential source of truth. +- `config/secrets-distribution.yaml` is the current shared distribution map and the canonical entrypoint is `bun scripts/cli.ts secrets plan|sync|status --config config/secrets-distribution.yaml --scope platform-infra`. +- The YAML maps local secret source files under the declared `sources.root` to target Kubernetes Secret names and keys. It is the source of authority for LangBot/n8n runtime Secret handoff and the pattern for future platform services; do not reverse-engineer passwords, API keys, JWT/encryption keys, database passwords or `DATABASE_URL` values from live pods or existing Kubernetes Secrets. +- `secrets plan` is read-only and may show sourceRef paths, required key names, generated-key intent, target Secret names, target keys, presence, missing keys and fingerprints. `secrets sync --confirm` may create missing local generated keys only when YAML explicitly allows `createIfMissing`; database passwords exported by `platform-db postgres` are not regenerated here. `secrets status` verifies live Secret key presence without decoding values. +- CLI output for Secret distribution may disclose key names, object names, sourceRef names, byte/count-style metadata and fingerprints only. It must not print base64 payloads, decoded values, full `DATABASE_URL`, API keys, JWT secrets, encryption keys, database passwords, copy-pastable credential mutation commands or remote raw transcripts. +- Service-specific `platform-infra apply` commands may read the declared local sourceRef files to render/apply runtime Secrets, but they must not infer missing values from the current runtime. If required local source keys are absent, the durable fix is the owning YAML/sourceRef/Secret generation entrypoint followed by `secrets sync` or the service apply path, not a runtime reverse lookup. +- When a runtime Secret already contains a value that is missing locally, treat that as drift to resolve through declared source authority. Do not decode it for local repair, do not copy it into YAML or env files, and do not make live Secret contents the bootstrap source for a new service. +- If a platform CLI, service error, log, issue, trace, or terminal transcript exposes a credential value, treat that credential as compromised. Rotate it from the declared YAML/sourceRef authority, push it through `secrets sync` and the relevant service `apply`/bootstrap entrypoint, then revoke stale service-side API keys or tokens without printing old or new values. + ## Sub2API Deployment Boundary - Sub2API is a platform service operated by UniDesk in namespace `platform-infra`. It is not a HWLAB lane workload, AgentRun workload, D601 user service, or master server daemon. @@ -26,6 +37,7 @@ - LangBot is a UniDesk-operated public platform service in namespace `platform-infra`. The canonical entrypoint is `bun scripts/cli.ts platform-infra langbot plan|apply|status|logs|validate|bootstrap-api-key|query`; G14 is the default runtime target. - LangBot configuration is YAML-first in `config/platform-infra/langbot.yaml`. Image tag, target namespace, PVCs, PK01 Caddy/FRP exposure, API key seed source, and official WeChat adapter metadata must stay in YAML rather than helper constants or manual runtime patches. +- LangBot runtime Secret handoff uses `config/secrets-distribution.yaml` and `bun scripts/cli.ts secrets ... --scope platform-infra`. `platform-infra langbot apply` must not create hidden passwords or reverse-read live Kubernetes Secret values to fill missing local source keys. - LangBot uses the existing PK01 host-native PostgreSQL instance through `config/platform-db/postgres-pk01.yaml` and `platform-db postgres`. Adding LangBot state means adding a dedicated database and role inside that existing instance; do not deploy a second PostgreSQL StatefulSet, container, or external DB instance for LangBot. - Public exposure uses PK01 Caddy plus FRP to the G14 ClusterIP service. Do not add Kubernetes Ingress, NodePort, LoadBalancer, host networking, or host ports for LangBot unless a later YAML-controlled platform decision changes the exposure model. - LangBot's built-in Web frontend and API share the same public HTTPS origin. CLI queries must use the YAML-declared API key source and must report key names/fingerprints only, never the API key value. @@ -40,6 +52,7 @@ - n8n is the UniDesk-operated workflow/automation layer for LangBot and platform service integration. It is a workflow bridge for webhook orchestration, service calls, manual approval flows and external integrations; it does not replace LangBot or become the chat runtime. - The canonical entrypoint is `bun scripts/cli.ts platform-infra n8n plan|apply|status|logs|validate`; G14 is the default runtime target and `config/platform-infra/n8n.yaml` is the YAML source of truth. +- n8n runtime Secret handoff uses `config/secrets-distribution.yaml` and `bun scripts/cli.ts secrets ... --scope platform-infra`. `platform-infra n8n apply` must not create hidden encryption keys or reverse-read live Kubernetes Secret values to fill missing local source keys. - n8n uses the existing Pika01/PK01 host-native PostgreSQL instance through `config/platform-db/postgres-pk01.yaml` and `platform-db postgres`. Adding n8n state means adding a dedicated `n8n` database and role inside that single external PostgreSQL instance; do not deploy an in-cluster PostgreSQL StatefulSet, a second PostgreSQL instance, or long-term SQLite state for n8n. - Public exposure uses PK01 Caddy plus FRP to the G14 ClusterIP service at `https://n8n.pikapython.com`. Do not add Kubernetes Ingress, NodePort, LoadBalancer, host networking, or host ports for n8n unless a later YAML-controlled platform decision changes the exposure model. - n8n reverse-proxy and webhook settings such as public base URL, `WEBHOOK_URL`, proxy hop trust and PostgreSQL connection fields must be rendered from YAML. Secret output may show key names, presence and fingerprints only; it must not print the database password, `N8N_ENCRYPTION_KEY`, or full `DATABASE_URL`. @@ -50,7 +63,11 @@ - WeChat-to-Baidu archive automation is a shared platform workflow, not a separate service-specific fork. Its durable source of truth is `config/platform-infra/wechat-archive.yaml`; the canonical entrypoint is `bun scripts/cli.ts platform-infra wechat-archive plan|apply|status|validate|pull`. - The workflow composes the existing LangBot public service, existing n8n public service, and the private `baidu-netdisk` microservice. LangBot remains the chat ingress, n8n owns webhook normalization/orchestration, and Baidu upload/download is performed through backend-core microservice proxy so Baidu OAuth tokens are never exposed in G14 or CLI output. - Text and image archive policy, remote path templates, staging roots, webhook path, timeout and validation fixtures must stay in YAML. CLI code may validate the YAML shape and render n8n workflow JSON, but it must not hard-code current path roots, credentials, message channel IDs, or Baidu account choices outside YAML/service runtime. +- The archive callback token is controlled by `archiveCallback.secretRoot`, `archiveCallback.tokenSourceRef`, and `archiveCallback.tokenKey` in YAML plus `config/secrets-distribution.yaml`. `secrets sync` may create the local source when YAML explicitly allows it; n8n receives the token only through controlled workflow rendering. Do not recover this token from the n8n database, frontend runtime, Baidu runtime, pod env, or logs. - For the current n8n runtime, production webhook reachability uses the registered path shape `workflowId/nodeName/webhookPath`; workflow node names used in generated webhooks should be ASCII path-safe, and `webhookPath` in YAML should remain one relative path segment. +- Generated n8n workflows should use n8n-native HTTP Request nodes for outbound service callbacks. Code nodes may normalize payloads, but must not assume sandbox globals such as `fetch` exist in the runtime. +- Personal WeChat ingestion must be read-only. The durable shape is a YAML-declared LangBot inbound webhook that mirrors messages to the archive workflow and returns `skip_pipeline=true`; the OpenClaw/LangBot bot must also have discard routing as fallback so webhook failure does not produce an automated reply. Do not connect personal WeChat through a normal reply pipeline, do not enable send-message surfaces for this purpose, and do not treat a successful archive upload as permission to reply. +- If LangBot or n8n public HTTPS fails while in-cluster service and FRP local-port probes are healthy, restore the PK01 Caddy managed blocks through `platform-infra langbot apply --confirm --wait` or `platform-infra n8n apply --confirm --wait`. Do not manually edit Caddy as the durable fix. - The archive uses the same single PK01/Pika01 PostgreSQL instance indirectly through the existing LangBot and n8n databases. Adding this workflow must not create another PostgreSQL instance, in-cluster PostgreSQL StatefulSet, or ad hoc database namespace. - `platform-infra-wechat-archive` and future similar public workflow CLIs should reuse the common platform-infra operations library for YAML parsing, target selection, workflow sync, private microservice proxy calls, transfer polling, staging path mapping, redaction and bounded output. Service-specific modules should keep only their business mapping and workflow payload rendering. - Closeout requires `platform-infra wechat-archive apply --confirm --wait`, `platform-infra wechat-archive status`, `platform-infra wechat-archive validate --full`, and a `platform-infra wechat-archive pull` command that retrieves an uploaded file by remote path or `fsId` and reports local path plus hash. diff --git a/scripts/cli.ts b/scripts/cli.ts index 128fec6c..3ec1ba5e 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -25,6 +25,7 @@ import { isHelpToken, rootHelp, serverHelp, sshHelp, staticNamespaceHelp } from import { runServerCleanupCommand } from "./src/server-cleanup"; import { runGcCommand } from "./src/gc"; import { runPlatformDbCommand } from "./src/platform-db"; +import { runSecretsCommand } from "./src/secrets"; const remoteOptions = extractRemoteCliOptions(process.argv.slice(2)); const args = remoteOptions.args; @@ -348,6 +349,14 @@ async function main(): Promise { return; } + if (top === "secrets") { + const result = await runSecretsCommand(readConfig(), args.slice(1)); + const ok = (result as { ok?: unknown }).ok !== false; + emitJson(commandName, result, ok); + if (!ok) process.exitCode = 1; + return; + } + const config = readConfig(); const autoRemoteCiPublishPlan = autoRemoteCiPublishUserServiceDryRunPlan(config, args); if (autoRemoteCiPublishPlan.enabled && autoRemoteCiPublishPlan.host !== null) { diff --git a/scripts/src/help.ts b/scripts/src/help.ts index a2d04e54..a4a8faaf 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -1,6 +1,7 @@ import { ghHelp } from "./gh"; import { authBrokerHelp } from "./auth-broker"; import { platformDbHelp } from "./platform-db"; +import { secretsHelp } from "./secrets"; export function rootHelp(): unknown { return { @@ -61,6 +62,7 @@ export function rootHelp(): unknown { { command: "hwlab g14 monitor-prs | hwlab g14 control-plane status|apply|trigger-current|runtime-migration|cleanup-runs|cleanup-released-pvs | hwlab g14 git-mirror status|apply|sync|flush | hwlab g14 tools-image status|build", description: "Start the legacy G14 PR monitor, run bounded v0.2 Tekton/Argo control-plane, manual PipelineRun trigger, runtime migration, CI workspace retention, manual devops-infra git mirror/relay maintenance, or fixed HWLAB CI tools image actions; long confirmed trigger/sync/flush actions return async jobs by default." }, { command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send|control-plane|git-mirror", description: "Use AgentRun v0.1 resource primitives with low-noise human output by default; session follow-up uses send only and the server decides internal steer vs turn." }, { command: "platform-infra sub2api|langbot|n8n|wechat-archive ...", description: "Deploy platform-infra services such as Sub2API, LangBot and n8n, manage YAML-controlled public FRP/Caddy exposure and WeChat archive workflows, and inspect status/logs without printing secrets." }, + { command: "secrets plan|sync|status", description: "Plan, push and inspect YAML-declared local secret source keys to Kubernetes Secrets without printing or reverse-engineering values." }, { command: "platform-db postgres plan|status|apply", description: "Manage YAML-declared host-native PostgreSQL 16 on PK01 for Sub2API/platform state, with PostgreSQL native TLS and redacted secret exports." }, { command: "hwlab cd audit --env dev | hwlab cd status --env dev | hwlab cd apply --env dev --dry-run", description: "Legacy D601 HWLAB DEV CD wrapper kept for explicit old-path diagnostics; current HWLAB rollout uses G14 GitOps." }, { command: "code-agent-sandbox", description: "Independent Code Agent Sandbox service skeleton for adapter, mode, and credential-boundary diagnostics." }, @@ -715,6 +717,7 @@ export async function staticNamespaceHelp(args: string[]): Promise (await import("./agentrun")).agentRunHelp(), agentRunHelpSummary()); if (top === "platform-infra") return loadHelp(async () => (await import("./platform-infra")).platformInfraHelp(), platformInfraHelpSummary()); if (top === "platform-db") return platformDbHelp(); + if (top === "secrets") return secretsHelp(); if (top === "hwlab" && (sub === "node" || sub === "nodes") && args[2] === "control-plane" && args[3] === "infra") { return loadHelp(async () => (await import("./hwlab-node-control-plane")).hwlabNodeControlPlaneInfraHelp(), hwlabNodeHelpSummary()); } diff --git a/scripts/src/platform-infra-langbot.ts b/scripts/src/platform-infra-langbot.ts index 86c55b5e..b6ca15b4 100644 --- a/scripts/src/platform-infra-langbot.ts +++ b/scripts/src/platform-infra-langbot.ts @@ -1,6 +1,6 @@ -import { createHash, randomBytes } from "node:crypto"; -import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, isAbsolute, join } from "node:path"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { isAbsolute, join } from "node:path"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; import { startJob } from "./jobs"; @@ -116,7 +116,7 @@ interface SecretMaterial { dbSourcePath: string; appSourceRef: string; appSourcePath: string; - action: "create" | "update" | "none"; + action: "read"; values: { dbUser: string; dbPassword: string; @@ -423,6 +423,7 @@ function plan(options: CommonOptions): Record { resourcePolicy: "No Kubernetes CPU/memory requests or limits are rendered.", }, next: { + secrets: "bun scripts/cli.ts secrets sync --config config/secrets-distribution.yaml --scope platform-infra --confirm", dryRun: `bun scripts/cli.ts platform-infra langbot apply --target ${target.id} --dry-run`, apply: `bun scripts/cli.ts platform-infra langbot apply --target ${target.id} --confirm`, status: `bun scripts/cli.ts platform-infra langbot status --target ${target.id}`, @@ -488,6 +489,7 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise existing[key] !== next[key]) ? "update" : "none"; - if (action !== "none") writeEnvFile(appSourcePath, next); + if (!existsSync(appSourcePath)) throw new Error(`LangBot app secret source ${redactRepoPath(appSourcePath)} is missing; run bun scripts/cli.ts secrets sync --config config/secrets-distribution.yaml --scope platform-infra --confirm first`); + const appValues = parseEnvFile(readFileSync(appSourcePath, "utf8")); + const jwtSecret = requiredEnvValue(appValues, "LANGBOT_JWT_SECRET", langbot.runtime.secrets.appSourceRef); + const apiKey = requiredEnvValue(appValues, langbot.apiKey.key, langbot.apiKey.sourceRef); const values = { dbUser, dbPassword, dbName, - jwtSecret: next.LANGBOT_JWT_SECRET, - apiKey: next[langbot.apiKey.key], + jwtSecret, + apiKey, }; return { dbSourceRef: langbot.runtime.database.sourceRef, dbSourcePath: redactRepoPath(dbSourcePath), appSourceRef: langbot.runtime.secrets.appSourceRef, appSourcePath: redactRepoPath(appSourcePath), - action, + action: "read", values, fingerprint: fingerprintValues({ dbPassword, jwtSecret: values.jwtSecret, apiKey: values.apiKey }, ["dbPassword", "jwtSecret", "apiKey"]), valuesPrinted: false, }; } +export function readLangBotRuntimeConfig(): Record { + const langbot = readLangBotConfig(); + const target = resolveTarget(langbot, "G14"); + return { + publicBaseUrl: target.publicExposure.publicBaseUrl, + expectedNamespace: target.namespace, + apiKeyName: langbot.apiKey.key, + valuesPrinted: false, + }; +} + +export function readLangBotSecretMaterial(): Record { + const langbot = readLangBotConfig(); + const secret = prepareSecretMaterial(langbot); + return { + apiKey: secret.values.apiKey, + apiKeyFingerprint: fingerprintValues({ [langbot.apiKey.key]: secret.values.apiKey }, [langbot.apiKey.key]), + sourceRef: langbot.apiKey.sourceRef, + keyName: langbot.apiKey.key, + valuesPrinted: false, + }; +} + function prepareFrpcSecret(langbot: LangBotConfig, target: LangBotTarget): FrpcSecretMaterial { const exposure = target.publicExposure; const sourcePath = join(secretRoot(langbot), exposure.frpc.tokenSourceRef); @@ -1558,7 +1588,7 @@ function publicHttpProbe(baseUrl: string, path: string, apiKey: string | null): url, status: Number.isInteger(status) ? status : null, bodyBytes: Buffer.byteLength(body, "utf8"), - bodyPreview: body.slice(0, 2000), + bodyPreview: redactText(body).slice(0, 2000), stderrTail: redactText(stderr).slice(-2000), apiKeyUsed: apiKey !== null, valuesPrinted: false, @@ -1663,18 +1693,6 @@ function unquoteEnvValue(value: string): string { return value; } -function writeEnvFile(path: string, values: Record): void { - mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - const lines = Object.keys(values).sort().map((key) => `${key}=${quoteEnv(values[key])}`); - writeFileSync(path, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); - chmodSync(path, 0o600); -} - -function quoteEnv(value: string): string { - if (/^[A-Za-z0-9_./:@%?&=+-]+$/u.test(value)) return value; - return `'${value.replaceAll("'", "'\"'\"'")}'`; -} - function sqlLiteral(value: string): string { return `'${value.replaceAll("'", "''")}'`; } @@ -1722,7 +1740,11 @@ function compactCapture(result: SshCaptureResult, options: { full?: boolean } = } function redactText(text: string): string { - return text.replace(/lbk_[A-Za-z0-9_-]+/gu, "lbk_").replace(/postgresql:\/\/[^@\s]+@/gu, "postgresql://@"); + return text + .replace(/lbk_[A-Za-z0-9_-]+/gu, "lbk_") + .replace(/(postgres(?:ql)?:\/\/)[^@\s"']+@/giu, "$1@") + .replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1") + .replace(/(["']?(?:token|password|secret|api[_-]?key|apikey|jwt[_-]?secret|database[_-]?url)["']?\s*[:=]\s*["']?)[^"',\s}]+(["']?)/giu, "$1$2"); } function redactRepoPath(path: string): string { diff --git a/scripts/src/platform-infra-n8n.ts b/scripts/src/platform-infra-n8n.ts index fc7e6f0e..a25b4e13 100644 --- a/scripts/src/platform-infra-n8n.ts +++ b/scripts/src/platform-infra-n8n.ts @@ -1,6 +1,6 @@ -import { createHash, randomBytes } from "node:crypto"; -import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, isAbsolute, join } from "node:path"; +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import { isAbsolute, join } from "node:path"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; import { startJob } from "./jobs"; @@ -97,7 +97,7 @@ interface SecretMaterial { dbSourcePath: string; appSourceRef: string; appSourcePath: string; - action: "create" | "update" | "none"; + action: "read"; values: { dbUser: string; dbPassword: string; @@ -231,6 +231,7 @@ function plan(options: CommonOptions): Record { }, next: { postgres: "bun scripts/cli.ts platform-db postgres apply --config config/platform-db/postgres-pk01.yaml --confirm", + secrets: "bun scripts/cli.ts secrets sync --config config/secrets-distribution.yaml --scope platform-infra --confirm", dryRun: `bun scripts/cli.ts platform-infra n8n apply --target ${target.id} --dry-run`, apply: `bun scripts/cli.ts platform-infra n8n apply --target ${target.id} --confirm`, status: `bun scripts/cli.ts platform-infra n8n status --target ${target.id}`, @@ -303,6 +304,7 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise): void { - mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - const lines = Object.keys(values).sort().map((key) => `${key}=${quoteEnv(values[key])}`); - writeFileSync(path, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); - chmodSync(path, 0o600); -} - -function quoteEnv(value: string): string { - if (/^[A-Za-z0-9_./:@%?&=+-]+$/u.test(value)) return value; - return `'${value.replaceAll("'", "'\"'\"'")}'`; -} - function requiredEnvValue(values: Record, key: string, sourceRef: string): string { const value = values[key]; if (value === undefined || value.length === 0) throw new Error(`${sourceRef} is missing required key ${key}`); diff --git a/scripts/src/platform-infra-wechat-archive.ts b/scripts/src/platform-infra-wechat-archive.ts index 0627fa00..2f5d5bc2 100644 --- a/scripts/src/platform-infra-wechat-archive.ts +++ b/scripts/src/platform-infra-wechat-archive.ts @@ -1,5 +1,5 @@ -import { createHash, randomUUID } from "node:crypto"; -import { existsSync } from "node:fs"; +import { randomUUID } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; import pathPosix from "node:path/posix"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; @@ -8,10 +8,8 @@ import { asRecord, assertProxyOk, booleanField, - compactProxyResponse, compactUnknown, containerPathToHostPath, - dateInTimeZone, fetchJsonWithTimeout, findBaiduFileByRemotePath, hostRootPath, @@ -24,8 +22,6 @@ import { readYamlRecord, redactSensitiveUnknown, recordField, - relativeStagingPath, - renderTemplate, repoRelative, resolveRepoPath, sanitizePathSegment, @@ -33,11 +29,11 @@ import { stringField, syncN8nWorkflow, waitForBaiduTransfer, - writeStagingBase64, - writeStagingText, type OpsApplyOptions, type OpsCommonOptions, } from "./platform-infra-ops-library"; +import { prepareLangBotSecretMaterial, readLangBotPostgresConninfo, readLangBotRuntimeConfig, readLangBotSecretMaterial } from "./platform-infra-langbot"; +import { fingerprintValues } from "./platform-infra-public-service"; const configFile = rootPath("config", "platform-infra", "wechat-archive.yaml"); const configLabel = "config/platform-infra/wechat-archive.yaml"; @@ -47,12 +43,27 @@ interface WechatArchiveConfig { kind: "platform-infra-wechat-archive"; metadata: { id: string; owner: string; relatedIssues: number[] }; target: { id: string; route: string; namespace: string }; - langbot: { configRef: string; publicBaseUrl: string; expectedAdapter: string; callbackPath: string; notes: string[] }; + langbot: { + configRef: string; + publicBaseUrl: string; + expectedAdapter: string; + callbackPath: string; + readOnlyRecord: { enabled: boolean; mode: string; webhookName: string; webhookDescription: string; discardPipelineUuid: string }; + pipeline: { name: string; description: string; runner: string; outputKey: string; timeoutSeconds: number }; + notes: string[]; + }; n8n: { configRef: string; publicBaseUrl: string; workflow: { name: string; id: string; webhookPath: string; active: boolean; timezone: string }; }; + archiveCallback: { + publicUrl: string; + secretRoot: string; + tokenSourceRef: string; + tokenKey: string; + timeoutMs: number; + }; baiduNetdisk: { serviceId: string; proxyMode: string; @@ -132,7 +143,7 @@ function parsePullOptions(args: string[]): PullOptions { function plan(options: OpsCommonOptions): Record { const archive = readConfig(); assertTarget(archive, options.targetId); - const workflow = renderN8nWorkflow(archive); + const workflow = renderN8nWorkflow(archive, null); const policy = policyChecks(archive); return { ok: policy.every((check) => check.ok), @@ -176,7 +187,8 @@ async function apply(config: UniDeskConfig, options: OpsApplyOptions): Promise { publicBaseUrl: config.langbot.publicBaseUrl, expectedAdapter: config.langbot.expectedAdapter, callbackPath: config.langbot.callbackPath, + readOnlyRecord: { + enabled: config.langbot.readOnlyRecord.enabled, + mode: config.langbot.readOnlyRecord.mode, + webhookName: config.langbot.readOnlyRecord.webhookName, + discardPipelineUuid: config.langbot.readOnlyRecord.discardPipelineUuid, + }, + pipeline: config.langbot.pipeline, }, n8n: { configRef: config.n8n.configRef, @@ -426,6 +479,14 @@ function configSummary(config: WechatArchiveConfig): Record { workflow: config.n8n.workflow, webhookUrl: webhookUrl(config), }, + archiveCallback: { + publicUrl: config.archiveCallback.publicUrl, + secretRoot: config.archiveCallback.secretRoot, + tokenSourceRef: config.archiveCallback.tokenSourceRef, + tokenKey: config.archiveCallback.tokenKey, + timeoutMs: config.archiveCallback.timeoutMs, + valuesPrinted: false, + }, baiduNetdisk: { serviceId: config.baiduNetdisk.serviceId, proxyMode: config.baiduNetdisk.proxyMode, @@ -443,6 +504,8 @@ function policyChecks(config: WechatArchiveConfig): Array): WechatArchiveConfig["langbot"]["readOnlyRecord"] { + return { + enabled: booleanField(raw, "enabled", `${configLabel}.langbot.readOnlyRecord`), + mode: stringField(raw, "mode", `${configLabel}.langbot.readOnlyRecord`), + webhookName: stringField(raw, "webhookName", `${configLabel}.langbot.readOnlyRecord`), + webhookDescription: stringField(raw, "webhookDescription", `${configLabel}.langbot.readOnlyRecord`), + discardPipelineUuid: stringField(raw, "discardPipelineUuid", `${configLabel}.langbot.readOnlyRecord`), + }; +} + +function parseLangBotPipeline(raw: Record): WechatArchiveConfig["langbot"]["pipeline"] { + const timeoutSeconds = numberField(raw, "timeoutSeconds", `${configLabel}.langbot.pipeline`); + return { + name: stringField(raw, "name", `${configLabel}.langbot.pipeline`), + description: stringField(raw, "description", `${configLabel}.langbot.pipeline`), + runner: stringField(raw, "runner", `${configLabel}.langbot.pipeline`), + outputKey: stringField(raw, "outputKey", `${configLabel}.langbot.pipeline`), + timeoutSeconds, + }; +} + function webhookUrl(config: WechatArchiveConfig): string { return `${config.n8n.publicBaseUrl.replace(/\/+$/u, "")}/webhook/${registeredWebhookPath(config)}`; } -function renderN8nWorkflow(config: WechatArchiveConfig): Record { +function langBotDryRun(config: WechatArchiveConfig): Record { + return { + ok: true, + mode: "dry-run", + pipeline: { + name: config.langbot.pipeline.name, + runner: config.langbot.pipeline.runner, + webhookUrl: webhookUrl(config), + outputKey: config.langbot.pipeline.outputKey, + timeoutSeconds: config.langbot.pipeline.timeoutSeconds, + }, + bot: { + expectedAdapter: config.langbot.expectedAdapter, + desiredRouting: "discard person/group messages after read-only webhook mirroring", + }, + valuesPrinted: false, + }; +} + +async function syncLangBotPipeline(config: WechatArchiveConfig): Promise> { + const runtime = readLangBotRuntimeConfig(); + const secret = readLangBotSecretMaterial(); + const apiKey = String(secret.apiKey || ""); + const baseUrl = String(runtime.publicBaseUrl || config.langbot.publicBaseUrl); + const pipelinesResp = await langBotRequest(baseUrl, apiKey, "GET", "/api/v1/pipelines"); + const pipelines = arrayFromPath(pipelinesResp.body, ["data", "pipelines"]); + let pipeline = pipelines.find((item) => String(asRecord(item, "pipeline").name || "") === config.langbot.pipeline.name); + let created = false; + if (pipeline === undefined) { + const createdResp = await langBotRequest(baseUrl, apiKey, "POST", "/api/v1/pipelines", { + name: config.langbot.pipeline.name, + description: config.langbot.pipeline.description, + emoji: "archive", + }); + const uuid = String(nestedValue(createdResp.body, ["data", "uuid"]) || ""); + if (!uuid) throw new Error("LangBot create pipeline did not return uuid"); + const getResp = await langBotRequest(baseUrl, apiKey, "GET", `/api/v1/pipelines/${encodeURIComponent(uuid)}`); + pipeline = asRecord(nestedValue(getResp.body, ["data", "pipeline"]), "created pipeline"); + created = true; + } + const pipelineRecord = asRecord(pipeline, "pipeline"); + const pipelineUuid = String(pipelineRecord.uuid || ""); + if (!pipelineUuid) throw new Error("LangBot pipeline record is missing uuid"); + const configRecord = asRecord(pipelineRecord.config, "pipeline.config"); + const nextConfig = langBotPipelineConfig(configRecord, config); + await langBotRequest(baseUrl, apiKey, "PUT", `/api/v1/pipelines/${encodeURIComponent(pipelineUuid)}`, { + name: config.langbot.pipeline.name, + description: config.langbot.pipeline.description, + config: nextConfig, + }); + const webhookBinding = syncLangBotWebhook(config, webhookUrl(config)); + const botBinding = await syncLangBotReadOnlyBot(config, baseUrl, apiKey); + return { + ok: webhookBinding.ok === true && botBinding.ok === true, + mode: "confirmed", + pipeline: { + uuid: pipelineUuid, + name: config.langbot.pipeline.name, + runner: config.langbot.pipeline.runner, + webhookUrl: webhookUrl(config), + created, + }, + readOnlyWebhook: webhookBinding, + bot: botBinding, + auth: { + sourceRef: secret.sourceRef, + keyName: secret.keyName, + apiKeyFingerprint: secret.apiKeyFingerprint, + valuesPrinted: false, + }, + valuesPrinted: false, + }; +} + +async function inspectLangBotBinding(config: WechatArchiveConfig): Promise> { + try { + const runtime = readLangBotRuntimeConfig(); + const secret = readLangBotSecretMaterial(); + const apiKey = String(secret.apiKey || ""); + const baseUrl = String(runtime.publicBaseUrl || config.langbot.publicBaseUrl); + const pipelinesResp = await langBotRequest(baseUrl, apiKey, "GET", "/api/v1/pipelines"); + const pipelines = arrayFromPath(pipelinesResp.body, ["data", "pipelines"]); + const pipeline = pipelines.find((item) => String(asRecord(item, "pipeline").name || "") === config.langbot.pipeline.name); + const pipelineRecord = pipeline === undefined ? null : asRecord(pipeline, "pipeline"); + const pipelineUuid = String(pipelineRecord?.uuid || ""); + const pipelineConfig = typeof pipelineRecord?.config === "object" && pipelineRecord.config !== null && !Array.isArray(pipelineRecord.config) + ? pipelineRecord.config as Record + : {}; + const runner = String(nestedValue(pipelineConfig, ["ai", "runner", "runner"]) || ""); + const webhook = String(nestedValue(pipelineConfig, ["ai", "n8n-service-api", "webhook-url"]) || ""); + const botsResp = await langBotRequest(baseUrl, apiKey, "GET", "/api/v1/platform/bots"); + const bots = arrayFromPath(botsResp.body, ["data", "bots"]); + const bot = bots.find((item) => String(asRecord(item, "bot").adapter || "") === config.langbot.expectedAdapter); + const botRecord = bot === undefined ? null : asRecord(bot, "bot"); + const routingRules = Array.isArray(botRecord?.pipeline_routing_rules) ? botRecord.pipeline_routing_rules as unknown[] : []; + const hasDiscardRouting = hasReadOnlyDiscardRouting(routingRules, config); + const readOnlyWebhook = inspectLangBotWebhook(config, webhookUrl(config)); + const ok = Boolean(pipelineUuid) + && runner === config.langbot.pipeline.runner + && webhook === webhookUrl(config) + && hasDiscardRouting + && readOnlyWebhook.ok === true; + return { + ok, + pipeline: { + exists: Boolean(pipelineUuid), + uuid: pipelineUuid || null, + name: config.langbot.pipeline.name, + runner, + webhookMatches: webhook === webhookUrl(config), + }, + bot: { + exists: botRecord !== null, + uuid: String(botRecord?.uuid || "") || null, + adapter: config.langbot.expectedAdapter, + usePipelineUuid: String(botRecord?.use_pipeline_uuid || "") || null, + readOnlyDiscardRouting: hasDiscardRouting, + }, + readOnlyWebhook, + auth: { + sourceRef: secret.sourceRef, + keyName: secret.keyName, + apiKeyFingerprint: secret.apiKeyFingerprint, + valuesPrinted: false, + }, + valuesPrinted: false, + }; + } catch (error) { + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + valuesPrinted: false, + }; + } +} + +function langBotPipelineConfig(current: Record, archive: WechatArchiveConfig): Record { + const next = JSON.parse(JSON.stringify(current)) as Record; + const ai = asMutableRecord(next, "ai"); + const runner = asMutableRecord(ai, "runner"); + runner.runner = archive.langbot.pipeline.runner; + runner["expire-time"] = 0; + const n8n = asMutableRecord(ai, "n8n-service-api"); + n8n["webhook-url"] = webhookUrl(archive); + n8n["auth-type"] = "none"; + n8n.timeout = archive.langbot.pipeline.timeoutSeconds; + n8n["output-key"] = archive.langbot.pipeline.outputKey; + return next; +} + +async function syncLangBotReadOnlyBot(config: WechatArchiveConfig, baseUrl: string, apiKey: string): Promise> { + const botsResp = await langBotRequest(baseUrl, apiKey, "GET", "/api/v1/platform/bots"); + const bots = arrayFromPath(botsResp.body, ["data", "bots"]); + const bot = bots.find((item) => String(asRecord(item, "bot").adapter || "") === config.langbot.expectedAdapter); + if (bot === undefined) throw new Error(`LangBot bot adapter ${config.langbot.expectedAdapter} not found`); + const botRecord = asRecord(bot, "bot"); + const botUuid = String(botRecord.uuid || ""); + if (!botUuid) throw new Error("LangBot bot record is missing uuid"); + const currentRules = Array.isArray(botRecord.pipeline_routing_rules) ? botRecord.pipeline_routing_rules : []; + const nextRules = readOnlyDiscardRoutingRules(config); + const alreadyDiscard = hasReadOnlyDiscardRouting(currentRules, config); + if (!alreadyDiscard || JSON.stringify(currentRules) !== JSON.stringify(nextRules)) { + await langBotRequest(baseUrl, apiKey, "PUT", `/api/v1/platform/bots/${encodeURIComponent(botUuid)}`, { pipeline_routing_rules: nextRules }); + } + return { + ok: true, + uuid: botUuid, + adapter: config.langbot.expectedAdapter, + mode: config.langbot.readOnlyRecord.mode, + usePipelineUuid: String(botRecord.use_pipeline_uuid || "") || null, + routingRules: nextRules, + readOnlyDiscardRouting: true, + hadDiscardRouting: alreadyDiscard, + valuesPrinted: false, + }; +} + +function readOnlyDiscardRoutingRules(config: WechatArchiveConfig): Array> { + const pipelineUuid = config.langbot.readOnlyRecord.discardPipelineUuid; + return [ + { type: "launcher_type", operator: "eq", value: "person", pipeline_uuid: pipelineUuid }, + { type: "launcher_type", operator: "eq", value: "group", pipeline_uuid: pipelineUuid }, + ]; +} + +function hasReadOnlyDiscardRouting(rules: unknown[], config: WechatArchiveConfig): boolean { + const desired = readOnlyDiscardRoutingRules(config); + return desired.every((expected) => rules.some((rule) => { + if (typeof rule !== "object" || rule === null || Array.isArray(rule)) return false; + const record = rule as Record; + return Object.entries(expected).every(([key, value]) => record[key] === value); + })); +} + +function syncLangBotWebhook(config: WechatArchiveConfig, url: string): Record { + const secret = prepareLangBotSecretMaterial(); + const conn = readLangBotPostgresConninfo(); + const sql = [ + "WITH updated AS (", + " UPDATE webhooks", + ` SET url = ${sqlLiteral(url)}, description = ${sqlLiteral(config.langbot.readOnlyRecord.webhookDescription)}, enabled = true, updated_at = now()`, + ` WHERE name = ${sqlLiteral(config.langbot.readOnlyRecord.webhookName)}`, + " RETURNING id, name, enabled, url", + "), inserted AS (", + " INSERT INTO webhooks (name, url, description, enabled)", + ` SELECT ${sqlLiteral(config.langbot.readOnlyRecord.webhookName)}, ${sqlLiteral(url)}, ${sqlLiteral(config.langbot.readOnlyRecord.webhookDescription)}, true`, + " WHERE NOT EXISTS (SELECT 1 FROM updated)", + " RETURNING id, name, enabled, url", + ")", + "SELECT id, name, enabled, url FROM updated UNION ALL SELECT id, name, enabled, url FROM inserted LIMIT 1;", + ].join("\n"); + const result = Bun.spawnSync(["psql", "-Atq", conn, "-c", sql], { + stdout: "pipe", + stderr: "pipe", + env: { ...process.env, PGPASSWORD: secret.values.dbPassword }, + }); + const stdout = new TextDecoder().decode(result.stdout).trim(); + const stderr = new TextDecoder().decode(result.stderr).trim(); + return { + ok: result.exitCode === 0, + mode: "db-upsert", + name: config.langbot.readOnlyRecord.webhookName, + url, + enabled: true, + row: result.exitCode === 0 ? compactWebhookRow(stdout) : "", + stderrTail: redactSensitiveUnknown(stderr).toString().slice(-1200), + auth: { + dbSourceRef: secret.dbSourceRef, + appSourceRef: secret.appSourceRef, + fingerprint: secret.fingerprint, + valuesPrinted: false, + }, + valuesPrinted: false, + }; +} + +function inspectLangBotWebhook(config: WechatArchiveConfig, url: string): Record { + const secret = prepareLangBotSecretMaterial(); + const conn = readLangBotPostgresConninfo(); + const sql = [ + "SELECT id, name, enabled, url FROM webhooks", + `WHERE url = ${sqlLiteral(url)} OR name = ${sqlLiteral(config.langbot.readOnlyRecord.webhookName)}`, + "ORDER BY enabled DESC, updated_at DESC LIMIT 1;", + ].join("\n"); + const result = Bun.spawnSync(["psql", "-Atq", conn, "-c", sql], { + stdout: "pipe", + stderr: "pipe", + env: { ...process.env, PGPASSWORD: secret.values.dbPassword }, + }); + const stdout = new TextDecoder().decode(result.stdout).trim(); + const stderr = new TextDecoder().decode(result.stderr).trim(); + const row = parseWebhookRow(stdout); + return { + ok: result.exitCode === 0 && row !== null && row.enabled === true && row.url === url, + exists: row !== null, + name: row?.name ?? config.langbot.readOnlyRecord.webhookName, + enabled: row?.enabled ?? false, + urlMatches: row?.url === url, + row: row === null ? null : { id: row.id, name: row.name, enabled: row.enabled }, + stderrTail: result.exitCode === 0 ? "" : redactSensitiveUnknown(stderr).toString().slice(-1200), + valuesPrinted: false, + }; +} + +function parseWebhookRow(value: string): { id: string; name: string; enabled: boolean; url: string } | null { + if (!value) return null; + const [id = "", name = "", enabled = "", url = ""] = value.split("|"); + if (!id) return null; + return { id, name, enabled: enabled === "t" || enabled === "true", url }; +} + +function compactWebhookRow(value: string): Record | null { + const row = parseWebhookRow(value); + if (row === null) return null; + return { id: row.id, name: row.name, enabled: row.enabled }; +} + +function sqlLiteral(value: string): string { + return `'${value.replace(/'/gu, "''")}'`; +} + +async function langBotRequest(baseUrl: string, apiKey: string, method: string, apiPath: string, body?: unknown): Promise> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 45_000); + try { + const response = await fetch(`${baseUrl.replace(/\/+$/u, "")}${apiPath}`, { + method, + headers: { "content-type": "application/json", "x-api-key": apiKey }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: controller.signal, + }); + const text = await response.text(); + let parsed: unknown = text; + try { + parsed = text.length > 0 ? JSON.parse(text) as unknown : null; + } catch { + parsed = text; + } + if (!response.ok) throw new Error(`LangBot ${method} ${apiPath} failed with HTTP ${response.status}: ${redactText(JSON.stringify(compactUnknown(parsed))).slice(0, 800)}`); + return { ok: true, status: response.status, body: parsed }; + } finally { + clearTimeout(timer); + } +} + +function redactText(text: string): string { + return text + .replace(/lbk_[A-Za-z0-9_-]+/gu, "lbk_") + .replace(/(postgres(?:ql)?:\/\/)[^@\s"']+@/giu, "$1@") + .replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1") + .replace(/(["']?(?:token|password|secret|api[_-]?key|apikey|jwt[_-]?secret|database[_-]?url)["']?\s*[:=]\s*["']?)[^"',\s}]+(["']?)/giu, "$1$2"); +} + +function nestedValue(value: unknown, path: string[]): unknown { + let current = value; + for (const part of path) { + if (typeof current !== "object" || current === null || Array.isArray(current)) return undefined; + current = (current as Record)[part]; + } + return current; +} + +function arrayFromPath(value: unknown, path: string[]): unknown[] { + const nested = nestedValue(value, path); + return Array.isArray(nested) ? nested : []; +} + +function asMutableRecord(parent: Record, key: string): Record { + const current = parent[key]; + if (typeof current === "object" && current !== null && !Array.isArray(current)) return current as Record; + const next: Record = {}; + parent[key] = next; + return next; +} + +function renderN8nWorkflow(config: WechatArchiveConfig, callbackToken: string | null): Record { + const tokenForWorkflow = callbackToken ?? "__UNIDESK_WECHAT_ARCHIVE_TOKEN_FROM_SOURCE_REF__"; const code = ` const input = $input.first()?.json ?? {}; const body = input.body && typeof input.body === 'object' ? input.body : input; +const data = body.data && typeof body.data === 'object' ? body.data : {}; +const langbotMessage = data.message && typeof data.message === 'object' ? data.message : {}; +const langbotComponents = Array.isArray(langbotMessage.root) + ? langbotMessage.root + : Array.isArray(langbotMessage) + ? langbotMessage + : []; +function firstComponent(type) { + return langbotComponents.find((item) => item && typeof item === 'object' && String(item.type || '').toLowerCase() === type.toLowerCase()) || {}; +} +function firstText() { + const plain = firstComponent('Plain'); + if (plain.text !== undefined && plain.text !== null) return String(plain.text); + return langbotComponents + .filter((item) => item && typeof item === 'object' && item.text !== undefined) + .map((item) => String(item.text)) + .join(''); +} +function firstImage() { + return firstComponent('Image'); +} function pick(...keys) { for (const key of keys) { const value = key.split('.').reduce((acc, part) => acc && typeof acc === 'object' ? acc[part] : undefined, body); @@ -465,11 +906,64 @@ function pick(...keys) { } return ''; } -const messageType = String(pick('messageType', 'type', 'msgType') || 'text').toLowerCase(); -const receivedAt = String(pick('receivedAt', 'timestamp') || new Date().toISOString()); -const messageId = String(pick('messageId', 'msgId', 'id') || String(Date.now())); -const conversationId = String(pick('conversationId', 'chatId', 'roomId', 'fromUser') || 'unknown-conversation'); -const senderId = String(pick('senderId', 'userId', 'fromUserName', 'fromUser') || 'unknown-sender'); +function sanitizePathSegment(value, fallback = 'unknown') { + const normalized = String(value || fallback) + .normalize('NFKC') + .replace(/[\\\\/\\0\\r\\n\\t]+/g, '-') + .replace(/[^A-Za-z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 120); + return normalized.length > 0 ? normalized : fallback; +} +function dateInTimeZone(date, timeZone) { + const parts = new Intl.DateTimeFormat('en-CA', { + timeZone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).formatToParts(date).reduce((acc, part) => { + acc[part.type] = part.value; + return acc; + }, {}); + return [parts.year, parts.month, parts.day].join('-'); +} +function renderTemplate(template, values) { + return template.replace(/\\{\\{\\s*([A-Za-z0-9_]+)\\s*\\}\\}/g, (_match, key) => values[key] || ''); +} +function normalizeRemotePath(path) { + const collapsed = String(path).replace(/\\/+/g, '/'); + return collapsed.startsWith('/') ? collapsed : '/' + collapsed; +} +const imageComponent = firstImage(); +const langbotText = firstText(); +const langbotSender = data.sender && typeof data.sender === 'object' ? data.sender : {}; +const langbotGroup = data.group && typeof data.group === 'object' ? data.group : {}; +const messageType = String(pick('messageType', 'type', 'msgType') || (Object.keys(imageComponent).length > 0 ? 'image' : 'text')).toLowerCase(); +const receivedAt = String(pick('receivedAt', 'timestamp', 'data.timestamp') || new Date().toISOString()); +const receivedDate = Number.isNaN(new Date(receivedAt).getTime()) ? new Date() : new Date(receivedAt); +const messageId = sanitizePathSegment(pick('messageId', 'msgId', 'id', 'message_id', 'uuid') || String(Date.now()), 'message'); +const conversationId = sanitizePathSegment(pick('conversationId', 'chatId', 'roomId', 'fromUser', 'conversation_id', 'session_id') || langbotGroup.id || langbotSender.id || 'unknown-conversation', 'conversation'); +const senderId = sanitizePathSegment(pick('senderId', 'userId', 'fromUserName', 'fromUser', 'user_id') || langbotSender.id || 'unknown-sender', 'sender'); +const media = body.media && typeof body.media === 'object' ? body.media : {}; +if (imageComponent.base64 && !media.dataBase64) media.dataBase64 = imageComponent.base64; +if (imageComponent.url && !media.url) media.url = imageComponent.url; +if (!media.filename && messageType === 'image') media.filename = messageId + '.${config.baiduNetdisk.archive.imageExtension}'; +const extension = messageType === 'image' ? '${config.baiduNetdisk.archive.imageExtension}' : '${config.baiduNetdisk.archive.textExtension}'; +const filename = messageType === 'image' + ? sanitizePathSegment(media.filename || messageId + '.' + extension, messageId + '.' + extension) + : messageId + '.' + extension; +const text = String(pick('text', 'content', 'message', 'chatInput', 'query', 'query_text') || langbotText || ''); +const remotePath = normalizeRemotePath(renderTemplate('${config.baiduNetdisk.archive.pathTemplate}', { + remoteRoot: '${config.baiduNetdisk.archive.remoteRoot}', + date: dateInTimeZone(receivedDate, '${config.baiduNetdisk.archive.timezone}'), + timestamp: receivedDate.toISOString().replace(/[-:.TZ]/g, ''), + conversationId, + senderId, + messageId, + type: messageType, + extension, + filename, +})); const normalized = { ok: true, source: 'n8n', @@ -479,13 +973,15 @@ const normalized = { conversationId, senderId, messageType, - text: String(pick('text', 'content') || ''), + text, receivedAt, - media: body.media && typeof body.media === 'object' ? body.media : {}, + media, }, archive: { remoteRoot: '${config.baiduNetdisk.archive.remoteRoot}', pathTemplate: '${config.baiduNetdisk.archive.pathTemplate}', + remotePath, + filename, }, }; return [{ json: normalized }]; @@ -516,11 +1012,38 @@ return [{ json: normalized }]; typeVersion: 2, position: [280, 0], }, + { + parameters: { + method: "POST", + url: config.archiveCallback.publicUrl, + sendHeaders: true, + headerParameters: { + parameters: [ + { name: "Authorization", value: `Bearer ${tokenForWorkflow}` }, + { name: "Content-Type", value: "application/json" }, + ], + }, + sendBody: true, + specifyBody: "json", + jsonBody: "={{ JSON.stringify($json) }}", + options: { + timeout: config.archiveCallback.timeoutMs, + }, + }, + id: "archive-callback", + name: "Archive To Baidu Callback", + type: "n8n-nodes-base.httpRequest", + typeVersion: 4.2, + position: [560, 0], + }, ], connections: { "wechat-archive-webhook": { main: [[{ node: "Normalize WeChat Message", type: "main", index: 0 }]], }, + "Normalize WeChat Message": { + main: [[{ node: "Archive To Baidu Callback", type: "main", index: 0 }]], + }, }, settings: { executionOrder: "v1", @@ -570,7 +1093,7 @@ function fixturePayload(config: WechatArchiveConfig, type: "text" | "image", obs } async function callWorkflow(config: WechatArchiveConfig, payload: Record): Promise> { - const response = await fetchJsonWithTimeout(webhookUrl(config), payload, 45_000); + const response = await fetchJsonWithTimeout(webhookUrl(config), payload, Math.max(45_000, config.archiveCallback.timeoutMs + 30_000)); const body = typeof response.body === "object" && response.body !== null && !Array.isArray(response.body) ? response.body as Record : {}; @@ -582,7 +1105,7 @@ async function callWorkflow(config: WechatArchiveConfig, payload: Record, n8nResponse: Record): Promise> { +function archiveFromWorkflowResponse(originalPayload: Record, n8nResponse: Record): Record { if (n8nResponse.ok !== true) { return { ok: false, @@ -594,59 +1117,24 @@ async function archiveMessage(config: WechatArchiveConfig, originalPayload: Reco }; } const body = asRecord(n8nResponse.body ?? {}, "n8n response body"); + const archive = asRecord(body.archive ?? {}, "n8n response archive"); const normalized = asRecord(body.message ?? originalPayload, "normalized message"); const type = String(normalized.messageType ?? originalPayload.messageType ?? "text").toLowerCase(); - const messageId = sanitizePathSegment(String(normalized.messageId ?? originalPayload.messageId ?? randomUUID())); - const conversationId = sanitizePathSegment(String(normalized.conversationId ?? originalPayload.conversationId ?? "unknown-conversation")); - const senderId = sanitizePathSegment(String(normalized.senderId ?? originalPayload.senderId ?? "unknown-sender")); - const receivedAt = new Date(String(normalized.receivedAt ?? originalPayload.receivedAt ?? new Date().toISOString())); - const extension = type === "image" ? config.baiduNetdisk.archive.imageExtension : config.baiduNetdisk.archive.textExtension; - const remotePath = normalizeRemotePath(renderTemplate(config.baiduNetdisk.archive.pathTemplate, { - remoteRoot: config.baiduNetdisk.archive.remoteRoot, - date: dateInTimeZone(receivedAt, config.baiduNetdisk.archive.timezone), - timestamp: receivedAt.toISOString().replace(/[-:.TZ]/gu, ""), - conversationId, - senderId, - messageId, - type, - extension, - filename: `${messageId}.${extension}`, - })); - const localRel = relativeStagingPath(config.baiduNetdisk.staging.outboxDir, `${messageId}.${extension}`); - const written = type === "image" - ? writeStagingBase64({ hostRoot: config.baiduNetdisk.staging.hostRoot, relativePath: localRel, dataBase64: String(asRecord(normalized.media ?? originalPayload.media ?? {}, "image media").dataBase64 ?? "") }) - : writeStagingText({ hostRoot: config.baiduNetdisk.staging.hostRoot, relativePath: localRel, content: messageText(normalized, originalPayload) }); - const created = assertProxyOk(microserviceProxy(config.baiduNetdisk.serviceId, "/api/transfers/upload-from-path", { - method: "POST", - body: { localPath: localRel, remotePath }, - timeoutMs: 60_000, - }), "baidu upload create"); - const job = asRecord(created.job, "baidu upload job"); - const jobId = String(job.id || ""); - const waited = await waitForBaiduTransfer(config.baiduNetdisk.serviceId, jobId, { - timeoutMs: config.validate.timeoutMs, - pollIntervalMs: config.validate.pollIntervalMs, - }); - const waitedJob = asRecord(waited.job ?? {}, "baidu waited upload job"); - const result = asRecord(waitedJob.result ?? {}, "baidu upload result"); - const baiduResult = asRecord(result.baidu ?? {}, "baidu upload result.baidu"); - let fsId = String(baiduResult.fs_id ?? baiduResult.fsId ?? waitedJob.fsId ?? ""); - let listed: Record | null = null; - if (!fsId) { - listed = findBaiduFileByRemotePath(config.baiduNetdisk.serviceId, remotePath); - fsId = String(listed?.fsId ?? listed?.fs_id ?? ""); - } + const remotePath = String(archive.remotePath || ""); + const fsId = String(archive.fsId || archive.fs_id || ""); return { - ok: waited.ok === true && Boolean(fsId), + ok: body.ok === true && Boolean(remotePath) && Boolean(fsId), + skipPipeline: body.skip_pipeline === true, + readOnly: body.readOnly === true, type, remotePath, fsId, - messageId, - conversationId, - senderId, - local: { relativePath: localRel, hostPath: written.hostPath, bytes: written.bytes, sha256: written.sha256 }, - uploadJob: waitedJob, - listed: redactBaiduFileEntry(listed), + messageId: normalized.messageId ?? originalPayload.messageId, + conversationId: normalized.conversationId ?? originalPayload.conversationId, + senderId: normalized.senderId ?? originalPayload.senderId, + local: typeof archive.local === "object" && archive.local !== null && !Array.isArray(archive.local) ? archive.local : {}, + uploadJob: typeof archive.uploadJob === "object" && archive.uploadJob !== null && !Array.isArray(archive.uploadJob) ? archive.uploadJob : {}, + listed: typeof archive.listed === "object" && archive.listed !== null && !Array.isArray(archive.listed) ? redactBaiduFileEntry(archive.listed as Record) : null, }; } @@ -678,10 +1166,19 @@ async function pullByFsId(config: WechatArchiveConfig, fsId: string, localRel: s }; } -function messageText(normalized: Record, originalPayload: Record): string { - const text = String(normalized.text ?? originalPayload.text ?? originalPayload.content ?? ""); - const payloadHash = createHash("sha256").update(JSON.stringify(originalPayload)).digest("hex"); - return `${text}\n\n---\nsource=unidesk-wechat-archive\npayloadSha256=${payloadHash}\n`; +async function pullArchiveIfReady(config: WechatArchiveConfig, archive: Record): Promise> { + const fsId = String(archive.fsId || ""); + const remotePath = String(archive.remotePath || ""); + if (archive.ok !== true || !fsId || !remotePath) { + return { + ok: false, + skipped: true, + reason: "archive-upload-not-ready", + fsId, + remotePath, + }; + } + return await pullByFsId(config, fsId, pullLocalRel(config, remotePath)); } function pullLocalRel(config: WechatArchiveConfig, remotePath: unknown): string { @@ -702,6 +1199,7 @@ function validationSummary( n8n: full ? n8n : { ok: n8n.ok, status: n8n.status, webhookUrl: n8n.webhookUrl }, archive: full ? archive : { ok: archive.ok, + skipPipeline: archive.skipPipeline, remotePath: archive.remotePath, fsId: archive.fsId, local: archive.local, @@ -720,6 +1218,101 @@ function compactDownload(downloaded: Record): Record): Record { + const body = typeof response.body === "object" && response.body !== null && !Array.isArray(response.body) + ? response.body as Record + : {}; + const storage = typeof body.storage === "object" && body.storage !== null && !Array.isArray(body.storage) ? body.storage as Record : {}; + const queue = typeof body.queue === "object" && body.queue !== null && !Array.isArray(body.queue) ? body.queue as Record : {}; + const transfers = typeof queue.transfers === "object" && queue.transfers !== null && !Array.isArray(queue.transfers) ? queue.transfers as Record : {}; + return { + ok: response.ok, + status: response.status, + service: body.service, + storage: { + postgres: storage.postgres, + stagingDir: storage.stagingDir, + }, + queue: { + transfers: { + succeeded: transfers.succeeded, + failed: transfers.failed, + }, + }, + }; +} + +function compactBaiduAuth(response: ReturnType): Record { + const body = typeof response.body === "object" && response.body !== null && !Array.isArray(response.body) + ? response.body as Record + : {}; + const auth = typeof body.auth === "object" && body.auth !== null && !Array.isArray(body.auth) ? body.auth as Record : {}; + const account = typeof auth.account === "object" && auth.account !== null && !Array.isArray(auth.account) ? auth.account as Record : {}; + return { + ok: response.ok, + status: response.status, + configured: auth.configured, + loggedIn: auth.loggedIn, + account: { + present: Object.keys(account).length > 0, + rootPath: account.rootPath, + quota: compactBaiduQuota(account.quota), + }, + valuesPrinted: false, + }; +} + +function compactBaiduQuota(value: unknown): Record | null { + if (typeof value !== "object" || value === null || Array.isArray(value)) return null; + const quota = value as Record; + return { + total: quota.total, + used: quota.used, + free: quota.free, + usedPercent: quota.usedPercent, + }; +} + +function compactBaiduTransfers(response: ReturnType): Record { + const body = typeof response.body === "object" && response.body !== null && !Array.isArray(response.body) + ? response.body as Record + : {}; + const counts = typeof body.counts === "object" && body.counts !== null && !Array.isArray(body.counts) ? body.counts as Record : {}; + const jobs = typeof body.jobs === "object" && body.jobs !== null && !Array.isArray(body.jobs) ? body.jobs as Record : {}; + const items = Array.isArray(jobs.items) + ? jobs.items + : Array.isArray(jobs.arrayPreview) + ? jobs.arrayPreview + : []; + return { + ok: response.ok, + status: response.status, + counts, + jobs: { + count: typeof jobs.count === "number" ? jobs.count : items.length, + items: items.slice(0, 10).map(compactBaiduTransferJob), + }, + }; +} + +function compactBaiduTransferJob(value: unknown): Record { + const job = typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; + const result = typeof job.result === "object" && job.result !== null && !Array.isArray(job.result) ? job.result as Record : {}; + const baidu = typeof result.baidu === "object" && result.baidu !== null && !Array.isArray(result.baidu) ? result.baidu as Record : {}; + return { + id: job.id, + direction: job.direction, + status: job.status, + createdAt: job.createdAt, + updatedAt: job.updatedAt, + remotePath: job.remotePath ?? result.remotePath, + fsId: job.fsId || baidu.fs_id || result.fsId, + sizeBytes: job.sizeBytes ?? result.sizeBytes ?? baidu.size, + progressPercent: job.progressPercent, + error: job.error || undefined, + }; +} + function redactBaiduFileEntry(value: Record | null): Record | null { if (value === null) return null; const copy: Record = { ...value }; @@ -727,3 +1320,38 @@ function redactBaiduFileEntry(value: Record | null): Record { + const result: Record = {}; + for (const rawLine of text.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq <= 0) continue; + const key = line.slice(0, eq).trim(); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) continue; + result[key] = unquoteEnvValue(line.slice(eq + 1).trim()); + } + return result; +} + +function unquoteEnvValue(value: string): string { + if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith("\"") && value.endsWith("\""))) return value.slice(1, -1); + return value; +} diff --git a/scripts/src/secrets.ts b/scripts/src/secrets.ts new file mode 100644 index 00000000..16087921 --- /dev/null +++ b/scripts/src/secrets.ts @@ -0,0 +1,826 @@ +import { randomBytes } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join } from "node:path"; +import type { UniDeskConfig } from "./config"; +import { rootPath } from "./config"; +import { startJob } from "./jobs"; +import type { SshCaptureResult } from "./ssh"; +import { capture, fingerprintValues, parseJsonOutput } from "./platform-infra-public-service"; + +const defaultConfigPath = "config/secrets-distribution.yaml"; +const fieldManager = "unidesk-secret-distribution"; + +interface SecretsOptions { + configPath: string; + scope: string | null; + targetId: string | null; + confirm: boolean; + dryRun: boolean; + wait: boolean; + full: boolean; + raw: boolean; +} + +interface SecretDistributionConfig { + configPath: string; + version: number; + kind: "unidesk-secret-distribution"; + metadata: { id: string; owner: string; relatedIssues: number[] }; + sources: { root: string; files: SourceFileConfig[] }; + targets: DistributionTarget[]; + kubernetesSecrets: KubernetesSecretConfig[]; +} + +interface SourceFileConfig { + sourceRef: string; + type: "env"; + requiredKeys: string[]; + createIfMissing: { + enabled: boolean; + values: Record; + randomHex: Record; + randomBase64Url: Record; + }; +} + +interface DistributionTarget { + id: string; + route: string; + namespace: string; + scope: string; + enabled: boolean; +} + +interface KubernetesSecretConfig { + name: string; + targetId: string; + secretName: string; + type: "Opaque"; + data: SecretDataMapping[]; +} + +interface SecretDataMapping { + sourceRef: string; + sourceKey: string; + targetKey: string; +} + +interface SourceMaterial { + sourceRef: string; + sourcePath: string; + exists: boolean; + requiredKeys: string[]; + presentKeys: string[]; + missingKeys: string[]; + action: "none" | "create" | "update" | "blocked"; + generatedKeys: string[]; + unmaterializedGeneratedKeys: string[]; + values: Record; + fingerprint: string | null; +} + +interface SourceInspection { + ok: boolean; + root: string; + entries: Array>; + materials: Map; +} + +interface DesiredSecret { + name: string; + target: DistributionTarget; + secretName: string; + type: "Opaque"; + data: Record; + keySources: Array<{ sourceRef: string; sourceKey: string; targetKey: string }>; + missingKeys: Array<{ sourceRef: string; sourceKey: string; targetKey: string }>; + pendingGeneratedKeys: string[]; + fingerprint: string | null; +} + +export function secretsHelp(): Record { + return { + command: "secrets plan|sync|status", + output: "json", + usage: [ + "bun scripts/cli.ts secrets plan --config config/secrets-distribution.yaml --scope platform-infra", + "bun scripts/cli.ts secrets sync --config config/secrets-distribution.yaml --scope platform-infra --confirm", + "bun scripts/cli.ts secrets status --config config/secrets-distribution.yaml --scope platform-infra", + ], + configTruth: defaultConfigPath, + secretPolicy: "Secret values are never printed or reverse-engineered from runtime. YAML sourceRef files are the authority; sync only pushes declared keys to declared Kubernetes Secret keys.", + }; +} + +export async function runSecretsCommand(config: UniDeskConfig, args: string[]): Promise> { + const [action = "plan"] = args; + if (action === "help" || action === "--help" || action === "-h") return secretsHelp(); + if (action === "plan") return plan(parseOptions(args.slice(1))); + if (action === "sync") return await sync(config, parseOptions(args.slice(1))); + if (action === "status") return await status(config, parseOptions(args.slice(1))); + return { ok: false, error: "unsupported-secrets-command", args, help: secretsHelp() }; +} + +function parseOptions(args: string[]): SecretsOptions { + let configPath = defaultConfigPath; + let scope: string | null = null; + let targetId: string | null = null; + let confirm = false; + let dryRun = false; + let wait = false; + let full = false; + let raw = false; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (arg === "--config") { + configPath = readOptionValue(args, index, "--config"); + index += 1; + } else if (arg.startsWith("--config=")) { + configPath = arg.slice("--config=".length); + } else if (arg === "--scope") { + scope = simpleId(readOptionValue(args, index, "--scope"), "--scope"); + index += 1; + } else if (arg.startsWith("--scope=")) { + scope = simpleId(arg.slice("--scope=".length), "--scope"); + } else if (arg === "--target") { + targetId = simpleId(readOptionValue(args, index, "--target"), "--target"); + index += 1; + } else if (arg.startsWith("--target=")) { + targetId = simpleId(arg.slice("--target=".length), "--target"); + } else if (arg === "--confirm") { + confirm = true; + } else if (arg === "--dry-run") { + dryRun = true; + } else if (arg === "--wait") { + wait = true; + } else if (arg === "--full") { + full = true; + } else if (arg === "--raw") { + raw = true; + full = true; + } else { + throw new Error(`unsupported secrets option: ${arg}`); + } + } + if (confirm && dryRun) throw new Error("secrets sync accepts only one of --confirm or --dry-run"); + return { configPath, scope, targetId, confirm, dryRun, wait, full, raw }; +} + +function plan(options: SecretsOptions): Record { + const distribution = readSecretDistributionConfig(options.configPath); + const sources = inspectSources(distribution, false); + const desired = desiredSecrets(distribution, options, sources); + return { + ok: sources.ok && desired.every((secret) => secret.missingKeys.length === 0), + action: "secrets-plan", + mutation: false, + config: configSummary(distribution, options), + localSources: sourceSummary(sources), + desiredSecrets: desired.map(desiredSecretSummary), + policy: { + sourceAuthority: "local YAML-declared sourceRef files under sources.root", + runtimeReverseEngineering: false, + valuesPrinted: false, + }, + next: { + sync: `bun scripts/cli.ts secrets sync --config ${distribution.configPath}${options.scope === null ? "" : ` --scope ${options.scope}`}${options.targetId === null ? "" : ` --target ${options.targetId}`} --confirm`, + status: `bun scripts/cli.ts secrets status --config ${distribution.configPath}${options.scope === null ? "" : ` --scope ${options.scope}`}${options.targetId === null ? "" : ` --target ${options.targetId}`}`, + }, + }; +} + +async function sync(config: UniDeskConfig, options: SecretsOptions): Promise> { + const distribution = readSecretDistributionConfig(options.configPath); + if (!options.confirm || options.dryRun) { + const planned = plan(options); + return { ...planned, action: "secrets-sync", mode: "dry-run", mutation: false }; + } + if (!options.wait) { + const jobArgs = ["bun", "scripts/cli.ts", "secrets", "sync", "--config", distribution.configPath, "--confirm", "--wait"]; + if (options.scope !== null) jobArgs.push("--scope", options.scope); + if (options.targetId !== null) jobArgs.push("--target", options.targetId); + const job = startJob("secrets_sync", jobArgs, "Sync YAML-declared local secret source keys into declared Kubernetes Secrets without printing values"); + return { + ok: true, + action: "secrets-sync", + mode: "async-job", + mutation: true, + config: configSummary(distribution, options), + job, + statusCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 12000`, + }; + } + const sources = inspectSources(distribution, true); + if (!sources.ok) return { ok: false, action: "secrets-sync", mode: "blocked-local-sources", mutation: true, config: configSummary(distribution, options), localSources: sourceSummary(sources) }; + const desired = desiredSecrets(distribution, options, sources); + const missing = desired.flatMap((secret) => secret.missingKeys); + if (missing.length > 0) { + return { ok: false, action: "secrets-sync", mode: "blocked-missing-secret-data", mutation: true, config: configSummary(distribution, options), localSources: sourceSummary(sources), missing, valuesPrinted: false }; + } + const perTarget = await Promise.all(groupDesiredSecretsByTarget(desired).map(async (group) => await applyTargetSecrets(config, group.target, group.secrets, options))); + return { + ok: perTarget.every((item) => item.ok === true), + action: "secrets-sync", + mode: "confirmed", + mutation: true, + config: configSummary(distribution, options), + localSources: sourceSummary(sources), + desiredSecrets: desired.map(desiredSecretSummary), + targets: perTarget, + valuesPrinted: false, + }; +} + +async function status(config: UniDeskConfig, options: SecretsOptions): Promise> { + const distribution = readSecretDistributionConfig(options.configPath); + const sources = inspectSources(distribution, false); + const desired = desiredSecrets(distribution, options, sources); + const perTarget = await Promise.all(groupDesiredSecretsByTarget(desired).map(async (group) => await statusTargetSecrets(config, group.target, group.secrets, options))); + return { + ok: perTarget.every((item) => item.ok === true), + action: "secrets-status", + mutation: false, + config: configSummary(distribution, options), + localSources: sourceSummary(sources), + desiredSecrets: desired.map(desiredSecretSummary), + targets: perTarget, + valuesPrinted: false, + }; +} + +function readSecretDistributionConfig(pathArg: string): SecretDistributionConfig { + const configPath = resolveConfigPath(pathArg); + const label = displayConfigPath(pathArg); + const root = asRecord(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown, label); + const version = integerField(root, "version", label); + const kind = stringField(root, "kind", label); + if (kind !== "unidesk-secret-distribution") throw new Error(`${label}.kind must be unidesk-secret-distribution`); + const metadata = objectField(root, "metadata", label); + const sources = objectField(root, "sources", label); + const config: SecretDistributionConfig = { + configPath: label, + version, + kind, + metadata: { + id: stringField(metadata, "id", `${label}.metadata`), + owner: stringField(metadata, "owner", `${label}.metadata`), + relatedIssues: numberArrayField(metadata, "relatedIssues", `${label}.metadata`), + }, + sources: { + root: stringField(sources, "root", `${label}.sources`), + files: arrayOfRecords(sources.files, `${label}.sources.files`).map((item, index) => parseSourceFile(item, `${label}.sources.files[${index}]`)), + }, + targets: arrayOfRecords(root.targets, `${label}.targets`).map((item, index) => parseTarget(item, `${label}.targets[${index}]`)), + kubernetesSecrets: arrayOfRecords(root.kubernetesSecrets, `${label}.kubernetesSecrets`).map((item, index) => parseKubernetesSecret(item, `${label}.kubernetesSecrets[${index}]`)), + }; + validateDistributionConfig(config); + return config; +} + +function parseSourceFile(record: Record, path: string): SourceFileConfig { + const type = stringField(record, "type", path); + if (type !== "env") throw new Error(`${path}.type must be env`); + const createRaw = record.createIfMissing === undefined ? {} : objectField(record, "createIfMissing", path); + const randomBase64UrlRaw = createRaw.randomBase64Url === undefined ? {} : objectField(createRaw, "randomBase64Url", `${path}.createIfMissing`); + return { + sourceRef: sourceRefField(record, "sourceRef", path), + type, + requiredKeys: stringArrayField(record, "requiredKeys", path).map((key, index) => envKeyValue(key, `${path}.requiredKeys[${index}]`)), + createIfMissing: { + enabled: createRaw.enabled === undefined ? false : booleanField(createRaw, "enabled", `${path}.createIfMissing`), + values: createRaw.values === undefined ? {} : stringMapField(createRaw, "values", `${path}.createIfMissing`), + randomHex: createRaw.randomHex === undefined ? {} : numberMapField(createRaw, "randomHex", `${path}.createIfMissing`), + randomBase64Url: Object.fromEntries(Object.entries(randomBase64UrlRaw).map(([key, value]) => [envKeyValue(key, `${path}.createIfMissing.randomBase64Url`), randomBase64UrlSpec(value, `${path}.createIfMissing.randomBase64Url.${key}`)])), + }, + }; +} + +function parseTarget(record: Record, path: string): DistributionTarget { + return { + id: simpleId(stringField(record, "id", path), `${path}.id`), + route: stringField(record, "route", path), + namespace: kubernetesNameField(record, "namespace", path), + scope: simpleId(stringField(record, "scope", path), `${path}.scope`), + enabled: booleanField(record, "enabled", path), + }; +} + +function parseKubernetesSecret(record: Record, path: string): KubernetesSecretConfig { + const type = stringField(record, "type", path); + if (type !== "Opaque") throw new Error(`${path}.type must be Opaque`); + return { + name: simpleId(stringField(record, "name", path), `${path}.name`), + targetId: simpleId(stringField(record, "targetId", path), `${path}.targetId`), + secretName: kubernetesNameField(record, "secretName", path), + type, + data: arrayOfRecords(record.data, `${path}.data`).map((item, index) => ({ + sourceRef: sourceRefField(item, "sourceRef", `${path}.data[${index}]`), + sourceKey: envKeyField(item, "sourceKey", `${path}.data[${index}]`), + targetKey: kubernetesSecretKeyField(item, "targetKey", `${path}.data[${index}]`), + })), + }; +} + +function validateDistributionConfig(config: SecretDistributionConfig): void { + if (config.sources.files.length === 0) throw new Error(`${config.configPath}.sources.files must not be empty`); + if (config.targets.length === 0) throw new Error(`${config.configPath}.targets must not be empty`); + const sources = new Map(config.sources.files.map((item) => [item.sourceRef, item])); + const targets = new Set(config.targets.map((item) => item.id)); + const targetSecrets = new Set(); + for (const secret of config.kubernetesSecrets) { + if (!targets.has(secret.targetId)) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name}.targetId is not declared in targets`); + const secretIdentity = `${secret.targetId}/${secret.secretName}`; + if (targetSecrets.has(secretIdentity)) throw new Error(`${config.configPath} declares duplicate target Secret ${secretIdentity}`); + targetSecrets.add(secretIdentity); + const targetKeys = new Set(); + for (const item of secret.data) { + const source = sources.get(item.sourceRef); + if (source === undefined) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} references undeclared sourceRef ${item.sourceRef}`); + if (!source.requiredKeys.includes(item.sourceKey)) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} maps ${item.sourceRef}.${item.sourceKey}, but that key is not listed in sources.files.requiredKeys`); + if (targetKeys.has(item.targetKey)) throw new Error(`${config.configPath}.kubernetesSecrets.${secret.name} maps duplicate target key ${item.targetKey}`); + targetKeys.add(item.targetKey); + } + } +} + +function inspectSources(config: SecretDistributionConfig, materialize: boolean): SourceInspection { + const root = secretRoot(config); + const materials = new Map(); + const entries = config.sources.files.map((source) => { + const sourcePath = join(root, source.sourceRef); + const exists = existsSync(sourcePath); + const existing = exists ? parseEnvFile(readFileSync(sourcePath, "utf8")) : {}; + const next = { ...existing }; + const generatedKeys: string[] = []; + const unmaterializedGeneratedKeys: string[] = []; + if (source.createIfMissing.enabled) { + for (const [key, value] of Object.entries(source.createIfMissing.values)) { + if (next[key] === undefined || next[key].length === 0) { + next[key] = value; + generatedKeys.push(key); + } + } + for (const [key, bytes] of Object.entries(source.createIfMissing.randomHex)) { + if (next[key] === undefined || next[key].length === 0) { + next[key] = materialize ? randomBytes(bytes).toString("hex") : ``; + generatedKeys.push(key); + if (!materialize) unmaterializedGeneratedKeys.push(key); + } + } + for (const [key, spec] of Object.entries(source.createIfMissing.randomBase64Url)) { + if (next[key] === undefined || next[key].length === 0) { + next[key] = materialize ? `${spec.prefix}${randomBytes(spec.bytes).toString("base64url")}` : ``; + generatedKeys.push(key); + if (!materialize) unmaterializedGeneratedKeys.push(key); + } + } + } + const missingBefore = source.requiredKeys.filter((key) => existing[key] === undefined || existing[key].length === 0); + const missingKeys = source.requiredKeys.filter((key) => next[key] === undefined || next[key].length === 0); + const action = missingKeys.length > 0 ? "blocked" : !exists ? "create" : missingBefore.length > 0 ? "update" : "none"; + if (materialize && action !== "blocked" && (action === "create" || action === "update")) writeEnvFile(sourcePath, next); + const material: SourceMaterial = { + sourceRef: source.sourceRef, + sourcePath, + exists, + requiredKeys: source.requiredKeys, + presentKeys: source.requiredKeys.filter((key) => next[key] !== undefined && next[key].length > 0), + missingKeys, + action, + generatedKeys, + unmaterializedGeneratedKeys, + values: next, + fingerprint: missingKeys.length === 0 && unmaterializedGeneratedKeys.length === 0 ? fingerprintValues(next, source.requiredKeys) : null, + }; + materials.set(source.sourceRef, material); + return { + sourceRef: source.sourceRef, + sourcePath: redactRepoPath(sourcePath), + exists, + requiredKeys: source.requiredKeys, + presentKeys: material.presentKeys, + missingKeys, + action, + generatedKeys, + unmaterializedGeneratedKeys, + fingerprint: material.fingerprint, + valuesPrinted: false, + }; + }); + return { + ok: entries.every((entry) => (entry.missingKeys as string[]).length === 0), + root: redactRepoPath(root), + entries, + materials, + }; +} + +function desiredSecrets(config: SecretDistributionConfig, options: SecretsOptions, sources: SourceInspection): DesiredSecret[] { + const targets = new Map(config.targets.map((target) => [target.id, target])); + return config.kubernetesSecrets + .map((secret) => ({ secret, target: targets.get(secret.targetId) })) + .filter((item): item is { secret: KubernetesSecretConfig; target: DistributionTarget } => item.target !== undefined && item.target.enabled) + .filter(({ target }) => options.scope === null || target.scope === options.scope) + .filter(({ target }) => options.targetId === null || target.id === options.targetId) + .map(({ secret, target }) => { + const data: Record = {}; + const missingKeys: DesiredSecret["missingKeys"] = []; + const pendingGeneratedKeys: string[] = []; + for (const item of secret.data) { + const source = sources.materials.get(item.sourceRef); + const value = source?.values[item.sourceKey]; + if (value === undefined || value.length === 0) { + missingKeys.push({ sourceRef: item.sourceRef, sourceKey: item.sourceKey, targetKey: item.targetKey }); + } else { + data[item.targetKey] = value; + } + if (source?.unmaterializedGeneratedKeys.includes(item.sourceKey) === true) pendingGeneratedKeys.push(item.targetKey); + } + const keys = Object.keys(data); + return { + name: secret.name, + target, + secretName: secret.secretName, + type: secret.type, + data, + keySources: secret.data, + missingKeys, + pendingGeneratedKeys, + fingerprint: missingKeys.length === 0 && pendingGeneratedKeys.length === 0 ? fingerprintValues(data, keys) : null, + }; + }); +} + +async function applyTargetSecrets(config: UniDeskConfig, target: DistributionTarget, secrets: DesiredSecret[], options: SecretsOptions): Promise> { + if (secrets.length === 0) return { ok: true, target: targetSummary(target), mode: "skipped-no-secrets" }; + const yaml = renderSecretManifest(target, secrets); + const result = await capture(config, target.route, ["script"], applySecretScript(target, secrets, yaml)); + const parsed = parseJsonOutput(result.stdout); + return { + ok: result.exitCode === 0 && boolField(parsed, "ok", false), + target: targetSummary(target), + summary: parsed, + remote: secretCaptureSummary(result), + ...(options.raw ? { rawCaptureOmitted: true, rawPolicy: "Secret distribution never returns raw SSH capture because remote output can contain credential-bearing diagnostics." } : {}), + }; +} + +async function statusTargetSecrets(config: UniDeskConfig, target: DistributionTarget, secrets: DesiredSecret[], options: SecretsOptions): Promise> { + const result = await capture(config, target.route, ["script"], statusSecretScript(target, secrets)); + const parsed = parseJsonOutput(result.stdout); + return { + ok: result.exitCode === 0 && boolField(parsed, "ok", false), + target: targetSummary(target), + summary: parsed, + remote: secretCaptureSummary(result), + ...(options.raw ? { rawCaptureOmitted: true, rawPolicy: "Secret distribution never returns raw SSH capture because remote output can contain credential-bearing diagnostics." } : {}), + }; +} + +function secretCaptureSummary(result: SshCaptureResult): Record { + return { + exitCode: result.exitCode, + stdoutBytes: Buffer.byteLength(result.stdout, "utf8"), + stderrBytes: Buffer.byteLength(result.stderr, "utf8"), + stdoutTailOmitted: true, + stderrTailOmitted: true, + valuesPrinted: false, + }; +} + +function renderSecretManifest(target: DistributionTarget, secrets: DesiredSecret[]): string { + return secrets.map((secret) => `apiVersion: v1 +kind: Secret +metadata: + name: ${secret.secretName} + namespace: ${target.namespace} + labels: + app.kubernetes.io/managed-by: unidesk + app.kubernetes.io/part-of: ${target.scope} +type: ${secret.type} +data: +${Object.entries(secret.data).sort(([a], [b]) => a.localeCompare(b)).map(([key, value]) => ` ${key}: ${Buffer.from(value, "utf8").toString("base64")}`).join("\n")} +`).join("---\n"); +} + +function applySecretScript(target: DistributionTarget, secrets: DesiredSecret[], yaml: string): string { + const manifestB64 = Buffer.from(yaml, "utf8").toString("base64"); + const summaryB64 = Buffer.from(JSON.stringify(secrets.map(remoteSecretSummary)), "utf8").toString("base64"); + return ` +set -u +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +manifest="$tmp/secrets.yaml" +printf '%s' '${manifestB64}' | base64 -d >"$manifest" +kubectl create namespace ${target.namespace} --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f - >"$tmp/ns.out" 2>"$tmp/ns.err" +ns_rc=$? +if [ "$ns_rc" -eq 0 ]; then + kubectl apply --server-side --force-conflicts --field-manager=${fieldManager} -f "$manifest" >"$tmp/apply.out" 2>"$tmp/apply.err" + apply_rc=$? +else + : >"$tmp/apply.out" + printf '%s\\n' 'skipped because namespace sync failed' >"$tmp/apply.err" + apply_rc=1 +fi +python3 - "$ns_rc" "$apply_rc" "$tmp/ns.out" "$tmp/ns.err" "$tmp/apply.out" "$tmp/apply.err" <<'PY' +import base64, json, sys +ns_rc, apply_rc = int(sys.argv[1]), int(sys.argv[2]) +def text(path, limit=5000): + try: + return open(path, encoding="utf-8", errors="replace").read()[-limit:] + except FileNotFoundError: + return "" +payload = { + "ok": ns_rc == 0 and apply_rc == 0, + "namespace": "${target.namespace}", + "secrets": json.loads(base64.b64decode("${summaryB64}").decode("utf-8")), + "valuesPrinted": False, + "steps": { + "namespace": {"exitCode": ns_rc, "stdout": text(sys.argv[3]), "stderr": text(sys.argv[4])}, + "apply": {"exitCode": apply_rc, "stdout": text(sys.argv[5]), "stderr": text(sys.argv[6])}, + }, +} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if payload["ok"] else 1) +PY +`; +} + +function statusSecretScript(target: DistributionTarget, secrets: DesiredSecret[]): string { + const summaryB64 = Buffer.from(JSON.stringify(secrets.map(remoteSecretSummary)), "utf8").toString("base64"); + const commands = secrets.map((secret, index) => [ + `kubectl -n ${target.namespace} get secret ${secret.secretName} -o json >"$tmp/secret.${index}.json" 2>"$tmp/secret.${index}.err"`, + `printf '%s' "$?" >"$tmp/secret.${index}.rc"`, + ].join("\n")).join("\n"); + return ` +set -u +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +${commands} +python3 - "$tmp" <<'PY' +import base64, json, os, sys +tmp = sys.argv[1] +expected = json.loads(base64.b64decode("${summaryB64}").decode("utf-8")) +items = [] +ok = True +for index, item in enumerate(expected): + try: + rc = int(open(os.path.join(tmp, f"secret.{index}.rc"), encoding="utf-8").read() or "1") + except FileNotFoundError: + rc = 1 + try: + observed = json.load(open(os.path.join(tmp, f"secret.{index}.json"), encoding="utf-8")) + except Exception: + observed = None + data = (observed or {}).get("data") or {} + observed_keys = sorted(data.keys()) + expected_keys = item.get("keys") or [] + missing = [key for key in expected_keys if key not in observed_keys] + exists = rc == 0 + item_ok = exists and len(missing) == 0 + ok = ok and item_ok + items.append({ + "name": item.get("name"), + "secretName": item.get("secretName"), + "exists": exists, + "keys": observed_keys, + "expectedKeys": expected_keys, + "missingKeys": missing, + "ok": item_ok, + "valuesPrinted": False, + }) +payload = {"ok": ok, "namespace": "${target.namespace}", "secrets": items, "valuesPrinted": False} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if ok else 1) +PY +`; +} + +function groupDesiredSecretsByTarget(secrets: DesiredSecret[]): Array<{ target: DistributionTarget; secrets: DesiredSecret[] }> { + const groups = new Map(); + for (const secret of secrets) { + const existing = groups.get(secret.target.id); + if (existing === undefined) groups.set(secret.target.id, { target: secret.target, secrets: [secret] }); + else existing.secrets.push(secret); + } + return Array.from(groups.values()); +} + +function configSummary(config: SecretDistributionConfig, options: SecretsOptions): Record { + return { + path: config.configPath, + metadata: config.metadata, + root: redactRepoPath(secretRoot(config)), + scope: options.scope, + targetId: options.targetId, + sources: config.sources.files.map((item) => ({ sourceRef: item.sourceRef, requiredKeys: item.requiredKeys, createIfMissing: item.createIfMissing.enabled })), + targets: config.targets.filter((target) => (options.scope === null || target.scope === options.scope) && (options.targetId === null || target.id === options.targetId)).map(targetSummary), + valuesPrinted: false, + }; +} + +function sourceSummary(sources: SourceInspection): Record { + return { ok: sources.ok, root: sources.root, entries: sources.entries, valuesPrinted: false }; +} + +function desiredSecretSummary(secret: DesiredSecret): Record { + return { + name: secret.name, + target: targetSummary(secret.target), + secretName: secret.secretName, + keys: Object.keys(secret.data).sort(), + keySources: secret.keySources, + missingKeys: secret.missingKeys, + pendingGeneratedKeys: secret.pendingGeneratedKeys, + fingerprint: secret.fingerprint, + valuesPrinted: false, + }; +} + +function remoteSecretSummary(secret: DesiredSecret): Record { + return { + name: secret.name, + secretName: secret.secretName, + keys: Object.keys(secret.data).sort(), + fingerprint: secret.fingerprint, + valuesPrinted: false, + }; +} + +function targetSummary(target: DistributionTarget): Record { + return { id: target.id, route: target.route, namespace: target.namespace, scope: target.scope }; +} + +function readOptionValue(args: string[], index: number, option: string): string { + const value = args[index + 1]; + if (value === undefined || value.length === 0 || value.startsWith("--")) throw new Error(`${option} requires a value`); + return value; +} + +function resolveConfigPath(pathArg: string): string { + if (pathArg.startsWith("/") || pathArg.includes("\0")) throw new Error("--config must be a repo-relative YAML path"); + if (pathArg.includes("..")) throw new Error("--config must not contain .."); + return rootPath(pathArg); +} + +function displayConfigPath(pathArg: string): string { + if (pathArg.startsWith("/") || pathArg.includes("..")) throw new Error("--config must be a repo-relative YAML path without .."); + return pathArg; +} + +function secretRoot(config: SecretDistributionConfig): string { + const root = config.sources.root; + return isAbsolute(root) ? root : rootPath(root); +} + +function parseEnvFile(text: string): Record { + const result: Record = {}; + for (const rawLine of text.split(/\r?\n/u)) { + const line = rawLine.trim(); + if (line.length === 0 || line.startsWith("#")) continue; + const eq = line.indexOf("="); + if (eq <= 0) continue; + const key = line.slice(0, eq).trim(); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) continue; + result[key] = unquoteEnvValue(line.slice(eq + 1).trim()); + } + return result; +} + +function unquoteEnvValue(value: string): string { + if ((value.startsWith("'") && value.endsWith("'")) || (value.startsWith("\"") && value.endsWith("\""))) return value.slice(1, -1); + return value; +} + +function writeEnvFile(path: string, values: Record): void { + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const lines = Object.keys(values).sort().map((key) => `${key}=${quoteEnv(values[key])}`); + writeFileSync(path, `${lines.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); +} + +function quoteEnv(value: string): string { + if (/^[A-Za-z0-9_./:@%?&=+-]+$/u.test(value)) return value; + return `'${value.replaceAll("'", "'\"'\"'")}'`; +} + +function redactRepoPath(path: string): string { + const root = rootPath(); + return path.startsWith(`${root}/`) ? path.slice(root.length + 1) : path; +} + +function boolField(value: Record | null, key: string, fallback: boolean): boolean { + return typeof value?.[key] === "boolean" ? value[key] : fallback; +} + +function asRecord(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be a YAML object`); + return value as Record; +} + +function objectField(obj: Record, key: string, path: string): Record { + return asRecord(obj[key], `${path}.${key}`); +} + +function arrayOfRecords(value: unknown, path: string): Record[] { + if (!Array.isArray(value)) throw new Error(`${path} must be an array`); + return value.map((item, index) => asRecord(item, `${path}[${index}]`)); +} + +function stringField(obj: Record, key: string, path: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.trim().length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value.trim(); +} + +function integerField(obj: Record, key: string, path: string): number { + const value = obj[key]; + if (typeof value !== "number" || !Number.isInteger(value)) throw new Error(`${path}.${key} must be an integer`); + return value; +} + +function booleanField(obj: Record, key: string, path: string): boolean { + const value = obj[key]; + if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); + return value; +} + +function stringArrayField(obj: Record, key: string, path: string): string[] { + const value = obj[key]; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.trim().length === 0)) throw new Error(`${path}.${key} must be a string array`); + return value.map((item) => (item as string).trim()); +} + +function numberArrayField(obj: Record, key: string, path: string): number[] { + const value = obj[key]; + if (!Array.isArray(value) || value.some((item) => typeof item !== "number" || !Number.isInteger(item))) throw new Error(`${path}.${key} must be an integer array`); + return value as number[]; +} + +function stringMapField(obj: Record, key: string, path: string): Record { + const record = objectField(obj, key, path); + const result: Record = {}; + for (const [itemKey, itemValue] of Object.entries(record)) { + result[envKeyValue(itemKey, `${path}.${key}`)] = stringValue(itemValue, `${path}.${key}.${itemKey}`); + } + return result; +} + +function numberMapField(obj: Record, key: string, path: string): Record { + const record = objectField(obj, key, path); + const result: Record = {}; + for (const [itemKey, itemValue] of Object.entries(record)) { + result[envKeyValue(itemKey, `${path}.${key}`)] = randomByteCount(itemValue, `${path}.${key}.${itemKey}`); + } + return result; +} + +function randomBase64UrlSpec(value: unknown, path: string): { bytes: number; prefix: string } { + const record = asRecord(value, path); + return { + bytes: randomByteCount(record.bytes, `${path}.bytes`), + prefix: record.prefix === undefined ? "" : stringValue(record.prefix, `${path}.prefix`), + }; +} + +function stringValue(value: unknown, path: string): string { + if (typeof value !== "string") throw new Error(`${path} must be a string`); + return value; +} + +function randomByteCount(value: unknown, path: string): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < 16 || value > 128) throw new Error(`${path} must be an integer in 16..128`); + return value; +} + +function sourceRefField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (value.startsWith("/") || value.includes("..") || value.includes("\0") || !/^[A-Za-z0-9_./-]+$/u.test(value)) throw new Error(`${path}.${key} must be a relative source ref without ..`); + return value; +} + +function envKeyField(obj: Record, key: string, path: string): string { + return envKeyValue(stringField(obj, key, path), `${path}.${key}`); +} + +function envKeyValue(value: string, path: string): string { + if (!/^[A-Z_][A-Z0-9_]*$/u.test(value)) throw new Error(`${path} must be an uppercase env key`); + return value; +} + +function kubernetesNameField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value)) throw new Error(`${path}.${key} must be a Kubernetes name`); + return value; +} + +function kubernetesSecretKeyField(obj: Record, key: string, path: string): string { + const value = stringField(obj, key, path); + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path}.${key} must be a Kubernetes Secret key`); + return value; +} + +function simpleId(value: string, path: string): string { + if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a simple id`); + return value; +} diff --git a/src/components/frontend/src/index.ts b/src/components/frontend/src/index.ts index fab3edea..538a0654 100644 --- a/src/components/frontend/src/index.ts +++ b/src/components/frontend/src/index.ts @@ -15,6 +15,7 @@ interface RuntimeConfig { providerToken: string | null; sshClientToken: string | null; sshClientRouteAllowlist: string[]; + wechatArchiveToken: string | null; sessionSecret: string; sessionTtlSeconds: number; logFile: string; @@ -211,6 +212,8 @@ function readConfig(): RuntimeConfig { sshClientToken: optionalEnv("UNIDESK_SSH_CLIENT_TOKEN") ?? optionalFileEnv("UNIDESK_SSH_CLIENT_TOKEN_FILE", "/run/secrets/unidesk_ssh_client_token"), sshClientRouteAllowlist: parseRouteAllowlist(optionalEnv("UNIDESK_SSH_CLIENT_ROUTE_ALLOWLIST") ?? "G14,G14:*,D601,D601:*"), + wechatArchiveToken: optionalEnv("UNIDESK_WECHAT_ARCHIVE_TOKEN") + ?? optionalFileEnv("UNIDESK_WECHAT_ARCHIVE_TOKEN_FILE", "/tmp/unidesk-wechat-archive-token"), sessionSecret: requiredEnv("SESSION_SECRET"), sessionTtlSeconds: readNumberEnv("SESSION_TTL_SECONDS"), logFile: requiredEnv("LOG_FILE"), @@ -319,6 +322,7 @@ function trimPerformanceBuffers(): void { function classifyRequestComponent(pathname: string): string { if (pathname === "/api/frontend-performance") return "webui_performance"; + if (pathname === "/webhooks/wechat-archive") return "wechat_archive_webhook"; if (pathname.startsWith("/api/") || pathname === "/logs") return "webui_api_proxy"; if (pathname === "/login" || pathname === "/logout" || pathname === "/api/session") return "webui_auth"; if (pathname === "/app.js" || pathname.startsWith("/vendor/") || /\/[^/]+\.[a-z0-9]+$/iu.test(pathname)) return "webui_static"; @@ -789,6 +793,152 @@ function sessionResponse(req: Request): Response { return jsonResponse({ ok: true, authenticated: true, user: { username: session.username }, expiresAt: new Date(session.expiresAt).toISOString() }); } +async function readJsonRecord(req: Request, maxBytes: number): Promise> { + const bytes = await req.arrayBuffer(); + if (bytes.byteLength > maxBytes) throw new Error(`request body exceeds ${maxBytes} bytes`); + const text = Buffer.from(bytes).toString("utf8"); + const parsed = JSON.parse(text) as unknown; + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error("request body must be a JSON object"); + return parsed as Record; +} + +function recordFrom(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; +} + +function stringFrom(value: unknown): string { + return typeof value === "string" ? value : value === undefined || value === null ? "" : String(value); +} + +function wechatArchiveTokenFromRequest(req: Request): string | null { + return bearerTokenFromRequest(req) ?? req.headers.get("x-unidesk-wechat-archive-token")?.trim() ?? null; +} + +function isValidArchiveRemotePath(path: string): boolean { + return path.startsWith("/UniDesk/WeChatArchive/") + && !path.includes("\0") + && !path.split("/").includes("..") + && path.length <= 1024; +} + +function archiveContentPayload(body: Record): Record { + const message = recordFrom(body.message); + const archive = recordFrom(body.archive); + const media = recordFrom(message.media); + const langbotData = recordFrom(body.data); + const langbotMessage = recordFrom(langbotData.message); + const langbotComponents = Array.isArray(langbotMessage.root) ? langbotMessage.root : []; + const langbotPlain = langbotComponents + .filter((item) => recordFrom(item).type === "Plain") + .map((item) => stringFrom(recordFrom(item).text)) + .join(""); + const langbotImage = recordFrom(langbotComponents.find((item) => recordFrom(item).type === "Image")); + const messageType = stringFrom(message.messageType || body.messageType || "text").toLowerCase(); + const remotePath = stringFrom(archive.remotePath); + if (!isValidArchiveRemotePath(remotePath)) throw new Error("archive.remotePath is outside the WeChat archive root"); + const filename = stringFrom(archive.filename || remotePath.split("/").filter(Boolean).pop() || "wechat-archive.txt"); + if (messageType === "image") { + const dataBase64 = stringFrom(media.dataBase64 || archive.dataBase64 || body.dataBase64 || langbotImage.base64); + if (!dataBase64) throw new Error("image archive payload is missing media.dataBase64"); + return { dataBase64, filename, remotePath, maxBytes: 10 * 1024 * 1024 }; + } + const text = stringFrom(message.text || body.text || body.message || body.content || langbotPlain); + const payloadHash = new Bun.CryptoHasher("sha256").update(JSON.stringify(body)).digest("hex"); + const content = `${text}\n\n---\nsource=unidesk-wechat-archive\npayloadSha256=${payloadHash}\n`; + return { content, filename, remotePath, maxBytes: 10 * 1024 * 1024 }; +} + +async function coreJson(path: string, init?: RequestInit): Promise> { + const response = await fetch(new URL(path, config.coreInternalUrl), { + ...init, + headers: { + "content-type": "application/json", + ...(init?.headers instanceof Headers ? Object.fromEntries(init.headers.entries()) : init?.headers as Record | undefined), + }, + }); + const text = await response.text(); + let parsed: unknown = text; + try { + parsed = text.length > 0 ? JSON.parse(text) : null; + } catch { + parsed = text; + } + if (!response.ok) return { ok: false, status: response.status, body: parsed }; + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) + ? { ok: true, status: response.status, ...(parsed as Record) } + : { ok: true, status: response.status, body: parsed }; +} + +async function waitBaiduTransfer(jobId: string, timeoutMs: number): Promise> { + const deadline = Date.now() + timeoutMs; + let last: Record = { ok: false, error: "not-polled" }; + while (Date.now() < deadline) { + last = await coreJson(`/api/microservices/baidu-netdisk/proxy/api/transfers/${encodeURIComponent(jobId)}`); + const job = recordFrom(last.job); + const status = stringFrom(job.status || last.status); + if (status === "succeeded") return { ok: true, job }; + if (status === "failed" || status === "canceled") return { ok: false, job }; + await Bun.sleep(1000); + } + return { ok: false, error: "baidu-transfer-timeout", last }; +} + +function baiduFsIdFromJob(job: Record): string { + const result = recordFrom(job.result); + const baidu = recordFrom(result.baidu); + return stringFrom(baidu.fs_id || baidu.fsId || job.fsId); +} + +async function wechatArchiveWebhook(req: Request): Promise { + if (req.method !== "POST") return jsonResponse({ ok: false, error: "method not allowed" }, 405); + const expected = config.wechatArchiveToken; + const supplied = wechatArchiveTokenFromRequest(req); + if (expected === null || supplied === null || !timingSafeStringEqual(supplied, expected)) { + return jsonResponse({ ok: false, error: "authentication required", valuesPrinted: false }, 401); + } + const started = performance.now(); + try { + const body = await readJsonRecord(req, 2 * 1024 * 1024); + const uploadBody = archiveContentPayload(body); + const created = await coreJson("/api/microservices/baidu-netdisk/proxy/api/transfers/upload-content", { + method: "POST", + body: JSON.stringify(uploadBody), + }); + if (created.ok !== true) { + recordOperationPerformance("frontend", "wechat_archive_upload_create", performance.now() - started, false, stringFrom(created.status)); + return jsonResponse({ ok: false, error: "baidu upload create failed", upstream: created, valuesPrinted: false }, 502); + } + const job = recordFrom(created.job); + const jobId = stringFrom(job.id); + if (!jobId) throw new Error("baidu upload create did not return job id"); + const waited = await waitBaiduTransfer(jobId, 90_000); + const waitedJob = recordFrom(waited.job); + const fsId = baiduFsIdFromJob(waitedJob); + const remotePath = stringFrom(uploadBody.remotePath); + const ok = waited.ok === true && fsId.length > 0; + recordOperationPerformance("frontend", "wechat_archive_upload", performance.now() - started, ok, `${jobId}:${remotePath}`); + logger(ok ? "info" : "warn", "wechat_archive_upload_finished", { ok, jobId, remotePath, fsIdPresent: fsId.length > 0 }); + return jsonResponse({ + ok, + skip_pipeline: ok, + readOnly: true, + archive: { + remotePath, + fsId, + local: recordFrom(created.local), + uploadJob: waitedJob, + listed: null, + }, + valuesPrinted: false, + }, ok ? 200 : 502); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + recordOperationPerformance("frontend", "wechat_archive_upload", performance.now() - started, false, message); + logger("warn", "wechat_archive_upload_failed", { error: message }); + return jsonResponse({ ok: false, error: message, valuesPrinted: false }, 400); + } +} + async function proxyApi(req: Request, url: URL): Promise { if (sessionFromRequest(req) === null) { return jsonResponse({ ok: false, error: "authentication required" }, 401); @@ -952,6 +1102,7 @@ async function handleRequest(req: Request, server: Server): Prom if (url.pathname === "/logout" && req.method === "POST") return logout(); if (url.pathname === "/api/session") return sessionResponse(req); if (url.pathname === "/api/frontend-performance") return frontendPerformanceResponse(req); + if (url.pathname === "/webhooks/wechat-archive") return wechatArchiveWebhook(req); if (url.pathname === "/ws/ssh") return proxySshWebSocket(req, server); if (url.pathname.startsWith("/api/") || url.pathname === "/logs") return proxyApi(req, url); if (url.pathname === "/docs" || url.pathname.startsWith("/docs/")) return docsResponse(req, url); diff --git a/src/components/microservices/baidu-netdisk/src/index.ts b/src/components/microservices/baidu-netdisk/src/index.ts index 6363b201..c23bd13e 100644 --- a/src/components/microservices/baidu-netdisk/src/index.ts +++ b/src/components/microservices/baidu-netdisk/src/index.ts @@ -1032,6 +1032,38 @@ async function createTransferJob(direction: TransferDirection, body: JsonRecord) return { ok: true, job: transferFromRow(row) }; } +function safeStagingFilename(value: string, fallback: string): string { + const base = basename(value || fallback || "upload.txt").replace(/[^A-Za-z0-9._-]+/gu, "_").replace(/^_+|_+$/gu, ""); + return base || fallback || "upload.txt"; +} + +function contentUploadBuffer(body: JsonRecord): { buffer: Buffer; source: string } { + if (typeof body.dataBase64 === "string" && body.dataBase64.length > 0) return { buffer: Buffer.from(body.dataBase64, "base64"), source: "dataBase64" }; + if (typeof body.content === "string") return { buffer: Buffer.from(body.content, "utf8"), source: "content" }; + throw new HttpError(400, "upload-content requires content or dataBase64"); +} + +async function createContentUpload(body: JsonRecord): Promise { + const { buffer, source } = contentUploadBuffer(body); + const maxBytes = Math.max(1, Math.min(10 * 1024 * 1024, asNumber(body.maxBytes, 10 * 1024 * 1024))); + if (buffer.byteLength > maxBytes) throw new HttpError(413, "upload-content body is too large", { maxBytes, bytes: buffer.byteLength }); + const remotePath = remoteFilePath(String(body.remotePath || ""), safeStagingFilename(String(body.filename || ""), "upload.txt")); + const filename = safeStagingFilename(String(body.filename || pathPosix.basename(remotePath)), pathPosix.basename(remotePath) || "upload.txt"); + const relativePath = pathPosix.join("content-uploads", `${Date.now()}-${randomUUID().slice(0, 8)}-${filename}`); + const localPath = resolveStagingPath(relativePath); + mkdirSync(dirname(localPath), { recursive: true }); + await writeFile(localPath, buffer); + const sha256 = createHash("sha256").update(buffer).digest("hex"); + const created = await createTransferJob("upload", { localPath: relativePath, remotePath }); + return { + ok: created.ok === true, + source, + remotePath, + local: { relativePath, bytes: buffer.byteLength, sha256 }, + job: asRecord(created.job), + }; +} + async function computeMd5Blocks(filePath: string, jobId: string): Promise<{ size: number; fullMd5: string; blocks: string[] }> { const info = await stat(filePath); if (!info.isFile()) throw new HttpError(400, "localPath must be a file inside staging directory", { localPath: filePath }); @@ -1363,6 +1395,7 @@ async function route(req: Request): Promise { if (path === "/api/folders" && req.method === "POST") return jsonResponse(await createFolder(await readJsonBody(req))); if (path === "/api/files/manage" && req.method === "POST") return jsonResponse(await manageFiles(await readJsonBody(req))); if (path === "/api/transfers/upload-from-path" && req.method === "POST") return jsonResponse(await createTransferJob("upload", await readJsonBody(req))); + if (path === "/api/transfers/upload-content" && req.method === "POST") return jsonResponse(await createContentUpload(await readJsonBody(req))); if (path === "/api/transfers/download-to-path" && req.method === "POST") return jsonResponse(await createTransferJob("download", await readJsonBody(req))); if (path === "/api/self-test" && req.method === "POST") return jsonResponse(await runSelfTest(await readJsonBody(req))); if (path === "/api/transfers" && req.method === "GET") return jsonResponse(await listTransfers(url));