From c8561d392be857e8a0b79d6dc1b27dee5ece6b88 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 19 May 2026 15:22:56 +0000 Subject: [PATCH] feat: add baidu netdisk artifact delivery --- AGENTS.md | 8 +- docker-compose.yml | 5 + docs/reference/artifact-registry.md | 35 ++-- docs/reference/ci.md | 13 +- docs/reference/cli.md | 10 +- docs/reference/deploy.md | 15 +- docs/reference/deployment.md | 4 +- docs/reference/dev-environment.md | 2 +- docs/reference/microservices.md | 1 + docs/reference/user-service-delivery.md | 13 +- scripts/src/artifact-registry.ts | 179 ++++++++++++++---- scripts/src/check.ts | 1 + scripts/src/ci.ts | 20 +- scripts/src/deploy.ts | 80 ++++++-- scripts/src/help.ts | 5 +- .../microservices/baidu-netdisk/Dockerfile | 3 +- .../microservices/baidu-netdisk/src/index.ts | 11 ++ 17 files changed, 302 insertions(+), 103 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 55815287..e161eeb4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,10 +38,10 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - `bun scripts/cli.ts microservice list/status/health/diagnostics/tunnel-self-test/proxy`：管理和验证挂载在主 server、计算节点 Docker 或 k3s 控制面上的用户服务，`proxy` 支持受控 JSON body，OA Event Flow/Todo Note/Baidu Netdisk/Code Queue Manager on main-server、k3s Control/Code Queue 执行面/MDTODO/Decision Center/FindJob/Pipeline/MET Nonlinear on D601 的规则见 `docs/reference/microservices.md`。 - `bun scripts/cli.ts decision upload/list/show/health`：通过 backend-core 用户服务代理上传会议记录/决议 Markdown、列出记录和查看详情；Decision Center 运行在 D601 k3s，规则见 `docs/reference/microservices.md`。 - `bun scripts/cli.ts decision diary import/list/months/show`：把带日期标题的工作日志 Markdown 拆成 `YYYY-MM/YYYY-MM-DD.md` 日记条目并写入 PostgreSQL，规则见 `docs/reference/microservices.md`。 -- `bun scripts/cli.ts deploy check/plan/apply [--file deploy.json|--env dev|prod] [--service ]`：按根目录 `deploy.json` 或 `origin/master:deploy.json#environments.` 的服务 repo 和 commit 期望状态校验或更新用户服务；`--env dev` 当前只开放 D601 `backend-core`/`frontend` persistent dev rollout，规则见 `docs/reference/deploy.md` 与 `docs/reference/dev-environment.md`。 +- `bun scripts/cli.ts deploy check/plan/apply [--file deploy.json|--env dev|prod] [--service ]`：按根目录 `deploy.json` 或 `origin/master:deploy.json#environments.` 的服务 repo 和 commit 期望状态校验或更新用户服务；`--env dev` 当前开放 D601 `backend-core`/`frontend` persistent dev rollout 和 `baidu-netdisk` artifact-consumer validation，规则见 `docs/reference/deploy.md` 与 `docs/reference/dev-environment.md`。 - `bun scripts/cli.ts dev-env validate [--manifest path] [--kubectl-dry-run]` / `dev-env prewarm-images`：离线校验 D601 `unidesk-dev` 生产隔离护栏，或把开发底座基础镜像预热到 D601 原生 k3s containerd，规则见 `docs/reference/deploy.md` 与 `docs/reference/microservices.md`。 -- `bun scripts/cli.ts artifact-registry plan|render|status|health|install|deploy-backend-core`：管理 D601 host-managed CNCF Distribution registry，并通过短生命周期 relay 做 production backend-core pull-only artifact CD，规则见 `docs/reference/artifact-registry.md`。 -- `bun scripts/cli.ts ci install/status/run/publish-backend-core/run-dev-e2e/logs`：在 D601 原生 k3s 上安装和运行 Tekton CI，支持每 commit 检查、Code Queue 只读性能门禁、backend-core commit-pinned 镜像发布和手动触发的 `origin/master:deploy.json#environments.dev` 临时 namespace e2e；`run-dev-e2e` 的 Git 控制 runner、短 launcher 和 no-CD 边界见 `docs/reference/dev-ci-runner.md`，Tekton 规则见 `docs/reference/ci.md`。 +- `bun scripts/cli.ts artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service`：管理 D601 host-managed CNCF Distribution registry，并通过短生命周期 relay 或 D601 pull/import 做 commit-pinned pull-only artifact CD，规则见 `docs/reference/artifact-registry.md`。 +- `bun scripts/cli.ts ci install/status/run/publish-backend-core/publish-user-service/run-dev-e2e/logs`：在 D601 原生 k3s 上安装和运行 Tekton CI，支持每 commit 检查、Code Queue 只读性能门禁、backend-core 与 user-service commit-pinned 镜像发布和手动触发的 `origin/master:deploy.json#environments.dev` 临时 namespace e2e；`run-dev-e2e` 的 Git 控制 runner、短 launcher 和 no-CD 边界见 `docs/reference/dev-ci-runner.md`，Tekton 规则见 `docs/reference/ci.md`。 - `bun scripts/cli.ts codex deploy `：旧 Code Queue 兼容部署入口已禁用，原因是它会绕过受控部署边界直连 D601 部署 Code Queue；规则见 `docs/reference/codex-deploy.md`。 - `bun scripts/cli.ts codex submit [prompt] [--prompt-file path|--prompt-stdin] [--queue ]`：通过 backend-core 私有代理提交 Code Queue 任务；控制面默认走主 server `code-queue-mgr` 写入 PostgreSQL，`--dry-run` 可只检查请求体不入队，规则见 `docs/reference/cli.md`。 - `bun scripts/cli.ts codex task `：按 Code Queue 任务 ID 查询初始 prompt、最后 assistant message、工具调用摘要、attempt/judge/error 和耗时，便于新任务引用历史 session。 @@ -79,7 +79,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台；本文 - `docs/reference/devops-hygiene.md`：Git-backed deployment truth、dirty worktree/manual repair 边界、受限手动操作和 CI 私有仓库 source-auth 规则。 - `docs/reference/release-governance.md`：`release/v1` 稳定维护线、`master` 集成线、CI/CD server 版本固定、master CLI 兼容和 feature flag 治理规则；决策记录见 GitHub issue #6。 - `docs/reference/artifact-registry.md`：D601 host-managed CNCF Distribution registry、loopback-only 边界和 backend-core artifact CD 目标流程。 -- `docs/reference/user-service-delivery.md`：用户服务默认交付流程、CI 镜像构建与 registry、dev 自动测试、prod 拉镜像部署和 Decision Center 产品化需求管理规则。 +- `docs/reference/user-service-delivery.md`：用户服务默认交付流程、CI 镜像构建与 registry、Baidu Netdisk 主 server 直管微服务样板、dev 自动测试、prod 拉镜像部署和 Decision Center 产品化需求管理规则。 - `docs/reference/dev-environment.md`：D601 `unidesk-dev` persistent dev 环境、18083 dev frontend proxy、`deploy apply --env dev` 服务范围和 Rust backend-core 只在 D601 编译的边界。 - `docs/reference/ci.md`：D601 k3s Tekton CI、只读主数据库性能门禁和 CLI 入口规则。 - `docs/reference/dev-ci-runner.md`：`ci run-dev-e2e` 的 Git 控制 runner、短 launcher、结果目录和 no-CD 边界。 diff --git a/docker-compose.yml b/docker-compose.yml index c77e45f1..2bbf9fde 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -233,6 +233,11 @@ services: BAIDU_NETDISK_STAGING_DIR: "/data/staging" LOG_FILE: "/var/log/unidesk/${UNIDESK_LOG_DAY}/${UNIDESK_LOG_PREFIX}_baidu-netdisk.jsonl" UNIDESK_LOG_RETENTION_BYTES: "${UNIDESK_LOG_RETENTION_BYTES:-1GiB}" + UNIDESK_DEPLOY_REF: "${UNIDESK_BAIDU_NETDISK_DEPLOY_REF:-deploy.json#environments.prod.services.baidu-netdisk}" + UNIDESK_DEPLOY_SERVICE_ID: "${UNIDESK_BAIDU_NETDISK_DEPLOY_SERVICE_ID:-baidu-netdisk}" + UNIDESK_DEPLOY_REPO: "${UNIDESK_BAIDU_NETDISK_DEPLOY_REPO:-}" + UNIDESK_DEPLOY_COMMIT: "${UNIDESK_BAIDU_NETDISK_DEPLOY_COMMIT:-}" + UNIDESK_DEPLOY_REQUESTED_COMMIT: "${UNIDESK_BAIDU_NETDISK_DEPLOY_REQUESTED_COMMIT:-}" volumes: - ${UNIDESK_LOG_DIR}:/var/log/unidesk - ./.state/baidu-netdisk/staging:/data/staging diff --git a/docs/reference/artifact-registry.md b/docs/reference/artifact-registry.md index d60796ba..9be6ef8b 100644 --- a/docs/reference/artifact-registry.md +++ b/docs/reference/artifact-registry.md @@ -1,8 +1,8 @@ # D601 Artifact Registry -D601 artifact registry 是为 backend-core 轻量 CD 准备的本地镜像制品入口。它使用开源、成熟的 CNCF Distribution Docker registry，不自定义镜像协议，也不把镜像托管到第三方服务。 +D601 artifact registry 是为 backend-core 和标准用户服务轻量 CD 准备的本地镜像制品入口。它使用开源、成熟的 CNCF Distribution Docker registry，不自定义镜像协议，也不把镜像托管到第三方服务。 -backend-core 的长期分工是：CI 在 D601 构建并发布 commit-pinned 镜像，CD 在 master server 只拉取、替换和验证该镜像。registry 是这条链路的本地制品缓存，不是构建编排器，也不是生产部署控制面。 +backend-core 和 reviewed user services 的长期分工是：CI 在 D601 构建并发布 commit-pinned 镜像，CD 在运行目标只拉取、替换/导入和验证该镜像。registry 是这条链路的本地制品缓存，不是构建编排器，也不是生产部署控制面。 Production CI/CD runtime pinning and release-line boundaries follow `docs/reference/release-governance.md` and [GitHub issue #6](https://github.com/pikasTech/unidesk/issues/6). The registry may cache commit-pinned artifacts, but it must not become a floating replacement for `deploy.json`, `release/v1`, or `master` source history. @@ -24,7 +24,7 @@ registry 运行在 D601 host/WSL OS 上，由 systemd 管理 Docker Compose 项 这个服务和 `k3sctl-adapter` 一样位于 k3s 故障域外，但职责不同： - `k3sctl-adapter` 是 UniDesk 到 native k3s 的控制桥，属于 UniDesk 直管服务。 -- artifact registry 是 D601 host-managed 制品缓存基础设施，D601 CI 向它推送 commit-pinned 镜像，master server CD 从它消费镜像。 +- artifact registry 是 D601 host-managed 制品缓存基础设施，D601 CI 向它推送 commit-pinned 镜像，master server Compose CD 和 D601 k3s CD 从它消费镜像。 ## Dependency Boundary @@ -40,8 +40,8 @@ registry 运行期依赖应保持低且可手动维护： 它不得依赖： - k3s 控制面或任意 k3s namespace。 -- 第三方镜像托管服务作为 backend-core artifact source of truth。 -- main server backend-core 本地 Rust 编译。 +- 第三方镜像托管服务作为 artifact source of truth。 +- main server backend-core 本地 Rust 编译或 artifact-consumer 服务本地 Docker build。 - 公开 registry 端口或公网反向代理。 - k3s Service、NodePort、Ingress 或任何长期暴露的 registry 代理。 @@ -56,6 +56,8 @@ bun scripts/cli.ts artifact-registry install bun scripts/cli.ts artifact-registry status bun scripts/cli.ts artifact-registry health bun scripts/cli.ts artifact-registry deploy-backend-core --commit +bun scripts/cli.ts artifact-registry deploy-service --service baidu-netdisk --commit +bun scripts/cli.ts artifact-registry deploy-service --service decision-center --commit ``` `plan` 输出架构边界、依赖项、默认路径和 backend-core artifact CD 流程。`render` 输出 systemd unit、Compose 文件和 registry config 的完整内容与 SHA-256。`install --dry-run` 只列出将要执行的远端动作，不写 D601 文件、不启动容器、不 reload systemd。 @@ -64,13 +66,14 @@ bun scripts/cli.ts artifact-registry deploy-backend-core --commit `deploy-backend-core` 是 production backend-core 的 CD 入口。它必须先通过 CNCF Distribution HTTP API 确认 D601 registry 中已经存在 `unidesk/backend-core:`，随后通过 provider-gateway Host SSH 流式执行 `docker save | gzip`，在 master server 侧 `docker load`、retag、Compose `--no-build` recreate 和 live commit 验证；如果镜像不存在，应失败并要求先运行 CI artifact publication。 -`deploy-service` 是标准化后的最小通用 artifact consumer。它目前只支持 `backend-core` 和 `decision-center`，并且必须先通过 D601 registry 的 commit-pinned manifest 校验，再执行拉取、导入、部署和健康验证： +`deploy-service` 是标准化后的最小通用 artifact consumer。它目前只支持 `backend-core`、`baidu-netdisk` 和 `decision-center`，并且必须先通过 D601 registry 的 commit-pinned manifest 校验，再执行拉取、导入/retag、部署和健康验证： ```bash +bun scripts/cli.ts artifact-registry deploy-service --service baidu-netdisk --commit --run-now bun scripts/cli.ts artifact-registry deploy-service --service decision-center --commit --run-now ``` -dry-run 输出会暴露 registry probe URL、required labels、目标 image、部署形态和回滚信息。`decision-center` 的 prod 路径会在 D601 上验证 `unidesk-decision-center:` 是否存在、导入 native k3s containerd、更新 `decision-center` Deployment image/env/annotations，并通过 Kubernetes API service proxy 验证 `/health` 中的 `deploy.commit`。回滚信息通过同一 artifact consumer 的 `rollback` 字段暴露，提示操作者重新对一个旧 commit 运行相同命令，而不是切回 legacy maintenance-channel 构建。 +dry-run 输出会暴露 registry probe URL、required labels、目标 image、部署形态和回滚信息。`baidu-netdisk` 的 Compose 路径会通过 provider-gateway Host SSH 把 `unidesk/baidu-netdisk:` 流式拉到 master server，retag 为 `baidu-netdisk` 和 `baidu-netdisk:`，写入 `UNIDESK_BAIDU_NETDISK_DEPLOY_*`，只 recreate `baidu-netdisk` service，并验证容器 image label 与 `/health.deploy.commit`。`decision-center` 的 prod 路径会在 D601 上验证 `unidesk-decision-center:` 是否存在、导入 native k3s containerd、更新 `decision-center` Deployment image/env/annotations，并通过 Kubernetes API service proxy 验证 `/health` 中的 `deploy.commit`。回滚信息通过同一 artifact consumer 的 `rollback` 字段暴露，提示操作者重新对一个旧 commit 运行相同命令，而不是切回 legacy maintenance-channel 构建。 `status` 和 `health` 通过： @@ -102,19 +105,19 @@ docker compose -p unidesk-artifact-registry -f /home/ubuntu/.unidesk/artifact-re 手动维护必须遵守 Git-backed deployment truth：运行态可以临时修复，但长期配置应回到 `artifact-registry render` 的声明文件和本文档。若 D601 runtime 文件 hash 与 CLI 渲染结果不一致，`status` / `health` 会显示 mismatch，操作者需要确认这是受控升级还是漂移。 -## Backend-Core Artifact CD +## Artifact CD 目标流程是： 1. D601 artifact registry 已安装并通过 `health`。 -2. D601 CI 从 pushed Git checkout 构建 `unidesk/backend-core:`。 -3. CI 将镜像 push 到 `127.0.0.1:5000/unidesk/backend-core:`，并记录 image ref 与 digest。 -4. CD 在 master server 上确认目标 commit/tag/digest 存在。 -5. master server 通过 provider-gateway Host SSH 从 D601 registry 流式读取 commit-pinned 镜像 tar，不开放 registry 端口，也不使用第三方镜像托管。 -6. master server retag 为 Compose 使用的 backend-core 镜像名，并执行 `docker compose up -d --no-build --no-deps --force-recreate backend-core`。 +2. D601 CI 从 pushed Git checkout 构建 `unidesk/:`。 +3. CI 将镜像 push 到 `127.0.0.1:5000/unidesk/:`，并记录 image ref 与 digest。 +4. CD 在运行目标上确认目标 commit/tag/digest 存在。 +5. Compose runtime 通过 provider-gateway Host SSH 从 D601 registry 流式读取 commit-pinned 镜像 tar，不开放 registry 端口，也不使用第三方镜像托管。 +6. Compose runtime retag 为 Compose 使用的镜像名，并执行 `docker compose up -d --no-build --no-deps --force-recreate `。 7. 部署后通过 image label、runtime env、health payload 验证 live commit。 -Decision Center follows the same artifact-consumer pattern, except the runtime target is native k3s on D601 instead of the master-server Compose stack. The consumer must check the registry manifest, pull the commit-pinned image, import it into `/run/k3s/containerd/containerd.sock`, set the Deployment image to the commit tag, stamp `UNIDESK_DEPLOY_*` env/annotations, and reject an old healthy revision if the live commit does not match. +Baidu Netdisk is the first main-server direct user-service sample in this flow. Its dev validation command and prod CD command both consume `127.0.0.1:5000/unidesk/baidu-netdisk:` and must not build source on the master server. Decision Center follows the same artifact-consumer pattern, except the runtime target is native k3s on D601 instead of the master-server Compose stack. The consumer must check the registry manifest, pull the commit-pinned image, import it into `/run/k3s/containerd/containerd.sock`, set the Deployment image to the commit tag, stamp `UNIDESK_DEPLOY_*` env/annotations, and reject an old healthy revision if the live commit does not match. 这个 CD 路径必须满足： @@ -122,5 +125,5 @@ Decision Center follows the same artifact-consumer pattern, except the runtime t - 镜像 tag 必须 commit-pinned，不能用 mutable latest 作为部署真相。 - provider-gateway SSH image stream 是临时控制动作，不开放长期公网 registry。 - CI 可以有较多依赖；CD 只做拉取、retag、recreate 和 live commit 验证。 -- CD 不执行 `cargo build`、`docker build`、`docker compose build backend-core` 或任何等价的 Rust 构建动作。 -- `server rebuild backend-core` 仍不得作为 master server Rust 编译路径。 +- CD 不执行 `cargo build`、`docker build`、`docker compose build ` 或任何等价构建动作。 +- `server rebuild backend-core` 和 `server rebuild baidu-netdisk` 仍是维护/非标准路径，不得作为 artifact CD 完成证据。 diff --git a/docs/reference/ci.md b/docs/reference/ci.md index c0fd1411..96294ea6 100644 --- a/docs/reference/ci.md +++ b/docs/reference/ci.md @@ -78,7 +78,7 @@ The artifact registry contract and CD consumption path are defined in `docs/refe ## User-Service Artifact Publication -User-service image creation uses the same CI producer boundary as backend-core, but the service identity and Dockerfile come from the registered `config.json.microservices[]` entry. The minimal supported service is `decision-center`. +User-service image creation uses the same CI producer boundary as backend-core, but the service identity and Dockerfile come from the registered `config.json.microservices[]` entry. The reviewed sample services are `baidu-netdisk` and `decision-center`. The CI user-service artifact task must follow these rules: @@ -89,6 +89,14 @@ The CI user-service artifact task must follow these rules: - The command output must include the image ref, tag, digest, source commit and service id. The digest ref is suitable as immutable input for later dev/prod deployment work. - CI is an artifact producer only. It must not restart production services, call production `deploy apply`, mutate the production namespace, or change `deploy.json`. +Publish a Baidu Netdisk artifact: + +```bash +bun scripts/cli.ts ci publish-user-service --service baidu-netdisk --commit --wait-ms 1200000 +``` + +This command creates the `unidesk-user-service-artifact-publish` Tekton PipelineRun and pushes `127.0.0.1:5000/unidesk/baidu-netdisk:`. + Publish a Decision Center artifact: ```bash @@ -146,10 +154,11 @@ This command creates the `unidesk-backend-core-artifact-publish` Tekton Pipeline Publish a user-service artifact: ```bash +bun scripts/cli.ts ci publish-user-service --service baidu-netdisk --commit --wait-ms 1200000 bun scripts/cli.ts ci publish-user-service --service decision-center --commit --wait-ms 1200000 ``` -This command is a CI producer action only. For Decision Center, it builds and pushes `127.0.0.1:5000/unidesk/decision-center:` and reports the immutable digest without deploying production. +This command is a CI producer action only. For Baidu Netdisk and Decision Center, it builds and pushes `127.0.0.1:5000/unidesk/:` and reports the immutable digest without deploying production. Run the dev namespace e2e harness manually: diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 92de6bde..ff7ddb0d 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -25,11 +25,11 @@ CLI 可以从 `master` 快速演进，但必须兼容 `deploy.json` 固定的 CI - `microservice list/status/health/diagnostics/tunnel-self-test/proxy` 通过 backend-core 内网 API 管理挂载在计算节点 Docker 或 k3s 控制面中的用户服务（底层命令名仍为 microservice）；`health`、`diagnostics`、`tunnel-self-test` 和 `proxy` 会走真实 backend-core -> provider-gateway 或 k3sctl-adapter -> 节点服务链路，`proxy` 支持受控 JSON 请求体并对超大响应 body 默认输出有界预览，规则见 `docs/reference/microservices.md`。 - `decision upload/list/show/health` 通过 backend-core 用户服务代理访问 D601 k3s Decision Center，用于上传会议记录/决议 Markdown、列出权威记录、查看详情和健康检查；`decision requirement list/upsert` 在同一 records 模型上管理 `goal|decision|blocker|debt|experiment` 需求记录。它们不得直连 D601 Service、NodePort 或 provider-gateway 业务 HTTP。 - `decision diary import ` 将带 `# YYYY年M月D日`、`# YYYY-MM-DD` 或 `# YYYY/M/D` 标题的工作日志拆成每天一篇 Markdown 日记，按 `YYYY-MM/YYYY-MM-DD.md` 虚拟路径写入 Decision Center PostgreSQL；`decision diary list/months/show` 分别用于按月/日期查询、列出月份和查看单日正文；`decision diary edit|upsert --body-file [--title text] [--source-file path] [--tag tag]` 通过 `PUT /api/diary/entries/:idOrDate` 创建当天或历史条目并编辑既有条目。 -- `deploy check/plan/apply` 默认从根目录 `deploy.json` 读取服务 repo 与 commit 期望状态，join `config.json` 和现有 manifest 后使用 target-side build 单一路径校验或更新已支持目标；`deploy plan --env dev|prod` 只从 `origin/master:deploy.json#environments.` 读取 manifest 并输出 dry-run 环境计划，不使用本地 dirty worktree；当前 `deploy apply --env dev` 只支持 D601 `backend-core` 与 `frontend` persistent dev rollout，dev desired-state smoke 使用 `ci run-dev-e2e`；规则见 `docs/reference/deploy.md`、`docs/reference/dev-environment.md` 和 `docs/reference/dev-ci-runner.md`。 +- `deploy check/plan/apply` 默认从根目录 `deploy.json` 读取服务 repo 与 commit 期望状态，join `config.json` 和现有 manifest 后使用 target-side build 单一路径校验或更新已支持目标；`deploy plan --env dev|prod` 只从 `origin/master:deploy.json#environments.` 读取 manifest 并输出 dry-run 环境计划，不使用本地 dirty worktree；当前 `deploy apply --env dev` 支持 D601 `backend-core`/`frontend` persistent dev rollout，以及 `baidu-netdisk` artifact-consumer validation，dev desired-state smoke 使用 `ci run-dev-e2e`；规则见 `docs/reference/deploy.md`、`docs/reference/dev-environment.md` 和 `docs/reference/dev-ci-runner.md`。 - `dev-env validate [--manifest path] [--kubectl-dry-run]` 离线校验 D601 `unidesk-dev` namespace、dev PostgreSQL 底座和 dev workload manifest。默认检查 `src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-foundation.k8s.yaml`；也可显式校验 `src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-core.k8s.yaml` 或 `src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-code-queue.k8s.yaml`。所有 namespaced 对象必须只落到 `unidesk-dev`，foundation manifest 必须包含 `postgres-dev` StatefulSet/Service、dev secret/config、迁移 Job 和 DB URL guard，core manifest 必须包含 `backend-core-dev`/`frontend-dev` Deployment/Service，Code Queue dev manifest 必须包含 `code-queue-scheduler-dev`、`code-queue-read-dev`、`code-queue-write-dev` 和 dev provider egress proxy。加 `--kubectl-dry-run` 时额外执行 `kubectl apply --dry-run=client --validate=false -f `，仍不 apply 资源。 - `dev-env prewarm-images [--image image] [--provider-id D601] [--no-pull] [--proxy-url URL] [--pull-timeout-ms N] [--dry-run]` 创建异步 job，通过 UniDesk SSH 维护桥在 D601 上把开发底座依赖镜像从 Docker 缓存导入原生 k3s containerd。默认镜像是 `postgres:16-alpine` 和 `rancher/mirrored-library-busybox:1.36.1`，用于避免 `postgres-dev` 与 local-path helper pod 卡在外部 registry 拉取。该命令固定验证 `/etc/rancher/k3s/k3s.yaml` 指向的 native k3s 上下文，并输出 `dev_env_containerd_image_ready=...` 作为成功判据；它不 apply manifest、不修改生产 `unidesk` namespace。 -- `artifact-registry plan|render|status|health|install|deploy-backend-core` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 production backend-core pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`，由 systemd + Docker Compose 管理，位于 native k3s 故障域外；`deploy-backend-core` 只通过短生命周期 localhost relay 拉取 CI 已发布的 commit-pinned 镜像、retag、`--no-build` recreate 和 live commit 验证，不构建 backend-core。长期规则见 `docs/reference/artifact-registry.md`。 -- `ci install|status|run|publish-backend-core|run-dev-e2e|logs` 管理 D601 原生 k3s 上的 Tekton CI。`run` 手动创建每 commit 检查和 Code Queue 只读性能门禁；`publish-backend-core` 从 pushed Git commit 构建并发布 `127.0.0.1:5000/unidesk/backend-core:`，但不部署生产；`run-dev-e2e` 的 Git 控制 runner、短 launcher、host fetch 边界、临时 smoke namespace 和 no-CD 规则只在 `docs/reference/dev-ci-runner.md` 定义；Tekton CI 通用规则见 `docs/reference/ci.md`。 +- `artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`，由 systemd + Docker Compose 管理，位于 native k3s 故障域外；`deploy-backend-core` 和 `deploy-service` 只拉取 CI 已发布的 commit-pinned 镜像、retag/recreate 或导入 native k3s，并做 live commit 验证，不构建 runtime source。长期规则见 `docs/reference/artifact-registry.md`。 +- `ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs` 管理 D601 原生 k3s 上的 Tekton CI。`run` 手动创建每 commit 检查和 Code Queue 只读性能门禁；`publish-backend-core` 与 `publish-user-service` 从 pushed Git commit 构建并发布 `127.0.0.1:5000/unidesk/:`，但不部署生产；`run-dev-e2e` 的 Git 控制 runner、短 launcher、host fetch 边界、临时 smoke namespace 和 no-CD 规则只在 `docs/reference/dev-ci-runner.md` 定义；Tekton CI 通用规则见 `docs/reference/ci.md`。 - `codex deploy ` 是旧 Code Queue 兼容部署入口，已禁用以防止维护通道直连 D601 部署 Code Queue；当前 dev 自动化只做 `ci run-dev-e2e` smoke，不提供 Code Queue CD，详细规则见 `docs/reference/codex-deploy.md`。 - `codex submit [prompt] [--prompt-file path|--prompt-stdin] [--queue queueId] [--provider-id id] [--cwd path] [--model model] [--reasoning-effort effort] [--execution-mode mode] [--max-attempts N] [--reference-task-id id] [--dry-run]` 通过 backend-core 私有代理向稳定 `code-queue` 用户服务路径提交任务；prompt 必须且只能来自位置参数、文件或 stdin 之一，`--dry-run` 只返回结构化请求且不实际入队。提交确认和 dry-run 必须返回完整 prompt、字符数和 `truncated=false`，不能套用任务详情的预览截断策略，否则长任务 prompt 无法被人工验收。backend-core 默认把提交、队列 CRUD、已读状态、历史摘要和轻量 Trace 读取分流到主 server `code-queue-mgr`，由它写入主 PostgreSQL；D601 scheduler 只轮询并执行已入库任务。 - `codex task ` 通过 Code Queue 私有代理按任务 ID 查询结构化执行摘要；默认只返回有界 prompt/response 预览、执行 Provider、工作目录、最后 assistant message、最近工具调用摘要、attempt、judge、错误、耗时和 trace 翻页提示，适合在新队列任务中引用历史 session 且避免噪声爆炸。该摘要读取默认由主 server `code-queue-mgr` 从 PostgreSQL 返回，不依赖 D601 `code-queue-read` Service 可用。 @@ -47,9 +47,9 @@ CLI 可以从 `master` 快速演进，但必须兼容 `deploy.json` 固定的 CI 长时操作采用 Fire-and-Forget 模式：CLI 创建 `.state/jobs/{jobId}.json`，后台进程执行真实命令，并将 stdout、stderr 分别写入 `.state/jobs/{jobId}.stdout.log` 与 `.state/jobs/{jobId}.stderr.log`。调用者通过 `bun scripts/cli.ts job status ` 查询进度和尾部输出。 -`server rebuild` 与 `server start`、`server stop` 一样必须通过返回的 job id 确认结果；不要把连续 `server rebuild` 命令理解成“前一个重建已完成”，因为两个命令只是在快速创建异步 job。重建 frontend 的标准流程是运行 `bun scripts/cli.ts server rebuild frontend`，随后轮询 `bun scripts/cli.ts job status ` 到 `succeeded`，再用 `server status` 或 `e2e run` 验证公网 frontend；重建 dev 入口薄代理使用 `bun scripts/cli.ts server rebuild dev-frontend-proxy`，随后验证 `server status` 的 `urls.devFrontend` 和 `http://127.0.0.1:18083/health`；重建 Todo Note 后端使用 `bun scripts/cli.ts server rebuild todo-note`，随后用 `microservice health todo-note` 和 `microservice proxy todo-note /api/instances` 验证；重建 Code Queue Manager 使用 `bun scripts/cli.ts server rebuild code-queue-mgr`，随后用 `microservice health code-queue-mgr`、`microservice health code-queue` 和 `codex submit --dry-run` 验证主 server 控制面路径；重建 Project Manager 后端使用 `bun scripts/cli.ts server rebuild project-manager`，随后用 `microservice health project-manager` 和 `microservice proxy project-manager /api/projects` 验证；重建 Baidu Netdisk 后端使用 `bun scripts/cli.ts server rebuild baidu-netdisk`，随后用 `microservice health baidu-netdisk` 和 `microservice proxy baidu-netdisk /api/transfers` 验证；重建 OA Event Flow 后端使用 `bun scripts/cli.ts server rebuild oa-event-flow`，随后用 `microservice health oa-event-flow` 和 `microservice proxy oa-event-flow /api/diagnostics` 验证。D601 Code Queue 执行面和 Decision Center 后端由 D601 k3s/k8s 控制面代管；persistent dev backend-core/frontend 只通过 `deploy apply --env dev --service ...`，当前 Code Queue/Decision Center 仍不得通过维护通道直连 D601 做部署。不得把 `docker rm` 手工兜底当成正式交付步骤。 +`server rebuild` 与 `server start`、`server stop` 一样必须通过返回的 job id 确认结果；不要把连续 `server rebuild` 命令理解成“前一个重建已完成”，因为两个命令只是在快速创建异步 job。重建 frontend 的标准流程是运行 `bun scripts/cli.ts server rebuild frontend`，随后轮询 `bun scripts/cli.ts job status ` 到 `succeeded`，再用 `server status` 或 `e2e run` 验证公网 frontend；重建 dev 入口薄代理使用 `bun scripts/cli.ts server rebuild dev-frontend-proxy`，随后验证 `server status` 的 `urls.devFrontend` 和 `http://127.0.0.1:18083/health`；重建 Todo Note 后端使用 `bun scripts/cli.ts server rebuild todo-note`，随后用 `microservice health todo-note` 和 `microservice proxy todo-note /api/instances` 验证；重建 Code Queue Manager 使用 `bun scripts/cli.ts server rebuild code-queue-mgr`，随后用 `microservice health code-queue-mgr`、`microservice health code-queue` 和 `codex submit --dry-run` 验证主 server 控制面路径；重建 Project Manager 后端使用 `bun scripts/cli.ts server rebuild project-manager`，随后用 `microservice health project-manager` 和 `microservice proxy project-manager /api/projects` 验证；重建 Baidu Netdisk 后端使用 `bun scripts/cli.ts server rebuild baidu-netdisk`，随后用 `microservice health baidu-netdisk` 和 `microservice proxy baidu-netdisk /api/transfers` 验证，但该命令只保留为维护/非标准路径；重建 OA Event Flow 后端使用 `bun scripts/cli.ts server rebuild oa-event-flow`，随后用 `microservice health oa-event-flow` 和 `microservice proxy oa-event-flow /api/diagnostics` 验证。D601 Code Queue 执行面和 Decision Center 后端由 D601 k3s/k8s 控制面代管；persistent dev backend-core/frontend 只通过 `deploy apply --env dev --service ...`，当前 Code Queue/Decision Center 仍不得通过维护通道直连 D601 做部署。不得把 `docker rm` 手工兜底当成正式交付步骤。 -新部署入口优先使用 `deploy apply`。`deploy apply --env dev --service backend-core|frontend` 已收敛到 D601 target-side dev 路径；旧的 `codex deploy` 已禁用；后续 Code Queue、Decision Center 等 D601 服务部署应另行收敛到同一类受控 target-side CD 路径：从 remote commit 导出源码，在目标节点一次性代理构建镜像，部署后用 live commit 校验证明不是旧服务。 +新部署入口优先使用 `deploy apply`。`deploy apply --env dev --service backend-core|frontend` 已收敛到 D601 target-side dev 路径；`deploy apply --env dev|prod --service baidu-netdisk` 是主 server 直管微服务的 artifact-consumer 样板；旧的 `codex deploy` 已禁用；后续 Code Queue、Decision Center 等 D601 服务部署应另行收敛到同一类受控 CD 路径，部署后用 live commit 校验证明不是旧服务。 ## Output Contract diff --git a/docs/reference/deploy.md b/docs/reference/deploy.md index 25bb227b..8ddfa9c3 100644 --- a/docs/reference/deploy.md +++ b/docs/reference/deploy.md @@ -40,7 +40,7 @@ The root `deploy.json` is the single desired-state source for both prod and dev. The optional non-service execution declaration under `environments.dev` is intentionally not specified here. The only currently allowed declaration is `ci`, and its authoritative `repo`, `scriptPath`, `timeoutMs`, short launcher, host fetch boundary and no-CD rules are defined only in `docs/reference/dev-ci-runner.md`. -Environment mode never reads the local dirty working tree manifest. `deploy check --env ...`, `deploy plan --env ...` and `deploy apply --env ...` fetch `origin/master`, read `origin/master:deploy.json`, select `environments.`, and report the manifest commit/blob, service commit IDs, target namespace, database fingerprint and Provider identity. `deploy apply --env dev` is currently enabled only for persistent D601 dev `backend-core` and `frontend`; all other D601 services remain rejected before runtime mutation. `deploy apply --env prod` remains disabled until the production environment executor and authorization policy are explicitly added. Production backend-core artifact CD is a separate executor because its build target is D601 CI while its runtime target is the master server. The default user-service delivery policy, including CI build, registry publication, dev validation, production CD and manual acceptance, is documented in `docs/reference/user-service-delivery.md`. +Environment mode never reads the local dirty working tree manifest. `deploy check --env ...`, `deploy plan --env ...` and `deploy apply --env ...` fetch `origin/master`, read `origin/master:deploy.json`, select `environments.`, and report the manifest commit/blob, service commit IDs, target namespace, database fingerprint and Provider identity. `deploy apply --env dev` is currently enabled for persistent D601 dev `backend-core`/`frontend` and for `baidu-netdisk` artifact-consumer validation; all other D601 services remain rejected before runtime mutation. `deploy apply --env prod` exposes only reviewed D601 registry artifact consumers (`backend-core`, `baidu-netdisk`, `decision-center`). The default user-service delivery policy, including CI build, registry publication, dev validation, production CD and manual acceptance, is documented in `docs/reference/user-service-delivery.md`. The current implementation has not yet enabled separate stable and integration dev lanes. Future lane names such as `dev-v1` and `dev-master`, or an equivalent nested schema, must be added as explicit `deploy.json` and CLI semantics before use. A deploy command must print the manifest ref it used and must not infer `release/v1` from a local branch, a dirty file, or an undocumented environment alias. @@ -106,7 +106,7 @@ Maintenance-channel direct D601 apply must not deploy dev Code Queue; the CLI re `bun scripts/cli.ts deploy plan --env dev [--service ]` reads `origin/master:deploy.json#environments.dev` and prints a dry-run environment plan without checking or mutating live runtime resources. `deploy check --env dev` uses the same dry-run environment plan. `--env prod` is available for parity as a dry-run planning path; it reads `origin/master:deploy.json#environments.prod` and must not use a dirty local `deploy.json`. -`bun scripts/cli.ts deploy apply [--file deploy.json | --env dev|prod] [--service ] [--commit ] [--dry-run] [--force]` starts an asynchronous job only for supported targets. Use `bun scripts/cli.ts job status --tail-bytes 30000` to observe progress. `--dry-run` resolves the same plan but does not build or replace runtime objects. `--force` rebuilds even when the live commit matches. Environment apply is not the dev e2e trigger; use `bun scripts/cli.ts ci run-dev-e2e` for the Git-controlled temporary namespace smoke flow. `--env dev` apply is enabled only for `backend-core` and `frontend`; `--env prod` apply now exposes the D601 registry artifact consumer for `backend-core` and `decision-center`. Unsupported prod services return a structured `unsupported` payload instead of silently falling back to a maintenance-channel source build. +`bun scripts/cli.ts deploy apply [--file deploy.json | --env dev|prod] [--service ] [--commit ] [--dry-run] [--force]` starts an asynchronous job only for supported targets. Use `bun scripts/cli.ts job status --tail-bytes 30000` to observe progress. `--dry-run` resolves the same plan but does not build or replace runtime objects. `--force` rebuilds even when the live commit matches. Environment apply is not the dev e2e trigger; use `bun scripts/cli.ts ci run-dev-e2e` for the Git-controlled temporary namespace smoke flow. `--env dev` apply is enabled for persistent D601 `backend-core`/`frontend` target-side rollout and for `baidu-netdisk` artifact-consumer validation. `--env prod` apply exposes the D601 registry artifact consumer for `backend-core`, `baidu-netdisk`, and `decision-center`. Unsupported prod services return a structured `unsupported` payload instead of silently falling back to a maintenance-channel source build. All deploy commands output JSON. Long operations must use `.state/jobs/` and bounded log tails; no deploy path may succeed with missing progress output. @@ -123,9 +123,9 @@ The reconciler distributes only repository URL, commit ID, Dockerfile path, buil Each target fetches the remote repository, resolves the requested commit to a full 40 character SHA and exports tracked files with `git archive`. Build contexts are created from that archive, not from the operator's current working tree. Environment applies such as `deploy apply --env dev` must not upload Kubernetes manifests or source files from the master server worktree; the target-side materialized commit is the source for Dockerfile, build context and k3s control manifests. The master server side may only do lightweight CLI orchestration, environment ref reading and remote command dispatch. -## Backend-Core Artifact Exception +## Artifact Consumer Exception -Production backend-core is the explicit exception to standard target-side build. The runtime target is the master server, but the build target is D601 CI because backend-core Rust compilation is too expensive for the low-resource master server. +Production backend-core and reviewed user-service samples are explicit exceptions to standard target-side build. The runtime target can be the master server Compose stack, but the build target is D601 CI; CD then consumes only commit-pinned images from the D601 artifact registry. The exception is narrow: @@ -133,6 +133,7 @@ The exception is narrow: - CD on the master server pulls that existing image through the controlled artifact-registry relay, retags it for the Compose service, recreates only `backend-core` with `--no-build --no-deps --force-recreate`, and verifies the running commit. - CD must not run Rust compilation, Docker build, Compose build or `server rebuild backend-core`. - The pushed Git commit remains the version source of truth. The image registry is a content cache and transfer boundary, not a replacement for `deploy.json` or Git. +- `baidu-netdisk` is the first main-server direct user-service sample for the same split: CI publishes `127.0.0.1:5000/unidesk/baidu-netdisk:` from `src/components/microservices/baidu-netdisk/Dockerfile`; dev validation and prod CD both pull that artifact, retag `baidu-netdisk`, recreate only `baidu-netdisk` with `--no-build --no-deps --force-recreate`, and verify image labels plus `/health.deploy.commit`. - This exception must not be generalized to other services unless their resource profile and runtime boundary are documented with the same CI-producer/CD-consumer split. The registry contract is defined in `docs/reference/artifact-registry.md`; the CI producer rules are defined in `docs/reference/ci.md`. @@ -157,7 +158,7 @@ Main server targets may build without a proxy unless a service explicitly requir The reconciler selects the executor from `config.json`: -- `deployment.mode=unidesk-direct` on `main-server`: build the image on the main server, then use the fixed UniDesk Compose project and `up -d --no-build --no-deps --force-recreate `. +- `deployment.mode=unidesk-direct` on `main-server`: the legacy/local manifest executor builds the image on the main server, then uses the fixed UniDesk Compose project and `up -d --no-build --no-deps --force-recreate `. Reviewed artifact-consumer services such as `baidu-netdisk` use the D601 registry pull-only path for `--env dev` and `--env prod` instead. - `deployment.mode=internal-sidecar` on `main-server`: use the same main-server target-side source export, Docker build, image label stamping, fixed Compose project replacement and live commit verification as direct Compose services. This class is for private sidecars such as `code-queue-mgr`; it is still versioned by `deploy.json.commitId`, not by the operator's current worktree. - `deployment.mode=unidesk-direct` on a provider: this executor is disabled for D601 service deployment. The historical behavior dispatched `host.ssh` to the provider, built on the provider, then used the service's provider-local compose file and project; that shape must not remain a second deployment control plane. - Control bridges that UniDesk needs in order to inspect or repair an orchestrator must stay in this direct class. In particular, `k3sctl-adapter` is a UniDesk-managed bridge to native k3s and must remain outside k3s; Docker packaging on Docker Desktop/WSL must create an explicit host-local bridge, currently an adapter-container SSH local tunnel, to reach `/etc/rancher/k3s/k3s.yaml` and WSL `127.0.0.1:6443`. @@ -169,11 +170,13 @@ Code Queue health and diagnostics must cover its k3s dependencies, not only sche Existing service-specific commands such as Code Queue deploy are disabled as direct D601 deploy paths. Their build/import/rollout semantics should converge later into one controlled target-side deployment path instead of keeping parallel implementations. +Baidu Netdisk is the main-server `unidesk-direct` sample for artifact CD. Controlled dev validation and prod CD use the D601 registry artifact consumer: it verifies `unidesk/baidu-netdisk:` exists in the registry, streams the image to the main server through provider-gateway Host SSH, retags `baidu-netdisk` and `baidu-netdisk:`, stamps `UNIDESK_BAIDU_NETDISK_DEPLOY_*`, recreates only Compose service `baidu-netdisk`, and verifies container health, image labels, service id, and `/health.deploy.commit`. It must not use `server rebuild baidu-netdisk`, mutable tags, dirty worktrees, hand-built images, or public `4244` exposure as deployment truth. + Decision Center is a standard `k3sctl-managed` service in this model, but D601 maintenance-channel direct apply must not deploy it. Controlled CD for Decision Center uses the D601 registry artifact consumer: it verifies `unidesk/decision-center:` exists in the registry, imports `unidesk-decision-center:` into native k3s containerd, applies `src/components/microservices/k3sctl-adapter/k3s/decision-center.k8s.yaml`, stamps the Deployment, and verifies health through `/api/microservices/decision-center/health`. It must not add a main-server Compose service, NodePort, hostPort, or provider-gateway direct HTTP backend for Decision Center. ## CI Separation -Continuous integration is intentionally separate from this deploy reconciler. D601 k3s hosts Tekton CI resources described in `docs/reference/ci.md`; PipelineRuns may clone, check, run read-only performance gates, create temporary CI-owned namespaces for dev manifest smoke e2e, or publish commit-pinned backend-core image artifacts to the D601 artifact registry. They must not call `deploy apply`, `codex deploy`, `kubectl rollout restart` for production services, mutate `deploy.json`, or write production namespaces. +Continuous integration is intentionally separate from this deploy reconciler. D601 k3s hosts Tekton CI resources described in `docs/reference/ci.md`; PipelineRuns may clone, check, run read-only performance gates, create temporary CI-owned namespaces for dev manifest smoke e2e, or publish commit-pinned backend-core/user-service image artifacts to the D601 artifact registry. They must not call `deploy apply`, `codex deploy`, `kubectl rollout restart` for production services, mutate `deploy.json`, or write production namespaces. The Code Queue performance gate may create a temporary `code-queue-ci-read` service and read the main PostgreSQL through the existing `d601-tcp-egress-gateway`. Because it runs with `CODE_QUEUE_SERVICE_ROLE=read`, scheduler/backfill/notification disabled and EmptyDir state, it is not deployment truth and does not need a temporary database for the current read-only checks. diff --git a/docs/reference/deployment.md b/docs/reference/deployment.md index dc9993ad..d75e5aa0 100644 --- a/docs/reference/deployment.md +++ b/docs/reference/deployment.md @@ -30,7 +30,7 @@ CLI 会优先使用 `docker compose` v2 plugin；当 v2 plugin 不存在时才 Compose v2 安装后仍然必须遵守 UniDesk 的服务控制入口：全栈生命周期用 `server start` / `server stop`，单服务重建用 `server rebuild `。不要因为 v2 可用就直接在生产栈上手工执行未纳入 CLI 的 `up --build`、`down -v` 或跨项目清理命令；所有会影响容器的动作都应保持 job 可观测、Compose project 固定、database named volume 保留。主 server Compose 命令必须从 `providerGateway.upgrade.hostProjectRoot` 指定的 canonical UniDesk 根目录运行，临时 worktree、Code Queue 导出目录或实验分支不得复用生产 `-p unidesk` 和固定 `container_name` 去替换生产容器。 -版本化用户服务部署优先使用 `bun scripts/cli.ts deploy apply` 已支持的受控路径；D601 persistent dev apply 当前只支持 `backend-core` 和 `frontend`，dev desired-state smoke 使用 `ci run-dev-e2e`。`deploy.json` 只声明服务 `id`、`repo` 和 `commitId`；目标节点、Dockerfile、Compose、Kubernetes manifest、健康检查和代理路径继续来自 `config.json` 与现有 manifest。主 server 直管微服务和内部 sidecar，例如 `code-queue-mgr`，也必须支持这一路径：`deploy apply --service code-queue-mgr` 从 `deploy.json` 指定 commit 导出源码、构建镜像、替换固定 Compose service 并验证运行中镜像/健康信息的 commit。部署默认遵循 target-side build：服务部署到哪台 target，就在哪台 target 从 remote commit 导出源码、一次性代理构建镜像并部署；不得把中心构建镜像作为默认分发路径，也不得用 `docker commit` 或脏 worktree 作为部署输入。production backend-core 是明确例外：D601 CI 构建并推送 commit-pinned 镜像到 D601 artifact registry，master server CD 只拉取、retag、recreate 和验证，不在 master server 编译 Rust 或执行 Compose build。完整规则见 `docs/reference/deploy.md`，D601 dev/Rust 边界见 `docs/reference/dev-environment.md`，artifact registry 见 `docs/reference/artifact-registry.md`。 +版本化用户服务部署优先使用 `bun scripts/cli.ts deploy apply` 已支持的受控路径；D601 persistent dev apply 当前只支持 `backend-core` 和 `frontend`，`baidu-netdisk` dev 验证使用 artifact consumer，dev desired-state smoke 使用 `ci run-dev-e2e`。`deploy.json` 只声明服务 `id`、`repo` 和 `commitId`；目标节点、Dockerfile、Compose、Kubernetes manifest、健康检查和代理路径继续来自 `config.json` 与现有 manifest。主 server 直管微服务和内部 sidecar，例如 `code-queue-mgr`，也必须支持这一路径：`deploy apply --service code-queue-mgr` 从 `deploy.json` 指定 commit 导出源码、构建镜像、替换固定 Compose service 并验证运行中镜像/健康信息的 commit。部署默认遵循 target-side build：服务部署到哪台 target，就在哪台 target 从 remote commit 导出源码、一次性代理构建镜像并部署；不得把中心构建镜像作为默认分发路径，也不得用 `docker commit` 或脏 worktree 作为部署输入。production backend-core 是明确例外；`baidu-netdisk` 是主 server 直管微服务的镜像化样板：D601 CI 构建并推送 commit-pinned 镜像到 D601 artifact registry，master server dev/prod CD 只拉取、retag、recreate 和验证，不在 master server 执行 Compose build。完整规则见 `docs/reference/deploy.md`，D601 dev/Rust 边界见 `docs/reference/dev-environment.md`，artifact registry 见 `docs/reference/artifact-registry.md`。 ## Main Server Swap @@ -46,7 +46,7 @@ swap 管理不能被强塞进所有热路径。`server start/status` 可以暴 ## Single Service Rebuild -前端、本机 provider-gateway、dev-frontend-proxy 或主 server 承载的 Todo Note/Code Queue Manager/Project Manager/Baidu Netdisk/OA Event Flow 用户服务需要非版本化本地重建时，统一使用 `bun scripts/cli.ts server rebuild `，其中 `` 只能是 `backend-core`、`frontend`、`dev-frontend-proxy`、`provider-gateway`、`todo-note`、`code-queue-mgr`、`project-manager`、`baidu-netdisk` 或 `oa-event-flow`。需要按 commit 上线或恢复到 desired-state 时必须改用 `bun scripts/cli.ts deploy apply --service ` 或 backend-core artifact CD；直管微服务也不能把脏工作树或手工重建作为部署真相。Rust backend-core 迭代不得在 master server 用 `server rebuild backend-core` 编译，生产 backend-core 也不得用该命令完成 Rust 构建，必须走 D601 dev deploy/CI 或 D601 artifact registry CD。D601 Code Queue 执行面、File Browser、FindJob、Pipeline、MET Nonlinear 和 ClaudeQQ 部署在计算节点，不属于主 server Compose 可重建服务；其中 D601 Code Queue 执行面不得再通过 `codex deploy` 或维护通道直连 D601 部署；未来正式 CD 必须经受控 target-side 路径执行 build-first、rollout 和 live commit 验证。 +前端、本机 provider-gateway、dev-frontend-proxy 或主 server 承载的 Todo Note/Code Queue Manager/Project Manager/Baidu Netdisk/OA Event Flow 用户服务需要非版本化本地重建时，统一使用 `bun scripts/cli.ts server rebuild `，其中 `` 只能是 `backend-core`、`frontend`、`dev-frontend-proxy`、`provider-gateway`、`todo-note`、`code-queue-mgr`、`project-manager`、`baidu-netdisk` 或 `oa-event-flow`。需要按 commit 上线或恢复到 desired-state 时必须改用 `bun scripts/cli.ts deploy apply --service `、backend-core artifact CD 或 `deploy apply --env dev|prod --service baidu-netdisk` artifact consumer；直管微服务也不能把脏工作树或手工重建作为部署真相。Rust backend-core 迭代不得在 master server 用 `server rebuild backend-core` 编译，生产 backend-core 也不得用该命令完成 Rust 构建，必须走 D601 dev deploy/CI 或 D601 artifact registry CD。`server rebuild baidu-netdisk` 只作为维护/非标准路径保留，不得作为标准发布完成证据。D601 Code Queue 执行面、File Browser、FindJob、Pipeline、MET Nonlinear 和 ClaudeQQ 部署在计算节点，不属于主 server Compose 可重建服务；其中 D601 Code Queue 执行面不得再通过 `codex deploy` 或维护通道直连 D601 部署；未来正式 CD 必须经受控 target-side 路径执行 build-first、rollout 和 live commit 验证。 frontend 改动必须明确上线到公网：修改 `src/components/frontend/src/`、`src/components/frontend/public/style.css`、frontend 使用的共享 TSX/TS 模块或 WebUI 导航后，必须在同一变更集中执行 `bun scripts/cli.ts server rebuild frontend`，并等待 job 成功。公网 WebUI 的 `/app.js` 是 `unidesk-frontend` 容器启动时从镜像内源码转译生成的运行时 bundle；只改工作区文件、只跑 `bun run check`、只跑 `Bun.build` 或只刷新浏览器都不会替换已经运行的容器。 diff --git a/docs/reference/dev-environment.md b/docs/reference/dev-environment.md index 1bab3552..a178a60b 100644 --- a/docs/reference/dev-environment.md +++ b/docs/reference/dev-environment.md @@ -43,7 +43,7 @@ The persistent dev rollout currently supports only: - `backend-core` - `frontend` -`code-queue` is present in `environments.dev.services` only so `ci run-dev-e2e` can build a Git-pinned Code Queue image and run a temporary namespace smoke. It is not part of persistent dev apply: `deploy apply --env dev --service code-queue` must still be rejected. Decision Center, k3sctl-adapter and other D601 services are not part of persistent dev apply yet. Their smoke validation stays under `ci run-dev-e2e` or service-specific future designs. The `environments.dev.ci` declaration and short launcher runner are owned by `docs/reference/dev-ci-runner.md`. +`baidu-netdisk` may appear in `environments.dev.services` as the first main-server direct user-service artifact validation sample. `deploy apply --env dev --service baidu-netdisk` consumes the D601 registry artifact and validates the main-server Compose service; it is not a persistent D601 k3s dev workload. `code-queue` is present in `environments.dev.services` only so `ci run-dev-e2e` can build a Git-pinned Code Queue image and run a temporary namespace smoke. It is not part of persistent dev apply: `deploy apply --env dev --service code-queue` must still be rejected. Decision Center, k3sctl-adapter and other D601 services are not part of persistent dev apply yet. Their smoke validation stays under `ci run-dev-e2e` or service-specific future designs. The `environments.dev.ci` declaration and short launcher runner are owned by `docs/reference/dev-ci-runner.md`. ## Rust Backend-Core Boundary diff --git a/docs/reference/microservices.md b/docs/reference/microservices.md index 6f4ddf1b..05c7b17c 100644 --- a/docs/reference/microservices.md +++ b/docs/reference/microservices.md @@ -110,6 +110,7 @@ Project Manager 在 UniDesk 语境中按纯后端服务管理：不得将 `4233` - Provider：`main-server`，由 backend-core 直接访问同一 Compose 网络内的 `http://baidu-netdisk:4244`，公网不发布 `4244`。 - 代码引用：`https://github.com/pikasTech/unidesk` 与配置中的 `repository.commitId`；服务源码位于 `src/components/microservices/baidu-netdisk`，属于 UniDesk 自有主 server 用户服务。 - 部署引用：UniDesk 根仓库 `docker-compose.yml` 中的 `baidu-netdisk` service，Dockerfile 为 `src/components/microservices/baidu-netdisk/Dockerfile`，容器名为 `baidu-netdisk-backend`。 +- 标准发布：`baidu-netdisk` 是主 server 直管微服务的镜像化样板。CI 使用 `bun scripts/cli.ts ci publish-user-service --service baidu-netdisk --commit ` 在 D601 registry 发布 `127.0.0.1:5000/unidesk/baidu-netdisk:`；dev 验证使用 `bun scripts/cli.ts deploy apply --env dev --service baidu-netdisk`；prod 发布使用 `bun scripts/cli.ts deploy apply --env prod --service baidu-netdisk`。两条 CD 都必须消费同一类 commit-pinned artifact、只 recreate `baidu-netdisk-backend`、并验证 image label 与 `/health.deploy.commit`；`server rebuild baidu-netdisk` 只保留为维护/非标准路径。 - 配置密钥：Compose 只透传 `UNIDESK_BAIDU_NETDISK_CLIENT_ID`、`UNIDESK_BAIDU_NETDISK_CLIENT_SECRET`、`UNIDESK_BAIDU_NETDISK_TOKEN_KEY` 与可选 `UNIDESK_BAIDU_NETDISK_APP_ROOT`；当前默认工作根目录为 `/`，如需收回到应用目录可显式设为 `/apps/`；不得把百度 AppSecret、token key、access token 或 refresh token 写入仓库文件。 - 配置步骤：`UNIDESK_BAIDU_NETDISK_TOKEN_KEY` 可由本机生成；百度 `client_id` 和 `client_secret` 必须由账号拥有者在百度网盘开放平台创建应用后提供，操作清单见 `docs/issue/baidu-netdisk-env-setup.md`。 - 数据库：OAuth 设备码会话、账号摘要、加密 token、传输任务和事件写入主 PostgreSQL 表 `baidu_netdisk_*`；服务启动时自动创建/补齐 schema，不依赖仅首次生效的 database init SQL。 diff --git a/docs/reference/user-service-delivery.md b/docs/reference/user-service-delivery.md index 6a8b4f63..d65a82a7 100644 --- a/docs/reference/user-service-delivery.md +++ b/docs/reference/user-service-delivery.md @@ -4,7 +4,7 @@ This document owns the default delivery path for UniDesk user services registere ## Scope -User services are non-core business services mounted onto the UniDesk platform. They must remain deliverable without changing the base platform strategy. The default policy here applies to user services that are expected to reach production after validation, including D601-managed services such as Decision Center. +User services are non-core business services mounted onto the UniDesk platform. They must remain deliverable without changing the base platform strategy. The default policy here applies to user services that are expected to reach production after validation, including main-server direct services such as Baidu Netdisk and D601-managed services such as Decision Center. This policy does not apply to: @@ -40,6 +40,17 @@ The default release flow for a user-service change is: Many user services are surfaced through the UniDesk frontend rather than through their own browser-exposed product. When a user-service backend has a paired UniDesk frontend page, the frontend change must be validated in the same dev/prod release window as the backend artifact, but it does not create a second deployment truth. The backend artifact remains the production deployment unit; the frontend page remains the user-visible acceptance surface. +## Baidu Netdisk + +Baidu Netdisk is the canonical main-server direct user-service sample. + +- The minimal standard artifact command is `bun scripts/cli.ts ci publish-user-service --service baidu-netdisk --commit --wait-ms 1200000`. +- The expected artifact is `127.0.0.1:5000/unidesk/baidu-netdisk:` plus its registry digest from the CI output. +- Dev validation consumes the same artifact with `bun scripts/cli.ts deploy apply --env dev --service baidu-netdisk`; this is a pull-only Compose validation path, not a source build on the master server. +- Production CD consumes the same artifact with `bun scripts/cli.ts deploy apply --env prod --service baidu-netdisk`; it recreates only `baidu-netdisk-backend` and verifies image labels plus `/health.deploy.commit`. +- `server rebuild baidu-netdisk` remains a maintenance/local rebuild path only. It is not the standard versioned release truth for Baidu Netdisk. +- Production acceptance must explicitly verify `microservice health baidu-netdisk`, `microservice proxy baidu-netdisk /api/transfers`, private `4244` exposure, and live commit / artifact information. + ## Decision Center Decision Center is the canonical example of a user service that doubles as a product workflow for requirements, decisions, and daily work diaries. diff --git a/scripts/src/artifact-registry.ts b/scripts/src/artifact-registry.ts index e87bb7d7..8a6a0f56 100644 --- a/scripts/src/artifact-registry.ts +++ b/scripts/src/artifact-registry.ts @@ -22,9 +22,10 @@ interface ArtifactRegistryOptions { dryRun: boolean; runNow: boolean; commit: string | null; - targetImage: string; + targetImage: string | null; serviceId: string | null; sourceRepo: string; + deployRef: string | null; } interface RenderedFile { @@ -60,11 +61,12 @@ const defaultOptions: ArtifactRegistryOptions = { dryRun: false, runNow: false, commit: null, - targetImage: "unidesk-backend-core", + targetImage: null, serviceId: null, sourceRepo: "https://github.com/pikasTech/unidesk", + deployRef: null, }; -const supportedArtifactConsumerServices = ["backend-core", "decision-center"] as const; +const supportedArtifactConsumerServices = ["backend-core", "baidu-netdisk", "decision-center"] as const; type SupportedArtifactConsumerService = typeof supportedArtifactConsumerServices[number]; interface ArtifactConsumerSpec { @@ -75,6 +77,13 @@ interface ArtifactConsumerSpec { targetImage: string; targetCommitImage: (commit: string) => string; deployRef: string; + compose?: { + serviceName: string; + containerName: string; + deployEnvPrefix: string; + healthProbeCommand: string; + requireHealthCommit: boolean; + }; k3s?: { namespace: string; manifestPath: string; @@ -95,6 +104,29 @@ const artifactConsumerSpecs: Record `unidesk-backend-core:${commit}`, deployRef: "deploy.json#environments.prod.services.backend-core", + compose: { + serviceName: "backend-core", + containerName: "unidesk-backend-core", + deployEnvPrefix: "UNIDESK_DEPLOY", + healthProbeCommand: "if command -v backend-core >/dev/null 2>&1; then backend-core --fetch-json http://127.0.0.1:8080/health --require-ok; elif command -v bun >/dev/null 2>&1; then bun -e \"fetch('http://127.0.0.1:8080/health').then(async r=>{const text=await r.text(); console.log(text); process.exit(r.ok?0:1)}).catch(e=>{console.error(e); process.exit(1)})\"; else exit 1; fi", + requireHealthCommit: false, + }, + }, + "baidu-netdisk": { + serviceId: "baidu-netdisk", + kind: "compose", + registryRepository: "unidesk/baidu-netdisk", + dockerfile: "src/components/microservices/baidu-netdisk/Dockerfile", + targetImage: "baidu-netdisk", + targetCommitImage: (commit: string) => `baidu-netdisk:${commit}`, + deployRef: "deploy.json#environments.prod.services.baidu-netdisk", + compose: { + serviceName: "baidu-netdisk", + containerName: "baidu-netdisk-backend", + deployEnvPrefix: "UNIDESK_BAIDU_NETDISK_DEPLOY", + healthProbeCommand: "bun -e \"fetch('http://127.0.0.1:4244/health').then(async r=>{const text=await r.text(); console.log(text); process.exit(r.ok?0:1)}).catch(e=>{console.error(e); process.exit(1)})\"", + requireHealthCommit: true, + }, }, "decision-center": { serviceId: "decision-center", @@ -185,6 +217,9 @@ function parseOptions(args: string[]): ArtifactRegistryOptions { } else if (arg === "--source-repo") { options.sourceRepo = requireValue(args, index, arg); index += 1; + } else if (arg === "--deploy-ref") { + options.deployRef = requireValue(args, index, arg); + index += 1; } else { throw new Error(`unknown artifact-registry option: ${arg}`); } @@ -202,6 +237,10 @@ function shellQuote(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'`; } +function safeName(value: string): string { + return value.replace(/[^A-Za-z0-9_.-]/gu, "-"); +} + function rootExecPrelude(): string { return [ "root_exec() {", @@ -328,16 +367,17 @@ function plan(options: ArtifactRegistryOptions): Record { "listen only on D601 host loopback 127.0.0.1:5000", "do not expose a public port, NodePort, hostPort, or third-party registry", "do not run inside k3s; keep the registry outside the native k3s failure domain", - "CI builds and publishes backend-core artifacts on D601", - "CD only pulls, retags, recreates backend-core, and verifies live commit", - "master server CD must not compile Rust or run docker compose build backend-core", + "CI builds and publishes backend-core and reviewed user-service artifacts on D601", + "CD only pulls, retags, recreates/imports artifacts, and verifies live commit", + "master server CD must not compile Rust or run docker compose build for artifact consumers", ], renderedPaths: bundle.paths, - backendCoreArtifactFlow: [ - "D601 CI builds unidesk/backend-core:", - "D601 CI pushes 127.0.0.1:5000/unidesk/backend-core:", - "main server pulls via a controlled short-lived localhost relay", - "prod CD retags, recreates backend-core, and verifies live commit metadata", + artifactConsumerFlow: [ + "D601 CI builds unidesk/:", + "D601 CI pushes 127.0.0.1:5000/unidesk/:", + "Compose consumers pull via a controlled provider-gateway SSH image stream", + "k3s consumers pull on D601 and import into native k3s containerd", + "CD retags/recreates or updates Deployment images and verifies live commit metadata", ], }; } @@ -471,6 +511,10 @@ function artifactImageRef(options: ArtifactRegistryOptions, spec: ArtifactConsum return `127.0.0.1:${options.port}/${spec.registryRepository}:${commit}`; } +function deployRefFor(options: ArtifactRegistryOptions, spec: ArtifactConsumerSpec): string { + return options.deployRef ?? spec.deployRef; +} + function runRemoteScript(options: ArtifactRegistryOptions, script: string, timeoutMs = options.timeoutMs): CommandResult { const command = [process.execPath, "scripts/cli.ts", "ssh", options.providerId, "argv", "bash", "-lc", script]; return runCommand(command, repoRoot, { timeoutMs }); @@ -742,21 +786,34 @@ function verifyLocalArtifactLabels( }; } -async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise> { +function composeArtifactEnvValues(spec: ArtifactConsumerSpec, options: ArtifactRegistryOptions, commit: string): Record { + if (spec.compose === undefined) throw new Error(`${spec.serviceId} missing compose artifact consumer config`); + const prefix = spec.compose.deployEnvPrefix; + return { + [`${prefix}_SERVICE_ID`]: spec.serviceId, + [`${prefix}_REF`]: deployRefFor(options, spec), + [`${prefix}_REPO`]: options.sourceRepo, + [`${prefix}_COMMIT`]: commit, + [`${prefix}_REQUESTED_COMMIT`]: commit, + }; +} + +async function deployComposeArtifactNow(options: ArtifactRegistryOptions, spec: ArtifactConsumerSpec): Promise> { const commit = options.commit; - if (commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit "); - const spec = artifactConsumerSpecs["backend-core"]; + if (commit === null) throw new Error("artifact-registry deploy-service requires --commit "); + if (spec.compose === undefined) throw new Error(`${spec.serviceId} missing compose artifact consumer config`); const health = runReadonlyStatus(options, true); if (health.ok !== true) { - return { ok: false, error: "D601 artifact registry is not healthy", health }; + return { ok: false, serviceId: spec.serviceId, error: "D601 artifact registry is not healthy", health }; } const sourceImage = artifactImageRef(options, spec, commit); - const composeImage = options.targetImage === defaultOptions.targetImage ? spec.targetImage : options.targetImage; - const commitImage = `${composeImage}:${commit}`; + const composeImage = options.targetImage ?? spec.targetImage; + const commitImage = options.targetImage === null ? spec.targetCommitImage(commit) : `${options.targetImage}:${commit}`; const registryProbe = runRemoteScript(options, registryArtifactProbeScript(options, spec, commit), Math.max(options.timeoutMs, 120_000)); if (registryProbe.exitCode !== 0 || registryProbe.timedOut) { return { ok: false, + serviceId: spec.serviceId, step: "registry-artifact-check", error: registryArtifactMissingMessage(spec), sourceImage, @@ -767,6 +824,7 @@ async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise/dev/null || true)", - " echo \"backend_core_container_probe attempt=$attempt cid=$cid health=$health\"", + ` echo "${serviceLogPrefix}_container_probe attempt=$attempt cid=$cid health=$health"`, " if [ \"$health\" = \"healthy\" ] || [ \"$health\" = \"running\" ]; then ready=1; break; fi", " else", - " echo \"backend_core_container_probe attempt=$attempt cid=missing\"", + ` echo "${serviceLogPrefix}_container_probe attempt=$attempt cid=missing"`, " fi", " sleep 1", "done", "test \"$ready\" = \"1\"", - "cid=$(docker ps -q --filter label=com.docker.compose.project=unidesk --filter label=com.docker.compose.service=backend-core --filter label=com.docker.compose.oneoff=False | head -1)", + `cid=$(docker ps -q --filter label=com.docker.compose.project=unidesk --filter label=com.docker.compose.service=${shellQuote(serviceName)} --filter label=com.docker.compose.oneoff=False | head -1)`, "image_id=$(docker inspect -f '{{.Image}}' \"$cid\")", "actual_commit=$(docker image inspect -f '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}' \"$image_id\")", `test "$actual_commit" = ${shellQuote(commit)}`, - "docker exec \"$cid\" bun -e \"fetch('http://127.0.0.1:8080/health').then(async r=>{const text=await r.text(); console.log(text); process.exit(r.ok?0:1)}).catch(e=>{console.error(e); process.exit(1)})\" >/tmp/unidesk-backend-core-health.json", - "cat /tmp/unidesk-backend-core-health.json", + `health_json=/tmp/unidesk-${safeName(spec.serviceId)}-health.json`, + `docker exec "$cid" sh -lc ${shellQuote(spec.compose.healthProbeCommand)} > "$health_json"`, + "cat \"$health_json\"", + ...(spec.compose.requireHealthCommit ? [ + "health_commit=$(python3 -c 'import json,sys; print(((json.load(open(sys.argv[1])).get(\"deploy\") or {}).get(\"commit\") or \"\"))' \"$health_json\")", + `test "$health_commit" = ${shellQuote(commit)}`, + ] : []), ].join("\n"); const deploy = runCommand(["bash", "-lc", composeLockScript(upScript)], repoRoot, { timeoutMs: Math.max(options.timeoutMs, 300_000) }); if (deploy.exitCode !== 0 || deploy.timedOut) { return { ok: false, + serviceId: spec.serviceId, step: "compose-recreate", sourceImage, targetImage: composeImage, @@ -829,9 +892,11 @@ async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise> { + if (options.commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit "); + return deployComposeArtifactNow(options, artifactConsumerSpecs["backend-core"]); +} + function deployBackendCoreJob(args: string[], options: ArtifactRegistryOptions): Record { if (options.commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit "); const runArgs = args.includes("--run-now") ? args : [...args, "--run-now"]; @@ -875,6 +951,7 @@ function dryRunArtifactConsumerPlan(options: ArtifactRegistryOptions, spec: Arti serviceId: spec.serviceId, commit, sourceRepo: options.sourceRepo, + deployRef: deployRefFor(options, spec), sourceImage, requiredLabels: { "unidesk.ai/service-id": spec.serviceId, @@ -885,23 +962,30 @@ function dryRunArtifactConsumerPlan(options: ArtifactRegistryOptions, spec: Arti method: "HEAD", url: `http://127.0.0.1:${options.port}/v2/${spec.registryRepository}/manifests/${commit}`, }, - boundary: "prod CD is artifact-consumer only: verify commit-pinned registry image, pull/import, deploy, then verify live commit/image/health; it never builds source on prod", + boundary: "artifact CD is artifact-consumer only: verify commit-pinned registry image, pull/import, deploy, then verify live commit/image/health; it never builds source on the runtime target", }; if (spec.kind === "compose") { + if (spec.compose === undefined) throw new Error(`${spec.serviceId} missing compose artifact consumer config`); return { ...common, target: { kind: "compose", - composeService: "backend-core", - targetImage: options.targetImage === defaultOptions.targetImage ? spec.targetImage : options.targetImage, - deployCommandShape: "docker compose up -d --no-build --no-deps --force-recreate backend-core", + composeService: spec.compose.serviceName, + containerName: spec.compose.containerName, + targetImage: options.targetImage ?? spec.targetImage, + runtimeImage: spec.targetCommitImage(commit), + deployEnvPrefix: spec.compose.deployEnvPrefix, + deployCommandShape: `docker compose up -d --no-build --no-deps --force-recreate ${spec.compose.serviceName}`, }, validation: [ "D601 registry /v2 manifest exists for the commit tag", "loaded image labels match service id, source commit, and Dockerfile", "running Compose container image label matches the requested commit", - "backend-core /health succeeds for the recreated container", + spec.compose.requireHealthCommit + ? `${spec.serviceId} /health succeeds and reports deploy.commit matching the artifact commit` + : `${spec.serviceId} /health succeeds for the recreated container`, ], + rollback: rollbackInfo(spec, commit), }; } return { @@ -929,9 +1013,12 @@ function dryRunArtifactConsumerPlan(options: ArtifactRegistryOptions, spec: Arti function rollbackInfo(spec: ArtifactConsumerSpec, commit: string): Record { if (spec.kind === "compose") { + const compose = spec.compose; return { type: "compose-retag-recreate", - previousImageHint: "Use docker image ls / docker inspect to find the previous labeled backend-core image id; Compose volumes are unchanged.", + composeService: compose?.serviceName, + previousImageHint: `Use docker image ls / docker inspect to find the previous labeled ${spec.serviceId} image id; Compose volumes are unchanged.`, + commandShape: `bun scripts/cli.ts deploy apply --env prod --service ${spec.serviceId} --commit `, }; } return { @@ -951,6 +1038,7 @@ function d601K3sArtifactDeployScript(options: ArtifactRegistryOptions, spec: Art const k3s = spec.k3s; const labels = [ `unidesk.ai/deploy-service-id=${spec.serviceId}`, + `unidesk.ai/deploy-ref=${deployRefFor(options, spec)}`, `unidesk.ai/deploy-repo=${options.sourceRepo}`, `unidesk.ai/deploy-commit=${commit}`, `unidesk.ai/deploy-requested-commit=${commit}`, @@ -964,6 +1052,7 @@ function d601K3sArtifactDeployScript(options: ArtifactRegistryOptions, spec: Art `commit_image=${shellQuote(commitImage)}`, `service_id=${shellQuote(spec.serviceId)}`, `source_repo=${shellQuote(options.sourceRepo)}`, + `deploy_ref=${shellQuote(deployRefFor(options, spec))}`, `commit=${shellQuote(commit)}`, `dockerfile=${shellQuote(spec.dockerfile)}`, `namespace=${shellQuote(k3s.namespace)}`, @@ -997,7 +1086,7 @@ function d601K3sArtifactDeployScript(options: ArtifactRegistryOptions, spec: Art "if [ -f \"$manifest\" ]; then kubectl apply -f \"$manifest\"; else echo artifact_cd_manifest_missing=$manifest; fi", "previous_commit=$(kubectl -n \"$namespace\" get deployment \"$deployment\" -o jsonpath='{.metadata.annotations.unidesk\\.ai/deploy-commit}' 2>/dev/null || true)", "kubectl -n \"$namespace\" set image \"deployment/$deployment\" \"$container_name=$commit_image\"", - "kubectl -n \"$namespace\" set env \"deployment/$deployment\" \"UNIDESK_DEPLOY_SERVICE_ID=$service_id\" \"UNIDESK_DEPLOY_REPO=$source_repo\" \"UNIDESK_DEPLOY_COMMIT=$commit\" \"UNIDESK_DEPLOY_REQUESTED_COMMIT=$commit\"", + "kubectl -n \"$namespace\" set env \"deployment/$deployment\" \"UNIDESK_DEPLOY_SERVICE_ID=$service_id\" \"UNIDESK_DEPLOY_REF=$deploy_ref\" \"UNIDESK_DEPLOY_REPO=$source_repo\" \"UNIDESK_DEPLOY_COMMIT=$commit\" \"UNIDESK_DEPLOY_REQUESTED_COMMIT=$commit\"", `kubectl -n "$namespace" annotate "deployment/$deployment" ${labels.map(shellQuote).join(" ")} --overwrite`, "if [ -n \"$previous_commit\" ] && [ \"$previous_commit\" != \"$commit\" ]; then kubectl -n \"$namespace\" annotate \"deployment/$deployment\" \"unidesk.ai/deploy-previous-commit=$previous_commit\" --overwrite; fi", "kubectl -n \"$namespace\" rollout status \"deployment/$deployment\" --timeout=180s", @@ -1109,6 +1198,7 @@ async function deployD601K3sArtifactNow(options: ArtifactRegistryOptions, spec: commit, providerId: options.providerId, sourceRepo: options.sourceRepo, + deployRef: deployRefFor(options, spec), sourceImage, stableImage: spec.targetImage, runtimeImage: spec.targetCommitImage(commit), @@ -1130,7 +1220,7 @@ async function deployServiceNow(options: ArtifactRegistryOptions): Promise { "bun scripts/cli.ts artifact-registry health [--provider-id D601]", "bun scripts/cli.ts artifact-registry install [--provider-id D601]", "bun scripts/cli.ts artifact-registry deploy-backend-core --commit [--run-now] [--provider-id D601]", + "bun scripts/cli.ts artifact-registry deploy-service --service baidu-netdisk --commit [--dry-run] [--run-now] [--provider-id D601]", "bun scripts/cli.ts artifact-registry deploy-service --service decision-center --commit [--dry-run] [--run-now] [--provider-id D601]", ], firstStage: "install now writes the rendered systemd/Compose/config files and starts the registry", artifactConsumers: { supportedServices: supportedArtifactConsumerServices, unsupportedPolicy: "return structured unsupported; never fall back to legacy maintenance-channel source builds", - prodCommand: "bun scripts/cli.ts deploy apply --env prod --service decision-center", + prodCommands: [ + "bun scripts/cli.ts deploy apply --env prod --service backend-core", + "bun scripts/cli.ts deploy apply --env prod --service baidu-netdisk", + "bun scripts/cli.ts deploy apply --env prod --service decision-center", + ], rollbackShape: "rerun the same artifact consumer with a previous commit-pinned image", }, defaults: defaultOptions, diff --git a/scripts/src/check.ts b/scripts/src/check.ts index dffd26dc..672da31c 100644 --- a/scripts/src/check.ts +++ b/scripts/src/check.ts @@ -28,6 +28,7 @@ const syntaxFiles = [ "src/components/frontend/src/decision-center.tsx", "src/components/provider-gateway/src/index.ts", "src/components/microservices/oa-event-flow/src/index.ts", + "src/components/microservices/baidu-netdisk/src/index.ts", "src/components/microservices/k3sctl-adapter/src/index.ts", "src/components/microservices/mdtodo/src/index.ts", "src/components/microservices/decision-center/src/index.ts", diff --git a/scripts/src/ci.ts b/scripts/src/ci.ts index b90cc421..cadc7499 100644 --- a/scripts/src/ci.ts +++ b/scripts/src/ci.ts @@ -196,11 +196,14 @@ function requireSupportedUserService(config: UniDeskConfig, serviceId: string): } const service = config.microservices.find((item) => item.id === serviceId); if (service === undefined) throw new Error(`unknown user service: ${serviceId}`); - if (service.providerId !== d601ProviderId || service.development.providerId !== d601ProviderId) { - throw new Error(`ci publish-user-service currently supports only D601 user services; ${serviceId} is assigned to ${service.providerId}`); - } - if (service.deployment.mode !== "k3sctl-managed") { - throw new Error(`ci publish-user-service currently supports k3sctl-managed D601 user services; ${serviceId} uses ${service.deployment.mode}`); + const isD601K3sService = service.providerId === d601ProviderId + && service.development.providerId === d601ProviderId + && service.deployment.mode === "k3sctl-managed"; + const isMainServerDirectService = service.providerId === "main-server" + && service.development.providerId === "main-server" + && service.deployment.mode === "unidesk-direct"; + if (!isD601K3sService && !isMainServerDirectService) { + throw new Error(`ci publish-user-service supports only reviewed k3sctl-managed D601 services or main-server unidesk-direct services; ${serviceId} is ${service.providerId}/${service.deployment.mode}`); } return service; } @@ -1307,6 +1310,7 @@ export function ciHelp(): Record { "bun scripts/cli.ts ci install", "bun scripts/cli.ts ci run --revision ", "bun scripts/cli.ts ci publish-backend-core --commit ", + "bun scripts/cli.ts ci publish-user-service --service baidu-netdisk --commit ", "bun scripts/cli.ts ci publish-user-service --service decision-center --commit ", "bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000", "bun scripts/cli.ts ci logs ", @@ -1328,7 +1332,7 @@ export function ciHelp(): Record { userServiceArtifact: { producer: "D601 CI", command: "bun scripts/cli.ts ci publish-user-service --service --commit ", - initiallySupportedServices: ["decision-center"], + initiallySupportedServices: ["baidu-netdisk", "decision-center"], registry: "127.0.0.1:5000/unidesk/:", outputFields: ["imageRef", "tag", "digest", "sourceCommit", "serviceId"], boundary: "artifact producer only; no prod deploy and no production namespace mutation", @@ -1373,8 +1377,8 @@ export async function runCiCommand(config: UniDeskConfig, args: string[]): Promi const waitMs = numberOption(args, "--wait-ms", 0); const dryRun = boolFlag(args, "--dry-run"); const dockerfile = requireRepoRelativePath(service.repository.dockerfile, `microservices.${serviceId}.repository.dockerfile`); - if (serviceId !== "decision-center") { - throw new Error("ci publish-user-service currently allows only decision-center until each user-service Dockerfile contract is reviewed"); + if (!["baidu-netdisk", "decision-center"].includes(serviceId)) { + throw new Error("ci publish-user-service currently allows only baidu-netdisk and decision-center until each user-service Dockerfile contract is reviewed"); } return publishUserServiceArtifact(config, { repoUrl, diff --git a/scripts/src/deploy.ts b/scripts/src/deploy.ts index 650c7ccf..6c753d68 100644 --- a/scripts/src/deploy.ts +++ b/scripts/src/deploy.ts @@ -135,7 +135,8 @@ const nativeK3sCtrAddress = "/run/k3s/containerd/containerd.sock"; const unideskRepoUrl = "https://github.com/pikasTech/unidesk"; const d601MaintenanceDeployAllowedServiceIds = new Set(["backend-core", "frontend", "k3sctl-adapter", "code-queue"]); const devApplySupportedServiceIds = new Set(["backend-core", "frontend"]); -const prodArtifactConsumerServiceIds = new Set(["backend-core", "decision-center"]); +const devArtifactConsumerServiceIds = new Set(["baidu-netdisk"]); +const prodArtifactConsumerServiceIds = new Set(["backend-core", "baidu-netdisk", "decision-center"]); const deployEnvironmentTargets: Record = { dev: { environment: "dev", @@ -198,7 +199,7 @@ export function deployHelp(action: string | undefined = undefined): Record", default: defaultDeployFile, description: "Desired-state manifest path relative to the repo root. JSON and ESM JS manifests are supported, for example deploy.json or develop.js. Local manifest apply allows k3sctl-adapter and explicit production code-queue controlled rollout on D601." }, - { name: "--env ", description: "Read the named environment from origin/master:deploy.json. Dev apply is enabled only for backend-core and frontend in D601 unidesk-dev; prod apply uses the D601 registry artifact consumer for backend-core and decision-center." }, + { name: "--env ", description: "Read the named environment from origin/master:deploy.json. Dev apply supports backend-core/frontend target-side rollout plus baidu-netdisk artifact validation; prod apply uses the D601 registry artifact consumer for backend-core, baidu-netdisk, and decision-center." }, { name: "--service ", description: "Limit reconcile to one service from the manifest." }, { name: "--commit ", description: "Prod artifact rollback/apply override for a selected service; the image must already exist in D601 registry." }, { name: "--dry-run", description: "Prepare and validate without mutating the target service." }, @@ -2476,12 +2477,17 @@ function environmentDryRunPlan( ? prodArtifactConsumerServiceIds.has(service.id) ? "d601-registry-artifact-consumer" : "unsupported" - : "d601-dev-target-side-build", - unsupported: environment === "prod" && !prodArtifactConsumerServiceIds.has(service.id) + : devArtifactConsumerServiceIds.has(service.id) + ? "d601-registry-artifact-consumer-dev-validation" + : "d601-dev-target-side-build", + unsupported: (environment === "prod" && !prodArtifactConsumerServiceIds.has(service.id)) + || (environment === "dev" && !devApplySupportedServiceIds.has(service.id) && !devArtifactConsumerServiceIds.has(service.id)) ? { ok: false, supported: false, - reason: "No standardized prod D601 registry artifact consumer is implemented for this service; legacy maintenance-channel deployment is not allowed.", + reason: environment === "prod" + ? "No standardized prod D601 registry artifact consumer is implemented for this service; legacy maintenance-channel deployment is not allowed." + : "No standardized dev deployment or artifact validation path is implemented for this service.", } : undefined, })), @@ -2491,7 +2497,7 @@ function environmentDryRunPlan( function unsupportedDevApplyServices(manifest: DeployManifest, serviceId: string | null): string[] { if (manifest.environment !== "dev") return []; const services = serviceId === null ? manifest.services : manifest.services.filter((service) => service.id === serviceId); - return services.map((service) => service.id).filter((id) => !devApplySupportedServiceIds.has(id)); + return services.map((service) => service.id).filter((id) => !devApplySupportedServiceIds.has(id) && !devArtifactConsumerServiceIds.has(id)); } function blockedD601MaintenanceDeployServices(config: UniDeskConfig, manifest: DeployManifest, serviceId: string | null): string[] { @@ -2505,6 +2511,16 @@ function d601MaintenanceDeployBlockMessage(blocked: string[]): string { return `D601 target-side deployment is enabled only for k3sctl-adapter and dev backend-core/frontend; blocked services: ${blocked.join(", ")}. Use ci run-dev-e2e for dev smoke verification.`; } +function selectedDevArtifactServices(manifest: DeployManifest, serviceId: string | null): DeployManifestService[] { + if (manifest.environment !== "dev") return []; + return selectedEnvironmentServices(manifest, serviceId).filter((service) => devArtifactConsumerServiceIds.has(service.id)); +} + +function selectedDevTargetServices(manifest: DeployManifest, serviceId: string | null): DeployManifestService[] { + if (manifest.environment !== "dev") return []; + return selectedEnvironmentServices(manifest, serviceId).filter((service) => devApplySupportedServiceIds.has(service.id)); +} + function selectedEnvironmentServices(manifest: DeployManifest, serviceId: string | null): DeployManifestService[] { const services = serviceId === null ? manifest.services : manifest.services.filter((service) => service.id === serviceId); if (serviceId !== null && services.length === 0) throw new Error(`deploy manifest does not contain service: ${serviceId}`); @@ -2532,10 +2548,12 @@ function prodArtifactUnsupportedResult(services: DeployManifestService[]): Recor }; } -async function runProdArtifactApplyNow(manifest: DeployManifest, options: DeployOptions): Promise> { - const selected = selectedEnvironmentServices(manifest, options.serviceId); - const unsupported = selected.filter((service) => !prodArtifactConsumerServiceIds.has(service.id)); - if (unsupported.length > 0) return prodArtifactUnsupportedResult(unsupported); +async function runArtifactConsumerApplyNow( + manifest: DeployManifest, + options: DeployOptions, + environment: "dev" | "prod", + selected: DeployManifestService[], +): Promise> { const startedAt = nowIso(); const results = []; for (const service of selected) { @@ -2545,11 +2563,12 @@ async function runProdArtifactApplyNow(manifest: DeployManifest, options: Deploy "--service", service.id, "--commit", commit, "--source-repo", service.repo, + "--deploy-ref", `deploy.json#environments.${environment}.services.${service.id}`, "--timeout-ms", String(options.timeoutMs), "--run-now", ...(options.dryRun ? ["--dry-run"] : []), ]; - progressLine("prod-artifact-cd", options.dryRun ? "dry-run" : "reconcile", { serviceId: service.id, commit }); + progressLine(`${environment}-artifact-cd`, options.dryRun ? "dry-run" : "reconcile", { serviceId: service.id, commit }); results.push(await runArtifactRegistryCommand(artifactArgs)); const latest = results.at(-1) as Record; if (latest.ok !== true) break; @@ -2557,7 +2576,7 @@ async function runProdArtifactApplyNow(manifest: DeployManifest, options: Deploy return { ok: results.every((result) => asRecord(result)?.ok === true), action: "apply", - environment: "prod", + environment, executor: "d601-registry-artifact-consumer", dryRun: options.dryRun, startedAt, @@ -2566,6 +2585,13 @@ async function runProdArtifactApplyNow(manifest: DeployManifest, options: Deploy }; } +async function runProdArtifactApplyNow(manifest: DeployManifest, options: DeployOptions): Promise> { + const selected = selectedEnvironmentServices(manifest, options.serviceId); + const unsupported = selected.filter((service) => !prodArtifactConsumerServiceIds.has(service.id)); + if (unsupported.length > 0) return prodArtifactUnsupportedResult(unsupported); + return runArtifactConsumerApplyNow(manifest, options, "prod", selected); +} + function prodArtifactApplyJob(args: string[], options: DeployOptions): Record { if (!options.runNow && options.dryRun) { throw new Error("deploy apply --env prod --dry-run should run in the foreground so the plan is visible immediately"); @@ -2585,6 +2611,25 @@ function prodArtifactApplyJob(args: string[], options: DeployOptions): Record { + if (!options.runNow && options.dryRun) { + throw new Error("deploy apply --env dev --dry-run should run in the foreground so the plan is visible immediately"); + } + const runArgs = args.includes("--run-now") ? args : [...args, "--run-now"]; + const command = [process.execPath, rootPath("scripts", "cli.ts"), "deploy", ...runArgs]; + const source = `${deployEnvironmentTargets.dev.gitRef}:deploy.json#environments.dev`; + const job = startJob("deploy_dev_artifact_apply", command, `Validate dev artifact consumer from ${source}${options.serviceId === null ? "" : ` service=${options.serviceId}`}`); + return { + ok: true, + mode: "async-job", + executor: "d601-registry-artifact-consumer-dev-validation", + job, + statusCommand: `bun scripts/cli.ts job status ${job.id}`, + tailCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`, + note: "Dev artifact validation continues in the background: D601 registry commit-pinned image check, pull-only Compose recreate, image-label and live health commit verification.", + }; +} + async function runApplyNow(config: UniDeskConfig, manifest: DeployManifest, options: DeployOptions): Promise> { const selected = selectServices(config, manifest, options.serviceId); const startedAt = nowIso(); @@ -2639,7 +2684,16 @@ export async function runDeployCommand(config: UniDeskConfig | null, args: strin } const unsupported = unsupportedDevApplyServices(manifest, options.serviceId); if (unsupported.length > 0) { - throw new Error(`deploy apply --env dev currently supports only backend-core and frontend; unsupported selected services: ${unsupported.join(", ")}. Use ci run-dev-e2e for smoke verification.`); + throw new Error(`deploy apply --env dev currently supports only backend-core/frontend target-side rollout and baidu-netdisk artifact validation; unsupported selected services: ${unsupported.join(", ")}. Use ci run-dev-e2e for smoke verification.`); + } + const devArtifactServices = selectedDevArtifactServices(manifest, options.serviceId); + const devTargetServices = selectedDevTargetServices(manifest, options.serviceId); + if (devArtifactServices.length > 0 && devTargetServices.length > 0) { + throw new Error("deploy apply --env dev cannot mix artifact validation services with target-side rollout services in one invocation; pass --service"); + } + if (devArtifactServices.length > 0) { + if (options.dryRun || options.runNow) return await runArtifactConsumerApplyNow(manifest, options, "dev", devArtifactServices); + return devArtifactApplyJob(args, options); } if (config === null) throw new Error("deploy apply --env dev requires config.json"); if (!options.dryRun) { diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 54054ada..8c0c92ab 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -54,7 +54,7 @@ export function rootHelp(): unknown { { command: "debug dispatch [providerId] [docker.ps|provider.upgrade|host.ssh|microservice.http|echo] [--wait-ms N]", description: "Submit a real internal-core dispatch request for CLI debugging." }, { command: "debug task ", description: "Read a dispatched task record from internal core for CLI debugging." }, { command: "network perf [--service code-queue --path /api/tasks/overview?limit=30 --count N --concurrency N --label before|after]", description: "Benchmark frontend -> backend-core -> provider/adapter user-service networking and report latency/proxy-mode distributions." }, - { command: "ci install|status|run|publish-backend-core|run-dev-e2e|logs", description: "Manage D601 k3s Tekton CI; publish-backend-core builds commit-pinned images in CI without deploying CD." }, + { command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs", description: "Manage D601 k3s Tekton CI; artifact publish commands build commit-pinned images in CI without deploying CD." }, { command: "e2e run [--only pattern[,pattern...]] [--skip pattern[,pattern...]]", description: "Run selected public/internal/Playwright E2E checks; use --only for focused iteration and rerun without filters for final regression." }, ], }; @@ -259,6 +259,7 @@ function artifactRegistryHelp(): unknown { "bun scripts/cli.ts artifact-registry health [--provider-id D601]", "bun scripts/cli.ts artifact-registry install [--provider-id D601]", "bun scripts/cli.ts artifact-registry deploy-backend-core --commit [--run-now] [--provider-id D601]", + "bun scripts/cli.ts artifact-registry deploy-service --service baidu-netdisk --commit [--dry-run] [--run-now] [--provider-id D601]", "bun scripts/cli.ts artifact-registry deploy-service --service decision-center --commit [--dry-run] [--run-now] [--provider-id D601]", ], description: "Manage the declaration, rendered files and readonly checks for the D601 host-managed CNCF Distribution artifact registry.", @@ -267,7 +268,7 @@ function artifactRegistryHelp(): unknown { "service is host-managed by systemd + Docker Compose, not k3s-managed", "install writes the rendered host unit/config and starts the registry", "deploy-backend-core only pulls commit-pinned backend-core artifacts and does not build backend-core on the master server", - "deploy-service currently supports backend-core and decision-center as standardized consumers", + "deploy-service currently supports backend-core, baidu-netdisk, and decision-center as standardized consumers", "status and health use provider-gateway Host SSH readonly checks", ], }; diff --git a/src/components/microservices/baidu-netdisk/Dockerfile b/src/components/microservices/baidu-netdisk/Dockerfile index d05feb5d..bc37b511 100644 --- a/src/components/microservices/baidu-netdisk/Dockerfile +++ b/src/components/microservices/baidu-netdisk/Dockerfile @@ -2,7 +2,8 @@ FROM oven/bun:1-alpine WORKDIR /app/src/components/microservices/baidu-netdisk COPY src/components/microservices/baidu-netdisk/package.json ./package.json -RUN bun install --production +COPY src/components/microservices/baidu-netdisk/bun.lock ./bun.lock +RUN bun install --production --frozen-lockfile COPY src/components/microservices/baidu-netdisk/tsconfig.json ./tsconfig.json COPY src/components/shared /app/src/components/shared COPY src/components/microservices/baidu-netdisk/src ./src diff --git a/src/components/microservices/baidu-netdisk/src/index.ts b/src/components/microservices/baidu-netdisk/src/index.ts index 32d9b94e..6363b201 100644 --- a/src/components/microservices/baidu-netdisk/src/index.ts +++ b/src/components/microservices/baidu-netdisk/src/index.ts @@ -99,6 +99,16 @@ class BaiduApiError extends Error { const serviceStartedAt = new Date().toISOString(); const recentLogs: JsonRecord[] = []; +function deployMetadata(): JsonRecord { + return { + serviceId: process.env.UNIDESK_DEPLOY_SERVICE_ID || "baidu-netdisk", + ref: process.env.UNIDESK_DEPLOY_REF || "", + repo: process.env.UNIDESK_DEPLOY_REPO || "", + commit: process.env.UNIDESK_DEPLOY_COMMIT || "", + requestedCommit: process.env.UNIDESK_DEPLOY_REQUESTED_COMMIT || "", + }; +} + function normalizedAppRoot(value: string): string { const raw = String(value || "/").trim() || "/"; const normalized = pathPosix.normalize(raw.startsWith("/") ? raw : `/${raw}`); @@ -790,6 +800,7 @@ async function health(): Promise { ok: schemaReady, service: "baidu-netdisk", startedAt: serviceStartedAt, + deploy: deployMetadata(), storage: { postgres: schemaReady ? "ready" : "starting", stagingDir: config.stagingDir }, auth: await authSummary(), baidu: { appRoot: config.appRoot, userAgent: config.userAgent, apiReachability: "not_checked" },