From 1faacb94e540cdb2f5aeb95a4f65359baabddc6a Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 20 May 2026 02:09:31 +0000 Subject: [PATCH] feat: add D601 direct artifact consumer --- CI.json | 12 +- deploy.json | 15 + docs/reference/artifact-registry.md | 10 +- docs/reference/ci.md | 3 +- docs/reference/cli.md | 2 +- docs/reference/deploy.md | 11 +- docs/reference/deployment.md | 2 +- docs/reference/microservices.md | 3 + docs/reference/user-service-delivery.md | 15 +- scripts/src/artifact-registry.ts | 368 +++++++++++++++++++++++- scripts/src/deploy.ts | 22 +- 11 files changed, 431 insertions(+), 32 deletions(-) diff --git a/CI.json b/CI.json index fd4dac28..9f3eaae6 100644 --- a/CI.json +++ b/CI.json @@ -159,7 +159,8 @@ }, "image": { "repository": "unidesk/findjob" - } + }, + "notes": "D601 direct Compose artifact consumer is implemented for plan/dry-run and dev validation. Live deploy requires a pre-existing labeled registry artifact for this external repo." }, { "serviceId": "pipeline", @@ -172,7 +173,8 @@ }, "image": { "repository": "unidesk/pipeline" - } + }, + "notes": "D601 direct Compose artifact consumer is implemented for plan/dry-run and dev validation. Live deploy requires a pre-existing labeled registry artifact for this external repo." }, { "serviceId": "met-nonlinear", @@ -185,7 +187,8 @@ }, "image": { "repository": "unidesk/met-nonlinear" - } + }, + "notes": "D601 direct Compose artifact consumer is plan/dry-run only until the long-running met-nonlinear-ts artifact contract is separated from the ML image Dockerfile." }, { "serviceId": "k3sctl-adapter", @@ -198,7 +201,8 @@ }, "image": { "repository": "unidesk/k3sctl-adapter" - } + }, + "notes": "D601 direct Compose artifact consumer exposes plan/dry-run only; real prod deployment requires supervisor confirmation." }, { "serviceId": "mdtodo", diff --git a/deploy.json b/deploy.json index bf2775b8..ed0197c7 100644 --- a/deploy.json +++ b/deploy.json @@ -102,6 +102,21 @@ "repo": "https://github.com/pikasTech/unidesk", "commitId": "c8561d392be857e8a0b79d6dc1b27dee5ece6b88" }, + { + "id": "findjob", + "repo": "https://gitee.com/Lyon1998/findjob", + "commitId": "2d43212c5f474df5d87820985a6c75a8c2e7ac42" + }, + { + "id": "pipeline", + "repo": "https://github.com/pikasTech/pipeline", + "commitId": "87811a8d43edf216a4f4d8efa55bbb96bad8df14" + }, + { + "id": "met-nonlinear", + "repo": "https://github.com/pikasTech/met_nonlinear", + "commitId": "9fcdfc0b505e52cc88cf51b196543dc055da2334" + }, { "id": "decision-center", "repo": "https://github.com/pikasTech/unidesk", diff --git a/docs/reference/artifact-registry.md b/docs/reference/artifact-registry.md index 5cdbc39f..8cc68668 100644 --- a/docs/reference/artifact-registry.md +++ b/docs/reference/artifact-registry.md @@ -67,6 +67,10 @@ bun scripts/cli.ts artifact-registry deploy-service --env prod --service project bun scripts/cli.ts artifact-registry deploy-service --env prod --service oa-event-flow --commit bun scripts/cli.ts artifact-registry deploy-service --env prod --service code-queue-mgr --commit --dry-run bun scripts/cli.ts artifact-registry deploy-service --env prod --service todo-note --commit +bun scripts/cli.ts artifact-registry deploy-service --env dev --service findjob --commit --dry-run +bun scripts/cli.ts artifact-registry deploy-service --env dev --service pipeline --commit --dry-run +bun scripts/cli.ts artifact-registry deploy-service --env dev --service met-nonlinear --commit --dry-run +bun scripts/cli.ts artifact-registry deploy-service --env prod --service k3sctl-adapter --commit --dry-run ``` `plan` 输出架构边界、依赖项、默认路径和 backend-core artifact CD 流程。`render` 输出 systemd unit、Compose 文件和 registry config 的完整内容与 SHA-256。`install --dry-run` 只列出将要执行的远端动作，不写 D601 文件、不启动容器、不 reload systemd。 @@ -75,7 +79,7 @@ bun scripts/cli.ts artifact-registry deploy-service --env prod --service todo-no `deploy-backend-core` 是旧兼容入口，当前作为标准路径已禁用。Production backend-core CD 必须使用 `bun scripts/cli.ts deploy apply --env prod --service backend-core --commit `，由 deploy reconciler 先执行共同的 artifact-consumer guardrail，再调用通用 `deploy-service` consumer。旧入口只能返回 structured deprecated 结果，不得绕过 `deploy apply --env prod`。 -`deploy-service` 是标准化后的最小通用 artifact consumer。它目前支持 `backend-core`、`baidu-netdisk`、prod/dev `frontend`、`decision-center`、`project-manager`、`oa-event-flow`、`code-queue-mgr` 和 `todo-note`，并且必须先通过 D601 registry 的 commit-pinned manifest 校验，再执行拉取、导入/retag、部署和健康验证；`code-queue-mgr` 的 prod live apply 仍需 supervisor 单独确认： +`deploy-service` 是标准化后的最小通用 artifact consumer。它目前支持 `backend-core`、`baidu-netdisk`、prod/dev `frontend`、`decision-center`、`project-manager`、`oa-event-flow`、`code-queue-mgr`、`todo-note`、`findjob`、`pipeline`、`met-nonlinear` 和 `k3sctl-adapter`。所有路径都必须先通过 D601 registry 的 commit-pinned manifest 校验，再执行拉取、导入/retag、部署和健康验证；`code-queue-mgr` 的 prod live apply 仍需 supervisor 单独确认，`met-nonlinear` 与 `k3sctl-adapter` 当前只提供 plan/dry-run： ```bash bun scripts/cli.ts artifact-registry deploy-service --service baidu-netdisk --commit --run-now @@ -86,9 +90,11 @@ bun scripts/cli.ts artifact-registry deploy-service --service decision-center -- bun scripts/cli.ts artifact-registry deploy-service --env prod --service project-manager --commit --run-now bun scripts/cli.ts artifact-registry deploy-service --env prod --service oa-event-flow --commit --run-now bun scripts/cli.ts artifact-registry deploy-service --env prod --service todo-note --commit --run-now +bun scripts/cli.ts artifact-registry deploy-service --env prod --service findjob --commit --run-now +bun scripts/cli.ts artifact-registry deploy-service --env prod --service pipeline --commit --run-now ``` -dry-run 输出会暴露 registry probe URL、required labels、目标 image、部署形态和回滚信息。`baidu-netdisk` 的 Compose 路径会通过 provider-gateway Host SSH 把 `unidesk/baidu-netdisk:` 流式拉到 master server，retag 为 `baidu-netdisk` 和 `baidu-netdisk:`，写入 `UNIDESK_BAIDU_NETDISK_DEPLOY_*`，只 recreate `baidu-netdisk` service，并验证容器 image label 与 `/health.deploy.commit`。`frontend --env prod` 使用同一 Compose artifact consumer，流式拉取 `unidesk/frontend:`，retag 为 `unidesk-frontend` 和 `unidesk-frontend:`，写入 `UNIDESK_FRONTEND_DEPLOY_*`，只 recreate `frontend` service，并验证 image label 与 `/health.deploy.commit`。`frontend --env dev` 和 `decision-center` 的 k3s 路径会在 D601 上验证 commit image、导入 native k3s containerd、更新 Deployment image/env/annotations，并通过 Kubernetes API service proxy 验证 `/health` 中的 `deploy.commit` 和 `deploy.requestedCommit`；dev frontend 还会在 rollout 前把主 server `config.json.auth` 同步到 `unidesk-dev` Secret/ConfigMap。`decision-center --env dev` 落到 `unidesk-dev/decision-center-dev`，prod 落到 `unidesk/decision-center`。回滚信息通过同一 artifact consumer 的 `rollback` 字段暴露，提示操作者重新对一个旧 commit 运行相同命令，而不是切回 legacy maintenance-channel 构建。 +dry-run 输出会暴露 registry probe URL、required labels、目标 image、部署形态和回滚信息。`baidu-netdisk` 的 Compose 路径会通过 provider-gateway Host SSH 把 `unidesk/baidu-netdisk:` 流式拉到 master server，retag 为 `baidu-netdisk` 和 `baidu-netdisk:`，写入 `UNIDESK_BAIDU_NETDISK_DEPLOY_*`，只 recreate `baidu-netdisk` service，并验证容器 image label 与 `/health.deploy.commit`。`findjob`、`pipeline` 和 `met-nonlinear` 的 D601 direct Compose 路径在 D601 本机验证 registry manifest、pull image、retag stable image、写入 `UNIDESK_*_DEPLOY_*` labels/env，并用 `docker compose up -d --no-build --no-deps --force-recreate ` 重新拉起对应 compose service；其中 `met-nonlinear` 当前因为 registered Dockerfile 和 long-running service contract 不一致而 live deploy blocked。`k3sctl-adapter` 是基础设施控制桥，只做 plan/dry-run，真实生产部署需要 supervisor 单独确认。`frontend --env prod` 使用同一 Compose artifact consumer，流式拉取 `unidesk/frontend:`，retag 为 `unidesk-frontend` 和 `unidesk-frontend:`，写入 `UNIDESK_FRONTEND_DEPLOY_*`，只 recreate `frontend` service，并验证 image label 与 `/health.deploy.commit`。`frontend --env dev` 和 `decision-center` 的 k3s 路径会在 D601 上验证 commit image、导入 native k3s containerd、更新 Deployment image/env/annotations，并通过 Kubernetes API service proxy 验证 `/health` 中的 `deploy.commit` 和 `deploy.requestedCommit`；dev frontend 还会在 rollout 前把主 server `config.json.auth` 同步到 `unidesk-dev` Secret/ConfigMap。`decision-center --env dev` 落到 `unidesk-dev/decision-center-dev`，prod 落到 `unidesk/decision-center`。D601 direct Compose consumer 与 k3s-managed consumer 的区别是：前者只接触 D601 Docker/Compose 项目和私有 backend health，不创建 Kubernetes 对象；后者只通过 native k3s Deployment/Service、containerd import 和 Kubernetes API service proxy 验证 live commit。回滚信息通过同一 artifact consumer 的 `rollback` 字段暴露，提示操作者重新对一个旧 commit 运行相同命令，而不是切回 legacy maintenance-channel 构建。 `status` 和 `health` 通过： diff --git a/docs/reference/ci.md b/docs/reference/ci.md index e0a6359e..c92c0038 100644 --- a/docs/reference/ci.md +++ b/docs/reference/ci.md @@ -50,7 +50,7 @@ Current catalog coverage: - `source-build/blocked`: `code-queue`. - `upstream-image/blocked`: `filebrowser`, `filebrowser-d601`. -`publish-user-service` reads `source.repo` and `source.dockerfile` from `CI.json`. The command rejects ad hoc `--repo` overrides; the catalog is the only source for producer build inputs. `publish-backend-core` also reads its producer inputs from `CI.json`, while preserving the dedicated backend-core command and Rust/D601 build boundary. +`publish-user-service` reads `source.repo` and `source.dockerfile` from `CI.json`. The command rejects ad hoc `--repo` overrides; the catalog is the only source for producer build inputs. `publish-backend-core` also reads its producer inputs from `CI.json`, while preserving the dedicated backend-core command and Rust/D601 build boundary. For `findjob`, `pipeline`, `met-nonlinear`, and `k3sctl-adapter`, the catalog can also carry consumer-only notes so CI producers and deploy consumers stay aligned on the live contract. Every successful image-producing CI task must expose a common `artifactSummary` contract: @@ -121,6 +121,7 @@ The CI user-service artifact task must follow these rules: - The image must carry `unidesk.ai/service-id`, `unidesk.ai/source-repo`, `unidesk.ai/source-commit` and `unidesk.ai/dockerfile` labels. - The command output must include the common `artifactSummary` fields: `serviceId`, `sourceCommit`, `sourceRepo`, `dockerfile`, `imageRef`, `tag`, `digest` and `digestRef`. The digest ref is suitable as immutable input for later dev/prod deployment work. - CI is an artifact producer only. It must not restart production services, call production `deploy apply`, mutate the production namespace, or change `deploy.json`. +- `CI.json` may also list downstream consumer-only catalog entries for D601 direct Compose services such as `findjob`, `pipeline`, `met-nonlinear`, and `k3sctl-adapter`; these entries describe the artifact contract and dry-run/support status, not new producer behavior. Publish a Baidu Netdisk artifact: diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 7bff6f01..3056ef6b 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -25,7 +25,7 @@ CLI 可以从 `master` 快速演进，但必须兼容 `deploy.json` 固定的 CI - `microservice list/status/health/diagnostics/tunnel-self-test/proxy` 通过 backend-core 内网 API 管理挂载在计算节点 Docker 或 k3s 控制面中的用户服务（底层命令名仍为 microservice）；`health`、`diagnostics`、`tunnel-self-test` 和 `proxy` 会走真实 backend-core -> provider-gateway 或 k3sctl-adapter -> 节点服务链路，`proxy` 支持受控 JSON 请求体并对超大响应 body 默认输出有界预览，规则见 `docs/reference/microservices.md`。 - `decision upload/list/show/health` 通过 backend-core 用户服务代理访问 D601 k3s Decision Center，用于上传会议记录/决议 Markdown、列出权威记录、查看详情和健康检查；`decision requirement list/upsert` 在同一 records 模型上管理 `goal|decision|blocker|debt|experiment` 需求记录。它们不得直连 D601 Service、NodePort 或 provider-gateway 业务 HTTP。 - `decision diary import ` 将带 `# YYYY年M月D日`、`# YYYY-MM-DD` 或 `# YYYY/M/D` 标题的工作日志拆成每天一篇 Markdown 日记，按 `YYYY-MM/YYYY-MM-DD.md` 虚拟路径写入 Decision Center PostgreSQL；`decision diary list/months/show` 分别用于按月/日期查询、列出月份和查看单日正文；`decision diary edit|upsert --body-file [--title text] [--source-file path] [--tag tag]` 通过 `PUT /api/diary/entries/:idOrDate` 创建当天或历史条目并编辑既有条目。 -- `deploy check/plan/apply` 默认从根目录 `deploy.json` 读取服务 repo 与 commit 期望状态，join `config.json` 和现有 manifest 后使用 target-side build 单一路径校验或更新已支持目标；`deploy plan --env dev|prod` 只从 `origin/master:deploy.json#environments.` 读取 manifest 并输出 dry-run 环境计划，不使用本地 dirty worktree；当前 `deploy apply --env dev` 支持 D601 `backend-core` target-side rollout，以及 `frontend`/`baidu-netdisk`/`decision-center` artifact consumers，dev desired-state smoke 使用 `ci run-dev-e2e`；规则见 `docs/reference/deploy.md`、`docs/reference/dev-environment.md` 和 `docs/reference/dev-ci-runner.md`。 +- `deploy check/plan/apply` 默认从根目录 `deploy.json` 读取服务 repo 与 commit 期望状态，join `config.json` 和现有 manifest 后使用 target-side build 单一路径校验或更新已支持目标；`deploy plan --env dev|prod` 只从 `origin/master:deploy.json#environments.` 读取 manifest 并输出 dry-run 环境计划，不使用本地 dirty worktree；当前 `deploy apply --env dev` 支持 D601 `backend-core` target-side rollout，以及 `frontend`/`baidu-netdisk`/`decision-center` artifact consumers，`findjob`/`pipeline`/`met-nonlinear` 为 D601 direct Compose artifact consumers，`k3sctl-adapter` 只提供 plan/dry-run；dev desired-state smoke 使用 `ci run-dev-e2e`；规则见 `docs/reference/deploy.md`、`docs/reference/dev-environment.md` 和 `docs/reference/dev-ci-runner.md`。`deploy apply --env prod` 同时覆盖 `findjob` 和 `pipeline` 的 pull-only Compose CD，但 `met-nonlinear` 仍然只允许 dry-run/plan，`k3sctl-adapter` 只允许 plan/dry-run。 - `dev-env validate [--manifest path] [--kubectl-dry-run]` 离线校验 D601 `unidesk-dev` namespace、dev PostgreSQL 底座和 dev workload manifest。默认检查 `src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-foundation.k8s.yaml`；也可显式校验 `src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-core.k8s.yaml` 或 `src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-code-queue.k8s.yaml`。所有 namespaced 对象必须只落到 `unidesk-dev`，foundation manifest 必须包含 `postgres-dev` StatefulSet/Service、dev secret/config、迁移 Job 和 DB URL guard，core manifest 必须包含 `backend-core-dev`/`frontend-dev` Deployment/Service，Code Queue dev manifest 必须包含 `code-queue-scheduler-dev`、`code-queue-read-dev`、`code-queue-write-dev` 和 dev provider egress proxy。加 `--kubectl-dry-run` 时额外执行 `kubectl apply --dry-run=client --validate=false -f `，仍不 apply 资源。 - `dev-env prewarm-images [--image image] [--provider-id D601] [--no-pull] [--proxy-url URL] [--pull-timeout-ms N] [--dry-run]` 创建异步 job，通过 UniDesk SSH 维护桥在 D601 上把开发底座依赖镜像从 Docker 缓存导入原生 k3s containerd。默认镜像是 `postgres:16-alpine` 和 `rancher/mirrored-library-busybox:1.36.1`，用于避免 `postgres-dev` 与 local-path helper pod 卡在外部 registry 拉取。该命令固定验证 `/etc/rancher/k3s/k3s.yaml` 指向的 native k3s 上下文，并输出 `dev_env_containerd_image_ready=...` 作为成功判据；它不 apply manifest、不修改生产 `unidesk` namespace。 - `artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`，由 systemd + Docker Compose 管理，位于 native k3s 故障域外；`deploy-service` 只拉取 CI 已发布的 commit-pinned 镜像、retag/recreate 或导入 native k3s，并做 live commit 验证，不构建 runtime source。`deploy-backend-core` 是 deprecated 兼容名，标准 backend-core prod CD 入口是 `deploy apply --env prod --service backend-core`。长期规则见 `docs/reference/artifact-registry.md`。 diff --git a/docs/reference/deploy.md b/docs/reference/deploy.md index 9a885dda..03f90ce6 100644 --- a/docs/reference/deploy.md +++ b/docs/reference/deploy.md @@ -40,7 +40,7 @@ The root `deploy.json` is the single desired-state source for both prod and dev. The optional non-service execution declaration under `environments.dev` is intentionally not specified here. The only currently allowed declaration is `ci`, and its authoritative `repo`, `scriptPath`, `timeoutMs`, short launcher, host fetch boundary and no-CD rules are defined only in `docs/reference/dev-ci-runner.md`. -Environment mode never reads the local dirty working tree manifest. `deploy check --env ...`, `deploy plan --env ...` and `deploy apply --env ...` fetch `origin/master`, read `origin/master:deploy.json`, select `environments.`, and report the manifest commit/blob, service commit IDs, target namespace, database fingerprint and Provider identity. `deploy apply --env dev` is currently enabled for persistent D601 dev `backend-core` target-side rollout and for reviewed artifact consumers `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, `code-queue-mgr`, and `todo-note`. `deploy apply --env prod` exposes reviewed registry artifact consumers (`backend-core`, `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, and `todo-note`), while `code-queue-mgr` remains supervisor-gated. Production backend-core artifact CD is a separate executor because its build target is D601 CI while its runtime target is the master server. The default user-service delivery policy, including CI build, registry publication, dev validation, production CD and manual acceptance, is documented in `docs/reference/user-service-delivery.md`. +Environment mode never reads the local dirty working tree manifest. `deploy check --env ...`, `deploy plan --env ...` and `deploy apply --env ...` fetch `origin/master`, read `origin/master:deploy.json`, select `environments.`, and report the manifest commit/blob, service commit IDs, target namespace, database fingerprint and Provider identity. `deploy apply --env dev` is currently enabled for persistent D601 dev `backend-core` target-side rollout and for reviewed artifact consumers `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, `code-queue-mgr`, `todo-note`, `findjob`, `pipeline` and `met-nonlinear`. `deploy apply --env prod` exposes reviewed registry artifact consumers (`backend-core`, `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, `todo-note`, `findjob`, `pipeline` and `met-nonlinear`), while `code-queue-mgr` remains supervisor-gated and `k3sctl-adapter` is plan/dry-run only. Production backend-core artifact CD is a separate executor because its build target is D601 CI while its runtime target is the master server. The default user-service delivery policy, including CI build, registry publication, dev validation, production CD and manual acceptance, is documented in `docs/reference/user-service-delivery.md`. For services with reviewed production artifact consumers, local-manifest `deploy apply --file ...` is not a production fallback. The CLI blocks `backend-core`, `frontend`, `baidu-netdisk` and `decision-center` before source materialization or Docker build and directs operators to `deploy apply --env prod --service --commit `. This prevents a dirty worktree, local manifest or target-side source build from bypassing the pull-only artifact CD guardrails. The broader precheck and legacy-path classification live in `docs/reference/cicd-standardization.md`. @@ -48,7 +48,7 @@ The current implementation has not yet enabled separate stable and integration d CI/CD server and control-plane services are normal deployable services for versioning purposes: production runtime must be pinned by `deploy.json` to a known commit. A CLI built from `master` may orchestrate the pinned server only through backward-compatible APIs and server-reported capabilities; it must not bypass server-side deploy policy when the pinned server does not support a requested operation. -The only D601 direct-service exception in local manifest mode is `k3sctl-adapter`, because it is the UniDesk-managed control bridge outside the k3s fault domain and owns the Kubernetes service catalog used by the dev public frontend path. Updating it must still use the normal target-side deploy reconciler from a pushed commit. D601 Code Queue, Decision Center, MDTODO, ClaudeQQ and future k3s-managed workloads remain blocked from maintenance-channel direct deploy. +The only D601 direct-service exception in local manifest mode is `k3sctl-adapter`, because it is the UniDesk-managed control bridge outside the k3s fault domain and owns the Kubernetes service catalog used by the dev public frontend path. Its artifact consumer path is plan/dry-run only and never performs real prod deployment without supervisor confirmation. D601 Code Queue, Decision Center, MDTODO, ClaudeQQ and future k3s-managed workloads remain blocked from maintenance-channel direct deploy. `config.json.microservices[].repository.commitId` is retained for catalog compatibility, but `deploy.json` is the deployment version authority for the reconciler. @@ -85,7 +85,7 @@ Phase 3 introduces the dev backend/frontend manifest at `src/components/microser `backend-core-dev` must use `unidesk-dev-runtime-config` and `unidesk-dev-runtime-secrets`, connect to `postgres-dev.../unidesk_dev`, expose HTTP on 8080 and provider ingress on 8081, and write logs under `/var/log/unidesk-dev`. `frontend-dev` must set `CORE_INTERNAL_URL=http://backend-core-dev.unidesk-dev.svc.cluster.local:8080` and must not proxy to production backend-core. -The manifest keeps placeholder image tags and deploy commit values in source control. The controlled `deploy apply --env dev --service backend-core` path fetches `origin/master:deploy.json`, materializes the requested source commit on D601, narrows the dev core control manifest to the selected Service/Deployment pair, replaces placeholders with the requested commit and dev image tag, builds on D601, imports the image into native k3s containerd, applies only the `unidesk-dev` objects and stamps the Deployment. `deploy apply --env dev --service frontend`, `decision-center`, `project-manager`, `oa-event-flow`, `code-queue-mgr`, and `todo-note` consume the existing D601 registry artifact instead of building source on the target. `code-queue-mgr` live prod apply remains supervisor-gated. Client dry-run and static validation remain useful checks before controlled apply: +The manifest keeps placeholder image tags and deploy commit values in source control. The controlled `deploy apply --env dev --service backend-core` path fetches `origin/master:deploy.json`, materializes the requested source commit on D601, narrows the dev core control manifest to the selected Service/Deployment pair, replaces placeholders with the requested commit and dev image tag, builds on D601, imports the image into native k3s containerd, applies only the `unidesk-dev` objects and stamps the Deployment. `deploy apply --env dev --service frontend`, `decision-center`, `project-manager`, `oa-event-flow`, `code-queue-mgr`, `todo-note`, `findjob`, `pipeline` and `met-nonlinear` consume the existing D601 registry artifact instead of building source on the target. The direct Docker/Compose services currently use D601 registry pull + label/config dry-run + health contract as dev validation; they are not separate parallel dev instances yet. `code-queue-mgr` live prod apply remains supervisor-gated. Client dry-run and static validation remain useful checks before controlled apply: - `bun scripts/cli.ts dev-env validate --manifest src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-core.k8s.yaml` - `KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl apply --dry-run=client --validate=false -f src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-core.k8s.yaml` @@ -108,7 +108,7 @@ Maintenance-channel direct D601 apply must not deploy dev Code Queue; the CLI re `bun scripts/cli.ts deploy plan --env dev [--service ]` reads `origin/master:deploy.json#environments.dev` and prints a dry-run environment plan without checking or mutating live runtime resources. `deploy check --env dev` uses the same dry-run environment plan. `--env prod` is available for parity as a dry-run planning path; it reads `origin/master:deploy.json#environments.prod` and must not use a dirty local `deploy.json`. -`bun scripts/cli.ts deploy apply [--file deploy.json | --env dev|prod] [--service ] [--commit ] [--dry-run] [--force]` starts an asynchronous job only for supported targets. Use `bun scripts/cli.ts job status --tail-bytes 30000` to observe progress. `--dry-run` resolves the same plan but does not build or replace runtime objects. `--force` rebuilds even when the live commit matches. Environment apply is not the dev e2e trigger; use `bun scripts/cli.ts ci run-dev-e2e` for the Git-controlled temporary namespace smoke flow. `--env dev` apply is enabled for persistent D601 `backend-core` target-side rollout and for `frontend`/`baidu-netdisk`/`decision-center`/`project-manager`/`oa-event-flow`/`code-queue-mgr`/`todo-note` artifact consumers. `--env prod` apply exposes the D601 registry artifact consumer for `backend-core`, `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, and `todo-note`; `code-queue-mgr` prod live apply is supervisor-gated. Unsupported prod services return a structured `unsupported` payload instead of silently falling back to a maintenance-channel source build. +`bun scripts/cli.ts deploy apply [--file deploy.json | --env dev|prod] [--service ] [--commit ] [--dry-run] [--force]` starts an asynchronous job only for supported targets. Use `bun scripts/cli.ts job status --tail-bytes 30000` to observe progress. `--dry-run` resolves the same plan but does not build or replace runtime objects. `--force` rebuilds even when the live commit matches. Environment apply is not the dev e2e trigger; use `bun scripts/cli.ts ci run-dev-e2e` for the Git-controlled temporary namespace smoke flow. `--env dev` apply is enabled for persistent D601 `backend-core` target-side rollout and for `frontend`/`baidu-netdisk`/`decision-center`/`project-manager`/`oa-event-flow`/`code-queue-mgr`/`todo-note`/`findjob`/`pipeline`/`met-nonlinear` artifact consumers. `--env prod` apply exposes the D601 registry artifact consumer for `backend-core`, `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, `todo-note`, `findjob`, `pipeline` and `met-nonlinear`; `code-queue-mgr` prod live apply is supervisor-gated and `k3sctl-adapter` is plan/dry-run only. Unsupported prod services return a structured `unsupported` payload instead of silently falling back to a maintenance-channel source build. All deploy commands output JSON. Long operations must use `.state/jobs/` and bounded log tails; no deploy path may succeed with missing progress output. @@ -138,6 +138,9 @@ The exception is narrow: - The pushed Git commit remains the version source of truth. The image registry is a content cache and transfer boundary, not a replacement for `deploy.json` or Git. - `baidu-netdisk` is the first main-server direct user-service sample for the same split: CI publishes `127.0.0.1:5000/unidesk/baidu-netdisk:` from `src/components/microservices/baidu-netdisk/Dockerfile`; dev validation and prod CD both pull that artifact, retag `baidu-netdisk`, recreate only `baidu-netdisk` with `--no-build --no-deps --force-recreate`, and verify image labels plus `/health.deploy.commit`. - `frontend` is the UniDesk UI artifact sample: CI publishes `127.0.0.1:5000/unidesk/frontend:` from `src/components/frontend/Dockerfile`; dev CD imports that artifact into native k3s `frontend-dev`, prod CD retags it as `unidesk-frontend` for the master-server Compose service, and both paths verify image labels plus `/health.deploy.commit`. +- `findjob` and `pipeline` are D601 direct Docker/Compose artifact consumers: CD runs on D601 through the existing provider-gateway/SSH maintenance bridge, verifies `127.0.0.1:5000/unidesk/:` labels, writes deploy env/labels, and recreates only the target Compose service with `--no-build --no-deps --force-recreate`. +- `met-nonlinear` has a D601 direct dry-run/plan contract, but live artifact deploy is blocked until the long-running `met-nonlinear-ts` image contract is separated from the ML image Dockerfile contract or otherwise proves the running container image label matches the requested commit. +- `k3sctl-adapter` exposes only artifact consumer plan/dry-run here because it is an infrastructure control bridge; real prod deployment requires supervisor confirmation outside the standard user-service CD path. - This exception must not be generalized to other services unless their resource profile and runtime boundary are documented with the same CI-producer/CD-consumer split. The registry contract is defined in `docs/reference/artifact-registry.md`; the CI producer rules are defined in `docs/reference/ci.md`. diff --git a/docs/reference/deployment.md b/docs/reference/deployment.md index f64c9ce9..10312617 100644 --- a/docs/reference/deployment.md +++ b/docs/reference/deployment.md @@ -30,7 +30,7 @@ CLI 会优先使用 `docker compose` v2 plugin；当 v2 plugin 不存在时才 Compose v2 安装后仍然必须遵守 UniDesk 的服务控制入口：全栈生命周期用 `server start` / `server stop`，单服务重建用 `server rebuild `。不要因为 v2 可用就直接在生产栈上手工执行未纳入 CLI 的 `up --build`、`down -v` 或跨项目清理命令；所有会影响容器的动作都应保持 job 可观测、Compose project 固定、database named volume 保留。主 server Compose 命令必须从 `providerGateway.upgrade.hostProjectRoot` 指定的 canonical UniDesk 根目录运行，临时 worktree、Code Queue 导出目录或实验分支不得复用生产 `-p unidesk` 和固定 `container_name` 去替换生产容器。 -版本化用户服务部署优先使用 `bun scripts/cli.ts deploy apply` 已支持的受控路径；D601 persistent dev apply 当前支持 `backend-core` target-side rollout，以及 `frontend`、`baidu-netdisk`、`decision-center`、`project-manager`、`oa-event-flow`、`code-queue-mgr` 和 `todo-note` 的 artifact consumer validation。`deploy.json` 只声明服务 `id`、`repo` 和 `commitId`；目标节点、Dockerfile、Compose、Kubernetes manifest、健康检查和代理路径继续来自 `config.json` 与现有 manifest。主 server 直管微服务和内部 sidecar，例如 `code-queue-mgr`，也必须支持这一路径：`deploy apply --service code-queue-mgr` 从 `deploy.json` 指定 commit 导出源码、构建镜像、替换固定 Compose service 并验证运行中镜像/健康信息的 commit，但 prod live apply 仍需 supervisor 确认。部署默认遵循 target-side build：服务部署到哪台 target，就在哪台 target 从 remote commit 导出源码、一次性代理构建镜像并部署；不得把中心构建镜像作为默认分发路径，也不得用 `docker commit` 或脏 worktree 作为部署输入。production backend-core 是明确例外；`frontend` 是用户服务 UI / 前端镜像化样板，`baidu-netdisk` 是主 server 直管微服务的镜像化样板：D601 CI 构建并推送 commit-pinned 镜像到 D601 artifact registry，dev/prod CD 只拉取、retag 或导入、recreate/rollout 和验证，不在 master server 或 dev target 执行 frontend Compose build。完整规则见 `docs/reference/deploy.md`，D601 dev/Rust 边界见 `docs/reference/dev-environment.md`，artifact registry 见 `docs/reference/artifact-registry.md`。 +版本化用户服务部署优先使用 `bun scripts/cli.ts deploy apply` 已支持的受控路径；D601 persistent dev apply 当前支持 `backend-core` target-side rollout，以及 `frontend`、`baidu-netdisk`、`decision-center`、`project-manager`、`oa-event-flow`、`code-queue-mgr`、`todo-note`、`findjob`、`pipeline` 和 `met-nonlinear` 的 artifact consumer validation。`deploy.json` 只声明服务 `id`、`repo` 和 `commitId`；目标节点、Dockerfile、Compose、Kubernetes manifest、健康检查和代理路径继续来自 `config.json` 与现有 manifest。主 server 直管微服务和内部 sidecar，例如 `code-queue-mgr`，也必须支持这一路径：`deploy apply --service code-queue-mgr` 从 `deploy.json` 指定 commit 导出源码、构建镜像、替换固定 Compose service 并验证运行中镜像/健康信息的 commit，但 prod live apply 仍需 supervisor 确认。部署默认遵循 target-side build：服务部署到哪台 target，就在哪台 target 从 remote commit 导出源码、一次性代理构建镜像并部署；不得把中心构建镜像作为默认分发路径，也不得用 `docker commit` 或脏 worktree 作为部署输入。production backend-core 是明确例外；`frontend` 是用户服务 UI / 前端镜像化样板，`baidu-netdisk` 是主 server 直管微服务的镜像化样板，`findjob` 和 `pipeline` 是 D601 direct Docker/Compose pull-only 样板：D601 CI 构建并推送 commit-pinned 镜像到 D601 artifact registry，CD 只拉取、retag 或导入、recreate/rollout 和验证，不在 master server 或 runtime target 执行 Compose build。`met-nonlinear` 当前只允许 dry-run/plan，`k3sctl-adapter` 只允许 plan/dry-run 且真实 prod 部署需要 supervisor 确认。完整规则见 `docs/reference/deploy.md`，D601 dev/Rust 边界见 `docs/reference/dev-environment.md`，artifact registry 见 `docs/reference/artifact-registry.md`。 ## Main Server Swap diff --git a/docs/reference/microservices.md b/docs/reference/microservices.md index 1927694d..77dab48f 100644 --- a/docs/reference/microservices.md +++ b/docs/reference/microservices.md @@ -346,12 +346,15 @@ ClaudeQQ 的业务源码和持久化数据仍在 D601，但正式运行由 k3s - `bun scripts/cli.ts microservice status findjob`：查看单个用户服务的配置与运行态。 - `bun scripts/cli.ts microservice health findjob`：通过 backend-core -> provider-gateway -> D601 本机后端链路探测 FindJob `/api/health`。 - `bun scripts/cli.ts microservice proxy findjob /api/summary`：通过同一私有代理读取业务 API，适合人工验证，不用于公开业务端口。 +- `findjob` 的 CD 形态是 D601 direct Compose artifact consumer：dev/prod 都消费 `127.0.0.1:5000/unidesk/findjob:`，并只重建 `server` compose service；dev validation 先做 registry pull、label、config dry-run 和 health contract，prod 只做 pull-only no-build recreate。 - `bun scripts/cli.ts microservice health pipeline`：通过 backend-core -> provider-gateway -> D601 本机后端链路探测 Pipeline `/health`。 - `bun scripts/cli.ts microservice proxy pipeline '/api/snapshot?__unideskArrayLimit=registry.components:8,runs:3'`：读取 Pipeline snapshot 的有界预览，适合人工验证，不用于公开业务端口；验证甘特图时应确认 run 和 procedureRun 摘要包含 `startedAt`、`finishedAt` 或 `durationMs`；若 body 仍超过 CLI 阈值，默认只输出 `bodyPreview`，需要完整 body 时显式追加 `--raw`。 +- `pipeline` 的 CD 形态是 D601 direct Compose artifact consumer：dev/prod 都消费 `127.0.0.1:5000/unidesk/pipeline:`，并只重建 `pipeline-control` compose service；dev validation 和 prod live 路径与 `findjob` 保持同样的 pull-only no-build 合约。 - Pipeline node 控制写入由 UniDesk frontend 调用同源 `/api/microservices/pipeline/proxy/api/node-control/...` 完成；通用 CLI `microservice proxy` 仍主要作为读取验证入口，不作为人工批量写入工具。 - `bun scripts/cli.ts microservice health met-nonlinear`：通过 backend-core -> provider-gateway -> D601 本机 TS 编排后端链路探测 MET Nonlinear `/health`。 - `bun scripts/cli.ts microservice proxy met-nonlinear /api/queue` 与 `bun scripts/cli.ts microservice proxy met-nonlinear /api/images`：读取 MET Nonlinear 队列、GPU 策略和训练镜像状态，适合人工验证，不用于公开业务端口。 - `bun scripts/cli.ts microservice proxy met-nonlinear '/api/projects?root=projects&limit=500'` 与 `bun scripts/cli.ts microservice proxy met-nonlinear '/api/projects/config?path=projects/' --raw`：验证项目库文件树输入和结构化项目详情；详情应包含 config、progress、data、model、metrics 字段，供前端渲染训练状态、模型参数量和指标。 +- `met-nonlinear` 的 CD 形态同样是 D601 direct Compose artifact consumer，但当前 live deploy 被 Dockerfile.contract 与 long-running service contract 不一致阻塞，只允许 dry-run/plan。 - `bun scripts/cli.ts microservice health claudeqq`、`bun scripts/cli.ts microservice proxy claudeqq /api/napcat/login`、`bun scripts/cli.ts microservice proxy claudeqq /api/events/recent` 和 `bun scripts/cli.ts microservice proxy claudeqq /api/events/subscriptions`：验证 ClaudeQQ 后端、NapCat 容器登录、事件订阅和私有代理链路；消息推送使用 `POST /api/push/text`，不得开放 D601 `3290/3000/3001/6099` 公网端口。 - `bun scripts/cli.ts microservice health todo-note` 与 `bun scripts/cli.ts microservice proxy todo-note /api/instances`：验证主 server Todo Note 后端、PostgreSQL 存储和本机 provider-gateway 私有代理链路。 - `bun scripts/cli.ts microservice health oa-event-flow`、`bun scripts/cli.ts microservice proxy oa-event-flow /api/diagnostics --raw` 与 `bun scripts/cli.ts microservice proxy oa-event-flow '/api/events?tags=service:code-queue&limit=20' --raw`：验证统一 OA 事件流、事件表、tag 查询和统计中心。 diff --git a/docs/reference/user-service-delivery.md b/docs/reference/user-service-delivery.md index 6c62c143..97e0f2f6 100644 --- a/docs/reference/user-service-delivery.md +++ b/docs/reference/user-service-delivery.md @@ -4,7 +4,7 @@ This document owns the default delivery path for UniDesk user services registere ## Scope -User services are non-core business services mounted onto the UniDesk platform. They must remain deliverable without changing the base platform strategy. The default policy here applies to user services that are expected to reach production after validation, including the UniDesk user-service UI artifact (`frontend`), main-server direct services such as Baidu Netdisk, and D601-managed services such as Decision Center. +User services are non-core business services mounted onto the UniDesk platform. They must remain deliverable without changing the base platform strategy. The default policy here applies to user services that are expected to reach production after validation, including the UniDesk user-service UI artifact (`frontend`), main-server direct services such as Baidu Netdisk, D601-managed k3s services such as Decision Center, and D601 direct Docker/Compose services such as FindJob and Pipeline. This policy does not apply to: @@ -51,7 +51,7 @@ Some registered user services are intentionally upstream-image consumers instead - CD must be pull-only and must verify the image identity, OCI labels and service health through the UniDesk private proxy. - Until the upstream digest has been resolved and mirrored or pinned for a future mirror producer, File Browser remains a recovery/diagnostic image-only path rather than a standard release path. -The current catalog covers `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, `todo-note`, `code-queue-mgr`, `findjob`, `pipeline`, `met-nonlinear`, `k3sctl-adapter`, `mdtodo` and `claudeqq` as supported `publish-user-service` source-build services. `code-queue` is cataloged but blocked by the D601 dev/prod boundary. `filebrowser` and `filebrowser-d601` are cataloged as pinned upstream images, not source builds. +The current catalog covers `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, `todo-note`, `code-queue-mgr`, `findjob`, `pipeline`, `met-nonlinear`, `k3sctl-adapter`, `mdtodo` and `claudeqq` as supported `publish-user-service` source-build services. `k3sctl-adapter` is cataloged for artifact visibility, but it is not a user-service live deploy target in this policy. `code-queue` is cataloged but blocked by the D601 dev/prod boundary. `filebrowser` and `filebrowser-d601` are cataloged as pinned upstream images, not source builds. ## Frontend Pairing @@ -120,6 +120,17 @@ Todo Note is an external-source main-server Compose service that follows the sta - The Compose runtime injects `UNIDESK_TODO_NOTE_DEPLOY_REF`, `UNIDESK_TODO_NOTE_DEPLOY_SERVICE_ID`, `UNIDESK_TODO_NOTE_DEPLOY_REPO`, `UNIDESK_TODO_NOTE_DEPLOY_COMMIT`, and `UNIDESK_TODO_NOTE_DEPLOY_REQUESTED_COMMIT`; the consumer health probe must return matching `deploy.commit` and `deploy.requestedCommit`. - `server rebuild todo-note` remains a maintenance/local rebuild path only. It is not the standard versioned release truth for Todo Note. +## D601 Direct Compose Consumers + +FindJob and Pipeline are D601 direct Docker/Compose user services. They differ from k3s-managed services because the artifact consumer operates on the D601 Docker host and the existing Compose project, not on Kubernetes manifests, containerd imports, or Kubernetes API service proxies. + +- `findjob` consumes `127.0.0.1:5000/unidesk/findjob:` and recreates only Compose service `server` in `/home/ubuntu/findjob`. +- `pipeline` consumes `127.0.0.1:5000/unidesk/pipeline:` and recreates only Compose service `pipeline-control` in `/home/ubuntu/pipeline`. +- Dev validation currently means D601 registry pull + required image labels + config/compose dry-run + health contract. It is not yet a separate parallel dev instance. +- Production CD is pull-only and no-build: it verifies the registry manifest and image labels, writes deploy env/labels, and runs `docker compose up -d --no-build --no-deps --force-recreate `. +- `met-nonlinear` has the same dry-run/plan shape, but live deploy is blocked until the long-running `met-nonlinear-ts` image contract can prove the requested commit label independently from the ML image Dockerfile. +- `k3sctl-adapter` has an artifact consumer plan/dry-run only. Real production deployment of that infrastructure bridge requires supervisor confirmation outside this policy. + ## Decision Center Decision Center is the canonical example of a user service that doubles as a product workflow for requirements, decisions, and daily work diaries. diff --git a/scripts/src/artifact-registry.ts b/scripts/src/artifact-registry.ts index e9f717fb..0a9fa5cd 100644 --- a/scripts/src/artifact-registry.ts +++ b/scripts/src/artifact-registry.ts @@ -71,14 +71,27 @@ const defaultOptions: ArtifactRegistryOptions = { sourceRepoExplicit: false, deployRef: null, }; -const supportedArtifactConsumerServices = ["backend-core", "baidu-netdisk", "code-queue-mgr", "decision-center", "frontend", "oa-event-flow", "project-manager", "todo-note"] as const; +const supportedArtifactConsumerServices = [ + "backend-core", + "baidu-netdisk", + "code-queue-mgr", + "decision-center", + "findjob", + "frontend", + "k3sctl-adapter", + "met-nonlinear", + "oa-event-flow", + "pipeline", + "project-manager", + "todo-note", +] as const; type SupportedArtifactConsumerService = typeof supportedArtifactConsumerServices[number]; const legacyDeployBackendCoreDisabled = true; interface ArtifactConsumerSpec { serviceId: SupportedArtifactConsumerService; environment?: ArtifactDeployEnvironment; - kind: "compose" | "d601-k3s"; + kind: "compose" | "d601-compose" | "d601-k3s"; registryRepository: string; sourceRepo?: string; dockerfile: string; @@ -99,6 +112,9 @@ interface ArtifactConsumerTarget { deployEnvPrefix: string; healthProbeCommand: string; requireHealthCommit: boolean; + workDir?: string; + composeFile?: string; + projectHint?: string; }; k3s?: { namespace: string; @@ -270,6 +286,117 @@ const artifactConsumerSpecs: Record = { }, }, }, + "findjob": { + serviceId: "findjob", + environment: "prod", + kind: "d601-compose", + sourceRepo: "https://gitee.com/Lyon1998/findjob", + registryRepository: "unidesk/findjob", + dockerfile: "Dockerfile", + prodLiveApply: "enabled", + targets: { + dev: { + targetImage: "findjob-server", + targetCommitImage: (commit: string) => `findjob-server:${commit}`, + deployRef: "deploy.json#environments.dev.services.findjob", + compose: { + serviceName: "server", + containerName: "findjob-server", + deployEnvPrefix: "UNIDESK_FINDJOB_DEPLOY", + workDir: "/home/ubuntu/findjob", + composeFile: "docker-compose.yml", + projectHint: "findjob", + healthProbeCommand: "curl -fsS --max-time 12 http://127.0.0.1:3254/api/health", + requireHealthCommit: false, + }, + }, + prod: { + targetImage: "findjob-server", + targetCommitImage: (commit: string) => `findjob-server:${commit}`, + deployRef: "deploy.json#environments.prod.services.findjob", + compose: { + serviceName: "server", + containerName: "findjob-server", + deployEnvPrefix: "UNIDESK_FINDJOB_DEPLOY", + workDir: "/home/ubuntu/findjob", + composeFile: "docker-compose.yml", + projectHint: "findjob", + healthProbeCommand: "curl -fsS --max-time 12 http://127.0.0.1:3254/api/health", + requireHealthCommit: false, + }, + }, + }, + }, + "k3sctl-adapter": { + serviceId: "k3sctl-adapter", + environment: "prod", + kind: "d601-compose", + registryRepository: "unidesk/k3sctl-adapter", + dockerfile: "src/components/microservices/k3sctl-adapter/Dockerfile", + prodLiveApply: "supervisor-only", + prodLiveBlockReason: "k3sctl-adapter is an infrastructure control bridge; this executor exposes artifact consumer plan/dry-run only. Real production deployment requires supervisor confirmation outside this task.", + targets: { + prod: { + targetImage: "unidesk-k3sctl-adapter:d601", + targetCommitImage: (commit: string) => `unidesk-k3sctl-adapter:${commit}`, + deployRef: "deploy.json#environments.prod.services.k3sctl-adapter", + compose: { + serviceName: "k3sctl-adapter", + containerName: "k3sctl-adapter", + deployEnvPrefix: "UNIDESK_K3SCTL_ADAPTER_DEPLOY", + workDir: "/home/ubuntu/cq-deploy", + composeFile: "src/components/microservices/k3sctl-adapter/docker-compose.d601.yml", + projectHint: "k3sctl-adapter", + healthProbeCommand: "curl -fsS --max-time 10 http://127.0.0.1:4266/health", + requireHealthCommit: false, + }, + }, + }, + }, + "met-nonlinear": { + serviceId: "met-nonlinear", + environment: "prod", + kind: "d601-compose", + sourceRepo: "https://github.com/pikasTech/met_nonlinear", + registryRepository: "unidesk/met-nonlinear", + dockerfile: "docker/unidesk/Dockerfile.ml", + prodLiveApply: "unsupported", + prodLiveBlockReason: "met-nonlinear is blocked for live artifact deploy because config.json points at docker/unidesk/Dockerfile.ml while the compose service is met-nonlinear-ts. The current compose contract does not let CD prove that the recreated long-running container image label equals the requested commit.", + runtimeVerification: "blocked", + runtimeVerificationBlockReason: "D601 direct artifact consumer is implemented, but this service's registered Dockerfile is the ML image contract while the long-running Compose service is met-nonlinear-ts. Publish a labeled artifact that matches the running service image contract before live deploy, or update the service contract to a server Dockerfile.", + targets: { + dev: { + targetImage: "met-nonlinear-ml:tf26", + targetCommitImage: (commit: string) => `met-nonlinear-ml:${commit}`, + deployRef: "deploy.json#environments.dev.services.met-nonlinear", + compose: { + serviceName: "met-nonlinear-ts", + containerName: "met-nonlinear-ts", + deployEnvPrefix: "UNIDESK_MET_NONLINEAR_DEPLOY", + workDir: "/home/ubuntu/met_nonlinear", + composeFile: "docker-compose.unidesk.yml", + projectHint: "met-nonlinear", + healthProbeCommand: "curl -fsS --max-time 20 http://127.0.0.1:3288/health", + requireHealthCommit: false, + }, + }, + prod: { + targetImage: "met-nonlinear-ml:tf26", + targetCommitImage: (commit: string) => `met-nonlinear-ml:${commit}`, + deployRef: "deploy.json#environments.prod.services.met-nonlinear", + compose: { + serviceName: "met-nonlinear-ts", + containerName: "met-nonlinear-ts", + deployEnvPrefix: "UNIDESK_MET_NONLINEAR_DEPLOY", + workDir: "/home/ubuntu/met_nonlinear", + composeFile: "docker-compose.unidesk.yml", + projectHint: "met-nonlinear", + healthProbeCommand: "curl -fsS --max-time 20 http://127.0.0.1:3288/health", + requireHealthCommit: false, + }, + }, + }, + }, "oa-event-flow": { serviceId: "oa-event-flow", environment: "prod", @@ -338,6 +465,47 @@ const artifactConsumerSpecs: Record = { }, }, }, + "pipeline": { + serviceId: "pipeline", + environment: "prod", + kind: "d601-compose", + sourceRepo: "https://github.com/pikasTech/pipeline", + registryRepository: "unidesk/pipeline", + dockerfile: "Dockerfile", + prodLiveApply: "enabled", + targets: { + dev: { + targetImage: "pipeline-v2-control", + targetCommitImage: (commit: string) => `pipeline-v2-control:${commit}`, + deployRef: "deploy.json#environments.dev.services.pipeline", + compose: { + serviceName: "pipeline-control", + containerName: "pipeline-v2-control", + deployEnvPrefix: "UNIDESK_PIPELINE_DEPLOY", + workDir: "/home/ubuntu/pipeline", + composeFile: "docker-compose.yml", + projectHint: "pipeline", + healthProbeCommand: "curl -fsS --max-time 20 http://127.0.0.1:18082/health", + requireHealthCommit: false, + }, + }, + prod: { + targetImage: "pipeline-v2-control", + targetCommitImage: (commit: string) => `pipeline-v2-control:${commit}`, + deployRef: "deploy.json#environments.prod.services.pipeline", + compose: { + serviceName: "pipeline-control", + containerName: "pipeline-v2-control", + deployEnvPrefix: "UNIDESK_PIPELINE_DEPLOY", + workDir: "/home/ubuntu/pipeline", + composeFile: "docker-compose.yml", + projectHint: "pipeline", + healthProbeCommand: "curl -fsS --max-time 20 http://127.0.0.1:18082/health", + requireHealthCommit: false, + }, + }, + }, + }, "todo-note": { serviceId: "todo-note", environment: "prod", @@ -1318,6 +1486,167 @@ async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise/dev/null", + "docker compose version >/dev/null", + `curl -fsSI -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' ${shellQuote(`http://127.0.0.1:${options.port}/v2/${spec.registryRepository}/manifests/${commit}`)} >/dev/null`, + "docker pull -q \"$registry_image\" >/dev/null", + "label_commit=$(docker image inspect \"$registry_image\" --format '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}')", + "label_service=$(docker image inspect \"$registry_image\" --format '{{ index .Config.Labels \"unidesk.ai/service-id\" }}')", + "label_dockerfile=$(docker image inspect \"$registry_image\" --format '{{ index .Config.Labels \"unidesk.ai/dockerfile\" }}')", + "test \"$label_commit\" = \"$commit\"", + "test \"$label_service\" = \"$service_id\"", + "test \"$label_dockerfile\" = \"$dockerfile\"", + "test -d \"$work_dir\"", + "test -f \"$work_dir/$compose_file\"", + "docker tag \"$registry_image\" \"$stable_image\"", + "docker tag \"$registry_image\" \"$commit_image\"", + "override=\"$work_dir/.unidesk-artifact-consumer.override.yml\"", + "printf '%s' \"$override_b64\" | base64 -d > \"$override\"", + "export UNIDESK_ARTIFACT_STABLE_IMAGE=\"$stable_image\"", + "project=$(docker inspect \"$container\" --format '{{ index .Config.Labels \"com.docker.compose.project\" }}' 2>/dev/null || true)", + "if [ -z \"$project\" ]; then project=\"$project_hint\"; fi", + "if [ -z \"$project\" ]; then project=$(basename \"$work_dir\"); fi", + "cd \"$work_dir\"", + "docker compose -p \"$project\" -f \"$compose_file\" -f \"$override\" up -d --no-build --no-deps --force-recreate \"$compose_service\"", + "ready=0", + "for attempt in $(seq 1 90); do", + " cid=$(docker ps -q --filter label=com.docker.compose.project=\"$project\" --filter label=com.docker.compose.service=\"$compose_service\" --filter label=com.docker.compose.oneoff=False | head -1 || true)", + " if [ -n \"$cid\" ]; then", + " health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' \"$cid\" 2>/dev/null || true)", + " echo \"artifact_cd_container_probe attempt=$attempt cid=$cid health=$health\"", + " if [ \"$health\" = \"healthy\" ] || [ \"$health\" = \"running\" ]; then ready=1; break; fi", + " else", + " echo \"artifact_cd_container_probe attempt=$attempt cid=missing\"", + " fi", + " sleep 2", + "done", + "test \"$ready\" = \"1\"", + "cid=$(docker ps -q --filter label=com.docker.compose.project=\"$project\" --filter label=com.docker.compose.service=\"$compose_service\" --filter label=com.docker.compose.oneoff=False | head -1)", + "image_id=$(docker inspect -f '{{.Image}}' \"$cid\")", + "actual_commit=$(docker image inspect -f '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}' \"$image_id\")", + "actual_service=$(docker image inspect -f '{{ index .Config.Labels \"unidesk.ai/service-id\" }}' \"$image_id\")", + "container_commit=$(docker inspect -f '{{ index .Config.Labels \"unidesk.ai/deploy-commit\" }}' \"$cid\")", + "test \"$actual_commit\" = \"$commit\"", + "test \"$actual_service\" = \"$service_id\"", + "test \"$container_commit\" = \"$commit\"", + "health_probe=$(printf '%s' \"$health_probe_b64\" | base64 -d)", + "docker exec \"$cid\" sh -lc \"$health_probe\" >/tmp/unidesk-artifact-health.out", + "cat /tmp/unidesk-artifact-health.out", + "printf 'artifact_cd_service=%s\\nartifact_cd_source_repo=%s\\nartifact_cd_deploy_ref=%s\\nartifact_cd_source_image=%s\\nartifact_cd_stable_image=%s\\nartifact_cd_runtime_image=%s\\nartifact_cd_commit=%s\\nartifact_cd_container=%s\\nartifact_cd_container_id=%s\\nartifact_cd_image_label_commit=%s\\nartifact_cd_container_label_commit=%s\\n' \"$service_id\" \"$source_repo\" \"$deploy_ref\" \"$registry_image\" \"$stable_image\" \"$commit_image\" \"$commit\" \"$container\" \"$cid\" \"$actual_commit\" \"$container_commit\"", + ].join("\n"); +} + +async function deployD601ComposeArtifactNow(options: ArtifactRegistryOptions, spec: ArtifactConsumerSpec, target: ArtifactConsumerTarget): Promise> { + const environment = options.environment ?? "prod"; + const commit = options.commit; + if (commit === null) throw new Error("artifact-registry deploy-service requires --commit "); + if (target.compose === undefined) throw new Error(`${spec.serviceId} missing compose artifact consumer config`); + const health = runReadonlyStatus(options, true); + if (health.ok !== true) return { ok: false, serviceId: spec.serviceId, error: "D601 artifact registry is not healthy", health }; + const sourceImage = artifactImageRef(options, spec, commit); + const registryProbe = runRemoteScript(options, registryArtifactProbeScript(options, spec, commit), Math.max(options.timeoutMs, 120_000)); + if (registryProbe.exitCode !== 0 || registryProbe.timedOut) { + return { + ok: false, + supported: true, + serviceId: spec.serviceId, + step: "registry-artifact-check", + error: registryArtifactMissingMessage(spec), + sourceImage, + registryProbe: commandTail(registryProbe), + }; + } + const deploy = runRemoteScript(options, d601ComposeArtifactDeployScript(options, spec, target, commit), Math.max(options.timeoutMs, 420_000)); + if (deploy.exitCode !== 0 || deploy.timedOut) { + return { + ok: false, + supported: true, + serviceId: spec.serviceId, + step: "d601-compose-artifact-deploy", + sourceImage, + registryProbe: commandTail(registryProbe), + deploy: commandTail(deploy), + rollback: rollbackInfo(spec, target, environment, commit), + }; + } + return { + ok: true, + supported: true, + serviceId: spec.serviceId, + environment, + commit, + providerId: options.providerId, + sourceRepo: sourceRepoFor(options, spec), + deployRef: deployRefFor(options, spec), + sourceImage, + targetImage: target.targetImage, + targetCommitImage: target.targetCommitImage(commit), + composeService: target.compose.serviceName, + containerName: target.compose.containerName, + registryProbe: commandTail(registryProbe), + deploy: commandTail(deploy), + validation: { + liveCommit: commit, + liveRequestedCommit: commit, + imageLabelCommit: commit, + containerDeployLabelCommit: commit, + serviceHealthCommit: target.compose.requireHealthCommit ? commit : "not-required", + serviceHealthRequestedCommit: target.compose.requireHealthCommit ? commit : "not-required", + healthyOldVersionAccepted: false, + }, + rollback: rollbackInfo(spec, target, environment, commit), + }; +} + function deployBackendCoreJob(args: string[], options: ArtifactRegistryOptions): Record { if (options.commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit "); const spec = artifactConsumerSpecs["backend-core"]; @@ -1383,12 +1712,15 @@ function dryRunArtifactConsumerPlan(options: ArtifactRegistryOptions, spec: Arti reason: spec.runtimeVerificationBlockReason ?? spec.prodLiveBlockReason ?? null, }, }; - if (spec.kind === "compose") { + if (spec.kind === "compose" || spec.kind === "d601-compose") { if (target.compose === undefined) throw new Error(`${spec.serviceId} missing compose artifact consumer config`); return { ...common, target: { - kind: "compose", + kind: spec.kind, + runtimeHost: spec.kind === "d601-compose" ? "D601" : "main-server", + workDir: target.compose.workDir, + composeFile: target.compose.composeFile, composeService: target.compose.serviceName, containerName: target.compose.containerName, targetImage: options.targetImage ?? target.targetImage, @@ -1398,8 +1730,13 @@ function dryRunArtifactConsumerPlan(options: ArtifactRegistryOptions, spec: Arti }, validation: [ "D601 registry /v2 manifest exists for the commit tag", - "loaded image labels match service id, source commit, and Dockerfile", - "running Compose container image label matches the requested commit", + spec.kind === "d601-compose" + ? "D601-pulled image labels match service id, source commit, and Dockerfile" + : "loaded image labels match service id, source commit, and Dockerfile", + spec.kind === "d601-compose" + ? "running D601 Compose container is recreated with a no-build override that points the service image at the artifact" + : "running Compose container image label matches the requested commit", + ...(spec.kind === "d601-compose" ? ["running Compose container image label matches the requested commit"] : []), verificationBlocked ? `blocked: ${spec.runtimeVerificationBlockReason}` : target.compose.requireHealthCommit @@ -1434,11 +1771,14 @@ function dryRunArtifactConsumerPlan(options: ArtifactRegistryOptions, spec: Arti } function rollbackInfo(spec: ArtifactConsumerSpec, target: ArtifactConsumerTarget, environment: ArtifactDeployEnvironment, commit: string): Record { - if (spec.kind === "compose") { + if (spec.kind === "compose" || spec.kind === "d601-compose") { const compose = target.compose; return { - type: "compose-retag-recreate", + type: spec.kind === "d601-compose" ? "d601-compose-retag-recreate" : "compose-retag-recreate", composeService: compose?.serviceName, + containerName: compose?.containerName, + workDir: compose?.workDir, + composeFile: compose?.composeFile, previousImageHint: `Use docker image ls / docker inspect to find the previous labeled ${spec.serviceId} image id; Compose volumes are unchanged.`, commandShape: `bun scripts/cli.ts deploy apply --env ${environment} --service ${spec.serviceId}${environment === "prod" ? " --commit " : ""}`, }; @@ -1670,6 +2010,7 @@ async function deployServiceNow(options: ArtifactRegistryOptions): Promise { "bun scripts/cli.ts artifact-registry deploy-service --env prod --service code-queue-mgr --commit --dry-run [--provider-id D601]", "bun scripts/cli.ts artifact-registry deploy-service --env prod --service todo-note --commit [--dry-run] [--run-now] [--provider-id D601]", "bun scripts/cli.ts artifact-registry deploy-service --env dev --service todo-note --commit [--dry-run] [--run-now] [--provider-id D601]", + "bun scripts/cli.ts artifact-registry deploy-service --env dev --service findjob --commit --dry-run [--provider-id D601]", + "bun scripts/cli.ts artifact-registry deploy-service --env dev --service pipeline --commit --dry-run [--provider-id D601]", + "bun scripts/cli.ts artifact-registry deploy-service --env dev --service met-nonlinear --commit --dry-run [--provider-id D601]", + "bun scripts/cli.ts artifact-registry deploy-service --env prod --service k3sctl-adapter --commit --dry-run [--provider-id D601]", ], firstStage: "install now writes the rendered systemd/Compose/config files and starts the registry", artifactConsumers: { @@ -1739,6 +2084,10 @@ function localHelp(): Record { "bun scripts/cli.ts deploy apply --env prod --service oa-event-flow", "bun scripts/cli.ts deploy apply --env prod --service code-queue-mgr --dry-run", "bun scripts/cli.ts deploy apply --env prod --service todo-note", + "bun scripts/cli.ts deploy apply --env prod --service findjob --dry-run", + "bun scripts/cli.ts deploy apply --env prod --service pipeline --dry-run", + "bun scripts/cli.ts deploy apply --env prod --service met-nonlinear --dry-run", + "bun scripts/cli.ts deploy apply --env prod --service k3sctl-adapter --dry-run", ], devCommands: [ "bun scripts/cli.ts deploy apply --env dev --service frontend", @@ -1747,6 +2096,9 @@ function localHelp(): Record { "bun scripts/cli.ts deploy apply --env dev --service oa-event-flow --dry-run", "bun scripts/cli.ts deploy apply --env dev --service code-queue-mgr --dry-run", "bun scripts/cli.ts deploy apply --env dev --service todo-note", + "bun scripts/cli.ts deploy apply --env dev --service findjob --dry-run", + "bun scripts/cli.ts deploy apply --env dev --service pipeline --dry-run", + "bun scripts/cli.ts deploy apply --env dev --service met-nonlinear --dry-run", ], rollbackShape: "rerun the same artifact consumer with a previous commit-pinned image", }, diff --git a/scripts/src/deploy.ts b/scripts/src/deploy.ts index 027de7bb..3c0f4788 100644 --- a/scripts/src/deploy.ts +++ b/scripts/src/deploy.ts @@ -136,12 +136,14 @@ const nativeK3sCtrAddress = "/run/k3s/containerd/containerd.sock"; const unideskRepoUrl = "https://github.com/pikasTech/unidesk"; const d601MaintenanceDeployAllowedServiceIds = new Set(["backend-core", "k3sctl-adapter", "code-queue"]); const devApplySupportedServiceIds = new Set(["backend-core"]); -const devArtifactConsumerServiceIds = new Set(["baidu-netdisk", "code-queue-mgr", "decision-center", "frontend", "oa-event-flow", "project-manager", "todo-note"]); +const devArtifactConsumerServiceIds = new Set(["baidu-netdisk", "code-queue-mgr", "decision-center", "findjob", "frontend", "met-nonlinear", "oa-event-flow", "pipeline", "project-manager", "todo-note"]); const devArtifactConsumerProdDesiredFallbackServiceIds = new Set(["code-queue-mgr", "oa-event-flow", "project-manager", "todo-note"]); -const prodArtifactConsumerServiceIds = new Set(["backend-core", "baidu-netdisk", "code-queue-mgr", "decision-center", "frontend", "oa-event-flow", "project-manager", "todo-note"]); -const prodForbiddenTargetSideBuildServiceIds = new Set(["backend-core", "baidu-netdisk", "decision-center", "frontend"]); +const prodArtifactConsumerServiceIds = new Set(["backend-core", "baidu-netdisk", "code-queue-mgr", "decision-center", "findjob", "frontend", "k3sctl-adapter", "met-nonlinear", "oa-event-flow", "pipeline", "project-manager", "todo-note"]); +const prodForbiddenTargetSideBuildServiceIds = new Set(["backend-core", "baidu-netdisk", "decision-center", "findjob", "frontend", "k3sctl-adapter", "met-nonlinear", "pipeline"]); const prodArtifactLiveApplyBlockedServiceIds = new Map([ ["code-queue-mgr", "code-queue-mgr is the main-server Code Queue control-plane sidecar; live production apply requires explicit supervisor confirmation."], + ["met-nonlinear", "met-nonlinear is blocked for live artifact deploy because config.json points at docker/unidesk/Dockerfile.ml while the compose service is met-nonlinear-ts."], + ["k3sctl-adapter", "k3sctl-adapter is an infrastructure control bridge; this executor exposes artifact consumer plan/dry-run only. Real production deployment requires supervisor confirmation outside this task."], ]); const deployEnvironmentTargets: Record = { dev: { @@ -205,7 +207,7 @@ export function deployHelp(action: string | undefined = undefined): Record", default: defaultDeployFile, description: "Desired-state manifest path relative to the repo root. JSON and ESM JS manifests are supported, for example deploy.json or develop.js. Local manifest apply allows k3sctl-adapter and explicit production code-queue controlled rollout on D601." }, - { name: "--env ", description: "Read the named environment from origin/master:deploy.json. Dev apply supports backend-core target-side rollout plus reviewed artifact consumers for frontend, baidu-netdisk, decision-center, project-manager, oa-event-flow, code-queue-mgr, and todo-note. Prod apply uses the D601 registry artifact consumer for reviewed services; code-queue-mgr live apply is supervisor-gated." }, + { name: "--env ", description: "Read the named environment from origin/master:deploy.json. Dev apply supports backend-core target-side rollout plus reviewed artifact consumers for frontend, baidu-netdisk, decision-center, project-manager, oa-event-flow, code-queue-mgr, todo-note, findjob, pipeline, and met-nonlinear. Prod apply uses the D601 registry artifact consumer for reviewed services; code-queue-mgr and D601 direct infrastructure/incomplete contracts are dry-run or supervisor-gated." }, { name: "--service ", description: "Limit reconcile to one service from the manifest." }, { name: "--commit ", description: "Prod artifact rollback/apply override for a selected service; the image must already exist in D601 registry." }, { name: "--dry-run", description: "Prepare and validate without mutating the target service." }, @@ -767,7 +769,7 @@ function isDevK3sDeployService(service: UniDeskMicroserviceConfig): boolean { } function isDevArtifactConsumerService(service: UniDeskMicroserviceConfig): boolean { - return service.deployment.mode === "k3sctl-managed" + return (service.deployment.mode === "k3sctl-managed" || isDirectComposeDeployMode(service)) && devArtifactConsumerServiceIds.has(service.id); } @@ -791,10 +793,12 @@ function selectServices(config: UniDeskConfig, manifest: DeployManifest, service return selected.map((desired) => { if (manifest.environment === "dev") { const service = devK3sDeployService(desired.id); - if (service === undefined) { - throw new Error(`deploy --env dev service ${desired.id} is not enabled for direct rollout in the current CI-only phase`); + if (service !== undefined) return { desired, config: service }; + const directService = configById.get(desired.id); + if (directService !== undefined && isDirectComposeDeployMode(directService) && devArtifactConsumerServiceIds.has(directService.id)) { + return { desired, config: directService }; } - return { desired, config: service }; + throw new Error(`deploy --env dev service ${desired.id} is not enabled for direct rollout in the current CI-only phase`); } const service = configById.get(desired.id) ?? coreDeployService(config, desired.id, manifest.environment); if (service === undefined) throw new Error(`deploy manifest service ${desired.id} is not present in config.json microservices or supported core deploy services`); @@ -2885,7 +2889,7 @@ export async function runDeployCommand(config: UniDeskConfig | null, args: strin } const unsupported = unsupportedDevApplyServices(manifest, options.serviceId); if (unsupported.length > 0) { - throw new Error(`deploy apply --env dev currently supports backend-core target-side rollout plus frontend/baidu-netdisk/decision-center/project-manager/oa-event-flow/code-queue-mgr/todo-note artifact consumers; unsupported selected services: ${unsupported.join(", ")}. Use ci run-dev-e2e for smoke verification.`); + throw new Error(`deploy apply --env dev currently supports backend-core target-side rollout plus frontend/baidu-netdisk/decision-center/project-manager/oa-event-flow/code-queue-mgr/todo-note/findjob/pipeline/met-nonlinear artifact consumers; unsupported selected services: ${unsupported.join(", ")}. Use ci run-dev-e2e for smoke verification.`); } const devArtifactServices = selectedDevArtifactServicesWithProdFallback(manifest, options.serviceId); const devTargetServices = selectedDevTargetServices(manifest, options.serviceId);