From 803a695d0a077383e33d7b011301e44110e3fc00 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 18 May 2026 15:54:01 +0000 Subject: [PATCH] feat: split backend-core artifact ci cd --- AGENTS.md | 4 +- deploy.json | 5 + docker-compose.yml | 10 +- docs/reference/artifact-registry.md | 31 +- docs/reference/ci.md | 24 ++ docs/reference/cli.md | 4 +- docs/reference/deploy.md | 20 +- docs/reference/deployment.md | 4 +- scripts/cli.ts | 2 +- scripts/src/artifact-registry.ts | 400 ++++++++++++++++-- scripts/src/check.ts | 2 + scripts/src/ci.ts | 82 +++- scripts/src/docker.ts | 5 + scripts/src/help.ts | 12 +- .../k3s/ci/unidesk-ci.pipeline.yaml | 164 +++++++ 15 files changed, 713 insertions(+), 56 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c2c43231..84c8f6b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -39,8 +39,8 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台;本文 - `bun scripts/cli.ts decision diary import/list/months/show`:把带日期标题的工作日志 Markdown 拆成 `YYYY-MM/YYYY-MM-DD.md` 日记条目并写入 PostgreSQL,规则见 `docs/reference/microservices.md`。 - `bun scripts/cli.ts deploy check/plan/apply [--file deploy.json|--env dev|prod] [--service ]`:按根目录 `deploy.json` 或 `origin/master:deploy.json#environments.` 的服务 repo 和 commit 期望状态校验或更新用户服务;`--env dev` 当前只开放 D601 `backend-core`/`frontend` persistent dev rollout,规则见 `docs/reference/deploy.md` 与 `docs/reference/dev-environment.md`。 - `bun scripts/cli.ts dev-env validate [--manifest path] [--kubectl-dry-run]` / `dev-env prewarm-images`:离线校验 D601 `unidesk-dev` 生产隔离护栏,或把开发底座基础镜像预热到 D601 原生 k3s containerd,规则见 `docs/reference/deploy.md` 与 `docs/reference/microservices.md`。 -- `bun scripts/cli.ts artifact-registry plan|render|status|health|install --dry-run`:声明和只读检查 D601 host-managed CNCF Distribution registry;第一阶段不写 runtime、不改变 backend-core 生产部署路径,规则见 `docs/reference/artifact-registry.md`。 -- `bun scripts/cli.ts ci install/status/run/run-dev-e2e/logs`:在 D601 原生 k3s 上安装和运行 Tekton CI,支持每 commit 检查、Code Queue 只读性能门禁和手动触发的 `origin/master:deploy.json#environments.dev` 临时 namespace e2e;`run-dev-e2e` 的 Git 控制 runner、短 launcher 和 no-CD 边界见 `docs/reference/dev-ci-runner.md`,Tekton 规则见 `docs/reference/ci.md`。 +- `bun scripts/cli.ts artifact-registry plan|render|status|health|install|deploy-backend-core`:管理 D601 host-managed CNCF Distribution registry,并通过短生命周期 relay 做 production backend-core pull-only artifact CD,规则见 `docs/reference/artifact-registry.md`。 +- `bun scripts/cli.ts ci install/status/run/publish-backend-core/run-dev-e2e/logs`:在 D601 原生 k3s 上安装和运行 Tekton CI,支持每 commit 检查、Code Queue 只读性能门禁、backend-core commit-pinned 镜像发布和手动触发的 `origin/master:deploy.json#environments.dev` 临时 namespace e2e;`run-dev-e2e` 的 Git 控制 runner、短 launcher 和 no-CD 边界见 `docs/reference/dev-ci-runner.md`,Tekton 规则见 `docs/reference/ci.md`。 - `bun scripts/cli.ts codex deploy `:旧 Code Queue 兼容部署入口已禁用,原因是它会绕过受控部署边界直连 D601 部署 Code Queue;规则见 `docs/reference/codex-deploy.md`。 - `bun scripts/cli.ts codex submit [prompt] [--prompt-file path|--prompt-stdin] [--queue ]`:通过 backend-core 私有代理提交 Code Queue 任务;控制面默认走主 server `code-queue-mgr` 写入 PostgreSQL,`--dry-run` 可只检查请求体不入队,规则见 `docs/reference/cli.md`。 - `bun scripts/cli.ts codex task `:按 Code Queue 任务 ID 查询初始 prompt、最后 assistant message、工具调用摘要、attempt/judge/error 和耗时,便于新任务引用历史 session。 diff --git a/deploy.json b/deploy.json index 130194c5..b5fe7457 100644 --- a/deploy.json +++ b/deploy.json @@ -3,6 +3,11 @@ "environments": { "prod": { "services": [ + { + "id": "backend-core", + "repo": "https://github.com/pikasTech/unidesk", + "commitId": "465f4a626b4aebb79b59e40c09f064c568128536" + }, { "id": "findjob", "repo": "https://gitee.com/Lyon1998/findjob", diff --git a/docker-compose.yml b/docker-compose.yml index 5e91b63e..c77e45f1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,7 @@ services: retries: 20 backend-core: + image: unidesk-backend-core build: context: . dockerfile: src/components/backend-core/Dockerfile @@ -66,13 +67,20 @@ services: BAIDU_NETDISK_INTERNAL_URL: "http://baidu-netdisk:4244" CODE_QUEUE_MGR_INTERNAL_URL: "http://code-queue-mgr:4278" MICROSERVICES_JSON: "${UNIDESK_MICROSERVICES_JSON:-[]}" + UNIDESK_DEPLOY_REF: "${UNIDESK_DEPLOY_REF:-deploy.json#environments.prod.services.backend-core}" + UNIDESK_DEPLOY_SERVICE_ID: "${UNIDESK_DEPLOY_SERVICE_ID:-backend-core}" + UNIDESK_DEPLOY_REPO: "${UNIDESK_DEPLOY_REPO:-}" + UNIDESK_DEPLOY_COMMIT: "${UNIDESK_DEPLOY_COMMIT:-}" + UNIDESK_DEPLOY_REQUESTED_COMMIT: "${UNIDESK_DEPLOY_REQUESTED_COMMIT:-}" LOG_FILE: "/var/log/unidesk/${UNIDESK_LOG_DAY}/${UNIDESK_LOG_PREFIX}_backend-core.jsonl" UNIDESK_LOG_RETENTION_BYTES: "${UNIDESK_LOG_RETENTION_BYTES:-1GiB}" volumes: - ${UNIDESK_LOG_DIR}:/var/log/unidesk - ./.state/baidu-netdisk/staging:/data/baidu-netdisk-staging healthcheck: - test: ["CMD", "backend-core", "--fetch-json", "http://127.0.0.1:8080/health", "--require-ok"] + test: + - "CMD-SHELL" + - "if command -v backend-core >/dev/null 2>&1; then backend-core --fetch-json http://127.0.0.1:8080/health --require-ok; elif command -v bun >/dev/null 2>&1; then bun -e \"fetch('http://127.0.0.1:8080/health').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))\"; else exit 1; fi" interval: 5s timeout: 3s retries: 20 diff --git a/docs/reference/artifact-registry.md b/docs/reference/artifact-registry.md index 33675e18..2c91385e 100644 --- a/docs/reference/artifact-registry.md +++ b/docs/reference/artifact-registry.md @@ -2,7 +2,7 @@ D601 artifact registry 是为 backend-core 轻量 CD 准备的本地镜像制品入口。它使用开源、成熟的 CNCF Distribution Docker registry,不自定义镜像协议,也不把镜像托管到第三方服务。 -第一阶段只提供声明、渲染、dry-run install 和只读状态检查;不会安装、启动或替换任何生产服务。backend-core 现有生产部署路径保持不变。 +backend-core 的长期分工是:CI 在 D601 构建并发布 commit-pinned 镜像,CD 在 master server 只拉取、替换和验证该镜像。registry 是这条链路的本地制品缓存,不是构建编排器,也不是生产部署控制面。 ## Architecture @@ -22,7 +22,7 @@ registry 运行在 D601 host/WSL OS 上,由 systemd 管理 Docker Compose 项 这个服务和 `k3sctl-adapter` 一样位于 k3s 故障域外,但职责不同: - `k3sctl-adapter` 是 UniDesk 到 native k3s 的控制桥,属于 UniDesk 直管服务。 -- artifact registry 是 D601 host-managed 制品缓存基础设施,后续服务 CD 会使用它拉取或推送 commit-pinned 镜像。 +- artifact registry 是 D601 host-managed 制品缓存基础设施,D601 CI 向它推送 commit-pinned 镜像,master server CD 从它消费镜像。 ## Dependency Boundary @@ -41,6 +41,7 @@ registry 运行期依赖应保持低且可手动维护: - 第三方镜像托管服务作为 backend-core artifact source of truth。 - main server backend-core 本地 Rust 编译。 - 公开 registry 端口或公网反向代理。 +- k3s Service、NodePort、Ingress 或任何长期暴露的 registry 代理。 ## CLI @@ -49,14 +50,17 @@ registry 运行期依赖应保持低且可手动维护: ```bash bun scripts/cli.ts artifact-registry plan bun scripts/cli.ts artifact-registry render -bun scripts/cli.ts artifact-registry install --dry-run +bun scripts/cli.ts artifact-registry install bun scripts/cli.ts artifact-registry status bun scripts/cli.ts artifact-registry health +bun scripts/cli.ts artifact-registry deploy-backend-core --commit ``` -`plan` 输出架构边界、依赖项、默认路径和未来 backend-core artifact CD 流程。`render` 输出 systemd unit、Compose 文件和 registry config 的完整内容与 SHA-256。`install --dry-run` 只列出将来要执行的远端动作,不写 D601 文件、不启动容器、不 reload systemd。 +`plan` 输出架构边界、依赖项、默认路径和 backend-core artifact CD 流程。`render` 输出 systemd unit、Compose 文件和 registry config 的完整内容与 SHA-256。`install --dry-run` 只列出将要执行的远端动作,不写 D601 文件、不启动容器、不 reload systemd。 -第一阶段 `install` 不带 `--dry-run` 必须失败。启用真实安装前,需要单独评审并补齐幂等写入、配置 hash 比对、回滚/停止策略和验收门禁。 +真实 `install` 必须是幂等动作:创建远端目录,写入 CLI 渲染出的 config/compose/unit,执行 `systemctl daemon-reload`,启用并启动 `unidesk-artifact-registry.service`,然后运行与 `health` 相同的验收检查。若远端文件 hash 与期望不一致,install 可以覆盖由本 CLI 管理的 unit/config/compose,但不得删除 registry storage。 + +`deploy-backend-core` 是 production backend-core 的 CD 入口。它必须先确认 D601 registry 中已经存在 `unidesk/backend-core:`,随后只执行短生命周期 relay、`docker pull`、retag、Compose `--no-build` recreate 和 live commit 验证;如果镜像不存在,应失败并要求先运行 CI artifact publication。 `status` 和 `health` 通过: @@ -88,20 +92,23 @@ docker compose -p unidesk-artifact-registry -f /home/ubuntu/.unidesk/artifact-re 手动维护必须遵守 Git-backed deployment truth:运行态可以临时修复,但长期配置应回到 `artifact-registry render` 的声明文件和本文档。若 D601 runtime 文件 hash 与 CLI 渲染结果不一致,`status` / `health` 会显示 mismatch,操作者需要确认这是受控升级还是漂移。 -## Future Backend-Core Artifact CD +## Backend-Core Artifact CD 目标流程是: -1. D601 CI/dev execution 在 D601 构建 `unidesk/backend-core:`。 -2. 构建产物 push 到 D601 loopback registry。 -3. main server 通过受控、短生命周期的 localhost relay 从 D601 registry 拉取 commit-pinned 镜像。 -4. main server retag 后替换 backend-core Compose 服务。 -5. 部署后通过 image label、runtime env、health payload 验证 live commit。 +1. D601 artifact registry 已安装并通过 `health`。 +2. D601 CI 从 pushed Git checkout 构建 `unidesk/backend-core:`。 +3. CI 将镜像 push 到 `127.0.0.1:5000/unidesk/backend-core:`,并记录 image ref 与 digest。 +4. CD 在 master server 上确认目标 commit/tag/digest 存在。 +5. master server 通过受控、短生命周期的 localhost relay 从 D601 registry 拉取 commit-pinned 镜像。 +6. master server retag 为 Compose 使用的 backend-core 镜像名,并执行 `docker compose up -d --no-build --no-deps --force-recreate backend-core`。 +7. 部署后通过 image label、runtime env、health payload 验证 live commit。 -这个未来 CD 路径仍必须满足: +这个 CD 路径必须满足: - source commit 来自 pushed Git,不来自 dirty worktree。 - 镜像 tag 必须 commit-pinned,不能用 mutable latest 作为部署真相。 - relay 是临时控制动作,不开放长期公网 registry。 - CI 可以有较多依赖;CD 只做拉取、retag、recreate 和 live commit 验证。 +- CD 不执行 `cargo build`、`docker build`、`docker compose build backend-core` 或任何等价的 Rust 构建动作。 - `server rebuild backend-core` 仍不得作为 master server Rust 编译路径。 diff --git a/docs/reference/ci.md b/docs/reference/ci.md index f0cb4e37..e9fd3164 100644 --- a/docs/reference/ci.md +++ b/docs/reference/ci.md @@ -19,6 +19,7 @@ Each commit CI run performs: - `git clone` and checkout of the requested repository revision. - `bun install --frozen-lockfile` at the repo root and `src/`, because `bun scripts/cli.ts check` compiles all `src/components` and needs the component workspace lockfile for frontend React dependencies. - `UNIDESK_D601_RUST_CHECK=1 bun scripts/cli.ts check --full --rust`, so Rust backend-core is checked only inside the D601 CI execution boundary. +- Backend-core production artifact publication when enabled: build the requested pushed commit on D601, stamp Docker image labels, and push a commit-pinned image to the D601 loopback artifact registry. - Temporary `code-queue-ci-read` Deployment and ClusterIP Service in `unidesk-ci`. - Code Queue read performance checks against the production PostgreSQL through `d601-tcp-egress-gateway`. - Manual dev desired-state smoke for Code Queue via `ci run-dev-e2e`, using the Git-pinned `code-queue` service commit from `origin/master:deploy.json#environments.dev`. @@ -45,6 +46,21 @@ The temporary Code Queue service uses: This means the CI service can read existing tasks, Trace summaries, Trace steps and Trace step details from the main database, but it must not schedule, mutate, notify, backfill or become deployment truth. +## Backend-Core Artifact Publication + +backend-core production image creation belongs to D601 CI, not to master server CD. The purpose is to keep Rust compilation, Docker build cache, dependency downloads and image push on the higher-resource D601 side while leaving production deployment with a small pull/recreate/verify surface. + +The CI artifact task must follow these rules: + +- Input revision comes from pushed Git and is resolved to a full 40-character commit. A dirty worktree or unpushed local tree must never be used as the image source. +- The source checkout, Rust build and Docker build run on D601 CI infrastructure. The master server must not run `cargo build`, `docker compose build backend-core` or `server rebuild backend-core` as part of production backend-core deployment. +- The image is tagged with the source commit, for example `unidesk/backend-core:`, and pushed to the D601 artifact registry as `127.0.0.1:5000/unidesk/backend-core:`. +- The image must carry at least `unidesk.ai/service-id=backend-core`, `unidesk.ai/source-repo`, `unidesk.ai/source-commit` and `unidesk.ai/dockerfile=src/components/backend-core/Dockerfile`. +- Publication must fail if the D601 artifact registry is not healthy. It must not fall back to a third-party registry or a mutable `latest` tag. +- CI may output the image ref and digest as deployment input, but it must not restart production Compose services, call production `deploy apply`, mutate production namespaces, or change `deploy.json`. + +The artifact registry contract and CD consumption path are defined in `docs/reference/artifact-registry.md`. CI is the producer of the backend-core image artifact; CD is only the consumer. + ## Dev Namespace E2E `ci run-dev-e2e` is the manual dev desired-state smoke flow. The single authoritative reference for its Git-controlled runner script, short launcher, result directory and no-CD boundary is `docs/reference/dev-ci-runner.md`. @@ -83,6 +99,14 @@ Run CI manually for a commit: bun scripts/cli.ts ci run --revision ``` +Publish a backend-core artifact for production CD: + +```bash +bun scripts/cli.ts ci publish-backend-core --commit --wait-ms 1200000 +``` + +This command creates the `unidesk-backend-core-artifact-publish` Tekton PipelineRun. It is a CI producer action only: it may build and push `127.0.0.1:5000/unidesk/backend-core:`, but it must not recreate the master server container. Production deployment is triggered separately with `artifact-registry deploy-backend-core`. + Run the dev namespace e2e harness manually: ```bash diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 772170da..fed92b29 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -26,8 +26,8 @@ UniDesk 的统一 CLI 入口是根目录 `scripts/cli.ts`,运行方式固定 - `deploy check/plan/apply` 默认从根目录 `deploy.json` 读取服务 repo 与 commit 期望状态,join `config.json` 和现有 manifest 后使用 target-side build 单一路径校验或更新已支持目标;`deploy plan --env dev|prod` 只从 `origin/master:deploy.json#environments.` 读取 manifest 并输出 dry-run 环境计划,不使用本地 dirty worktree;当前 `deploy apply --env dev` 只支持 D601 `backend-core` 与 `frontend` persistent dev rollout,dev desired-state smoke 使用 `ci run-dev-e2e`;规则见 `docs/reference/deploy.md`、`docs/reference/dev-environment.md` 和 `docs/reference/dev-ci-runner.md`。 - `dev-env validate [--manifest path] [--kubectl-dry-run]` 离线校验 D601 `unidesk-dev` namespace、dev PostgreSQL 底座和 dev workload manifest。默认检查 `src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-foundation.k8s.yaml`;也可显式校验 `src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-core.k8s.yaml` 或 `src/components/microservices/k3sctl-adapter/k3s/dev/unidesk-dev-code-queue.k8s.yaml`。所有 namespaced 对象必须只落到 `unidesk-dev`,foundation manifest 必须包含 `postgres-dev` StatefulSet/Service、dev secret/config、迁移 Job 和 DB URL guard,core manifest 必须包含 `backend-core-dev`/`frontend-dev` Deployment/Service,Code Queue dev manifest 必须包含 `code-queue-scheduler-dev`、`code-queue-read-dev`、`code-queue-write-dev` 和 dev provider egress proxy。加 `--kubectl-dry-run` 时额外执行 `kubectl apply --dry-run=client --validate=false -f `,仍不 apply 资源。 - `dev-env prewarm-images [--image image] [--provider-id D601] [--no-pull] [--proxy-url URL] [--pull-timeout-ms N] [--dry-run]` 创建异步 job,通过 UniDesk SSH 维护桥在 D601 上把开发底座依赖镜像从 Docker 缓存导入原生 k3s containerd。默认镜像是 `postgres:16-alpine` 和 `rancher/mirrored-library-busybox:1.36.1`,用于避免 `postgres-dev` 与 local-path helper pod 卡在外部 registry 拉取。该命令固定验证 `/etc/rancher/k3s/k3s.yaml` 指向的 native k3s 上下文,并输出 `dev_env_containerd_image_ready=...` 作为成功判据;它不 apply manifest、不修改生产 `unidesk` namespace。 -- `artifact-registry plan|render|status|health|install --dry-run` 管理 D601 host-managed CNCF Distribution registry 的声明和只读检查。该 registry 固定为 D601 loopback `127.0.0.1:5000`,由 systemd + Docker Compose 管理,位于 native k3s 故障域外;第一阶段只允许渲染和 dry-run install,不写 D601 runtime,不改变 backend-core 生产部署路径。长期规则见 `docs/reference/artifact-registry.md`。 -- `ci install|status|run|run-dev-e2e|logs` 管理 D601 原生 k3s 上的 Tekton CI。`run` 手动创建每 commit 检查和 Code Queue 只读性能门禁;`run-dev-e2e` 的 Git 控制 runner、短 launcher、host fetch 边界、临时 smoke namespace 和 no-CD 规则只在 `docs/reference/dev-ci-runner.md` 定义;Tekton CI 通用规则见 `docs/reference/ci.md`。 +- `artifact-registry plan|render|status|health|install|deploy-backend-core` 管理 D601 host-managed CNCF Distribution registry 的声明、安装、只读检查和 production backend-core pull-only artifact CD。该 registry 固定为 D601 loopback `127.0.0.1:5000`,由 systemd + Docker Compose 管理,位于 native k3s 故障域外;`deploy-backend-core` 只通过短生命周期 localhost relay 拉取 CI 已发布的 commit-pinned 镜像、retag、`--no-build` recreate 和 live commit 验证,不构建 backend-core。长期规则见 `docs/reference/artifact-registry.md`。 +- `ci install|status|run|publish-backend-core|run-dev-e2e|logs` 管理 D601 原生 k3s 上的 Tekton CI。`run` 手动创建每 commit 检查和 Code Queue 只读性能门禁;`publish-backend-core` 从 pushed Git commit 构建并发布 `127.0.0.1:5000/unidesk/backend-core:`,但不部署生产;`run-dev-e2e` 的 Git 控制 runner、短 launcher、host fetch 边界、临时 smoke namespace 和 no-CD 规则只在 `docs/reference/dev-ci-runner.md` 定义;Tekton CI 通用规则见 `docs/reference/ci.md`。 - `codex deploy ` 是旧 Code Queue 兼容部署入口,已禁用以防止维护通道直连 D601 部署 Code Queue;当前 dev 自动化只做 `ci run-dev-e2e` smoke,不提供 Code Queue CD,详细规则见 `docs/reference/codex-deploy.md`。 - `codex submit [prompt] [--prompt-file path|--prompt-stdin] [--queue queueId] [--provider-id id] [--cwd path] [--model model] [--reasoning-effort effort] [--execution-mode mode] [--max-attempts N] [--reference-task-id id] [--dry-run]` 通过 backend-core 私有代理向稳定 `code-queue` 用户服务路径提交任务;prompt 必须且只能来自位置参数、文件或 stdin 之一,`--dry-run` 只返回结构化请求且不实际入队。提交确认和 dry-run 必须返回完整 prompt、字符数和 `truncated=false`,不能套用任务详情的预览截断策略,否则长任务 prompt 无法被人工验收。backend-core 默认把提交、队列 CRUD、已读状态、历史摘要和轻量 Trace 读取分流到主 server `code-queue-mgr`,由它写入主 PostgreSQL;D601 scheduler 只轮询并执行已入库任务。 - `codex task ` 通过 Code Queue 私有代理按任务 ID 查询结构化执行摘要;默认只返回有界 prompt/response 预览、执行 Provider、工作目录、最后 assistant message、最近工具调用摘要、attempt、judge、错误、耗时和 trace 翻页提示,适合在新队列任务中引用历史 session 且避免噪声爆炸。该摘要读取默认由主 server `code-queue-mgr` 从 PostgreSQL 返回,不依赖 D601 `code-queue-read` Service 可用。 diff --git a/docs/reference/deploy.md b/docs/reference/deploy.md index de6ac40c..cc4c6603 100644 --- a/docs/reference/deploy.md +++ b/docs/reference/deploy.md @@ -40,7 +40,7 @@ The root `deploy.json` is the single desired-state source for both prod and dev. The optional non-service execution declaration under `environments.dev` is intentionally not specified here. The only currently allowed declaration is `ci`, and its authoritative `repo`, `scriptPath`, `timeoutMs`, short launcher, host fetch boundary and no-CD rules are defined only in `docs/reference/dev-ci-runner.md`. -Environment mode never reads the local dirty working tree manifest. `deploy check --env ...`, `deploy plan --env ...` and `deploy apply --env ...` fetch `origin/master`, read `origin/master:deploy.json`, select `environments.`, and report the manifest commit/blob, service commit IDs, target namespace, database fingerprint and Provider identity. `deploy apply --env dev` is currently enabled only for persistent D601 dev `backend-core` and `frontend`; all other D601 services remain rejected before runtime mutation. `deploy apply --env prod` remains disabled until the production environment executor and authorization policy are explicitly added. +Environment mode never reads the local dirty working tree manifest. `deploy check --env ...`, `deploy plan --env ...` and `deploy apply --env ...` fetch `origin/master`, read `origin/master:deploy.json`, select `environments.`, and report the manifest commit/blob, service commit IDs, target namespace, database fingerprint and Provider identity. `deploy apply --env dev` is currently enabled only for persistent D601 dev `backend-core` and `frontend`; all other D601 services remain rejected before runtime mutation. `deploy apply --env prod` remains disabled until the production environment executor and authorization policy are explicitly added. Production backend-core artifact CD is a separate executor because its build target is D601 CI while its runtime target is the master server. The only D601 direct-service exception in local manifest mode is `k3sctl-adapter`, because it is the UniDesk-managed control bridge outside the k3s fault domain and owns the Kubernetes service catalog used by the dev public frontend path. Updating it must still use the normal target-side deploy reconciler from a pushed commit. D601 Code Queue, Decision Center, MDTODO, ClaudeQQ and future k3s-managed workloads remain blocked from maintenance-channel direct deploy. @@ -108,7 +108,7 @@ All deploy commands output JSON. Long operations must use `.state/jobs/` and bou ## Target-Side Build -Target-side build is the only standard deployment mode. The controller may run on the main server, but source materialization, compile/build, Docker image creation and deployment happen on the target node that will run the service. +Target-side build is the standard deployment mode. The controller may run on the main server, but source materialization, compile/build, Docker image creation and deployment normally happen on the target node that will run the service. - Main server services are fetched, built and deployed on the main server. - D601 services are fetched, built and deployed on D601. @@ -119,6 +119,20 @@ The reconciler distributes only repository URL, commit ID, Dockerfile path, buil Each target fetches the remote repository, resolves the requested commit to a full 40 character SHA and exports tracked files with `git archive`. Build contexts are created from that archive, not from the operator's current working tree. Environment applies such as `deploy apply --env dev` must not upload Kubernetes manifests or source files from the master server worktree; the target-side materialized commit is the source for Dockerfile, build context and k3s control manifests. The master server side may only do lightweight CLI orchestration, environment ref reading and remote command dispatch. +## Backend-Core Artifact Exception + +Production backend-core is the explicit exception to standard target-side build. The runtime target is the master server, but the build target is D601 CI because backend-core Rust compilation is too expensive for the low-resource master server. + +The exception is narrow: + +- CI on D601 builds `src/components/backend-core/Dockerfile` from a pushed commit, stamps image labels and publishes `127.0.0.1:5000/unidesk/backend-core:` to the D601 artifact registry. +- CD on the master server pulls that existing image through the controlled artifact-registry relay, retags it for the Compose service, recreates only `backend-core` with `--no-build --no-deps --force-recreate`, and verifies the running commit. +- CD must not run Rust compilation, Docker build, Compose build or `server rebuild backend-core`. +- The pushed Git commit remains the version source of truth. The image registry is a content cache and transfer boundary, not a replacement for `deploy.json` or Git. +- This exception must not be generalized to other services unless their resource profile and runtime boundary are documented with the same CI-producer/CD-consumer split. + +The registry contract is defined in `docs/reference/artifact-registry.md`; the CI producer rules are defined in `docs/reference/ci.md`. + ## One-Shot Build Proxy Target-side source fetches and Docker builds that need external network access use a one-shot proxy scope through provider-gateway WS egress. Provider targets connect only to their node-local provider-gateway egress endpoint, normally `http://127.0.0.1:18789`; provider-gateway carries the TCP stream over the already-authenticated provider WebSocket to the main server, and the main server opens the final outbound TCP connection. This is the only allowed proxy channel for provider-side deploy source fetches and builds. The deploy path must not mutate host-global proxy settings: @@ -151,7 +165,7 @@ Decision Center is a standard `k3sctl-managed` service in this model, but D601 m ## CI Separation -Continuous integration is intentionally separate from this deploy reconciler. D601 k3s hosts Tekton CI resources described in `docs/reference/ci.md`, but those PipelineRuns only clone, check, run read-only performance gates, or create temporary CI-owned namespaces for dev manifest smoke e2e. They must not call `deploy apply`, `codex deploy`, `kubectl rollout restart` for production services, mutate `deploy.json`, or write production namespaces. +Continuous integration is intentionally separate from this deploy reconciler. D601 k3s hosts Tekton CI resources described in `docs/reference/ci.md`; PipelineRuns may clone, check, run read-only performance gates, create temporary CI-owned namespaces for dev manifest smoke e2e, or publish commit-pinned backend-core image artifacts to the D601 artifact registry. They must not call `deploy apply`, `codex deploy`, `kubectl rollout restart` for production services, mutate `deploy.json`, or write production namespaces. The Code Queue performance gate may create a temporary `code-queue-ci-read` service and read the main PostgreSQL through the existing `d601-tcp-egress-gateway`. Because it runs with `CODE_QUEUE_SERVICE_ROLE=read`, scheduler/backfill/notification disabled and EmptyDir state, it is not deployment truth and does not need a temporary database for the current read-only checks. diff --git a/docs/reference/deployment.md b/docs/reference/deployment.md index d7c64c2a..60765b0b 100644 --- a/docs/reference/deployment.md +++ b/docs/reference/deployment.md @@ -28,7 +28,7 @@ CLI 会优先使用 `docker compose` v2 plugin;当 v2 plugin 不存在时才 Compose v2 安装后仍然必须遵守 UniDesk 的服务控制入口:全栈生命周期用 `server start` / `server stop`,单服务重建用 `server rebuild `。不要因为 v2 可用就直接在生产栈上手工执行未纳入 CLI 的 `up --build`、`down -v` 或跨项目清理命令;所有会影响容器的动作都应保持 job 可观测、Compose project 固定、database named volume 保留。主 server Compose 命令必须从 `providerGateway.upgrade.hostProjectRoot` 指定的 canonical UniDesk 根目录运行,临时 worktree、Code Queue 导出目录或实验分支不得复用生产 `-p unidesk` 和固定 `container_name` 去替换生产容器。 -版本化用户服务部署优先使用 `bun scripts/cli.ts deploy apply` 已支持的受控路径;D601 persistent dev apply 当前只支持 `backend-core` 和 `frontend`,dev desired-state smoke 使用 `ci run-dev-e2e`。`deploy.json` 只声明服务 `id`、`repo` 和 `commitId`;目标节点、Dockerfile、Compose、Kubernetes manifest、健康检查和代理路径继续来自 `config.json` 与现有 manifest。主 server 直管微服务和内部 sidecar,例如 `code-queue-mgr`,也必须支持这一路径:`deploy apply --service code-queue-mgr` 从 `deploy.json` 指定 commit 导出源码、构建镜像、替换固定 Compose service 并验证运行中镜像/健康信息的 commit。部署必须遵循 target-side build:服务部署到哪台 target,就在哪台 target 从 remote commit 导出源码、一次性代理构建镜像并部署;不得把中心构建镜像作为默认分发路径,也不得用 `docker commit` 或脏 worktree 作为部署输入。完整规则见 `docs/reference/deploy.md`,D601 dev/Rust 边界见 `docs/reference/dev-environment.md`。 +版本化用户服务部署优先使用 `bun scripts/cli.ts deploy apply` 已支持的受控路径;D601 persistent dev apply 当前只支持 `backend-core` 和 `frontend`,dev desired-state smoke 使用 `ci run-dev-e2e`。`deploy.json` 只声明服务 `id`、`repo` 和 `commitId`;目标节点、Dockerfile、Compose、Kubernetes manifest、健康检查和代理路径继续来自 `config.json` 与现有 manifest。主 server 直管微服务和内部 sidecar,例如 `code-queue-mgr`,也必须支持这一路径:`deploy apply --service code-queue-mgr` 从 `deploy.json` 指定 commit 导出源码、构建镜像、替换固定 Compose service 并验证运行中镜像/健康信息的 commit。部署默认遵循 target-side build:服务部署到哪台 target,就在哪台 target 从 remote commit 导出源码、一次性代理构建镜像并部署;不得把中心构建镜像作为默认分发路径,也不得用 `docker commit` 或脏 worktree 作为部署输入。production backend-core 是明确例外:D601 CI 构建并推送 commit-pinned 镜像到 D601 artifact registry,master server CD 只拉取、retag、recreate 和验证,不在 master server 编译 Rust 或执行 Compose build。完整规则见 `docs/reference/deploy.md`,D601 dev/Rust 边界见 `docs/reference/dev-environment.md`,artifact registry 见 `docs/reference/artifact-registry.md`。 ## Main Server Swap @@ -44,7 +44,7 @@ swap 管理不能被强塞进所有热路径。`server start/status` 可以暴 ## Single Service Rebuild -前端、backend-core、本机 provider-gateway、dev-frontend-proxy 或主 server 承载的 Todo Note/Code Queue Manager/Project Manager/Baidu Netdisk/OA Event Flow 用户服务需要非版本化本地重建时,统一使用 `bun scripts/cli.ts server rebuild `,其中 `` 只能是 `backend-core`、`frontend`、`dev-frontend-proxy`、`provider-gateway`、`todo-note`、`code-queue-mgr`、`project-manager`、`baidu-netdisk` 或 `oa-event-flow`。需要按 commit 上线或恢复到 desired-state 时必须改用 `bun scripts/cli.ts deploy apply --service `;直管微服务也不能把脏工作树或手工重建作为部署真相。Rust backend-core 迭代不得在 master server 用 `server rebuild backend-core` 编译,必须走 D601 dev deploy/CI。D601 Code Queue 执行面、File Browser、FindJob、Pipeline、MET Nonlinear 和 ClaudeQQ 部署在计算节点,不属于主 server Compose 可重建服务;其中 D601 Code Queue 执行面不得再通过 `codex deploy` 或维护通道直连 D601 部署;未来正式 CD 必须经受控 target-side 路径执行 build-first、rollout 和 live commit 验证。 +前端、本机 provider-gateway、dev-frontend-proxy 或主 server 承载的 Todo Note/Code Queue Manager/Project Manager/Baidu Netdisk/OA Event Flow 用户服务需要非版本化本地重建时,统一使用 `bun scripts/cli.ts server rebuild `,其中 `` 只能是 `backend-core`、`frontend`、`dev-frontend-proxy`、`provider-gateway`、`todo-note`、`code-queue-mgr`、`project-manager`、`baidu-netdisk` 或 `oa-event-flow`。需要按 commit 上线或恢复到 desired-state 时必须改用 `bun scripts/cli.ts deploy apply --service ` 或 backend-core artifact CD;直管微服务也不能把脏工作树或手工重建作为部署真相。Rust backend-core 迭代不得在 master server 用 `server rebuild backend-core` 编译,生产 backend-core 也不得用该命令完成 Rust 构建,必须走 D601 dev deploy/CI 或 D601 artifact registry CD。D601 Code Queue 执行面、File Browser、FindJob、Pipeline、MET Nonlinear 和 ClaudeQQ 部署在计算节点,不属于主 server Compose 可重建服务;其中 D601 Code Queue 执行面不得再通过 `codex deploy` 或维护通道直连 D601 部署;未来正式 CD 必须经受控 target-side 路径执行 build-first、rollout 和 live commit 验证。 frontend 改动必须明确上线到公网:修改 `src/components/frontend/src/`、`src/components/frontend/public/style.css`、frontend 使用的共享 TSX/TS 模块或 WebUI 导航后,必须在同一变更集中执行 `bun scripts/cli.ts server rebuild frontend`,并等待 job 成功。公网 WebUI 的 `/app.js` 是 `unidesk-frontend` 容器启动时从镜像内源码转译生成的运行时 bundle;只改工作区文件、只跑 `bun run check`、只跑 `Bun.build` 或只刷新浏览器都不会替换已经运行的容器。 diff --git a/scripts/cli.ts b/scripts/cli.ts index 8f0ed3f0..df005b27 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -136,7 +136,7 @@ async function main(): Promise { } if (top === "artifact-registry") { - const result = runArtifactRegistryCommand(args.slice(1)); + const result = await runArtifactRegistryCommand(args.slice(1)); const ok = (result as { ok?: unknown }).ok !== false; emitJson(commandName, result, ok); if (!ok) process.exitCode = 1; diff --git a/scripts/src/artifact-registry.ts b/scripts/src/artifact-registry.ts index b1102c04..9b3475a9 100644 --- a/scripts/src/artifact-registry.ts +++ b/scripts/src/artifact-registry.ts @@ -1,8 +1,13 @@ import { createHash } from "node:crypto"; +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { createServer, type AddressInfo, type Server, type Socket } from "node:net"; +import { readFileSync, writeFileSync } from "node:fs"; import { runCommand, type CommandResult } from "./command"; -import { repoRoot } from "./config"; +import { readConfig, repoRoot, rootPath } from "./config"; +import { resolveComposeCommand, writeComposeEnv } from "./docker"; +import { startJob } from "./jobs"; -type ArtifactRegistryAction = "plan" | "render" | "status" | "health" | "install"; +type ArtifactRegistryAction = "plan" | "render" | "status" | "health" | "install" | "deploy-backend-core"; interface ArtifactRegistryOptions { providerId: string; @@ -17,6 +22,10 @@ interface ArtifactRegistryOptions { containerName: string; timeoutMs: number; dryRun: boolean; + runNow: boolean; + commit: string | null; + localPullPort: number | null; + targetImage: string; } interface RenderedFile { @@ -50,6 +59,10 @@ const defaultOptions: ArtifactRegistryOptions = { containerName: "unidesk-artifact-registry", timeoutMs: 30_000, dryRun: false, + runNow: false, + commit: null, + localPullPort: null, + targetImage: "unidesk-backend-core", }; function isHelpArg(value: string | undefined): boolean { @@ -68,18 +81,32 @@ function positiveInt(value: string, option: string): number { return parsed; } +function optionalPort(value: string, option: string): number { + const parsed = positiveInt(value, option); + if (parsed > 65535) throw new Error(`${option} must be <= 65535`); + return parsed; +} + function absolutePath(value: string, option: string): string { if (!value.startsWith("/")) throw new Error(`${option} must be an absolute path`); if (value.includes("\n") || value.includes("\0")) throw new Error(`${option} must not contain control characters`); return value.replace(/\/+$/u, ""); } +function commitValue(value: string, option: string): string { + const normalized = value.toLowerCase(); + if (!/^[0-9a-f]{40}$/u.test(normalized)) throw new Error(`${option} must be a full 40-character commit SHA`); + return normalized; +} + function parseOptions(args: string[]): ArtifactRegistryOptions { const options = { ...defaultOptions }; for (let index = 0; index < args.length; index += 1) { const arg = args[index]; if (arg === "--dry-run") { options.dryRun = true; + } else if (arg === "--run-now") { + options.runNow = true; } else if (arg === "--provider-id") { options.providerId = requireValue(args, index, arg); index += 1; @@ -101,6 +128,15 @@ function parseOptions(args: string[]): ArtifactRegistryOptions { } else if (arg === "--timeout-ms") { options.timeoutMs = positiveInt(requireValue(args, index, arg), arg); index += 1; + } else if (arg === "--commit") { + options.commit = commitValue(requireValue(args, index, arg), arg); + index += 1; + } else if (arg === "--local-pull-port") { + options.localPullPort = optionalPort(requireValue(args, index, arg), arg); + index += 1; + } else if (arg === "--target-image") { + options.targetImage = requireValue(args, index, arg); + index += 1; } else { throw new Error(`unknown artifact-registry option: ${arg}`); } @@ -231,13 +267,14 @@ function plan(options: ArtifactRegistryOptions): Record { "listen only on D601 host loopback 127.0.0.1:5000", "do not expose a public port, NodePort, hostPort, or third-party registry", "do not run inside k3s; keep the registry outside the native k3s failure domain", - "first-stage CLI does not mutate D601 runtime; install is dry-run only", - "backend-core production deploy behavior is unchanged in this issue", + "CI builds and publishes backend-core artifacts on D601", + "CD only pulls, retags, recreates backend-core, and verifies live commit", + "master server CD must not compile Rust or run docker compose build backend-core", ], renderedPaths: bundle.paths, - futureFlow: [ - "D601 CI/dev builds unidesk/backend-core:", - "D601 pushes the image into the loopback registry", + backendCoreArtifactFlow: [ + "D601 CI builds unidesk/backend-core:", + "D601 CI pushes 127.0.0.1:5000/unidesk/backend-core:", "main server pulls via a controlled short-lived localhost relay", "prod CD retags, recreates backend-core, and verifies live commit metadata", ], @@ -352,6 +389,11 @@ function commandTail(result: CommandResult): Record { }; } +function runRemoteScript(options: ArtifactRegistryOptions, script: string, timeoutMs = options.timeoutMs): CommandResult { + const command = [process.execPath, "scripts/cli.ts", "ssh", options.providerId, "argv", "bash", "-lc", script]; + return runCommand(command, repoRoot, { timeoutMs }); +} + function statusFromValues(options: ArtifactRegistryOptions, values: Record, command: CommandResult, healthMode: boolean): Record { const commandOk = command.exitCode === 0 && !command.timedOut; const checks = { @@ -427,8 +469,7 @@ function statusFromValues(options: ArtifactRegistryOptions, values: Record { const bundle = renderBundle(options); const script = statusScript(options, bundle); - const command = [process.execPath, "scripts/cli.ts", "ssh", options.providerId, "argv", "bash", "-lc", script]; - const result = runCommand(command, repoRoot, { timeoutMs: options.timeoutMs }); + const result = runRemoteScript(options, script); if (result.exitCode !== 0 || result.timedOut) { return { ok: false, @@ -447,6 +488,54 @@ function runReadonlyStatus(options: ArtifactRegistryOptions, healthMode: boolean return statusFromValues(options, parseKeyValueOutput(result.stdout), result, healthMode); } +function remoteWriteFileCommand(item: RenderedFile): string { + const encoded = Buffer.from(item.content, "utf8").toString("base64"); + return [ + `target=${shellQuote(item.path)}`, + "tmp=$(mktemp /tmp/unidesk-artifact-registry.XXXXXX)", + "trap 'rm -f \"$tmp\"' EXIT", + `printf %s ${shellQuote(encoded)} | base64 -d > "$tmp"`, + "if [ \"$(id -u)\" = \"0\" ]; then", + " mkdir -p \"$(dirname \"$target\")\"", + ` install -m ${shellQuote(item.mode)} "$tmp" "$target"`, + "else", + " sudo mkdir -p \"$(dirname \"$target\")\"", + ` sudo install -m ${shellQuote(item.mode)} "$tmp" "$target"`, + "fi", + `echo ${shellQuote(`artifact_registry_file_written path=${item.path} sha256=${item.sha256}`)}`, + ].join("\n"); +} + +function install(options: ArtifactRegistryOptions): Record { + const bundle = renderBundle(options); + const script = [ + "set -euo pipefail", + "command -v docker >/dev/null", + "docker compose version >/dev/null", + "command -v systemctl >/dev/null", + `mkdir -p ${shellQuote(bundle.paths.baseDir)} ${shellQuote(bundle.paths.storage)}`, + ...bundle.files.map(remoteWriteFileCommand), + "if [ \"$(id -u)\" = \"0\" ]; then systemctl daemon-reload; else sudo systemctl daemon-reload; fi", + `if [ "$(id -u)" = "0" ]; then systemctl enable --now ${shellQuote(options.unitName)}; else sudo systemctl enable --now ${shellQuote(options.unitName)}; fi`, + "sleep 2", + `curl -fsS http://${options.host}:${options.port}/v2/ >/dev/null`, + ].join("\n"); + const command = runRemoteScript(options, script, Math.max(options.timeoutMs, 120_000)); + const status = runReadonlyStatus(options, true); + return { + ok: command.exitCode === 0 && !command.timedOut && status.ok === true, + dryRun: false, + mutation: true, + providerId: options.providerId, + render: { + paths: bundle.paths, + files: bundle.files.map((item) => ({ path: item.path, mode: item.mode, sha256: item.sha256 })), + }, + installCommand: commandTail(command), + health: status, + }; +} + function installDryRun(options: ArtifactRegistryOptions): Record { const bundle = renderBundle(options); return { @@ -466,32 +555,297 @@ function installDryRun(options: ArtifactRegistryOptions): Record Promise; +}> { + const children = new Set(); + const sockets = new Set(); + const remoteProxyScript = String.raw` +import select +import socket +import sys + +target = socket.create_connection(("127.0.0.1", 5000), timeout=10) +target.setblocking(False) +sys.stdin.buffer.raw.fileno() +sys.stdout.buffer.raw.fileno() +stdin_fd = sys.stdin.fileno() +stdout_fd = sys.stdout.fileno() +target_fd = target.fileno() + +while True: + readable, _, _ = select.select([stdin_fd, target_fd], [], []) + if stdin_fd in readable: + data = sys.stdin.buffer.read1(65536) + if not data: + try: + target.shutdown(socket.SHUT_WR) + except OSError: + pass + else: + target.sendall(data) + if target_fd in readable: + try: + data = target.recv(65536) + except BlockingIOError: + data = b"" + if not data: + break + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() +`; + const server = createServer((socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + const child = spawn(process.execPath, [ + "scripts/cli.ts", + "ssh", + options.providerId, + "argv", + "python3", + "-u", + "-c", + remoteProxyScript, + ], { + cwd: repoRoot, + stdio: ["pipe", "pipe", "pipe"], + }); + children.add(child); + child.on("close", () => children.delete(child)); + child.on("error", () => socket.destroy()); + socket.pipe(child.stdin); + child.stdout.pipe(socket); + child.stderr.on("data", (chunk) => process.stderr.write(`[artifact-registry-relay] ${chunk.toString("utf8")}`)); + socket.on("close", () => child.kill()); + }); + const listen = new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(options.localPullPort ?? 0, "127.0.0.1", () => { + server.off("error", reject); + const address = server.address() as AddressInfo; + resolve(address.port); + }); + }); + const port = await listen; + return { + server, + port, + close: async () => { + for (const socket of sockets) socket.destroy(); + for (const child of children) child.kill(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +} + +function composeLockScript(innerScript: string): string { + const lockDir = rootPath(".state", "locks"); + const lockPath = rootPath(".state", "locks", "server-compose.lock"); + return [ + "set -euo pipefail", + `mkdir -p ${shellQuote(lockDir)}`, + `echo ${shellQuote(`compose_lock_wait ${lockPath}`)}`, + `flock ${shellQuote(lockPath)} bash -lc ${shellQuote(innerScript)}`, + ].join("; "); +} + +function upsertEnvFileValues(path: string, values: Record): void { + const existing = readFileSync(path, "utf8"); + const seen = new Set(); + const lines = existing.split(/\n/u).filter((line, index, array) => index < array.length - 1 || line.length > 0).map((line) => { + const match = /^([A-Za-z0-9_]+)=/u.exec(line); + if (match === null || values[match[1]] === undefined) return line; + seen.add(match[1]); + return `${match[1]}=${values[match[1]]}`; + }); + for (const [key, value] of Object.entries(values)) { + if (!seen.has(key)) lines.push(`${key}=${value}`); + } + writeFileSync(path, `${lines.join("\n")}\n`, "utf8"); +} + +async function deployBackendCoreNow(options: ArtifactRegistryOptions): Promise> { + const commit = options.commit; + if (commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit "); + const health = runReadonlyStatus(options, true); + if (health.ok !== true) { + return { ok: false, error: "D601 artifact registry is not healthy", health }; + } + const sourceImage = `127.0.0.1:${options.port}/unidesk/backend-core:${commit}`; + const composeImage = options.targetImage; + const commitImage = `${options.targetImage}:${commit}`; + const registryProbeScript = [ + "set -euo pipefail", + `image=${shellQuote(`unidesk/backend-core:${commit}`)}`, + `registry_image=${shellQuote(sourceImage)}`, + "docker manifest inspect \"$registry_image\" >/tmp/unidesk-backend-core-manifest.json", + "manifest_digest=$(docker manifest inspect --verbose \"$registry_image\" 2>/dev/null | awk -F'\"' '/Digest/ {print $4; exit}' || true)", + "printf 'registry_image=%s\\nmanifest_digest=%s\\n' \"$registry_image\" \"$manifest_digest\"", + ].join("\n"); + const registryProbe = runRemoteScript(options, registryProbeScript, Math.max(options.timeoutMs, 120_000)); + if (registryProbe.exitCode !== 0 || registryProbe.timedOut) { + return { + ok: false, + step: "registry-artifact-check", + error: "backend-core image artifact is missing from D601 registry; run CI artifact publication first", + sourceImage, + registryProbe: commandTail(registryProbe), + }; + } + const relay = await startRegistryRelay(options); + const localSourceImage = `127.0.0.1:${relay.port}/unidesk/backend-core:${commit}`; + try { + const pull = runCommand(["docker", "pull", localSourceImage], repoRoot, { timeoutMs: Math.max(options.timeoutMs, 900_000) }); + if (pull.exitCode !== 0 || pull.timedOut) { + return { + ok: false, + step: "docker-pull", + sourceImage, + localRelayImage: localSourceImage, + registryProbe: commandTail(registryProbe), + pull: commandTail(pull), + }; + } + const inspectPulled = runCommand(["docker", "image", "inspect", localSourceImage, "--format", "{{json .Config.Labels}}"], repoRoot); + const labelCommit = runCommand(["docker", "image", "inspect", localSourceImage, "--format", "{{ index .Config.Labels \"unidesk.ai/source-commit\" }}"], repoRoot); + const labelService = runCommand(["docker", "image", "inspect", localSourceImage, "--format", "{{ index .Config.Labels \"unidesk.ai/service-id\" }}"], repoRoot); + if (labelCommit.stdout.trim() !== commit || labelService.stdout.trim() !== "backend-core") { + return { + ok: false, + step: "image-label-verify", + expected: { commit, serviceId: "backend-core" }, + observed: { commit: labelCommit.stdout.trim(), serviceId: labelService.stdout.trim(), labels: inspectPulled.stdout.trim() }, + registryProbe: commandTail(registryProbe), + }; + } + const tag = runCommand(["docker", "tag", localSourceImage, composeImage], repoRoot); + if (tag.exitCode !== 0 || tag.timedOut) { + return { ok: false, step: "docker-tag", targetImage: composeImage, tag: commandTail(tag), registryProbe: commandTail(registryProbe) }; + } + const tagCommit = runCommand(["docker", "tag", localSourceImage, commitImage], repoRoot); + if (tagCommit.exitCode !== 0 || tagCommit.timedOut) { + return { ok: false, step: "docker-tag-commit", targetImage: commitImage, tag: commandTail(tagCommit), registryProbe: commandTail(registryProbe) }; + } + const config = readConfig(); + const runtimeEnv = writeComposeEnv(config, false); + upsertEnvFileValues(runtimeEnv.envFile, { + UNIDESK_DEPLOY_REF: "deploy.json#environments.prod.services.backend-core", + UNIDESK_DEPLOY_SERVICE_ID: "backend-core", + UNIDESK_DEPLOY_REPO: "https://github.com/pikasTech/unidesk", + UNIDESK_DEPLOY_COMMIT: commit, + UNIDESK_DEPLOY_REQUESTED_COMMIT: commit, + }); + const compose = resolveComposeCommand(config, runtimeEnv.envFile); + const upScript = [ + "set -euo pipefail", + `echo ${shellQuote(`backend_core_artifact_cd source=${sourceImage} target=${composeImage}`)}`, + `${compose.map(shellQuote).join(" ")} up -d --no-build --no-deps --force-recreate backend-core`, + "ready=0", + "for attempt in $(seq 1 60); do", + " cid=$(docker ps -q --filter label=com.docker.compose.project=unidesk --filter label=com.docker.compose.service=backend-core --filter label=com.docker.compose.oneoff=False | head -1 || true)", + " if [ -n \"$cid\" ]; then", + " health=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' \"$cid\" 2>/dev/null || true)", + " echo \"backend_core_container_probe attempt=$attempt cid=$cid health=$health\"", + " if [ \"$health\" = \"healthy\" ] || [ \"$health\" = \"running\" ]; then ready=1; break; fi", + " else", + " echo \"backend_core_container_probe attempt=$attempt cid=missing\"", + " fi", + " sleep 1", + "done", + "test \"$ready\" = \"1\"", + "cid=$(docker ps -q --filter label=com.docker.compose.project=unidesk --filter label=com.docker.compose.service=backend-core --filter label=com.docker.compose.oneoff=False | head -1)", + "image_id=$(docker inspect -f '{{.Image}}' \"$cid\")", + "actual_commit=$(docker image inspect -f '{{ index .Config.Labels \"unidesk.ai/source-commit\" }}' \"$image_id\")", + `test "$actual_commit" = ${shellQuote(commit)}`, + "if docker exec \"$cid\" sh -lc 'command -v backend-core >/dev/null 2>&1'; then", + " docker exec \"$cid\" backend-core --fetch-json http://127.0.0.1:8080/health --require-ok >/tmp/unidesk-backend-core-health.json", + "else", + " docker exec \"$cid\" bun -e \"fetch('http://127.0.0.1:8080/health').then(async r=>{console.log(await r.text()); process.exit(r.ok?0:1)}).catch(e=>{console.error(e); process.exit(1)})\" >/tmp/unidesk-backend-core-health.json", + "fi", + "cat /tmp/unidesk-backend-core-health.json", + ].join("\n"); + const deploy = runCommand(["bash", "-lc", composeLockScript(upScript)], repoRoot, { timeoutMs: Math.max(options.timeoutMs, 300_000) }); + if (deploy.exitCode !== 0 || deploy.timedOut) { + return { + ok: false, + step: "compose-recreate", + sourceImage, + targetImage: composeImage, + registryProbe: commandTail(registryProbe), + deploy: commandTail(deploy), + }; + } + const running = runCommand(["docker", "inspect", "unidesk-backend-core", "--format", "image={{.Config.Image}} imageId={{.Image}} status={{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}"], repoRoot); + return { + ok: true, + commit, + providerId: options.providerId, + sourceImage, + registryProbe: commandTail(registryProbe), + localRelayImage: localSourceImage, + targetImage: composeImage, + targetCommitImage: commitImage, + relay: { + bind: `127.0.0.1:${relay.port}`, + persistent: false, + }, + pull: commandTail(pull), + deploy: commandTail(deploy), + running: running.stdout.trim(), + rollback: { + previousImageHint: "Use docker image ls and docker inspect logs for the previous backend-core image id; Compose named volumes were not changed.", + commandShape: `${compose.map(shellQuote).join(" ")} up -d --no-build --no-deps --force-recreate backend-core`, + }, + }; + } finally { + await relay.close(); + } +} + +function deployBackendCoreJob(args: string[], options: ArtifactRegistryOptions): Record { + if (options.commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit "); + const runArgs = args.includes("--run-now") ? args : [...args, "--run-now"]; + const command = [process.execPath, rootPath("scripts", "cli.ts"), "artifact-registry", ...runArgs]; + const job = startJob("artifact_registry_backend_core_cd", command, `Pull and deploy backend-core artifact ${options.commit} from D601 registry`); + return { + ok: true, + mode: "async-job", + job, + statusCommand: `bun scripts/cli.ts job status ${job.id}`, + tailCommand: `bun scripts/cli.ts job status ${job.id} --tail-bytes 30000`, + note: "Backend-core CD continues in the background: D601 registry health, short-lived local relay, docker pull, retag, no-build recreate, live commit verification.", }; } function localHelp(): Record { return { ok: true, - command: "artifact-registry plan|render|status|health|install", + command: "artifact-registry plan|render|status|health|install|deploy-backend-core", output: "json", usage: [ "bun scripts/cli.ts artifact-registry plan [--provider-id D601]", "bun scripts/cli.ts artifact-registry render [--provider-id D601]", "bun scripts/cli.ts artifact-registry status [--provider-id D601]", "bun scripts/cli.ts artifact-registry health [--provider-id D601]", - "bun scripts/cli.ts artifact-registry install --dry-run [--provider-id D601]", + "bun scripts/cli.ts artifact-registry install [--provider-id D601]", + "bun scripts/cli.ts artifact-registry deploy-backend-core --commit [--run-now] [--provider-id D601]", ], - firstStage: "install without --dry-run is intentionally disabled", + firstStage: "install now writes the rendered systemd/Compose/config files and starts the registry", defaults: defaultOptions, }; } -export function runArtifactRegistryCommand(args: string[]): unknown { +export async function runArtifactRegistryCommand(args: string[]): Promise { const action = args[0]; if (isHelpArg(action)) return localHelp(); - if (action !== "plan" && action !== "render" && action !== "status" && action !== "health" && action !== "install") { - throw new Error("artifact-registry usage: plan|render|status|health|install"); + if (action !== "plan" && action !== "render" && action !== "status" && action !== "health" && action !== "install" && action !== "deploy-backend-core") { + throw new Error("artifact-registry usage: plan|render|status|health|install|deploy-backend-core"); } const options = parseOptions(args.slice(1)); if (action === "plan") return plan(options); @@ -499,15 +853,11 @@ export function runArtifactRegistryCommand(args: string[]): unknown { if (action === "status") return runReadonlyStatus(options, false); if (action === "health") return runReadonlyStatus(options, true); if (action === "install") { - if (!options.dryRun) { - return { - ok: false, - error: "artifact-registry install is first-stage dry-run only; pass --dry-run", - firstStageOnly: true, - mutation: false, - }; - } - return installDryRun(options); + return options.dryRun ? installDryRun(options) : install(options); + } + if (action === "deploy-backend-core") { + if (options.commit === null) throw new Error("artifact-registry deploy-backend-core requires --commit "); + return options.runNow ? await deployBackendCoreNow(options) : deployBackendCoreJob(args, options); } throw new Error("unreachable artifact-registry action"); } diff --git a/scripts/src/check.ts b/scripts/src/check.ts index a60cdb87..01551816 100644 --- a/scripts/src/check.ts +++ b/scripts/src/check.ts @@ -236,9 +236,11 @@ export function runChecks(config: UniDeskConfig, options: CheckOptions = default fileItem("src/components/microservices/code-queue-mgr/src/index.ts"), fileItem("src/components/microservices/code-queue-mgr/src/prompt-observation.ts"), fileItem("scripts/src/deploy.ts"), + fileItem("scripts/src/ci.ts"), fileItem("scripts/src/e2e.ts"), fileItem("scripts/code-queue-prompt-observation-test.ts"), fileItem("scripts/src/artifact-registry.ts"), + fileItem("src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml"), ); } else { items.push(skippedItem("files:required-entrypoints", "required file presence scan is opt-in", "--files or --full")); diff --git a/scripts/src/ci.ts b/scripts/src/ci.ts index 1752e284..c38e894e 100644 --- a/scripts/src/ci.ts +++ b/scripts/src/ci.ts @@ -34,6 +34,12 @@ interface CiOptions { waitMs: number; } +interface CiPublishBackendCoreOptions { + repoUrl: string; + commit: string; + waitMs: number; +} + interface CiDevE2EOptions { repoUrl: string; desiredRef: string; @@ -92,6 +98,12 @@ function requireRevision(value: string | null): string { return value; } +function requireFullCommit(value: string | null, option = "--commit"): string { + const commit = value?.toLowerCase() ?? ""; + if (!/^[0-9a-f]{40}$/u.test(commit)) throw new Error(`${option} requires a full 40-character pushed Git commit SHA`); + return commit; +} + function requireDesiredRef(value: string | null): string { const ref = value ?? "master"; if (!/^[A-Za-z0-9._/-]{1,160}$/u.test(ref) || ref.startsWith("-") || ref.includes("..")) { @@ -474,6 +486,35 @@ spec: `; } +function backendCoreArtifactPipelineRunManifest(options: CiPublishBackendCoreOptions): string { + const safeSuffix = new Date().toISOString().replace(/[-:.TZ]/g, "").slice(0, 14).toLowerCase(); + return `apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: backend-core-artifact-${safeSuffix}- + namespace: unidesk-ci + labels: + app.kubernetes.io/name: unidesk-backend-core-artifact-publish + app.kubernetes.io/part-of: unidesk + unidesk.ai/service-id: backend-core + unidesk.ai/revision: ${JSON.stringify(options.commit)} +spec: + pipelineRef: + name: unidesk-backend-core-artifact-publish + taskRunTemplate: + serviceAccountName: unidesk-ci-runner + params: + - name: repo-url + value: ${JSON.stringify(options.repoUrl)} + - name: revision + value: ${JSON.stringify(options.commit)} + workspaces: + - name: shared-workspace + persistentVolumeClaim: + claimName: unidesk-ci-cache +`; +} + async function remoteCreatePipelineRun(manifest: string): Promise { const encoded = Buffer.from(manifest, "utf8").toString("base64"); const token = randomUUID().replace(/-/gu, "").slice(0, 12); @@ -540,6 +581,29 @@ async function run(options: CiOptions): Promise> { }; } +async function publishBackendCoreArtifact(options: CiPublishBackendCoreOptions): Promise> { + const name = await remoteCreatePipelineRun(backendCoreArtifactPipelineRunManifest(options)); + const wait = await waitForPipelineRun(name, options.waitMs); + const waitSucceeded = wait === null || wait.exitCode === 0 || wait.stdout.trimStart().startsWith("True\tSucceeded\t"); + return { + ok: waitSucceeded, + pipelineRun: name, + namespace: "unidesk-ci", + repoUrl: options.repoUrl, + commit: options.commit, + artifact: `127.0.0.1:5000/unidesk/backend-core:${options.commit}`, + boundary: "CI publishes the image to D601 registry; CD must pull it and must not build backend-core", + wait: wait === null ? null : { + stdoutTail: wait.stdout.slice(-6000), + stderrTail: wait.stderr.slice(-6000), + }, + next: [ + `bun scripts/cli.ts ci logs ${name}`, + `bun scripts/cli.ts artifact-registry deploy-backend-core --commit ${options.commit}`, + ], + }; +} + async function runRemoteDevE2ELauncher(options: CiDevE2EOptions): Promise { const scriptTimeoutMs = Math.max(options.scriptTimeoutMs, options.waitMs, 60_000); const remoteTimeoutMs = 45_000; @@ -788,11 +852,12 @@ async function logs(name: string): Promise> { export function ciHelp(): Record { return { - command: "ci install|status|run|run-dev-e2e|logs", - description: "Manage the D601 k3s Tekton CI gate. This intentionally does not deploy CD.", + command: "ci install|status|run|publish-backend-core|run-dev-e2e|logs", + description: "Manage the D601 k3s Tekton CI gate. CI may publish backend-core image artifacts, but it intentionally does not deploy CD.", examples: [ "bun scripts/cli.ts ci install", "bun scripts/cli.ts ci run --revision ", + "bun scripts/cli.ts ci publish-backend-core --commit ", "bun scripts/cli.ts ci run-dev-e2e --wait-ms 600000", "bun scripts/cli.ts ci logs ", ], @@ -805,6 +870,11 @@ export function ciHelp(): Record { interceptors: tektonTriggersInterceptorsUrl, }, }, + backendCoreArtifact: { + producer: "D601 CI", + registry: "127.0.0.1:5000/unidesk/backend-core:", + cdCommand: "bun scripts/cli.ts artifact-registry deploy-backend-core --commit ", + }, runDevE2E: { defaultTriggerMode: "commit-pinned-ssh-launcher", desiredState: "origin/master:deploy.json#environments.dev", @@ -831,6 +901,12 @@ export async function runCiCommand(_config: UniDeskConfig, args: string[]): Prom const waitMs = numberOption(args, "--wait-ms", 0); return run({ repoUrl, revision, waitMs }); } + if (action === "publish-backend-core") { + const repoUrl = stringOption(args, "--repo") ?? stringOption(args, "--repo-url") ?? "https://github.com/pikasTech/unidesk"; + const commit = requireFullCommit(stringOption(args, "--commit") ?? stringOption(args, "--revision")); + const waitMs = numberOption(args, "--wait-ms", 0); + return publishBackendCoreArtifact({ repoUrl, commit, waitMs }); + } if (action === "run-dev-e2e") { const repoUrl = stringOption(args, "--repo") ?? stringOption(args, "--repo-url") ?? "https://github.com/pikasTech/unidesk"; const desiredRef = requireDesiredRef(stringOption(args, "--desired-ref") ?? stringOption(args, "--deploy-branch")); @@ -852,7 +928,7 @@ export async function runCiCommand(_config: UniDeskConfig, args: string[]): Prom }); } if (action === "logs") return logs(nameArg ?? ""); - throw new Error("ci command must be one of: install, status, run, run-dev-e2e, logs"); + throw new Error("ci command must be one of: install, status, run, publish-backend-core, run-dev-e2e, logs"); } export function startCiInstallJob(): Record { diff --git a/scripts/src/docker.ts b/scripts/src/docker.ts index 7904db32..e01bcd78 100644 --- a/scripts/src/docker.ts +++ b/scripts/src/docker.ts @@ -106,6 +106,11 @@ export function writeComposeEnv(config: UniDeskConfig, freshLogPrefix: boolean): UNIDESK_PROVIDER_NAME: config.providerGateway.name, UNIDESK_PROVIDER_LABELS_JSON: labels, UNIDESK_MICROSERVICES_JSON: microservices, + UNIDESK_DEPLOY_REF: runtimeSecret("UNIDESK_DEPLOY_REF"), + UNIDESK_DEPLOY_SERVICE_ID: runtimeSecret("UNIDESK_DEPLOY_SERVICE_ID") || "backend-core", + UNIDESK_DEPLOY_REPO: runtimeSecret("UNIDESK_DEPLOY_REPO"), + UNIDESK_DEPLOY_COMMIT: runtimeSecret("UNIDESK_DEPLOY_COMMIT"), + UNIDESK_DEPLOY_REQUESTED_COMMIT: runtimeSecret("UNIDESK_DEPLOY_REQUESTED_COMMIT"), UNIDESK_AUTH_USERNAME: config.auth.username, UNIDESK_AUTH_PASSWORD: config.auth.password, UNIDESK_SESSION_SECRET: config.auth.sessionSecret, diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 9f269495..62632678 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -36,7 +36,7 @@ export function rootHelp(): unknown { { command: "decision show ", description: "Show one Decision Center record." }, { command: "deploy check|plan|apply [--file deploy.json|--env dev|prod] [--service id] [--dry-run] [--force]", description: "Reconcile services from a repo+commit manifest; --env reads origin/master:deploy.json environments and can apply supported dev services." }, { command: "dev-env validate|prewarm-images", description: "Validate D601 unidesk-dev guardrails or prewarm dev foundation images into native k3s containerd through a bounded async job." }, - { command: "artifact-registry plan|render|status|health|install --dry-run", description: "Plan and inspect the D601 host-managed CNCF Distribution registry for future backend-core artifact CD; first-stage install is dry-run only." }, + { command: "artifact-registry plan|render|status|health|install|deploy-backend-core", description: "Manage the D601 host-managed CNCF Distribution registry and run pull-only production backend-core artifact CD." }, { command: "schedule list|get|runs|run|delete", description: "Manage backend-core scheduled tasks and run history; schedule run supports --wait-ms N." }, { command: "schedule upsert-pgdata-backup [--time HH:MM] [--remote-base /SERVER_DATA/UNIDESK_PG_DATA]", description: "Create or update the daily PGDATA physical backup task that uploads monthly rotated archives to Baidu Netdisk." }, { command: "codex deploy [--provider-id D601] [--timeout-ms N]", description: "Compatibility wrapper for deploy apply --service code-queue with a temporary repo+commit manifest." }, @@ -52,7 +52,7 @@ export function rootHelp(): unknown { { command: "debug dispatch [providerId] [docker.ps|provider.upgrade|host.ssh|microservice.http|echo] [--wait-ms N]", description: "Submit a real internal-core dispatch request for CLI debugging." }, { command: "debug task ", description: "Read a dispatched task record from internal core for CLI debugging." }, { command: "network perf [--service code-queue --path /api/tasks/overview?limit=30 --count N --concurrency N --label before|after]", description: "Benchmark frontend -> backend-core -> provider/adapter user-service networking and report latency/proxy-mode distributions." }, - { command: "ci install|status|run|run-dev-e2e|logs", description: "Manage D601 k3s Tekton CI only; run-dev-e2e manually validates master deploy.json dev state in an isolated temporary namespace." }, + { command: "ci install|status|run|publish-backend-core|run-dev-e2e|logs", description: "Manage D601 k3s Tekton CI; publish-backend-core builds commit-pinned images in CI without deploying CD." }, { command: "e2e run [--only pattern[,pattern...]] [--skip pattern[,pattern...]]", description: "Run selected public/internal/Playwright E2E checks; use --only for focused iteration and rerun without filters for final regression." }, ], }; @@ -247,20 +247,22 @@ function devEnvHelp(): unknown { function artifactRegistryHelp(): unknown { return { - command: "artifact-registry plan|render|status|health|install", + command: "artifact-registry plan|render|status|health|install|deploy-backend-core", output: "json", usage: [ "bun scripts/cli.ts artifact-registry plan [--provider-id D601]", "bun scripts/cli.ts artifact-registry render [--provider-id D601]", "bun scripts/cli.ts artifact-registry status [--provider-id D601]", "bun scripts/cli.ts artifact-registry health [--provider-id D601]", - "bun scripts/cli.ts artifact-registry install --dry-run [--provider-id D601]", + "bun scripts/cli.ts artifact-registry install [--provider-id D601]", + "bun scripts/cli.ts artifact-registry deploy-backend-core --commit [--run-now] [--provider-id D601]", ], description: "Manage the declaration, rendered files and readonly checks for the D601 host-managed CNCF Distribution artifact registry.", boundary: [ "registry endpoint is D601 loopback 127.0.0.1:5000 only", "service is host-managed by systemd + Docker Compose, not k3s-managed", - "first-stage install without --dry-run is rejected", + "install writes the rendered host unit/config and starts the registry", + "deploy-backend-core only pulls commit-pinned backend-core artifacts and does not build backend-core on the master server", "status and health use provider-gateway Host SSH readonly checks", ], }; diff --git a/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml b/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml index 19898bff..f7f78179 100644 --- a/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml +++ b/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml @@ -258,6 +258,170 @@ spec: --- apiVersion: tekton.dev/v1 kind: Task +metadata: + name: unidesk-backend-core-artifact-publish + namespace: unidesk-ci + labels: + app.kubernetes.io/name: unidesk-ci + app.kubernetes.io/component: backend-core-artifact +spec: + params: + - name: repo-url + type: string + - name: revision + type: string + - name: app-image + type: string + default: unidesk-code-queue:dev + - name: registry + type: string + default: 127.0.0.1:5000 + workspaces: + - name: source + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + type: Socket + steps: + - name: clone + image: alpine/git:2.45.2 + env: + - name: HTTP_PROXY + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: HTTPS_PROXY + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: ALL_PROXY + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: NO_PROXY + value: "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local" + - name: http_proxy + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: https_proxy + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: all_proxy + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: no_proxy + value: "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local" + script: | + #!/bin/sh + set -eu + case "$(params.revision)" in + [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; + *) echo "backend_core_artifact_revision_must_be_full_sha=$(params.revision)" >&2; exit 2 ;; + esac + rm -rf "$(workspaces.source.path)/backend-core-artifact-repo" + git clone --filter=blob:none "$(params.repo-url)" "$(workspaces.source.path)/backend-core-artifact-repo" + cd "$(workspaces.source.path)/backend-core-artifact-repo" + git fetch --depth=1 origin "$(params.revision)" + git checkout --detach FETCH_HEAD + resolved="$(git rev-parse HEAD)" + test "$resolved" = "$(params.revision)" + git cat-file -e "$resolved:src/components/backend-core/Dockerfile" + git cat-file -e "$resolved:src/components/backend-core/src" + git rev-parse HEAD | tee "$(workspaces.source.path)/backend-core-artifact-commit.txt" + - name: build-and-push + image: "$(params.app-image)" + imagePullPolicy: Never + workingDir: "$(workspaces.source.path)/backend-core-artifact-repo" + env: + - name: DOCKER_HOST + value: unix:///var/run/docker.sock + - name: HTTP_PROXY + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: HTTPS_PROXY + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: ALL_PROXY + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: NO_PROXY + value: "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local" + - name: http_proxy + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: https_proxy + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: all_proxy + value: "http://d601-provider-egress-proxy.unidesk.svc.cluster.local:18789" + - name: no_proxy + value: "localhost,127.0.0.1,::1,host.docker.internal,d601-provider-egress-proxy,d601-provider-egress-proxy.unidesk,d601-provider-egress-proxy.unidesk.svc,d601-provider-egress-proxy.unidesk.svc.cluster.local" + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + script: | + #!/usr/bin/env bash + set -euo pipefail + commit="$(cat "$(workspaces.source.path)/backend-core-artifact-commit.txt")" + registry="$(params.registry)" + local_image="unidesk/backend-core:$commit" + registry_image="$registry/unidesk/backend-core:$commit" + command -v docker + docker buildx version >/dev/null 2>&1 || docker buildx inspect default >/dev/null 2>&1 + docker run --rm --network host rancher/mirrored-library-busybox:1.36.1 wget -q -O- "http://$registry/v2/" >/dev/null + docker buildx build \ + --builder default \ + --load \ + --network host \ + --progress=plain \ + --build-arg HTTP_PROXY=http://127.0.0.1:18789 \ + --build-arg HTTPS_PROXY=http://127.0.0.1:18789 \ + --build-arg ALL_PROXY=http://127.0.0.1:18789 \ + --build-arg NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal \ + --label "unidesk.ai/service-id=backend-core" \ + --label "unidesk.ai/source-repo=$(params.repo-url)" \ + --label "unidesk.ai/source-commit=$commit" \ + --label "unidesk.ai/dockerfile=src/components/backend-core/Dockerfile" \ + -t "$local_image" \ + -t "$registry_image" \ + -f src/components/backend-core/Dockerfile \ + . + actual_commit="$(docker image inspect "$registry_image" --format '{{ index .Config.Labels "unidesk.ai/source-commit" }}')" + test "$actual_commit" = "$commit" + docker push "$registry_image" + repo_digests="$(docker image inspect "$registry_image" --format '{{json .RepoDigests}}')" + printf 'backend_core_artifact_image=%s\nbackend_core_artifact_repo_digests=%s\n' "$registry_image" "$repo_digests" +--- +apiVersion: tekton.dev/v1 +kind: Pipeline +metadata: + name: unidesk-backend-core-artifact-publish + namespace: unidesk-ci + labels: + app.kubernetes.io/name: unidesk-ci + app.kubernetes.io/component: backend-core-artifact + app.kubernetes.io/part-of: unidesk +spec: + params: + - name: repo-url + type: string + default: https://github.com/pikasTech/unidesk + - name: revision + type: string + - name: app-image + type: string + default: unidesk-code-queue:dev + - name: registry + type: string + default: 127.0.0.1:5000 + workspaces: + - name: shared-workspace + tasks: + - name: publish-backend-core-artifact + taskRef: + name: unidesk-backend-core-artifact-publish + params: + - name: repo-url + value: "$(params.repo-url)" + - name: revision + value: "$(params.revision)" + - name: app-image + value: "$(params.app-image)" + - name: registry + value: "$(params.registry)" + workspaces: + - name: source + workspace: shared-workspace +--- +apiVersion: tekton.dev/v1 +kind: Task metadata: name: unidesk-code-queue-read-perf namespace: unidesk-ci