diff --git a/AGENTS.md b/AGENTS.md index 557f353a..7d4106d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -41,7 +41,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台;本文 - `bun scripts/cli.ts deploy check/plan/apply [--file deploy.json|--env dev|prod] [--service ]`:按根目录 `deploy.json` 或 `origin/master:deploy.json#environments.` 的服务 repo 和 commit 期望状态校验或更新用户服务;`--env dev` 当前开放 D601 `backend-core` persistent dev rollout 以及 `frontend`/`baidu-netdisk`/`decision-center`/`project-manager`/`oa-event-flow`/`code-queue-mgr` artifact-consumer validation,`todo-note` 仅 dry-run,规则见 `docs/reference/deploy.md` 与 `docs/reference/dev-environment.md`。 - `bun scripts/cli.ts dev-env validate [--manifest path] [--kubectl-dry-run]` / `dev-env prewarm-images`:离线校验 D601 `unidesk-dev` 生产隔离护栏,或把开发底座基础镜像预热到 D601 原生 k3s containerd,规则见 `docs/reference/deploy.md` 与 `docs/reference/microservices.md`。 - `bun scripts/cli.ts artifact-registry plan|render|status|health|install|deploy-backend-core|deploy-service`:管理 D601 host-managed CNCF Distribution registry,并通过短生命周期 relay 或 D601 pull/import 做 commit-pinned pull-only artifact CD;`deploy-backend-core` 是 deprecated 兼容名,backend-core prod CD 标准入口是 `deploy apply --env prod --service backend-core`,规则见 `docs/reference/artifact-registry.md`。 -- `bun scripts/cli.ts ci install/status/run/publish-backend-core/publish-user-service/run-dev-e2e/logs`:在 D601 原生 k3s 上安装和运行 Tekton CI,支持每 commit 检查、Code Queue 只读性能门禁、backend-core 与 user-service commit-pinned 镜像发布和手动触发的 `origin/master:deploy.json#environments.dev` 临时 namespace e2e;`run-dev-e2e` 的 Git 控制 runner、短 launcher 和 no-CD 边界见 `docs/reference/dev-ci-runner.md`,Tekton 规则见 `docs/reference/ci.md`。 +- `bun scripts/cli.ts ci install/status/run/publish-backend-core/publish-user-service/run-dev-e2e/logs`:在 D601 原生 k3s 上安装和运行 Tekton CI,支持每 commit 检查、Code Queue 只读性能门禁、`CI.json` catalog 驱动的 backend-core 与 user-service commit-pinned 镜像发布和手动触发的 `origin/master:deploy.json#environments.dev` 临时 namespace e2e;catalog/producer/consumer 分工见 `docs/reference/cicd-standardization.md`,`run-dev-e2e` 的 Git 控制 runner、短 launcher 和 no-CD 边界见 `docs/reference/dev-ci-runner.md`,Tekton 规则见 `docs/reference/ci.md`。 - `bun scripts/cli.ts codex deploy `:旧 Code Queue 兼容部署入口已禁用,原因是它会绕过受控部署边界直连 D601 部署 Code Queue;规则见 `docs/reference/codex-deploy.md`。 - `bun scripts/cli.ts codex submit [prompt] [--prompt-file path|--prompt-stdin] [--queue ]`:通过 backend-core 私有代理提交 Code Queue 任务;控制面默认走主 server `code-queue-mgr` 写入 PostgreSQL,`--dry-run` 可只检查请求体不入队,规则见 `docs/reference/cli.md`。 - `bun scripts/cli.ts codex task `:按 Code Queue 任务 ID 查询初始 prompt、最后 assistant message、工具调用摘要、attempt/judge/error 和耗时,便于新任务引用历史 session。 @@ -78,7 +78,7 @@ UniDesk 是一个以主 server 为统一入口的分布式工作平台;本文 - `docs/reference/pipeline-model-proxy.md`:Pipeline v2 model proxy 链路架构、D601 宿主 proxy 服务部署、harness token 注入规则和 smoke test 验证流程。 - `docs/reference/deploy.md`:`deploy.json` desired-state、target-side build、一次性构建 proxy、直管/代管服务部署 executor 和 live commit 验证规则。 - `docs/reference/devops-hygiene.md`:Git-backed deployment truth、dirty worktree/manual repair 边界、受限手动操作和 CI 私有仓库 source-auth 规则。 -- `docs/reference/cicd-standardization.md`:镜像 artifact 标准化目标、File Browser 上游镜像例外、legacy CI/CD 路径分类和本轮 guardrail。 +- `docs/reference/cicd-standardization.md`:`CI.json` catalog、CI producer summary、blocked/upstream-image 服务、File Browser 上游镜像例外、legacy CI/CD 路径分类和 CD consumer 分工。 - `docs/reference/release-governance.md`:`release/v1` 稳定维护线、`master` 集成线、CI/CD server 版本固定、master CLI 兼容和 feature flag 治理规则;决策记录见 GitHub issue #6。 - `docs/reference/artifact-registry.md`:D601 host-managed CNCF Distribution registry、loopback-only 边界和 backend-core artifact CD 目标流程。 - `docs/reference/user-service-delivery.md`:用户服务默认交付流程、CI 镜像构建与 registry、Baidu Netdisk 主 server 直管微服务样板、dev 自动测试、prod 拉镜像部署和 Decision Center 产品化需求管理规则。 diff --git a/CI.json b/CI.json index 5719ea81..fd4dac28 100644 --- a/CI.json +++ b/CI.json @@ -1,7 +1,7 @@ { - "schemaVersion": 1, + "schemaVersion": 2, "kind": "ci-artifact-catalog", - "purpose": "CI artifact catalog only. This file describes build inputs and image artifact naming; it does not describe runtime topology and does not replace deploy.json.", + "purpose": "CI artifact catalog only. This file describes build inputs and image artifact naming; it does not describe runtime topology and does not replace deploy.json, config.json, or runtime manifests.", "summaryContract": { "requiredOnSuccess": [ "serviceId", @@ -27,7 +27,7 @@ "defaults": { "producer": "D601 Tekton CI", "registry": "127.0.0.1:5000", - "tag": "{{sourceCommit}}", + "tagTemplate": "{{sourceCommit}}", "mutableTagsAllowed": false, "runtimeFieldsForbidden": [ "providerId", @@ -41,127 +41,236 @@ "volumes" ] }, - "upstreamImageConsumers": [ - { - "serviceId": "filebrowser", - "source": "upstream-image", - "upstreamImageRef": "docker.io/filebrowser/filebrowser:v2.63.3", - "upstreamSourceRepo": "https://github.com/filebrowser/filebrowser", - "upstreamSourceRevision": "ca5e249e3c0c94159c2136a0cd431a424eb18472", - "digestPin": { - "required": true, - "status": "pending-network-verification", - "expectedRefShape": "docker.io/filebrowser/filebrowser@sha256:" - }, - "mirrorStrategy": { - "mode": "mirror-after-digest-verification", - "targetRepository": "127.0.0.1:5000/upstream/filebrowser/filebrowser", - "targetDigestRefShape": "127.0.0.1:5000/upstream/filebrowser/filebrowser@sha256:" - }, - "ciBuild": { - "dockerfileBuild": false, - "publishCommand": null, - "reason": "third-party upstream image; CD may only pull a verified digest or mirror digest" - }, - "pullOnlyCdValidation": [ - "resolve tag to upstream manifest digest before mirroring or rollout", - "pull by digest or from the digest-verified local mirror", - "verify container image id/digest and OCI labels report filebrowser 2.63.3 / ca5e249e3c0c94159c2136a0cd431a424eb18472", - "verify provider-private File Browser health through the UniDesk microservice proxy", - "do not run docker build, docker compose up --build, or a CI Dockerfile producer" - ] - }, - { - "serviceId": "filebrowser-d601", - "source": "upstream-image", - "upstreamImageRef": "docker.io/filebrowser/filebrowser:v2.63.3", - "upstreamSourceRepo": "https://github.com/filebrowser/filebrowser", - "upstreamSourceRevision": "ca5e249e3c0c94159c2136a0cd431a424eb18472", - "digestPin": { - "required": true, - "status": "pending-network-verification", - "expectedRefShape": "docker.io/filebrowser/filebrowser@sha256:" - }, - "mirrorStrategy": { - "mode": "mirror-after-digest-verification", - "targetRepository": "127.0.0.1:5000/upstream/filebrowser/filebrowser", - "targetDigestRefShape": "127.0.0.1:5000/upstream/filebrowser/filebrowser@sha256:" - }, - "ciBuild": { - "dockerfileBuild": false, - "publishCommand": null, - "reason": "third-party upstream image; CD may only pull a verified digest or mirror digest" - }, - "pullOnlyCdValidation": [ - "resolve tag to upstream manifest digest before mirroring or rollout", - "pull by digest or from the digest-verified local mirror", - "verify container image id/digest and OCI labels report filebrowser 2.63.3 / ca5e249e3c0c94159c2136a0cd431a424eb18472", - "verify provider-private File Browser health through the UniDesk microservice proxy", - "do not run docker build, docker compose up --build, or a CI Dockerfile producer" - ] - } - ], "artifacts": [ { - "serviceId": "baidu-netdisk", - "sourceRepo": "https://github.com/pikasTech/unidesk", - "dockerfile": "src/components/microservices/baidu-netdisk/Dockerfile", - "imageRepository": "unidesk/baidu-netdisk", - "imageRef": "127.0.0.1:5000/unidesk/baidu-netdisk:{{sourceCommit}}", - "digestRef": "127.0.0.1:5000/unidesk/baidu-netdisk@{{digest}}" - }, - { - "serviceId": "decision-center", - "sourceRepo": "https://github.com/pikasTech/unidesk", - "dockerfile": "src/components/microservices/decision-center/Dockerfile", - "imageRepository": "unidesk/decision-center", - "imageRef": "127.0.0.1:5000/unidesk/decision-center:{{sourceCommit}}", - "digestRef": "127.0.0.1:5000/unidesk/decision-center@{{digest}}", - "publishCommand": "bun scripts/cli.ts ci publish-user-service --service decision-center --commit " + "serviceId": "backend-core", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-backend-core", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/backend-core/Dockerfile" + }, + "image": { + "repository": "unidesk/backend-core" + }, + "notes": "Rust backend-core image creation is CI producer only. It is limited to dev image validation and artifact publication; this catalog does not authorize prod deployment verification." }, { "serviceId": "frontend", - "sourceRepo": "https://github.com/pikasTech/unidesk", - "dockerfile": "src/components/frontend/Dockerfile", - "imageRepository": "unidesk/frontend", - "imageRef": "127.0.0.1:5000/unidesk/frontend:{{sourceCommit}}", - "digestRef": "127.0.0.1:5000/unidesk/frontend@{{digest}}" + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/frontend/Dockerfile" + }, + "image": { + "repository": "unidesk/frontend" + } + }, + { + "serviceId": "baidu-netdisk", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/microservices/baidu-netdisk/Dockerfile" + }, + "image": { + "repository": "unidesk/baidu-netdisk" + } + }, + { + "serviceId": "decision-center", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/microservices/decision-center/Dockerfile" + }, + "image": { + "repository": "unidesk/decision-center" + } }, { "serviceId": "project-manager", - "sourceRepo": "https://github.com/pikasTech/unidesk", - "dockerfile": "src/components/microservices/project-manager/Dockerfile", - "imageRepository": "unidesk/project-manager", - "imageRef": "127.0.0.1:5000/unidesk/project-manager:{{sourceCommit}}", - "digestRef": "127.0.0.1:5000/unidesk/project-manager@{{digest}}", - "publishCommand": "bun scripts/cli.ts ci publish-user-service --service project-manager --commit " + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/microservices/project-manager/Dockerfile" + }, + "image": { + "repository": "unidesk/project-manager" + } }, { "serviceId": "oa-event-flow", - "sourceRepo": "https://github.com/pikasTech/unidesk", - "dockerfile": "src/components/microservices/oa-event-flow/Dockerfile", - "imageRepository": "unidesk/oa-event-flow", - "imageRef": "127.0.0.1:5000/unidesk/oa-event-flow:{{sourceCommit}}", - "digestRef": "127.0.0.1:5000/unidesk/oa-event-flow@{{digest}}", - "publishCommand": "bun scripts/cli.ts ci publish-user-service --service oa-event-flow --commit " + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/microservices/oa-event-flow/Dockerfile" + }, + "image": { + "repository": "unidesk/oa-event-flow" + } + }, + { + "serviceId": "todo-note", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://gitee.com/Lyon1998/todo_note", + "dockerfile": "Dockerfile" + }, + "image": { + "repository": "unidesk/todo-note" + } }, { "serviceId": "code-queue-mgr", - "sourceRepo": "https://github.com/pikasTech/unidesk", - "dockerfile": "src/components/microservices/code-queue-mgr/Dockerfile", - "imageRepository": "unidesk/code-queue-mgr", - "imageRef": "127.0.0.1:5000/unidesk/code-queue-mgr:{{sourceCommit}}", - "digestRef": "127.0.0.1:5000/unidesk/code-queue-mgr@{{digest}}", - "publishCommand": "bun scripts/cli.ts ci publish-user-service --service code-queue-mgr --commit " + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/microservices/code-queue-mgr/Dockerfile" + }, + "image": { + "repository": "unidesk/code-queue-mgr" + }, + "notes": "Main-server internal sidecar artifact producer. Dev artifact consumer validation is supported; prod live apply is supervisor-gated by the deploy/artifact-registry consumer." }, { - "serviceId": "backend-core", - "sourceRepo": "https://github.com/pikasTech/unidesk", - "dockerfile": "src/components/backend-core/Dockerfile", - "imageRepository": "unidesk/backend-core", - "imageRef": "127.0.0.1:5000/unidesk/backend-core:{{sourceCommit}}", - "digestRef": "127.0.0.1:5000/unidesk/backend-core@{{digest}}", - "publishCommand": "bun scripts/cli.ts ci publish-backend-core --commit " + "serviceId": "findjob", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://gitee.com/Lyon1998/findjob", + "dockerfile": "Dockerfile" + }, + "image": { + "repository": "unidesk/findjob" + } + }, + { + "serviceId": "pipeline", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/pipeline", + "dockerfile": "Dockerfile" + }, + "image": { + "repository": "unidesk/pipeline" + } + }, + { + "serviceId": "met-nonlinear", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/met_nonlinear", + "dockerfile": "docker/unidesk/Dockerfile.ml" + }, + "image": { + "repository": "unidesk/met-nonlinear" + } + }, + { + "serviceId": "k3sctl-adapter", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/microservices/k3sctl-adapter/Dockerfile" + }, + "image": { + "repository": "unidesk/k3sctl-adapter" + } + }, + { + "serviceId": "mdtodo", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/microservices/mdtodo/Dockerfile" + }, + "image": { + "repository": "unidesk/mdtodo" + } + }, + { + "serviceId": "claudeqq", + "kind": "source-build", + "status": "supported", + "producer": "ci publish-user-service", + "source": { + "repo": "https://gitee.com/lyon1998/agent_skills", + "dockerfile": "claudeqq/Dockerfile" + }, + "image": { + "repository": "unidesk/claudeqq" + } + }, + { + "serviceId": "code-queue", + "kind": "source-build", + "status": "blocked", + "producer": "ci publish-user-service", + "source": { + "repo": "https://github.com/pikasTech/unidesk", + "dockerfile": "src/components/microservices/code-queue/Dockerfile" + }, + "image": { + "repository": "unidesk/code-queue" + }, + "blockedReason": "D601 code-queue is limited to dev image validation in this phase. The catalog records its producer input, but publish-user-service must not run prod-oriented artifact publication for it yet." + }, + { + "serviceId": "filebrowser", + "kind": "upstream-image", + "status": "blocked", + "producer": "ci publish-user-service", + "upstream": { + "imageRef": "docker.io/filebrowser/filebrowser:v2.63.3", + "digestRef": "docker.io/filebrowser/filebrowser@sha256:289c5dd677c56662440f26eeb44266ed9746fe563d2e9100f546bff558534d70", + "sourceRepo": "https://github.com/filebrowser/filebrowser", + "sourceRevision": "ca5e249e3c0c94159c2136a0cd431a424eb18472", + "mirrorRepository": "upstream/filebrowser/filebrowser", + "mirrorTag": "upstream-v2.63.3", + "mirrorDigestRef": "127.0.0.1:5000/upstream/filebrowser/filebrowser@{{digest}}" + }, + "blockedReason": "File Browser uses an upstream image and must not be modeled as a UniDesk Dockerfile source build. Add a future upstream mirror producer before publishing it through CI." + }, + { + "serviceId": "filebrowser-d601", + "kind": "upstream-image", + "status": "blocked", + "producer": "ci publish-user-service", + "upstream": { + "imageRef": "docker.io/filebrowser/filebrowser:v2.63.3", + "digestRef": "docker.io/filebrowser/filebrowser@sha256:289c5dd677c56662440f26eeb44266ed9746fe563d2e9100f546bff558534d70", + "sourceRepo": "https://github.com/filebrowser/filebrowser", + "sourceRevision": "ca5e249e3c0c94159c2136a0cd431a424eb18472", + "mirrorRepository": "upstream/filebrowser/filebrowser", + "mirrorTag": "upstream-v2.63.3", + "mirrorDigestRef": "127.0.0.1:5000/upstream/filebrowser/filebrowser@{{digest}}" + }, + "blockedReason": "File Browser D601 uses the same pinned upstream image as filebrowser and must not be modeled as a UniDesk Dockerfile source build. Add a future upstream mirror producer before publishing it through CI." } ] } diff --git a/docs/reference/artifact-registry.md b/docs/reference/artifact-registry.md index b8855e20..0089fb90 100644 --- a/docs/reference/artifact-registry.md +++ b/docs/reference/artifact-registry.md @@ -8,6 +8,8 @@ Production CI/CD runtime pinning and release-line boundaries follow `docs/refere The CI-side artifact catalog is root `CI.json`. That file describes only artifact producer inputs and naming; registry consumers still verify the real image labels, manifest digest and live runtime separately. The producer summary contract is owned by `docs/reference/ci.md` and includes `serviceId`, `sourceCommit`, `sourceRepo`, `dockerfile`, `imageRef`, `tag`, `digest` and `digestRef`. +`CI.json` may also record image-only upstream services as `upstream-image` entries with upstream digest and future mirror naming. Those entries are catalog coverage only until a mirror producer exists. Registry CD must not infer a deployable artifact from an upstream-image entry unless the corresponding D601 registry manifest already exists and a reviewed consumer supports that service. + ## Architecture registry 运行在 D601 host/WSL OS 上,由 systemd 管理 Docker Compose 项目: diff --git a/docs/reference/ci.md b/docs/reference/ci.md index 8c972184..e0a6359e 100644 --- a/docs/reference/ci.md +++ b/docs/reference/ci.md @@ -35,7 +35,22 @@ Private repository source authentication is part of the CI contract and follows ## Artifact Catalog And Summary Contract -`CI.json` is the reusable CI artifact catalog. It must remain artifact-only: `serviceId`, `sourceRepo`, `dockerfile`, registry repository naming, tag policy and summary-field semantics are allowed; provider ids, runtime namespaces, ports, compose services, Kubernetes Services, health paths, env, volumes and desired deploy commits are not allowed. `deploy.json` remains the version intent for deployments and must not be replaced by `CI.json`. +`CI.json` is the reusable CI artifact catalog. It must remain artifact-only: `serviceId`, artifact `kind`, producer command, source repository URL, optional repo root, repo-relative Dockerfile path, registry repository naming, upstream image digest/mirror metadata and summary-field semantics are allowed; provider ids, runtime namespaces, ports, compose services, Kubernetes Services, health paths, env, volumes and desired deploy commits are not allowed. `deploy.json` remains the version intent for deployments and must not be replaced by `CI.json`. + +`CI.json` schema version 2 uses these artifact kinds: + +- `source-build`: CI builds a Dockerfile from a pushed Git commit. UniDesk repo Dockerfiles, external Git repositories and Dockerfiles in repository subdirectories all use this kind. +- `upstream-image`: CI records an image-only service that comes from an upstream image digest and optional D601 mirror rule. It is not a Dockerfile build producer. + +Each catalog artifact also has a `status`. `supported` means the matching producer command may start a dry-run or real CI producer action. `blocked` means the service is intentionally listed for coverage but the producer must return a structured blocked result instead of silently building, skipping or falling back. `filebrowser` and `filebrowser-d601` are `upstream-image` blocked entries pinned to `docker.io/filebrowser/filebrowser@sha256:289c5dd677c56662440f26eeb44266ed9746fe563d2e9100f546bff558534d70`; they must not be represented as source-build services. + +Current catalog coverage: + +- `source-build/supported`: `backend-core`, `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, `todo-note`, `code-queue-mgr`, `findjob`, `pipeline`, `met-nonlinear`, `k3sctl-adapter`, `mdtodo`, `claudeqq`. +- `source-build/blocked`: `code-queue`. +- `upstream-image/blocked`: `filebrowser`, `filebrowser-d601`. + +`publish-user-service` reads `source.repo` and `source.dockerfile` from `CI.json`. The command rejects ad hoc `--repo` overrides; the catalog is the only source for producer build inputs. `publish-backend-core` also reads its producer inputs from `CI.json`, while preserving the dedicated backend-core command and Rust/D601 build boundary. Every successful image-producing CI task must expose a common `artifactSummary` contract: @@ -96,11 +111,11 @@ The artifact registry contract and CD consumption path are defined in `docs/refe ## User-Service Artifact Publication -User-service image creation uses the same CI producer boundary as backend-core. Most service identities and Dockerfiles come from the registered `config.json.microservices[]` entry; `frontend` is the reviewed UniDesk UI artifact sample and uses `src/components/frontend/Dockerfile`. The reviewed sample services are `baidu-netdisk`, `decision-center` and `frontend`. +User-service image creation uses the same CI producer boundary as backend-core. Service identities, source repositories, Dockerfiles and image repositories come from root `CI.json`; runtime topology still comes from `config.json`, `deploy.json` and existing manifests. The reviewed sample services are `baidu-netdisk`, `decision-center` and `frontend`, and the catalog now also covers the other source-build services listed above. The CI user-service artifact task must follow these rules: -- Inputs are a pushed full 40-character Git commit and a registered service id. Dirty worktrees, operator-uploaded source trees and local-only commits are not valid artifact sources. +- Inputs are a pushed full 40-character Git commit and a service id registered in `CI.json`. Dirty worktrees, operator-uploaded source trees, command-line repo overrides and local-only commits are not valid artifact sources. - D601 prepares a commit-pinned source export under `/home/ubuntu/.unidesk/ci/user-service-artifacts//` using the existing GitHub SSH deploy identity and node-local provider-gateway WS egress proxy. Tekton consumes that export through a read-only hostPath. - The image is tagged only with the source commit and pushed to the D601 registry as `127.0.0.1:5000/unidesk/:`. The producer must reject third-party registries and must not publish or consume a mutable `latest` tag. - The image must carry `unidesk.ai/service-id`, `unidesk.ai/source-repo`, `unidesk.ai/source-commit` and `unidesk.ai/dockerfile` labels. diff --git a/docs/reference/cicd-standardization.md b/docs/reference/cicd-standardization.md index 78a1dbec..071bb609 100644 --- a/docs/reference/cicd-standardization.md +++ b/docs/reference/cicd-standardization.md @@ -1,6 +1,6 @@ -# CI/CD Standardization Precheck +# CI/CD Standardization -This document classifies CI/CD paths while UniDesk converges on image-artifact delivery. It is a precheck and guardrail document, not a mass-deletion plan. +This document defines the stable split between CI artifact producers, artifact catalog data, legacy guardrails and CD consumers. Detailed Tekton rules remain in `docs/reference/ci.md`; registry runtime and consumers remain in `docs/reference/artifact-registry.md`; user-service delivery order remains in `docs/reference/user-service-delivery.md`. ## Target Shape @@ -14,16 +14,66 @@ The standard release shape is: `backend-core` and D601 `code-queue` may be validated only in dev in this phase. This document must not be used to introduce production deploy validation for either service. +## Artifact Catalog + +Root `CI.json` is the CI producer catalog. It is not a deployment manifest. + +Allowed catalog data: + +- stable `serviceId`; +- artifact `kind`: `source-build` or `upstream-image`; +- producer command: `ci publish-backend-core` or `ci publish-user-service`; +- source repository URL, optional repo root and repo-relative Dockerfile path; +- image repository naming and commit tag policy; +- upstream image digest, upstream source revision and D601 mirror intent for image-only services; +- the required success summary contract. + +Forbidden catalog data: + +- provider IDs; +- runtime namespace, Compose service, Kubernetes Service or health path; +- ports, environment variables, replicas or volumes; +- desired deployment commits or rollout targets. + +Runtime topology belongs to `config.json`, `deploy.json`, existing Kubernetes manifests and the artifact-registry executor. + +## Producer Contract + +`bun scripts/cli.ts ci publish-user-service --service --commit ` reads `source.repo`, `source.dockerfile` and image repository naming from `CI.json`. It rejects command-line repo overrides. Successful source-build producers must label the image with: + +- `unidesk.ai/service-id`; +- `unidesk.ai/source-commit`; +- `unidesk.ai/source-repo`; +- `unidesk.ai/dockerfile`. + +The successful `artifactSummary` must contain `serviceId`, `sourceCommit`, `sourceRepo`, `dockerfile`, `imageRef`, `tag`, `digest` and `digestRef`. + +`blocked` catalog entries must return a structured blocked result. They must not silently skip, build from a dirty worktree, fall back to target-side maintenance deployment, or mutate production. + +## Current Coverage + +Supported source-build artifact producers: + +- `backend-core` through `ci publish-backend-core`; +- `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, `todo-note`, `code-queue-mgr`, `findjob`, `pipeline`, `met-nonlinear`, `k3sctl-adapter`, `mdtodo`, `claudeqq` through `ci publish-user-service`. + +Cataloged but blocked: + +- `code-queue`: source input is known, but this phase allows only dev image validation and not prod-oriented artifact publication. +- `filebrowser` and `filebrowser-d601`: upstream image-only services pinned to `docker.io/filebrowser/filebrowser@sha256:289c5dd677c56662440f26eeb44266ed9746fe563d2e9100f546bff558534d70`; they need a future upstream mirror producer before CI can publish them. + +`code-queue-mgr` is a supported CI producer because the source-build input is known and the remote consumer commit already added a reviewed artifact consumer shape. Its production live apply remains supervisor-gated by deploy/artifact-registry and is not authorized by `CI.json`. + ## Upstream Image Consumers `filebrowser` and `filebrowser-d601` are upstream-image consumers, not source-built UniDesk services. | Service | Upstream image | Source revision | Catalog home | CI Dockerfile build | Digest / mirror strategy | CD validation | | --- | --- | --- | --- | --- | --- | --- | -| `filebrowser` | `docker.io/filebrowser/filebrowser:v2.63.3` | `ca5e249e3c0c94159c2136a0cd431a424eb18472` | `CI.json.upstreamImageConsumers[]` and `config.json.microservices[].repository.artifactSource` | forbidden | resolve tag to `docker.io/filebrowser/filebrowser@sha256:`, then optionally mirror to `127.0.0.1:5000/upstream/filebrowser/filebrowser@sha256:` | pull by digest or mirror digest, verify OCI labels, container image id/digest, and private proxy health | -| `filebrowser-d601` | `docker.io/filebrowser/filebrowser:v2.63.3` | `ca5e249e3c0c94159c2136a0cd431a424eb18472` | `CI.json.upstreamImageConsumers[]` and `config.json.microservices[].repository.artifactSource` | forbidden | same as `filebrowser` | same as `filebrowser` | +| `filebrowser` | `docker.io/filebrowser/filebrowser:v2.63.3` | `ca5e249e3c0c94159c2136a0cd431a424eb18472` | `CI.json.artifacts[]` with `kind=upstream-image` plus `config.json.microservices[].repository.artifactSource` | forbidden | resolve tag to `docker.io/filebrowser/filebrowser@sha256:`, then optionally mirror to `127.0.0.1:5000/upstream/filebrowser/filebrowser@sha256:` | pull by digest or mirror digest, verify OCI labels, container image id/digest, and private proxy health | +| `filebrowser-d601` | `docker.io/filebrowser/filebrowser:v2.63.3` | `ca5e249e3c0c94159c2136a0cd431a424eb18472` | `CI.json.artifacts[]` with `kind=upstream-image` plus `config.json.microservices[].repository.artifactSource` | forbidden | same as `filebrowser` | same as `filebrowser` | -The current precheck could inspect the locally cached image labels and image id, but Docker Hub and registry HTTP requests timed out from this container. Therefore the catalog records `digestPin.status=pending-network-verification`; rollout must remain blocked until a reachable registry path resolves the manifest digest and records the mirror digest. The locally cached image shows `org.opencontainers.image.version=2.63.3`, `org.opencontainers.image.revision=ca5e249e3c0c94159c2136a0cd431a424eb18472`, `linux/amd64`, and image id `sha256:6a4d051140ef9313ad87b443f55ccb1cd6331e7463b4becbec2174b494ea533c`, but a local image id is not a registry digest pin. +The catalog records the resolved upstream digest for the current image. If a future tag refresh cannot resolve the registry manifest digest, rollout must remain blocked until a reachable registry path resolves the manifest digest and records the mirror digest. A local Docker image id is supporting evidence only and not a registry digest pin. ### Upstream Image Evidence @@ -31,11 +81,10 @@ The catalog expression is intentionally minimal and parseable: | Evidence command | Required result shape | | --- | --- | -| `jq '.upstreamImageConsumers[] | {serviceId, upstreamImageRef, digestPin, mirrorStrategy, ciBuild, pullOnlyCdValidation}' CI.json` | both File Browser services show `upstreamImageRef=docker.io/filebrowser/filebrowser:v2.63.3`, `digestPin.required=true`, `digestPin.status=pending-network-verification`, `mirrorStrategy.mode=mirror-after-digest-verification`, `ciBuild.dockerfileBuild=false`, and `publishCommand=null` | +| `jq '.artifacts[] | select(.kind=="upstream-image") | {serviceId, upstream, status}' CI.json` | both File Browser services show `upstream.imageRef=docker.io/filebrowser/filebrowser:v2.63.3`, a sha256 `upstream.digestRef`, `sourceRevision=ca5e249e3c0c94159c2136a0cd431a424eb18472`, mirror intent under `upstream/filebrowser/filebrowser`, and `status=blocked` | | `bun scripts/cli.ts config show` with the File Browser `artifactSource` projection | both services parse as `kind=upstream-image`, `digestPinRequired=true`, `mirrorRepository=127.0.0.1:5000/upstream/filebrowser/filebrowser`, `ciDockerfileBuild=false`, and `pullOnlyCd=true` | -| `docker image inspect filebrowser/filebrowser:v2.63.3` | local cache evidence may show the image id and OCI labels for version `2.63.3` and revision `ca5e249e3c0c94159c2136a0cd431a424eb18472`; this is not a registry digest | -| `docker manifest inspect --verbose docker.io/filebrowser/filebrowser:v2.63.3` | must resolve an upstream manifest digest before rollout; when the registry request times out, rollout remains blocked and the catalog stays at `pending-network-verification` | -| `bun scripts/cli.ts ci publish-user-service --service filebrowser --commit --dry-run` | returns `ok=false` with the message that File Browser is an upstream image consumer and must not be built by Dockerfile CI | +| `docker manifest inspect --verbose docker.io/filebrowser/filebrowser:v2.63.3` | must resolve the upstream manifest digest before rollout; if the registry request times out, rollout remains blocked | +| `bun scripts/cli.ts ci publish-user-service --service filebrowser --commit --dry-run` | returns `ok=false` with `status=blocked`, upstream digest/mirror metadata, and no Dockerfile source build | The digest/mirror dry-run contract is: @@ -70,8 +119,8 @@ Pull-only CD validation must be expressed as concrete checks: ## Guardrails Added -- Upstream-image services are represented in `CI.json.upstreamImageConsumers[]` and in `config.json.microservices[].repository.artifactSource`; they are explicitly outside `CI.json.artifacts[]`. -- `ci publish-user-service` rejects registered `upstream-image` services instead of trying to interpret `repository.dockerfile` as a source Dockerfile. +- Upstream-image services are represented in `CI.json.artifacts[]` with `kind=upstream-image` and in `config.json.microservices[].repository.artifactSource`; they are explicitly outside source-build producers. +- `ci publish-user-service` returns a structured blocked result for registered `upstream-image` services instead of trying to interpret `repository.dockerfile` as a source Dockerfile. - Local-manifest production deploy for reviewed artifact consumers is blocked before source materialization/build, so prod cannot silently fall back to target-side source build or a dirty worktree. - `artifact-registry deploy-backend-core` is demoted to a structured deprecated result; backend-core production CD must enter through `deploy apply --env prod`. @@ -79,7 +128,7 @@ Pull-only CD validation must be expressed as concrete checks: | Guardrail name / result key | Command evidence | Legacy path covered | Deletion status | | --- | --- | --- | --- | -| `upstream-image` CI publish rejection | `bun scripts/cli.ts ci publish-user-service --service filebrowser --commit --dry-run` returns `ok=false` and says not to build an upstream image consumer | File Browser accidentally entering `CI.json.artifacts[]` or Dockerfile CI | keep; deletion of docker-run repair waits for digest/mirror CD | +| `upstream-image` CI publish rejection | `bun scripts/cli.ts ci publish-user-service --service filebrowser --commit --dry-run` returns `ok=false`, `status=blocked`, and upstream metadata | File Browser accidentally entering Dockerfile CI | keep; deletion of docker-run repair waits for digest/mirror CD | | `prod-artifact-consumer-local-manifest-blocked` | `bun scripts/cli.ts deploy apply --file deploy.json --service frontend --dry-run` returns `ok=false`, this error key, and points to `deploy apply --env prod --service --commit ` | prod source-build fallback for reviewed artifact consumers | keep; local manifest mode may still be needed for non-prod/recovery until runbooks are replaced | | `artifact-registry deploy-backend-core` deprecated result | `bun scripts/cli.ts artifact-registry deploy-backend-core --commit ` returns `ok=false`, `deprecated=true`, and replacement `deploy apply --env prod --service backend-core --commit ` | backend-core prod CD bypassing deploy reconciler guardrails | keep name only as compatibility until all callers stop using it | | prod unsupported result for services without artifact consumers | `deploy apply --env prod --service --dry-run` must return unsupported instead of falling back to source build | target-side source build/maintenance-channel prod deploy | keep disabled until service-specific artifact consumers exist | @@ -87,9 +136,11 @@ Pull-only CD validation must be expressed as concrete checks: The guarded-but-not-deletable paths are: `server rebuild backend-core`, `server rebuild frontend`, `server rebuild baidu-netdisk`, provider-gateway protected upgrade, native k3s bootstrap, k3sctl-adapter bridge repair, File Browser provider-local docker-run repair, and D601 dev/backend target-side rollout. They remain because they are bootstrap, recovery, diagnostic, or controlled dev paths; deleting them requires replacement runbooks or reviewed artifact consumers. -## Not Removed Yet +## Safety Boundary -Bootstrap and repair paths remain because they still protect recovery: native k3s initialization, provider-gateway protected upgrade, k3sctl-adapter control bridge repair, main-server Compose maintenance rebuilds, and File Browser docker-run operations. These paths must be replaced by reviewed artifact consumers or explicit recovery runbooks before deletion. +CI may build images, push to the D601 loopback registry and report immutable digests. CI must not run production CD, call `deploy apply` for production, mutate production namespaces, recreate production Compose services or update `deploy.json`. + +backend-core and D601 `code-queue` remain restricted to dev image validation in this phase. Any future production rollout for them must be implemented as an explicit CD consumer change, not as a CI producer side effect. ## Validation Boundary diff --git a/docs/reference/deploy.md b/docs/reference/deploy.md index 936728d7..62743c12 100644 --- a/docs/reference/deploy.md +++ b/docs/reference/deploy.md @@ -144,7 +144,7 @@ The registry contract is defined in `docs/reference/artifact-registry.md`; the C ## Upstream Image Exception -`filebrowser` and `filebrowser-d601` are not source-built UniDesk services and must not be moved into `CI.json.artifacts[]`. Their minimal catalog expression is `CI.json.upstreamImageConsumers[]` plus `config.json.microservices[].repository.artifactSource`: +`filebrowser` and `filebrowser-d601` are not source-built UniDesk services and must not be modeled as Dockerfile producers. Their minimal catalog expression is `CI.json.artifacts[]` entries with `kind=upstream-image` plus `config.json.microservices[].repository.artifactSource`: - upstream image: `docker.io/filebrowser/filebrowser:v2.63.3`; - upstream source revision: `ca5e249e3c0c94159c2136a0cd431a424eb18472`; diff --git a/docs/reference/user-service-delivery.md b/docs/reference/user-service-delivery.md index ed9ff9be..3ad7fa1d 100644 --- a/docs/reference/user-service-delivery.md +++ b/docs/reference/user-service-delivery.md @@ -34,9 +34,10 @@ The default release flow for a user-service change is: - No user-service artifact may rely on a third-party registry as source of truth. - No production deploy may rebuild the source from a dirty worktree. - Commit-pinned image tags are the deployment truth; mutable `latest` tags are not. -- Root `CI.json` is an artifact catalog only. It can list user-service CI build inputs such as `serviceId`, `sourceRepo`, `dockerfile`, image repository naming and the required artifact summary fields; it must not carry runtime topology or replace `deploy.json`. -- The standard CI artifact producer is `bun scripts/cli.ts ci publish-user-service --service --commit `. It accepts only a pushed Git commit and a registered service id, and reports `serviceId`, `sourceCommit`, `sourceRepo`, `dockerfile`, `imageRef`, `tag`, `digest` and `digestRef`. +- Root `CI.json` is an artifact catalog only. It lists CI producer inputs such as `serviceId`, artifact kind, source repository, repo-relative Dockerfile, image repository naming, upstream image digest/mirror metadata and the required artifact summary fields; it must not carry runtime topology or replace `deploy.json`. +- The standard CI artifact producer is `bun scripts/cli.ts ci publish-user-service --service --commit `. It accepts only a pushed Git commit and a service id registered in `CI.json`, reads `source.repo` and `source.dockerfile` from that catalog, rejects ad hoc `--repo` overrides, and reports `serviceId`, `sourceCommit`, `sourceRepo`, `dockerfile`, `imageRef`, `tag`, `digest` and `digestRef`. - The CI artifact producer is not a deploy executor. It must not mutate the production namespace, restart production services, or update `deploy.json`. +- `CI.json` may list `blocked` source-build entries when the source input is known but the publish/CD boundary is not yet reviewed. It may also list `upstream-image` entries for image-only services such as File Browser; those entries pin upstream digest and mirror intent but must not be treated as Dockerfile builds. - Every production release must finish with a manual acceptance step after the automated checks pass. - Multi-service delivery programs may use Code Queue parallelization, but the supervisor must follow `docs/reference/code-queue-supervision.md`: tasks need self-contained prompts, isolated worktrees, bounded queue concurrency, explicit acceptance evidence, and infrastructure defects split into separate follow-up tasks when they block several lanes. @@ -44,11 +45,13 @@ The default release flow for a user-service change is: Some registered user services are intentionally upstream-image consumers instead of source-built services. `filebrowser` and `filebrowser-d601` are in this class. -- They must be cataloged as upstream images, not as `CI.json.artifacts[]` Dockerfile producers. -- `ci publish-user-service` must reject them; there is no UniDesk Dockerfile build for these services. +- They must be cataloged as `CI.json.artifacts[]` entries with `kind=upstream-image`, not as Dockerfile `source-build` producers. +- `ci publish-user-service` must return a structured blocked result for them; there is no UniDesk Dockerfile build for these services. - The release input is an upstream manifest digest or a digest-verified mirror in the D601 registry, not a Git commit tag built by Tekton. - CD must be pull-only and must verify the image identity, OCI labels and service health through the UniDesk private proxy. -- Until the upstream digest has been resolved and mirrored or pinned, File Browser remains a recovery/diagnostic image-only path rather than a standard release path. +- Until the upstream digest has been resolved and mirrored or pinned for a future mirror producer, File Browser remains a recovery/diagnostic image-only path rather than a standard release path. + +The current catalog covers `frontend`, `baidu-netdisk`, `decision-center`, `project-manager`, `oa-event-flow`, `todo-note`, `code-queue-mgr`, `findjob`, `pipeline`, `met-nonlinear`, `k3sctl-adapter`, `mdtodo` and `claudeqq` as supported `publish-user-service` source-build services. `code-queue` is cataloged but blocked by the D601 dev/prod boundary. `filebrowser` and `filebrowser-d601` are cataloged as pinned upstream images, not source builds. ## Frontend Pairing diff --git a/scripts/src/ci-catalog.ts b/scripts/src/ci-catalog.ts new file mode 100644 index 00000000..e9d74fc4 --- /dev/null +++ b/scripts/src/ci-catalog.ts @@ -0,0 +1,266 @@ +import { existsSync, readFileSync } from "node:fs"; + +import { rootPath } from "./config"; + +export type CiArtifactStatus = "supported" | "blocked"; + +export type CiCatalogArtifact = CiSourceBuildCatalogArtifact | CiUpstreamImageCatalogArtifact; + +export interface CiCatalog { + schemaVersion: number; + kind: "ci-artifact-catalog"; + purpose: string; + summaryContract: { + requiredOnSuccess: string[]; + fieldSemantics: Record; + }; + defaults: { + registry: string; + tagTemplate: string; + mutableTagsAllowed: boolean; + runtimeFieldsForbidden: string[]; + }; + artifacts: CiCatalogArtifact[]; +} + +export interface CiSourceBuildCatalogArtifact { + serviceId: string; + kind: "source-build"; + status: CiArtifactStatus; + producer: "ci publish-backend-core" | "ci publish-user-service"; + source: { + repo: string; + dockerfile: string; + root?: string; + }; + image: { + repository: string; + }; + notes?: string; + blockedReason?: string; +} + +export interface CiUpstreamImageCatalogArtifact { + serviceId: string; + kind: "upstream-image"; + status: "blocked"; + producer: "ci publish-user-service"; + upstream: { + imageRef: string; + digestRef: string; + sourceRepo?: string; + sourceRevision?: string; + mirrorRepository: string; + mirrorTag: string; + mirrorDigestRef: string; + }; + notes?: string; + blockedReason: string; +} + +let cachedCatalog: CiCatalog | null = null; + +function asRecord(value: unknown, name: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${name} must be an object`); + } + return value as Record; +} + +function stringField(obj: Record, key: string, path: string): string { + const value = obj[key]; + if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value; +} + +function stringArrayField(obj: Record, key: string, path: string): string[] { + const value = obj[key]; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) { + throw new Error(`${path}.${key} must be an array of non-empty strings`); + } + return value as string[]; +} + +function optionalStringField(obj: Record, key: string, path: string): string | undefined { + const value = obj[key]; + if (value === undefined) return undefined; + if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`); + return value; +} + +function optionalStringArrayField(obj: Record, key: string, path: string): string[] | undefined { + const value = obj[key]; + if (value === undefined) return undefined; + return stringArrayField(obj, key, path); +} + +function optionalBooleanField(obj: Record, key: string, path: string): boolean | undefined { + const value = obj[key]; + if (value === undefined) return undefined; + if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`); + return value; +} + +function requiredCatalogPath(value: string, label: string): string { + if (value.length === 0 || value.startsWith("/") || value.includes("\0") || value.split("/").includes("..")) { + throw new Error(`${label} must be a repo-relative path`); + } + return value; +} + +function requiredImageRepository(value: string, label: string): string { + if (value.length === 0 || value.startsWith("/") || value.includes("..") || value.includes(":") || value.includes("@") || value.includes("latest") || !/^[a-z0-9._/-]+$/u.test(value)) { + throw new Error(`${label} must be a registry repository path without registry host, tag, digest, latest, or uppercase characters`); + } + return value; +} + +function stringRecordField(obj: Record, key: string, path: string): Record { + const value = asRecord(obj[key], `${path}.${key}`); + for (const [entryKey, entryValue] of Object.entries(value)) { + if (typeof entryValue !== "string" || entryValue.length === 0) { + throw new Error(`${path}.${key}.${entryKey} must be a non-empty string`); + } + } + return value as Record; +} + +function validateSourceBuildArtifact(item: Record, index: number): CiSourceBuildCatalogArtifact { + const path = `artifacts[${index}]`; + const source = asRecord(item.source, `${path}.source`); + const image = asRecord(item.image, `${path}.image`); + const status = stringField(item, "status", path); + if (status !== "supported" && status !== "blocked") throw new Error(`${path}.status must be supported or blocked`); + const producer = stringField(item, "producer", path); + if (producer !== "ci publish-backend-core" && producer !== "ci publish-user-service") { + throw new Error(`${path}.producer must be ci publish-backend-core or ci publish-user-service`); + } + const artifact: CiSourceBuildCatalogArtifact = { + serviceId: stringField(item, "serviceId", path), + kind: "source-build", + status, + producer, + source: { + repo: stringField(source, "repo", `${path}.source`), + dockerfile: requiredCatalogPath(stringField(source, "dockerfile", `${path}.source`), `${path}.source.dockerfile`), + ...(source.root === undefined ? {} : { root: requiredCatalogPath(stringField(source, "root", `${path}.source`), `${path}.source.root`) }), + }, + image: { + repository: requiredImageRepository(stringField(image, "repository", `${path}.image`), `${path}.image.repository`), + }, + ...(optionalStringField(item, "notes", path) === undefined ? {} : { notes: optionalStringField(item, "notes", path) }), + ...(optionalStringField(item, "blockedReason", path) === undefined ? {} : { blockedReason: optionalStringField(item, "blockedReason", path) }), + }; + if (artifact.status === "blocked" && artifact.blockedReason === undefined) { + throw new Error(`${path}.blockedReason is required when status=blocked`); + } + if (artifact.status === "supported" && artifact.blockedReason !== undefined) { + throw new Error(`${path}.blockedReason is only allowed when status=blocked`); + } + return artifact; +} + +function validateUpstreamImageArtifact(item: Record, index: number): CiUpstreamImageCatalogArtifact { + const path = `artifacts[${index}]`; + const upstream = asRecord(item.upstream, `${path}.upstream`); + const producer = stringField(item, "producer", path); + if (producer !== "ci publish-user-service") throw new Error(`${path}.producer must be ci publish-user-service`); + const artifact: CiUpstreamImageCatalogArtifact = { + serviceId: stringField(item, "serviceId", path), + kind: "upstream-image", + status: "blocked", + producer, + upstream: { + imageRef: stringField(upstream, "imageRef", `${path}.upstream`), + digestRef: stringField(upstream, "digestRef", `${path}.upstream`), + ...(optionalStringField(upstream, "sourceRepo", `${path}.upstream`) === undefined ? {} : { sourceRepo: optionalStringField(upstream, "sourceRepo", `${path}.upstream`) }), + ...(optionalStringField(upstream, "sourceRevision", `${path}.upstream`) === undefined ? {} : { sourceRevision: optionalStringField(upstream, "sourceRevision", `${path}.upstream`) }), + mirrorRepository: requiredImageRepository(stringField(upstream, "mirrorRepository", `${path}.upstream`), `${path}.upstream.mirrorRepository`), + mirrorTag: stringField(upstream, "mirrorTag", `${path}.upstream`), + mirrorDigestRef: stringField(upstream, "mirrorDigestRef", `${path}.upstream`), + }, + blockedReason: stringField(item, "blockedReason", path), + ...(optionalStringField(item, "notes", path) === undefined ? {} : { notes: optionalStringField(item, "notes", path) }), + }; + return artifact; +} + +function validateCatalogArtifact(item: unknown, index: number): CiCatalogArtifact { + const record = asRecord(item, `artifacts[${index}]`); + const kind = stringField(record, "kind", `artifacts[${index}]`); + if (kind === "source-build") return validateSourceBuildArtifact(record, index); + if (kind === "upstream-image") return validateUpstreamImageArtifact(record, index); + throw new Error(`artifacts[${index}].kind must be source-build or upstream-image`); +} + +function assertUniqueServiceIds(artifacts: CiCatalogArtifact[]): void { + const seen = new Set(); + for (const artifact of artifacts) { + if (seen.has(artifact.serviceId)) throw new Error(`CI.json.artifacts contains duplicate serviceId: ${artifact.serviceId}`); + seen.add(artifact.serviceId); + } +} + +export function loadCiCatalog(): CiCatalog { + if (cachedCatalog !== null) return cachedCatalog; + const path = rootPath("CI.json"); + if (!existsSync(path)) throw new Error(`CI.json not found at ${path}`); + const parsed = asRecord(JSON.parse(readFileSync(path, "utf8")) as unknown, "CI.json"); + const schemaVersion = parsed.schemaVersion; + if (schemaVersion !== 2) throw new Error("CI.json schemaVersion must be 2"); + const kind = stringField(parsed, "kind", "CI.json"); + if (kind !== "ci-artifact-catalog") throw new Error("CI.json kind must be ci-artifact-catalog"); + const summaryContract = asRecord(parsed.summaryContract, "CI.json.summaryContract"); + const defaults = asRecord(parsed.defaults, "CI.json.defaults"); + const artifacts = Array.isArray(parsed.artifacts) ? parsed.artifacts.map((item, index) => validateCatalogArtifact(item, index)) : []; + if (artifacts.length === 0) throw new Error("CI.json.artifacts must not be empty"); + assertUniqueServiceIds(artifacts); + cachedCatalog = { + schemaVersion, + kind, + purpose: stringField(parsed, "purpose", "CI.json"), + summaryContract: { + requiredOnSuccess: stringArrayField(summaryContract, "requiredOnSuccess", "CI.json.summaryContract"), + fieldSemantics: stringRecordField(summaryContract, "fieldSemantics", "CI.json.summaryContract"), + }, + defaults: { + registry: stringField(defaults, "registry", "CI.json.defaults"), + tagTemplate: stringField(defaults, "tagTemplate", "CI.json.defaults"), + mutableTagsAllowed: optionalBooleanField(defaults, "mutableTagsAllowed", "CI.json.defaults") ?? false, + runtimeFieldsForbidden: optionalStringArrayField(defaults, "runtimeFieldsForbidden", "CI.json.defaults") ?? [], + }, + artifacts, + }; + return cachedCatalog; +} + +export function findCiCatalogArtifact(serviceId: string): CiCatalogArtifact | null { + return loadCiCatalog().artifacts.find((artifact) => artifact.serviceId === serviceId) ?? null; +} + +export function supportedSourceBuildArtifactIds(): string[] { + return loadCiCatalog().artifacts + .filter((artifact): artifact is CiSourceBuildCatalogArtifact => artifact.kind === "source-build" && artifact.status === "supported") + .map((artifact) => artifact.serviceId); +} + +export function blockedCatalogArtifactIds(): string[] { + return loadCiCatalog().artifacts.filter((artifact) => artifact.status === "blocked").map((artifact) => artifact.serviceId); +} + +export function catalogSummary(): { + totalArtifacts: number; + sourceBuildArtifacts: number; + supportedSourceBuildArtifacts: number; + upstreamImageArtifacts: number; + blockedArtifacts: number; +} { + const catalog = loadCiCatalog(); + return { + totalArtifacts: catalog.artifacts.length, + sourceBuildArtifacts: catalog.artifacts.filter((artifact) => artifact.kind === "source-build").length, + supportedSourceBuildArtifacts: catalog.artifacts.filter((artifact) => artifact.kind === "source-build" && artifact.status === "supported").length, + upstreamImageArtifacts: catalog.artifacts.filter((artifact) => artifact.kind === "upstream-image").length, + blockedArtifacts: catalog.artifacts.filter((artifact) => artifact.status === "blocked").length, + }; +} diff --git a/scripts/src/ci.ts b/scripts/src/ci.ts index 58a81cb0..df4149e8 100644 --- a/scripts/src/ci.ts +++ b/scripts/src/ci.ts @@ -1,8 +1,9 @@ import { randomUUID } from "node:crypto"; import { existsSync, readFileSync } from "node:fs"; import { posix as posixPath } from "node:path"; +import { blockedCatalogArtifactIds, catalogSummary, findCiCatalogArtifact, loadCiCatalog, supportedSourceBuildArtifactIds, type CiCatalogArtifact, type CiSourceBuildCatalogArtifact, type CiUpstreamImageCatalogArtifact } from "./ci-catalog"; import { runCommand } from "./command"; -import { type UniDeskConfig, type UniDeskMicroserviceConfig, repoRoot, rootPath } from "./config"; +import { type UniDeskConfig, repoRoot, rootPath } from "./config"; import { ensureGithubSshIdentityForProvider, gitSshHttpConnectProxySource } from "./deploy-ssh-identity"; import { startJob } from "./jobs"; import { coreInternalFetch } from "./microservices"; @@ -29,14 +30,6 @@ const ciRuntimeImages = [ "alpine/git:2.45.2", ciCodeQueueImage, ]; -const publishUserServiceArtifactAllowedServiceIds = new Set([ - "baidu-netdisk", - "code-queue-mgr", - "decision-center", - "frontend", - "oa-event-flow", - "project-manager", -]); interface CiOptions { repoUrl: string; @@ -49,6 +42,8 @@ interface CiPublishBackendCoreOptions { commit: string; waitMs: number; sourceHostPath: string; + dockerfile: string; + imageRepository: string; dryRun: boolean; } @@ -59,6 +54,7 @@ interface CiPublishUserServiceArtifactOptions { sourceHostPath: string; serviceId: string; dockerfile: string; + imageRepository: string; dryRun: boolean; } @@ -130,6 +126,7 @@ interface ArtifactSummaryContext { commit: string; repoUrl: string; dockerfile: string; + imageRepository: string; } function stringOption(args: string[], name: string): string | null { @@ -188,6 +185,10 @@ function shellQuote(value: string): string { return `'${value.replace(/'/gu, "'\\''")}'`; } +function safePathToken(value: string): string { + return value.replace(/[^a-z0-9._-]/giu, "-").toLowerCase().replace(/^-+|-+$/gu, "").slice(0, 80) || "artifact"; +} + function repoSshUrl(repoUrl: string): string { if (repoUrl.startsWith("git@")) return repoUrl; if (repoUrl.startsWith("https://github.com/")) { @@ -197,6 +198,10 @@ function repoSshUrl(repoUrl: string): string { return repoUrl; } +function repoNeedsGithubSshIdentity(repoFetchUrl: string): boolean { + return repoFetchUrl.startsWith("git@github.com:"); +} + function requireRepoRelativePath(path: string, label: string): string { if (path.length === 0 || path.startsWith("/") || path.includes("\0") || path.split("/").includes("..")) { throw new Error(`${label} must be a repo-relative path`); @@ -206,35 +211,45 @@ function requireRepoRelativePath(path: string, label: string): string { return normalized; } -function requireSupportedUserService(config: UniDeskConfig, serviceId: string): UniDeskMicroserviceConfig { - if (serviceId === "backend-core") { - throw new Error("backend-core uses ci publish-backend-core; publish-user-service is for registered user services"); +function resolveCatalogArtifact(serviceId: string): CiCatalogArtifact { + const artifact = findCiCatalogArtifact(serviceId); + if (artifact === null) { + const known = loadCiCatalog().artifacts.map((item) => item.serviceId).sort().join(", "); + throw new Error(`unknown CI artifact service: ${serviceId}. Known services: ${known}`); } - const service = config.microservices.find((item) => item.id === serviceId); - if (service === undefined) throw new Error(`unknown user service: ${serviceId}`); - if (service.repository.artifactSource?.kind === "upstream-image") { - throw new Error(`ci publish-user-service does not build ${serviceId}: it is an upstream image consumer (${service.repository.artifactSource.imageRef}). Use the upstream-image digest/mirror governance path; do not add it to Dockerfile CI artifacts.`); - } - const isD601K3sService = service.providerId === d601ProviderId - && service.development.providerId === d601ProviderId - && service.deployment.mode === "k3sctl-managed"; - const isMainServerDirectService = service.providerId === "main-server" - && service.development.providerId === "main-server" - && service.deployment.mode === "unidesk-direct"; - const isMainServerInternalSidecar = service.providerId === "main-server" - && service.development.providerId === "main-server" - && service.deployment.mode === "internal-sidecar"; - if (!isD601K3sService && !isMainServerDirectService && !isMainServerInternalSidecar) { - throw new Error(`ci publish-user-service supports only reviewed k3sctl-managed D601 services or main-server unidesk-direct services; ${serviceId} is ${service.providerId}/${service.deployment.mode}`); - } - return service; + return artifact; } -function frontendArtifactTarget(repoOverride: string | null): { repoUrl: string; dockerfile: string } { - return { - repoUrl: repoOverride ?? "https://github.com/pikasTech/unidesk", - dockerfile: "src/components/frontend/Dockerfile", +function blockedArtifactResult(artifact: CiUpstreamImageCatalogArtifact | CiSourceBuildCatalogArtifact, commit: string, note: string): Record { + const base = { + ok: false, + status: "blocked", + error: "blocked", + serviceId: artifact.serviceId, + commit, + reason: note, + catalogArtifact: artifact, + boundary: "CI catalog marks this service as blocked; it must not be treated as a source-build artifact producer", }; + return artifact.kind === "upstream-image" + ? { + ...base, + upstream: artifact.upstream, + next: [ + `document the upstream image contract in CI.json for ${artifact.serviceId}`, + ], + } + : { + ...base, + next: [ + `unblock ${artifact.serviceId} in CI.json before attempting source-build publication`, + ], + }; +} + +function blockedReason(artifact: CiSourceBuildCatalogArtifact): string { + if (artifact.blockedReason === undefined) throw new Error(`${artifact.serviceId} is blocked in CI.json but has no blockedReason`); + return artifact.blockedReason; } function chunks(value: string, size: number): string[] { @@ -621,6 +636,10 @@ spec: value: ${JSON.stringify(options.repoUrl)} - name: revision value: ${JSON.stringify(options.commit)} + - name: dockerfile + value: ${JSON.stringify(options.dockerfile)} + - name: image-repository + value: ${JSON.stringify(options.imageRepository)} - name: source-host-path value: ${JSON.stringify(options.sourceHostPath)} workspaces: @@ -656,6 +675,8 @@ spec: value: ${JSON.stringify(options.serviceId)} - name: dockerfile value: ${JSON.stringify(options.dockerfile)} + - name: image-repository + value: ${JSON.stringify(options.imageRepository)} - name: source-host-path value: ${JSON.stringify(options.sourceHostPath)} workspaces: @@ -674,18 +695,20 @@ function userServiceArtifactSourceHostPath(serviceId: string, commit: string): s } async function prepareBackendCoreArtifactSource(config: UniDeskConfig, options: CiPublishBackendCoreOptions): Promise> { - const sshIdentity = await ensureGithubSshIdentityForProvider(config, d601ProviderId); - if (!sshIdentity.ok) throw new Error(sshIdentity.detail); - const proxyPython = gitSshHttpConnectProxySource(); const sourceRoot = "/home/ubuntu/.unidesk/ci/backend-core-artifacts"; const sourceHostPath = options.sourceHostPath; const repoCache = "/home/ubuntu/.unidesk/ci/git/unidesk.git"; const repoFetchUrl = repoSshUrl(options.repoUrl); + const sshIdentity = repoNeedsGithubSshIdentity(repoFetchUrl) ? await ensureGithubSshIdentityForProvider(config, d601ProviderId) : null; + if (sshIdentity !== null && !sshIdentity.ok) throw new Error(sshIdentity.detail); + const proxyPython = gitSshHttpConnectProxySource(); + const dockerfile = requireRepoRelativePath(options.dockerfile, "CI.json.artifacts.backend-core.source.dockerfile"); const script = [ "set -euo pipefail", `commit=${shellQuote(options.commit)}`, `repo_url=${shellQuote(options.repoUrl)}`, `repo_fetch_url=${shellQuote(repoFetchUrl)}`, + `dockerfile=${shellQuote(dockerfile)}`, `source_root=${shellQuote(sourceRoot)}`, `source_dir=${shellQuote(sourceHostPath)}`, `repo_cache=${shellQuote(repoCache)}`, @@ -708,7 +731,7 @@ async function prepareBackendCoreArtifactSource(config: UniDeskConfig, options: "git -C \"$repo_cache\" fetch --no-tags origin \"$commit\" || git -C \"$repo_cache\" fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*'", "resolved=$(git -C \"$repo_cache\" rev-parse --verify \"$commit^{commit}\")", "test \"$resolved\" = \"$commit\" || { echo \"backend_core_artifact_resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }", - "git -C \"$repo_cache\" cat-file -e \"$commit:src/components/backend-core/Dockerfile\"", + "git -C \"$repo_cache\" cat-file -e \"$commit:$dockerfile\"", "git -C \"$repo_cache\" cat-file -e \"$commit:src/components/backend-core/src\"", "tmp_dir=\"$source_root/.tmp-$commit-$$\"", "rm -rf \"$tmp_dir\"", @@ -718,7 +741,7 @@ async function prepareBackendCoreArtifactSource(config: UniDeskConfig, options: "printf '%s\\n' \"$repo_url\" > \"$tmp_dir/.unidesk-source-repo\"", "rm -rf \"$source_dir\"", "mv \"$tmp_dir\" \"$source_dir\"", - "test -f \"$source_dir/src/components/backend-core/Dockerfile\"", + "test -f \"$source_dir/$dockerfile\"", "test -d \"$source_dir/src/components/backend-core/src\"", "echo backend_core_artifact_source_host_path=$source_dir", ].join("\n"); @@ -726,13 +749,14 @@ async function prepareBackendCoreArtifactSource(config: UniDeskConfig, options: if (!result.ok) throw new Error(`failed to prepare backend-core source on D601: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`); return { ok: true, - mode: "d601-host-github-ssh-export", + mode: repoNeedsGithubSshIdentity(repoFetchUrl) ? "d601-host-git-ssh-export" : "d601-host-git-https-export", providerId: d601ProviderId, repoUrl: options.repoUrl, repoFetchUrl, commit: options.commit, sourceHostPath, - identity: { + dockerfile, + identity: sshIdentity === null ? null : { fingerprint: sshIdentity.fingerprint, seededFromLocal: sshIdentity.seededFromLocal, }, @@ -741,14 +765,13 @@ async function prepareBackendCoreArtifactSource(config: UniDeskConfig, options: } async function prepareUserServiceArtifactSource(config: UniDeskConfig, options: CiPublishUserServiceArtifactOptions): Promise> { - const sshIdentity = await ensureGithubSshIdentityForProvider(config, d601ProviderId); - if (!sshIdentity.ok) throw new Error(sshIdentity.detail); - const proxyPython = gitSshHttpConnectProxySource(); const sourceRoot = `/home/ubuntu/.unidesk/ci/user-service-artifacts/${options.serviceId}`; const sourceHostPath = options.sourceHostPath; - const repoCache = "/home/ubuntu/.unidesk/ci/git/unidesk.git"; + const repoCache = `/home/ubuntu/.unidesk/ci/git/${safePathToken(options.serviceId)}.git`; const repoFetchUrl = repoSshUrl(options.repoUrl); - const dockerfileDir = posixPath.dirname(options.dockerfile); + const sshIdentity = repoNeedsGithubSshIdentity(repoFetchUrl) ? await ensureGithubSshIdentityForProvider(config, d601ProviderId) : null; + if (sshIdentity !== null && !sshIdentity.ok) throw new Error(sshIdentity.detail); + const proxyPython = gitSshHttpConnectProxySource(); const script = [ "set -euo pipefail", `service_id=${shellQuote(options.serviceId)}`, @@ -756,7 +779,6 @@ async function prepareUserServiceArtifactSource(config: UniDeskConfig, options: `repo_url=${shellQuote(options.repoUrl)}`, `repo_fetch_url=${shellQuote(repoFetchUrl)}`, `dockerfile=${shellQuote(options.dockerfile)}`, - `dockerfile_dir=${shellQuote(dockerfileDir)}`, `source_root=${shellQuote(sourceRoot)}`, `source_dir=${shellQuote(sourceHostPath)}`, `repo_cache=${shellQuote(repoCache)}`, @@ -781,7 +803,6 @@ async function prepareUserServiceArtifactSource(config: UniDeskConfig, options: "resolved=$(git -C \"$repo_cache\" rev-parse --verify \"$commit^{commit}\")", "test \"$resolved\" = \"$commit\" || { echo \"user_service_artifact_resolved_commit_mismatch=$resolved expected=$commit\" >&2; exit 1; }", "git -C \"$repo_cache\" cat-file -e \"$commit:$dockerfile\"", - "git -C \"$repo_cache\" cat-file -e \"$commit:$dockerfile_dir/src\"", "tmp_dir=\"$source_root/.tmp-$commit-$$\"", "rm -rf \"$tmp_dir\"", "mkdir -p \"$tmp_dir\"", @@ -793,14 +814,13 @@ async function prepareUserServiceArtifactSource(config: UniDeskConfig, options: "rm -rf \"$source_dir\"", "mv \"$tmp_dir\" \"$source_dir\"", "test -f \"$source_dir/$dockerfile\"", - "test -d \"$source_dir/$dockerfile_dir/src\"", "echo user_service_artifact_source_host_path=$source_dir", ].join("\n"); const result = await runRemoteBackground(`prepare-${options.serviceId}-source`, script, 300_000); if (!result.ok) throw new Error(`failed to prepare ${options.serviceId} source on D601: ${result.stderr || result.stdout || JSON.stringify(result.raw)}`); return { ok: true, - mode: "d601-host-github-ssh-export", + mode: repoNeedsGithubSshIdentity(repoFetchUrl) ? "d601-host-git-ssh-export" : "d601-host-git-https-export", providerId: d601ProviderId, repoUrl: options.repoUrl, repoFetchUrl, @@ -808,7 +828,7 @@ async function prepareUserServiceArtifactSource(config: UniDeskConfig, options: serviceId: options.serviceId, dockerfile: options.dockerfile, sourceHostPath, - identity: { + identity: sshIdentity === null ? null : { fingerprint: sshIdentity.fingerprint, seededFromLocal: sshIdentity.seededFromLocal, }, @@ -894,7 +914,7 @@ function pipelineRunWaitSucceeded(wait: DispatchResult | null, condition: Pipeli function artifactSummaryDefaults(context: ArtifactSummaryContext): ArtifactSummary { const registry = "127.0.0.1:5000"; - const repository = `${registry}/unidesk/${context.serviceId}`; + const repository = `${registry}/${context.imageRepository}`; return { serviceId: context.serviceId, sourceCommit: context.commit, @@ -917,7 +937,7 @@ function artifactSummaryField(fields: Map, suffix: string): stri function parseArtifactSummaryFromFields(fields: Map, context: ArtifactSummaryContext): ArtifactSummary { const planned = artifactSummaryDefaults(context); const registry = artifactSummaryField(fields, "registry") ?? planned.registry; - const repository = artifactSummaryField(fields, "repository") ?? `${registry}/unidesk/${context.serviceId}`; + const repository = artifactSummaryField(fields, "repository") ?? planned.repository; const tag = artifactSummaryField(fields, "tag") ?? planned.tag; const imageRef = artifactSummaryField(fields, "image") ?? (repository.length > 0 && tag.length > 0 ? `${repository}:${tag}` : planned.imageRef); const digest = artifactSummaryField(fields, "digest"); @@ -1075,9 +1095,10 @@ async function run(options: CiOptions): Promise> { async function publishBackendCoreArtifact(config: UniDeskConfig, options: CiPublishBackendCoreOptions): Promise> { const summaryContext: ArtifactSummaryContext = { serviceId: "backend-core", - dockerfile: "src/components/backend-core/Dockerfile", commit: options.commit, repoUrl: options.repoUrl, + dockerfile: options.dockerfile, + imageRepository: options.imageRepository, }; const plannedArtifact = artifactSummaryDefaults(summaryContext); if (options.dryRun) { @@ -1096,7 +1117,8 @@ async function publishBackendCoreArtifact(config: UniDeskConfig, options: CiPubl repoUrl: options.repoUrl, repoFetchUrl: repoSshUrl(options.repoUrl), commit: options.commit, - dockerfile: "src/components/backend-core/Dockerfile", + dockerfile: options.dockerfile, + imageRepository: options.imageRepository, sourceHostPath: options.sourceHostPath, }, artifact: plannedArtifact.imageRef, @@ -1148,6 +1170,7 @@ async function publishUserServiceArtifact(config: UniDeskConfig, options: CiPubl dockerfile: options.dockerfile, commit: options.commit, repoUrl: options.repoUrl, + imageRepository: options.imageRepository, }; const plannedArtifact = artifactSummaryDefaults(summaryContext); if (options.dryRun) { @@ -1169,6 +1192,7 @@ async function publishUserServiceArtifact(config: UniDeskConfig, options: CiPubl commit: options.commit, serviceId: options.serviceId, dockerfile: options.dockerfile, + imageRepository: options.imageRepository, sourceHostPath: options.sourceHostPath, }, artifact: plannedArtifact.imageRef, @@ -1461,7 +1485,32 @@ async function logs(name: string): Promise> { }; } +function catalogArtifactDescriptor(artifact: CiCatalogArtifact): Record { + if (artifact.kind === "source-build") { + return { + serviceId: artifact.serviceId, + kind: artifact.kind, + status: artifact.status, + producer: artifact.producer, + source: artifact.source, + image: artifact.image, + ...(artifact.notes === undefined ? {} : { notes: artifact.notes }), + ...(artifact.blockedReason === undefined ? {} : { blockedReason: artifact.blockedReason }), + }; + } + return { + serviceId: artifact.serviceId, + kind: artifact.kind, + status: artifact.status, + producer: artifact.producer, + upstream: artifact.upstream, + blockedReason: artifact.blockedReason, + ...(artifact.notes === undefined ? {} : { notes: artifact.notes }), + }; +} + export function ciHelp(): Record { + const catalog = loadCiCatalog(); return { command: "ci install|status|run|publish-backend-core|publish-user-service|run-dev-e2e|logs", description: "Manage the D601 k3s Tekton CI gate. CI may publish commit-pinned image artifacts, but it intentionally does not deploy CD.", @@ -1492,9 +1541,13 @@ export function ciHelp(): Record { userServiceArtifact: { producer: "D601 CI", command: "bun scripts/cli.ts ci publish-user-service --service --commit ", - initiallySupportedServices: ["baidu-netdisk", "decision-center", "frontend"], + supportedServices: supportedSourceBuildArtifactIds().filter((serviceId) => serviceId !== "backend-core"), + blockedServices: blockedCatalogArtifactIds(), registry: "127.0.0.1:5000/unidesk/:", outputFields: ["serviceId", "sourceCommit", "sourceRepo", "dockerfile", "imageRef", "tag", "digest", "digestRef"], + summaryContract: catalog.summaryContract, + catalogSummary: catalogSummary(), + catalog: catalog.artifacts.map(catalogArtifactDescriptor), boundary: "artifact producer only; no prod deploy and no production namespace mutation", frontendNext: [ "bun scripts/cli.ts deploy apply --env dev --service frontend", @@ -1528,37 +1581,82 @@ export async function runCiCommand(config: UniDeskConfig, args: string[]): Promi return run({ repoUrl, revision, waitMs }); } if (action === "publish-backend-core") { - const repoUrl = stringOption(args, "--repo") ?? stringOption(args, "--repo-url") ?? "https://github.com/pikasTech/unidesk"; + if (stringOption(args, "--repo") !== null || stringOption(args, "--repo-url") !== null) { + throw new Error("ci publish-backend-core reads source repo from CI.json; edit CI.json instead of using --repo"); + } const commit = requireFullCommit(stringOption(args, "--commit") ?? stringOption(args, "--revision")); const waitMs = numberOption(args, "--wait-ms", 0); const dryRun = boolFlag(args, "--dry-run"); - return publishBackendCoreArtifact(config, { repoUrl, commit, waitMs, sourceHostPath: backendCoreArtifactSourceHostPath(commit), dryRun }); + const artifact = resolveCatalogArtifact("backend-core"); + if (artifact.kind !== "source-build") throw new Error("backend-core must be modeled as a source-build artifact in CI.json"); + if (artifact.status === "blocked") return blockedArtifactResult(artifact, commit, blockedReason(artifact)); + return publishBackendCoreArtifact(config, { + repoUrl: artifact.source.repo, + commit, + waitMs, + sourceHostPath: backendCoreArtifactSourceHostPath(commit), + dockerfile: artifact.source.dockerfile, + imageRepository: artifact.image.repository, + dryRun, + }); } if (action === "publish-user-service") { const serviceId = requireServiceId(stringOption(args, "--service") ?? stringOption(args, "--service-id")); - const repoOverride = stringOption(args, "--repo") ?? stringOption(args, "--repo-url"); - const target = serviceId === "frontend" - ? frontendArtifactTarget(repoOverride) - : (() => { - const service = requireSupportedUserService(config, serviceId); - return { - repoUrl: repoOverride ?? service.repository.url, - dockerfile: service.repository.dockerfile, - }; - })(); const commit = requireFullCommit(stringOption(args, "--commit") ?? stringOption(args, "--revision")); const waitMs = numberOption(args, "--wait-ms", 0); const dryRun = boolFlag(args, "--dry-run"); - const dockerfile = requireRepoRelativePath(target.dockerfile, serviceId === "frontend" ? "frontend.dockerfile" : `microservices.${serviceId}.repository.dockerfile`); - if (!publishUserServiceArtifactAllowedServiceIds.has(serviceId)) { - throw new Error(`ci publish-user-service currently allows only ${Array.from(publishUserServiceArtifactAllowedServiceIds).join(", ")} until each user-service Dockerfile contract is reviewed`); + if (stringOption(args, "--repo") !== null || stringOption(args, "--repo-url") !== null) { + throw new Error("ci publish-user-service reads source repo from CI.json; edit CI.json instead of using --repo"); + } + const artifact = resolveCatalogArtifact(serviceId); + if (artifact.kind === "source-build" && artifact.serviceId === "backend-core") { + throw new Error("backend-core uses ci publish-backend-core; publish-user-service is for registered user services"); + } + if (artifact.kind === "upstream-image") { + return blockedArtifactResult(artifact, commit, artifact.blockedReason); + } + if (artifact.status === "blocked") { + return blockedArtifactResult(artifact, commit, blockedReason(artifact)); + } + const repoUrl = artifact.source.repo; + const dockerfile = requireRepoRelativePath(artifact.source.dockerfile, `CI.json.artifacts.${serviceId}.source.dockerfile`); + const configService = config.microservices.find((item) => item.id === serviceId); + if (configService !== undefined) { + const isD601K3sService = configService.providerId === d601ProviderId + && configService.development.providerId === d601ProviderId + && configService.deployment.mode === "k3sctl-managed"; + const isD601DirectService = configService.providerId === d601ProviderId + && configService.development.providerId === d601ProviderId + && configService.deployment.mode === "unidesk-direct"; + const isMainServerDirectService = configService.providerId === "main-server" + && configService.development.providerId === "main-server" + && configService.deployment.mode === "unidesk-direct"; + const isMainServerInternalSidecar = configService.providerId === "main-server" + && configService.development.providerId === "main-server" + && configService.deployment.mode === "internal-sidecar"; + if (!isD601K3sService && !isD601DirectService && !isMainServerDirectService && !isMainServerInternalSidecar) { + return { + ok: false, + status: "blocked", + error: "blocked", + serviceId, + commit, + reason: `config.json marks ${serviceId} as ${configService.providerId}/${configService.deployment.mode}, which is outside the reviewed CI artifact producer boundary`, + catalogArtifact: artifact, + configService: { + providerId: configService.providerId, + deploymentMode: configService.deployment.mode, + }, + }; + } } return publishUserServiceArtifact(config, { - repoUrl: target.repoUrl, + repoUrl, commit, waitMs, serviceId, dockerfile, + imageRepository: artifact.image.repository, sourceHostPath: userServiceArtifactSourceHostPath(serviceId, commit), dryRun, }); diff --git a/scripts/src/deploy.ts b/scripts/src/deploy.ts index dd7caae9..237357a3 100644 --- a/scripts/src/deploy.ts +++ b/scripts/src/deploy.ts @@ -139,6 +139,7 @@ const devApplySupportedServiceIds = new Set(["backend-core"]); const devArtifactConsumerServiceIds = new Set(["baidu-netdisk", "code-queue-mgr", "decision-center", "frontend", "oa-event-flow", "project-manager", "todo-note"]); const devArtifactConsumerProdDesiredFallbackServiceIds = new Set(["code-queue-mgr", "oa-event-flow", "project-manager", "todo-note"]); const prodArtifactConsumerServiceIds = new Set(["backend-core", "baidu-netdisk", "code-queue-mgr", "decision-center", "frontend", "oa-event-flow", "project-manager", "todo-note"]); +const prodForbiddenTargetSideBuildServiceIds = new Set(["backend-core", "baidu-netdisk", "decision-center", "frontend"]); const prodArtifactLiveApplyBlockedServiceIds = new Map([ ["code-queue-mgr", "code-queue-mgr is the main-server Code Queue control-plane sidecar; live production apply requires explicit supervisor confirmation."], ["todo-note", "todo-note source is external to this repository and the current checked-in contract cannot prove /api/health deploy.commit/deploy.requestedCommit support."], @@ -2700,6 +2701,22 @@ function prodArtifactUnsupportedResult(services: DeployManifestService[]): Recor }; } +function prodArtifactConsumerLocalManifestResult(services: DeployManifestService[]): Record { + return { + ok: false, + supported: false, + error: "prod-artifact-consumer-local-manifest-blocked", + services: services.map((service) => ({ + id: service.id, + repo: service.repo, + commitId: service.commitId, + replacement: `bun scripts/cli.ts deploy apply --env prod --service ${service.id} --commit ${service.commitId}`, + reason: "production artifact consumers must use the Git-backed environment manifest and D601 registry artifact consumer, not local-manifest target-side source build", + })), + policy: "prod deploy must not silently fall back to a dirty worktree, local manifest, target-side source build, or maintenance-channel deployment", + }; +} + function prodArtifactLiveApplyBlockedResult(services: DeployManifestService[]): Record { return { ok: false, diff --git a/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml b/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml index cd795275..4f06d240 100644 --- a/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml +++ b/src/components/microservices/k3sctl-adapter/k3s/ci/unidesk-ci.pipeline.yaml @@ -320,6 +320,10 @@ spec: type: string - name: revision type: string + - name: dockerfile + type: string + - name: image-repository + type: string - name: app-image type: string default: unidesk-code-queue:dev @@ -370,10 +374,14 @@ spec: script: | #!/bin/sh set -eu + dockerfile="$(params.dockerfile)" case "$(params.revision)" in [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; *) echo "backend_core_artifact_revision_must_be_full_sha=$(params.revision)" >&2; exit 2 ;; esac + case "$dockerfile" in + /*|*..*|""|*latest*) echo "backend_core_artifact_dockerfile_invalid=$dockerfile" >&2; exit 2 ;; + esac mkdir -p "$(workspaces.source.path)/backend-core-artifact-repo" find "$(workspaces.source.path)/backend-core-artifact-repo" -mindepth 1 -maxdepth 1 -exec rm -rf {} + case "$(params.source-host-path)" in @@ -387,9 +395,10 @@ spec: test "$prepared_commit" = "$(params.revision)" cp -a "$source_dir/." "$(workspaces.source.path)/backend-core-artifact-repo/" cd "$(workspaces.source.path)/backend-core-artifact-repo" - test -f src/components/backend-core/Dockerfile + test -f "$dockerfile" test -d src/components/backend-core/src printf '%s\n' "$prepared_commit" | tee "$(workspaces.source.path)/backend-core-artifact-commit.txt" + printf '%s\n' "$dockerfile" | tee "$(workspaces.source.path)/backend-core-artifact-dockerfile.txt" - name: build-and-push image: "$(params.app-image)" imagePullPolicy: Never @@ -420,11 +429,16 @@ spec: #!/usr/bin/env bash set -euo pipefail commit="$(cat "$(workspaces.source.path)/backend-core-artifact-commit.txt")" + dockerfile="$(cat "$(workspaces.source.path)/backend-core-artifact-dockerfile.txt")" registry="$(params.registry)" + image_repository="$(params.image-repository)" test "$registry" = "127.0.0.1:5000" || { echo "backend_core_artifact_registry_must_be_d601_loopback=$registry" >&2; exit 2; } - local_image="unidesk/backend-core:$commit" - registry_image="$registry/unidesk/backend-core:$commit" - repository="$registry/unidesk/backend-core" + case "$image_repository" in + ""|/*|*..*|*:*|*@*|*latest*|*[!a-z0-9._/-]*) echo "backend_core_artifact_image_repository_invalid=$image_repository" >&2; exit 2 ;; + esac + local_image="$image_repository:$commit" + registry_image="$registry/$image_repository:$commit" + repository="$registry/$image_repository" command -v docker docker version >/dev/null docker run --rm --network host rancher/mirrored-library-busybox:1.36.1 wget -q -O- "http://$registry/v2/" >/dev/null @@ -437,10 +451,10 @@ spec: --label "unidesk.ai/service-id=backend-core" \ --label "unidesk.ai/source-repo=$(params.repo-url)" \ --label "unidesk.ai/source-commit=$commit" \ - --label "unidesk.ai/dockerfile=src/components/backend-core/Dockerfile" \ + --label "unidesk.ai/dockerfile=$dockerfile" \ -t "$local_image" \ -t "$registry_image" \ - -f src/components/backend-core/Dockerfile \ + -f "$dockerfile" \ . actual_commit="$(docker image inspect "$registry_image" --format '{{ index .Config.Labels "unidesk.ai/source-commit" }}')" test "$actual_commit" = "$commit" @@ -453,14 +467,14 @@ spec: printf '%s' "backend-core" > "$(results.backend_core_artifact_service_id.path)" printf '%s' "$commit" > "$(results.backend_core_artifact_source_commit.path)" printf '%s' "$(params.repo-url)" > "$(results.backend_core_artifact_source_repo.path)" - printf '%s' "src/components/backend-core/Dockerfile" > "$(results.backend_core_artifact_dockerfile.path)" + printf '%s' "$dockerfile" > "$(results.backend_core_artifact_dockerfile.path)" printf '%s' "$registry" > "$(results.backend_core_artifact_registry.path)" printf '%s' "$repository" > "$(results.backend_core_artifact_repository.path)" printf '%s' "$registry_image" > "$(results.backend_core_artifact_image.path)" printf '%s' "$commit" > "$(results.backend_core_artifact_tag.path)" printf '%s' "$digest" > "$(results.backend_core_artifact_digest.path)" printf '%s' "$digest_ref" > "$(results.backend_core_artifact_digest_ref.path)" - printf 'backend_core_artifact_service_id=backend-core\nbackend_core_artifact_image=%s\nbackend_core_artifact_repository=%s\nbackend_core_artifact_tag=%s\nbackend_core_artifact_digest=%s\nbackend_core_artifact_digest_ref=%s\nbackend_core_artifact_source_commit=%s\nbackend_core_artifact_source_repo=%s\nbackend_core_artifact_dockerfile=src/components/backend-core/Dockerfile\nbackend_core_artifact_registry=%s\nbackend_core_artifact_repo_digests=%s\n' "$registry_image" "$repository" "$commit" "$digest" "$digest_ref" "$commit" "$(params.repo-url)" "$registry" "$repo_digests" + printf 'backend_core_artifact_service_id=backend-core\nbackend_core_artifact_image=%s\nbackend_core_artifact_repository=%s\nbackend_core_artifact_tag=%s\nbackend_core_artifact_digest=%s\nbackend_core_artifact_digest_ref=%s\nbackend_core_artifact_source_commit=%s\nbackend_core_artifact_source_repo=%s\nbackend_core_artifact_dockerfile=%s\nbackend_core_artifact_registry=%s\nbackend_core_artifact_repo_digests=%s\n' "$registry_image" "$repository" "$commit" "$digest" "$digest_ref" "$commit" "$(params.repo-url)" "$dockerfile" "$registry" "$repo_digests" --- apiVersion: tekton.dev/v1 kind: Pipeline @@ -478,6 +492,12 @@ spec: default: https://github.com/pikasTech/unidesk - name: revision type: string + - name: dockerfile + type: string + default: src/components/backend-core/Dockerfile + - name: image-repository + type: string + default: unidesk/backend-core - name: app-image type: string default: unidesk-code-queue:dev @@ -497,6 +517,10 @@ spec: value: "$(params.repo-url)" - name: revision value: "$(params.revision)" + - name: dockerfile + value: "$(params.dockerfile)" + - name: image-repository + value: "$(params.image-repository)" - name: app-image value: "$(params.app-image)" - name: registry @@ -525,6 +549,8 @@ spec: type: string - name: dockerfile type: string + - name: image-repository + type: string - name: app-image type: string default: unidesk-code-queue:dev @@ -607,7 +633,6 @@ spec: cp -a "$source_dir/." "$(workspaces.source.path)/user-service-artifact-repo/" cd "$(workspaces.source.path)/user-service-artifact-repo" test -f "$dockerfile" - test -d "$(dirname "$dockerfile")/src" printf '%s\n' "$prepared_commit" | tee "$(workspaces.source.path)/user-service-artifact-commit.txt" printf '%s\n' "$service_id" | tee "$(workspaces.source.path)/user-service-artifact-service-id.txt" printf '%s\n' "$dockerfile" | tee "$(workspaces.source.path)/user-service-artifact-dockerfile.txt" @@ -644,13 +669,17 @@ spec: service_id="$(cat "$(workspaces.source.path)/user-service-artifact-service-id.txt")" dockerfile="$(cat "$(workspaces.source.path)/user-service-artifact-dockerfile.txt")" registry="$(params.registry)" + image_repository="$(params.image-repository)" test "$registry" = "127.0.0.1:5000" || { echo "user_service_artifact_registry_must_be_d601_loopback=$registry" >&2; exit 2; } + case "$image_repository" in + ""|/*|*..*|*:*|*@*|*latest*|*[!a-z0-9._/-]*) echo "user_service_artifact_image_repository_invalid=$image_repository" >&2; exit 2 ;; + esac case "$commit" in [0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]) ;; *) echo "user_service_artifact_commit_invalid=$commit" >&2; exit 2 ;; esac - local_image="unidesk/$service_id:$commit" - repository="$registry/unidesk/$service_id" + local_image="$image_repository:$commit" + repository="$registry/$image_repository" registry_image="$repository:$commit" command -v docker docker version >/dev/null @@ -711,6 +740,8 @@ spec: type: string - name: dockerfile type: string + - name: image-repository + type: string - name: app-image type: string default: unidesk-code-queue:dev @@ -734,6 +765,8 @@ spec: value: "$(params.service-id)" - name: dockerfile value: "$(params.dockerfile)" + - name: image-repository + value: "$(params.image-repository)" - name: app-image value: "$(params.app-image)" - name: registry