From 2873a367d26204ffb2da34a089aa5f8db11eb32c Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 16 May 2026 09:51:59 +0000 Subject: [PATCH] fix(deploy): use provider ws egress build cache --- docs/reference/codex-deploy.md | 2 +- docs/reference/deploy.md | 9 +++++--- docs/reference/provider-gateway.md | 4 ++-- scripts/src/deploy.ts | 35 +++++++++++++++++++++--------- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/docs/reference/codex-deploy.md b/docs/reference/codex-deploy.md index 7ed249b1..37a2ee51 100644 --- a/docs/reference/codex-deploy.md +++ b/docs/reference/codex-deploy.md @@ -21,7 +21,7 @@ bun scripts/cli.ts job status --tail-bytes 30000 1. 在 D601 的 deploy cache 中 `git fetch` remote，并用 `git archive ` 导出 tracked files 到一次性 export 目录。 2. 用 `rsync --delete` 同步导出的 repo 到 `/home/ubuntu/cq-deploy`，保留 `.state/`、`logs/`、`.git/`、`node_modules/` 和 `dist/`。 -3. 在 D601 用目标 Docker daemon 的本地 BuildKit builder 构建 `unidesk-code-queue:d601`，复用 D601 上已有基础镜像和 layer cache；provider-gateway egress proxy 只作为本次 build 的环境变量与 build-arg 注入，并配合本次 build 的 `--network host` 让 RUN 阶段访问 D601 宿主 loopback proxy，不能污染 D601 宿主 Docker/HTTP proxy 配置。 +3. 在 D601 用目标 Docker daemon 的本地 BuildKit builder 构建 `unidesk-code-queue:d601`，复用 D601 上已有基础镜像、inline cache 和 Code Queue build-base；provider-gateway WS egress 是唯一允许的构建代理通道，只作为本次 build 的环境变量与 build-arg 注入，并配合本次 build 的 `--network host` 让 RUN 阶段访问 D601 宿主 loopback proxy，不能污染 D601 宿主 Docker/HTTP proxy 配置，不能新建 SSH SOCKS、公网 master proxy 或直连 fallback。 4. `docker save` 镜像并导入 k3s containerd：`docker exec -i unidesk-v8s-server ctr -n k8s.io images import -`。 5. `kubectl apply -f src/components/microservices/v3sctl-adapter/v3s/code-queue.k8s.yaml`，其中包含 Code Queue 和 `d601-tcp-egress-gateway`。 6. 将解析后的 40 位 remote commit 写入 `deployment/code-queue` 的 `CODE_QUEUE_DEPLOY_COMMIT` / `CODE_QUEUE_DEPLOY_REQUESTED_COMMIT`，并记录到 Deployment annotation。 diff --git a/docs/reference/deploy.md b/docs/reference/deploy.md index 89f7476e..0ee80e35 100644 --- a/docs/reference/deploy.md +++ b/docs/reference/deploy.md @@ -48,16 +48,19 @@ Each target fetches the remote repository, resolves the requested commit to a fu ## One-Shot Build Proxy -Target-side Docker builds that need external network access use a one-shot build proxy scope to the main server network environment. The build path must not mutate host-global proxy settings: +Target-side Docker builds that need external network access use a one-shot proxy scope through provider-gateway WS egress. Provider targets connect only to their node-local provider-gateway egress endpoint, normally `http://127.0.0.1:18789`; provider-gateway carries the TCP stream over the already-authenticated provider WebSocket to the main server, and the main server opens the final outbound TCP connection. This is the only allowed proxy channel for provider-side deploy builds. The build path must not mutate host-global proxy settings: - Do not edit `/etc/docker/daemon.json`. - Do not edit shell profiles or global Docker CLI config. - Do not leave long-lived host `HTTP_PROXY`, `HTTPS_PROXY` or `ALL_PROXY`. - Do not silently fall back to target local direct internet. +- Do not create a separate SSH SOCKS proxy, public master proxy port, or direct backend-core/provider-ingress connection for Docker build egress. -The standard implementation first uses the target Docker daemon's local BuildKit builder so target-side base image and layer caches are reused. Proxy variables are scoped to the current build process and passed as matching `--build-arg` values for Dockerfile `RUN` steps; they are not written to daemon or shell configuration. Provider targets also use `docker buildx build --network host` so `127.0.0.1:` inside `RUN` resolves to the target host's loopback proxy. If a service later needs an isolated `docker-container` builder, it may use one only as a service-specific fallback and must still log proxy resolution, proxy probe result and builder cleanup. The default path should not discard target-local image cache by creating a fresh builder for every deploy. +The standard implementation first uses the target Docker daemon's local BuildKit builder so target-side base image and layer caches are reused. Proxy variables are scoped to the current build process and passed as matching `--build-arg` values for Dockerfile `RUN` steps; they are not written to daemon or shell configuration. Provider targets also use `docker buildx build --network host` so `127.0.0.1:` inside `RUN` resolves to the target host's loopback provider-gateway egress proxy. Each deploy build must log the proxy channel and probe result, for example `target_build_proxy=provider-gateway-ws-egress:http://127.0.0.1:18789` and `target_build_proxy_probe=ok`. -Provider targets should use their local provider-gateway egress proxy endpoint when available, such as `http://127.0.0.1:18789`. Main server targets may build without a proxy unless a service explicitly requires one. +Build cache is part of the deployment contract, not an optimization left to Docker defaults. The deploy reconciler must pass inline BuildKit cache metadata (`--cache-to type=inline`) and import the current target image as cache source when it exists (`--cache-from `). Dockerfiles that intentionally expose a warm build-base argument, such as Code Queue's `CODE_QUEUE_BASE_IMAGE`, may use the target-local `-build-base` image to avoid re-running large apt/npm/Playwright setup layers; this is still target-local build cache and must be logged as `target_build_base_image=-build-base`. If a service later needs an isolated `docker-container` builder or a local cache directory backend, it may use one only as a service-specific fallback and must still log proxy resolution, proxy probe result, cache source, cache destination and builder cleanup. The default path must not discard target-local image cache by creating a fresh builder for every deploy. + +Main server targets may build without a proxy unless a service explicitly requires one. Provider targets must not bypass provider-gateway WS egress for GitHub, Debian apt, npm, Playwright, model downloads or any other external build dependency. ## Deployment Executors diff --git a/docs/reference/provider-gateway.md b/docs/reference/provider-gateway.md index fb1992df..f5489f3d 100644 --- a/docs/reference/provider-gateway.md +++ b/docs/reference/provider-gateway.md @@ -92,11 +92,11 @@ provider ingress 是唯一允许公网暴露的 provider 连接接口，当前 ## Egress Proxy -provider-gateway 可以提供 egress HTTP CONNECT 代理，用于让 Code Queue、Pipeline runner 等节点侧执行环境通过既有 provider WebSocket 通道出网。代理默认监听容器内 `0.0.0.0:18789`，节点部署必须只发布为宿主 loopback `127.0.0.1:18789->18789/tcp`，不得开放公网端口；普通 Docker 执行容器可通过同一私有 Docker network 访问 provider-gateway 容器名，v3s/k8s Pod 必须通过显式 Kubernetes Service/EndpointSlice 暴露同节点 provider-gateway 私有 endpoint，例如 D601 Code Queue 使用 `d601-provider-egress-proxy.unidesk.svc.cluster.local:18789`，不得把该 egress Service 当作业务 HTTP 入口。代理只负责把本地 CONNECT/absolute HTTP 请求转换为 `egress_tcp_open`、`egress_tcp_data`、`egress_tcp_close` 消息；backend-core 在主 server 侧建立真实 TCP 连接并把数据回传，避免 D601 等计算节点本地网络不可达时卡死 Codex/Git/NPM。 +provider-gateway 可以提供 egress HTTP CONNECT 代理，用于让 Code Queue、Pipeline runner、target-side Docker build 等节点侧执行环境通过既有 provider WebSocket 通道出网。代理默认监听容器内 `0.0.0.0:18789`，节点部署必须只发布为宿主 loopback `127.0.0.1:18789->18789/tcp`，不得开放公网端口；普通 Docker 执行容器可通过同一私有 Docker network 访问 provider-gateway 容器名，v3s/k8s Pod 必须通过显式 Kubernetes Service/EndpointSlice 暴露同节点 provider-gateway 私有 endpoint，例如 D601 Code Queue 使用 `d601-provider-egress-proxy.unidesk.svc.cluster.local:18789`，不得把该 egress Service 当作业务 HTTP 入口。代理只负责把本地 CONNECT/absolute HTTP 请求转换为 `egress_tcp_open`、`egress_tcp_data`、`egress_tcp_close` 消息；backend-core 在主 server 侧建立真实 TCP 连接并把数据回传，避免 D601 等计算节点本地网络不可达时卡死 Codex/Git/NPM/apt/Playwright。 该能力属于 provider-gateway 通道能力，register/heartbeat 的 `unideskCapabilities` 必须包含 `network.egress-proxy`，labels 必须上报 `providerGatewayEgressProxy*` 状态。不得再为某个用户服务单独注册伪 provider 来实现出网代理；否则节点列表会出现虚假 provider，且代理、统计、升级路径会形成多套通道。代理健康检查使用 `GET /__unidesk/egress-proxy/health`，返回 `connected`、`providerId`、`activeTunnels` 和监听端口；业务服务自己的 `/health` 应把该结果作为排障证据透出。 -egress proxy 的长期边界是“统一 provider 通道，不引入第二控制面”。backend-core 只接受在线 provider socket 上的 `egress_tcp_*` 消息，并在该 socket 关闭时销毁全部对应 TCP relay；provider-gateway 只维护本地 HTTP proxy 与 WebSocket 消息映射，不保存业务状态，不参与任务调度、统计或节点注册以外的控制面。执行容器、用户服务和 Pipeline runner 不允许直接连接 backend-core provider ingress，也不允许携带 provider token 自行注册；需要出网时只能连接同节点 provider-gateway 的私有 proxy endpoint。当前 v3s/k8s Code Queue 通过 `d601-provider-egress-proxy` Kubernetes Service 连接 D601 provider-gateway egress endpoint，这是 Pod 内的出网入口，不是业务 HTTP 代理入口，也不能替代 Kubernetes API service proxy。 +egress proxy 的长期边界是“统一 provider 通道，不引入第二控制面”。backend-core 只接受在线 provider socket 上的 `egress_tcp_*` 消息，并在该 socket 关闭时销毁全部对应 TCP relay；provider-gateway 只维护本地 HTTP proxy 与 WebSocket 消息映射，不保存业务状态，不参与任务调度、统计或节点注册以外的控制面。执行容器、用户服务、Pipeline runner 和 provider-side deploy build 不允许直接连接 backend-core provider ingress，也不允许携带 provider token 自行注册；需要出网时只能连接同节点 provider-gateway 的私有 proxy endpoint。当前 v3s/k8s Code Queue 通过 `d601-provider-egress-proxy` Kubernetes Service 连接 D601 provider-gateway egress endpoint，这是 Pod 内的出网入口，不是业务 HTTP 代理入口，也不能替代 Kubernetes API service proxy。部署构建同样不得新建 SSH SOCKS、公网 master proxy 或宿主全局代理；构建脚本只能把 provider-gateway WS egress 作为短生命周期环境变量和 Docker build-arg 注入，并配合目标节点本地 BuildKit/image cache 避免重复下载大依赖层。 故障语义必须显式，不允许静默 fallback。provider-gateway 到 backend-core 的 WebSocket 未连接时，本地 proxy 必须返回 503；执行容器不能自动绕过到 D601 本地直连公网、外部公共代理或主 server 公网 HTTP 端口。`NO_PROXY` 只用于 PostgreSQL、OA Event Flow、ClaudeQQ、frontend/backend-core 内网代理、provider-gateway health 等明确内网链路，不能把 GitHub、模型 API、npm registry 等外部目标加入绕过列表。`hyueapi.com` 是明确的模型 API 例外：该上游会拒绝 provider-gateway egress proxy 出口，Code Queue 必须用 `CODE_QUEUE_EGRESS_PROXY_NO_PROXY` / `NO_PROXY` 将 `hyueapi.com,.hyueapi.com` 配成直连，其它模型 API 仍不得默认绕过 proxy。验收必须同时证明 provider-gateway labels、业务服务 `/health` 和执行容器内 `curl -I https://...` 都走同一 proxy path，hyueapi 例外则以 Code Queue `/health.egressProxy.noProxy` 和目标任务成功完成作为证据。 diff --git a/scripts/src/deploy.ts b/scripts/src/deploy.ts index 7d7a3711..9b88485d 100644 --- a/scripts/src/deploy.ts +++ b/scripts/src/deploy.ts @@ -81,7 +81,7 @@ const k8sNamespace = "unidesk"; const k8sContainer = "unidesk-v8s-server"; const k8sKubeconfig = "/home/ubuntu/cq-deploy/.state/v8s/kubeconfig"; const v3sDeployDir = "/home/ubuntu/cq-deploy"; -const dockerProxyUrl = "http://127.0.0.1:18789"; +const providerGatewayWsEgressProxyUrl = "http://127.0.0.1:18789"; function nowIso(): string { return new Date().toISOString(); @@ -276,14 +276,26 @@ function k8sManifestPath(service: UniDeskMicroserviceConfig): string { function sourceProxyPrelude(service: UniDeskMicroserviceConfig): string { if (targetIsMain(service)) return ""; return [ - `build_proxy=${shellQuote(dockerProxyUrl)}`, + `build_proxy=${shellQuote(providerGatewayWsEgressProxyUrl)}`, "export HTTP_PROXY=\"$build_proxy\" HTTPS_PROXY=\"$build_proxy\" ALL_PROXY=\"$build_proxy\"", "export NO_PROXY=\"localhost,127.0.0.1,::1,host.docker.internal\"", "curl -fsSI --max-time 20 -x \"$build_proxy\" https://github.com >/dev/null", + "echo target_build_proxy=provider-gateway-ws-egress:$build_proxy", "echo target_build_proxy_probe=ok", ].join("\n"); } +function buildCachePrelude(dockerfileVariable: string): string[] { + return [ + "cache_args=(--cache-to type=inline --build-arg BUILDKIT_INLINE_CACHE=1)", + "if docker image inspect \"$image\" >/dev/null 2>&1; then cache_args+=(--cache-from \"$image\"); echo target_build_cache_from_image=$image; else echo target_build_cache_from_image=missing:$image; fi", + "echo target_build_cache_to=inline", + "base_args=()", + "build_base_image=\"${image}-build-base\"", + `if grep -Eq '^ARG[[:space:]]+CODE_QUEUE_BASE_IMAGE([=[:space:]]|$)' "${dockerfileVariable}" 2>/dev/null; then if docker image inspect "$build_base_image" >/dev/null 2>&1; then base_args=(--build-arg "CODE_QUEUE_BASE_IMAGE=$build_base_image"); echo target_build_base_image=$build_base_image; else echo target_build_base_image=default; fi; else echo target_build_base_image=unsupported; fi`, + ]; +} + function prepareSourceScript(service: UniDeskMicroserviceConfig, desired: DeployManifestService, exportDir: string): string { if (targetIsMain(service) && desired.repo === "https://github.com/pikasTech/unidesk") { return [ @@ -363,21 +375,23 @@ function buildImageScript(service: UniDeskMicroserviceConfig, desired: DeployMan ? [] : [ "--network", "host", - "--build-arg", `HTTP_PROXY=${dockerProxyUrl}`, - "--build-arg", `HTTPS_PROXY=${dockerProxyUrl}`, - "--build-arg", `ALL_PROXY=${dockerProxyUrl}`, + "--build-arg", `HTTP_PROXY=${providerGatewayWsEgressProxyUrl}`, + "--build-arg", `HTTPS_PROXY=${providerGatewayWsEgressProxyUrl}`, + "--build-arg", `ALL_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", "NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal", ]; return [ "set -euo pipefail", sourceProxyPrelude(service), `image=${shellQuote(image)}`, + `dockerfile=${shellQuote(dockerfile)}`, "docker buildx version >/dev/null", "builder_args=()", "if docker buildx inspect --builder default >/dev/null 2>&1; then builder_args=(--builder default); echo target_build_builder=default; else echo target_build_builder=implicit; fi", "docker buildx inspect \"${builder_args[@]}\" --bootstrap || true", "echo target_build_builder_cleanup=not-required", - `docker buildx build "\${builder_args[@]}" --load ${[...proxyBuildArgs, ...commonArgs].map(shellQuote).join(" ")}`, + ...buildCachePrelude("$dockerfile"), + `docker buildx build "\${builder_args[@]}" --load "\${cache_args[@]}" "\${base_args[@]}" ${[...proxyBuildArgs, ...commonArgs].map(shellQuote).join(" ")}`, "docker image inspect \"$image\" --format 'image_id={{.Id}} labels={{json .Config.Labels}}'", ].filter((line) => line.length > 0).join("\n"); } @@ -427,9 +441,9 @@ function buildDirectImageScript(service: UniDeskMicroserviceConfig, desired: Dep ? [] : [ "--network", "host", - "--build-arg", `HTTP_PROXY=${dockerProxyUrl}`, - "--build-arg", `HTTPS_PROXY=${dockerProxyUrl}`, - "--build-arg", `ALL_PROXY=${dockerProxyUrl}`, + "--build-arg", `HTTP_PROXY=${providerGatewayWsEgressProxyUrl}`, + "--build-arg", `HTTPS_PROXY=${providerGatewayWsEgressProxyUrl}`, + "--build-arg", `ALL_PROXY=${providerGatewayWsEgressProxyUrl}`, "--build-arg", "NO_PROXY=localhost,127.0.0.1,::1,host.docker.internal", ]; const labelArgs = [ @@ -445,11 +459,12 @@ function buildDirectImageScript(service: UniDeskMicroserviceConfig, desired: Dep "builder_args=()", "if docker buildx inspect --builder default >/dev/null 2>&1; then builder_args=(--builder default); echo target_build_builder=default; else echo target_build_builder=implicit; fi", "docker buildx inspect \"${builder_args[@]}\" --bootstrap || true", + ...buildCachePrelude("$dockerfile_abs"), "compose_build_args=()", "while IFS= read -r item; do [ -n \"$item\" ] && compose_build_args+=(--build-arg \"$item\"); done < \"$build_args_file\"", "target_args=()", "if [ -n \"$build_target\" ]; then target_args=(--target \"$build_target\"); fi", - `docker buildx build "\${builder_args[@]}" --load ${[...proxyBuildArgs, ...labelArgs].map(shellQuote).join(" ")} "\${compose_build_args[@]}" "\${target_args[@]}" --progress=plain -t "$image" -f "$dockerfile_abs" "$build_context"`, + `docker buildx build "\${builder_args[@]}" --load "\${cache_args[@]}" "\${base_args[@]}" ${[...proxyBuildArgs, ...labelArgs].map(shellQuote).join(" ")} "\${compose_build_args[@]}" "\${target_args[@]}" --progress=plain -t "$image" -f "$dockerfile_abs" "$build_context"`, "docker image inspect \"$image\" --format 'image_id={{.Id}} labels={{json .Config.Labels}}'", ].filter((line) => line.length > 0).join("\n"); }