diff --git a/.agents/skills/unidesk-ops/SKILL.md b/.agents/skills/unidesk-ops/SKILL.md index 2175284d..e400820a 100644 --- a/.agents/skills/unidesk-ops/SKILL.md +++ b/.agents/skills/unidesk-ops/SKILL.md @@ -1,6 +1,6 @@ --- name: unidesk-ops -description: UniDesk 手动运维 CLI — `server`、`gc`、PK01 `platform-db postgres`、platform-infra egress proxy 和 k3s dependency proxy benchmark 运维。覆盖主 server 启停、健康检查、swap、日志、Docker 镜像清理、磁盘 GC、服务重建/重启、PK01 host PostgreSQL、D601/D518 proxyserver 视角流量测速和 k3s 真实依赖拉取 benchmark。用户提到 server start、server status、server swap、server rebuild、server restart、gc、磁盘清理、platform-db、PK01 PostgreSQL、egress-proxy traffic、proxy 测速、k3s benchmark、apk/npm/go 拉取测速时使用。 +description: UniDesk 手动运维 CLI — `server`、`gc`、PK01 `platform-db postgres`、platform-infra egress proxy 和 k3s dependency proxy benchmark 运维。覆盖主 server 启停、健康检查、swap、日志、Docker 镜像清理、磁盘 GC、服务重建/重启、PK01 host PostgreSQL、D601/D518 proxyserver 视角流量测速和 k3s 真实依赖拉取 benchmark。用户提到 server start、server status、server swap、server rebuild、server restart、gc、磁盘清理、platform-db、PK01 PostgreSQL、egress-proxy traffic、proxy 测速、k3s benchmark、apk/npm/go/git mirror 拉取测速时使用。 --- # UniDesk Ops @@ -24,7 +24,7 @@ bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark --targets D60 ## K3s Dependency Proxy Benchmark -用于验证 k3s CI/CD 构建出网性能时，必须用真实远程依赖，不用 Cloudflare synthetic。标准 profile 是 `real-deps-500m`：k3s 远程拉 `alpine:3.20`、`node:22-bookworm`、`golang:1.24-bookworm`，然后在 Pod 内跑 `apk add`、`npm install`、`go mod download`；proxyserver 视角累计/窗口流量至少要能支撑 500MiB+ 验收。 +用于验证 k3s CI/CD 构建出网性能时，必须用真实远程依赖，不用 Cloudflare synthetic。标准 profile 是 `real-deps-500m`：k3s 远程拉 `alpine:3.20`、`node:22-bookworm`、`golang:1.24-bookworm`，然后在 Pod 内跑 `apk add`、`npm install`、`go mod download`、`git clone --mirror` 和 `remote update --prune`；proxyserver 视角累计/窗口流量至少要能支撑 500MiB+ 验收。 ```bash # 计划预览：确认 D601/D518、镜像、pull policy、最小 proxy 流量和依赖集合。 @@ -35,11 +35,11 @@ bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark \ bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark \ --targets D601,D518 --profile real-deps-500m --confirm -# 状态表：看 APK/NPM/GO/REAL_DEPS、failure family 和 proxyserver 采样。 +# 状态表：看 APK/NPM/GO/GIT_MIRROR/REAL_DEPS、failure family 和 proxyserver 采样。 bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark status \ --targets D601,D518 --profile real-deps-500m --traffic-sample-seconds 15 -# 日志 drill-down：按 init container 展开 image pull、apk、npm、go 阶段尾部。 +# 日志 drill-down：按 init container 展开 image pull、apk、npm、go、git mirror 阶段尾部。 bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark logs \ --targets D601,D518 --profile real-deps-500m --tail-lines 160 @@ -52,7 +52,7 @@ bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark cleanup \ --targets D601,D518 --profile real-deps-500m --confirm ``` -D601/D518 结果必须分表记录：`STATE`、Job/run、duration、`APK/NPM/GO/REAL_DEPS`、`TRAFFIC_WINDOW`、`TRAFFIC_RATE`、`PROXY_CUM`、`TOP_CLIENT`、`TOP_DEST` 和 `FAILURE`。D518 通过不代表 D601 通过；D601 只证明 k3s/containerd 走到 proxy 也不等于性能达标。未完成 500MiB+ 的真实 k3s image pull + apk/npm/go 测试前，不关闭对应 issue，不合并标记为等待运行面验收的 PR。 +D601/D518 结果必须分表记录：`STATE`、Job/run、duration、`APK/NPM/GO/GIT_MIRROR/REAL_DEPS`、`TRAFFIC_WINDOW`、`TRAFFIC_RATE`、`PROXY_CUM`、`TOP_CLIENT`、`TOP_DEST` 和 `FAILURE`。D518 通过不代表 D601 通过；D601 只证明 k3s/containerd 走到 proxy 也不等于性能达标。未完成 500MiB+ 的真实 k3s image pull + apk/npm/go/git mirror 测试前，不关闭对应 issue，不合并标记为等待运行面验收的 PR。 ## Egress Proxy 运行面修复入口 @@ -69,8 +69,8 @@ D601 若需要让 `sub2api-egress-proxy` 绕开 pod overlay，可在 YAML 中显 常见判读： - `TOP_CLIENT=10.42.0.1` 且 `TOP_DEST=registry-1.docker.io:443`：k3s/containerd image pull 已从 proxyserver 视角可见。 - `TOP_DEST=dl-cdn.alpinelinux.org:443`：Pod 内 `apk` 阶段已走 proxy。 -- `registry.npmjs.org:443`、`proxy.golang.org:443`：分别对应 `npm install` 和 Go module 拉取。 -- `image-pull` failure 表示还卡在 kubelet/containerd 拉镜像；`apk-download`、`npm-download`、`go-download` 分别表示 Pod 内依赖阶段失败。 +- `registry.npmjs.org:443`、`proxy.golang.org:443`、`github.com:443`：分别对应 `npm install`、Go module 拉取和 Git mirror clone/sync。 +- `image-pull` failure 表示还卡在 kubelet/containerd 拉镜像；`apk-download`、`npm-download`、`go-download`、`git-mirror` 分别表示 Pod 内依赖阶段失败。 - proxy 窗口 `0 B/s` 但 active cumulative 增长很慢时，按性能不达标处理；先清理 Job，再继续查上游，不要让慢速 benchmark 长时间占用资源。 ## P0 边界 diff --git a/config/platform-infra/egress-proxy-benchmarks.yaml b/config/platform-infra/egress-proxy-benchmarks.yaml index 80e75a89..58a0a8eb 100644 --- a/config/platform-infra/egress-proxy-benchmarks.yaml +++ b/config/platform-infra/egress-proxy-benchmarks.yaml @@ -6,12 +6,13 @@ metadata: relatedIssues: - 1032 - 1048 + - 1077 profiles: real-deps-500m: enabled: true workload: k3s-real-deps - description: Real k3s dependency egress benchmark; kubelet pulls remote alpine/node/golang images, then pod stages run apk, npm and go downloads through the YAML-declared proxy. + description: Real k3s dependency egress benchmark; kubelet pulls remote alpine/node/golang images, then pod stages run apk, npm, go and git mirror clone/sync through the YAML-declared proxy. image: docker.io/library/alpine:3.20 imagePullPolicy: Always targetOverrides: {} @@ -72,6 +73,10 @@ profiles: - google.golang.org/grpc@v1.69.4 - k8s.io/client-go@v0.32.4 expectedMiB: 180 + gitMirror: + image: docker.io/library/alpine:3.20 + remote: https://github.com/git/git.git + expectedMiB: 30 no-mirror-600m: enabled: true diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 009feaed..d8111df8 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -169,7 +169,7 @@ When target-level `egressProxy.enabled=true`, the D601 target renders an in-clus The egress proxy Deployment may opt into `hostNetwork: true` per target via `config/platform-infra/sub2api.yaml` `targets[].egressProxy.hostNetwork`. When enabled, the manifest renders `hostNetwork: true`, `dnsPolicy: ClusterFirstWithHostNet`, and a RollingUpdate strategy of `maxSurge=0`/`maxUnavailable=1` so the sing-box client bypasses the pod overlay and connects the master upstream directly from the node network; this is the durable fix for a target whose pod-overlay path to the upstream is the throughput bottleneck. It is a per-target YAML decision, not a D601-only default: a target whose pod overlay is already fast enough must keep `hostNetwork: false`, and the `no-host-network` policy check only permits `hostNetwork: true` on the single YAML-declared egress proxy Deployment for a target whose `egressProxy.hostNetwork=true`. Do not generalize one target's hostNetwork experiment to other nodes, and do not leave a one-off `kubectl patch` as the final state; promote or demote hostNetwork only by editing the target YAML and running `platform-infra sub2api apply --target `. -`platform-infra egress-proxy k3s-build-benchmark --targets --profile real-deps-500m` is the production-ready egress proxy throughput acceptance entry. The `real-deps-500m` profile in `config/platform-infra/egress-proxy-benchmarks.yaml` is the only acceptance profile: it renders one Job per target whose kubelet/containerd pulls remote `alpine`, `node` and `golang` images with `imagePullPolicy: Always`, then runs Pod-internal `apk add`, `npm install` and `go mod download` stages through the YAML-declared proxy env. Acceptance requires `STATE=succeeded`, `REAL_DEPS >= 500 MiB` (the profile's `realDeps.minProxyMiB`), image-pull plus apk/npm/go evidence, and a proxyserver-observed cumulative traffic above the profile minimum. Cloudflare synthetic downloads and curl-only probes are bypass diagnostics, never acceptance evidence. Status/logs/traffic are short-polled; a started benchmark is fire-and-forget and must be `cleanup`-ed when it stalls or after acceptance to release k3s resources. D601 and D518 must both pass the same profile: a single node passing does not close a cross-node proxy issue, and an optimization on one target must not regress the other. +`platform-infra egress-proxy k3s-build-benchmark --targets --profile real-deps-500m` is the production-ready egress proxy throughput acceptance entry. The `real-deps-500m` profile in `config/platform-infra/egress-proxy-benchmarks.yaml` is the only acceptance profile: it renders one Job per target whose kubelet/containerd pulls remote `alpine`, `node` and `golang` images with `imagePullPolicy: Always`, then runs Pod-internal `apk add`, `npm install`, `go mod download` and `git clone --mirror` plus `remote update --prune` stages through the YAML-declared proxy env. Acceptance requires `STATE=succeeded`, `REAL_DEPS >= 500 MiB` (the profile's `realDeps.minProxyMiB`), image-pull plus apk/npm/go/git-mirror evidence, and a proxyserver-observed cumulative traffic above the profile minimum. Cloudflare synthetic downloads and curl-only probes are bypass diagnostics, never acceptance evidence. Status/logs/traffic are short-polled; a started benchmark is fire-and-forget and must be `cleanup`-ed when it stalls or after acceptance to release k3s resources. D601 and D518 must both pass the same profile: a single node passing does not close a cross-node proxy issue, and an optimization on one target must not regress the other. `platform-infra sub2api validate --target D601 --full` must prove the proxy Deployment/Service is ready and that an app pod can complete the YAML-declared health probe through the proxy. This target-level injection does not by itself bind manually created Sub2API accounts to that proxy; account tests and account-specific upstream transports still need a YAML-declared `manualAccounts.protected[].proxyBinding` when the account must avoid direct egress. Proxy credentials, subscription contents, and generated proxy configs are Secret material and must not be printed. If a direct D601-to-upstream TLS/SNI path is reset, do not leave a one-off plain HTTP CONNECT or JS proxy as the durable fix; use a mature encrypted proxy source, currently master `shadowsocks-rust` plus D601 `sing-box`, through YAML/compose. diff --git a/project-management/PJ2026-01/specs/PJ2026-01060310-real-k3s-deps-proxy-benchmark.md b/project-management/PJ2026-01/specs/PJ2026-01060310-real-k3s-deps-proxy-benchmark.md index 454d1634..3a5520f4 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060310-real-k3s-deps-proxy-benchmark.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060310-real-k3s-deps-proxy-benchmark.md @@ -4,12 +4,13 @@ This SPEC covers pikasTech/unidesk#1048. It supersedes synthetic Cloudflare download evidence for proxy acceleration decisions and adds a real k3s dependency benchmark profile named `real-deps-500m`. -The benchmark must prove the target k3s cluster can use the platform-infra egress proxy for real dependency acquisition. It has four required stages: +The benchmark must prove the target k3s cluster can use the platform-infra egress proxy for real dependency acquisition. It has five required stages: - Kubernetes image pull: kubelet/containerd must directly pull remote `alpine`, `node`, and `golang` images with `imagePullPolicy: Always`. - Pod `apk add`: the Alpine stage must fetch packages from upstream apk repositories through proxy environment variables. - Pod `npm install`: the Node stage must install packages from `https://registry.npmjs.org/` through the proxy. - Pod `go mod download`: the Go stage must download modules through `GOPROXY=https://proxy.golang.org,direct` and the proxy. +- Pod Git mirror sync/clone: the Git stage must run `git clone --mirror` against the configured public remote, then `remote update --prune`, with Git HTTP(S) proxy settings bound to the YAML-declared proxy. If the Kubernetes image pull stage fails, the benchmark result is not an application dependency failure; it is an image-pull proxy failure in the k3s/containerd path and must be fixed there. @@ -22,15 +23,16 @@ The `real-deps-500m` profile renders a multi-stage Kubernetes Job: - `initContainer/apk-add`: image `docker.io/library/alpine:3.20`. - `initContainer/npm-install`: image `docker.io/library/node:22-bookworm`. - `initContainer/go-download`: image `docker.io/library/golang:1.24-bookworm`. +- `initContainer/git-mirror`: image and remote declared by `realDeps.gitMirror`. - `container/summary`: emits a bounded result marker after all init containers finish. -All three init containers receive the YAML-declared `sub2api-egress-proxy` service URL through `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and lowercase variants. The image pull itself happens before Pod process execution; therefore image pull proxy evidence must come from the k3s/containerd path and proxyserver-side traffic sampling, not from in-container env alone. +All dependency init containers receive the YAML-declared `sub2api-egress-proxy` service URL through `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and lowercase variants. The image pull itself happens before Pod process execution; therefore image pull proxy evidence must come from the k3s/containerd path and proxyserver-side traffic sampling, not from in-container env alone. ## Observability The source of truth for traffic is `platform-infra egress-proxy traffic --target --sample-seconds N`. Benchmark status may include this traffic sample. The final evidence table must include proxyserver window bytes/rate/cumulative bytes, top client, and top destination. -For image pull traffic, the observed proxy client may be the node/k3s/containerd path rather than the benchmark Pod IP. For `apk`, `npm`, and `go` stages, the observed proxy client should correspond to the benchmark Pod network path. This distinction must be preserved in issue evidence. +For image pull traffic, the observed proxy client may be the node/k3s/containerd path rather than the benchmark Pod IP. For `apk`, `npm`, `go`, and `git-mirror` stages, the observed proxy client should correspond to the benchmark Pod network path. This distinction must be preserved in issue evidence. Status output must classify failures into at least: @@ -38,6 +40,7 @@ Status output must classify failures into at least: - `apk-download`: Pod started but apk fetch/install failed. - `npm-download`: Pod started but npm install failed. - `go-download`: Pod started but Go module download failed. +- `git-mirror`: Pod started but Git mirror clone/sync failed. - `none`: all stages succeeded. ## Boundaries @@ -47,15 +50,15 @@ This benchmark must not: - use Cloudflare speed-test downloads as acceptance evidence; - install Node or Go only as a substitute for Kubernetes pulling `node`/`golang` images; - rewrite apk/npm/go sources to regional mirrors; -- use HWLAB source repositories, Tekton, Argo, git-mirror, or previous build caches; +- use HWLAB source repositories, Tekton, Argo, the production git-mirror service, or previous build caches; - hide image pull failures behind local image overrides. -`payloadMiB: 500` in the `real-deps-500m` profile means the minimum proxyserver-observed traffic required for acceptance. The Pod result marker may report apk/npm/go workspace sizes, but those sizes do not replace proxyserver traffic evidence because image pull bytes are outside the Pod filesystem. +`payloadMiB: 500` in the `real-deps-500m` profile means the minimum proxyserver-observed traffic required for acceptance. The Pod result marker may report apk/npm/go/git-mirror workspace sizes, but those sizes do not replace proxyserver traffic evidence because image pull bytes are outside the Pod filesystem. ## Acceptance - `bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark --targets D601,D518 --profile real-deps-500m --dry-run` prints both node plans and the remote image set. - `--confirm` creates one unique Job per node and returns immediately. -- `status --traffic-sample-seconds 15` reports Job state, `image-pull`/`apk`/`npm`/`go` failure family when applicable, and proxyserver traffic columns. -- D601 and D518 both have final rows with target, profile, job, state, duration, apk MiB, npm MiB, go MiB, proxy traffic window/rate/cumulative, top client, top destination, and failure family. +- `status --traffic-sample-seconds 15` reports Job state, `image-pull`/`apk`/`npm`/`go`/`git-mirror` failure family when applicable, and proxyserver traffic columns. +- D601 and D518 both have final rows with target, profile, job, state, duration, apk MiB, npm MiB, go MiB, git mirror MiB, proxy traffic window/rate/cumulative, top client, top destination, and failure family. - Acceptance requires at least 500 MiB of proxyserver-observed traffic per successful node run. If a node cannot reach that point because image pull fails, the issue remains open until the k3s/containerd image pull proxy path is fixed or a blocker is explicitly documented. diff --git a/scripts/src/platform-infra-k3s-build-benchmark.ts b/scripts/src/platform-infra-k3s-build-benchmark.ts index c4530ea0..c6111974 100644 --- a/scripts/src/platform-infra-k3s-build-benchmark.ts +++ b/scripts/src/platform-infra-k3s-build-benchmark.ts @@ -74,6 +74,11 @@ interface K3sRealDepsSpec { modules: readonly string[]; expectedMiB: number; }; + gitMirror: { + image: string; + remote: string; + expectedMiB: number; + }; } interface K3sBuildBenchmarkTargetOverride { @@ -113,6 +118,7 @@ interface TargetStatus { apkMiB: number | null; npmMiB: number | null; goMiB: number | null; + gitMirrorMiB: number | null; realDepsMiB: number | null; failureFamily: string; logTail: string; @@ -276,6 +282,7 @@ function statusBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchma status.apkMiB === null ? "-" : `${status.apkMiB}MiB`, status.npmMiB === null ? "-" : `${status.npmMiB}MiB`, status.goMiB === null ? "-" : `${status.goMiB}MiB`, + status.gitMirrorMiB === null ? "-" : `${status.gitMirrorMiB}MiB`, status.realDepsMiB === null ? "-" : `${status.realDepsMiB}MiB`, status.traffic === undefined ? "-" : bytes(status.traffic.windowBytes), status.traffic === undefined ? "-" : rate(status.traffic.rateBps), @@ -294,7 +301,7 @@ function statusBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchma renderedText: [ "PLATFORM-INFRA K3S BUILD BENCHMARK STATUS", "", - ...table(["TARGET", "PROFILE", "STATE", "JOB", "DURATION", "OUTPUT", "DOWNLOAD", "APK", "NPM", "GO", "REAL_DEPS", "TRAFFIC_WINDOW", "TRAFFIC_RATE", "PROXY_CUM", "TOP_CLIENT", "TOP_DEST", "FAILURE"], rows), + ...table(["TARGET", "PROFILE", "STATE", "JOB", "DURATION", "OUTPUT", "DOWNLOAD", "APK", "NPM", "GO", "GIT_MIRROR", "REAL_DEPS", "TRAFFIC_WINDOW", "TRAFFIC_RATE", "PROXY_CUM", "TOP_CLIENT", "TOP_DEST", "FAILURE"], rows), ...logSections, "", "NEXT", @@ -371,7 +378,7 @@ function renderDryRun(plans: readonly TargetPlan[], options: K3sBuildBenchmarkOp function dryRunImages(profile: K3sBuildBenchmarkProfile): string { if (profile.workload === "k3s-real-deps" && profile.realDeps !== undefined) { - return [profile.realDeps.apk.image, profile.realDeps.npm.image, profile.realDeps.go.image].join(","); + return [profile.realDeps.apk.image, profile.realDeps.npm.image, profile.realDeps.go.image, profile.realDeps.gitMirror.image].join(","); } return profile.image; } @@ -383,13 +390,13 @@ function dryRunPullPolicy(profile: K3sBuildBenchmarkProfile): string { function dependencySummary(profile: K3sBuildBenchmarkProfile): string { if (profile.workload === "k3s-real-deps" && profile.realDeps !== undefined) { - return `apk~${profile.realDeps.apk.expectedMiB} npm~${profile.realDeps.npm.expectedMiB} go~${profile.realDeps.go.expectedMiB}`; + return `apk~${profile.realDeps.apk.expectedMiB} npm~${profile.realDeps.npm.expectedMiB} go~${profile.realDeps.go.expectedMiB} gitMirror~${profile.realDeps.gitMirror.expectedMiB}`; } return `${profile.dependencyDownload.expectedMiB}MiB`; } function dryRunDetail(profile: K3sBuildBenchmarkProfile | undefined): string { - if (profile?.workload === "k3s-real-deps") return "kubelet-image-pull + apk/npm/go through proxy"; + if (profile?.workload === "k3s-real-deps") return "kubelet-image-pull + apk/npm/go/git-mirror through proxy"; return "no-mirror emptyDir unique-job"; } @@ -491,7 +498,8 @@ function realDepsJobManifest(target: Sub2ApiTargetConfig, profile: K3sBuildBench "unidesk.ai/workload": profile.workload, "unidesk.ai/min-proxy-mib": String(realDeps.minProxyMiB), "unidesk.ai/image-pull-mode": "kubelet-containerd", - "unidesk.ai/remote-images": [realDeps.apk.image, realDeps.npm.image, realDeps.go.image].join(","), + "unidesk.ai/remote-images": [realDeps.apk.image, realDeps.npm.image, realDeps.go.image, realDeps.gitMirror.image].join(","), + "unidesk.ai/git-mirror-remote": realDeps.gitMirror.remote, }, }, spec: { @@ -543,6 +551,19 @@ function realDepsJobManifest(target: Sub2ApiTargetConfig, profile: K3sBuildBench ], volumeMounts: [{ name: "work", mountPath: "/work" }], }, + { + name: "git-mirror", + image: realDeps.gitMirror.image, + imagePullPolicy: realDeps.imagePullPolicy, + command: ["/bin/sh", "-lc"], + args: [realDepsGitMirrorScript()], + env: [ + ...commonEnv, + { name: "GIT_MIRROR_IMAGE", value: realDeps.gitMirror.image }, + { name: "GIT_MIRROR_REMOTE", value: realDeps.gitMirror.remote }, + ], + volumeMounts: [{ name: "work", mountPath: "/work" }], + }, ], containers: [{ name: "summary", @@ -630,23 +651,53 @@ printf 'UNIDESK_K3S_REAL_DEPS_STAGE {"stage":"go","ok":true,"image":"%s","downlo `; } +function realDepsGitMirrorScript(): string { + return `set -eu +mkdir -p /work/stages /work/git-mirror +printf 'UNIDESK_K3S_REAL_DEPS_EVENT stage=git-mirror target=%s profile=%s run=%s image=%s remote=%s\\n' "$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$GIT_MIRROR_IMAGE" "$GIT_MIRROR_REMOTE" +if ! command -v git >/dev/null 2>&1; then + if ! command -v apk >/dev/null 2>&1; then + echo "git-runtime-missing image=$GIT_MIRROR_IMAGE" >&2 + exit 127 + fi + if grep -R -E 'npmmirror|daocloud|aliyun|tuna|ustc|huaweicloud' /etc/apk/repositories >/tmp/git-apk-mirror-check.out 2>/dev/null; then + cat /tmp/git-apk-mirror-check.out >&2 + echo "unexpected apk mirror in git mirror base image" >&2 + exit 42 + fi + apk update + apk add --no-cache git ca-certificates +fi +git --version +git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=120 clone --mirror --filter=blob:none "$GIT_MIRROR_REMOTE" /work/git-mirror/repo.git +git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" --git-dir=/work/git-mirror/repo.git remote update --prune +git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" --git-dir=/work/git-mirror/repo.git remote -v +git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" ls-remote --heads "$GIT_MIRROR_REMOTE" >/work/git-mirror/ls-remote-heads.txt +git_mib="$(du -sk /work/git-mirror/repo.git /work/git-mirror/ls-remote-heads.txt 2>/dev/null | awk '{s+=$1} END {printf "%d", int((s+1023)/1024)}')" +printf 'gitMirrorMiB=%s\\n' "$git_mib" > /work/stages/git-mirror.env +printf 'UNIDESK_K3S_REAL_DEPS_STAGE {"stage":"git-mirror","ok":true,"image":"%s","remote":"%s","mirrorMiB":%s}\\n' "$GIT_MIRROR_IMAGE" "$GIT_MIRROR_REMOTE" "$git_mib" +`; +} + function realDepsSummaryScript(realDeps: K3sRealDepsSpec): string { return `set -eu apkMiB=0 npmMiB=0 goMiB=0 +gitMirrorMiB=0 realDepsStartedEpoch="$(date +%s)" realDepsStartedAt="$(date -u +%Y-%m-%dT%H:%M:%SZ)" [ -f /work/stages/apk.env ] && . /work/stages/apk.env [ -f /work/stages/npm.env ] && . /work/stages/npm.env [ -f /work/stages/go.env ] && . /work/stages/go.env +[ -f /work/stages/git-mirror.env ] && . /work/stages/git-mirror.env completed_epoch="$(date +%s)" completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -real_deps_mib=$((apkMiB + npmMiB + goMiB)) +real_deps_mib=$((apkMiB + npmMiB + goMiB + gitMirrorMiB)) duration=$((completed_epoch - realDepsStartedEpoch)) pod_ip="$(hostname -i 2>/dev/null | awk '{print $1}')" -printf 'UNIDESK_K3S_REAL_DEPS_RESULT {"ok":true,"target":"%s","profile":"%s","runId":"%s","startedAt":"%s","completedAt":"%s","durationSeconds":%s,"podIp":"%s","apkMiB":%s,"npmMiB":%s,"goMiB":%s,"realDepsMiB":%s,"minProxyMiB":%s,"imagePullMode":"kubelet-containerd","apkImage":"%s","npmImage":"%s","goImage":"%s"}\\n' \\ - "$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$realDepsStartedAt" "$completed_at" "$duration" "$pod_ip" "$apkMiB" "$npmMiB" "$goMiB" "$real_deps_mib" "$MIN_PROXY_MIB" ${shQuote(realDeps.apk.image)} ${shQuote(realDeps.npm.image)} ${shQuote(realDeps.go.image)} +printf 'UNIDESK_K3S_REAL_DEPS_RESULT {"ok":true,"target":"%s","profile":"%s","runId":"%s","startedAt":"%s","completedAt":"%s","durationSeconds":%s,"podIp":"%s","apkMiB":%s,"npmMiB":%s,"goMiB":%s,"gitMirrorMiB":%s,"realDepsMiB":%s,"minProxyMiB":%s,"imagePullMode":"kubelet-containerd","apkImage":"%s","npmImage":"%s","goImage":"%s","gitMirrorImage":"%s","gitMirrorRemote":"%s"}\\n' \\ + "$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$realDepsStartedAt" "$completed_at" "$duration" "$pod_ip" "$apkMiB" "$npmMiB" "$goMiB" "$gitMirrorMiB" "$real_deps_mib" "$MIN_PROXY_MIB" ${shQuote(realDeps.apk.image)} ${shQuote(realDeps.npm.image)} ${shQuote(realDeps.go.image)} ${shQuote(realDeps.gitMirror.image)} ${shQuote(realDeps.gitMirror.remote)} `; } @@ -952,6 +1003,8 @@ elif "npm-install" in tail_text and ("npm ERR!" in tail_text or "EAI_AGAIN" in t failure_family = "npm-download" elif "go-download" in tail_text and ("terminated exit=" in tail_text or "i/o timeout" in tail_text or "connection refused" in tail_text or "no such host" in tail_text or "proxyconnect tcp" in tail_text or "TLS handshake timeout" in tail_text or "go-runtime-missing" in tail_text): failure_family = "go-download" +elif "git-mirror" in tail_text and ("terminated exit=" in tail_text or "fatal:" in tail_text or "Could not resolve host" in tail_text or "Failed to connect" in tail_text or "connection refused" in tail_text or "proxyconnect tcp" in tail_text or "TLS handshake timeout" in tail_text or "git-runtime-missing" in tail_text): + failure_family = "git-mirror" elif "curl:" in tail_text or "urllib.error.HTTPError" in tail_text or "urllib.error.URLError" in tail_text: failure_family = "dependency-download" elif "payload-too-small" in tail_text: @@ -1032,6 +1085,7 @@ function normalizeStatus(plan: TargetPlan, parsed: unknown, result: CommandResul apkMiB: null, npmMiB: null, goMiB: null, + gitMirrorMiB: null, realDepsMiB: null, failureFamily: result.timedOut ? "transport-timeout" : state, logTail: (result.stderr || result.stdout).slice(-4000), @@ -1057,6 +1111,7 @@ function normalizeStatus(plan: TargetPlan, parsed: unknown, result: CommandResul apkMiB: nullableNumber(jobResult.apkMiB), npmMiB: nullableNumber(jobResult.npmMiB), goMiB: nullableNumber(jobResult.goMiB), + gitMirrorMiB: nullableNumber(jobResult.gitMirrorMiB), realDepsMiB: nullableNumber(jobResult.realDepsMiB), failureFamily: text(data.failureFamily, data.ok === true ? "none" : state === "running" || state === "pending" ? "in-progress" : text(data.reason, "unknown")), logTail: text(data.logTail, result.stderr.slice(-2000)), @@ -1081,6 +1136,7 @@ function blockedStatus(plan: TargetPlan, profile: string): TargetStatus { apkMiB: null, npmMiB: null, goMiB: null, + gitMirrorMiB: null, realDepsMiB: null, failureFamily: plan.blocker ?? "blocked", logTail: plan.detail ?? "", @@ -1151,6 +1207,7 @@ function realDepsSpec(raw: Record, path: string): K3sRealDepsSp const apk = asRecord(raw.apk, `${path}.apk`); const npm = asRecord(raw.npm, `${path}.npm`); const go = asRecord(raw.go, `${path}.go`); + const gitMirror = asRecord(raw.gitMirror, `${path}.gitMirror`); return { minProxyMiB: integerField(raw, "minProxyMiB", path), imagePullPolicy: imagePullPolicyField(raw, "imagePullPolicy", path), @@ -1171,6 +1228,11 @@ function realDepsSpec(raw: Record, path: string): K3sRealDepsSp modules: stringArrayField(go, "modules", `${path}.go`), expectedMiB: integerField(go, "expectedMiB", `${path}.go`), }, + gitMirror: { + image: stringField(gitMirror, "image", `${path}.gitMirror`), + remote: stringField(gitMirror, "remote", `${path}.gitMirror`), + expectedMiB: integerField(gitMirror, "expectedMiB", `${path}.gitMirror`), + }, }; }