feat: add git mirror proxy benchmark stage
This commit is contained in:
Codex
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: unidesk-ops
|
||||
description: UniDesk 手动运维 CLI — `server`、`gc`、PK01 `platform-db postgres`、platform-infra egress proxy 和 k3s dependency proxy benchmark 运维。覆盖主 server 启停、健康检查、swap、日志、Docker 镜像清理、磁盘 GC、服务重建/重启、PK01 host PostgreSQL、D601/D518 proxyserver 视角流量测速和 k3s 真实依赖拉取 benchmark。用户提到 server start、server status、server swap、server rebuild、server restart、gc、磁盘清理、platform-db、PK01 PostgreSQL、egress-proxy traffic、proxy 测速、k3s benchmark、apk/npm/go 拉取测速时使用。
|
||||
description: UniDesk 手动运维 CLI — `server`、`gc`、PK01 `platform-db postgres`、platform-infra egress proxy 和 k3s dependency proxy benchmark 运维。覆盖主 server 启停、健康检查、swap、日志、Docker 镜像清理、磁盘 GC、服务重建/重启、PK01 host PostgreSQL、D601/D518 proxyserver 视角流量测速和 k3s 真实依赖拉取 benchmark。用户提到 server start、server status、server swap、server rebuild、server restart、gc、磁盘清理、platform-db、PK01 PostgreSQL、egress-proxy traffic、proxy 测速、k3s benchmark、apk/npm/go/git mirror 拉取测速时使用。
|
||||
---
|
||||
|
||||
# UniDesk Ops
|
||||
@@ -24,7 +24,7 @@ bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark --targets D60
|
||||
|
||||
## K3s Dependency Proxy Benchmark
|
||||
|
||||
用于验证 k3s CI/CD 构建出网性能时，必须用真实远程依赖，不用 Cloudflare synthetic。标准 profile 是 `real-deps-500m`：k3s 远程拉 `alpine:3.20`、`node:22-bookworm`、`golang:1.24-bookworm`，然后在 Pod 内跑 `apk add`、`npm install`、`go mod download`；proxyserver 视角累计/窗口流量至少要能支撑 500MiB+ 验收。
|
||||
用于验证 k3s CI/CD 构建出网性能时，必须用真实远程依赖，不用 Cloudflare synthetic。标准 profile 是 `real-deps-500m`：k3s 远程拉 `alpine:3.20`、`node:22-bookworm`、`golang:1.24-bookworm`，然后在 Pod 内跑 `apk add`、`npm install`、`go mod download`、`git clone --mirror` 和 `remote update --prune`；proxyserver 视角累计/窗口流量至少要能支撑 500MiB+ 验收。
|
||||
|
||||
```bash
|
||||
# 计划预览：确认 D601/D518、镜像、pull policy、最小 proxy 流量和依赖集合。
|
||||
@@ -35,11 +35,11 @@ bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark \
|
||||
bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark \
|
||||
--targets D601,D518 --profile real-deps-500m --confirm
|
||||
|
||||
# 状态表：看 APK/NPM/GO/REAL_DEPS、failure family 和 proxyserver 采样。
|
||||
# 状态表：看 APK/NPM/GO/GIT_MIRROR/REAL_DEPS、failure family 和 proxyserver 采样。
|
||||
bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark status \
|
||||
--targets D601,D518 --profile real-deps-500m --traffic-sample-seconds 15
|
||||
|
||||
# 日志 drill-down：按 init container 展开 image pull、apk、npm、go 阶段尾部。
|
||||
# 日志 drill-down：按 init container 展开 image pull、apk、npm、go、git mirror 阶段尾部。
|
||||
bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark logs \
|
||||
--targets D601,D518 --profile real-deps-500m --tail-lines 160
|
||||
|
||||
@@ -52,7 +52,7 @@ bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark cleanup \
|
||||
--targets D601,D518 --profile real-deps-500m --confirm
|
||||
```
|
||||
|
||||
D601/D518 结果必须分表记录：`STATE`、Job/run、duration、`APK/NPM/GO/REAL_DEPS`、`TRAFFIC_WINDOW`、`TRAFFIC_RATE`、`PROXY_CUM`、`TOP_CLIENT`、`TOP_DEST` 和 `FAILURE`。D518 通过不代表 D601 通过；D601 只证明 k3s/containerd 走到 proxy 也不等于性能达标。未完成 500MiB+ 的真实 k3s image pull + apk/npm/go 测试前，不关闭对应 issue，不合并标记为等待运行面验收的 PR。
|
||||
D601/D518 结果必须分表记录：`STATE`、Job/run、duration、`APK/NPM/GO/GIT_MIRROR/REAL_DEPS`、`TRAFFIC_WINDOW`、`TRAFFIC_RATE`、`PROXY_CUM`、`TOP_CLIENT`、`TOP_DEST` 和 `FAILURE`。D518 通过不代表 D601 通过；D601 只证明 k3s/containerd 走到 proxy 也不等于性能达标。未完成 500MiB+ 的真实 k3s image pull + apk/npm/go/git mirror 测试前，不关闭对应 issue，不合并标记为等待运行面验收的 PR。
|
||||
|
||||
## Egress Proxy 运行面修复入口
|
||||
|
||||
@@ -69,8 +69,8 @@ D601 若需要让 `sub2api-egress-proxy` 绕开 pod overlay，可在 YAML 中显
|
||||
常见判读：
|
||||
- `TOP_CLIENT=10.42.0.1` 且 `TOP_DEST=registry-1.docker.io:443`：k3s/containerd image pull 已从 proxyserver 视角可见。
|
||||
- `TOP_DEST=dl-cdn.alpinelinux.org:443`：Pod 内 `apk` 阶段已走 proxy。
|
||||
- `registry.npmjs.org:443`、`proxy.golang.org:443`：分别对应 `npm install` 和 Go module 拉取。
|
||||
- `image-pull` failure 表示还卡在 kubelet/containerd 拉镜像；`apk-download`、`npm-download`、`go-download` 分别表示 Pod 内依赖阶段失败。
|
||||
- `registry.npmjs.org:443`、`proxy.golang.org:443`、`github.com:443`：分别对应 `npm install`、Go module 拉取和 Git mirror clone/sync。
|
||||
- `image-pull` failure 表示还卡在 kubelet/containerd 拉镜像；`apk-download`、`npm-download`、`go-download`、`git-mirror` 分别表示 Pod 内依赖阶段失败。
|
||||
- proxy 窗口 `0 B/s` 但 active cumulative 增长很慢时，按性能不达标处理；先清理 Job，再继续查上游，不要让慢速 benchmark 长时间占用资源。
|
||||
|
||||
## P0 边界
|
||||
|
||||
@@ -6,12 +6,13 @@ metadata:
|
||||
relatedIssues:
|
||||
- 1032
|
||||
- 1048
|
||||
- 1077
|
||||
|
||||
profiles:
|
||||
real-deps-500m:
|
||||
enabled: true
|
||||
workload: k3s-real-deps
|
||||
description: Real k3s dependency egress benchmark; kubelet pulls remote alpine/node/golang images, then pod stages run apk, npm and go downloads through the YAML-declared proxy.
|
||||
description: Real k3s dependency egress benchmark; kubelet pulls remote alpine/node/golang images, then pod stages run apk, npm, go and git mirror clone/sync through the YAML-declared proxy.
|
||||
image: docker.io/library/alpine:3.20
|
||||
imagePullPolicy: Always
|
||||
targetOverrides: {}
|
||||
@@ -72,6 +73,10 @@ profiles:
|
||||
- google.golang.org/grpc@v1.69.4
|
||||
- k8s.io/client-go@v0.32.4
|
||||
expectedMiB: 180
|
||||
gitMirror:
|
||||
image: docker.io/library/alpine:3.20
|
||||
remote: https://github.com/git/git.git
|
||||
expectedMiB: 30
|
||||
|
||||
no-mirror-600m:
|
||||
enabled: true
|
||||
|
||||
@@ -169,7 +169,7 @@ When target-level `egressProxy.enabled=true`, the D601 target renders an in-clus
|
||||
|
||||
The egress proxy Deployment may opt into `hostNetwork: true` per target via `config/platform-infra/sub2api.yaml` `targets[].egressProxy.hostNetwork`. When enabled, the manifest renders `hostNetwork: true`, `dnsPolicy: ClusterFirstWithHostNet`, and a RollingUpdate strategy of `maxSurge=0`/`maxUnavailable=1` so the sing-box client bypasses the pod overlay and connects the master upstream directly from the node network; this is the durable fix for a target whose pod-overlay path to the upstream is the throughput bottleneck. It is a per-target YAML decision, not a D601-only default: a target whose pod overlay is already fast enough must keep `hostNetwork: false`, and the `no-host-network` policy check only permits `hostNetwork: true` on the single YAML-declared egress proxy Deployment for a target whose `egressProxy.hostNetwork=true`. Do not generalize one target's hostNetwork experiment to other nodes, and do not leave a one-off `kubectl patch` as the final state; promote or demote hostNetwork only by editing the target YAML and running `platform-infra sub2api apply --target <id>`.
|
||||
|
||||
`platform-infra egress-proxy k3s-build-benchmark --targets <ids> --profile real-deps-500m` is the production-ready egress proxy throughput acceptance entry. The `real-deps-500m` profile in `config/platform-infra/egress-proxy-benchmarks.yaml` is the only acceptance profile: it renders one Job per target whose kubelet/containerd pulls remote `alpine`, `node` and `golang` images with `imagePullPolicy: Always`, then runs Pod-internal `apk add`, `npm install` and `go mod download` stages through the YAML-declared proxy env. Acceptance requires `STATE=succeeded`, `REAL_DEPS >= 500 MiB` (the profile's `realDeps.minProxyMiB`), image-pull plus apk/npm/go evidence, and a proxyserver-observed cumulative traffic above the profile minimum. Cloudflare synthetic downloads and curl-only probes are bypass diagnostics, never acceptance evidence. Status/logs/traffic are short-polled; a started benchmark is fire-and-forget and must be `cleanup`-ed when it stalls or after acceptance to release k3s resources. D601 and D518 must both pass the same profile: a single node passing does not close a cross-node proxy issue, and an optimization on one target must not regress the other.
|
||||
`platform-infra egress-proxy k3s-build-benchmark --targets <ids> --profile real-deps-500m` is the production-ready egress proxy throughput acceptance entry. The `real-deps-500m` profile in `config/platform-infra/egress-proxy-benchmarks.yaml` is the only acceptance profile: it renders one Job per target whose kubelet/containerd pulls remote `alpine`, `node` and `golang` images with `imagePullPolicy: Always`, then runs Pod-internal `apk add`, `npm install`, `go mod download` and `git clone --mirror` plus `remote update --prune` stages through the YAML-declared proxy env. Acceptance requires `STATE=succeeded`, `REAL_DEPS >= 500 MiB` (the profile's `realDeps.minProxyMiB`), image-pull plus apk/npm/go/git-mirror evidence, and a proxyserver-observed cumulative traffic above the profile minimum. Cloudflare synthetic downloads and curl-only probes are bypass diagnostics, never acceptance evidence. Status/logs/traffic are short-polled; a started benchmark is fire-and-forget and must be `cleanup`-ed when it stalls or after acceptance to release k3s resources. D601 and D518 must both pass the same profile: a single node passing does not close a cross-node proxy issue, and an optimization on one target must not regress the other.
|
||||
|
||||
`platform-infra sub2api validate --target D601 --full` must prove the proxy Deployment/Service is ready and that an app pod can complete the YAML-declared health probe through the proxy. This target-level injection does not by itself bind manually created Sub2API accounts to that proxy; account tests and account-specific upstream transports still need a YAML-declared `manualAccounts.protected[].proxyBinding` when the account must avoid direct egress. Proxy credentials, subscription contents, and generated proxy configs are Secret material and must not be printed. If a direct D601-to-upstream TLS/SNI path is reset, do not leave a one-off plain HTTP CONNECT or JS proxy as the durable fix; use a mature encrypted proxy source, currently master `shadowsocks-rust` plus D601 `sing-box`, through YAML/compose.
|
||||
|
||||
|
||||
+10
-7
@@ -4,12 +4,13 @@
|
||||
|
||||
This SPEC covers pikasTech/unidesk#1048. It supersedes synthetic Cloudflare download evidence for proxy acceleration decisions and adds a real k3s dependency benchmark profile named `real-deps-500m`.
|
||||
|
||||
The benchmark must prove the target k3s cluster can use the platform-infra egress proxy for real dependency acquisition. It has four required stages:
|
||||
The benchmark must prove the target k3s cluster can use the platform-infra egress proxy for real dependency acquisition. It has five required stages:
|
||||
|
||||
- Kubernetes image pull: kubelet/containerd must directly pull remote `alpine`, `node`, and `golang` images with `imagePullPolicy: Always`.
|
||||
- Pod `apk add`: the Alpine stage must fetch packages from upstream apk repositories through proxy environment variables.
|
||||
- Pod `npm install`: the Node stage must install packages from `https://registry.npmjs.org/` through the proxy.
|
||||
- Pod `go mod download`: the Go stage must download modules through `GOPROXY=https://proxy.golang.org,direct` and the proxy.
|
||||
- Pod Git mirror sync/clone: the Git stage must run `git clone --mirror` against the configured public remote, then `remote update --prune`, with Git HTTP(S) proxy settings bound to the YAML-declared proxy.
|
||||
|
||||
If the Kubernetes image pull stage fails, the benchmark result is not an application dependency failure; it is an image-pull proxy failure in the k3s/containerd path and must be fixed there.
|
||||
|
||||
@@ -22,15 +23,16 @@ The `real-deps-500m` profile renders a multi-stage Kubernetes Job:
|
||||
- `initContainer/apk-add`: image `docker.io/library/alpine:3.20`.
|
||||
- `initContainer/npm-install`: image `docker.io/library/node:22-bookworm`.
|
||||
- `initContainer/go-download`: image `docker.io/library/golang:1.24-bookworm`.
|
||||
- `initContainer/git-mirror`: image and remote declared by `realDeps.gitMirror`.
|
||||
- `container/summary`: emits a bounded result marker after all init containers finish.
|
||||
|
||||
All three init containers receive the YAML-declared `sub2api-egress-proxy` service URL through `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and lowercase variants. The image pull itself happens before Pod process execution; therefore image pull proxy evidence must come from the k3s/containerd path and proxyserver-side traffic sampling, not from in-container env alone.
|
||||
All dependency init containers receive the YAML-declared `sub2api-egress-proxy` service URL through `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, and lowercase variants. The image pull itself happens before Pod process execution; therefore image pull proxy evidence must come from the k3s/containerd path and proxyserver-side traffic sampling, not from in-container env alone.
|
||||
|
||||
## Observability
|
||||
|
||||
The source of truth for traffic is `platform-infra egress-proxy traffic --target <node> --sample-seconds N`. Benchmark status may include this traffic sample. The final evidence table must include proxyserver window bytes/rate/cumulative bytes, top client, and top destination.
|
||||
|
||||
For image pull traffic, the observed proxy client may be the node/k3s/containerd path rather than the benchmark Pod IP. For `apk`, `npm`, and `go` stages, the observed proxy client should correspond to the benchmark Pod network path. This distinction must be preserved in issue evidence.
|
||||
For image pull traffic, the observed proxy client may be the node/k3s/containerd path rather than the benchmark Pod IP. For `apk`, `npm`, `go`, and `git-mirror` stages, the observed proxy client should correspond to the benchmark Pod network path. This distinction must be preserved in issue evidence.
|
||||
|
||||
Status output must classify failures into at least:
|
||||
|
||||
@@ -38,6 +40,7 @@ Status output must classify failures into at least:
|
||||
- `apk-download`: Pod started but apk fetch/install failed.
|
||||
- `npm-download`: Pod started but npm install failed.
|
||||
- `go-download`: Pod started but Go module download failed.
|
||||
- `git-mirror`: Pod started but Git mirror clone/sync failed.
|
||||
- `none`: all stages succeeded.
|
||||
|
||||
## Boundaries
|
||||
@@ -47,15 +50,15 @@ This benchmark must not:
|
||||
- use Cloudflare speed-test downloads as acceptance evidence;
|
||||
- install Node or Go only as a substitute for Kubernetes pulling `node`/`golang` images;
|
||||
- rewrite apk/npm/go sources to regional mirrors;
|
||||
- use HWLAB source repositories, Tekton, Argo, git-mirror, or previous build caches;
|
||||
- use HWLAB source repositories, Tekton, Argo, the production git-mirror service, or previous build caches;
|
||||
- hide image pull failures behind local image overrides.
|
||||
|
||||
`payloadMiB: 500` in the `real-deps-500m` profile means the minimum proxyserver-observed traffic required for acceptance. The Pod result marker may report apk/npm/go workspace sizes, but those sizes do not replace proxyserver traffic evidence because image pull bytes are outside the Pod filesystem.
|
||||
`payloadMiB: 500` in the `real-deps-500m` profile means the minimum proxyserver-observed traffic required for acceptance. The Pod result marker may report apk/npm/go/git-mirror workspace sizes, but those sizes do not replace proxyserver traffic evidence because image pull bytes are outside the Pod filesystem.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- `bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark --targets D601,D518 --profile real-deps-500m --dry-run` prints both node plans and the remote image set.
|
||||
- `--confirm` creates one unique Job per node and returns immediately.
|
||||
- `status --traffic-sample-seconds 15` reports Job state, `image-pull`/`apk`/`npm`/`go` failure family when applicable, and proxyserver traffic columns.
|
||||
- D601 and D518 both have final rows with target, profile, job, state, duration, apk MiB, npm MiB, go MiB, proxy traffic window/rate/cumulative, top client, top destination, and failure family.
|
||||
- `status --traffic-sample-seconds 15` reports Job state, `image-pull`/`apk`/`npm`/`go`/`git-mirror` failure family when applicable, and proxyserver traffic columns.
|
||||
- D601 and D518 both have final rows with target, profile, job, state, duration, apk MiB, npm MiB, go MiB, git mirror MiB, proxy traffic window/rate/cumulative, top client, top destination, and failure family.
|
||||
- Acceptance requires at least 500 MiB of proxyserver-observed traffic per successful node run. If a node cannot reach that point because image pull fails, the issue remains open until the k3s/containerd image pull proxy path is fixed or a blocker is explicitly documented.
|
||||
|
||||
@@ -74,6 +74,11 @@ interface K3sRealDepsSpec {
|
||||
modules: readonly string[];
|
||||
expectedMiB: number;
|
||||
};
|
||||
gitMirror: {
|
||||
image: string;
|
||||
remote: string;
|
||||
expectedMiB: number;
|
||||
};
|
||||
}
|
||||
|
||||
interface K3sBuildBenchmarkTargetOverride {
|
||||
@@ -113,6 +118,7 @@ interface TargetStatus {
|
||||
apkMiB: number | null;
|
||||
npmMiB: number | null;
|
||||
goMiB: number | null;
|
||||
gitMirrorMiB: number | null;
|
||||
realDepsMiB: number | null;
|
||||
failureFamily: string;
|
||||
logTail: string;
|
||||
@@ -276,6 +282,7 @@ function statusBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchma
|
||||
status.apkMiB === null ? "-" : `${status.apkMiB}MiB`,
|
||||
status.npmMiB === null ? "-" : `${status.npmMiB}MiB`,
|
||||
status.goMiB === null ? "-" : `${status.goMiB}MiB`,
|
||||
status.gitMirrorMiB === null ? "-" : `${status.gitMirrorMiB}MiB`,
|
||||
status.realDepsMiB === null ? "-" : `${status.realDepsMiB}MiB`,
|
||||
status.traffic === undefined ? "-" : bytes(status.traffic.windowBytes),
|
||||
status.traffic === undefined ? "-" : rate(status.traffic.rateBps),
|
||||
@@ -294,7 +301,7 @@ function statusBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchma
|
||||
renderedText: [
|
||||
"PLATFORM-INFRA K3S BUILD BENCHMARK STATUS",
|
||||
"",
|
||||
...table(["TARGET", "PROFILE", "STATE", "JOB", "DURATION", "OUTPUT", "DOWNLOAD", "APK", "NPM", "GO", "REAL_DEPS", "TRAFFIC_WINDOW", "TRAFFIC_RATE", "PROXY_CUM", "TOP_CLIENT", "TOP_DEST", "FAILURE"], rows),
|
||||
...table(["TARGET", "PROFILE", "STATE", "JOB", "DURATION", "OUTPUT", "DOWNLOAD", "APK", "NPM", "GO", "GIT_MIRROR", "REAL_DEPS", "TRAFFIC_WINDOW", "TRAFFIC_RATE", "PROXY_CUM", "TOP_CLIENT", "TOP_DEST", "FAILURE"], rows),
|
||||
...logSections,
|
||||
"",
|
||||
"NEXT",
|
||||
@@ -371,7 +378,7 @@ function renderDryRun(plans: readonly TargetPlan[], options: K3sBuildBenchmarkOp
|
||||
|
||||
function dryRunImages(profile: K3sBuildBenchmarkProfile): string {
|
||||
if (profile.workload === "k3s-real-deps" && profile.realDeps !== undefined) {
|
||||
return [profile.realDeps.apk.image, profile.realDeps.npm.image, profile.realDeps.go.image].join(",");
|
||||
return [profile.realDeps.apk.image, profile.realDeps.npm.image, profile.realDeps.go.image, profile.realDeps.gitMirror.image].join(",");
|
||||
}
|
||||
return profile.image;
|
||||
}
|
||||
@@ -383,13 +390,13 @@ function dryRunPullPolicy(profile: K3sBuildBenchmarkProfile): string {
|
||||
|
||||
function dependencySummary(profile: K3sBuildBenchmarkProfile): string {
|
||||
if (profile.workload === "k3s-real-deps" && profile.realDeps !== undefined) {
|
||||
return `apk~${profile.realDeps.apk.expectedMiB} npm~${profile.realDeps.npm.expectedMiB} go~${profile.realDeps.go.expectedMiB}`;
|
||||
return `apk~${profile.realDeps.apk.expectedMiB} npm~${profile.realDeps.npm.expectedMiB} go~${profile.realDeps.go.expectedMiB} gitMirror~${profile.realDeps.gitMirror.expectedMiB}`;
|
||||
}
|
||||
return `${profile.dependencyDownload.expectedMiB}MiB`;
|
||||
}
|
||||
|
||||
function dryRunDetail(profile: K3sBuildBenchmarkProfile | undefined): string {
|
||||
if (profile?.workload === "k3s-real-deps") return "kubelet-image-pull + apk/npm/go through proxy";
|
||||
if (profile?.workload === "k3s-real-deps") return "kubelet-image-pull + apk/npm/go/git-mirror through proxy";
|
||||
return "no-mirror emptyDir unique-job";
|
||||
}
|
||||
|
||||
@@ -491,7 +498,8 @@ function realDepsJobManifest(target: Sub2ApiTargetConfig, profile: K3sBuildBench
|
||||
"unidesk.ai/workload": profile.workload,
|
||||
"unidesk.ai/min-proxy-mib": String(realDeps.minProxyMiB),
|
||||
"unidesk.ai/image-pull-mode": "kubelet-containerd",
|
||||
"unidesk.ai/remote-images": [realDeps.apk.image, realDeps.npm.image, realDeps.go.image].join(","),
|
||||
"unidesk.ai/remote-images": [realDeps.apk.image, realDeps.npm.image, realDeps.go.image, realDeps.gitMirror.image].join(","),
|
||||
"unidesk.ai/git-mirror-remote": realDeps.gitMirror.remote,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
@@ -543,6 +551,19 @@ function realDepsJobManifest(target: Sub2ApiTargetConfig, profile: K3sBuildBench
|
||||
],
|
||||
volumeMounts: [{ name: "work", mountPath: "/work" }],
|
||||
},
|
||||
{
|
||||
name: "git-mirror",
|
||||
image: realDeps.gitMirror.image,
|
||||
imagePullPolicy: realDeps.imagePullPolicy,
|
||||
command: ["/bin/sh", "-lc"],
|
||||
args: [realDepsGitMirrorScript()],
|
||||
env: [
|
||||
...commonEnv,
|
||||
{ name: "GIT_MIRROR_IMAGE", value: realDeps.gitMirror.image },
|
||||
{ name: "GIT_MIRROR_REMOTE", value: realDeps.gitMirror.remote },
|
||||
],
|
||||
volumeMounts: [{ name: "work", mountPath: "/work" }],
|
||||
},
|
||||
],
|
||||
containers: [{
|
||||
name: "summary",
|
||||
@@ -630,23 +651,53 @@ printf 'UNIDESK_K3S_REAL_DEPS_STAGE {"stage":"go","ok":true,"image":"%s","downlo
|
||||
`;
|
||||
}
|
||||
|
||||
function realDepsGitMirrorScript(): string {
|
||||
return `set -eu
|
||||
mkdir -p /work/stages /work/git-mirror
|
||||
printf 'UNIDESK_K3S_REAL_DEPS_EVENT stage=git-mirror target=%s profile=%s run=%s image=%s remote=%s\\n' "$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$GIT_MIRROR_IMAGE" "$GIT_MIRROR_REMOTE"
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
if ! command -v apk >/dev/null 2>&1; then
|
||||
echo "git-runtime-missing image=$GIT_MIRROR_IMAGE" >&2
|
||||
exit 127
|
||||
fi
|
||||
if grep -R -E 'npmmirror|daocloud|aliyun|tuna|ustc|huaweicloud' /etc/apk/repositories >/tmp/git-apk-mirror-check.out 2>/dev/null; then
|
||||
cat /tmp/git-apk-mirror-check.out >&2
|
||||
echo "unexpected apk mirror in git mirror base image" >&2
|
||||
exit 42
|
||||
fi
|
||||
apk update
|
||||
apk add --no-cache git ca-certificates
|
||||
fi
|
||||
git --version
|
||||
git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=120 clone --mirror --filter=blob:none "$GIT_MIRROR_REMOTE" /work/git-mirror/repo.git
|
||||
git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" --git-dir=/work/git-mirror/repo.git remote update --prune
|
||||
git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" --git-dir=/work/git-mirror/repo.git remote -v
|
||||
git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" ls-remote --heads "$GIT_MIRROR_REMOTE" >/work/git-mirror/ls-remote-heads.txt
|
||||
git_mib="$(du -sk /work/git-mirror/repo.git /work/git-mirror/ls-remote-heads.txt 2>/dev/null | awk '{s+=$1} END {printf "%d", int((s+1023)/1024)}')"
|
||||
printf 'gitMirrorMiB=%s\\n' "$git_mib" > /work/stages/git-mirror.env
|
||||
printf 'UNIDESK_K3S_REAL_DEPS_STAGE {"stage":"git-mirror","ok":true,"image":"%s","remote":"%s","mirrorMiB":%s}\\n' "$GIT_MIRROR_IMAGE" "$GIT_MIRROR_REMOTE" "$git_mib"
|
||||
`;
|
||||
}
|
||||
|
||||
function realDepsSummaryScript(realDeps: K3sRealDepsSpec): string {
|
||||
return `set -eu
|
||||
apkMiB=0
|
||||
npmMiB=0
|
||||
goMiB=0
|
||||
gitMirrorMiB=0
|
||||
realDepsStartedEpoch="$(date +%s)"
|
||||
realDepsStartedAt="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
[ -f /work/stages/apk.env ] && . /work/stages/apk.env
|
||||
[ -f /work/stages/npm.env ] && . /work/stages/npm.env
|
||||
[ -f /work/stages/go.env ] && . /work/stages/go.env
|
||||
[ -f /work/stages/git-mirror.env ] && . /work/stages/git-mirror.env
|
||||
completed_epoch="$(date +%s)"
|
||||
completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
real_deps_mib=$((apkMiB + npmMiB + goMiB))
|
||||
real_deps_mib=$((apkMiB + npmMiB + goMiB + gitMirrorMiB))
|
||||
duration=$((completed_epoch - realDepsStartedEpoch))
|
||||
pod_ip="$(hostname -i 2>/dev/null | awk '{print $1}')"
|
||||
printf 'UNIDESK_K3S_REAL_DEPS_RESULT {"ok":true,"target":"%s","profile":"%s","runId":"%s","startedAt":"%s","completedAt":"%s","durationSeconds":%s,"podIp":"%s","apkMiB":%s,"npmMiB":%s,"goMiB":%s,"realDepsMiB":%s,"minProxyMiB":%s,"imagePullMode":"kubelet-containerd","apkImage":"%s","npmImage":"%s","goImage":"%s"}\\n' \\
|
||||
"$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$realDepsStartedAt" "$completed_at" "$duration" "$pod_ip" "$apkMiB" "$npmMiB" "$goMiB" "$real_deps_mib" "$MIN_PROXY_MIB" ${shQuote(realDeps.apk.image)} ${shQuote(realDeps.npm.image)} ${shQuote(realDeps.go.image)}
|
||||
printf 'UNIDESK_K3S_REAL_DEPS_RESULT {"ok":true,"target":"%s","profile":"%s","runId":"%s","startedAt":"%s","completedAt":"%s","durationSeconds":%s,"podIp":"%s","apkMiB":%s,"npmMiB":%s,"goMiB":%s,"gitMirrorMiB":%s,"realDepsMiB":%s,"minProxyMiB":%s,"imagePullMode":"kubelet-containerd","apkImage":"%s","npmImage":"%s","goImage":"%s","gitMirrorImage":"%s","gitMirrorRemote":"%s"}\\n' \\
|
||||
"$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$realDepsStartedAt" "$completed_at" "$duration" "$pod_ip" "$apkMiB" "$npmMiB" "$goMiB" "$gitMirrorMiB" "$real_deps_mib" "$MIN_PROXY_MIB" ${shQuote(realDeps.apk.image)} ${shQuote(realDeps.npm.image)} ${shQuote(realDeps.go.image)} ${shQuote(realDeps.gitMirror.image)} ${shQuote(realDeps.gitMirror.remote)}
|
||||
`;
|
||||
}
|
||||
|
||||
@@ -952,6 +1003,8 @@ elif "npm-install" in tail_text and ("npm ERR!" in tail_text or "EAI_AGAIN" in t
|
||||
failure_family = "npm-download"
|
||||
elif "go-download" in tail_text and ("terminated exit=" in tail_text or "i/o timeout" in tail_text or "connection refused" in tail_text or "no such host" in tail_text or "proxyconnect tcp" in tail_text or "TLS handshake timeout" in tail_text or "go-runtime-missing" in tail_text):
|
||||
failure_family = "go-download"
|
||||
elif "git-mirror" in tail_text and ("terminated exit=" in tail_text or "fatal:" in tail_text or "Could not resolve host" in tail_text or "Failed to connect" in tail_text or "connection refused" in tail_text or "proxyconnect tcp" in tail_text or "TLS handshake timeout" in tail_text or "git-runtime-missing" in tail_text):
|
||||
failure_family = "git-mirror"
|
||||
elif "curl:" in tail_text or "urllib.error.HTTPError" in tail_text or "urllib.error.URLError" in tail_text:
|
||||
failure_family = "dependency-download"
|
||||
elif "payload-too-small" in tail_text:
|
||||
@@ -1032,6 +1085,7 @@ function normalizeStatus(plan: TargetPlan, parsed: unknown, result: CommandResul
|
||||
apkMiB: null,
|
||||
npmMiB: null,
|
||||
goMiB: null,
|
||||
gitMirrorMiB: null,
|
||||
realDepsMiB: null,
|
||||
failureFamily: result.timedOut ? "transport-timeout" : state,
|
||||
logTail: (result.stderr || result.stdout).slice(-4000),
|
||||
@@ -1057,6 +1111,7 @@ function normalizeStatus(plan: TargetPlan, parsed: unknown, result: CommandResul
|
||||
apkMiB: nullableNumber(jobResult.apkMiB),
|
||||
npmMiB: nullableNumber(jobResult.npmMiB),
|
||||
goMiB: nullableNumber(jobResult.goMiB),
|
||||
gitMirrorMiB: nullableNumber(jobResult.gitMirrorMiB),
|
||||
realDepsMiB: nullableNumber(jobResult.realDepsMiB),
|
||||
failureFamily: text(data.failureFamily, data.ok === true ? "none" : state === "running" || state === "pending" ? "in-progress" : text(data.reason, "unknown")),
|
||||
logTail: text(data.logTail, result.stderr.slice(-2000)),
|
||||
@@ -1081,6 +1136,7 @@ function blockedStatus(plan: TargetPlan, profile: string): TargetStatus {
|
||||
apkMiB: null,
|
||||
npmMiB: null,
|
||||
goMiB: null,
|
||||
gitMirrorMiB: null,
|
||||
realDepsMiB: null,
|
||||
failureFamily: plan.blocker ?? "blocked",
|
||||
logTail: plan.detail ?? "",
|
||||
@@ -1151,6 +1207,7 @@ function realDepsSpec(raw: Record<string, unknown>, path: string): K3sRealDepsSp
|
||||
const apk = asRecord(raw.apk, `${path}.apk`);
|
||||
const npm = asRecord(raw.npm, `${path}.npm`);
|
||||
const go = asRecord(raw.go, `${path}.go`);
|
||||
const gitMirror = asRecord(raw.gitMirror, `${path}.gitMirror`);
|
||||
return {
|
||||
minProxyMiB: integerField(raw, "minProxyMiB", path),
|
||||
imagePullPolicy: imagePullPolicyField(raw, "imagePullPolicy", path),
|
||||
@@ -1171,6 +1228,11 @@ function realDepsSpec(raw: Record<string, unknown>, path: string): K3sRealDepsSp
|
||||
modules: stringArrayField(go, "modules", `${path}.go`),
|
||||
expectedMiB: integerField(go, "expectedMiB", `${path}.go`),
|
||||
},
|
||||
gitMirror: {
|
||||
image: stringField(gitMirror, "image", `${path}.gitMirror`),
|
||||
remote: stringField(gitMirror, "remote", `${path}.gitMirror`),
|
||||
expectedMiB: integerField(gitMirror, "expectedMiB", `${path}.gitMirror`),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user