mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add git mirror proxy benchmark stage

Browse Source
This commit is contained in:
Codex
2026-06-27 02:23:46 +00:00
parent 1114517e98
commit cb99ad969b
5 changed files with 94 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-k3s-build-benchmark.ts
+70 -8
View File
@@ -74,6 +74,11 @@ interface K3sRealDepsSpec {
modules: readonly string[];
expectedMiB: number;
};
gitMirror: {
image: string;
remote: string;
expectedMiB: number;
};
}
interface K3sBuildBenchmarkTargetOverride {
@@ -113,6 +118,7 @@ interface TargetStatus {
apkMiB: number | null;
npmMiB: number | null;
goMiB: number | null;
gitMirrorMiB: number | null;
realDepsMiB: number | null;
failureFamily: string;
logTail: string;
@@ -276,6 +282,7 @@ function statusBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchma
status.apkMiB === null ? "-" : `${status.apkMiB}MiB`,
status.npmMiB === null ? "-" : `${status.npmMiB}MiB`,
status.goMiB === null ? "-" : `${status.goMiB}MiB`,
status.gitMirrorMiB === null ? "-" : `${status.gitMirrorMiB}MiB`,
status.realDepsMiB === null ? "-" : `${status.realDepsMiB}MiB`,
status.traffic === undefined ? "-" : bytes(status.traffic.windowBytes),
status.traffic === undefined ? "-" : rate(status.traffic.rateBps),
@@ -294,7 +301,7 @@ function statusBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchma
renderedText: [
"PLATFORM-INFRA K3S BUILD BENCHMARK STATUS",
"",
...table(["TARGET", "PROFILE", "STATE", "JOB", "DURATION", "OUTPUT", "DOWNLOAD", "APK", "NPM", "GO", "REAL_DEPS", "TRAFFIC_WINDOW", "TRAFFIC_RATE", "PROXY_CUM", "TOP_CLIENT", "TOP_DEST", "FAILURE"], rows),
...table(["TARGET", "PROFILE", "STATE", "JOB", "DURATION", "OUTPUT", "DOWNLOAD", "APK", "NPM", "GO", "GIT_MIRROR", "REAL_DEPS", "TRAFFIC_WINDOW", "TRAFFIC_RATE", "PROXY_CUM", "TOP_CLIENT", "TOP_DEST", "FAILURE"], rows),
...logSections,
"",
"NEXT",
@@ -371,7 +378,7 @@ function renderDryRun(plans: readonly TargetPlan[], options: K3sBuildBenchmarkOp
function dryRunImages(profile: K3sBuildBenchmarkProfile): string {
if (profile.workload === "k3s-real-deps" && profile.realDeps !== undefined) {
return [profile.realDeps.apk.image, profile.realDeps.npm.image, profile.realDeps.go.image].join(",");
return [profile.realDeps.apk.image, profile.realDeps.npm.image, profile.realDeps.go.image, profile.realDeps.gitMirror.image].join(",");
}
return profile.image;
}
@@ -383,13 +390,13 @@ function dryRunPullPolicy(profile: K3sBuildBenchmarkProfile): string {
function dependencySummary(profile: K3sBuildBenchmarkProfile): string {
if (profile.workload === "k3s-real-deps" && profile.realDeps !== undefined) {
return `apk~${profile.realDeps.apk.expectedMiB} npm~${profile.realDeps.npm.expectedMiB} go~${profile.realDeps.go.expectedMiB}`;
return `apk~${profile.realDeps.apk.expectedMiB} npm~${profile.realDeps.npm.expectedMiB} go~${profile.realDeps.go.expectedMiB} gitMirror~${profile.realDeps.gitMirror.expectedMiB}`;
}
return `${profile.dependencyDownload.expectedMiB}MiB`;
}
function dryRunDetail(profile: K3sBuildBenchmarkProfile | undefined): string {
if (profile?.workload === "k3s-real-deps") return "kubelet-image-pull + apk/npm/go through proxy";
if (profile?.workload === "k3s-real-deps") return "kubelet-image-pull + apk/npm/go/git-mirror through proxy";
return "no-mirror emptyDir unique-job";
}
@@ -491,7 +498,8 @@ function realDepsJobManifest(target: Sub2ApiTargetConfig, profile: K3sBuildBench
"unidesk.ai/workload": profile.workload,
"unidesk.ai/min-proxy-mib": String(realDeps.minProxyMiB),
"unidesk.ai/image-pull-mode": "kubelet-containerd",
"unidesk.ai/remote-images": [realDeps.apk.image, realDeps.npm.image, realDeps.go.image].join(","),
"unidesk.ai/remote-images": [realDeps.apk.image, realDeps.npm.image, realDeps.go.image, realDeps.gitMirror.image].join(","),
"unidesk.ai/git-mirror-remote": realDeps.gitMirror.remote,
},
},
spec: {
@@ -543,6 +551,19 @@ function realDepsJobManifest(target: Sub2ApiTargetConfig, profile: K3sBuildBench
],
volumeMounts: [{ name: "work", mountPath: "/work" }],
},
{
name: "git-mirror",
image: realDeps.gitMirror.image,
imagePullPolicy: realDeps.imagePullPolicy,
command: ["/bin/sh", "-lc"],
args: [realDepsGitMirrorScript()],
env: [
...commonEnv,
{ name: "GIT_MIRROR_IMAGE", value: realDeps.gitMirror.image },
{ name: "GIT_MIRROR_REMOTE", value: realDeps.gitMirror.remote },
],
volumeMounts: [{ name: "work", mountPath: "/work" }],
},
],
containers: [{
name: "summary",
@@ -630,23 +651,53 @@ printf 'UNIDESK_K3S_REAL_DEPS_STAGE {"stage":"go","ok":true,"image":"%s","downlo
`;
}
function realDepsGitMirrorScript(): string {
return `set -eu
mkdir -p /work/stages /work/git-mirror
printf 'UNIDESK_K3S_REAL_DEPS_EVENT stage=git-mirror target=%s profile=%s run=%s image=%s remote=%s\\n' "$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$GIT_MIRROR_IMAGE" "$GIT_MIRROR_REMOTE"
if ! command -v git >/dev/null 2>&1; then
if ! command -v apk >/dev/null 2>&1; then
echo "git-runtime-missing image=$GIT_MIRROR_IMAGE" >&2
exit 127
fi
if grep -R -E 'npmmirror|daocloud|aliyun|tuna|ustc|huaweicloud' /etc/apk/repositories >/tmp/git-apk-mirror-check.out 2>/dev/null; then
cat /tmp/git-apk-mirror-check.out >&2
echo "unexpected apk mirror in git mirror base image" >&2
exit 42
fi
apk update
apk add --no-cache git ca-certificates
fi
git --version
git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" -c http.lowSpeedLimit=1024 -c http.lowSpeedTime=120 clone --mirror --filter=blob:none "$GIT_MIRROR_REMOTE" /work/git-mirror/repo.git
git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" --git-dir=/work/git-mirror/repo.git remote update --prune
git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" --git-dir=/work/git-mirror/repo.git remote -v
git -c "http.proxy=\${HTTP_PROXY:-}" -c "https.proxy=\${HTTPS_PROXY:-}" ls-remote --heads "$GIT_MIRROR_REMOTE" >/work/git-mirror/ls-remote-heads.txt
git_mib="$(du -sk /work/git-mirror/repo.git /work/git-mirror/ls-remote-heads.txt 2>/dev/null | awk '{s+=$1} END {printf "%d", int((s+1023)/1024)}')"
printf 'gitMirrorMiB=%s\\n' "$git_mib" > /work/stages/git-mirror.env
printf 'UNIDESK_K3S_REAL_DEPS_STAGE {"stage":"git-mirror","ok":true,"image":"%s","remote":"%s","mirrorMiB":%s}\\n' "$GIT_MIRROR_IMAGE" "$GIT_MIRROR_REMOTE" "$git_mib"
`;
}
function realDepsSummaryScript(realDeps: K3sRealDepsSpec): string {
return `set -eu
apkMiB=0
npmMiB=0
goMiB=0
gitMirrorMiB=0
realDepsStartedEpoch="$(date +%s)"
realDepsStartedAt="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
[ -f /work/stages/apk.env ] && . /work/stages/apk.env
[ -f /work/stages/npm.env ] && . /work/stages/npm.env
[ -f /work/stages/go.env ] && . /work/stages/go.env
[ -f /work/stages/git-mirror.env ] && . /work/stages/git-mirror.env
completed_epoch="$(date +%s)"
completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
real_deps_mib=$((apkMiB + npmMiB + goMiB))
real_deps_mib=$((apkMiB + npmMiB + goMiB + gitMirrorMiB))
duration=$((completed_epoch - realDepsStartedEpoch))
pod_ip="$(hostname -i 2>/dev/null | awk '{print $1}')"
printf 'UNIDESK_K3S_REAL_DEPS_RESULT {"ok":true,"target":"%s","profile":"%s","runId":"%s","startedAt":"%s","completedAt":"%s","durationSeconds":%s,"podIp":"%s","apkMiB":%s,"npmMiB":%s,"goMiB":%s,"realDepsMiB":%s,"minProxyMiB":%s,"imagePullMode":"kubelet-containerd","apkImage":"%s","npmImage":"%s","goImage":"%s"}\\n' \\
"$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$realDepsStartedAt" "$completed_at" "$duration" "$pod_ip" "$apkMiB" "$npmMiB" "$goMiB" "$real_deps_mib" "$MIN_PROXY_MIB" ${shQuote(realDeps.apk.image)} ${shQuote(realDeps.npm.image)} ${shQuote(realDeps.go.image)}
printf 'UNIDESK_K3S_REAL_DEPS_RESULT {"ok":true,"target":"%s","profile":"%s","runId":"%s","startedAt":"%s","completedAt":"%s","durationSeconds":%s,"podIp":"%s","apkMiB":%s,"npmMiB":%s,"goMiB":%s,"gitMirrorMiB":%s,"realDepsMiB":%s,"minProxyMiB":%s,"imagePullMode":"kubelet-containerd","apkImage":"%s","npmImage":"%s","goImage":"%s","gitMirrorImage":"%s","gitMirrorRemote":"%s"}\\n' \\
"$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$realDepsStartedAt" "$completed_at" "$duration" "$pod_ip" "$apkMiB" "$npmMiB" "$goMiB" "$gitMirrorMiB" "$real_deps_mib" "$MIN_PROXY_MIB" ${shQuote(realDeps.apk.image)} ${shQuote(realDeps.npm.image)} ${shQuote(realDeps.go.image)} ${shQuote(realDeps.gitMirror.image)} ${shQuote(realDeps.gitMirror.remote)}
`;
}
@@ -952,6 +1003,8 @@ elif "npm-install" in tail_text and ("npm ERR!" in tail_text or "EAI_AGAIN" in t
failure_family = "npm-download"
elif "go-download" in tail_text and ("terminated exit=" in tail_text or "i/o timeout" in tail_text or "connection refused" in tail_text or "no such host" in tail_text or "proxyconnect tcp" in tail_text or "TLS handshake timeout" in tail_text or "go-runtime-missing" in tail_text):
failure_family = "go-download"
elif "git-mirror" in tail_text and ("terminated exit=" in tail_text or "fatal:" in tail_text or "Could not resolve host" in tail_text or "Failed to connect" in tail_text or "connection refused" in tail_text or "proxyconnect tcp" in tail_text or "TLS handshake timeout" in tail_text or "git-runtime-missing" in tail_text):
failure_family = "git-mirror"
elif "curl:" in tail_text or "urllib.error.HTTPError" in tail_text or "urllib.error.URLError" in tail_text:
failure_family = "dependency-download"
elif "payload-too-small" in tail_text:
@@ -1032,6 +1085,7 @@ function normalizeStatus(plan: TargetPlan, parsed: unknown, result: CommandResul
apkMiB: null,
npmMiB: null,
goMiB: null,
gitMirrorMiB: null,
realDepsMiB: null,
failureFamily: result.timedOut ? "transport-timeout" : state,
logTail: (result.stderr || result.stdout).slice(-4000),
@@ -1057,6 +1111,7 @@ function normalizeStatus(plan: TargetPlan, parsed: unknown, result: CommandResul
apkMiB: nullableNumber(jobResult.apkMiB),
npmMiB: nullableNumber(jobResult.npmMiB),
goMiB: nullableNumber(jobResult.goMiB),
gitMirrorMiB: nullableNumber(jobResult.gitMirrorMiB),
realDepsMiB: nullableNumber(jobResult.realDepsMiB),
failureFamily: text(data.failureFamily, data.ok === true ? "none" : state === "running" || state === "pending" ? "in-progress" : text(data.reason, "unknown")),
logTail: text(data.logTail, result.stderr.slice(-2000)),
@@ -1081,6 +1136,7 @@ function blockedStatus(plan: TargetPlan, profile: string): TargetStatus {
apkMiB: null,
npmMiB: null,
goMiB: null,
gitMirrorMiB: null,
realDepsMiB: null,
failureFamily: plan.blocker ?? "blocked",
logTail: plan.detail ?? "",
@@ -1151,6 +1207,7 @@ function realDepsSpec(raw: Record<string, unknown>, path: string): K3sRealDepsSp
const apk = asRecord(raw.apk, `${path}.apk`);
const npm = asRecord(raw.npm, `${path}.npm`);
const go = asRecord(raw.go, `${path}.go`);
const gitMirror = asRecord(raw.gitMirror, `${path}.gitMirror`);
return {
minProxyMiB: integerField(raw, "minProxyMiB", path),
imagePullPolicy: imagePullPolicyField(raw, "imagePullPolicy", path),
@@ -1171,6 +1228,11 @@ function realDepsSpec(raw: Record<string, unknown>, path: string): K3sRealDepsSp
modules: stringArrayField(go, "modules", `${path}.go`),
expectedMiB: integerField(go, "expectedMiB", `${path}.go`),
},
gitMirror: {
image: stringField(gitMirror, "image", `${path}.gitMirror`),
remote: stringField(gitMirror, "remote", `${path}.gitMirror`),
expectedMiB: integerField(gitMirror, "expectedMiB", `${path}.gitMirror`),
},
};
}