mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add real dependency proxy benchmark

Browse Source
This commit is contained in:
Codex
2026-06-26 16:34:23 +00:00
parent 9bce21b9ef
commit 678dc427d6
4 changed files with 562 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-egress-proxy.ts
+1 -1
View File
@@ -150,7 +150,7 @@ function platformTrafficSpec(options: TrafficOptions): EgressProxyTrafficSpec {
function parseEgressProxyOptions(args: string[]): EgressProxyOptions {
const actionRaw = args[0] ?? "benchmark";
if (!isEgressProxyAction(actionRaw)) {
throw new Error("platform-infra egress-proxy usage: benchmark|benchmark-status|benchmark-logs|traffic|k3s-build-benchmark --target D601|D518 [--profile no-mirror|no-mirror-600m] [--sample-seconds N]");
throw new Error("platform-infra egress-proxy usage: benchmark|benchmark-status|benchmark-logs|traffic|k3s-build-benchmark --target D601|D518 [--profile no-mirror|no-mirror-600m|real-deps-500m] [--sample-seconds N]");
}
const action = actionRaw;
const rest = args.slice(1);
scripts/src/platform-infra-k3s-build-benchmark.ts
+434 -29
View File
@@ -12,8 +12,9 @@ import { resolveTarget } from "./platform-infra/manifest";
const BENCHMARK_CONFIG_PATH = "config/platform-infra/egress-proxy-benchmarks.yaml";
const BENCHMARK_APP = "unidesk-k3s-build-benchmark";
type K3sBuildAction = "start" | "status" | "logs";
type K3sBuildAction = "start" | "status" | "logs" | "cleanup";
type ImagePullPolicy = "Always" | "IfNotPresent" | "Never";
type K3sBuildWorkload = "k3s-build" | "k3s-real-deps";
interface K3sBuildBenchmarkOptions {
action: K3sBuildAction;
@@ -29,7 +30,7 @@ interface K3sBuildBenchmarkOptions {
interface K3sBuildBenchmarkProfile {
id: string;
enabled: boolean;
workload: "k3s-build";
workload: K3sBuildWorkload;
description: string;
image: string;
imagePullPolicy: ImagePullPolicy;
@@ -50,6 +51,29 @@ interface K3sBuildBenchmarkProfile {
chunks: number;
expectedMiB: number;
};
realDeps?: K3sRealDepsSpec;
}
interface K3sRealDepsSpec {
minProxyMiB: number;
imagePullPolicy: ImagePullPolicy;
apk: {
image: string;
packages: readonly string[];
expectedMiB: number;
};
npm: {
image: string;
registry: string;
packages: Record<string, string>;
expectedMiB: number;
};
go: {
image: string;
goProxy: string;
modules: readonly string[];
expectedMiB: number;
};
}
interface K3sBuildBenchmarkTargetOverride {
@@ -86,6 +110,10 @@ interface TargetStatus {
outputMiB: number | null;
downloadMiB: number | null;
payloadMiB: number | null;
apkMiB: number | null;
npmMiB: number | null;
goMiB: number | null;
realDepsMiB: number | null;
failureFamily: string;
logTail: string;
traffic?: TrafficSummary;
@@ -107,15 +135,16 @@ export function runK3sBuildBenchmarkCommand(args: string[]): RenderedCliResult {
const plans = resolvePlans(config, options);
if (options.action === "start" && options.dryRun) return renderDryRun(plans, options);
if (options.action === "start") return startBenchmarks(plans, options);
if (options.action === "cleanup") return cleanupBenchmarks(plans, options);
return statusBenchmarks(plans, options);
}
function parseK3sBuildBenchmarkOptions(args: string[]): K3sBuildBenchmarkOptions {
const first = args[0];
const action: K3sBuildAction = first === "status" || first === "logs" ? first : "start";
const action: K3sBuildAction = first === "status" || first === "logs" || first === "cleanup" ? first : "start";
const rest = action === "start" ? args : args.slice(1);
if (first === "--help" || first === "-h" || first === "help") {
throw new Error("platform-infra egress-proxy k3s-build-benchmark usage: k3s-build-benchmark [status|logs] --targets D601,D518 --profile no-mirror-600m [--dry-run|--confirm]");
throw new Error("platform-infra egress-proxy k3s-build-benchmark usage: k3s-build-benchmark [status|logs|cleanup] --targets D601,D518 --profile real-deps-500m [--dry-run|--confirm]");
}
const confirm = rest.includes("--confirm");
const explicitDryRun = rest.includes("--dry-run");
@@ -152,6 +181,7 @@ function resolvePlans(config: K3sBuildBenchmarkConfig, options: K3sBuildBenchmar
if (profile === undefined) return { ok: false, targetId, blocker: "profile-missing", detail: `${BENCHMARK_CONFIG_PATH}.profiles.${options.profile} is missing` };
if (!profile.enabled) return { ok: false, targetId, profile, blocker: "profile-disabled", detail: `${BENCHMARK_CONFIG_PATH}.profiles.${profile.id}.enabled=false` };
if (profile.payloadMiB < 500) return { ok: false, targetId, profile, blocker: "payload-too-small", detail: `${BENCHMARK_CONFIG_PATH}.profiles.${profile.id}.payloadMiB must be >= 500` };
if (profile.workload === "k3s-real-deps" && profile.realDeps === undefined) return { ok: false, targetId, profile, blocker: "real-deps-missing", detail: `${BENCHMARK_CONFIG_PATH}.profiles.${profile.id}.realDeps is missing` };
try {
const target = resolveTarget(sub2api, targetId);
if (target.egressProxy === null || !target.egressProxy.enabled) return { ok: false, targetId: target.id, target, profile, blocker: "egress-proxy-disabled", detail: `config/platform-infra/sub2api.yaml target ${target.id} has no enabled egressProxy` };
@@ -240,6 +270,10 @@ function statusBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchma
status.durationSeconds === null ? "-" : `${status.durationSeconds}s`,
status.outputMiB === null ? "-" : `${status.outputMiB}MiB`,
status.downloadMiB === null ? "-" : `${status.downloadMiB}MiB`,
status.apkMiB === null ? "-" : `${status.apkMiB}MiB`,
status.npmMiB === null ? "-" : `${status.npmMiB}MiB`,
status.goMiB === null ? "-" : `${status.goMiB}MiB`,
status.realDepsMiB === null ? "-" : `${status.realDepsMiB}MiB`,
status.traffic === undefined ? "-" : bytes(status.traffic.windowBytes),
status.traffic === undefined ? "-" : rate(status.traffic.rateBps),
status.traffic === undefined ? "-" : bytes(status.traffic.processTotalBytes),
@@ -257,7 +291,7 @@ function statusBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchma
renderedText: [
"PLATFORM-INFRA K3S BUILD BENCHMARK STATUS",
"",
...table(["TARGET", "PROFILE", "STATE", "JOB", "DURATION", "OUTPUT", "DOWNLOAD", "TRAFFIC_WINDOW", "TRAFFIC_RATE", "PROXY_CUM", "TOP_CLIENT", "TOP_DEST", "FAILURE"], rows),
...table(["TARGET", "PROFILE", "STATE", "JOB", "DURATION", "OUTPUT", "DOWNLOAD", "APK", "NPM", "GO", "REAL_DEPS", "TRAFFIC_WINDOW", "TRAFFIC_RATE", "PROXY_CUM", "TOP_CLIENT", "TOP_DEST", "FAILURE"], rows),
...logSections,
"",
"NEXT",
@@ -269,6 +303,38 @@ function statusBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchma
};
}
function cleanupBenchmarks(plans: readonly TargetPlan[], options: K3sBuildBenchmarkOptions): RenderedCliResult {
const rows = plans.map((plan) => {
if (!plan.ok || plan.target === undefined || plan.profile === undefined) {
return { targetId: plan.targetId, profile: options.profile, state: "blocked", deleted: "-", detail: plan.detail ?? plan.blocker ?? "blocked" };
}
if (!options.confirm) return { targetId: plan.targetId, profile: plan.profile.id, state: "dry-run", deleted: "-", detail: "pass --confirm to delete matching benchmark Jobs" };
const result = runTrans(plan.target.route, cleanupScript(plan.target, plan.profile), options.timeoutSeconds);
const parsed = parseJson(result.stdout);
const data = record(parsed);
return {
targetId: plan.targetId,
profile: plan.profile.id,
state: result.exitCode === 0 && data.ok !== false ? "deleted" : "failed",
deleted: text(data.deleted, "-"),
detail: text(data.detail, result.stderr.slice(-1000) || result.stdout.slice(-1000)),
};
});
return {
ok: rows.every((row) => row.state === "deleted" || row.state === "dry-run"),
command: "platform-infra egress-proxy k3s-build-benchmark cleanup",
contentType: "text/plain",
renderedText: [
"PLATFORM-INFRA K3S BUILD BENCHMARK CLEANUP",
"",
...table(["TARGET", "PROFILE", "STATE", "DELETED", "DETAIL"], rows.map((row) => [row.targetId, row.profile, row.state, row.deleted, row.detail])),
"",
"NEXT",
` bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark --targets ${rows.map((row) => row.targetId).join(",")} --profile ${options.profile} --confirm`,
].join("\n"),
};
}
function renderDryRun(plans: readonly TargetPlan[], options: K3sBuildBenchmarkOptions): RenderedCliResult {
const rows = plans.map((plan) => [
plan.targetId,
@@ -277,11 +343,11 @@ function renderDryRun(plans: readonly TargetPlan[], options: K3sBuildBenchmarkOp
plan.target?.route ?? "-",
plan.target?.namespace ?? "-",
plan.target?.egressProxy?.serviceName ?? "-",
plan.target !== undefined && plan.profile !== undefined ? effectiveImage(plan.target, plan.profile).image : "-",
plan.target !== undefined && plan.profile !== undefined ? effectiveImage(plan.target, plan.profile).imagePullPolicy : "-",
plan.profile !== undefined ? dryRunImages(plan.profile) : "-",
plan.profile !== undefined ? dryRunPullPolicy(plan.profile) : "-",
plan.profile === undefined ? "-" : `${plan.profile.payloadMiB}MiB`,
plan.profile === undefined ? "-" : `${plan.profile.dependencyDownload.expectedMiB}MiB`,
plan.detail ?? "no-mirror emptyDir unique-job",
plan.profile === undefined ? "-" : dependencySummary(plan.profile),
plan.detail ?? dryRunDetail(plan.profile),
]);
return {
ok: plans.every((plan) => plan.ok),
@@ -290,7 +356,7 @@ function renderDryRun(plans: readonly TargetPlan[], options: K3sBuildBenchmarkOp
renderedText: [
"PLATFORM-INFRA K3S BUILD BENCHMARK DRY-RUN",
"",
...table(["TARGET", "PROFILE", "STATUS", "ROUTE", "NAMESPACE", "PROXY", "IMAGE", "PULL", "PAYLOAD", "DOWNLOAD", "DETAIL"], rows),
...table(["TARGET", "PROFILE", "STATUS", "ROUTE", "NAMESPACE", "PROXY", "IMAGES", "PULL", "MIN_PROXY", "DEPS", "DETAIL"], rows),
"",
"NEXT",
` bun scripts/cli.ts platform-infra egress-proxy k3s-build-benchmark --targets ${plans.map((plan) => plan.targetId).join(",")} --profile ${options.profile} --confirm`,
@@ -300,7 +366,32 @@ function renderDryRun(plans: readonly TargetPlan[], options: K3sBuildBenchmarkOp
};
}
function dryRunImages(profile: K3sBuildBenchmarkProfile): string {
if (profile.workload === "k3s-real-deps" && profile.realDeps !== undefined) {
return [profile.realDeps.apk.image, profile.realDeps.npm.image, profile.realDeps.go.image].join(",");
}
return profile.image;
}
function dryRunPullPolicy(profile: K3sBuildBenchmarkProfile): string {
if (profile.workload === "k3s-real-deps" && profile.realDeps !== undefined) return profile.realDeps.imagePullPolicy;
return profile.imagePullPolicy;
}
function dependencySummary(profile: K3sBuildBenchmarkProfile): string {
if (profile.workload === "k3s-real-deps" && profile.realDeps !== undefined) {
return `apk~${profile.realDeps.apk.expectedMiB} npm~${profile.realDeps.npm.expectedMiB} go~${profile.realDeps.go.expectedMiB}`;
}
return `${profile.dependencyDownload.expectedMiB}MiB`;
}
function dryRunDetail(profile: K3sBuildBenchmarkProfile | undefined): string {
if (profile?.workload === "k3s-real-deps") return "kubelet-image-pull + apk/npm/go through proxy";
return "no-mirror emptyDir unique-job";
}
function benchmarkJobManifest(target: Sub2ApiTargetConfig, profile: K3sBuildBenchmarkProfile, runId: string, jobName: string): Record<string, unknown> {
if (profile.workload === "k3s-real-deps") return realDepsJobManifest(target, profile, runId, jobName);
const proxy = target.egressProxy;
if (proxy === null) throw new Error(`target ${target.id} has no egressProxy`);
const proxyUrl = `http://${proxy.serviceName}.${target.namespace}.svc.cluster.local:${proxy.listenPort}`;
@@ -365,6 +456,197 @@ function benchmarkJobManifest(target: Sub2ApiTargetConfig, profile: K3sBuildBenc
};
}
function realDepsJobManifest(target: Sub2ApiTargetConfig, profile: K3sBuildBenchmarkProfile, runId: string, jobName: string): Record<string, unknown> {
const proxy = target.egressProxy;
if (proxy === null) throw new Error(`target ${target.id} has no egressProxy`);
const realDeps = requireRealDeps(profile);
const proxyUrl = `http://${proxy.serviceName}.${target.namespace}.svc.cluster.local:${proxy.listenPort}`;
const noProxy = proxy.noProxy.join(",");
const labels = benchmarkLabels(target, profile, runId);
const commonEnv = [
{ name: "HTTP_PROXY", value: proxyUrl },
{ name: "HTTPS_PROXY", value: proxyUrl },
{ name: "ALL_PROXY", value: proxyUrl },
{ name: "http_proxy", value: proxyUrl },
{ name: "https_proxy", value: proxyUrl },
{ name: "all_proxy", value: proxyUrl },
{ name: "NO_PROXY", value: noProxy },
{ name: "no_proxy", value: noProxy },
{ name: "BENCHMARK_TARGET", value: target.id },
{ name: "BENCHMARK_PROFILE", value: profile.id },
{ name: "BENCHMARK_RUN_ID", value: runId },
{ name: "MIN_PROXY_MIB", value: String(realDeps.minProxyMiB) },
];
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: {
name: jobName,
namespace: target.namespace,
labels,
annotations: {
"unidesk.ai/workload": profile.workload,
"unidesk.ai/min-proxy-mib": String(realDeps.minProxyMiB),
"unidesk.ai/image-pull-mode": "kubelet-containerd",
"unidesk.ai/remote-images": [realDeps.apk.image, realDeps.npm.image, realDeps.go.image].join(","),
},
},
spec: {
backoffLimit: 0,
activeDeadlineSeconds: profile.timeoutSeconds,
ttlSecondsAfterFinished: profile.ttlSecondsAfterFinished,
template: {
metadata: { labels },
spec: {
restartPolicy: "Never",
initContainers: [
{
name: "apk-add",
image: realDeps.apk.image,
imagePullPolicy: realDeps.imagePullPolicy,
command: ["/bin/sh", "-lc"],
args: [realDepsApkScript()],
env: [
...commonEnv,
{ name: "APK_PACKAGES", value: realDeps.apk.packages.join(" ") },
{ name: "APK_IMAGE", value: realDeps.apk.image },
],
volumeMounts: [{ name: "work", mountPath: "/work" }],
},
{
name: "npm-install",
image: realDeps.npm.image,
imagePullPolicy: realDeps.imagePullPolicy,
command: ["/bin/sh", "-lc"],
args: [realDepsNpmScript(realDeps)],
env: [
...commonEnv,
{ name: "NPM_CONFIG_REGISTRY", value: realDeps.npm.registry },
{ name: "NPM_IMAGE", value: realDeps.npm.image },
],
volumeMounts: [{ name: "work", mountPath: "/work" }],
},
{
name: "go-download",
image: realDeps.go.image,
imagePullPolicy: realDeps.imagePullPolicy,
command: ["/bin/sh", "-lc"],
args: [realDepsGoScript()],
env: [
...commonEnv,
{ name: "GOPROXY", value: realDeps.go.goProxy },
{ name: "GO_MODULES", value: realDeps.go.modules.join(" ") },
{ name: "GO_IMAGE", value: realDeps.go.image },
],
volumeMounts: [{ name: "work", mountPath: "/work" }],
},
],
containers: [{
name: "summary",
image: realDeps.apk.image,
imagePullPolicy: "IfNotPresent",
command: ["/bin/sh", "-lc"],
args: [realDepsSummaryScript(realDeps)],
env: commonEnv,
volumeMounts: [{ name: "work", mountPath: "/work" }],
}],
volumes: [{ name: "work", emptyDir: { sizeLimit: "6Gi" } }],
},
},
},
};
}
function requireRealDeps(profile: K3sBuildBenchmarkProfile): K3sRealDepsSpec {
if (profile.realDeps === undefined) throw new Error(`profiles.${profile.id}.realDeps is required for workload=${profile.workload}`);
return profile.realDeps;
}
function realDepsApkScript(): string {
return `set -eu
mkdir -p /work/stages
started_epoch="$(date +%s)"
started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
printf 'UNIDESK_K3S_REAL_DEPS_EVENT stage=apk target=%s profile=%s run=%s image=%s packages="%s"\\n' "$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$APK_IMAGE" "$APK_PACKAGES"
if grep -R -E 'npmmirror|daocloud|aliyun|tuna|ustc|huaweicloud' /etc/apk/repositories >/tmp/apk-mirror-check.out 2>/dev/null; then
cat /tmp/apk-mirror-check.out >&2
echo "unexpected apk mirror in base image" >&2
exit 42
fi
apk update
apk add --no-cache $APK_PACKAGES
apk_mib="$(du -sk /usr/bin /usr/lib /usr/include /usr/local 2>/dev/null | awk '{s+=$1} END {printf "%d", int((s+1023)/1024)}')"
{
printf 'realDepsStartedEpoch=%s\\n' "$started_epoch"
printf 'realDepsStartedAt=%s\\n' "$started_at"
printf 'apkMiB=%s\\n' "$apk_mib"
} > /work/stages/apk.env
printf 'UNIDESK_K3S_REAL_DEPS_STAGE {"stage":"apk","ok":true,"image":"%s","installedMiB":%s}\\n' "$APK_IMAGE" "$apk_mib"
`;
}
function realDepsNpmScript(realDeps: K3sRealDepsSpec): string {
const packageJson = JSON.stringify({ private: true, dependencies: realDeps.npm.packages }, null, 2);
return `set -eu
mkdir -p /work/stages /work/npm/project /work/npm/cache
cd /work/npm/project
cat > package.json <<'JSON'
${packageJson}
JSON
printf 'UNIDESK_K3S_REAL_DEPS_EVENT stage=npm target=%s profile=%s run=%s image=%s registry=%s\\n' "$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$NPM_IMAGE" "$NPM_CONFIG_REGISTRY"
npm install --ignore-scripts --no-audit --no-fund --cache /work/npm/cache --registry "$NPM_CONFIG_REGISTRY"
npm_mib="$(du -sk /work/npm/cache /work/npm/project/node_modules 2>/dev/null | awk '{s+=$1} END {printf "%d", int((s+1023)/1024)}')"
printf 'npmMiB=%s\\n' "$npm_mib" > /work/stages/npm.env
printf 'UNIDESK_K3S_REAL_DEPS_STAGE {"stage":"npm","ok":true,"image":"%s","installedMiB":%s}\\n' "$NPM_IMAGE" "$npm_mib"
`;
}
function realDepsGoScript(): string {
return `set -eu
mkdir -p /work/stages /work/go/module /work/go/gopath /work/go/gomodcache
cd /work/go/module
if ! command -v go >/dev/null 2>&1 && [ -x /usr/local/go/bin/go ]; then
export PATH="/usr/local/go/bin:\${PATH:-/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}"
fi
if ! command -v go >/dev/null 2>&1; then
echo "go-runtime-missing PATH=\${PATH:-}" >&2
exit 127
fi
go version
go mod init unidesk.local/proxy-benchmark
printf 'UNIDESK_K3S_REAL_DEPS_EVENT stage=go target=%s profile=%s run=%s image=%s goproxy=%s modules="%s"\\n' "$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$GO_IMAGE" "$GOPROXY" "$GO_MODULES"
export GOPATH=/work/go/gopath
export GOMODCACHE=/work/go/gomodcache
for module in $GO_MODULES; do
go get "$module"
done
go mod download -x all
go_mib="$(du -sk /work/go/gomodcache /work/go/gopath/pkg 2>/dev/null | awk '{s+=$1} END {printf "%d", int((s+1023)/1024)}')"
printf 'goMiB=%s\\n' "$go_mib" > /work/stages/go.env
printf 'UNIDESK_K3S_REAL_DEPS_STAGE {"stage":"go","ok":true,"image":"%s","downloadedMiB":%s}\\n' "$GO_IMAGE" "$go_mib"
`;
}
function realDepsSummaryScript(realDeps: K3sRealDepsSpec): string {
return `set -eu
apkMiB=0
npmMiB=0
goMiB=0
realDepsStartedEpoch="$(date +%s)"
realDepsStartedAt="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
[ -f /work/stages/apk.env ] && . /work/stages/apk.env
[ -f /work/stages/npm.env ] && . /work/stages/npm.env
[ -f /work/stages/go.env ] && . /work/stages/go.env
completed_epoch="$(date +%s)"
completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
real_deps_mib=$((apkMiB + npmMiB + goMiB))
duration=$((completed_epoch - realDepsStartedEpoch))
pod_ip="$(hostname -i 2>/dev/null | awk '{print $1}')"
printf 'UNIDESK_K3S_REAL_DEPS_RESULT {"ok":true,"target":"%s","profile":"%s","runId":"%s","startedAt":"%s","completedAt":"%s","durationSeconds":%s,"podIp":"%s","apkMiB":%s,"npmMiB":%s,"goMiB":%s,"realDepsMiB":%s,"minProxyMiB":%s,"imagePullMode":"kubelet-containerd","apkImage":"%s","npmImage":"%s","goImage":"%s"}\\n' \\
"$BENCHMARK_TARGET" "$BENCHMARK_PROFILE" "$BENCHMARK_RUN_ID" "$realDepsStartedAt" "$completed_at" "$duration" "$pod_ip" "$apkMiB" "$npmMiB" "$goMiB" "$real_deps_mib" "$MIN_PROXY_MIB" ${shQuote(realDeps.apk.image)} ${shQuote(realDeps.npm.image)} ${shQuote(realDeps.go.image)}
`;
}
function workloadScript(profile: K3sBuildBenchmarkProfile): string {
return `set -eu
started_epoch="$(date +%s)"
@@ -530,6 +812,24 @@ printf '{"ok":true,"jobName":"%s","namespace":"%s","target":"%s","runId":"%s","p
`;
}
function cleanupScript(target: Sub2ApiTargetConfig, profile: K3sBuildBenchmarkProfile): string {
const selector = `app.kubernetes.io/name=${BENCHMARK_APP},unidesk.ai/benchmark-profile=${profile.id},unidesk.ai/runtime-node=${target.id.toLowerCase()}`;
return `
set -eu
namespace=${shQuote(target.namespace)}
selector=${shQuote(selector)}
before="$(kubectl -n "$namespace" get jobs -l "$selector" --no-headers 2>/dev/null | wc -l | tr -d ' ')"
kubectl -n "$namespace" delete jobs -l "$selector" --ignore-not-found >/tmp/k3s-build-benchmark-cleanup.out 2>/tmp/k3s-build-benchmark-cleanup.err
after="$(kubectl -n "$namespace" get jobs -l "$selector" --no-headers 2>/dev/null | wc -l | tr -d ' ')"
deleted=$((before - after))
python3 - "$before" "$after" "$deleted" <<'PY'
import json, sys
before, after, deleted = sys.argv[1:4]
print(json.dumps({"ok": True, "before": int(before), "after": int(after), "deleted": int(deleted), "detail": "matching jobs deleted"}, ensure_ascii=False))
PY
`;
}
function statusScript(target: Sub2ApiTargetConfig, profile: K3sBuildBenchmarkProfile, tailLines: number): string {
const selector = `app.kubernetes.io/name=${BENCHMARK_APP},unidesk.ai/benchmark-profile=${profile.id},unidesk.ai/runtime-node=${target.id.toLowerCase()}`;
return `
@@ -561,26 +861,55 @@ pods = json.loads(pods_result.stdout or "{}").get("items", []) if pods_result.re
pods.sort(key=lambda item: item.get("metadata", {}).get("creationTimestamp", ""))
pod_name = pods[-1].get("metadata", {}).get("name") if pods else None
waiting_reasons = []
container_names = []
if pods:
for container_status in (pods[-1].get("status", {}) or {}).get("containerStatuses", []) or []:
waiting = ((container_status.get("state") or {}).get("waiting") or {})
pod_status = pods[-1].get("status", {}) or {}
status_groups = []
status_groups.extend((pod_status.get("initContainerStatuses") or []))
status_groups.extend((pod_status.get("containerStatuses") or []))
for container_status in status_groups:
container_name = container_status.get("name") or "container"
container_names.append(container_name)
image_name = container_status.get("image") or "-"
state_record = container_status.get("state") or {}
waiting = (state_record.get("waiting") or {})
if waiting:
reason = waiting.get("reason") or "waiting"
message = waiting.get("message") or ""
waiting_reasons.append((reason + " " + message).strip())
logs = ""
if pod_name:
logs_result = kubectl(["logs", pod_name, "--tail", str(tail_lines)])
logs = logs_result.stdout if logs_result.returncode == 0 else logs_result.stderr
full_logs = ""
if pod_name:
full_result = kubectl(["logs", pod_name, "--tail", "800"])
full_logs = full_result.stdout if full_result.returncode == 0 else ""
waiting_reasons.append((container_name + " " + image_name + " " + reason + " " + message).strip())
terminated = (state_record.get("terminated") or {})
if terminated and int(terminated.get("exitCode") or 0) != 0:
reason = terminated.get("reason") or "terminated"
message = terminated.get("message") or ""
waiting_reasons.append(f"{container_name} {image_name} terminated exit={terminated.get('exitCode')} reason={reason} {message}".strip())
def collect_logs(lines):
if not pod_name:
return ""
chunks = []
seen = set()
for name in container_names:
if name in seen:
continue
seen.add(name)
log_result = kubectl(["logs", pod_name, "-c", name, "--tail", str(lines)])
text = log_result.stdout if log_result.returncode == 0 else log_result.stderr
if text:
chunks.append(f"[{name}]\\n{text}")
return "\\n".join(chunks)
logs = collect_logs(tail_lines)
full_logs = collect_logs(800)
match = None
for line in reversed(full_logs.splitlines()):
if line.startswith("UNIDESK_K3S_BUILD_BENCHMARK_RESULT "):
marker = None
if "UNIDESK_K3S_BUILD_BENCHMARK_RESULT " in line:
marker = "UNIDESK_K3S_BUILD_BENCHMARK_RESULT "
elif "UNIDESK_K3S_REAL_DEPS_RESULT " in line:
marker = "UNIDESK_K3S_REAL_DEPS_RESULT "
if marker is not None:
try:
match = json.loads(line.split(" ", 1)[1])
match = json.loads(line.split(marker, 1)[1])
except Exception:
match = None
break
@@ -604,6 +933,12 @@ elif "ImagePullBackOff" in tail_text or "ErrImagePull" in tail_text:
failure_family = "image-pull"
elif "apt-get" in tail_text and ("Failed" in tail_text or "Unable to" in tail_text):
failure_family = "apt-download"
elif "apk-add" in tail_text and ("ERROR:" in tail_text or "temporary error" in tail_text or "Permission denied" in tail_text):
failure_family = "apk-download"
elif "npm-install" in tail_text and ("npm ERR!" in tail_text or "EAI_AGAIN" in tail_text or "ETIMEDOUT" in tail_text):
failure_family = "npm-download"
elif "go-download" in tail_text and ("terminated exit=" in tail_text or "i/o timeout" in tail_text or "connection refused" in tail_text or "no such host" in tail_text or "proxyconnect tcp" in tail_text or "TLS handshake timeout" in tail_text or "go-runtime-missing" in tail_text):
failure_family = "go-download"
elif "curl:" in tail_text or "urllib.error.HTTPError" in tail_text or "urllib.error.URLError" in tail_text:
failure_family = "dependency-download"
elif "payload-too-small" in tail_text:
@@ -666,6 +1001,29 @@ function trafficSpec(target: Sub2ApiTargetConfig): EgressProxyTrafficSpec {
}
function normalizeStatus(plan: TargetPlan, parsed: unknown, result: CommandResult): TargetStatus {
if (typeof parsed !== "object" || parsed === null) {
const state = result.exitCode === 0 ? "transport-unparseable" : "transport-failed";
return {
ok: false,
targetId: plan.targetId,
profile: plan.profile?.id ?? "-",
state,
jobName: "-",
runId: "-",
startedAt: "-",
completedAt: "-",
durationSeconds: null,
outputMiB: null,
downloadMiB: null,
payloadMiB: plan.profile?.payloadMiB ?? null,
apkMiB: null,
npmMiB: null,
goMiB: null,
realDepsMiB: null,
failureFamily: result.timedOut ? "transport-timeout" : state,
logTail: (result.stderr || result.stdout).slice(-4000),
};
}
const data = typeof parsed === "object" && parsed !== null ? parsed as Record<string, unknown> : {};
const jobResult = record(data.result);
const state = text(data.state, result.exitCode === 0 ? "unknown" : "failed");
@@ -683,6 +1041,10 @@ function normalizeStatus(plan: TargetPlan, parsed: unknown, result: CommandResul
outputMiB: nullableNumber(jobResult.outputMiB),
downloadMiB: nullableNumber(jobResult.downloadMiB),
payloadMiB: nullableNumber(jobResult.payloadMiB),
apkMiB: nullableNumber(jobResult.apkMiB),
npmMiB: nullableNumber(jobResult.npmMiB),
goMiB: nullableNumber(jobResult.goMiB),
realDepsMiB: nullableNumber(jobResult.realDepsMiB),
failureFamily: text(data.failureFamily, data.ok === true ? "none" : state === "running" || state === "pending" ? "in-progress" : text(data.reason, "unknown")),
logTail: text(data.logTail, result.stderr.slice(-2000)),
};
@@ -703,6 +1065,10 @@ function blockedStatus(plan: TargetPlan, profile: string): TargetStatus {
outputMiB: null,
downloadMiB: null,
payloadMiB: plan.profile?.payloadMiB ?? null,
apkMiB: null,
npmMiB: null,
goMiB: null,
realDepsMiB: null,
failureFamily: plan.blocker ?? "blocked",
logTail: plan.detail ?? "",
};
@@ -734,9 +1100,8 @@ function readK3sBuildBenchmarkConfig(): K3sBuildBenchmarkConfig {
function profileSpec(id: string, raw: Record<string, unknown>): K3sBuildBenchmarkProfile {
const workload = stringField(raw, "workload", `profiles.${id}`);
if (workload !== "k3s-build") throw new Error(`profiles.${id}.workload must be k3s-build`);
const imagePullPolicy = stringField(raw, "imagePullPolicy", `profiles.${id}`);
if (imagePullPolicy !== "Always" && imagePullPolicy !== "IfNotPresent" && imagePullPolicy !== "Never") throw new Error(`profiles.${id}.imagePullPolicy must be Always, IfNotPresent, or Never`);
if (workload !== "k3s-build" && workload !== "k3s-real-deps") throw new Error(`profiles.${id}.workload must be k3s-build or k3s-real-deps`);
const imagePullPolicy = imagePullPolicyField(raw, "imagePullPolicy", `profiles.${id}`);
const noMirror = asRecord(raw.noMirror, `profiles.${id}.noMirror`);
const registryMirror = stringField(noMirror, "registryMirror", `profiles.${id}.noMirror`);
if (registryMirror !== "forbidden") throw new Error(`profiles.${id}.noMirror.registryMirror must be forbidden`);
@@ -765,6 +1130,34 @@ function profileSpec(id: string, raw: Record<string, unknown>): K3sBuildBenchmar
chunks: integerField(dependencyDownload, "chunks", `profiles.${id}.dependencyDownload`),
expectedMiB: integerField(dependencyDownload, "expectedMiB", `profiles.${id}.dependencyDownload`),
},
realDeps: raw.realDeps === undefined ? undefined : realDepsSpec(asRecord(raw.realDeps, `profiles.${id}.realDeps`), `profiles.${id}.realDeps`),
};
}
function realDepsSpec(raw: Record<string, unknown>, path: string): K3sRealDepsSpec {
const apk = asRecord(raw.apk, `${path}.apk`);
const npm = asRecord(raw.npm, `${path}.npm`);
const go = asRecord(raw.go, `${path}.go`);
return {
minProxyMiB: integerField(raw, "minProxyMiB", path),
imagePullPolicy: imagePullPolicyField(raw, "imagePullPolicy", path),
apk: {
image: stringField(apk, "image", `${path}.apk`),
packages: stringArrayField(apk, "packages", `${path}.apk`),
expectedMiB: integerField(apk, "expectedMiB", `${path}.apk`),
},
npm: {
image: stringField(npm, "image", `${path}.npm`),
registry: stringField(npm, "registry", `${path}.npm`),
packages: stringRecordField(npm, "packages", `${path}.npm`),
expectedMiB: integerField(npm, "expectedMiB", `${path}.npm`),
},
go: {
image: stringField(go, "image", `${path}.go`),
goProxy: stringField(go, "goProxy", `${path}.go`),
modules: stringArrayField(go, "modules", `${path}.go`),
expectedMiB: integerField(go, "expectedMiB", `${path}.go`),
},
};
}
@@ -776,9 +1169,7 @@ function targetOverridesField(raw: Record<string, unknown>, path: string): Recor
const override: K3sBuildBenchmarkTargetOverride = {};
if (recordValue.image !== undefined) override.image = stringField(recordValue, "image", `${path}.targetOverrides.${targetId}`);
if (recordValue.imagePullPolicy !== undefined) {
const pullPolicy = stringField(recordValue, "imagePullPolicy", `${path}.targetOverrides.${targetId}`);
if (pullPolicy !== "Always" && pullPolicy !== "IfNotPresent" && pullPolicy !== "Never") throw new Error(`${path}.targetOverrides.${targetId}.imagePullPolicy must be Always, IfNotPresent, or Never`);
override.imagePullPolicy = pullPolicy;
override.imagePullPolicy = imagePullPolicyField(recordValue, "imagePullPolicy", `${path}.targetOverrides.${targetId}`);
}
return [targetId, override];
}));
@@ -856,6 +1247,20 @@ function stringArrayField(obj: Record<string, unknown>, key: string, path: strin
return [...value] as string[];
}
function stringRecordField(obj: Record<string, unknown>, key: string, path: string): Record<string, string> {
const value = obj[key];
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path}.${key} must be an object`);
const entries = Object.entries(value as Record<string, unknown>);
if (entries.some(([name, item]) => name.length === 0 || typeof item !== "string" || item.length === 0)) throw new Error(`${path}.${key} must contain string values`);
return Object.fromEntries(entries) as Record<string, string>;
}
function imagePullPolicyField(obj: Record<string, unknown>, key: string, path: string): ImagePullPolicy {
const value = stringField(obj, key, path);
if (value !== "Always" && value !== "IfNotPresent" && value !== "Never") throw new Error(`${path}.${key} must be Always, IfNotPresent, or Never`);
return value;
}
function table(headers: string[], rows: string[][]): string[] {
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0)));
const render = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd();