|
|
|
@@ -4,6 +4,7 @@ import { rootPath } from "./config";
|
|
|
|
|
import { runCommand, type CommandResult } from "./command";
|
|
|
|
|
import { egressBenchmarkCompactResult, egressBenchmarkDryRun, egressBenchmarkStartScript, egressBenchmarkStatusScript, type EgressBenchmarkSpec } from "./egress-proxy-benchmark";
|
|
|
|
|
import { resolveEgressProxySourceRef } from "./egress-proxy-sources";
|
|
|
|
|
import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
|
|
|
|
import type { RenderedCliResult } from "./output";
|
|
|
|
|
import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets";
|
|
|
|
|
|
|
|
|
@@ -13,6 +14,7 @@ type InfraAction = "plan" | "status" | "apply";
|
|
|
|
|
type ToolsImageAction = "status" | "build" | "logs";
|
|
|
|
|
type ArgoAction = "status" | "apply" | "logs";
|
|
|
|
|
type EgressBenchmarkAction = "benchmark" | "status" | "logs";
|
|
|
|
|
type CiBuildBenchmarkAction = "benchmark" | "status" | "logs";
|
|
|
|
|
|
|
|
|
|
interface InfraOptions {
|
|
|
|
|
action: InfraAction;
|
|
|
|
@@ -56,6 +58,39 @@ interface EgressBenchmarkOptions {
|
|
|
|
|
tailLines: number;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface CiBuildBenchmarkOptions {
|
|
|
|
|
action: CiBuildBenchmarkAction;
|
|
|
|
|
node: string;
|
|
|
|
|
lane: string;
|
|
|
|
|
profile: string;
|
|
|
|
|
dryRun: boolean;
|
|
|
|
|
confirm: boolean;
|
|
|
|
|
timeoutSeconds: number;
|
|
|
|
|
tailLines: number;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface CiBuildBenchmarkCachePolicy {
|
|
|
|
|
noPipelineRunReuse: boolean;
|
|
|
|
|
forceFullBuild: boolean;
|
|
|
|
|
forbidGitopsCatalogReuse: boolean;
|
|
|
|
|
forbidDependencyCache: boolean;
|
|
|
|
|
forbidBuildkitCache: boolean;
|
|
|
|
|
forbidRegistryMirror: boolean;
|
|
|
|
|
forbidLocalPreheatedImages: boolean;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface CiBuildBenchmarkProfileSpec {
|
|
|
|
|
profile: string;
|
|
|
|
|
runtimeLaneConfigRef: string;
|
|
|
|
|
pipelineRunPrefix: string;
|
|
|
|
|
catalogPathTemplate: string;
|
|
|
|
|
imageTagMode: "full";
|
|
|
|
|
pipelineTimeoutSeconds: number;
|
|
|
|
|
cachePolicy: CiBuildBenchmarkCachePolicy;
|
|
|
|
|
requiredTimings: readonly string[];
|
|
|
|
|
failureFamilies: readonly string[];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
interface ControlPlaneEgressProxySpec {
|
|
|
|
|
mode: "k8s-service-cluster-ip";
|
|
|
|
|
clientName: string;
|
|
|
|
@@ -162,6 +197,7 @@ interface ControlPlaneTargetSpec {
|
|
|
|
|
buildMode: string;
|
|
|
|
|
};
|
|
|
|
|
};
|
|
|
|
|
ciBuildBenchmarks: readonly CiBuildBenchmarkProfileSpec[];
|
|
|
|
|
argo: {
|
|
|
|
|
namespace: string;
|
|
|
|
|
projectName: string;
|
|
|
|
@@ -216,6 +252,11 @@ export function runHwlabNodeControlPlaneInfra(args: string[]): Record<string, un
|
|
|
|
|
const { config, node, target } = controlPlaneContext(options.node, options.lane);
|
|
|
|
|
return runEgressBenchmarkCommand(config, node, target, options);
|
|
|
|
|
}
|
|
|
|
|
if (args[0] === "ci-build-benchmark") {
|
|
|
|
|
const options = parseCiBuildBenchmarkOptions(args.slice(1));
|
|
|
|
|
const { config, node, target } = controlPlaneContext(options.node, options.lane);
|
|
|
|
|
return runCiBuildBenchmarkCommand(config, node, target, options);
|
|
|
|
|
}
|
|
|
|
|
const options = parseInfraOptions(args);
|
|
|
|
|
const { config, node, target } = controlPlaneContext(options.node, options.lane);
|
|
|
|
|
|
|
|
|
@@ -255,6 +296,9 @@ export function hwlabNodeControlPlaneInfraHelp(): Record<string, unknown> {
|
|
|
|
|
"bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node D601 --lane v03",
|
|
|
|
|
"bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark --node D601 --lane v03 --profile no-mirror --confirm",
|
|
|
|
|
"bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark status --node D601 --lane v03 --profile no-mirror",
|
|
|
|
|
"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark --node D601 --lane v03 --profile no-mirror-full --confirm",
|
|
|
|
|
"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node D601 --lane v03 --profile no-mirror-full",
|
|
|
|
|
"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node D601 --lane v03 --profile no-mirror-full",
|
|
|
|
|
],
|
|
|
|
|
g14Consistency: "D601 target fields mirror the existing G14 runtime lane control-plane vocabulary: source branch, gitops branch/path, Pipeline, PipelineRun prefix, ServiceAccount, Argo Application, and git-mirror read/write/sync/flush status concepts.",
|
|
|
|
|
};
|
|
|
|
@@ -539,6 +583,272 @@ function renderControlPlaneBenchmarkResult(result: Record<string, unknown>): Ren
|
|
|
|
|
return { ok: result.ok !== false, command: renderCell(result.command, "hwlab nodes control-plane infra egress-benchmark"), renderedText: lines.join("\n"), contentType: "text/plain" };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function runCiBuildBenchmarkCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: CiBuildBenchmarkOptions): RenderedCliResult {
|
|
|
|
|
const profile = ciBuildBenchmarkProfileForTarget(target, options.profile);
|
|
|
|
|
const runtime = ciBuildBenchmarkRuntimeSpec(node, target, profile);
|
|
|
|
|
if (options.action === "status" || options.action === "logs") {
|
|
|
|
|
const result = runTransK3s(node.kubeRoute, ciBuildBenchmarkStatusScript(target, profile, options.tailLines, options.action === "logs"), options.timeoutSeconds);
|
|
|
|
|
const parsed = parseRemoteJson(result.stdout);
|
|
|
|
|
const job = typeof parsed === "object" && parsed !== null ? parsed as Record<string, unknown> : { stdoutPreview: result.stdout.slice(0, 2000) };
|
|
|
|
|
return renderCiBuildBenchmarkResult({
|
|
|
|
|
ok: result.exitCode === 0 && ciBuildBenchmarkLiveOk(job, runtime.serviceIds),
|
|
|
|
|
command: `hwlab nodes control-plane infra ci-build-benchmark ${options.action}`,
|
|
|
|
|
configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH,
|
|
|
|
|
node: node.id,
|
|
|
|
|
lane: target.lane,
|
|
|
|
|
profile: profile.profile,
|
|
|
|
|
mode: options.action,
|
|
|
|
|
mutation: false,
|
|
|
|
|
benchmark: ciBuildBenchmarkDefinitionSummary(runtime, target, profile),
|
|
|
|
|
job,
|
|
|
|
|
result: compactCommandResult(result),
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const head = resolveCiBuildBenchmarkSourceHead(runtime);
|
|
|
|
|
if (head.sourceCommit === null) {
|
|
|
|
|
throw new Error(`failed to resolve ${runtime.gitUrl} refs/heads/${runtime.sourceBranch}: ${head.result.stderr || head.result.stdout || `exit ${head.result.exitCode}`}`);
|
|
|
|
|
}
|
|
|
|
|
const pipelineRun = ciBuildBenchmarkPipelineRunName(profile.pipelineRunPrefix, head.sourceCommit);
|
|
|
|
|
const catalogPath = ciBuildBenchmarkCatalogPath(profile, pipelineRun);
|
|
|
|
|
const manifest = ciBuildBenchmarkPipelineRunManifest(runtime, target, profile, head.sourceCommit, pipelineRun, catalogPath);
|
|
|
|
|
const plan = {
|
|
|
|
|
...ciBuildBenchmarkDefinitionSummary(runtime, target, profile),
|
|
|
|
|
pipelineRun,
|
|
|
|
|
sourceCommit: head.sourceCommit,
|
|
|
|
|
catalogPath,
|
|
|
|
|
manifest: manifestObjectSummary([manifest]),
|
|
|
|
|
manifestSha256: sha256Short(JSON.stringify(manifest)),
|
|
|
|
|
sourceHead: compactCommandResult(head.result),
|
|
|
|
|
};
|
|
|
|
|
if (options.dryRun) {
|
|
|
|
|
return renderCiBuildBenchmarkResult({
|
|
|
|
|
ok: true,
|
|
|
|
|
command: "hwlab nodes control-plane infra ci-build-benchmark",
|
|
|
|
|
configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH,
|
|
|
|
|
node: node.id,
|
|
|
|
|
lane: target.lane,
|
|
|
|
|
profile: profile.profile,
|
|
|
|
|
mode: "dry-run",
|
|
|
|
|
mutation: false,
|
|
|
|
|
plan,
|
|
|
|
|
next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark --node ${node.id} --lane ${target.lane} --profile ${profile.profile} --confirm` },
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const result = runTransK3s(node.kubeRoute, ciBuildBenchmarkStartScript(target, profile, manifest, runtime.pipeline, pipelineRun, head.sourceCommit, catalogPath), options.timeoutSeconds);
|
|
|
|
|
const parsed = parseRemoteJson(result.stdout);
|
|
|
|
|
return renderCiBuildBenchmarkResult({
|
|
|
|
|
ok: result.exitCode === 0,
|
|
|
|
|
command: "hwlab nodes control-plane infra ci-build-benchmark",
|
|
|
|
|
configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH,
|
|
|
|
|
node: node.id,
|
|
|
|
|
lane: target.lane,
|
|
|
|
|
profile: profile.profile,
|
|
|
|
|
mode: "confirmed-start",
|
|
|
|
|
mutation: result.exitCode === 0,
|
|
|
|
|
benchmark: ciBuildBenchmarkDefinitionSummary(runtime, target, profile),
|
|
|
|
|
start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) },
|
|
|
|
|
result: compactCommandResult(result),
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkProfileForTarget(target: ControlPlaneTargetSpec, profileName: string): CiBuildBenchmarkProfileSpec {
|
|
|
|
|
const profile = target.ciBuildBenchmarks.find((item) => item.profile === profileName);
|
|
|
|
|
if (profile === undefined) {
|
|
|
|
|
const known = target.ciBuildBenchmarks.map((item) => item.profile).join(", ") || "<none>";
|
|
|
|
|
throw new Error(`ci-build-benchmark profile ${profileName} is not declared for target ${target.id}; known profiles: ${known}`);
|
|
|
|
|
}
|
|
|
|
|
return profile;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkRuntimeSpec(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec): HwlabRuntimeLaneSpec {
|
|
|
|
|
if (!isHwlabRuntimeLane(target.lane)) throw new Error(`target ${target.id}.lane=${target.lane} is not a runtime lane in config/hwlab-node-lanes.yaml`);
|
|
|
|
|
const runtime = hwlabRuntimeLaneSpecForNode(target.lane, node.id);
|
|
|
|
|
if (runtime.nodeId !== node.id || runtime.lane !== target.lane) throw new Error(`runtime lane mismatch for ${node.id}/${target.lane}`);
|
|
|
|
|
if (!profile.runtimeLaneConfigRef.startsWith("config/hwlab-node-lanes.yaml#")) {
|
|
|
|
|
throw new Error(`targets.${target.id}.ciBuildBenchmarks.${profile.profile}.runtimeLaneConfigRef must point at config/hwlab-node-lanes.yaml`);
|
|
|
|
|
}
|
|
|
|
|
return runtime;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function resolveCiBuildBenchmarkSourceHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } {
|
|
|
|
|
const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], rootPath(), { timeoutMs: 45_000 });
|
|
|
|
|
if (result.exitCode !== 0 || result.timedOut) return { sourceCommit: null, result };
|
|
|
|
|
const match = /[0-9a-f]{40}/iu.exec(`${result.stdout}\n${result.stderr}`);
|
|
|
|
|
return { sourceCommit: match?.[0].toLowerCase() ?? null, result };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkPipelineRunName(prefix: string, sourceCommit: string): string {
|
|
|
|
|
const suffix = `${sourceCommit.slice(0, 12)}-${Date.now().toString(36)}`;
|
|
|
|
|
return `${prefix}-${suffix}`.slice(0, 63).replace(/-+$/u, "");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkCatalogPath(profile: CiBuildBenchmarkProfileSpec, pipelineRun: string): string {
|
|
|
|
|
return profile.catalogPathTemplate.replace(/\{profile\}/gu, profile.profile).replace(/\{pipelineRun\}/gu, pipelineRun);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkDefinitionSummary(runtime: HwlabRuntimeLaneSpec, target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec): Record<string, unknown> {
|
|
|
|
|
return {
|
|
|
|
|
targetId: target.id,
|
|
|
|
|
profile: profile.profile,
|
|
|
|
|
runtimeLaneConfigRef: profile.runtimeLaneConfigRef,
|
|
|
|
|
namespace: target.ciNamespace,
|
|
|
|
|
pipeline: runtime.pipeline,
|
|
|
|
|
serviceAccountName: runtime.serviceAccountName,
|
|
|
|
|
sourceBranch: runtime.sourceBranch,
|
|
|
|
|
gitReadUrl: runtime.gitReadUrl,
|
|
|
|
|
gitWriteUrl: runtime.gitWriteUrl,
|
|
|
|
|
gitopsBranch: runtime.gitopsBranch,
|
|
|
|
|
runtimePath: runtime.runtimePath,
|
|
|
|
|
registryPrefix: runtime.registryPrefix,
|
|
|
|
|
baseImage: runtime.baseImage,
|
|
|
|
|
services: runtime.serviceIds,
|
|
|
|
|
imageTagMode: profile.imageTagMode,
|
|
|
|
|
cachePolicy: profile.cachePolicy,
|
|
|
|
|
requiredTimings: profile.requiredTimings,
|
|
|
|
|
failureFamilies: profile.failureFamilies,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkPipelineRunManifest(
|
|
|
|
|
runtime: HwlabRuntimeLaneSpec,
|
|
|
|
|
target: ControlPlaneTargetSpec,
|
|
|
|
|
profile: CiBuildBenchmarkProfileSpec,
|
|
|
|
|
sourceCommit: string,
|
|
|
|
|
pipelineRun: string,
|
|
|
|
|
catalogPath: string,
|
|
|
|
|
): Record<string, unknown> {
|
|
|
|
|
return {
|
|
|
|
|
apiVersion: "tekton.dev/v1",
|
|
|
|
|
kind: "PipelineRun",
|
|
|
|
|
metadata: {
|
|
|
|
|
name: pipelineRun,
|
|
|
|
|
namespace: target.ciNamespace,
|
|
|
|
|
labels: {
|
|
|
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
|
|
|
"hwlab.pikastech.local/gitops-target": runtime.lane,
|
|
|
|
|
"hwlab.pikastech.local/source-commit": sourceCommit,
|
|
|
|
|
"hwlab.pikastech.local/trigger": "unidesk-ci-build-benchmark",
|
|
|
|
|
"unidesk.ai/benchmark": "ci-build",
|
|
|
|
|
"unidesk.ai/benchmark-profile": profile.profile,
|
|
|
|
|
},
|
|
|
|
|
annotations: {
|
|
|
|
|
"hwlab.pikastech.local/node": runtime.nodeId,
|
|
|
|
|
"hwlab.pikastech.local/source-branch": runtime.sourceBranch,
|
|
|
|
|
"hwlab.pikastech.local/gitops-branch": runtime.gitopsBranch,
|
|
|
|
|
"hwlab.pikastech.local/runtime-path": runtime.runtimePath,
|
|
|
|
|
"hwlab.pikastech.local/network-profile": runtime.networkProfileId,
|
|
|
|
|
"hwlab.pikastech.local/download-profile": runtime.downloadProfileId,
|
|
|
|
|
"unidesk.ai/issue": "pikasTech/unidesk#1010",
|
|
|
|
|
"unidesk.ai/cache-policy": JSON.stringify(profile.cachePolicy),
|
|
|
|
|
"unidesk.ai/catalog-path": catalogPath,
|
|
|
|
|
"unidesk.ai/runtime-lane-config-ref": profile.runtimeLaneConfigRef,
|
|
|
|
|
"unidesk.ai/required-timings": profile.requiredTimings.join(","),
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
spec: {
|
|
|
|
|
pipelineRef: { name: runtime.pipeline },
|
|
|
|
|
timeouts: { pipeline: `${profile.pipelineTimeoutSeconds}s` },
|
|
|
|
|
taskRunTemplate: {
|
|
|
|
|
serviceAccountName: runtime.serviceAccountName,
|
|
|
|
|
podTemplate: {
|
|
|
|
|
hostNetwork: true,
|
|
|
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
|
|
|
securityContext: { fsGroup: 1000 },
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
params: [
|
|
|
|
|
{ name: "git-url", value: runtime.gitUrl },
|
|
|
|
|
{ name: "git-read-url", value: runtime.gitReadUrl },
|
|
|
|
|
{ name: "git-write-url", value: runtime.gitWriteUrl },
|
|
|
|
|
{ name: "source-branch", value: runtime.sourceBranch },
|
|
|
|
|
{ name: "gitops-branch", value: runtime.gitopsBranch },
|
|
|
|
|
{ name: "lane", value: runtime.lane },
|
|
|
|
|
{ name: "catalog-path", value: catalogPath },
|
|
|
|
|
{ name: "image-tag-mode", value: profile.imageTagMode },
|
|
|
|
|
{ name: "runtime-path", value: runtime.runtimePath },
|
|
|
|
|
{ name: "revision", value: sourceCommit },
|
|
|
|
|
{ name: "registry-prefix", value: runtime.registryPrefix },
|
|
|
|
|
{ name: "services", value: runtime.serviceIds.join(",") },
|
|
|
|
|
{ name: "base-image", value: runtime.baseImage },
|
|
|
|
|
],
|
|
|
|
|
workspaces: [
|
|
|
|
|
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
|
|
|
|
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } },
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function renderCiBuildBenchmarkResult(result: Record<string, unknown>): RenderedCliResult {
|
|
|
|
|
const start = record(result.start);
|
|
|
|
|
const job = record(result.job);
|
|
|
|
|
const plan = record(result.plan);
|
|
|
|
|
const benchmark = record(result.benchmark ?? plan);
|
|
|
|
|
const next = record(result.next);
|
|
|
|
|
const pipelineRun = record(job.pipelineRun);
|
|
|
|
|
const taskRows = ciBuildBenchmarkTaskRows(job);
|
|
|
|
|
const serviceRows = ciBuildBenchmarkServiceRows(job, benchmark.services);
|
|
|
|
|
const failures = ciBuildBenchmarkFailureRows(job, serviceRows);
|
|
|
|
|
const profile = renderCell(result.profile ?? benchmark.profile ?? plan.profile, "unknown");
|
|
|
|
|
const node = renderCell(result.node);
|
|
|
|
|
const lane = renderCell(result.lane);
|
|
|
|
|
const target = `${node}/${lane}`;
|
|
|
|
|
const statusText = renderCell(job.state ?? pipelineRun.status ?? start.state, result.ok === false ? "failed" : "ok");
|
|
|
|
|
const pipelineRunName = renderCell(job.pipelineRunName ?? pipelineRun.name ?? start.pipelineRun ?? plan.pipelineRun);
|
|
|
|
|
const sourceCommit = renderCell(pipelineRun.sourceCommit ?? start.sourceCommit ?? plan.sourceCommit);
|
|
|
|
|
const statusCommand = renderCell(start.statusCommand, `bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node ${node} --lane ${lane} --profile ${profile}`);
|
|
|
|
|
const logsCommand = renderCell(start.logsCommand, `bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node ${node} --lane ${lane} --profile ${profile}`);
|
|
|
|
|
const logTail = typeof job.logTail === "string" ? job.logTail.trimEnd() : "";
|
|
|
|
|
const lines = [
|
|
|
|
|
"HWLAB K3S CI BUILD BENCHMARK",
|
|
|
|
|
"",
|
|
|
|
|
...renderTable(["TARGET", "PROFILE", "MODE", "STATUS", "PIPELINERUN", "SOURCE"], [[target, profile, renderCell(result.mode ?? optionsModeFromCommand(result.command)), statusText, pipelineRunName, shortDisplay(sourceCommit)]]),
|
|
|
|
|
"",
|
|
|
|
|
"POLICY",
|
|
|
|
|
...renderTable(["FIELD", "VALUE"], [
|
|
|
|
|
["pipeline", renderCell(benchmark.pipeline)],
|
|
|
|
|
["catalogPath", renderCell(start.catalogPath ?? plan.catalogPath ?? pipelineRun.catalogPath)],
|
|
|
|
|
["services", String((Array.isArray(benchmark.services) ? benchmark.services : []).length)],
|
|
|
|
|
["cachePolicy", JSON.stringify(benchmark.cachePolicy ?? {})],
|
|
|
|
|
["requiredTimings", Array.isArray(benchmark.requiredTimings) ? benchmark.requiredTimings.join(",") : "-"],
|
|
|
|
|
]),
|
|
|
|
|
"",
|
|
|
|
|
serviceRows.length === 0 ? "SERVICES\n-" : [
|
|
|
|
|
"SERVICES",
|
|
|
|
|
...renderTable(["SERVICE", "TASK", "STATUS", "DURATION", "FAILURE"], serviceRows.map((row) => [
|
|
|
|
|
renderCell(row.service),
|
|
|
|
|
renderCell(row.task),
|
|
|
|
|
renderCell(row.status),
|
|
|
|
|
renderCell(row.duration),
|
|
|
|
|
renderCell(row.failure),
|
|
|
|
|
])),
|
|
|
|
|
].join("\n"),
|
|
|
|
|
"",
|
|
|
|
|
taskRows.length === 0 ? "TIMINGS\n-" : [
|
|
|
|
|
"TIMINGS",
|
|
|
|
|
...renderTable(["TASK", "STATUS", "DURATION", "START", "END"], taskRows.map((row) => [
|
|
|
|
|
renderCell(row.task),
|
|
|
|
|
renderCell(row.status),
|
|
|
|
|
renderCell(row.duration),
|
|
|
|
|
renderCell(row.start),
|
|
|
|
|
renderCell(row.end),
|
|
|
|
|
])),
|
|
|
|
|
].join("\n"),
|
|
|
|
|
...(failures.length === 0 ? [] : ["", "FAILURE FAMILIES", ...renderTable(["FAMILY", "COUNT", "SCOPE"], failures.map((row) => [renderCell(row.family), renderCell(row.count), renderCell(row.scope)]))]),
|
|
|
|
|
...(logTail.length === 0 ? [] : ["", "LOG TAIL", logTail]),
|
|
|
|
|
"",
|
|
|
|
|
"NEXT",
|
|
|
|
|
` ${renderCell(next.confirm, "")}`,
|
|
|
|
|
` ${statusCommand}`,
|
|
|
|
|
` ${logsCommand}`,
|
|
|
|
|
"",
|
|
|
|
|
"Disclosure: output is bounded; Git/proxy/Secret values are not expanded. A succeeded PipelineRun with missing build-<service> TaskRuns is reported as cache-hit-forbidden.",
|
|
|
|
|
].filter((line) => line !== " ");
|
|
|
|
|
return { ok: result.ok !== false, command: renderCell(result.command, "hwlab nodes control-plane infra ci-build-benchmark"), renderedText: lines.join("\n"), contentType: "text/plain" };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function toolsImageCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ToolsImageOptions): Record<string, unknown> {
|
|
|
|
|
const registry = toolsImageStatus(node, target, options.timeoutSeconds);
|
|
|
|
|
const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "tools-image", options.tailLines), options.timeoutSeconds);
|
|
|
|
@@ -793,6 +1103,30 @@ function parseEgressBenchmarkOptions(args: string[]): EgressBenchmarkOptions {
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function parseCiBuildBenchmarkOptions(args: string[]): CiBuildBenchmarkOptions {
|
|
|
|
|
const first = args[0];
|
|
|
|
|
const action: CiBuildBenchmarkAction = first === "status" || first === "logs"
|
|
|
|
|
? first
|
|
|
|
|
: "benchmark";
|
|
|
|
|
const effectiveArgs = action === "benchmark" ? args : args.slice(1);
|
|
|
|
|
if (first === "--help" || first === "-h" || first === "help") {
|
|
|
|
|
throw new Error("infra ci-build-benchmark usage: ci-build-benchmark [status|logs] --node NODE --lane vNN --profile PROFILE [--dry-run|--confirm]");
|
|
|
|
|
}
|
|
|
|
|
const confirm = effectiveArgs.includes("--confirm");
|
|
|
|
|
const explicitDryRun = effectiveArgs.includes("--dry-run");
|
|
|
|
|
if (confirm && explicitDryRun) throw new Error("ci-build-benchmark accepts only one of --confirm or --dry-run");
|
|
|
|
|
return {
|
|
|
|
|
action,
|
|
|
|
|
node: requiredOption(effectiveArgs, "--node"),
|
|
|
|
|
lane: requiredOption(effectiveArgs, "--lane"),
|
|
|
|
|
profile: requiredOption(effectiveArgs, "--profile"),
|
|
|
|
|
confirm,
|
|
|
|
|
dryRun: action === "benchmark" ? explicitDryRun || !confirm : false,
|
|
|
|
|
timeoutSeconds: positiveIntegerOption(effectiveArgs, "--timeout-seconds", 60, 60),
|
|
|
|
|
tailLines: positiveIntegerOption(effectiveArgs, "--tail-lines", 120, 1000),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function readControlPlaneConfig(): ControlPlaneConfig {
|
|
|
|
|
const parsed = asRecord(Bun.YAML.parse(readFileSync(rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH), "utf8")) as unknown, HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
|
|
|
|
|
const version = numberField(parsed, "version", HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH);
|
|
|
|
@@ -903,6 +1237,54 @@ function argoInstallSpec(raw: Record<string, unknown>, path: string): ControlPla
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkProfileSpecs(raw: unknown, path: string): readonly CiBuildBenchmarkProfileSpec[] {
|
|
|
|
|
if (raw === undefined) return [];
|
|
|
|
|
if (!Array.isArray(raw)) throw new Error(`${path} must be an array`);
|
|
|
|
|
const profiles = raw.map((item, index) => ciBuildBenchmarkProfileSpec(asRecord(item, `${path}[${index}]`), `${path}[${index}]`));
|
|
|
|
|
const names = new Set<string>();
|
|
|
|
|
for (const profile of profiles) {
|
|
|
|
|
if (names.has(profile.profile)) throw new Error(`${path} contains duplicate profile ${profile.profile}`);
|
|
|
|
|
names.add(profile.profile);
|
|
|
|
|
}
|
|
|
|
|
return profiles;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkProfileSpec(raw: Record<string, unknown>, path: string): CiBuildBenchmarkProfileSpec {
|
|
|
|
|
const profile = stringField(raw, "profile", path);
|
|
|
|
|
validateBenchmarkProfileName(profile, `${path}.profile`);
|
|
|
|
|
const pipelineRunPrefix = stringField(raw, "pipelineRunPrefix", path);
|
|
|
|
|
validateKubernetesName(pipelineRunPrefix, `${path}.pipelineRunPrefix`);
|
|
|
|
|
if (pipelineRunPrefix.length > 40) throw new Error(`${path}.pipelineRunPrefix must leave room for source and nonce suffix`);
|
|
|
|
|
const catalogPathTemplate = stringField(raw, "catalogPathTemplate", path);
|
|
|
|
|
validateBenchmarkCatalogPathTemplate(catalogPathTemplate, `${path}.catalogPathTemplate`);
|
|
|
|
|
const imageTagMode = stringField(raw, "imageTagMode", path);
|
|
|
|
|
if (imageTagMode !== "full") throw new Error(`${path}.imageTagMode currently must be full`);
|
|
|
|
|
const timings = asRecord(raw.timings, `${path}.timings`);
|
|
|
|
|
return {
|
|
|
|
|
profile,
|
|
|
|
|
runtimeLaneConfigRef: stringField(raw, "runtimeLaneConfigRef", path),
|
|
|
|
|
pipelineRunPrefix,
|
|
|
|
|
catalogPathTemplate,
|
|
|
|
|
imageTagMode,
|
|
|
|
|
pipelineTimeoutSeconds: positiveConfigIntegerField(raw, "pipelineTimeoutSeconds", path),
|
|
|
|
|
cachePolicy: ciBuildBenchmarkCachePolicy(asRecord(raw.cachePolicy, `${path}.cachePolicy`), `${path}.cachePolicy`),
|
|
|
|
|
requiredTimings: stringArrayField(timings, "requiredStages", `${path}.timings`),
|
|
|
|
|
failureFamilies: stringArrayField(raw, "failureFamilies", path),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkCachePolicy(raw: Record<string, unknown>, path: string): CiBuildBenchmarkCachePolicy {
|
|
|
|
|
return {
|
|
|
|
|
noPipelineRunReuse: booleanField(raw, "noPipelineRunReuse", path),
|
|
|
|
|
forceFullBuild: booleanField(raw, "forceFullBuild", path),
|
|
|
|
|
forbidGitopsCatalogReuse: booleanField(raw, "forbidGitopsCatalogReuse", path),
|
|
|
|
|
forbidDependencyCache: booleanField(raw, "forbidDependencyCache", path),
|
|
|
|
|
forbidBuildkitCache: booleanField(raw, "forbidBuildkitCache", path),
|
|
|
|
|
forbidRegistryMirror: booleanField(raw, "forbidRegistryMirror", path),
|
|
|
|
|
forbidLocalPreheatedImages: booleanField(raw, "forbidLocalPreheatedImages", path),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function imageRewriteSpec(raw: Record<string, unknown>, path: string): ImageRewriteSpec {
|
|
|
|
|
const rewrite = {
|
|
|
|
|
source: stringField(raw, "source", path),
|
|
|
|
@@ -1130,6 +1512,7 @@ function targetSpec(raw: Record<string, unknown>, index: number): ControlPlaneTa
|
|
|
|
|
pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${path}.tekton`),
|
|
|
|
|
toolsImage: toolsImageSpec(toolsImage, `${path}.tekton.toolsImage`),
|
|
|
|
|
},
|
|
|
|
|
ciBuildBenchmarks: ciBuildBenchmarkProfileSpecs(raw.ciBuildBenchmarks, `${path}.ciBuildBenchmarks`),
|
|
|
|
|
argo: {
|
|
|
|
|
namespace: stringField(argo, "namespace", `${path}.argo`),
|
|
|
|
|
projectName: stringField(argo, "projectName", `${path}.argo`),
|
|
|
|
@@ -2632,6 +3015,233 @@ printf '{"started":true,"pid":%s,"stateDir":"%s","statusCommand":"bun scripts/cl
|
|
|
|
|
`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkStartScript(
|
|
|
|
|
target: ControlPlaneTargetSpec,
|
|
|
|
|
profile: CiBuildBenchmarkProfileSpec,
|
|
|
|
|
manifest: Record<string, unknown>,
|
|
|
|
|
pipelineName: string,
|
|
|
|
|
pipelineRun: string,
|
|
|
|
|
sourceCommit: string,
|
|
|
|
|
catalogPath: string,
|
|
|
|
|
): string {
|
|
|
|
|
const stateDir = ciBuildBenchmarkStateDir(target, profile.profile);
|
|
|
|
|
const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64");
|
|
|
|
|
return `
|
|
|
|
|
set -eu
|
|
|
|
|
state_dir=${shQuote(stateDir)}
|
|
|
|
|
status_file="$state_dir/status.json"
|
|
|
|
|
ns=${shQuote(target.ciNamespace)}
|
|
|
|
|
profile=${shQuote(profile.profile)}
|
|
|
|
|
pipeline=${shQuote(pipelineName)}
|
|
|
|
|
pipeline_run=${shQuote(pipelineRun)}
|
|
|
|
|
source_commit=${shQuote(sourceCommit)}
|
|
|
|
|
catalog_path=${shQuote(catalogPath)}
|
|
|
|
|
mkdir -p "$state_dir"
|
|
|
|
|
previous_run=
|
|
|
|
|
if [ -s "$status_file" ]; then
|
|
|
|
|
previous_run=$(python3 - "$status_file" <<'PY' || true
|
|
|
|
|
import json, sys
|
|
|
|
|
try:
|
|
|
|
|
data=json.load(open(sys.argv[1], encoding="utf-8"))
|
|
|
|
|
print(data.get("pipelineRun") or "")
|
|
|
|
|
except Exception:
|
|
|
|
|
print("")
|
|
|
|
|
PY
|
|
|
|
|
)
|
|
|
|
|
fi
|
|
|
|
|
if [ -n "$previous_run" ]; then
|
|
|
|
|
previous_status=$(kubectl -n "$ns" get pipelinerun "$previous_run" -o 'jsonpath={.status.conditions[?(@.type=="Succeeded")].status}' 2>/dev/null || true)
|
|
|
|
|
if [ -n "$previous_status" ] && [ "$previous_status" != "True" ] && [ "$previous_status" != "False" ]; then
|
|
|
|
|
python3 - "$state_dir" "$previous_run" "$profile" ${shQuote(target.node)} ${shQuote(target.lane)} <<'PY'
|
|
|
|
|
import json, sys
|
|
|
|
|
state_dir, previous_run, profile, node, lane = sys.argv[1:6]
|
|
|
|
|
print(json.dumps({
|
|
|
|
|
"started": False,
|
|
|
|
|
"state": "already-running",
|
|
|
|
|
"pipelineRun": previous_run,
|
|
|
|
|
"stateDir": state_dir,
|
|
|
|
|
"statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node {node} --lane {lane} --profile {profile}",
|
|
|
|
|
"logsCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node {node} --lane {lane} --profile {profile}",
|
|
|
|
|
}, ensure_ascii=False))
|
|
|
|
|
PY
|
|
|
|
|
exit 0
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
manifest_path="$state_dir/$pipeline_run.json"
|
|
|
|
|
printf '%s' ${shQuote(manifestB64)} | base64 -d >"$manifest_path"
|
|
|
|
|
set +e
|
|
|
|
|
pipeline_check=$(kubectl -n "$ns" get pipeline "$pipeline" -o name 2>&1)
|
|
|
|
|
pipeline_check_rc=$?
|
|
|
|
|
create_output=
|
|
|
|
|
create_rc=0
|
|
|
|
|
if [ "$pipeline_check_rc" = 0 ]; then
|
|
|
|
|
create_output=$(kubectl create -f "$manifest_path" 2>&1)
|
|
|
|
|
create_rc=$?
|
|
|
|
|
else
|
|
|
|
|
create_output="$pipeline_check"
|
|
|
|
|
create_rc="$pipeline_check_rc"
|
|
|
|
|
fi
|
|
|
|
|
printf '%s\\n' "$create_output" >"$state_dir/create.log"
|
|
|
|
|
python3 - "$status_file" "$state_dir" "$pipeline_run" "$source_commit" "$profile" "$catalog_path" "$create_rc" "$create_output" ${shQuote(target.node)} ${shQuote(target.lane)} <<'PY'
|
|
|
|
|
import datetime, json, sys
|
|
|
|
|
status_file, state_dir, pipeline_run, source_commit, profile, catalog_path, rc_raw, output, node, lane = sys.argv[1:11]
|
|
|
|
|
rc=int(rc_raw or "0")
|
|
|
|
|
payload={
|
|
|
|
|
"started": rc == 0,
|
|
|
|
|
"state": "started" if rc == 0 else "failed",
|
|
|
|
|
"pipelineRun": pipeline_run,
|
|
|
|
|
"sourceCommit": source_commit,
|
|
|
|
|
"profile": profile,
|
|
|
|
|
"catalogPath": catalog_path,
|
|
|
|
|
"stateDir": state_dir,
|
|
|
|
|
"createdAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
|
|
|
"exitCode": rc,
|
|
|
|
|
"createOutputTail": output[-2000:],
|
|
|
|
|
"statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node {node} --lane {lane} --profile {profile}",
|
|
|
|
|
"logsCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node {node} --lane {lane} --profile {profile}",
|
|
|
|
|
}
|
|
|
|
|
open(status_file, "w", encoding="utf-8").write(json.dumps(payload, ensure_ascii=False))
|
|
|
|
|
print(json.dumps(payload, ensure_ascii=False))
|
|
|
|
|
PY
|
|
|
|
|
exit "$create_rc"
|
|
|
|
|
`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkStatusScript(target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec, tailLines: number, includeLogs: boolean): string {
|
|
|
|
|
const stateDir = ciBuildBenchmarkStateDir(target, profile.profile);
|
|
|
|
|
return `
|
|
|
|
|
set +e
|
|
|
|
|
state_dir=${shQuote(stateDir)}
|
|
|
|
|
status_file="$state_dir/status.json"
|
|
|
|
|
ns=${shQuote(target.ciNamespace)}
|
|
|
|
|
profile=${shQuote(profile.profile)}
|
|
|
|
|
tail_lines=${shQuote(String(tailLines))}
|
|
|
|
|
include_logs=${includeLogs ? "true" : "false"}
|
|
|
|
|
tmp_dir=$(mktemp -d)
|
|
|
|
|
pipeline_run=
|
|
|
|
|
if [ -s "$status_file" ]; then
|
|
|
|
|
pipeline_run=$(python3 - "$status_file" <<'PY' || true
|
|
|
|
|
import json, sys
|
|
|
|
|
try:
|
|
|
|
|
data=json.load(open(sys.argv[1], encoding="utf-8"))
|
|
|
|
|
print(data.get("pipelineRun") or "")
|
|
|
|
|
except Exception:
|
|
|
|
|
print("")
|
|
|
|
|
PY
|
|
|
|
|
)
|
|
|
|
|
fi
|
|
|
|
|
if [ -z "$pipeline_run" ]; then
|
|
|
|
|
pipeline_run=$(kubectl -n "$ns" get pipelinerun -l "unidesk.ai/benchmark=ci-build,unidesk.ai/benchmark-profile=$profile" -o 'jsonpath={range .items[*]}{.metadata.creationTimestamp}{" "}{.metadata.name}{"\\n"}{end}' 2>/dev/null | sort | tail -n 1 | awk '{print $2}')
|
|
|
|
|
fi
|
|
|
|
|
if [ -n "$pipeline_run" ]; then
|
|
|
|
|
kubectl -n "$ns" get pipelinerun "$pipeline_run" -o json >"$tmp_dir/pipelinerun.json" 2>"$tmp_dir/pipelinerun.err"
|
|
|
|
|
kubectl -n "$ns" get taskrun -l "tekton.dev/pipelineRun=$pipeline_run" -o json >"$tmp_dir/taskruns.json" 2>"$tmp_dir/taskruns.err"
|
|
|
|
|
kubectl -n "$ns" get pod -l "tekton.dev/pipelineRun=$pipeline_run" -o json >"$tmp_dir/pods.json" 2>"$tmp_dir/pods.err"
|
|
|
|
|
if [ "$include_logs" = true ]; then
|
|
|
|
|
kubectl -n "$ns" logs -l "tekton.dev/pipelineRun=$pipeline_run" --all-containers --tail="$tail_lines" --prefix=true >"$tmp_dir/logs.txt" 2>"$tmp_dir/logs.err" || true
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
python3 - "$state_dir" "$status_file" "$tmp_dir" "$pipeline_run" "$include_logs" "$tail_lines" <<'PY'
|
|
|
|
|
import json, pathlib, sys
|
|
|
|
|
state_dir=pathlib.Path(sys.argv[1])
|
|
|
|
|
status_path=pathlib.Path(sys.argv[2])
|
|
|
|
|
tmp_dir=pathlib.Path(sys.argv[3])
|
|
|
|
|
pipeline_run=sys.argv[4]
|
|
|
|
|
include_logs=sys.argv[5] == "true"
|
|
|
|
|
tail_lines=int(sys.argv[6])
|
|
|
|
|
|
|
|
|
|
def read_json(path):
|
|
|
|
|
try:
|
|
|
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
except Exception:
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
def read_text(path, limit=4000):
|
|
|
|
|
try:
|
|
|
|
|
return path.read_text(encoding="utf-8", errors="replace")[-limit:]
|
|
|
|
|
except Exception:
|
|
|
|
|
return ""
|
|
|
|
|
|
|
|
|
|
def succeeded_condition(obj):
|
|
|
|
|
for cond in obj.get("status", {}).get("conditions", []) or []:
|
|
|
|
|
if cond.get("type") == "Succeeded":
|
|
|
|
|
return cond
|
|
|
|
|
return {}
|
|
|
|
|
|
|
|
|
|
status=None
|
|
|
|
|
if status_path.exists():
|
|
|
|
|
status=read_json(status_path)
|
|
|
|
|
pr=read_json(tmp_dir / "pipelinerun.json") if pipeline_run else None
|
|
|
|
|
trs=read_json(tmp_dir / "taskruns.json") if pipeline_run else None
|
|
|
|
|
pods=read_json(tmp_dir / "pods.json") if pipeline_run else None
|
|
|
|
|
pr_cond=succeeded_condition(pr or {})
|
|
|
|
|
task_runs=[]
|
|
|
|
|
for item in (trs or {}).get("items", []) or []:
|
|
|
|
|
cond=succeeded_condition(item)
|
|
|
|
|
labels=item.get("metadata", {}).get("labels", {}) or {}
|
|
|
|
|
task_runs.append({
|
|
|
|
|
"name": item.get("metadata", {}).get("name"),
|
|
|
|
|
"pipelineTask": labels.get("tekton.dev/pipelineTask"),
|
|
|
|
|
"status": cond.get("status"),
|
|
|
|
|
"reason": cond.get("reason"),
|
|
|
|
|
"message": cond.get("message"),
|
|
|
|
|
"startTime": item.get("status", {}).get("startTime"),
|
|
|
|
|
"completionTime": item.get("status", {}).get("completionTime"),
|
|
|
|
|
"podName": item.get("status", {}).get("podName"),
|
|
|
|
|
})
|
|
|
|
|
pod_rows=[]
|
|
|
|
|
for item in (pods or {}).get("items", []) or []:
|
|
|
|
|
phase=item.get("status", {}).get("phase")
|
|
|
|
|
pod_rows.append({
|
|
|
|
|
"name": item.get("metadata", {}).get("name"),
|
|
|
|
|
"phase": phase,
|
|
|
|
|
"startTime": item.get("status", {}).get("startTime"),
|
|
|
|
|
})
|
|
|
|
|
state="not-started"
|
|
|
|
|
if pr:
|
|
|
|
|
status_value=pr_cond.get("status")
|
|
|
|
|
if status_value == "True":
|
|
|
|
|
state="succeeded"
|
|
|
|
|
elif status_value == "False":
|
|
|
|
|
state="failed"
|
|
|
|
|
elif status_value:
|
|
|
|
|
state="running"
|
|
|
|
|
else:
|
|
|
|
|
state="pending"
|
|
|
|
|
elif pipeline_run:
|
|
|
|
|
state="missing"
|
|
|
|
|
payload={
|
|
|
|
|
"stateDir": str(state_dir),
|
|
|
|
|
"status": status,
|
|
|
|
|
"pipelineRunName": pipeline_run or None,
|
|
|
|
|
"state": state,
|
|
|
|
|
"pipelineRun": None if not pr else {
|
|
|
|
|
"name": pr.get("metadata", {}).get("name"),
|
|
|
|
|
"status": pr_cond.get("status"),
|
|
|
|
|
"reason": pr_cond.get("reason"),
|
|
|
|
|
"message": pr_cond.get("message"),
|
|
|
|
|
"createdAt": pr.get("metadata", {}).get("creationTimestamp"),
|
|
|
|
|
"startTime": pr.get("status", {}).get("startTime"),
|
|
|
|
|
"completionTime": pr.get("status", {}).get("completionTime"),
|
|
|
|
|
"sourceCommit": (pr.get("metadata", {}).get("labels", {}) or {}).get("hwlab.pikastech.local/source-commit"),
|
|
|
|
|
"catalogPath": (pr.get("metadata", {}).get("annotations", {}) or {}).get("unidesk.ai/catalog-path"),
|
|
|
|
|
},
|
|
|
|
|
"taskRuns": task_runs,
|
|
|
|
|
"pods": pod_rows,
|
|
|
|
|
"errors": {
|
|
|
|
|
"pipelinerun": read_text(tmp_dir / "pipelinerun.err"),
|
|
|
|
|
"taskruns": read_text(tmp_dir / "taskruns.err"),
|
|
|
|
|
"pods": read_text(tmp_dir / "pods.err"),
|
|
|
|
|
"logs": read_text(tmp_dir / "logs.err") if include_logs else "",
|
|
|
|
|
},
|
|
|
|
|
"logTail": read_text(tmp_dir / "logs.txt", max(4000, tail_lines * 300)) if include_logs else "",
|
|
|
|
|
}
|
|
|
|
|
print(json.dumps(payload, ensure_ascii=False))
|
|
|
|
|
PY
|
|
|
|
|
rm -rf "$tmp_dir"
|
|
|
|
|
`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function remoteJobStatusScript(target: ControlPlaneTargetSpec, name: "tools-image" | "argo", tailLines: number): string {
|
|
|
|
|
const stateDir = remoteJobStateDir(target, name);
|
|
|
|
|
return `
|
|
|
|
@@ -2734,6 +3344,159 @@ function parseRemoteJson(text: string): unknown {
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkStateDir(target: ControlPlaneTargetSpec, profile: string): string {
|
|
|
|
|
return `/tmp/unidesk-hwlab-node-control-plane/${target.id}/ci-build-benchmark-${profile}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkLiveOk(job: Record<string, unknown>, expectedServices: readonly string[]): boolean {
|
|
|
|
|
const pipelineRun = record(job.pipelineRun);
|
|
|
|
|
const pipelineStatus = renderCell(pipelineRun.status, "");
|
|
|
|
|
if (pipelineStatus === "False") return false;
|
|
|
|
|
if (pipelineStatus !== "True") return true;
|
|
|
|
|
const taskRuns = ciBuildBenchmarkTaskRunRecords(job);
|
|
|
|
|
for (const service of expectedServices) {
|
|
|
|
|
if (!taskRuns.some((task) => task.pipelineTask === `build-${service}`)) return false;
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkTaskRows(job: Record<string, unknown>): Record<string, string>[] {
|
|
|
|
|
const pipelineRun = record(job.pipelineRun);
|
|
|
|
|
const rows: Record<string, string>[] = [];
|
|
|
|
|
if (Object.keys(pipelineRun).length > 0) {
|
|
|
|
|
rows.push({
|
|
|
|
|
task: "pipeline-total",
|
|
|
|
|
status: ciBuildBenchmarkStatusText(pipelineRun.status),
|
|
|
|
|
duration: durationBetweenIso(pipelineRun.startTime, pipelineRun.completionTime),
|
|
|
|
|
start: shortIsoTime(pipelineRun.startTime),
|
|
|
|
|
end: shortIsoTime(pipelineRun.completionTime),
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
const taskRuns = ciBuildBenchmarkTaskRunRecords(job).sort((left, right) => renderCell(left.startTime, "").localeCompare(renderCell(right.startTime, "")));
|
|
|
|
|
for (const task of taskRuns) {
|
|
|
|
|
rows.push({
|
|
|
|
|
task: renderCell(task.pipelineTask ?? task.name),
|
|
|
|
|
status: ciBuildBenchmarkStatusText(task.status),
|
|
|
|
|
duration: durationBetweenIso(task.startTime, task.completionTime),
|
|
|
|
|
start: shortIsoTime(task.startTime),
|
|
|
|
|
end: shortIsoTime(task.completionTime),
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
return rows;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkServiceRows(job: Record<string, unknown>, servicesValue: unknown): Record<string, string>[] {
|
|
|
|
|
const services = ciBuildBenchmarkExpectedServices(servicesValue);
|
|
|
|
|
if (services.length === 0) return [];
|
|
|
|
|
const pipelineRun = record(job.pipelineRun);
|
|
|
|
|
const pipelineTerminal = pipelineRun.status === "True" || pipelineRun.status === "False";
|
|
|
|
|
const taskRuns = ciBuildBenchmarkTaskRunRecords(job);
|
|
|
|
|
return services.map((service) => {
|
|
|
|
|
const task = taskRuns.find((item) => item.pipelineTask === `build-${service}`);
|
|
|
|
|
if (task === undefined) {
|
|
|
|
|
const status = pipelineTerminal ? "missing" : "pending";
|
|
|
|
|
return {
|
|
|
|
|
service,
|
|
|
|
|
task: `build-${service}`,
|
|
|
|
|
status,
|
|
|
|
|
duration: "-",
|
|
|
|
|
failure: pipelineRun.status === "True" ? "cache-hit-forbidden" : "-",
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const taskStatus = ciBuildBenchmarkStatusText(task.status);
|
|
|
|
|
const failure = task.status === "False" ? classifyCiBuildBenchmarkFailure(`${renderCell(task.reason, "")}\n${renderCell(task.message, "")}`) : "-";
|
|
|
|
|
return {
|
|
|
|
|
service,
|
|
|
|
|
task: renderCell(task.pipelineTask ?? task.name),
|
|
|
|
|
status: taskStatus,
|
|
|
|
|
duration: durationBetweenIso(task.startTime, task.completionTime),
|
|
|
|
|
failure,
|
|
|
|
|
};
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkFailureRows(job: Record<string, unknown>, serviceRows: readonly Record<string, string>[]): Record<string, string>[] {
|
|
|
|
|
const counts = new Map<string, { count: number; scopes: string[] }>();
|
|
|
|
|
const add = (family: string, scope: string): void => {
|
|
|
|
|
if (family === "-" || family.length === 0) return;
|
|
|
|
|
const existing = counts.get(family) ?? { count: 0, scopes: [] };
|
|
|
|
|
existing.count += 1;
|
|
|
|
|
if (!existing.scopes.includes(scope)) existing.scopes.push(scope);
|
|
|
|
|
counts.set(family, existing);
|
|
|
|
|
};
|
|
|
|
|
for (const row of serviceRows) add(row.failure, row.service);
|
|
|
|
|
const pipelineRun = record(job.pipelineRun);
|
|
|
|
|
if (pipelineRun.status === "False") {
|
|
|
|
|
add(classifyCiBuildBenchmarkFailure(`${renderCell(pipelineRun.reason, "")}\n${renderCell(pipelineRun.message, "")}`), "pipeline");
|
|
|
|
|
}
|
|
|
|
|
return [...counts.entries()].map(([family, value]) => ({ family, count: String(value.count), scope: value.scopes.join(",") }));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkTaskRunRecords(job: Record<string, unknown>): Record<string, unknown>[] {
|
|
|
|
|
return Array.isArray(job.taskRuns) ? job.taskRuns.map(record) : [];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkExpectedServices(value: unknown): string[] {
|
|
|
|
|
return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : [];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ciBuildBenchmarkStatusText(value: unknown): string {
|
|
|
|
|
if (value === "True") return "succeeded";
|
|
|
|
|
if (value === "False") return "failed";
|
|
|
|
|
if (value === "Unknown") return "running";
|
|
|
|
|
return renderCell(value, "pending");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function classifyCiBuildBenchmarkFailure(text: string): string {
|
|
|
|
|
const value = text.toLowerCase();
|
|
|
|
|
if (/cache-hit-forbidden|reused-from|reuse/i.test(text)) return "cache-hit-forbidden";
|
|
|
|
|
if (/no such host|could not resolve|enotfound|dns/i.test(text)) return "dns";
|
|
|
|
|
if (/429|rate limit|too many requests|toomanyrequests/i.test(text)) return "rate-limit";
|
|
|
|
|
if (/tls|certificate|x509|timeout|timed out|i\/o timeout/i.test(text)) return "tls-timeout";
|
|
|
|
|
if (/proxy|connect|connection reset|connection refused|econn/i.test(text)) return "proxy-connect";
|
|
|
|
|
if (/unauthorized|authentication required|permission denied|forbidden|denied/i.test(text)) return "auth";
|
|
|
|
|
if (/imagepullbackoff|errimagepull|imagepolicy|pull access denied/i.test(text)) return "image-policy";
|
|
|
|
|
if (/push|registry|blob upload|manifest invalid|manifest unknown/i.test(text)) return "registry-push";
|
|
|
|
|
return value.trim().length === 0 ? "unknown" : "build-script";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function durationBetweenIso(startValue: unknown, endValue: unknown): string {
|
|
|
|
|
if (typeof startValue !== "string" || startValue.length === 0) return "-";
|
|
|
|
|
const start = Date.parse(startValue);
|
|
|
|
|
if (!Number.isFinite(start)) return "-";
|
|
|
|
|
const end = typeof endValue === "string" && endValue.length > 0 ? Date.parse(endValue) : Date.now();
|
|
|
|
|
if (!Number.isFinite(end) || end < start) return "-";
|
|
|
|
|
return formatDurationMs(end - start);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function formatDurationMs(ms: number): string {
|
|
|
|
|
const seconds = Math.round(ms / 1000);
|
|
|
|
|
const minutes = Math.floor(seconds / 60);
|
|
|
|
|
const rest = seconds % 60;
|
|
|
|
|
return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function shortIsoTime(value: unknown): string {
|
|
|
|
|
if (typeof value !== "string" || value.length === 0) return "-";
|
|
|
|
|
return value.replace(/^\d{4}-\d{2}-\d{2}T/u, "").replace(/Z$/u, "Z");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function shortDisplay(value: string): string {
|
|
|
|
|
return /^[0-9a-f]{40}$/iu.test(value) ? value.slice(0, 12).toLowerCase() : value;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function validateBenchmarkProfileName(value: string, path: string): void {
|
|
|
|
|
if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 63) throw new Error(`${path} must be a DNS-label style benchmark profile`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function validateBenchmarkCatalogPathTemplate(value: string, path: string): void {
|
|
|
|
|
if (!value.includes("{profile}") || !value.includes("{pipelineRun}")) throw new Error(`${path} must include {profile} and {pipelineRun}`);
|
|
|
|
|
if (value.startsWith("/") || value.includes("\n") || value.includes("\r")) throw new Error(`${path} must be a relative repo path template`);
|
|
|
|
|
const rendered = value.replace(/\{profile\}/gu, "profile").replace(/\{pipelineRun\}/gu, "pipeline-run");
|
|
|
|
|
if (rendered.split("/").some((segment) => segment === ".." || segment.length === 0)) throw new Error(`${path} must not contain empty or parent path segments`);
|
|
|
|
|
if (!rendered.endsWith(".json")) throw new Error(`${path} must render to a JSON catalog path`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function asRecord(value: unknown, path: string): Record<string, unknown> {
|
|
|
|
|
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`);
|
|
|
|
|
return value as Record<string, unknown>;
|
|
|
|
|
Lyon