diff --git a/.agents/skills/unidesk-cicd/references/full.md b/.agents/skills/unidesk-cicd/references/full.md index 582124fb..afc07ecb 100644 --- a/.agents/skills/unidesk-cicd/references/full.md +++ b/.agents/skills/unidesk-cicd/references/full.md @@ -91,6 +91,9 @@ bun scripts/cli.ts hwlab nodes control-plane infra runtime-image preload --node bun scripts/cli.ts hwlab nodes control-plane infra runtime-image logs --node D601 --lane v03 bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03 bun scripts/cli.ts hwlab nodes control-plane infra argo apply --node D601 --lane v03 --confirm +bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark --node D601 --lane v03 --profile no-mirror-full --confirm +bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node D601 --lane v03 --profile no-mirror-full +bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node D601 --lane v03 --profile no-mirror-full bun scripts/cli.ts hwlab nodes control-plane status --node D601 --lane v03 [--pipeline-run |--source-commit ] [--full|--raw] bun scripts/cli.ts hwlab nodes control-plane trigger-current --node D601 --lane v03 --confirm --wait bun scripts/cli.ts hwlab nodes control-plane sync --node D601 --lane v03 --confirm @@ -98,6 +101,8 @@ bun scripts/cli.ts hwlab nodes control-plane sync --node D601 --lane v03 --confi 从 `config/hwlab-node-control-plane.yaml` 渲染 D601 HWLAB v03 的节点本地 CI/CD、git-mirror、Tekton、runtime dependency image preload 和 Argo 前置对象。confirmed apply 只做 control-plane bootstrap,不触发 runtime rollout,不创建 PK01 DB,也不修改 Caddy/FRP。node-local registry 镜像只能作为 tools image 或 runtime dependency 的输出 artifact;输入 base/pull image 必须是 YAML 中声明的公开 registry 来源,缺失 output image 时通过 `status.next.blockers` 或 `runtime-image status` 暴露。D601 Argo CD 安装也必须由 YAML 声明:官方 manifest URL、版本、镜像 rewrite/preload、CRD、期望 workload 和 AppProject/Application 都来自 YAML,不能使用手工 kubectl/argo CLI 作为正式安装路径。 +`ci-build-benchmark` 是 HWLAB v0.3 k3s CI/CD 全量无缓存构建出网测速入口。profile、独立 catalog path 模板、cache policy、必须输出的 timing 阶段和失败族来自 `config/hwlab-node-control-plane.yaml`;实际 service set、git mirror URL、Pipeline、ServiceAccount、registry prefix 和 base image 仍以 `config/hwlab-node-lanes.yaml` 为准。confirmed benchmark 只创建唯一 PipelineRun 并返回 status/logs 轮询命令;通过证据必须包含每个 `build-` TaskRun,PipelineRun 成功但缺少 build task 要按 `cache-hit-forbidden` 处理。 + `hwlab nodes control-plane status` 默认返回 compact commander summary,只保留 source commit、PipelineRun、Argo、runtime readiness、public probe 和 next action;完整 expected YAML/render target、kubectl result tail、Secret/sourceRef 详情和 probe 原始结果只在 `--full` 或 `--raw` 下展开。 `hwlab nodes control-plane sync --confirm` 是 Argo runtime 收敛修复入口:会先按 YAML `runtimeStore.postgres.mode=local-k3s` 同步本地 postgres bootstrap Secret,再终止卡住的 running Argo operation、删除失败 hook Job,并在 StatefulSet template 已更新但旧 controller-revision pod 因 `ImagePullBackOff` / `ErrImagePull` / `CrashLoopBackOff` 卡住时受控删除该旧 pod,让 StatefulSet 按最新 revision 重建。不要手工裸删 pod;需要解除这类死锁时走该入口。 diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index cdf7c826..38a114b2 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -5,6 +5,7 @@ metadata: relatedIssues: - 290 - 491 + - 1010 - 1119 imagePolicy: requireReproducibleBuildSource: true @@ -155,6 +156,39 @@ targets: - docker.io/docker:29-cli buildOwner: D601 buildMode: node-local + ciBuildBenchmarks: + - profile: no-mirror-full + runtimeLaneConfigRef: config/hwlab-node-lanes.yaml#lanes.v03.targets.D601 + pipelineRunPrefix: hwlab-v03-ci-bench + catalogPathTemplate: .unidesk/ci-build-benchmark/{profile}/{pipelineRun}/artifact-catalog.json + imageTagMode: full + pipelineTimeoutSeconds: 7200 + cachePolicy: + noPipelineRunReuse: true + forceFullBuild: true + forbidGitopsCatalogReuse: true + forbidDependencyCache: true + forbidBuildkitCache: true + forbidRegistryMirror: true + forbidLocalPreheatedImages: true + timings: + requiredStages: + - source-fetch + - dependency-install + - base-image-pull + - service-image-build + - registry-push + - pipeline-total + failureFamilies: + - dns + - proxy-connect + - tls-timeout + - rate-limit + - auth + - cache-hit-forbidden + - image-policy + - build-script + - registry-push argo: namespace: argocd projectName: hwlab-d601 diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 86bb36d8..5628742e 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -32,6 +32,8 @@ G14/D601 v03 的 bootstrap admin password 是 HWLAB runtime Secret 生命周期 `hwlab nodes control-plane infra plan|status|apply --node D601 --lane v03` 是 D601 HWLAB v03 节点本地 k3s、CI/CD 与 git-mirror 前置控制面的 YAML 驱动入口,配置真相源是 `config/hwlab-node-control-plane.yaml`。`plan` 只读展示 YAML target、host k3s node config 摘要和将渲染的 control-plane 对象;`status` 只读观察 k3s systemd drop-in 与 node `capacity/allocatable.pods`、D601 Tekton、CI namespace、git-mirror、Argo、node-local registry 和 tools image readiness;`apply --dry-run` 只输出 manifest 与 host config 摘要;`apply --confirm` 按 YAML 收敛 D601 host k3s drop-in 和 control-plane bootstrap 对象,只有 host k3s 配置或 live pod capacity 未收敛时才重启 k3s,不触发 HWLAB runtime rollout,不创建 PK01 DB,也不修改 Caddy/FRP。D601 host 侧 k3s pre-start 修正也必须写成 YAML `execStartPre` argv,不做手工 systemd 热改;当 kube API 已不可用时,`apply` 可用同一 YAML 渲染出的 host 脚本经 node-local tools image/Docker fallback 恢复 systemd drop-in,输出仍只给对象名、SHA、exit code 和摘要。k3s pod capacity 等可调数值只以 YAML 为准,长期参考不复制具体数值;tools image 的 node-local registry 地址只能作为输出 artifact,输入 base image 必须由 YAML 声明为公开 registry 来源,缺少 output image 时应在 `status.next.blockers` 中体现,而不是把现有 node-local image 当成输入基础镜像。 +`hwlab nodes control-plane infra ci-build-benchmark --node D601 --lane v03 --profile --confirm` 是 HWLAB v0.3 k3s CI/CD 全量无缓存构建出网测速入口,profile、cache policy、独立 catalog path 模板、PipelineRun prefix、必须输出的 timing 阶段和失败族都来自 `config/hwlab-node-control-plane.yaml`。confirmed benchmark 只创建一次唯一 PipelineRun,使用 node-lane YAML 中的实际 HWLAB v0.3 service set、git mirror read/write URL、registry prefix、base image 和 Tekton pipeline;status/logs 通过短连接轮询 PipelineRun/TaskRun 摘要和有界日志。成功的 benchmark 必须出现每个 `build-` TaskRun;如果 PipelineRun 成功但缺少任一 service build task,CLI 必须把该 service 报为 `cache-hit-forbidden`,不能把 catalog/env reuse 当作 #1010 这类性能验收的通过证据。 + `hwlab nodes git-mirror status|sync|flush --node --lane ` 是 node-scoped runtime lane 的 Git mirror 维护入口。`status` 的 `githubSource` / `githubGitops` 来自本地 mirror cache 的 `refs/mirror-stage/...`,不是实时 GitHub API;输出中的 `refSources.githubFieldsAreMirrorStageCache=true` 和 `refSources.cacheRefresh` 给出这一来源和刷新命令。`sync --confirm --wait` 的 k3s Job 遇到 GitHub SSH transient 时,应通过目标 workspace fallback 拉取 GitHub source/gitops 并写回 node-local mirror,输出只披露 commit、mirror write URL 和 fallback 状态。`flush --confirm --wait` 如果已经把 GitOps ref push 到 GitHub,但 post-push fetch/recheck 因 transient SSH 失败而无法刷新 mirror-stage,会标记 `partialSuccess=push-succeeded-fetch-failed`;CLI 应自动执行一次受控 sync 刷新 mirror-stage,若恢复后 `pendingFlush=false` 且 `githubInSync=true`,结果应为 `ok=true` 并输出 `partialSuccessRecovered` / `postPushRecovery`,否则才保留 `degradedReason=node-runtime-git-mirror-flush-post-push-fetch-failed` 和下一步 `sync --confirm --wait`。不要把这种 partial success 解读为需要连续盲目 flush。`hwlab nodes control-plane trigger-current --node --lane --confirm --wait` 会在 source sync 后自动执行必要的 pre-flush,在 PipelineRun terminal 后自动执行必要的 post-flush;progress 事件必须显式输出 `git-mirror-pre-flush` / `git-mirror-post-flush` 的 executed/skipped、jobName、local/github source、local/github GitOps、`pendingFlush` 和 `githubInSync`,且已恢复的 partial success 不能让顶层 trigger-current false-fail。`control-plane status` 仍是只读入口,只暴露 compact `gitMirror` 摘要和下一步 flush 命令,不隐式执行写操作。 PR 合并后触发 node-scoped runtime lane 时,`control-plane status --pipeline-run ` 是某次 PipelineRun 的定点观察入口,但同一输出中的 `sourceHead` / `summary.sourceCommit` 仍可能反映当前分支最新 head;如果触发后又有后续 PR 合并,当前 head 可能已经不是该 PipelineRun 名称中的短 SHA。closeout 证据必须同时写明:PR merge commit、定点 PipelineRun 名称和状态、最终 runtime/GitOps revision、当前 branch tip,以及当前 branch tip 是否包含本次 PR merge commit。不要只凭 `summary.sourceCommit` 反推某个旧 PipelineRun 的源码身份。 diff --git a/scripts/src/help.ts b/scripts/src/help.ts index a1d54456..01e36ee0 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -685,6 +685,8 @@ function hwlabNodeHelpSummary(): unknown { "bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra argo status --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark --node D601 --lane v03 --profile no-mirror --confirm", + "bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark --node D601 --lane v03 --profile no-mirror-full --confirm", + "bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node D601 --lane v03 --profile no-mirror-full", "bun scripts/cli.ts hwlab nodes control-plane status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes git-mirror status --node G14 --lane v03", "bun scripts/cli.ts hwlab nodes hwpod-preinstall plan --node D601 --lane v03 --dry-run", diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 8e225bbc..7f55cb64 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -4,6 +4,7 @@ import { rootPath } from "./config"; import { runCommand, type CommandResult } from "./command"; import { egressBenchmarkCompactResult, egressBenchmarkDryRun, egressBenchmarkStartScript, egressBenchmarkStatusScript, type EgressBenchmarkSpec } from "./egress-proxy-benchmark"; import { resolveEgressProxySourceRef } from "./egress-proxy-sources"; +import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import type { RenderedCliResult } from "./output"; import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./secrets"; @@ -13,6 +14,7 @@ type InfraAction = "plan" | "status" | "apply"; type ToolsImageAction = "status" | "build" | "logs"; type ArgoAction = "status" | "apply" | "logs"; type EgressBenchmarkAction = "benchmark" | "status" | "logs"; +type CiBuildBenchmarkAction = "benchmark" | "status" | "logs"; interface InfraOptions { action: InfraAction; @@ -56,6 +58,39 @@ interface EgressBenchmarkOptions { tailLines: number; } +interface CiBuildBenchmarkOptions { + action: CiBuildBenchmarkAction; + node: string; + lane: string; + profile: string; + dryRun: boolean; + confirm: boolean; + timeoutSeconds: number; + tailLines: number; +} + +interface CiBuildBenchmarkCachePolicy { + noPipelineRunReuse: boolean; + forceFullBuild: boolean; + forbidGitopsCatalogReuse: boolean; + forbidDependencyCache: boolean; + forbidBuildkitCache: boolean; + forbidRegistryMirror: boolean; + forbidLocalPreheatedImages: boolean; +} + +interface CiBuildBenchmarkProfileSpec { + profile: string; + runtimeLaneConfigRef: string; + pipelineRunPrefix: string; + catalogPathTemplate: string; + imageTagMode: "full"; + pipelineTimeoutSeconds: number; + cachePolicy: CiBuildBenchmarkCachePolicy; + requiredTimings: readonly string[]; + failureFamilies: readonly string[]; +} + interface ControlPlaneEgressProxySpec { mode: "k8s-service-cluster-ip"; clientName: string; @@ -162,6 +197,7 @@ interface ControlPlaneTargetSpec { buildMode: string; }; }; + ciBuildBenchmarks: readonly CiBuildBenchmarkProfileSpec[]; argo: { namespace: string; projectName: string; @@ -216,6 +252,11 @@ export function runHwlabNodeControlPlaneInfra(args: string[]): Record { "bun scripts/cli.ts hwlab nodes control-plane infra argo logs --node D601 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark --node D601 --lane v03 --profile no-mirror --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra egress-benchmark status --node D601 --lane v03 --profile no-mirror", + "bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark --node D601 --lane v03 --profile no-mirror-full --confirm", + "bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node D601 --lane v03 --profile no-mirror-full", + "bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node D601 --lane v03 --profile no-mirror-full", ], g14Consistency: "D601 target fields mirror the existing G14 runtime lane control-plane vocabulary: source branch, gitops branch/path, Pipeline, PipelineRun prefix, ServiceAccount, Argo Application, and git-mirror read/write/sync/flush status concepts.", }; @@ -539,6 +583,272 @@ function renderControlPlaneBenchmarkResult(result: Record): Ren return { ok: result.ok !== false, command: renderCell(result.command, "hwlab nodes control-plane infra egress-benchmark"), renderedText: lines.join("\n"), contentType: "text/plain" }; } +function runCiBuildBenchmarkCommand(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: CiBuildBenchmarkOptions): RenderedCliResult { + const profile = ciBuildBenchmarkProfileForTarget(target, options.profile); + const runtime = ciBuildBenchmarkRuntimeSpec(node, target, profile); + if (options.action === "status" || options.action === "logs") { + const result = runTransK3s(node.kubeRoute, ciBuildBenchmarkStatusScript(target, profile, options.tailLines, options.action === "logs"), options.timeoutSeconds); + const parsed = parseRemoteJson(result.stdout); + const job = typeof parsed === "object" && parsed !== null ? parsed as Record : { stdoutPreview: result.stdout.slice(0, 2000) }; + return renderCiBuildBenchmarkResult({ + ok: result.exitCode === 0 && ciBuildBenchmarkLiveOk(job, runtime.serviceIds), + command: `hwlab nodes control-plane infra ci-build-benchmark ${options.action}`, + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + profile: profile.profile, + mode: options.action, + mutation: false, + benchmark: ciBuildBenchmarkDefinitionSummary(runtime, target, profile), + job, + result: compactCommandResult(result), + }); + } + + const head = resolveCiBuildBenchmarkSourceHead(runtime); + if (head.sourceCommit === null) { + throw new Error(`failed to resolve ${runtime.gitUrl} refs/heads/${runtime.sourceBranch}: ${head.result.stderr || head.result.stdout || `exit ${head.result.exitCode}`}`); + } + const pipelineRun = ciBuildBenchmarkPipelineRunName(profile.pipelineRunPrefix, head.sourceCommit); + const catalogPath = ciBuildBenchmarkCatalogPath(profile, pipelineRun); + const manifest = ciBuildBenchmarkPipelineRunManifest(runtime, target, profile, head.sourceCommit, pipelineRun, catalogPath); + const plan = { + ...ciBuildBenchmarkDefinitionSummary(runtime, target, profile), + pipelineRun, + sourceCommit: head.sourceCommit, + catalogPath, + manifest: manifestObjectSummary([manifest]), + manifestSha256: sha256Short(JSON.stringify(manifest)), + sourceHead: compactCommandResult(head.result), + }; + if (options.dryRun) { + return renderCiBuildBenchmarkResult({ + ok: true, + command: "hwlab nodes control-plane infra ci-build-benchmark", + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + profile: profile.profile, + mode: "dry-run", + mutation: false, + plan, + next: { confirm: `bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark --node ${node.id} --lane ${target.lane} --profile ${profile.profile} --confirm` }, + }); + } + + const result = runTransK3s(node.kubeRoute, ciBuildBenchmarkStartScript(target, profile, manifest, runtime.pipeline, pipelineRun, head.sourceCommit, catalogPath), options.timeoutSeconds); + const parsed = parseRemoteJson(result.stdout); + return renderCiBuildBenchmarkResult({ + ok: result.exitCode === 0, + command: "hwlab nodes control-plane infra ci-build-benchmark", + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + profile: profile.profile, + mode: "confirmed-start", + mutation: result.exitCode === 0, + benchmark: ciBuildBenchmarkDefinitionSummary(runtime, target, profile), + start: typeof parsed === "object" && parsed !== null ? parsed : { stdoutPreview: result.stdout.slice(0, 2000) }, + result: compactCommandResult(result), + }); +} + +function ciBuildBenchmarkProfileForTarget(target: ControlPlaneTargetSpec, profileName: string): CiBuildBenchmarkProfileSpec { + const profile = target.ciBuildBenchmarks.find((item) => item.profile === profileName); + if (profile === undefined) { + const known = target.ciBuildBenchmarks.map((item) => item.profile).join(", ") || ""; + throw new Error(`ci-build-benchmark profile ${profileName} is not declared for target ${target.id}; known profiles: ${known}`); + } + return profile; +} + +function ciBuildBenchmarkRuntimeSpec(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec): HwlabRuntimeLaneSpec { + if (!isHwlabRuntimeLane(target.lane)) throw new Error(`target ${target.id}.lane=${target.lane} is not a runtime lane in config/hwlab-node-lanes.yaml`); + const runtime = hwlabRuntimeLaneSpecForNode(target.lane, node.id); + if (runtime.nodeId !== node.id || runtime.lane !== target.lane) throw new Error(`runtime lane mismatch for ${node.id}/${target.lane}`); + if (!profile.runtimeLaneConfigRef.startsWith("config/hwlab-node-lanes.yaml#")) { + throw new Error(`targets.${target.id}.ciBuildBenchmarks.${profile.profile}.runtimeLaneConfigRef must point at config/hwlab-node-lanes.yaml`); + } + return runtime; +} + +function resolveCiBuildBenchmarkSourceHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandResult } { + const result = runCommand(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], rootPath(), { timeoutMs: 45_000 }); + if (result.exitCode !== 0 || result.timedOut) return { sourceCommit: null, result }; + const match = /[0-9a-f]{40}/iu.exec(`${result.stdout}\n${result.stderr}`); + return { sourceCommit: match?.[0].toLowerCase() ?? null, result }; +} + +function ciBuildBenchmarkPipelineRunName(prefix: string, sourceCommit: string): string { + const suffix = `${sourceCommit.slice(0, 12)}-${Date.now().toString(36)}`; + return `${prefix}-${suffix}`.slice(0, 63).replace(/-+$/u, ""); +} + +function ciBuildBenchmarkCatalogPath(profile: CiBuildBenchmarkProfileSpec, pipelineRun: string): string { + return profile.catalogPathTemplate.replace(/\{profile\}/gu, profile.profile).replace(/\{pipelineRun\}/gu, pipelineRun); +} + +function ciBuildBenchmarkDefinitionSummary(runtime: HwlabRuntimeLaneSpec, target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec): Record { + return { + targetId: target.id, + profile: profile.profile, + runtimeLaneConfigRef: profile.runtimeLaneConfigRef, + namespace: target.ciNamespace, + pipeline: runtime.pipeline, + serviceAccountName: runtime.serviceAccountName, + sourceBranch: runtime.sourceBranch, + gitReadUrl: runtime.gitReadUrl, + gitWriteUrl: runtime.gitWriteUrl, + gitopsBranch: runtime.gitopsBranch, + runtimePath: runtime.runtimePath, + registryPrefix: runtime.registryPrefix, + baseImage: runtime.baseImage, + services: runtime.serviceIds, + imageTagMode: profile.imageTagMode, + cachePolicy: profile.cachePolicy, + requiredTimings: profile.requiredTimings, + failureFamilies: profile.failureFamilies, + }; +} + +function ciBuildBenchmarkPipelineRunManifest( + runtime: HwlabRuntimeLaneSpec, + target: ControlPlaneTargetSpec, + profile: CiBuildBenchmarkProfileSpec, + sourceCommit: string, + pipelineRun: string, + catalogPath: string, +): Record { + return { + apiVersion: "tekton.dev/v1", + kind: "PipelineRun", + metadata: { + name: pipelineRun, + namespace: target.ciNamespace, + labels: { + "app.kubernetes.io/part-of": "hwlab", + "hwlab.pikastech.local/gitops-target": runtime.lane, + "hwlab.pikastech.local/source-commit": sourceCommit, + "hwlab.pikastech.local/trigger": "unidesk-ci-build-benchmark", + "unidesk.ai/benchmark": "ci-build", + "unidesk.ai/benchmark-profile": profile.profile, + }, + annotations: { + "hwlab.pikastech.local/node": runtime.nodeId, + "hwlab.pikastech.local/source-branch": runtime.sourceBranch, + "hwlab.pikastech.local/gitops-branch": runtime.gitopsBranch, + "hwlab.pikastech.local/runtime-path": runtime.runtimePath, + "hwlab.pikastech.local/network-profile": runtime.networkProfileId, + "hwlab.pikastech.local/download-profile": runtime.downloadProfileId, + "unidesk.ai/issue": "pikasTech/unidesk#1010", + "unidesk.ai/cache-policy": JSON.stringify(profile.cachePolicy), + "unidesk.ai/catalog-path": catalogPath, + "unidesk.ai/runtime-lane-config-ref": profile.runtimeLaneConfigRef, + "unidesk.ai/required-timings": profile.requiredTimings.join(","), + }, + }, + spec: { + pipelineRef: { name: runtime.pipeline }, + timeouts: { pipeline: `${profile.pipelineTimeoutSeconds}s` }, + taskRunTemplate: { + serviceAccountName: runtime.serviceAccountName, + podTemplate: { + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + securityContext: { fsGroup: 1000 }, + }, + }, + params: [ + { name: "git-url", value: runtime.gitUrl }, + { name: "git-read-url", value: runtime.gitReadUrl }, + { name: "git-write-url", value: runtime.gitWriteUrl }, + { name: "source-branch", value: runtime.sourceBranch }, + { name: "gitops-branch", value: runtime.gitopsBranch }, + { name: "lane", value: runtime.lane }, + { name: "catalog-path", value: catalogPath }, + { name: "image-tag-mode", value: profile.imageTagMode }, + { name: "runtime-path", value: runtime.runtimePath }, + { name: "revision", value: sourceCommit }, + { name: "registry-prefix", value: runtime.registryPrefix }, + { name: "services", value: runtime.serviceIds.join(",") }, + { name: "base-image", value: runtime.baseImage }, + ], + workspaces: [ + { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, + { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }, + ], + }, + }; +} + +function renderCiBuildBenchmarkResult(result: Record): RenderedCliResult { + const start = record(result.start); + const job = record(result.job); + const plan = record(result.plan); + const benchmark = record(result.benchmark ?? plan); + const next = record(result.next); + const pipelineRun = record(job.pipelineRun); + const taskRows = ciBuildBenchmarkTaskRows(job); + const serviceRows = ciBuildBenchmarkServiceRows(job, benchmark.services); + const failures = ciBuildBenchmarkFailureRows(job, serviceRows); + const profile = renderCell(result.profile ?? benchmark.profile ?? plan.profile, "unknown"); + const node = renderCell(result.node); + const lane = renderCell(result.lane); + const target = `${node}/${lane}`; + const statusText = renderCell(job.state ?? pipelineRun.status ?? start.state, result.ok === false ? "failed" : "ok"); + const pipelineRunName = renderCell(job.pipelineRunName ?? pipelineRun.name ?? start.pipelineRun ?? plan.pipelineRun); + const sourceCommit = renderCell(pipelineRun.sourceCommit ?? start.sourceCommit ?? plan.sourceCommit); + const statusCommand = renderCell(start.statusCommand, `bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node ${node} --lane ${lane} --profile ${profile}`); + const logsCommand = renderCell(start.logsCommand, `bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node ${node} --lane ${lane} --profile ${profile}`); + const logTail = typeof job.logTail === "string" ? job.logTail.trimEnd() : ""; + const lines = [ + "HWLAB K3S CI BUILD BENCHMARK", + "", + ...renderTable(["TARGET", "PROFILE", "MODE", "STATUS", "PIPELINERUN", "SOURCE"], [[target, profile, renderCell(result.mode ?? optionsModeFromCommand(result.command)), statusText, pipelineRunName, shortDisplay(sourceCommit)]]), + "", + "POLICY", + ...renderTable(["FIELD", "VALUE"], [ + ["pipeline", renderCell(benchmark.pipeline)], + ["catalogPath", renderCell(start.catalogPath ?? plan.catalogPath ?? pipelineRun.catalogPath)], + ["services", String((Array.isArray(benchmark.services) ? benchmark.services : []).length)], + ["cachePolicy", JSON.stringify(benchmark.cachePolicy ?? {})], + ["requiredTimings", Array.isArray(benchmark.requiredTimings) ? benchmark.requiredTimings.join(",") : "-"], + ]), + "", + serviceRows.length === 0 ? "SERVICES\n-" : [ + "SERVICES", + ...renderTable(["SERVICE", "TASK", "STATUS", "DURATION", "FAILURE"], serviceRows.map((row) => [ + renderCell(row.service), + renderCell(row.task), + renderCell(row.status), + renderCell(row.duration), + renderCell(row.failure), + ])), + ].join("\n"), + "", + taskRows.length === 0 ? "TIMINGS\n-" : [ + "TIMINGS", + ...renderTable(["TASK", "STATUS", "DURATION", "START", "END"], taskRows.map((row) => [ + renderCell(row.task), + renderCell(row.status), + renderCell(row.duration), + renderCell(row.start), + renderCell(row.end), + ])), + ].join("\n"), + ...(failures.length === 0 ? [] : ["", "FAILURE FAMILIES", ...renderTable(["FAMILY", "COUNT", "SCOPE"], failures.map((row) => [renderCell(row.family), renderCell(row.count), renderCell(row.scope)]))]), + ...(logTail.length === 0 ? [] : ["", "LOG TAIL", logTail]), + "", + "NEXT", + ` ${renderCell(next.confirm, "")}`, + ` ${statusCommand}`, + ` ${logsCommand}`, + "", + "Disclosure: output is bounded; Git/proxy/Secret values are not expanded. A succeeded PipelineRun with missing build- TaskRuns is reported as cache-hit-forbidden.", + ].filter((line) => line !== " "); + return { ok: result.ok !== false, command: renderCell(result.command, "hwlab nodes control-plane infra ci-build-benchmark"), renderedText: lines.join("\n"), contentType: "text/plain" }; +} + function toolsImageCommandStatus(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: ToolsImageOptions): Record { const registry = toolsImageStatus(node, target, options.timeoutSeconds); const jobResult = runTransK3s(node.kubeRoute, remoteJobStatusScript(target, "tools-image", options.tailLines), options.timeoutSeconds); @@ -793,6 +1103,30 @@ function parseEgressBenchmarkOptions(args: string[]): EgressBenchmarkOptions { }; } +function parseCiBuildBenchmarkOptions(args: string[]): CiBuildBenchmarkOptions { + const first = args[0]; + const action: CiBuildBenchmarkAction = first === "status" || first === "logs" + ? first + : "benchmark"; + const effectiveArgs = action === "benchmark" ? args : args.slice(1); + if (first === "--help" || first === "-h" || first === "help") { + throw new Error("infra ci-build-benchmark usage: ci-build-benchmark [status|logs] --node NODE --lane vNN --profile PROFILE [--dry-run|--confirm]"); + } + const confirm = effectiveArgs.includes("--confirm"); + const explicitDryRun = effectiveArgs.includes("--dry-run"); + if (confirm && explicitDryRun) throw new Error("ci-build-benchmark accepts only one of --confirm or --dry-run"); + return { + action, + node: requiredOption(effectiveArgs, "--node"), + lane: requiredOption(effectiveArgs, "--lane"), + profile: requiredOption(effectiveArgs, "--profile"), + confirm, + dryRun: action === "benchmark" ? explicitDryRun || !confirm : false, + timeoutSeconds: positiveIntegerOption(effectiveArgs, "--timeout-seconds", 60, 60), + tailLines: positiveIntegerOption(effectiveArgs, "--tail-lines", 120, 1000), + }; +} + function readControlPlaneConfig(): ControlPlaneConfig { const parsed = asRecord(Bun.YAML.parse(readFileSync(rootPath(HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH), "utf8")) as unknown, HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); const version = numberField(parsed, "version", HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH); @@ -903,6 +1237,54 @@ function argoInstallSpec(raw: Record, path: string): ControlPla }; } +function ciBuildBenchmarkProfileSpecs(raw: unknown, path: string): readonly CiBuildBenchmarkProfileSpec[] { + if (raw === undefined) return []; + if (!Array.isArray(raw)) throw new Error(`${path} must be an array`); + const profiles = raw.map((item, index) => ciBuildBenchmarkProfileSpec(asRecord(item, `${path}[${index}]`), `${path}[${index}]`)); + const names = new Set(); + for (const profile of profiles) { + if (names.has(profile.profile)) throw new Error(`${path} contains duplicate profile ${profile.profile}`); + names.add(profile.profile); + } + return profiles; +} + +function ciBuildBenchmarkProfileSpec(raw: Record, path: string): CiBuildBenchmarkProfileSpec { + const profile = stringField(raw, "profile", path); + validateBenchmarkProfileName(profile, `${path}.profile`); + const pipelineRunPrefix = stringField(raw, "pipelineRunPrefix", path); + validateKubernetesName(pipelineRunPrefix, `${path}.pipelineRunPrefix`); + if (pipelineRunPrefix.length > 40) throw new Error(`${path}.pipelineRunPrefix must leave room for source and nonce suffix`); + const catalogPathTemplate = stringField(raw, "catalogPathTemplate", path); + validateBenchmarkCatalogPathTemplate(catalogPathTemplate, `${path}.catalogPathTemplate`); + const imageTagMode = stringField(raw, "imageTagMode", path); + if (imageTagMode !== "full") throw new Error(`${path}.imageTagMode currently must be full`); + const timings = asRecord(raw.timings, `${path}.timings`); + return { + profile, + runtimeLaneConfigRef: stringField(raw, "runtimeLaneConfigRef", path), + pipelineRunPrefix, + catalogPathTemplate, + imageTagMode, + pipelineTimeoutSeconds: positiveConfigIntegerField(raw, "pipelineTimeoutSeconds", path), + cachePolicy: ciBuildBenchmarkCachePolicy(asRecord(raw.cachePolicy, `${path}.cachePolicy`), `${path}.cachePolicy`), + requiredTimings: stringArrayField(timings, "requiredStages", `${path}.timings`), + failureFamilies: stringArrayField(raw, "failureFamilies", path), + }; +} + +function ciBuildBenchmarkCachePolicy(raw: Record, path: string): CiBuildBenchmarkCachePolicy { + return { + noPipelineRunReuse: booleanField(raw, "noPipelineRunReuse", path), + forceFullBuild: booleanField(raw, "forceFullBuild", path), + forbidGitopsCatalogReuse: booleanField(raw, "forbidGitopsCatalogReuse", path), + forbidDependencyCache: booleanField(raw, "forbidDependencyCache", path), + forbidBuildkitCache: booleanField(raw, "forbidBuildkitCache", path), + forbidRegistryMirror: booleanField(raw, "forbidRegistryMirror", path), + forbidLocalPreheatedImages: booleanField(raw, "forbidLocalPreheatedImages", path), + }; +} + function imageRewriteSpec(raw: Record, path: string): ImageRewriteSpec { const rewrite = { source: stringField(raw, "source", path), @@ -1130,6 +1512,7 @@ function targetSpec(raw: Record, index: number): ControlPlaneTa pipelineRunPrefix: stringField(tekton, "pipelineRunPrefix", `${path}.tekton`), toolsImage: toolsImageSpec(toolsImage, `${path}.tekton.toolsImage`), }, + ciBuildBenchmarks: ciBuildBenchmarkProfileSpecs(raw.ciBuildBenchmarks, `${path}.ciBuildBenchmarks`), argo: { namespace: stringField(argo, "namespace", `${path}.argo`), projectName: stringField(argo, "projectName", `${path}.argo`), @@ -2632,6 +3015,233 @@ printf '{"started":true,"pid":%s,"stateDir":"%s","statusCommand":"bun scripts/cl `; } +function ciBuildBenchmarkStartScript( + target: ControlPlaneTargetSpec, + profile: CiBuildBenchmarkProfileSpec, + manifest: Record, + pipelineName: string, + pipelineRun: string, + sourceCommit: string, + catalogPath: string, +): string { + const stateDir = ciBuildBenchmarkStateDir(target, profile.profile); + const manifestB64 = Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"); + return ` +set -eu +state_dir=${shQuote(stateDir)} +status_file="$state_dir/status.json" +ns=${shQuote(target.ciNamespace)} +profile=${shQuote(profile.profile)} +pipeline=${shQuote(pipelineName)} +pipeline_run=${shQuote(pipelineRun)} +source_commit=${shQuote(sourceCommit)} +catalog_path=${shQuote(catalogPath)} +mkdir -p "$state_dir" +previous_run= +if [ -s "$status_file" ]; then + previous_run=$(python3 - "$status_file" <<'PY' || true +import json, sys +try: + data=json.load(open(sys.argv[1], encoding="utf-8")) + print(data.get("pipelineRun") or "") +except Exception: + print("") +PY +) +fi +if [ -n "$previous_run" ]; then + previous_status=$(kubectl -n "$ns" get pipelinerun "$previous_run" -o 'jsonpath={.status.conditions[?(@.type=="Succeeded")].status}' 2>/dev/null || true) + if [ -n "$previous_status" ] && [ "$previous_status" != "True" ] && [ "$previous_status" != "False" ]; then + python3 - "$state_dir" "$previous_run" "$profile" ${shQuote(target.node)} ${shQuote(target.lane)} <<'PY' +import json, sys +state_dir, previous_run, profile, node, lane = sys.argv[1:6] +print(json.dumps({ + "started": False, + "state": "already-running", + "pipelineRun": previous_run, + "stateDir": state_dir, + "statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node {node} --lane {lane} --profile {profile}", + "logsCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node {node} --lane {lane} --profile {profile}", +}, ensure_ascii=False)) +PY + exit 0 + fi +fi +manifest_path="$state_dir/$pipeline_run.json" +printf '%s' ${shQuote(manifestB64)} | base64 -d >"$manifest_path" +set +e +pipeline_check=$(kubectl -n "$ns" get pipeline "$pipeline" -o name 2>&1) +pipeline_check_rc=$? +create_output= +create_rc=0 +if [ "$pipeline_check_rc" = 0 ]; then + create_output=$(kubectl create -f "$manifest_path" 2>&1) + create_rc=$? +else + create_output="$pipeline_check" + create_rc="$pipeline_check_rc" +fi +printf '%s\\n' "$create_output" >"$state_dir/create.log" +python3 - "$status_file" "$state_dir" "$pipeline_run" "$source_commit" "$profile" "$catalog_path" "$create_rc" "$create_output" ${shQuote(target.node)} ${shQuote(target.lane)} <<'PY' +import datetime, json, sys +status_file, state_dir, pipeline_run, source_commit, profile, catalog_path, rc_raw, output, node, lane = sys.argv[1:11] +rc=int(rc_raw or "0") +payload={ + "started": rc == 0, + "state": "started" if rc == 0 else "failed", + "pipelineRun": pipeline_run, + "sourceCommit": source_commit, + "profile": profile, + "catalogPath": catalog_path, + "stateDir": state_dir, + "createdAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "exitCode": rc, + "createOutputTail": output[-2000:], + "statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark status --node {node} --lane {lane} --profile {profile}", + "logsCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra ci-build-benchmark logs --node {node} --lane {lane} --profile {profile}", +} +open(status_file, "w", encoding="utf-8").write(json.dumps(payload, ensure_ascii=False)) +print(json.dumps(payload, ensure_ascii=False)) +PY +exit "$create_rc" +`; +} + +function ciBuildBenchmarkStatusScript(target: ControlPlaneTargetSpec, profile: CiBuildBenchmarkProfileSpec, tailLines: number, includeLogs: boolean): string { + const stateDir = ciBuildBenchmarkStateDir(target, profile.profile); + return ` +set +e +state_dir=${shQuote(stateDir)} +status_file="$state_dir/status.json" +ns=${shQuote(target.ciNamespace)} +profile=${shQuote(profile.profile)} +tail_lines=${shQuote(String(tailLines))} +include_logs=${includeLogs ? "true" : "false"} +tmp_dir=$(mktemp -d) +pipeline_run= +if [ -s "$status_file" ]; then + pipeline_run=$(python3 - "$status_file" <<'PY' || true +import json, sys +try: + data=json.load(open(sys.argv[1], encoding="utf-8")) + print(data.get("pipelineRun") or "") +except Exception: + print("") +PY +) +fi +if [ -z "$pipeline_run" ]; then + pipeline_run=$(kubectl -n "$ns" get pipelinerun -l "unidesk.ai/benchmark=ci-build,unidesk.ai/benchmark-profile=$profile" -o 'jsonpath={range .items[*]}{.metadata.creationTimestamp}{" "}{.metadata.name}{"\\n"}{end}' 2>/dev/null | sort | tail -n 1 | awk '{print $2}') +fi +if [ -n "$pipeline_run" ]; then + kubectl -n "$ns" get pipelinerun "$pipeline_run" -o json >"$tmp_dir/pipelinerun.json" 2>"$tmp_dir/pipelinerun.err" + kubectl -n "$ns" get taskrun -l "tekton.dev/pipelineRun=$pipeline_run" -o json >"$tmp_dir/taskruns.json" 2>"$tmp_dir/taskruns.err" + kubectl -n "$ns" get pod -l "tekton.dev/pipelineRun=$pipeline_run" -o json >"$tmp_dir/pods.json" 2>"$tmp_dir/pods.err" + if [ "$include_logs" = true ]; then + kubectl -n "$ns" logs -l "tekton.dev/pipelineRun=$pipeline_run" --all-containers --tail="$tail_lines" --prefix=true >"$tmp_dir/logs.txt" 2>"$tmp_dir/logs.err" || true + fi +fi +python3 - "$state_dir" "$status_file" "$tmp_dir" "$pipeline_run" "$include_logs" "$tail_lines" <<'PY' +import json, pathlib, sys +state_dir=pathlib.Path(sys.argv[1]) +status_path=pathlib.Path(sys.argv[2]) +tmp_dir=pathlib.Path(sys.argv[3]) +pipeline_run=sys.argv[4] +include_logs=sys.argv[5] == "true" +tail_lines=int(sys.argv[6]) + +def read_json(path): + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return None + +def read_text(path, limit=4000): + try: + return path.read_text(encoding="utf-8", errors="replace")[-limit:] + except Exception: + return "" + +def succeeded_condition(obj): + for cond in obj.get("status", {}).get("conditions", []) or []: + if cond.get("type") == "Succeeded": + return cond + return {} + +status=None +if status_path.exists(): + status=read_json(status_path) +pr=read_json(tmp_dir / "pipelinerun.json") if pipeline_run else None +trs=read_json(tmp_dir / "taskruns.json") if pipeline_run else None +pods=read_json(tmp_dir / "pods.json") if pipeline_run else None +pr_cond=succeeded_condition(pr or {}) +task_runs=[] +for item in (trs or {}).get("items", []) or []: + cond=succeeded_condition(item) + labels=item.get("metadata", {}).get("labels", {}) or {} + task_runs.append({ + "name": item.get("metadata", {}).get("name"), + "pipelineTask": labels.get("tekton.dev/pipelineTask"), + "status": cond.get("status"), + "reason": cond.get("reason"), + "message": cond.get("message"), + "startTime": item.get("status", {}).get("startTime"), + "completionTime": item.get("status", {}).get("completionTime"), + "podName": item.get("status", {}).get("podName"), + }) +pod_rows=[] +for item in (pods or {}).get("items", []) or []: + phase=item.get("status", {}).get("phase") + pod_rows.append({ + "name": item.get("metadata", {}).get("name"), + "phase": phase, + "startTime": item.get("status", {}).get("startTime"), + }) +state="not-started" +if pr: + status_value=pr_cond.get("status") + if status_value == "True": + state="succeeded" + elif status_value == "False": + state="failed" + elif status_value: + state="running" + else: + state="pending" +elif pipeline_run: + state="missing" +payload={ + "stateDir": str(state_dir), + "status": status, + "pipelineRunName": pipeline_run or None, + "state": state, + "pipelineRun": None if not pr else { + "name": pr.get("metadata", {}).get("name"), + "status": pr_cond.get("status"), + "reason": pr_cond.get("reason"), + "message": pr_cond.get("message"), + "createdAt": pr.get("metadata", {}).get("creationTimestamp"), + "startTime": pr.get("status", {}).get("startTime"), + "completionTime": pr.get("status", {}).get("completionTime"), + "sourceCommit": (pr.get("metadata", {}).get("labels", {}) or {}).get("hwlab.pikastech.local/source-commit"), + "catalogPath": (pr.get("metadata", {}).get("annotations", {}) or {}).get("unidesk.ai/catalog-path"), + }, + "taskRuns": task_runs, + "pods": pod_rows, + "errors": { + "pipelinerun": read_text(tmp_dir / "pipelinerun.err"), + "taskruns": read_text(tmp_dir / "taskruns.err"), + "pods": read_text(tmp_dir / "pods.err"), + "logs": read_text(tmp_dir / "logs.err") if include_logs else "", + }, + "logTail": read_text(tmp_dir / "logs.txt", max(4000, tail_lines * 300)) if include_logs else "", +} +print(json.dumps(payload, ensure_ascii=False)) +PY +rm -rf "$tmp_dir" +`; +} + function remoteJobStatusScript(target: ControlPlaneTargetSpec, name: "tools-image" | "argo", tailLines: number): string { const stateDir = remoteJobStateDir(target, name); return ` @@ -2734,6 +3344,159 @@ function parseRemoteJson(text: string): unknown { return null; } +function ciBuildBenchmarkStateDir(target: ControlPlaneTargetSpec, profile: string): string { + return `/tmp/unidesk-hwlab-node-control-plane/${target.id}/ci-build-benchmark-${profile}`; +} + +function ciBuildBenchmarkLiveOk(job: Record, expectedServices: readonly string[]): boolean { + const pipelineRun = record(job.pipelineRun); + const pipelineStatus = renderCell(pipelineRun.status, ""); + if (pipelineStatus === "False") return false; + if (pipelineStatus !== "True") return true; + const taskRuns = ciBuildBenchmarkTaskRunRecords(job); + for (const service of expectedServices) { + if (!taskRuns.some((task) => task.pipelineTask === `build-${service}`)) return false; + } + return true; +} + +function ciBuildBenchmarkTaskRows(job: Record): Record[] { + const pipelineRun = record(job.pipelineRun); + const rows: Record[] = []; + if (Object.keys(pipelineRun).length > 0) { + rows.push({ + task: "pipeline-total", + status: ciBuildBenchmarkStatusText(pipelineRun.status), + duration: durationBetweenIso(pipelineRun.startTime, pipelineRun.completionTime), + start: shortIsoTime(pipelineRun.startTime), + end: shortIsoTime(pipelineRun.completionTime), + }); + } + const taskRuns = ciBuildBenchmarkTaskRunRecords(job).sort((left, right) => renderCell(left.startTime, "").localeCompare(renderCell(right.startTime, ""))); + for (const task of taskRuns) { + rows.push({ + task: renderCell(task.pipelineTask ?? task.name), + status: ciBuildBenchmarkStatusText(task.status), + duration: durationBetweenIso(task.startTime, task.completionTime), + start: shortIsoTime(task.startTime), + end: shortIsoTime(task.completionTime), + }); + } + return rows; +} + +function ciBuildBenchmarkServiceRows(job: Record, servicesValue: unknown): Record[] { + const services = ciBuildBenchmarkExpectedServices(servicesValue); + if (services.length === 0) return []; + const pipelineRun = record(job.pipelineRun); + const pipelineTerminal = pipelineRun.status === "True" || pipelineRun.status === "False"; + const taskRuns = ciBuildBenchmarkTaskRunRecords(job); + return services.map((service) => { + const task = taskRuns.find((item) => item.pipelineTask === `build-${service}`); + if (task === undefined) { + const status = pipelineTerminal ? "missing" : "pending"; + return { + service, + task: `build-${service}`, + status, + duration: "-", + failure: pipelineRun.status === "True" ? "cache-hit-forbidden" : "-", + }; + } + const taskStatus = ciBuildBenchmarkStatusText(task.status); + const failure = task.status === "False" ? classifyCiBuildBenchmarkFailure(`${renderCell(task.reason, "")}\n${renderCell(task.message, "")}`) : "-"; + return { + service, + task: renderCell(task.pipelineTask ?? task.name), + status: taskStatus, + duration: durationBetweenIso(task.startTime, task.completionTime), + failure, + }; + }); +} + +function ciBuildBenchmarkFailureRows(job: Record, serviceRows: readonly Record[]): Record[] { + const counts = new Map(); + const add = (family: string, scope: string): void => { + if (family === "-" || family.length === 0) return; + const existing = counts.get(family) ?? { count: 0, scopes: [] }; + existing.count += 1; + if (!existing.scopes.includes(scope)) existing.scopes.push(scope); + counts.set(family, existing); + }; + for (const row of serviceRows) add(row.failure, row.service); + const pipelineRun = record(job.pipelineRun); + if (pipelineRun.status === "False") { + add(classifyCiBuildBenchmarkFailure(`${renderCell(pipelineRun.reason, "")}\n${renderCell(pipelineRun.message, "")}`), "pipeline"); + } + return [...counts.entries()].map(([family, value]) => ({ family, count: String(value.count), scope: value.scopes.join(",") })); +} + +function ciBuildBenchmarkTaskRunRecords(job: Record): Record[] { + return Array.isArray(job.taskRuns) ? job.taskRuns.map(record) : []; +} + +function ciBuildBenchmarkExpectedServices(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.length > 0) : []; +} + +function ciBuildBenchmarkStatusText(value: unknown): string { + if (value === "True") return "succeeded"; + if (value === "False") return "failed"; + if (value === "Unknown") return "running"; + return renderCell(value, "pending"); +} + +function classifyCiBuildBenchmarkFailure(text: string): string { + const value = text.toLowerCase(); + if (/cache-hit-forbidden|reused-from|reuse/i.test(text)) return "cache-hit-forbidden"; + if (/no such host|could not resolve|enotfound|dns/i.test(text)) return "dns"; + if (/429|rate limit|too many requests|toomanyrequests/i.test(text)) return "rate-limit"; + if (/tls|certificate|x509|timeout|timed out|i\/o timeout/i.test(text)) return "tls-timeout"; + if (/proxy|connect|connection reset|connection refused|econn/i.test(text)) return "proxy-connect"; + if (/unauthorized|authentication required|permission denied|forbidden|denied/i.test(text)) return "auth"; + if (/imagepullbackoff|errimagepull|imagepolicy|pull access denied/i.test(text)) return "image-policy"; + if (/push|registry|blob upload|manifest invalid|manifest unknown/i.test(text)) return "registry-push"; + return value.trim().length === 0 ? "unknown" : "build-script"; +} + +function durationBetweenIso(startValue: unknown, endValue: unknown): string { + if (typeof startValue !== "string" || startValue.length === 0) return "-"; + const start = Date.parse(startValue); + if (!Number.isFinite(start)) return "-"; + const end = typeof endValue === "string" && endValue.length > 0 ? Date.parse(endValue) : Date.now(); + if (!Number.isFinite(end) || end < start) return "-"; + return formatDurationMs(end - start); +} + +function formatDurationMs(ms: number): string { + const seconds = Math.round(ms / 1000); + const minutes = Math.floor(seconds / 60); + const rest = seconds % 60; + return minutes > 0 ? `${minutes}m${String(rest).padStart(2, "0")}s` : `${seconds}s`; +} + +function shortIsoTime(value: unknown): string { + if (typeof value !== "string" || value.length === 0) return "-"; + return value.replace(/^\d{4}-\d{2}-\d{2}T/u, "").replace(/Z$/u, "Z"); +} + +function shortDisplay(value: string): string { + return /^[0-9a-f]{40}$/iu.test(value) ? value.slice(0, 12).toLowerCase() : value; +} + +function validateBenchmarkProfileName(value: string, path: string): void { + if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 63) throw new Error(`${path} must be a DNS-label style benchmark profile`); +} + +function validateBenchmarkCatalogPathTemplate(value: string, path: string): void { + if (!value.includes("{profile}") || !value.includes("{pipelineRun}")) throw new Error(`${path} must include {profile} and {pipelineRun}`); + if (value.startsWith("/") || value.includes("\n") || value.includes("\r")) throw new Error(`${path} must be a relative repo path template`); + const rendered = value.replace(/\{profile\}/gu, "profile").replace(/\{pipelineRun\}/gu, "pipeline-run"); + if (rendered.split("/").some((segment) => segment === ".." || segment.length === 0)) throw new Error(`${path} must not contain empty or parent path segments`); + if (!rendered.endsWith(".json")) throw new Error(`${path} must render to a JSON catalog path`); +} + function asRecord(value: unknown, path: string): Record { if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`); return value as Record;