diff --git a/scripts/hwlab-g14-contract-test.ts b/scripts/hwlab-g14-contract-test.ts index 65a30742..b3956b93 100644 --- a/scripts/hwlab-g14-contract-test.ts +++ b/scripts/hwlab-g14-contract-test.ts @@ -1,4 +1,5 @@ import { activeV02PipelineRuns, g14ObservabilityQueryAssertion, gitMirrorFlushJobManifest, gitMirrorStatusSummary, gitMirrorSyncJobManifest, gitMirrorV02SyncRequirement, hwlabG14Help, hwlabG14MonitorStateFileName, parseGitMirrorStatusRefs, parseK8sCpuMillicores, parseK8sMemoryMiB, parsePipelineTaskRunMetrics, parseV02TriggerSnapshot, rolloutRecordBody, semanticChangelogBullets, summarizeV02CdStatus, v02CloseoutVerdict, v02CommitAlignment, v02ControlPlaneRefreshScriptHash, v02ControlPlaneRenderScript, v02ExistingPipelineRunReuseDecision, v02FalseGreenGuard, v02GitMirrorPreSyncWaitMs, v02LatestOnlyTargetValidation, v02PipelineServiceIds, v02PrAutomationCommentBody, v02ReusableGitMirrorPreSyncMarker, v02ReusableRefreshMarker, v02StatusHistoryPolicy, v02TaskRunPerformanceSummary } from "./src/hwlab-g14"; +import { hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec } from "./src/hwlab-g14-lanes"; import { runCommand } from "./src/command"; function assertCondition(condition: unknown, message: string, detail: unknown = {}): void { @@ -44,6 +45,29 @@ assertCondition( "v0.2 control-plane help must expose targeted PipelineRun/source-commit status, closeout inspection, and cleanup", hwlabHelpUsage, ); +assertCondition( + hwlabHelpUsage.some((line) => line.includes("control-plane status --lane v03")) + && hwlabHelpUsage.some((line) => line.includes("control-plane apply --lane v03 --dry-run")) + && hwlabHelpUsage.some((line) => line.includes("control-plane trigger-current --lane v03 --dry-run")) + && JSON.stringify(hwlabG14Help()).includes("runtime lane v02/v03"), + "v0.3 control-plane help must expose the runtime lane bootstrap status/apply/trigger entrypoints", + hwlabHelpUsage, +); +const v03LaneSpec = hwlabRuntimeLaneSpec("v03"); +assertCondition( + JSON.stringify(hwlabRuntimeLaneIds()) === JSON.stringify(["v02", "v03"]) + && v03LaneSpec.sourceBranch === "v0.3" + && v03LaneSpec.gitopsBranch === "v0.3-gitops" + && v03LaneSpec.workspace === "/root/hwlab-v03" + && v03LaneSpec.cicdRepo === "/root/hwlab-v03-cicd.git" + && v03LaneSpec.runtimeNamespace === "hwlab-v03" + && v03LaneSpec.runtimeRenderDir === "runtime-v03" + && v03LaneSpec.pipeline === "hwlab-v03-ci-image-publish" + && v03LaneSpec.publicWebUrl.endsWith(":19766") + && v03LaneSpec.publicApiUrl.endsWith(":19767"), + "runtime lane spec must make v0.3 expansion a config-driven lane instead of scattered literals", + v03LaneSpec, +); assertCondition( hwlabHelpUsage.some((line) => line.includes("monitor-prs --lane v02")) && hwlabHelpUsage.some((line) => line.includes("monitor-prs --lane v02 --status")) @@ -52,6 +76,12 @@ assertCondition( "v0.2 PR monitor help must expose the auto CI/CD lane, status query, latest-only CD, and dedupe comment state", hwlabG14Help(), ); +assertCondition( + hwlabHelpUsage.some((line) => line.includes("git-mirror apply --lane v02 --confirm")) + && hwlabHelpUsage.some((line) => line.includes("git-mirror apply --lane v03 --confirm")), + "git mirror help must expose lane-selected apply so v0.3 mirror config is not rendered through v0.2", + hwlabHelpUsage, +); assertCondition( hwlabHelpUsage.some((line) => line.includes("secret status --lane v02 --name hwlab-v02-openfga")) && hwlabHelpUsage.some((line) => line.includes("secret ensure --lane v02 --name hwlab-v02-openfga --confirm")) @@ -268,8 +298,10 @@ assertCondition( && renderScript.includes("git -C \"$worktree_dir\" checkout --detach \"$source_commit\"") && renderScript.includes("npm ci --ignore-scripts --no-audit --prefer-offline") && renderScript.includes("bun install --frozen-lockfile --ignore-scripts") + && renderScript.includes("scripts/run-bun.mjs") + && renderScript.includes("--lane \"$render_lane\"") && renderScript.includes("/tmp/hwlab-v02-control-plane-source-aaaaaaaaaaaa-"), - "v0.2 control-plane render must use an isolated temp clone with lockfile-based dependencies so same-commit concurrent triggers do not share worktree metadata", + "v0.2 control-plane render must use an isolated temp clone with lockfile-based dependencies and the modular Bun renderer fallback", renderScript, ); assertCondition( @@ -308,10 +340,11 @@ assertCondition( ); assertCondition( sourceText.includes("function runG14K3sRemoteAsync") - && sourceText.includes("label: \"v02-control-plane-apply\"") + && sourceText.includes("function applyRuntimeLaneControlPlaneFiles") + && sourceText.includes("label: `${spec.lane}-control-plane-apply`") && sourceText.includes("remote async command timed out after") && sourceText.includes("return runG14K3sRemoteAsync({"), - "v0.2 control-plane apply must use short start/poll remote-async semantics instead of one long G14:k3s apply call", + "runtime lane control-plane apply must use short start/poll remote-async semantics instead of one long G14:k3s apply call", ); const existingPipelineRunReuse = v02ExistingPipelineRunReuseDecision({ sourceCommit: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", diff --git a/scripts/src/hwlab-g14-lanes.ts b/scripts/src/hwlab-g14-lanes.ts new file mode 100644 index 00000000..bf245b0e --- /dev/null +++ b/scripts/src/hwlab-g14-lanes.ts @@ -0,0 +1,104 @@ +export type HwlabRuntimeLane = "v02" | "v03"; + +export interface HwlabRuntimeLaneConfig { + readonly lane: HwlabRuntimeLane; + readonly minor: number; +} + +export interface HwlabRuntimeLaneSpec { + readonly lane: HwlabRuntimeLane; + readonly minor: number; + readonly version: string; + readonly sourceBranch: string; + readonly workspace: string; + readonly cicdRepo: string; + readonly cicdRepoLock: string; + readonly app: string; + readonly pipeline: string; + readonly pipelineRunPrefix: string; + readonly serviceAccountName: string; + readonly controlPlaneFieldManager: string; + readonly gitUrl: string; + readonly gitReadUrl: string; + readonly gitWriteUrl: string; + readonly gitopsBranch: string; + readonly catalogPath: string; + readonly runtimePath: string; + readonly runtimeNamespace: string; + readonly runtimeRenderDir: string; + readonly tektonDir: string; + readonly argoApplicationFile: string; + readonly registryPrefix: string; + readonly baseImage: string; + readonly serviceIds: readonly string[]; + readonly publicWebUrl: string; + readonly publicApiUrl: string; +} + +export const HWLAB_RUNTIME_LANE_CONFIGS = [ + { lane: "v02", minor: 2 }, + { lane: "v03", minor: 3 }, +] as const satisfies readonly HwlabRuntimeLaneConfig[]; + +const HWLAB_GIT_URL = "git@github.com:pikasTech/HWLAB.git"; +const HWLAB_GIT_READ_URL = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; +const HWLAB_GIT_WRITE_URL = "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; +const HWLAB_REGISTRY_PREFIX = "127.0.0.1:5000/hwlab"; +const HWLAB_BASE_IMAGE = "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim"; +const HWLAB_SERVICE_IDS = [ + "hwlab-cloud-api", + "hwlab-cloud-web", + "hwlab-gateway", + "hwlab-edge-proxy", + "hwlab-agent-skills", +] as const; + +function buildRuntimeLaneSpec(config: HwlabRuntimeLaneConfig): HwlabRuntimeLaneSpec { + const version = `v0.${config.minor}`; + const lane = config.lane; + return { + lane, + minor: config.minor, + version, + sourceBranch: version, + workspace: `/root/hwlab-${lane}`, + cicdRepo: `/root/hwlab-${lane}-cicd.git`, + cicdRepoLock: `/tmp/hwlab-${lane}-cicd-repo.lock`, + app: `hwlab-g14-${lane}`, + pipeline: `hwlab-${lane}-ci-image-publish`, + pipelineRunPrefix: `hwlab-${lane}-ci-poll`, + serviceAccountName: `hwlab-${lane}-tekton-runner`, + controlPlaneFieldManager: `unidesk-hwlab-${lane}-control-plane`, + gitUrl: HWLAB_GIT_URL, + gitReadUrl: HWLAB_GIT_READ_URL, + gitWriteUrl: HWLAB_GIT_WRITE_URL, + gitopsBranch: `${version}-gitops`, + catalogPath: `deploy/artifact-catalog.${lane}.json`, + runtimePath: `deploy/gitops/g14/runtime-${lane}`, + runtimeNamespace: `hwlab-${lane}`, + runtimeRenderDir: `runtime-${lane}`, + tektonDir: `tekton-${lane}`, + argoApplicationFile: `application-${lane}.yaml`, + registryPrefix: HWLAB_REGISTRY_PREFIX, + baseImage: HWLAB_BASE_IMAGE, + serviceIds: HWLAB_SERVICE_IDS, + publicWebUrl: `http://74.48.78.17:${19466 + config.minor * 100}`, + publicApiUrl: `http://74.48.78.17:${19467 + config.minor * 100}`, + }; +} + +const RUNTIME_LANE_SPECS = Object.fromEntries( + HWLAB_RUNTIME_LANE_CONFIGS.map((config) => [config.lane, buildRuntimeLaneSpec(config)]), +) as Record; + +export function isHwlabRuntimeLane(value: string): value is HwlabRuntimeLane { + return Object.prototype.hasOwnProperty.call(RUNTIME_LANE_SPECS, value); +} + +export function hwlabRuntimeLaneSpec(lane: HwlabRuntimeLane): HwlabRuntimeLaneSpec { + return RUNTIME_LANE_SPECS[lane]; +} + +export function hwlabRuntimeLaneIds(): HwlabRuntimeLane[] { + return HWLAB_RUNTIME_LANE_CONFIGS.map((config) => config.lane); +} diff --git a/scripts/src/hwlab-g14.ts b/scripts/src/hwlab-g14.ts index 2ecfe53b..f97eab00 100644 --- a/scripts/src/hwlab-g14.ts +++ b/scripts/src/hwlab-g14.ts @@ -4,32 +4,34 @@ import { createHash, randomBytes } from "node:crypto"; import { repoRoot, rootPath, type Config } from "./config"; import { runCommand } from "./command"; import { readJob, startJob } from "./jobs"; +import { hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-g14-lanes"; const HWLAB_REPO = "pikasTech/HWLAB"; const G14_SOURCE_BRANCH = "G14"; const G14_PROVIDER = "G14"; const G14_WORKSPACE = "/root/hwlab"; -const V02_SOURCE_BRANCH = "v0.2"; -const V02_WORKSPACE = "/root/hwlab-v02"; -const V02_CICD_REPO = "/root/hwlab-v02-cicd.git"; +const V02_LANE_SPEC = hwlabRuntimeLaneSpec("v02"); +const V02_SOURCE_BRANCH = V02_LANE_SPEC.sourceBranch; +const V02_WORKSPACE = V02_LANE_SPEC.workspace; +const V02_CICD_REPO = V02_LANE_SPEC.cicdRepo; const DEV_NAMESPACE = "hwlab-dev"; const CI_NAMESPACE = "hwlab-ci"; const ARGO_NAMESPACE = "argocd"; const DEV_APP = "hwlab-g14-dev"; -const V02_APP = "hwlab-g14-v02"; -const V02_PIPELINE = "hwlab-v02-ci-image-publish"; +const V02_APP = V02_LANE_SPEC.app; +const V02_PIPELINE = V02_LANE_SPEC.pipeline; const V02_POLLER = "hwlab-v02-branch-poller"; const V02_RECONCILER = "hwlab-v02-control-plane-reconciler"; -const V02_PIPELINERUN_PREFIX = "hwlab-v02-ci-poll"; -const V02_CONTROL_PLANE_FIELD_MANAGER = "unidesk-hwlab-v02-control-plane"; +const V02_PIPELINERUN_PREFIX = V02_LANE_SPEC.pipelineRunPrefix; +const V02_CONTROL_PLANE_FIELD_MANAGER = V02_LANE_SPEC.controlPlaneFieldManager; const V02_SECRET_FIELD_MANAGER = "unidesk-hwlab-v02-secret"; -const V02_GIT_URL = "git@github.com:pikasTech/HWLAB.git"; -const V02_GIT_READ_URL = "http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; -const V02_GIT_WRITE_URL = "http://git-mirror-write.devops-infra.svc.cluster.local/pikasTech/HWLAB.git"; -const V02_GITOPS_BRANCH = "v0.2-gitops"; -const V02_CATALOG_PATH = "deploy/artifact-catalog.v02.json"; -const V02_RUNTIME_PATH = "deploy/gitops/g14/runtime-v02"; -const V02_RUNTIME_NAMESPACE = "hwlab-v02"; +const V02_GIT_URL = V02_LANE_SPEC.gitUrl; +const V02_GIT_READ_URL = V02_LANE_SPEC.gitReadUrl; +const V02_GIT_WRITE_URL = V02_LANE_SPEC.gitWriteUrl; +const V02_GITOPS_BRANCH = V02_LANE_SPEC.gitopsBranch; +const V02_CATALOG_PATH = V02_LANE_SPEC.catalogPath; +const V02_RUNTIME_PATH = V02_LANE_SPEC.runtimePath; +const V02_RUNTIME_NAMESPACE = V02_LANE_SPEC.runtimeNamespace; const V02_OPENFGA_SECRET = "hwlab-v02-openfga"; const V02_OPENFGA_AUTHN_SECRET_KEY = "authn-preshared-key"; const V02_OPENFGA_DATASTORE_URI_SECRET_KEY = "datastore-uri"; @@ -39,8 +41,8 @@ const V02_OPENFGA_DB_USER = "hwlab_openfga"; const V02_MASTER_ADMIN_API_KEY_SECRET = "hwlab-v02-master-server-admin-api-key"; const V02_MASTER_ADMIN_API_KEY_SECRET_KEY = "api-key"; const V02_MASTER_ADMIN_API_KEY_LOCAL_ENV = "/root/.config/hwlab-v02/master-server-admin-api-key.env"; -const V02_REGISTRY_PREFIX = "127.0.0.1:5000/hwlab"; -const V02_BASE_IMAGE = "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim"; +const V02_REGISTRY_PREFIX = V02_LANE_SPEC.registryPrefix; +const V02_BASE_IMAGE = V02_LANE_SPEC.baseImage; const GIT_MIRROR_NAMESPACE = "devops-infra"; const GIT_MIRROR_MANIFEST_FIELD_MANAGER = "unidesk-hwlab-git-mirror"; const GIT_MIRROR_SYNC_JOB_PREFIX = "git-mirror-hwlab-sync-manual"; @@ -53,15 +55,9 @@ const G14_PROMETHEUS_NAME = "g14-shared"; const G14_PROMETHEUS_SERVICE = "prometheus-g14-shared"; const G14_PROMETHEUS_SERVICE_ACCOUNT = "g14-observability-prometheus"; const G14_PROMETHEUS_OPERATOR_RELEASE_ASSET = `https://github.com/prometheus-operator/prometheus-operator/releases/download/${G14_PROMETHEUS_OPERATOR_VERSION}/bundle.yaml`; -const V02_SERVICE_IDS = [ - "hwlab-cloud-api", - "hwlab-cloud-web", - "hwlab-gateway", - "hwlab-edge-proxy", - "hwlab-agent-skills", -]; -const V02_CLOUD_WEB_URL = "http://74.48.78.17:19666"; -const V02_CLOUD_API_URL = "http://74.48.78.17:19667"; +const V02_SERVICE_IDS = [...V02_LANE_SPEC.serviceIds]; +const V02_CLOUD_WEB_URL = V02_LANE_SPEC.publicWebUrl; +const V02_CLOUD_API_URL = V02_LANE_SPEC.publicApiUrl; const V02_OBSERVABILITY_SERVICE_IDS = [ "hwlab-agent-skills", "hwlab-cloud-api", @@ -128,7 +124,7 @@ interface G14RecordRolloutOptions { interface G14ControlPlaneOptions { action: "status" | "closeout" | "apply" | "trigger-current" | "cleanup-runs" | "cleanup-released-pvs" | "runtime-migration"; - lane: "v02" | "g14" | "all"; + lane: HwlabRuntimeLane | "g14" | "all"; dryRun: boolean; confirm: boolean; wait: boolean; @@ -171,6 +167,7 @@ interface G14UpstreamImageOptions { interface G14GitMirrorOptions { action: "status" | "apply" | "sync" | "flush"; + lane: HwlabRuntimeLane; dryRun: boolean; confirm: boolean; wait: boolean; @@ -327,6 +324,15 @@ function validateV02PipelineRunOption(value: string): string { return value.toLowerCase(); } +function validateRuntimeLanePipelineRunOption(lane: HwlabRuntimeLane, value: string): string { + const spec = hwlabRuntimeLaneSpec(lane); + const escapedPrefix = spec.pipelineRunPrefix.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + if (!new RegExp(`^${escapedPrefix}-[0-9a-f]{7,40}$`, "iu").test(value)) { + throw new Error(`--pipeline-run must be a ${spec.pipelineRunPrefix}- PipelineRun name for --lane ${lane}`); + } + return value.toLowerCase(); +} + function parseRecordRolloutOptions(args: string[]): G14RecordRolloutOptions { const prRaw = optionValue(args, "--pr") ?? optionValue(args, "--number"); const prNumber = Number(prRaw); @@ -355,15 +361,23 @@ function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions { actionRaw !== "cleanup-released-pvs" && actionRaw !== "runtime-migration" ) { - throw new Error("control-plane usage: status|closeout|apply|trigger-current|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]"); + throw new Error("control-plane usage: status|apply|trigger-current --lane v02|v03 | closeout|runtime-migration --lane v02 | cleanup-runs --lane v02|g14|all | cleanup-released-pvs --lane all [--dry-run|--confirm]"); } - const lane = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02"); + const laneRaw = optionValue(args, "--lane") ?? (actionRaw === "cleanup-released-pvs" ? "all" : "v02"); + let lane: G14ControlPlaneOptions["lane"]; if (actionRaw === "cleanup-runs") { - if (lane !== "v02" && lane !== "g14" && lane !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all"); + if (laneRaw !== "v02" && laneRaw !== "g14" && laneRaw !== "all") throw new Error("control-plane cleanup-runs requires --lane v02|g14|all"); + lane = laneRaw; } else if (actionRaw === "cleanup-released-pvs") { - if (lane !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane"); - } else if (lane !== "v02") { - throw new Error("control-plane status/closeout/apply/trigger-current/runtime-migration currently requires --lane v02"); + if (laneRaw !== "all") throw new Error("control-plane cleanup-released-pvs requires --lane all because released PVs no longer preserve the v02/g14 PipelineRun lane"); + lane = laneRaw; + } else if (actionRaw === "closeout" || actionRaw === "runtime-migration") { + if (laneRaw !== "v02") throw new Error("control-plane closeout/runtime-migration currently requires --lane v02"); + lane = laneRaw; + } else if (!isHwlabRuntimeLane(laneRaw)) { + throw new Error(`control-plane ${actionRaw} requires --lane ${hwlabRuntimeLaneIds().join("|")}`); + } else { + lane = laneRaw; } const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); @@ -391,7 +405,11 @@ function parseControlPlaneOptions(args: string[]): G14ControlPlaneOptions { minAgeMinutes: positiveIntegerOption(args, "--min-age-minutes", 60, 10080), limit: positiveIntegerOption(args, "--limit", 20, 200), sourceCommit: sourceCommitRaw === undefined ? undefined : validateFullShaOption(sourceCommitRaw, "--source-commit"), - pipelineRun: pipelineRunRaw === undefined ? undefined : validateV02PipelineRunOption(pipelineRunRaw), + pipelineRun: pipelineRunRaw === undefined + ? undefined + : isHwlabRuntimeLane(lane) + ? validateRuntimeLanePipelineRunOption(lane, pipelineRunRaw) + : validateV02PipelineRunOption(pipelineRunRaw), history: args.includes("--history"), }; } @@ -446,13 +464,16 @@ function parseUpstreamImageOptions(args: string[]): G14UpstreamImageOptions { function parseGitMirrorOptions(args: string[]): G14GitMirrorOptions { const [actionRaw] = args; if (actionRaw !== "status" && actionRaw !== "apply" && actionRaw !== "sync" && actionRaw !== "flush") { - throw new Error("git-mirror usage: status|apply|sync|flush [--dry-run|--confirm]"); + throw new Error("git-mirror usage: status|apply|sync|flush [--lane v02|v03] [--dry-run|--confirm]"); } + const laneRaw = optionValue(args, "--lane") ?? "v02"; + if (!isHwlabRuntimeLane(laneRaw)) throw new Error(`git-mirror --lane must be ${hwlabRuntimeLaneIds().join("|")}`); const confirm = args.includes("--confirm"); const explicitDryRun = args.includes("--dry-run"); if (confirm && explicitDryRun) throw new Error("git-mirror accepts only one of --confirm or --dry-run"); return { action: actionRaw, + lane: laneRaw, confirm, wait: args.includes("--wait"), dryRun: actionRaw === "status" ? true : explicitDryRun || !confirm, @@ -1082,36 +1103,37 @@ function runG14K3sRemoteAsync(spec: RemoteAsyncCommandSpec): CommandJsonResult { }; } -function v02CicdRepoEnsureScript(): string { +function runtimeLaneCicdRepoEnsureScript(spec: HwlabRuntimeLaneSpec): string { return [ - `cicd_repo=${shellQuote(V02_CICD_REPO)}`, - `cicd_url=${shellQuote(V02_GIT_URL)}`, - "cicd_repo_lock=/tmp/hwlab-v02-cicd-repo.lock", + `cicd_repo=${shellQuote(spec.cicdRepo)}`, + `cicd_url=${shellQuote(spec.gitUrl)}`, + `cicd_branch=${shellQuote(spec.sourceBranch)}`, + `cicd_repo_lock=${shellQuote(spec.cicdRepoLock)}`, "cicd_repo_ensure_fetch() {", " mkdir -p \"$(dirname \"$cicd_repo\")\" || return $?", " if [ -d \"$cicd_repo/objects\" ] && [ -f \"$cicd_repo/HEAD\" ]; then", " :", " elif [ -e \"$cicd_repo\" ]; then", - " echo \"v0.2 CI/CD repo path exists but is not a bare git repo: $cicd_repo\" >&2", + ` echo "${spec.version} CI/CD repo path exists but is not a bare git repo: $cicd_repo" >&2`, " return 41", " else", " git clone --bare \"$cicd_url\" \"$cicd_repo\" || return $?", " fi", " git --git-dir=\"$cicd_repo\" remote set-url origin \"$cicd_url\" 2>/dev/null || git --git-dir=\"$cicd_repo\" remote add origin \"$cicd_url\" || return $?", " git --git-dir=\"$cicd_repo\" config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' || return $?", - " git --git-dir=\"$cicd_repo\" fetch origin '+refs/heads/v0.2:refs/remotes/origin/v0.2' --prune || return $?", + " git --git-dir=\"$cicd_repo\" fetch origin \"+refs/heads/$cicd_branch:refs/remotes/origin/$cicd_branch\" --prune || return $?", "}", "cicd_repo_lock_mode=none", "if command -v flock >/dev/null 2>&1; then", " exec 9>\"$cicd_repo_lock\"", - " flock -w 120 9 || { echo \"timed out waiting for v0.2 CI/CD repo lock: $cicd_repo_lock\" >&2; exit 42; }", + ` flock -w 120 9 || { echo "timed out waiting for ${spec.version} CI/CD repo lock: $cicd_repo_lock" >&2; exit 42; }`, " cicd_repo_lock_mode=flock", "else", " cicd_repo_lock_dir=\"$cicd_repo_lock.d\"", " cicd_repo_lock_attempt=0", " while ! mkdir \"$cicd_repo_lock_dir\" 2>/dev/null; do", " cicd_repo_lock_attempt=$((cicd_repo_lock_attempt + 1))", - " if [ \"$cicd_repo_lock_attempt\" -ge 120 ]; then echo \"timed out waiting for v0.2 CI/CD repo lock: $cicd_repo_lock_dir\" >&2; exit 42; fi", + ` if [ "$cicd_repo_lock_attempt" -ge 120 ]; then echo "timed out waiting for ${spec.version} CI/CD repo lock: $cicd_repo_lock_dir" >&2; exit 42; fi`, " sleep 1", " done", " cicd_repo_lock_mode=dir", @@ -1120,10 +1142,14 @@ function v02CicdRepoEnsureScript(): string { "cicd_repo_ensure_fetch || cicd_repo_status=$?", "if [ \"$cicd_repo_lock_mode\" = flock ]; then flock -u 9 >/dev/null 2>&1 || true; fi", "if [ \"$cicd_repo_lock_mode\" = dir ]; then rmdir \"$cicd_repo_lock_dir\" >/dev/null 2>&1 || true; fi", - "if [ \"$cicd_repo_status\" -ne 0 ]; then echo \"v0.2 CI/CD repo refresh failed status=$cicd_repo_status repo=$cicd_repo\" >&2; exit \"$cicd_repo_status\"; fi", + `if [ "$cicd_repo_status" -ne 0 ]; then echo "${spec.version} CI/CD repo refresh failed status=$cicd_repo_status repo=$cicd_repo" >&2; exit "$cicd_repo_status"; fi`, ].join("\n"); } +function v02CicdRepoEnsureScript(): string { + return runtimeLaneCicdRepoEnsureScript(V02_LANE_SPEC); +} + function resolveV02Head(): { sourceCommit: string | null; result: CommandJsonResult } { const result = g14HostScript([ "set -eu", @@ -1148,8 +1174,32 @@ function resolveV02LatestRemoteHead(): { sourceCommit: string | null; result: Co return { sourceCommit: match?.[0] ?? null, result }; } +function resolveRuntimeLaneHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandJsonResult } { + const result = g14HostScript([ + "set -eu", + runtimeLaneCicdRepoEnsureScript(spec), + `git --git-dir="$cicd_repo" rev-parse refs/remotes/origin/${spec.sourceBranch}`, + ].join("\n"), 180_000); + if (!isCommandSuccess(result)) return { sourceCommit: null, result }; + const output = String(nested(result.parsed, ["data", "stdout"]) ?? result.stdout).trim(); + const match = /[0-9a-f]{40}/iu.exec(output); + return { sourceCommit: match?.[0] ?? null, result }; +} + +function resolveRuntimeLaneLatestRemoteHead(spec: HwlabRuntimeLaneSpec): { sourceCommit: string | null; result: CommandJsonResult } { + const result = commandJson(["git", "ls-remote", spec.gitUrl, `refs/heads/${spec.sourceBranch}`], 30_000); + if (!isCommandSuccess(result)) return { sourceCommit: null, result }; + const output = statusText(result); + const match = /^[0-9a-f]{40}/imu.exec(output); + return { sourceCommit: match?.[0] ?? null, result }; +} + +function runtimeLanePipelineRunName(spec: HwlabRuntimeLaneSpec, sourceCommit: string): string { + return `${spec.pipelineRunPrefix}-${shortSha(sourceCommit)}`; +} + function v02PipelineRunName(sourceCommit: string): string { - return `${V02_PIPELINERUN_PREFIX}-${shortSha(sourceCommit)}`; + return runtimeLanePipelineRunName(V02_LANE_SPEC, sourceCommit); } interface V02TriggerSnapshot { @@ -1314,7 +1364,7 @@ function secondsBetween(start: unknown, end: unknown): number | null { return Math.round((endMs - startMs) / 1000); } -function pipelineRunStatusRowFromLine(line: string, nowMs: number): Record | null { +function pipelineRunStatusRowFromLine(line: string, nowMs: number, pipelineRunPrefix = V02_PIPELINERUN_PREFIX): Record | null { const [ name = "", sourceCommit = "", @@ -1324,7 +1374,7 @@ function pipelineRunStatusRowFromLine(line: string, nowMs: number): Record 0 ? completionTime : null; const conditionStatus = status.trim().length > 0 ? status : null; @@ -1342,10 +1392,10 @@ function pipelineRunStatusRowFromLine(line: string, nowMs: number): Record { +function parsePipelineRunRows(text: string, limit: number, nowMs = Date.now(), pipelineRunPrefix = V02_PIPELINERUN_PREFIX): Record { const rows = text .split(/\r?\n/u) - .map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs)) + .map((line) => pipelineRunStatusRowFromLine(line.trim(), nowMs, pipelineRunPrefix)) .filter((item): item is Record => item !== null) .sort((left, right) => { return (timestampMs(right.createdAt) ?? 0) - (timestampMs(left.createdAt) ?? 0); @@ -2495,6 +2545,9 @@ function tailText(value: unknown, maxBytes = 2000): string { } function listCleanupPipelineRuns(options: G14ControlPlaneOptions): Record[] { + if (options.lane !== "v02" && options.lane !== "g14" && options.lane !== "all") { + throw new Error("control-plane cleanup-runs requires --lane v02|g14|all"); + } const result = g14K3s([ "kubectl", "get", @@ -2794,18 +2847,19 @@ function runControlPlaneCleanup(options: G14ControlPlaneOptions): Record&2", + ` echo "${spec.version} control-plane render cannot install dependencies: no lockfile found" >&2`, " exit 42", " fi", "fi", - `node scripts/g14-gitops-render.mjs --lane v02 --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, + "if [ -f scripts/run-bun.mjs ]; then", + ` node scripts/run-bun.mjs scripts/g14-gitops-render.mjs --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, + "else", + ` node scripts/g14-gitops-render.mjs --lane "$render_lane" --source-revision ${shellQuote(sourceCommit)} --out "$render_dir"`, + "fi", ].join("\n"); } +export function v02ControlPlaneRenderScript(sourceCommit: string, token = v02RenderToken()): string { + return runtimeLaneControlPlaneRenderScript(V02_LANE_SPEC, sourceCommit, token); +} + interface V02RenderTempResult { result: CommandJsonResult; renderDir: string; @@ -2834,68 +2896,112 @@ interface V02RenderTempResult { token: string; } -function runV02RenderToTemp(sourceCommit: string): V02RenderTempResult { - const token = v02RenderToken(); +function runRuntimeLaneRenderToTemp(spec: HwlabRuntimeLaneSpec, sourceCommit: string): V02RenderTempResult { + const token = runtimeLaneRenderToken(); return { - result: g14HostScript(v02ControlPlaneRenderScript(sourceCommit, token), 180_000), - renderDir: v02RenderDir(sourceCommit, token), - worktreeDir: v02RenderWorktreeDir(sourceCommit, token), + result: g14HostScript(runtimeLaneControlPlaneRenderScript(spec, sourceCommit, token), 180_000), + renderDir: runtimeLaneRenderDir(spec, sourceCommit, token), + worktreeDir: runtimeLaneRenderWorktreeDir(spec, sourceCommit, token), token, }; } -function v02RenderToken(): string { +function runV02RenderToTemp(sourceCommit: string): V02RenderTempResult { + return runRuntimeLaneRenderToTemp(V02_LANE_SPEC, sourceCommit); +} + +function runtimeLaneRenderToken(): string { const random = Math.random().toString(16).slice(2, 10); return `${process.pid}-${Date.now().toString(36)}-${random}`.replace(/[^a-zA-Z0-9_.-]/gu, "-"); } +function v02RenderToken(): string { + return runtimeLaneRenderToken(); +} + +function runtimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string { + return `/tmp/hwlab-${spec.lane}-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`; +} + function v02RenderDir(sourceCommit: string, token?: string): string { - return `/tmp/hwlab-v02-control-plane-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`; + return runtimeLaneRenderDir(V02_LANE_SPEC, sourceCommit, token); +} + +function runtimeLaneRenderWorktreeDir(spec: HwlabRuntimeLaneSpec, sourceCommit: string, token?: string): string { + return `/tmp/hwlab-${spec.lane}-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`; } function v02RenderWorktreeDir(sourceCommit: string, token?: string): string { - return `/tmp/hwlab-v02-control-plane-source-${shortSha(sourceCommit)}${token ? `-${token}` : ""}`; + return runtimeLaneRenderWorktreeDir(V02_LANE_SPEC, sourceCommit, token); } -function cleanupV02RenderDir(renderDir: string): CommandJsonResult { +function cleanupRuntimeLaneRenderDir(spec: HwlabRuntimeLaneSpec, renderDir: string): CommandJsonResult { return g14HostScript([ "set -eu", `render_dir=${shellQuote(renderDir)}`, "case \"$render_dir\" in", - " /tmp/hwlab-v02-control-plane-*) rm -rf \"$render_dir\" ;;", - " *) echo \"refusing to cleanup unexpected v0.2 render dir: $render_dir\" >&2; exit 44 ;;", + ` /tmp/hwlab-${spec.lane}-control-plane-*) rm -rf "$render_dir" ;;`, + ` *) echo "refusing to cleanup unexpected ${spec.version} render dir: $render_dir" >&2; exit 44 ;;`, "esac", ].join("\n"), 60_000); } -function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { - const args = [ +function cleanupV02RenderDir(renderDir: string): CommandJsonResult { + return cleanupRuntimeLaneRenderDir(V02_LANE_SPEC, renderDir); +} + +function applyRuntimeLaneControlPlaneFiles(spec: HwlabRuntimeLaneSpec, renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { + const namespaceFile = `${renderDir}/${spec.runtimeRenderDir}/namespace.yaml`; + const rbacFile = `${renderDir}/${spec.tektonDir}/rbac.yaml`; + const pipelineFile = `${renderDir}/${spec.tektonDir}/pipeline.yaml`; + const projectFile = `${renderDir}/argocd/project.yaml`; + const applicationFile = `${renderDir}/argocd/${spec.argoApplicationFile}`; + const resourceFiles = [namespaceFile, rbacFile, pipelineFile, projectFile, applicationFile]; + const serverSideArgs = [ "kubectl", "apply", "--server-side", "--force-conflicts", - `--field-manager=${V02_CONTROL_PLANE_FIELD_MANAGER}`, + `--field-manager=${spec.controlPlaneFieldManager}`, ...(dryRun ? ["--dry-run=server"] : []), - "-f", - `${renderDir}/tekton-v02/rbac.yaml`, - "-f", - `${renderDir}/tekton-v02/pipeline.yaml`, - "-f", - `${renderDir}/argocd/project.yaml`, - "-f", - `${renderDir}/argocd/application-v02.yaml`, + ...resourceFiles.flatMap((file) => ["-f", file]), ]; - const command = ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, ...args]; - const shellCommand = args.map(shellQuote).join(" "); + const command = ["bun", "scripts/cli.ts", "ssh", `${G14_PROVIDER}:k3s`, ...serverSideArgs]; + const shellCommand = dryRun && spec.lane !== "v02" + ? [ + "set -eu", + [ + ...serverSideArgs.slice(0, 6), + "-f", + namespaceFile, + "-f", + pipelineFile, + "-f", + projectFile, + "-f", + applicationFile, + ].map(shellQuote).join(" "), + [ + "kubectl", + "apply", + "--dry-run=client", + ...resourceFiles.flatMap((file) => ["-f", file]), + ].map(shellQuote).join(" "), + ].join("\n") + : `exec ${serverSideArgs.map(shellQuote).join(" ")}`; return runG14K3sRemoteAsync({ - script: `exec ${shellCommand}`, + script: shellCommand, timeoutSeconds, - label: "v02-control-plane-apply", - token: v02RenderToken(), + label: `${spec.lane}-control-plane-apply`, + token: runtimeLaneRenderToken(), command, }); } +function applyV02ControlPlaneFiles(renderDir: string, dryRun: boolean, timeoutSeconds: number): CommandJsonResult { + return applyRuntimeLaneControlPlaneFiles(V02_LANE_SPEC, renderDir, dryRun, timeoutSeconds); +} + interface V02RefreshMarker { ok: boolean; sourceCommit: string; @@ -3102,8 +3208,8 @@ function deleteV02ObsoleteCronJobs(dryRun: boolean): CommandJsonResult { ], 60_000); } -function v02PipelineRunManifest(sourceCommit: string): Record { - const pipelineRun = v02PipelineRunName(sourceCommit); +function runtimeLanePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string): Record { + const pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit); return { apiVersion: "tekton.dev/v1", kind: "PipelineRun", @@ -3112,20 +3218,20 @@ function v02PipelineRunManifest(sourceCommit: string): Record { namespace: CI_NAMESPACE, labels: { "app.kubernetes.io/part-of": "hwlab", - "hwlab.pikastech.local/gitops-target": "v02", + "hwlab.pikastech.local/gitops-target": spec.lane, "hwlab.pikastech.local/source-commit": sourceCommit, "hwlab.pikastech.local/trigger": "manual-cli", }, annotations: { - "hwlab.pikastech.local/source-branch": V02_SOURCE_BRANCH, - "hwlab.pikastech.local/gitops-branch": V02_GITOPS_BRANCH, + "hwlab.pikastech.local/source-branch": spec.sourceBranch, + "hwlab.pikastech.local/gitops-branch": spec.gitopsBranch, "hwlab.pikastech.local/triggered-by": "unidesk-cli", }, }, spec: { - pipelineRef: { name: V02_PIPELINE }, + pipelineRef: { name: spec.pipeline }, taskRunTemplate: { - serviceAccountName: "hwlab-v02-tekton-runner", + serviceAccountName: spec.serviceAccountName, podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", @@ -3133,18 +3239,18 @@ function v02PipelineRunManifest(sourceCommit: string): Record { }, }, params: [ - { name: "git-url", value: V02_GIT_URL }, - { name: "git-read-url", value: V02_GIT_READ_URL }, - { name: "source-branch", value: V02_SOURCE_BRANCH }, - { name: "gitops-branch", value: V02_GITOPS_BRANCH }, - { name: "lane", value: "v02" }, - { name: "catalog-path", value: V02_CATALOG_PATH }, + { name: "git-url", value: spec.gitUrl }, + { name: "git-read-url", value: spec.gitReadUrl }, + { name: "source-branch", value: spec.sourceBranch }, + { name: "gitops-branch", value: spec.gitopsBranch }, + { name: "lane", value: spec.lane }, + { name: "catalog-path", value: spec.catalogPath }, { name: "image-tag-mode", value: "full" }, - { name: "runtime-path", value: V02_RUNTIME_PATH }, + { name: "runtime-path", value: spec.runtimePath }, { name: "revision", value: sourceCommit }, - { name: "registry-prefix", value: V02_REGISTRY_PREFIX }, - { name: "services", value: V02_SERVICE_IDS.join(",") }, - { name: "base-image", value: V02_BASE_IMAGE }, + { name: "registry-prefix", value: spec.registryPrefix }, + { name: "services", value: spec.serviceIds.join(",") }, + { name: "base-image", value: spec.baseImage }, ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, @@ -3154,10 +3260,14 @@ function v02PipelineRunManifest(sourceCommit: string): Record { }; } -function createV02PipelineRun(sourceCommit: string, timeoutSeconds: number): CommandJsonResult { - const manifest = JSON.stringify(v02PipelineRunManifest(sourceCommit)); +function v02PipelineRunManifest(sourceCommit: string): Record { + return runtimeLanePipelineRunManifest(V02_LANE_SPEC, sourceCommit); +} + +function createRuntimeLanePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number): CommandJsonResult { + const manifest = JSON.stringify(runtimeLanePipelineRunManifest(spec, sourceCommit)); const manifestB64 = Buffer.from(manifest, "utf8").toString("base64"); - const pipelineRun = v02PipelineRunName(sourceCommit); + const pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit); const script = [ "set -eu", `manifest_b64=${shellQuote(manifestB64)}`, @@ -3178,6 +3288,10 @@ function createV02PipelineRun(sourceCommit: string, timeoutSeconds: number): Com return g14K3s(["script", "--", script], timeoutSeconds * 1000); } +function createV02PipelineRun(sourceCommit: string, timeoutSeconds: number): CommandJsonResult { + return createRuntimeLanePipelineRun(V02_LANE_SPEC, sourceCommit, timeoutSeconds); +} + function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record { const targetMode: V02StatusTargetMode = target.mode ?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head"); @@ -3404,6 +3518,360 @@ function v02ControlPlaneStatus(target: V02ControlPlaneStatusTarget = {}): Record }; } +function runtimeLaneControlPlaneStatusBundle(spec: HwlabRuntimeLaneSpec, target: V02ControlPlaneStatusTarget = {}): CommandJsonResult { + const targetMode: V02StatusTargetMode = target.mode + ?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head"); + const targetInit = target.pipelineRun !== undefined && target.pipelineRun !== null + ? [ + `pipeline_run=${shellQuote(target.pipelineRun)}`, + `source_commit=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.metadata.labels.hwlab\\.pikastech\\.local/source-commit}' 2>/dev/null || true)`, + `if [ -z "$source_commit" ]; then source_commit=$(printf '%s' "$pipeline_run" | sed 's/^${spec.pipelineRunPrefix}-//'); fi`, + ].join("\n") + : [ + target.sourceCommit === undefined + ? `source_commit=$(git --git-dir=${shellQuote(spec.cicdRepo)} rev-parse refs/remotes/origin/${spec.sourceBranch} 2>/dev/null || true)` + : `source_commit=${shellQuote(target.sourceCommit ?? "")}`, + "pipeline_run=", + ].join("\n"); + const controlPlaneProbe = [ + `kubectl get serviceaccount -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(spec.serviceAccountName)} -o name`, + `kubectl get pipeline -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(spec.pipeline)} -o name`, + ].join("; "); + const script = [ + "set +e", + targetInit, + `target_mode=${shellQuote(targetMode)}`, + `if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run="${spec.pipelineRunPrefix}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`, + "section() {", + " name=\"$1\"", + " shift", + " printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"", + " \"$@\"", + " code=$?", + " printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"", + "}", + "printf '__UNIDESK_SECTION_BEGIN__ statusTarget\\n'", + "printf 'mode\\t%s\\n' \"$target_mode\"", + "printf 'sourceCommit\\t%s\\n' \"$source_commit\"", + "printf 'pipelineRun\\t%s\\n' \"$pipeline_run\"", + "printf '__UNIDESK_SECTION_END__ statusTarget exit=0\\n'", + `section sourceHead git --git-dir=${shellQuote(spec.cicdRepo)} rev-parse refs/remotes/origin/${spec.sourceBranch}`, + `section controlPlane sh -lc ${shellQuote(controlPlaneProbe)}`, + `section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(spec.app)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`, + `section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'`, + `section runtimeWorkloads kubectl get deploy,statefulset,svc,ingress,configmap -n ${shellQuote(spec.runtimeNamespace)} -l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)} -o name`, + [ + `section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)}`, + `-l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)}`, + "--sort-by=.metadata.creationTimestamp", + "-o", + shellQuote(pipelineRunRowsJsonPath()), + ].join(" "), + ].join("\n"); + return g14K3s(["script", "--", script], 120_000); +} + +function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: V02ControlPlaneStatusTarget = {}): Record { + const targetMode: V02StatusTargetMode = target.mode + ?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head"); + const bundle = runtimeLaneControlPlaneStatusBundle(spec, target); + const sections = parseShellSections(statusText(bundle)); + const statusTargetFields = keyValueLinesFromText(sections.statusTarget?.stdout ?? ""); + const sourceCommit = stringOrNull(statusTargetFields.sourceCommit) ?? stringOrNull(sections.sourceHead?.stdout) ?? null; + const pipelineRun = stringOrNull(statusTargetFields.pipelineRun) ?? (sourceCommit === null ? null : runtimeLanePipelineRunName(spec, sourceCommit)); + const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(sections.argo?.stdout ?? "").split(/\r?\n/u); + const pipelineRunInfo = pipelineRun === null + ? null + : pipelineRunCompactFromText( + pipelineRun, + sections.pipelineRun?.stdout ?? "", + shellSectionOk(sections.pipelineRun), + `kubectl get pipelinerun -n ${CI_NAMESPACE} ${pipelineRun}`, + sections.pipelineRun?.exitCode ?? null, + bundle.stderr, + ); + const runtimeWorkloadNames = String(sections.runtimeWorkloads?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); + const recentPipelineRuns = parsePipelineRunRows(sections.recentPipelineRuns?.stdout ?? "", 8, Date.now(), spec.pipelineRunPrefix); + const baseOk = isCommandSuccess(bundle) && sourceCommit !== null && shellSectionOk(sections.controlPlane) && shellSectionOk(sections.argo); + return { + ok: baseOk, + command: `hwlab g14 control-plane status --lane ${spec.lane}`, + lane: spec.lane, + mode: "runtime-lane-status", + statusTarget: { + mode: statusTargetFields.mode || targetMode, + sourceCommit, + pipelineRun, + note: `${spec.lane} status is a bootstrap visibility check; v0.2 closeout/false-green verdicts are not reused for new lanes`, + }, + sourceCommit, + expected: { + sourceRepo: spec.cicdRepo, + workspace: spec.workspace, + branch: spec.sourceBranch, + namespace: CI_NAMESPACE, + runtimeNamespace: spec.runtimeNamespace, + pipeline: spec.pipeline, + serviceAccount: spec.serviceAccountName, + argoApplication: spec.app, + gitopsBranch: spec.gitopsBranch, + runtimePath: spec.runtimePath, + }, + sourceHead: { + ok: shellSectionOk(sections.sourceHead), + value: stringOrNull(sections.sourceHead?.stdout), + exitCode: sections.sourceHead?.exitCode ?? null, + }, + controlPlane: { + ok: shellSectionOk(sections.controlPlane), + raw: sections.controlPlane?.stdout ?? "", + exitCode: sections.controlPlane?.exitCode ?? null, + }, + argo: { + ok: shellSectionOk(sections.argo), + raw: sections.argo?.stdout ?? "", + fields: { targetRevision, path, syncRevision, syncStatus, health }, + exitCode: sections.argo?.exitCode ?? null, + }, + pipelineRun: pipelineRunInfo, + runtimeWorkloads: { + ok: shellSectionOk(sections.runtimeWorkloads), + namespace: spec.runtimeNamespace, + names: runtimeWorkloadNames, + count: runtimeWorkloadNames.length, + exitCode: sections.runtimeWorkloads?.exitCode ?? null, + }, + recentPipelineRuns, + query: { + ok: isCommandSuccess(bundle), + exitCode: bundle.exitCode, + stderr: bundle.stderr.trim().slice(0, 2000), + }, + next: { + apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm`, + triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm`, + }, + }; +} + +function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: G14ControlPlaneOptions): Record { + if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-runs" || options.action === "cleanup-released-pvs") { + return { + ok: false, + command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`, + lane: spec.lane, + degradedReason: "unsupported-runtime-lane-action", + message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current`, + }; + } + if (options.action === "status" && options.pipelineRun !== undefined) { + return runtimeLaneControlPlaneStatus(spec, { pipelineRun: options.pipelineRun, mode: "pipeline-run", includeHistory: options.history }); + } + if (options.action === "status" && options.sourceCommit !== undefined) { + return runtimeLaneControlPlaneStatus(spec, { sourceCommit: options.sourceCommit, mode: "source-commit", includeHistory: options.history }); + } + const latestHead = options.action === "trigger-current" ? resolveRuntimeLaneLatestRemoteHead(spec) : null; + const head = resolveRuntimeLaneHead(spec); + let sourceCommit = head.sourceCommit; + if (options.action === "trigger-current" && latestHead !== null && latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) { + const refreshedHead = resolveRuntimeLaneHead(spec); + sourceCommit = refreshedHead.sourceCommit; + if (sourceCommit !== latestHead.sourceCommit) { + return { + ok: false, + command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, + lane: spec.lane, + mode: "confirmed-trigger", + phase: "source-head-refresh", + sourceCommit, + latestSourceHeadProbe: { + sourceCommit: latestHead.sourceCommit, + result: compactCommandResult(latestHead.result), + }, + refreshedHeadProbe: compactCommandResult(refreshedHead.result), + degradedReason: `${spec.lane}-head-refresh-did-not-reach-latest-source`, + }; + } + } + if (sourceCommit === null) { + return { + ok: false, + command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`, + lane: spec.lane, + degradedReason: `${spec.lane}-head-unresolved`, + sourceRepo: spec.cicdRepo, + workspace: spec.workspace, + headProbe: compactCommandResult(head.result), + }; + } + if (options.action === "status") { + return runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }); + } + if (options.action === "apply") { + const render = runRuntimeLaneRenderToTemp(spec, sourceCommit); + if (!isCommandSuccess(render.result)) { + return { + ok: false, + command: `hwlab g14 control-plane apply --lane ${spec.lane}`, + lane: spec.lane, + phase: "source-render", + sourceCommit, + renderDir: render.renderDir, + render: compactCommandResult(render.result), + }; + } + const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, options.dryRun, options.timeoutSeconds); + const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null; + return { + ok: isCommandSuccess(apply), + command: `hwlab g14 control-plane apply --lane ${spec.lane}`, + lane: spec.lane, + mode: options.dryRun ? "dry-run" : "confirmed-apply", + sourceCommit, + renderDir: render.renderDir, + render: compactCommandResult(render.result), + apply: compactCommandResult(apply), + cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir), + status: runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }), + next: options.dryRun + ? { apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm` } + : { triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm` }, + }; + } + const pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit); + const before = getPipelineRunCompact(pipelineRun); + if (options.dryRun) { + return { + ok: true, + command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, + lane: spec.lane, + mode: "dry-run", + sourceCommit, + pipelineRun, + before, + gitMirrorSync: { + mode: "skipped-dry-run", + wouldSyncBeforeTrigger: true, + command: "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm", + }, + controlPlaneRefresh: { + mode: "skipped-dry-run", + wouldRefreshBeforeCreate: true, + resources: [`${spec.runtimeRenderDir}/namespace.yaml`, `${spec.tektonDir}/rbac.yaml`, `${spec.tektonDir}/pipeline.yaml`, "argocd/project.yaml", `argocd/${spec.argoApplicationFile}`], + }, + manifest: runtimeLanePipelineRunManifest(spec, sourceCommit), + next: { triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm` }, + }; + } + const beforeStatus = stringOrNull(before.status); + if (before.exists === true && (beforeStatus === "True" || beforeStatus === "Unknown")) { + return { + ok: true, + command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, + lane: spec.lane, + mode: "confirmed-trigger", + sourceCommit, + pipelineRun, + before, + skipped: true, + reason: "existing-pipelinerun-reused", + latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default", + }; + } + if (before.exists === true && beforeStatus !== null) { + return { + ok: false, + command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, + lane: spec.lane, + mode: "confirmed-trigger", + sourceCommit, + pipelineRun, + before, + skipped: true, + reason: "existing-pipelinerun-terminal-failed", + degradedReason: "existing-pipelinerun-terminal-failed", + next: { status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --pipeline-run ${pipelineRun}` }, + }; + } + const gitMirrorSync = runGitMirrorSync({ + action: "sync", + lane: spec.lane, + confirm: true, + dryRun: false, + wait: true, + timeoutSeconds: options.timeoutSeconds, + }); + if (gitMirrorSync.ok !== true) { + return { + ok: false, + command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, + lane: spec.lane, + mode: "confirmed-trigger", + sourceCommit, + pipelineRun, + before, + gitMirrorSync: compactGitMirrorSync(gitMirrorSync), + degradedReason: "git-mirror-sync-before-trigger-failed", + }; + } + const render = runRuntimeLaneRenderToTemp(spec, sourceCommit); + if (!isCommandSuccess(render.result)) { + return { + ok: false, + command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, + lane: spec.lane, + mode: "confirmed-trigger", + phase: "source-render", + sourceCommit, + pipelineRun, + before, + gitMirrorSync: compactGitMirrorSync(gitMirrorSync), + renderDir: render.renderDir, + render: compactCommandResult(render.result), + degradedReason: "control-plane-render-failed", + }; + } + const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, false, options.timeoutSeconds); + const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null; + if (!isCommandSuccess(apply)) { + return { + ok: false, + command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, + lane: spec.lane, + mode: "confirmed-trigger", + phase: "control-plane-apply", + sourceCommit, + pipelineRun, + before, + gitMirrorSync: compactGitMirrorSync(gitMirrorSync), + renderDir: render.renderDir, + render: compactCommandResult(render.result), + apply: compactCommandResult(apply), + cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir), + degradedReason: "control-plane-apply-before-trigger-failed", + }; + } + const create = createRuntimeLanePipelineRun(spec, sourceCommit, options.timeoutSeconds); + return { + ok: isCommandSuccess(create), + command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, + lane: spec.lane, + mode: "confirmed-trigger", + sourceCommit, + pipelineRun, + before, + gitMirrorSync: compactGitMirrorSync(gitMirrorSync), + renderDir: render.renderDir, + render: compactCommandResult(render.result), + apply: compactCommandResult(apply), + cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir), + create: compactCommandResult(create), + status: runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }), + degradedReason: isCommandSuccess(create) ? undefined : "pipelinerun-create-failed", + next: { status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --source-commit ${sourceCommit}` }, + }; +} + function runV02ControlPlane(options: G14ControlPlaneOptions): Record { if (options.action === "cleanup-runs") return runControlPlaneCleanup(options); if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options); @@ -4690,8 +5158,10 @@ function preSyncV02GitMirror(sourceCommit: string, options: Pick { } function runGitMirrorApply(options: G14GitMirrorOptions): Record { - const head = resolveV02Head(); + const spec = hwlabRuntimeLaneSpec(options.lane); + const head = resolveRuntimeLaneHead(spec); const sourceCommit = head.sourceCommit; if (sourceCommit === null) { - return { ok: false, command: "hwlab g14 git-mirror apply", degradedReason: "v02-head-unresolved", sourceRepo: V02_CICD_REPO, workspace: V02_WORKSPACE, headProbe: compactCommandResult(head.result) }; + return { ok: false, command: `hwlab g14 git-mirror apply --lane ${spec.lane}`, lane: spec.lane, degradedReason: `${spec.lane}-head-unresolved`, sourceRepo: spec.cicdRepo, workspace: spec.workspace, headProbe: compactCommandResult(head.result) }; } - const render = runV02RenderToTemp(sourceCommit); + const render = runRuntimeLaneRenderToTemp(spec, sourceCommit); if (!isCommandSuccess(render.result)) { return { ok: false, - command: "hwlab g14 git-mirror apply", + command: `hwlab g14 git-mirror apply --lane ${spec.lane}`, + lane: spec.lane, phase: "source-render", sourceCommit, renderDir: render.renderDir, @@ -4841,10 +5313,11 @@ function runGitMirrorApply(options: G14GitMirrorOptions): Record, timeoutSecon if (pendingFlush !== true) return { ok: true, skipped: true, reason: "git-mirror-already-flushed" }; return runG14GitMirror({ action: "flush", + lane: "v02", confirm: true, dryRun: false, wait: true, @@ -7907,6 +8384,12 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm", "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm", "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm --wait", + "bun scripts/cli.ts hwlab g14 control-plane status --lane v03", + "bun scripts/cli.ts hwlab g14 control-plane apply --lane v03 --dry-run", + "bun scripts/cli.ts hwlab g14 control-plane apply --lane v03 --confirm", + "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v03 --dry-run", + "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v03 --confirm", + "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v03 --confirm --wait", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --dry-run", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --min-age-minutes 30 --limit 20 --confirm", "bun scripts/cli.ts hwlab g14 control-plane cleanup-runs --lane v02 --pipeline-run hwlab-v02-ci-poll- --dry-run", @@ -7924,7 +8407,8 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name --dry-run", "bun scripts/cli.ts hwlab g14 secret delete --lane v02 --name --confirm", "bun scripts/cli.ts hwlab g14 git-mirror status", - "bun scripts/cli.ts hwlab g14 git-mirror apply --confirm", + "bun scripts/cli.ts hwlab g14 git-mirror apply --lane v02 --confirm", + "bun scripts/cli.ts hwlab g14 git-mirror apply --lane v03 --confirm", "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm", "bun scripts/cli.ts hwlab g14 git-mirror flush --confirm", "bun scripts/cli.ts hwlab g14 git-mirror sync --confirm --wait", @@ -7945,18 +8429,21 @@ export function hwlabG14Help(): Record { "bun scripts/cli.ts hwlab g14 upstream-image ensure --name openfga --tag v1.17.0 --confirm", "bun scripts/cli.ts job status --tail-bytes 30000", ], - description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, v0.2 runtime SecretRef bootstrap, devops-infra git mirror and observability maintenance, controlled CI tools image build/status entry, and allowlisted upstream image mirroring. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane status/closeout/apply/cleanup-runs/cleanup-released-pvs/runtime-migration uses UniDesk G14:k3s routes for v0.2 Tekton/Argo control resources, runtime migration, historical PipelineRun/source-commit closeout verdicts, GitOps mirror flush state, and completed CI workspace retention only. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. upstream-image status/ensure only mirrors allowlisted upstream runtime images into the G14 local registry. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob. observability status/apply/query/targets/boundary/closeout owns the shared Prometheus Operator and Prometheus instance in devops-infra, adds bounded PromQL assertions and semantic closeout summaries, while HWLAB lane manifests own only ServiceMonitor and PrometheusRule objects.", + description: "G14 HWLAB PR monitor, DEV rollout command, bounded v0.2 control-plane bootstrap/cleanup/runtime-migration helper, runtime lane v02/v03 control-plane apply/status/trigger entry, v0.2 runtime SecretRef bootstrap, devops-infra git mirror and observability maintenance, controlled CI tools image build/status entry, and allowlisted upstream image mirroring. The public monitor starts a fire-and-forget job. Default monitor lane is base=G14; --lane v02 monitors base=v0.2 PRs, waits for GitHub preflight/CI readiness, automatically merges ready PRs without waiting for other active v0.2 PipelineRuns, triggers v0.2 CD with latest-only GitOps writeback, flushes the git mirror when needed, and posts deduplicated PR comments for pending, blocked/conflict, success, superseded, failure, or timeout states. confirmed control-plane trigger-current and git-mirror sync/flush also return async jobs by default, with --wait reserved for explicit synchronous debugging. control-plane v02 keeps the full closeout/cleanup/runtime-migration verdict path; v03+ uses the runtime lane spec for Tekton/Argo apply, status visibility, and commit-pinned PipelineRun trigger. secret status/ensure is the standard v0.2 runtime SecretRef bootstrap path; it never reads or prints secret values. upstream-image status/ensure only mirrors allowlisted upstream runtime images into the G14 local registry. git-mirror status/apply/sync/flush is the manual devops-infra mirror/relay control path and does not install a CronJob. observability status/apply/query/targets/boundary/closeout owns the shared Prometheus Operator and Prometheus instance in devops-infra, adds bounded PromQL assertions and semantic closeout summaries, while HWLAB lane manifests own only ServiceMonitor and PrometheusRule objects.", defaults: { repo: HWLAB_REPO, base: G14_SOURCE_BRANCH, v02Base: V02_SOURCE_BRANCH, + runtimeLanes: hwlabRuntimeLaneIds(), provider: G14_PROVIDER, workspace: G14_WORKSPACE, v02Workspace: V02_WORKSPACE, + v03Workspace: hwlabRuntimeLaneSpec("v03").workspace, ciToolsImageRepo: G14_CI_TOOLS_IMAGE_REPO, intervalSeconds: DEFAULT_INTERVAL_SECONDS, devApplication: DEV_APP, v02Application: V02_APP, + v03Application: hwlabRuntimeLaneSpec("v03").app, briefIndexIssue: G14_BRIEF_INDEX_ISSUE, observabilityNamespace: G14_OBSERVABILITY_NAMESPACE, prometheusOperatorVersion: G14_PROMETHEUS_OPERATOR_VERSION, @@ -8049,6 +8536,9 @@ export async function runHwlabG14Command(_config: Config, args: string[]): Promi if (options.action === "trigger-current" && options.confirm && !options.dryRun && !options.wait) { return startControlPlaneTriggerJob(options); } + if (isHwlabRuntimeLane(options.lane) && options.lane !== "v02") { + return runRuntimeLaneControlPlane(hwlabRuntimeLaneSpec(options.lane), options); + } return runV02ControlPlane(options); } if (action === "secret") {