// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. control-plane module for scripts/src/hwlab-legacy-runtime.ts. // Moved mechanically from scripts/src/hwlab-legacy-runtime.ts:3737-5054 for #903. import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { createHash, randomBytes } from "node:crypto"; import { repoRoot, rootPath, type Config } from "../config"; import { runCommand } from "../command"; import { cancelJob, listJobs, readJob, startJob } from "../jobs"; import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes"; import type { CommandJsonResult, LegacyRuntimeControlPlaneOptions, DefaultRuntimeLaneControlPlaneStatusTarget, DefaultRuntimeLaneStatusTargetMode } from "./types"; import { runtimeLaneCdPassedWithMirror, runtimeLaneStatusWithGitMirror } from "./cd-trigger"; import { runControlPlaneCleanup, runControlPlaneReleasedPvCleanup, runDefaultRuntimeLaneRuntimeMigration } from "./cleanup"; import { compactGitMirrorSync, gitMirrorStatusSummary, preSyncDefaultRuntimeLaneGitMirror, runGitMirrorStatus, runGitMirrorSync, runtimeLaneGitMirrorSourceInSync } from "./git-mirror"; import { statusText, summarizeRuntimeLaneCdStatus, defaultRuntimeLaneControlPlaneCloseout } from "./pr-monitor"; import { compactCommandResult, compactControlPlaneRefresh, compactTriggerCommandResult, legacyRuntimeK3s, isCommandSuccess, nested, printProgressEvent, printRuntimeLaneTriggerProgress, record, shellQuote, stringOrNull } from "./remote"; import { applyRuntimeLaneControlPlaneFiles, applyDefaultRuntimeLaneControlPlaneFiles, cleanupRuntimeLaneRenderDir, cleanupDefaultRuntimeLaneRenderDir, deleteDefaultRuntimeLaneObsoleteCronJobs, refreshDefaultRuntimeLaneControlPlaneBeforeTrigger, runRuntimeLaneRenderToTemp, runDefaultRuntimeLaneRenderToTemp } from "./render"; import { resolveRuntimeLaneHead, resolveRuntimeLaneLatestRemoteHead, resolveDefaultRuntimeLaneHead, resolveDefaultRuntimeLaneLatestRemoteHead, runtimeLanePipelineRunName, runtimeLaneRerunPipelineRunName, defaultRuntimeLanePipelineRunName } from "./source"; import { ARGO_NAMESPACE, CI_NAMESPACE, DEFAULT_RUNTIME_LANE_APP, DEFAULT_RUNTIME_LANE_CICD_REPO, DEFAULT_RUNTIME_LANE_LANE_SPEC, DEFAULT_RUNTIME_LANE_PIPELINE, DEFAULT_RUNTIME_LANE_SOURCE_BRANCH, DEFAULT_RUNTIME_LANE_WORKSPACE } from "./types"; import { getPipelineRunCompact, getDefaultRuntimeLaneTriggerSnapshot, keyValueLinesFromText, listDefaultRuntimeLanePipelineRunsCompactFromText, numericField, parsePipelineRunRows, parseShellSections, pipelineRunCompactFromText, pipelineRunRowsJsonPath, runtimeLaneTaskRunDiagnosticsFromText, runtimeLaneTaskRunDiagnosticsScript, shellSectionOk, taskRunsCompactFromText, timestampMs, defaultRuntimeLaneCloseoutVerdict, defaultRuntimeLaneCommitAlignment, defaultRuntimeLaneControlPlaneStatusBundle, defaultRuntimeLaneExistingPipelineRunReuseDecision, defaultRuntimeLaneFalseGreenGuard, defaultRuntimeLaneLatestOnlyTargetValidation, defaultRuntimeLanePlanArtifactsFromText, defaultRuntimeLaneRuntimeWorkloadsFromText, defaultRuntimeLaneSourceHeadsFromText, defaultRuntimeLaneTargetValidation, defaultRuntimeLaneWebAssetsFromText } from "./runtime-status"; export function runtimeLanePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit)): Record { return { apiVersion: "tekton.dev/v1", kind: "PipelineRun", metadata: { name: pipelineRun, namespace: CI_NAMESPACE, labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/gitops-target": spec.lane, "hwlab.pikastech.local/source-commit": sourceCommit, "hwlab.pikastech.local/trigger": "manual-cli", }, annotations: { "hwlab.pikastech.local/source-branch": spec.sourceBranch, "hwlab.pikastech.local/gitops-branch": spec.gitopsBranch, "hwlab.pikastech.local/node": spec.nodeId, "hwlab.pikastech.local/network-profile": spec.networkProfileId, "hwlab.pikastech.local/download-profile": spec.downloadProfileId, "hwlab.pikastech.local/no-proxy-required": hwlabRequiredNoProxyEntries().join(","), "hwlab.pikastech.local/triggered-by": "unidesk-cli", }, }, spec: { pipelineRef: { name: spec.pipeline }, taskRunTemplate: { serviceAccountName: spec.serviceAccountName, podTemplate: { hostNetwork: true, dnsPolicy: "ClusterFirstWithHostNet", securityContext: { fsGroup: 1000 }, }, }, params: [ { name: "git-url", value: spec.gitUrl }, { name: "git-read-url", value: spec.gitReadUrl }, { name: "source-branch", value: spec.sourceBranch }, { name: "gitops-branch", value: spec.gitopsBranch }, { name: "lane", value: spec.lane }, { name: "catalog-path", value: spec.catalogPath }, { name: "image-tag-mode", value: "full" }, { name: "runtime-path", value: spec.runtimePath }, { name: "revision", value: sourceCommit }, { name: "registry-prefix", value: spec.registryPrefix }, { name: "services", value: spec.serviceIds.join(",") }, { name: "base-image", value: spec.baseImage }, ], workspaces: [ { name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } }, { name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } }, ], }, }; } export function defaultRuntimeLanePipelineRunManifest(sourceCommit: string): Record { return runtimeLanePipelineRunManifest(DEFAULT_RUNTIME_LANE_LANE_SPEC, sourceCommit); } export function createRuntimeLanePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number, pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit)): CommandJsonResult { const manifest = JSON.stringify(runtimeLanePipelineRunManifest(spec, sourceCommit, pipelineRun)); const manifestB64 = Buffer.from(manifest, "utf8").toString("base64"); const script = [ "set -eu", `manifest_b64=${shellQuote(manifestB64)}`, `manifest_path=/tmp/${pipelineRun}.json`, "printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"", "if kubectl create -f \"$manifest_path\"; then", " :", "else", " code=$?", ` if kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} >/dev/null 2>&1; then`, ` printf 'PipelineRun %s already exists; reusing existing object\\n' ${shellQuote(pipelineRun)} >&2`, " else", " exit \"$code\"", " fi", "fi", `kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'`, ].join("\n"); return legacyRuntimeK3s(["sh", "--", script], timeoutSeconds * 1000); } export function createDefaultRuntimeLanePipelineRun(sourceCommit: string, timeoutSeconds: number): CommandJsonResult { return createRuntimeLanePipelineRun(DEFAULT_RUNTIME_LANE_LANE_SPEC, sourceCommit, timeoutSeconds); } export function defaultRuntimeLaneControlPlaneStatus(target: DefaultRuntimeLaneControlPlaneStatusTarget = {}): Record { const targetMode: DefaultRuntimeLaneStatusTargetMode = target.mode ?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head"); const includeHistory = target.includeHistory === true; const strictHeadAlignment = targetMode === "latest-source-head"; const bundle = defaultRuntimeLaneControlPlaneStatusBundle({ ...target, mode: targetMode }); const sections = parseShellSections(statusText(bundle)); const sourceCommit = stringOrNull(sections.sourceCommit?.stdout) ?? null; const pipelineRun = stringOrNull(sections.pipelineRunName?.stdout) ?? (sourceCommit === null ? null : defaultRuntimeLanePipelineRunName(sourceCommit)); const statusTargetFields = keyValueLinesFromText(sections.statusTarget?.stdout ?? ""); const queryNowMs = timestampMs(sections.queryNow?.stdout) ?? Date.now(); const sourceHeadsSection = sections.sourceHeads; const controlPlane = sections.controlPlane; const obsoleteCronJobs = sections.obsoleteCronJobs; const argo = sections.argo; const pipelineRunSection = sections.pipelineRun; const taskRunsSection = sections.taskRuns; const planArtifactsSection = sections.planArtifacts; const runtimeWorkloadsSection = sections.runtimeWorkloads; const gitMirrorCacheSection = sections.gitMirrorCache; const webAssetsSection = sections.webAssets; const recentPipelineRuns = listDefaultRuntimeLanePipelineRunsCompactFromText( sections.recentPipelineRuns?.stdout ?? "", shellSectionOk(sections.recentPipelineRuns), "kubectl get pipelinerun -n hwlab-ci -l hwlab.pikastech.local/gitops-target=v02 -o jsonpath=", sections.recentPipelineRuns?.exitCode ?? null, bundle.stderr, 8, queryNowMs, ); const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : []; const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u); const sourceHeads = defaultRuntimeLaneSourceHeadsFromText( sourceHeadsSection?.stdout ?? "", shellSectionOk(sourceHeadsSection), sourceHeadsSection?.exitCode ?? null, bundle.stderr, ); const pipelineRunInfo = pipelineRun === null ? null : pipelineRunCompactFromText( pipelineRun, pipelineRunSection?.stdout ?? "", shellSectionOk(pipelineRunSection), `kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`, pipelineRunSection?.exitCode ?? null, bundle.stderr, ); const taskRuns = taskRunsCompactFromText( taskRunsSection?.stdout ?? "", shellSectionOk(taskRunsSection), pipelineRun, taskRunsSection?.exitCode ?? null, bundle.stderr, ); const planArtifacts = defaultRuntimeLanePlanArtifactsFromText( planArtifactsSection?.stdout ?? "", shellSectionOk(planArtifactsSection), pipelineRun, planArtifactsSection?.exitCode ?? null, bundle.stderr, ); const runtimeWorkloads = defaultRuntimeLaneRuntimeWorkloadsFromText( runtimeWorkloadsSection?.stdout ?? "", shellSectionOk(runtimeWorkloadsSection), runtimeWorkloadsSection?.exitCode ?? null, bundle.stderr, ); const webAssets = defaultRuntimeLaneWebAssetsFromText( webAssetsSection?.stdout ?? "", shellSectionOk(webAssetsSection), sourceCommit, syncRevision, webAssetsSection?.exitCode ?? null, bundle.stderr, activePipelineRuns, ); const gitMirror = { ok: shellSectionOk(gitMirrorCacheSection), summary: gitMirrorStatusSummary(String(gitMirrorCacheSection?.stdout ?? "").trim()), raw: String(gitMirrorCacheSection?.stdout ?? "").trim(), exitCode: gitMirrorCacheSection?.exitCode ?? null, stderr: shellSectionOk(gitMirrorCacheSection) ? "" : bundle.stderr.trim().slice(0, 2000), }; const commitAlignment = defaultRuntimeLaneCommitAlignment({ expectedSourceHead: sourceCommit, sourceHeads, gitMirrorSummary: record(gitMirror.summary), pipelineRun: pipelineRunInfo, recentPipelineRuns, planArtifacts, runtimeWorkloads, webAssets, }); const targetValidationBase = defaultRuntimeLaneTargetValidation({ targetMode, sourceCommit, pipelineRun: pipelineRunInfo, argo: { ok: shellSectionOk(argo), fields: { targetRevision, path, syncRevision, syncStatus, health }, }, planArtifacts, runtimeWorkloads, webAssets, gitMirror, recentPipelineRuns, }); const targetValidation = defaultRuntimeLaneLatestOnlyTargetValidation({ targetMode, sourceCommit, pipelineRun: pipelineRunInfo, commitAlignment, targetValidation: targetValidationBase, }); const sourceHeadsTarget = record(sourceHeads.target); const falseGreenGuard = targetValidation.state === "superseded" ? { ok: null, state: "superseded", summary: "target PipelineRun succeeded but runtime now reflects a newer v0.2 PipelineRun; current-runtime false-green guard is not applicable to this historical target", sourceCommit, targetValidationState: targetValidation.state, newerSucceededRuns: targetValidation.newerSucceededRuns ?? [], supersededRuntimeServices: targetValidation.supersededRuntimeServices ?? [], } : defaultRuntimeLaneFalseGreenGuard({ sourceCommit, pipelineRun: pipelineRunInfo, taskRuns, planArtifacts, runtimeWorkloads }); const baseOk = sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo); const targetPipelineRunOk = strictHeadAlignment ? true : pipelineRunInfo !== null && pipelineRunInfo.ok === true && pipelineRunInfo.exists !== false; const targetDegradedReason = !strictHeadAlignment && !targetPipelineRunOk ? "target-pipelinerun-not-found-or-unreadable" : undefined; const statusCommand = targetMode === "pipeline-run" && pipelineRun !== null ? `hwlab g14 control-plane status --lane v02 --pipeline-run ${pipelineRun}` : targetMode === "source-commit" && sourceCommit !== null ? `hwlab g14 control-plane status --lane v02 --source-commit ${sourceCommit}` : "hwlab g14 control-plane status --lane v02"; const state = strictHeadAlignment ? commitAlignment.state : `target-${targetMode}`; const sourceCommitSource = targetMode === "pipeline-run" ? "PipelineRun metadata label hwlab.pikastech.local/source-commit" : targetMode === "source-commit" ? "explicit --source-commit" : "G14 CI/CD dedicated bare repo refs/remotes/origin/v0.2; workspace checkout is observable but isolated"; const status: Record = { ok: baseOk && targetPipelineRunOk && (!strictHeadAlignment || commitAlignment.aligned !== false), command: statusCommand, lane: "v02", state, degradedReason: strictHeadAlignment && commitAlignment.aligned === false ? commitAlignment.state : targetDegradedReason, statusTarget: { mode: statusTargetFields.mode || targetMode, sourceCommit, pipelineRun, strictHeadAlignment, ancestorOfLatest: sourceHeadsTarget.ancestorOfOriginHead ?? null, note: strictHeadAlignment ? "default status validates the latest v0.2 source head alignment" : "targeted status inspects the requested PipelineRun/source commit without failing merely because origin/v0.2 advanced later", }, sourceCommit, sourceCommitSource, expected: { sourceRepo: DEFAULT_RUNTIME_LANE_CICD_REPO, workspace: DEFAULT_RUNTIME_LANE_WORKSPACE, workspaceIsolation: "workspace dirty/stale state must not select CI/CD source commits", branch: DEFAULT_RUNTIME_LANE_SOURCE_BRANCH, namespace: CI_NAMESPACE, runtimeNamespace: "hwlab-v02", pipeline: DEFAULT_RUNTIME_LANE_PIPELINE, trigger: "manual UniDesk CLI PipelineRun creation", argoApplication: DEFAULT_RUNTIME_LANE_APP, }, commitAlignment, targetValidation, sourceHeads, sourceHeadsTargetAncestorOfOriginHead: sourceHeadsTarget.ancestorOfOriginHead ?? null, gitMirror, controlPlane: { ok: shellSectionOk(controlPlane), names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), exitCode: controlPlane?.exitCode ?? null, }, obsoleteCronJobs: { ok: shellSectionOk(obsoleteCronJobs), names: String(obsoleteCronJobs?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean), exitCode: obsoleteCronJobs?.exitCode ?? null, cleanupCommand: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm", }, argo: { ok: shellSectionOk(argo), raw: argo?.stdout ?? "", fields: { targetRevision, path, syncRevision, syncStatus, health }, exitCode: argo?.exitCode ?? null, }, pipelineRun: pipelineRunInfo, taskRuns, planArtifacts, runtimeWorkloads, falseGreenGuard, webAssets, activePipelineRuns, history: { included: includeHistory, activePipelineRunCount: activePipelineRuns.length, recentPipelineRunCount: Array.isArray(recentPipelineRuns.items) ? recentPipelineRuns.items.length : 0, note: includeHistory ? "recentPipelineRuns is included because --history was requested; historical runs are diagnostic evidence, not the default current-target verdict" : "recent PipelineRun history is hidden by default so old manifest/service-id failures do not pollute the current-target verdict; rerun with --history to inspect it", command: statusCommand.includes("--history") ? statusCommand : `${statusCommand} --history`, }, ...(includeHistory ? { recentPipelineRuns } : {}), query: { ok: isCommandSuccess(bundle), exitCode: bundle.exitCode, stderr: bundle.stderr.trim().slice(0, 2000), }, visibilityHint: activePipelineRuns.length > 0 ? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger" : "no active v02 PipelineRun observed", }; return { ...status, closeout: defaultRuntimeLaneCloseoutVerdict(status), }; } export function runtimeLaneControlPlaneStatusBundle(spec: HwlabRuntimeLaneSpec, target: DefaultRuntimeLaneControlPlaneStatusTarget = {}): CommandJsonResult { const targetMode: DefaultRuntimeLaneStatusTargetMode = target.mode ?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head"); const targetInit = target.pipelineRun !== undefined && target.pipelineRun !== null ? [ `pipeline_run=${shellQuote(target.pipelineRun)}`, `source_commit=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.metadata.labels.hwlab\\.pikastech\\.local/source-commit}' 2>/dev/null || true)`, `if [ -z "$source_commit" ]; then source_commit=$(printf '%s' "$pipeline_run" | sed 's/^${spec.pipelineRunPrefix}-//'); fi`, ].join("\n") : [ target.sourceCommit === undefined ? `source_commit=$(git --git-dir=${shellQuote(spec.cicdRepo)} rev-parse refs/remotes/origin/${spec.sourceBranch} 2>/dev/null || true)` : `source_commit=${shellQuote(target.sourceCommit ?? "")}`, "pipeline_run=", ].join("\n"); const controlPlaneProbe = [ `kubectl get serviceaccount -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(spec.serviceAccountName)} -o name`, `kubectl get pipeline -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(spec.pipeline)} -o name`, ].join("; "); const script = [ "set +e", targetInit, `target_mode=${shellQuote(targetMode)}`, `if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l "hwlab.pikastech.local/gitops-target=${spec.lane},hwlab.pikastech.local/source-commit=$source_commit" --sort-by=.metadata.creationTimestamp -o name 2>/dev/null | sed 's#.*/##' | tail -n 1 || true); fi`, `if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run="${spec.pipelineRunPrefix}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`, "section() {", " name=\"$1\"", " shift", " printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"", " \"$@\"", " code=$?", " printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"", "}", "printf '__UNIDESK_SECTION_BEGIN__ statusTarget\\n'", "printf 'mode\\t%s\\n' \"$target_mode\"", "printf 'sourceCommit\\t%s\\n' \"$source_commit\"", "printf 'pipelineRun\\t%s\\n' \"$pipeline_run\"", "printf '__UNIDESK_SECTION_END__ statusTarget exit=0\\n'", `section sourceHead git --git-dir=${shellQuote(spec.cicdRepo)} rev-parse refs/remotes/origin/${spec.sourceBranch}`, `section controlPlane sh -lc ${shellQuote(controlPlaneProbe)}`, `section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(spec.app)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`, `section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'`, `section taskRunDiagnostics env PIPELINE_RUN="$pipeline_run" CI_NAMESPACE=${shellQuote(CI_NAMESPACE)} TASKRUN_LOG_TAIL_LINES=60 TASKRUN_LOG_MAX_CHARS=4000 sh -lc ${shellQuote(runtimeLaneTaskRunDiagnosticsScript())}`, `section runtimeWorkloads kubectl get deploy,statefulset,svc,ingress,configmap -n ${shellQuote(spec.runtimeNamespace)} -l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)} -o name`, `section publicProbes sh -lc ${shellQuote(runtimeLanePublicProbeScript(spec))}`, [ `section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)}`, `-l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)}`, "--sort-by=.metadata.creationTimestamp", "-o", shellQuote(pipelineRunRowsJsonPath()), ].join(" "), ].join("\n"); return legacyRuntimeK3s(["sh", "--", script], 120_000); } export function runtimeLanePublicProbeScript(spec: HwlabRuntimeLaneSpec): string { const apiUrl = `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`; const webUrl = spec.publicWebUrl.replace(/\/+$/u, "/"); return [ "set +e", `api_url=${shellQuote(apiUrl)}`, `web_url=${shellQuote(webUrl)}`, "api_output=$(curl -fsS --connect-timeout 3 --max-time 15 \"$api_url\" 2>&1)", "api_exit=$?", "api_status_ok=no", "if [ \"$api_exit\" = 0 ] && printf '%s' \"$api_output\" | grep -Eq '\"status\"[[:space:]]*:[[:space:]]*\"ok\"'; then api_status_ok=yes; fi", "web_tmp=$(mktemp /tmp/hwlab-lane-web-probe.XXXXXX)", "web_output=$(curl -fsS --connect-timeout 3 --max-time 15 -o \"$web_tmp\" \"$web_url\" 2>&1)", "web_exit=$?", "web_bytes=0", "web_has_html=no", "if [ -f \"$web_tmp\" ]; then", " web_bytes=$(wc -c < \"$web_tmp\" | tr -d ' ')", " if grep -Eiq ' { const targetMode: DefaultRuntimeLaneStatusTargetMode = target.mode ?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head"); const bundle = runtimeLaneControlPlaneStatusBundle(spec, target); const sections = parseShellSections(statusText(bundle)); const statusTargetFields = keyValueLinesFromText(sections.statusTarget?.stdout ?? ""); const sourceCommit = stringOrNull(statusTargetFields.sourceCommit) ?? stringOrNull(sections.sourceHead?.stdout) ?? null; const pipelineRun = stringOrNull(statusTargetFields.pipelineRun) ?? (sourceCommit === null ? null : runtimeLanePipelineRunName(spec, sourceCommit)); const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(sections.argo?.stdout ?? "").split(/\r?\n/u); const pipelineRunInfo = pipelineRun === null ? null : pipelineRunCompactFromText( pipelineRun, sections.pipelineRun?.stdout ?? "", shellSectionOk(sections.pipelineRun), `kubectl get pipelinerun -n ${CI_NAMESPACE} ${pipelineRun}`, sections.pipelineRun?.exitCode ?? null, bundle.stderr, ); const taskRuns = pipelineRun === null ? null : runtimeLaneTaskRunDiagnosticsFromText(pipelineRun, sections.taskRunDiagnostics, bundle.stderr); const runtimeWorkloadNames = String(sections.runtimeWorkloads?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean); const publicProbeFields = keyValueLinesFromText(sections.publicProbes?.stdout ?? ""); const publicProbesOk = shellSectionOk(sections.publicProbes) && publicProbeFields.apiExitCode === "0" && publicProbeFields.apiStatusOk === "yes" && publicProbeFields.webExitCode === "0" && Number(publicProbeFields.webBytes ?? 0) > 0; const recentPipelineRuns = parsePipelineRunRows(sections.recentPipelineRuns?.stdout ?? "", 8, Date.now(), spec.pipelineRunPrefix); const pipelineRunOk = pipelineRunInfo !== null && pipelineRunInfo.status === "True"; const argoOk = shellSectionOk(sections.argo) && syncStatus === "Synced" && health === "Healthy"; const runtimeWorkloadsOk = shellSectionOk(sections.runtimeWorkloads); const baseOk = isCommandSuccess(bundle) && sourceCommit !== null && shellSectionOk(sections.controlPlane) && pipelineRunOk && argoOk && runtimeWorkloadsOk && publicProbesOk; return { ok: baseOk, command: `hwlab g14 control-plane status --lane ${spec.lane}`, lane: spec.lane, mode: "runtime-lane-status", statusTarget: { mode: statusTargetFields.mode || targetMode, sourceCommit, pipelineRun, note: `${spec.lane} status is a bootstrap visibility check; v0.2 closeout/false-green verdicts are not reused for new lanes`, }, sourceCommit, expected: { sourceRepo: spec.cicdRepo, configPath: hwlabRuntimeLaneConfigPath(), node: spec.nodeId, nodeRoute: spec.nodeRoute, workspace: spec.workspace, branch: spec.sourceBranch, namespace: CI_NAMESPACE, runtimeNamespace: spec.runtimeNamespace, pipeline: spec.pipeline, serviceAccount: spec.serviceAccountName, argoApplication: spec.app, gitopsBranch: spec.gitopsBranch, runtimePath: spec.runtimePath, networkProfile: { id: spec.networkProfileId, httpProxy: spec.networkProfile.proxy.http, noProxy: spec.networkProfile.proxy.noProxy.join(","), dockerBuildHttpProxy: spec.networkProfile.dockerBuildProxy.http, dockerBuildNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","), }, downloadProfile: spec.downloadProfile, }, sourceHead: { ok: shellSectionOk(sections.sourceHead), value: stringOrNull(sections.sourceHead?.stdout), exitCode: sections.sourceHead?.exitCode ?? null, }, controlPlane: { ok: shellSectionOk(sections.controlPlane), raw: sections.controlPlane?.stdout ?? "", exitCode: sections.controlPlane?.exitCode ?? null, }, argo: { ok: shellSectionOk(sections.argo), raw: sections.argo?.stdout ?? "", fields: { targetRevision, path, syncRevision, syncStatus, health }, exitCode: sections.argo?.exitCode ?? null, }, pipelineRun: pipelineRunInfo, taskRuns, runtimeWorkloads: { ok: shellSectionOk(sections.runtimeWorkloads), namespace: spec.runtimeNamespace, names: runtimeWorkloadNames, count: runtimeWorkloadNames.length, exitCode: sections.runtimeWorkloads?.exitCode ?? null, }, publicProbes: { ok: publicProbesOk, apiUrl: publicProbeFields.apiUrl || `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`, apiExitCode: numericField(publicProbeFields.apiExitCode), apiStatusOk: publicProbeFields.apiStatusOk === "yes", apiPreview: publicProbeFields.apiPreview || null, webUrl: publicProbeFields.webUrl || spec.publicWebUrl, webExitCode: numericField(publicProbeFields.webExitCode), webBytes: numericField(publicProbeFields.webBytes), webHasHtml: publicProbeFields.webHasHtml === "yes", webErrorPreview: publicProbeFields.webErrorPreview || null, exitCode: sections.publicProbes?.exitCode ?? null, }, recentPipelineRuns, query: { ok: isCommandSuccess(bundle), exitCode: bundle.exitCode, timedOut: bundle.timedOut ?? false, durationMs: bundle.durationMs ?? null, stdoutBytes: bundle.stdoutBytes ?? Buffer.byteLength(bundle.stdout, "utf8"), stderrBytes: bundle.stderrBytes ?? Buffer.byteLength(bundle.stderr, "utf8"), stderr: bundle.stderr.trim().slice(0, 2000), }, next: { apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm`, refresh: `bun scripts/cli.ts hwlab g14 control-plane refresh --lane ${spec.lane} --confirm`, triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm`, }, }; } export function runtimeLaneArgoRefreshScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string { const operationPatch = JSON.stringify({ operation: null }); return [ "set +e", `app=${shellQuote(spec.app)}`, `argo_namespace=${shellQuote(ARGO_NAMESPACE)}`, `runtime_namespace=${shellQuote(spec.runtimeNamespace)}`, `lane=${shellQuote(spec.lane)}`, `dry_run=${shellQuote(dryRun ? "true" : "false")}`, `operation_patch=${shellQuote(operationPatch)}`, "app_jsonpath() { kubectl -n \"$argo_namespace\" get application \"$app\" -o \"jsonpath=$1\" 2>/dev/null || true; }", "job_jsonpath() { kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o \"jsonpath=$1\" 2>/dev/null || true; }", "before_operation_revision=$(app_jsonpath '{.operation.sync.revision}')", "before_operation_phase=$(app_jsonpath '{.status.operationState.phase}')", "before_operation_message=$(app_jsonpath '{.status.operationState.message}')", "before_sync_revision=$(app_jsonpath '{.status.sync.revision}')", "before_sync_status=$(app_jsonpath '{.status.sync.status}')", "before_health=$(app_jsonpath '{.status.health.status}')", "before_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')", "before_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)", "before_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')", "before_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')", "before_hook_active=$(job_jsonpath '{.status.active}')", "before_operation_present=$([ -n \"$before_operation_revision$before_operation_phase$before_operation_message\" ] && printf yes || printf no)", "patch_exit=", "patch_output=skipped-no-live-operation", "annotate_exit=", "annotate_output=", "if [ \"$before_operation_present\" = yes ]; then", " if [ \"$dry_run\" = true ]; then", " patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" --dry-run=server -o name 2>&1)", " else", " patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" -o name 2>&1)", " fi", " patch_exit=$?", "fi", "if [ \"$dry_run\" = true ]; then", " annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)", "else", " annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)", "fi", "annotate_exit=$?", "if [ \"$dry_run\" != true ]; then sleep 2; fi", "after_operation_revision=$(app_jsonpath '{.operation.sync.revision}')", "after_operation_phase=$(app_jsonpath '{.status.operationState.phase}')", "after_operation_message=$(app_jsonpath '{.status.operationState.message}')", "after_sync_revision=$(app_jsonpath '{.status.sync.revision}')", "after_sync_status=$(app_jsonpath '{.status.sync.status}')", "after_health=$(app_jsonpath '{.status.health.status}')", "after_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')", "after_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)", "after_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')", "after_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')", "after_hook_active=$(job_jsonpath '{.status.active}')", "after_operation_present=$([ -n \"$after_operation_revision$after_operation_phase$after_operation_message\" ] && printf yes || printf no)", "printf 'app\\t%s\\n' \"$app\"", "printf 'lane\\t%s\\n' \"$lane\"", "printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"", "printf 'dryRun\\t%s\\n' \"$dry_run\"", "printf 'beforeOperationPresent\\t%s\\n' \"$before_operation_present\"", "printf 'beforeOperationRevision\\t%s\\n' \"$before_operation_revision\"", "printf 'beforeOperationPhase\\t%s\\n' \"$before_operation_phase\"", "printf 'beforeOperationMessage\\t%s\\n' \"$before_operation_message\"", "printf 'beforeSyncRevision\\t%s\\n' \"$before_sync_revision\"", "printf 'beforeSyncStatus\\t%s\\n' \"$before_sync_status\"", "printf 'beforeHealth\\t%s\\n' \"$before_health\"", "printf 'beforeResourceVersion\\t%s\\n' \"$before_resource_version\"", "printf 'beforeHookJob\\t%s\\n' \"$before_hook_job\"", "printf 'beforeHookSourceCommit\\t%s\\n' \"$before_hook_source\"", "printf 'beforeHookPhase\\t%s\\n' \"$before_hook_phase\"", "printf 'beforeHookActive\\t%s\\n' \"$before_hook_active\"", "printf 'patchExitCode\\t%s\\n' \"$patch_exit\"", "printf 'patchOutput\\t%s\\n' \"$patch_output\"", "printf 'annotateExitCode\\t%s\\n' \"$annotate_exit\"", "printf 'annotateOutput\\t%s\\n' \"$annotate_output\"", "printf 'afterOperationPresent\\t%s\\n' \"$after_operation_present\"", "printf 'afterOperationRevision\\t%s\\n' \"$after_operation_revision\"", "printf 'afterOperationPhase\\t%s\\n' \"$after_operation_phase\"", "printf 'afterOperationMessage\\t%s\\n' \"$after_operation_message\"", "printf 'afterSyncRevision\\t%s\\n' \"$after_sync_revision\"", "printf 'afterSyncStatus\\t%s\\n' \"$after_sync_status\"", "printf 'afterHealth\\t%s\\n' \"$after_health\"", "printf 'afterResourceVersion\\t%s\\n' \"$after_resource_version\"", "printf 'afterHookJob\\t%s\\n' \"$after_hook_job\"", "printf 'afterHookSourceCommit\\t%s\\n' \"$after_hook_source\"", "printf 'afterHookPhase\\t%s\\n' \"$after_hook_phase\"", "printf 'afterHookActive\\t%s\\n' \"$after_hook_active\"", "if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi", "if [ -n \"$annotate_exit\" ] && [ \"$annotate_exit\" != 0 ]; then exit \"$annotate_exit\"; fi", ].join("\n"); } export function refreshRuntimeLaneArgoApplication(spec: HwlabRuntimeLaneSpec, options: LegacyRuntimeControlPlaneOptions): Record { const result = legacyRuntimeK3s(["sh", "--", runtimeLaneArgoRefreshScript(spec, options.dryRun)], options.timeoutSeconds * 1000); const fields = keyValueLinesFromText(statusText(result)); const ok = isCommandSuccess(result); return { ok, command: `hwlab g14 control-plane refresh --lane ${spec.lane}`, lane: spec.lane, mode: options.dryRun ? "dry-run" : "confirmed-refresh", argoApplication: spec.app, runtimeNamespace: spec.runtimeNamespace, dryRun: options.dryRun, action: { terminatedOperation: fields.beforeOperationPresent === "yes", hardRefreshRequested: numericField(fields.annotateExitCode) === 0, mutation: !options.dryRun && ok, }, before: { operationPresent: fields.beforeOperationPresent === "yes", operationRevision: fields.beforeOperationRevision || null, operationPhase: fields.beforeOperationPhase || null, operationMessage: fields.beforeOperationMessage || null, syncRevision: fields.beforeSyncRevision || null, syncStatus: fields.beforeSyncStatus || null, health: fields.beforeHealth || null, resourceVersion: fields.beforeResourceVersion || null, hookJob: fields.beforeHookJob || null, hookSourceCommit: fields.beforeHookSourceCommit || null, hookPhase: fields.beforeHookPhase || null, hookActive: fields.beforeHookActive || null, }, after: { operationPresent: fields.afterOperationPresent === "yes", operationRevision: fields.afterOperationRevision || null, operationPhase: fields.afterOperationPhase || null, operationMessage: fields.afterOperationMessage || null, syncRevision: fields.afterSyncRevision || null, syncStatus: fields.afterSyncStatus || null, health: fields.afterHealth || null, resourceVersion: fields.afterResourceVersion || null, hookJob: fields.afterHookJob || null, hookSourceCommit: fields.afterHookSourceCommit || null, hookPhase: fields.afterHookPhase || null, hookActive: fields.afterHookActive || null, }, patchExitCode: numericField(fields.patchExitCode), annotateExitCode: numericField(fields.annotateExitCode), result: compactCommandResult(result), next: { status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane}`, }, }; } export function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: LegacyRuntimeControlPlaneOptions): Record { if (options.action === "cleanup-runs") return runControlPlaneCleanup(options); if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-released-pvs") { return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`, lane: spec.lane, degradedReason: "unsupported-runtime-lane-action", message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current/refresh/cleanup-runs`, }; } if (options.action === "status" && options.pipelineRun !== undefined) { return runtimeLaneControlPlaneStatus(spec, { pipelineRun: options.pipelineRun, mode: "pipeline-run", includeHistory: options.history }); } if (options.action === "status" && options.sourceCommit !== undefined) { return runtimeLaneControlPlaneStatus(spec, { sourceCommit: options.sourceCommit, mode: "source-commit", includeHistory: options.history }); } const latestHead = options.action === "trigger-current" ? resolveRuntimeLaneLatestRemoteHead(spec) : null; const head = resolveRuntimeLaneHead(spec); let sourceCommit = head.sourceCommit; if (options.action === "trigger-current" && latestHead !== null && latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) { const refreshedHead = resolveRuntimeLaneHead(spec); sourceCommit = refreshedHead.sourceCommit; if (sourceCommit !== latestHead.sourceCommit) { return { ok: false, command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "confirmed-trigger", phase: "source-head-refresh", sourceCommit, latestSourceHeadProbe: { sourceCommit: latestHead.sourceCommit, result: compactCommandResult(latestHead.result), }, refreshedHeadProbe: compactCommandResult(refreshedHead.result), degradedReason: `${spec.lane}-head-refresh-did-not-reach-latest-source`, }; } } if (sourceCommit === null) { return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`, lane: spec.lane, degradedReason: `${spec.lane}-head-unresolved`, sourceRepo: spec.cicdRepo, workspace: spec.workspace, headProbe: compactCommandResult(head.result), }; } if (options.action === "status") { return runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }); } if (options.action === "refresh") { return refreshRuntimeLaneArgoApplication(spec, options); } if (options.action === "apply") { const render = runRuntimeLaneRenderToTemp(spec, sourceCommit); if (!isCommandSuccess(render.result)) { return { ok: false, command: `hwlab g14 control-plane apply --lane ${spec.lane}`, lane: spec.lane, phase: "source-render", sourceCommit, renderDir: render.renderDir, render: compactCommandResult(render.result), }; } const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, options.dryRun, options.timeoutSeconds); const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null; return { ok: isCommandSuccess(apply), command: `hwlab g14 control-plane apply --lane ${spec.lane}`, lane: spec.lane, mode: options.dryRun ? "dry-run" : "confirmed-apply", sourceCommit, renderDir: render.renderDir, render: compactCommandResult(render.result), apply: compactCommandResult(apply), cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir), status: runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }), next: options.dryRun ? { apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm` } : { triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm` }, }; } let pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit); let before = getPipelineRunCompact(pipelineRun); const originalPipelineRun = pipelineRun; if (options.rerun && before.exists === true) { pipelineRun = runtimeLaneRerunPipelineRunName(spec, sourceCommit); before = getPipelineRunCompact(pipelineRun); } if (options.dryRun) { return { ok: true, command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "dry-run", sourceCommit, pipelineRun, rerun: options.rerun, originalPipelineRun: options.rerun ? originalPipelineRun : undefined, before, gitMirrorSync: { mode: "skipped-dry-run", wouldSyncBeforeTrigger: true, command: `bun scripts/cli.ts hwlab g14 git-mirror sync --lane ${spec.lane} --confirm`, }, controlPlaneRefresh: { mode: "skipped-dry-run", wouldRefreshBeforeCreate: true, resources: [`${spec.runtimeRenderDir}/namespace.yaml`, `${spec.tektonDir}/rbac.yaml`, `${spec.tektonDir}/pipeline.yaml`, "argocd/project.yaml", `argocd/${spec.argoApplicationFile}`], }, manifest: runtimeLanePipelineRunManifest(spec, sourceCommit), next: { triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm` }, }; } const beforeStatus = stringOrNull(before.status); if (before.exists === true && beforeStatus === "True") { const currentCd = runtimeLaneStatusWithGitMirror(spec, sourceCommit); if (runtimeLaneCdPassedWithMirror(spec, currentCd)) { return { ok: true, command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "confirmed-trigger", sourceCommit, pipelineRun, before, skipped: true, reason: "existing-pipelinerun-reused", cd: summarizeRuntimeLaneCdStatus(spec, currentCd), latestOnlyPolicy: "same source commit is idempotent only after PipelineRun, Argo/runtime, public probes, and Git mirror are already aligned", }; } pipelineRun = runtimeLaneRerunPipelineRunName(spec, sourceCommit); before = getPipelineRunCompact(pipelineRun); printRuntimeLaneTriggerProgress(spec, { stage: "existing-pipelinerun", status: "rerun-required", sourceCommit, pipelineRun, previousPipelineRun: runtimeLanePipelineRunName(spec, sourceCommit), previousCd: summarizeRuntimeLaneCdStatus(spec, currentCd), }); } const effectiveBeforeStatus = stringOrNull(before.status); if (before.exists === true && (effectiveBeforeStatus === "True" || effectiveBeforeStatus === "Unknown")) { return { ok: true, command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "confirmed-trigger", sourceCommit, pipelineRun, before, skipped: true, reason: "existing-rerun-pipelinerun-reused", latestOnlyPolicy: "same source commit reuses an active or already successful rerun PipelineRun after the deterministic run failed CD alignment", }; } if (before.exists === true && effectiveBeforeStatus !== null) { if (options.rerun === true) { pipelineRun = runtimeLaneRerunPipelineRunName(spec, sourceCommit); before = getPipelineRunCompact(pipelineRun); printRuntimeLaneTriggerProgress(spec, { stage: "existing-pipelinerun", status: "explicit-rerun-required", sourceCommit, pipelineRun, previousPipelineRun: originalPipelineRun, previousStatus: effectiveBeforeStatus, }); } else { return { ok: false, command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "confirmed-trigger", sourceCommit, pipelineRun, before, skipped: true, reason: "existing-pipelinerun-terminal-failed", degradedReason: "existing-pipelinerun-terminal-failed", next: { rerun: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm --rerun`, status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --pipeline-run ${pipelineRun}`, }, }; } } printRuntimeLaneTriggerProgress(spec, { stage: "git-mirror-sync", status: "started", sourceCommit, pipelineRun, }); const gitMirrorPreSyncStatus = runGitMirrorStatus(spec.lane); const gitMirrorSourceAlreadyCurrent = runtimeLaneGitMirrorSourceInSync(spec, sourceCommit, gitMirrorPreSyncStatus); const gitMirrorSync = gitMirrorSourceAlreadyCurrent ? { ok: true, command: "hwlab nodes git-mirror sync", mode: "skipped-source-already-current", skipped: true, reason: "local-mirror-source-current", status: gitMirrorPreSyncStatus, } : runGitMirrorSync({ action: "sync", lane: spec.lane, confirm: true, dryRun: false, wait: true, timeoutSeconds: options.timeoutSeconds, }); printRuntimeLaneTriggerProgress(spec, { stage: "git-mirror-sync", status: gitMirrorSync.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun, jobName: gitMirrorSync.jobName ?? null, elapsedMs: gitMirrorSync.elapsedMs ?? null, }); if (gitMirrorSync.ok !== true) { return { ok: false, command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "confirmed-trigger", sourceCommit, pipelineRun, before, gitMirrorSync: compactGitMirrorSync(gitMirrorSync), degradedReason: "git-mirror-sync-before-trigger-failed", }; } const gitMirrorPostSyncStatus = gitMirrorSourceAlreadyCurrent ? gitMirrorPreSyncStatus : record(gitMirrorSync.status); if (!runtimeLaneGitMirrorSourceInSync(spec, sourceCommit, gitMirrorPostSyncStatus)) { return { ok: false, command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "confirmed-trigger", sourceCommit, pipelineRun, before, gitMirrorSync: compactGitMirrorSync(gitMirrorSync), gitMirror: { ok: gitMirrorPostSyncStatus.ok === true, summary: record(nested(gitMirrorPostSyncStatus, ["cache", "summary"])), }, degradedReason: `${spec.lane}-git-mirror-source-still-stale-after-sync`, next: { apply: `bun scripts/cli.ts hwlab nodes git-mirror apply --node ${spec.nodeId} --lane ${spec.lane} --confirm`, sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`, }, }; } printRuntimeLaneTriggerProgress(spec, { stage: "source-render", status: "started", sourceCommit, pipelineRun, }); const render = runRuntimeLaneRenderToTemp(spec, sourceCommit); printRuntimeLaneTriggerProgress(spec, { stage: "source-render", status: isCommandSuccess(render.result) ? "succeeded" : "failed", sourceCommit, pipelineRun, renderDir: render.renderDir, exitCode: render.result.exitCode, }); if (!isCommandSuccess(render.result)) { return { ok: false, command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "confirmed-trigger", phase: "source-render", sourceCommit, pipelineRun, before, gitMirrorSync: compactGitMirrorSync(gitMirrorSync), renderDir: render.renderDir, render: compactCommandResult(render.result), degradedReason: "control-plane-render-failed", }; } printRuntimeLaneTriggerProgress(spec, { stage: "control-plane-apply", status: "started", sourceCommit, pipelineRun, renderDir: render.renderDir, }); const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, false, options.timeoutSeconds); const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null; printRuntimeLaneTriggerProgress(spec, { stage: "control-plane-apply", status: isCommandSuccess(apply) ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: apply.exitCode, }); if (!isCommandSuccess(apply)) { return { ok: false, command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "confirmed-trigger", phase: "control-plane-apply", sourceCommit, pipelineRun, before, gitMirrorSync: compactGitMirrorSync(gitMirrorSync), renderDir: render.renderDir, render: compactCommandResult(render.result), apply: compactCommandResult(apply), cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir), degradedReason: "control-plane-apply-before-trigger-failed", }; } printRuntimeLaneTriggerProgress(spec, { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun, }); const create = createRuntimeLanePipelineRun(spec, sourceCommit, options.timeoutSeconds, pipelineRun); printRuntimeLaneTriggerProgress(spec, { stage: "create-pipelinerun", status: isCommandSuccess(create) ? "succeeded" : "failed", sourceCommit, pipelineRun, exitCode: create.exitCode, }); return { ok: isCommandSuccess(create), command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`, lane: spec.lane, mode: "confirmed-trigger", sourceCommit, pipelineRun, before, gitMirrorSync: compactGitMirrorSync(gitMirrorSync), renderDir: render.renderDir, render: compactCommandResult(render.result), apply: compactCommandResult(apply), cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir), create: compactCommandResult(create), status: runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }), degradedReason: isCommandSuccess(create) ? undefined : "pipelinerun-create-failed", next: { status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --source-commit ${sourceCommit}` }, }; } export function runDefaultRuntimeLaneControlPlane(options: LegacyRuntimeControlPlaneOptions): Record { if (options.action === "cleanup-runs") return runControlPlaneCleanup(options); if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options); if ((options.action === "status" || options.action === "closeout") && options.pipelineRun !== undefined) { const status = defaultRuntimeLaneControlPlaneStatus({ pipelineRun: options.pipelineRun, mode: "pipeline-run", includeHistory: options.history }); if (options.action === "closeout") return defaultRuntimeLaneControlPlaneCloseout(status); return status; } if ((options.action === "status" || options.action === "closeout") && options.sourceCommit !== undefined) { const status = defaultRuntimeLaneControlPlaneStatus({ sourceCommit: options.sourceCommit, mode: "source-commit", includeHistory: options.history }); if (options.action === "closeout") return defaultRuntimeLaneControlPlaneCloseout(status); return status; } if (options.action === "closeout") { const status = defaultRuntimeLaneControlPlaneStatus({ includeHistory: options.history }); return defaultRuntimeLaneControlPlaneCloseout(status); } const snapshot = options.action === "trigger-current" ? getDefaultRuntimeLaneTriggerSnapshot() : null; const latestHead = options.action === "trigger-current" ? resolveDefaultRuntimeLaneLatestRemoteHead() : null; const head = snapshot === null ? resolveDefaultRuntimeLaneHead() : snapshot.sourceCommit === null ? resolveDefaultRuntimeLaneHead() : { sourceCommit: snapshot.sourceCommit, result: snapshot.result }; let sourceCommit = head.sourceCommit; if (options.action === "trigger-current" && latestHead !== null && latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) { const refreshedHead = resolveDefaultRuntimeLaneHead(); sourceCommit = refreshedHead.sourceCommit; if (sourceCommit !== latestHead.sourceCommit) { return { ok: false, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", phase: "source-head-refresh", sourceCommit, latestSourceHeadProbe: { sourceCommit: latestHead.sourceCommit, result: compactCommandResult(latestHead.result), }, refreshedHeadProbe: compactCommandResult(refreshedHead.result), degradedReason: "v02-head-refresh-did-not-reach-latest-source", }; } } if (sourceCommit === null) { return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", sourceRepo: DEFAULT_RUNTIME_LANE_CICD_REPO, workspace: DEFAULT_RUNTIME_LANE_WORKSPACE, headProbe: compactCommandResult(head.result) }; } if (options.action === "runtime-migration") return runDefaultRuntimeLaneRuntimeMigration(options, sourceCommit); if (options.action === "status") return defaultRuntimeLaneControlPlaneStatus({ sourceCommit, mode: "latest-source-head", includeHistory: options.history }); if (options.action === "refresh") return refreshRuntimeLaneArgoApplication(DEFAULT_RUNTIME_LANE_LANE_SPEC, options); if (options.action === "apply") { const render = runDefaultRuntimeLaneRenderToTemp(sourceCommit); if (!isCommandSuccess(render.result)) { return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, phase: "source-render", sourceCommit, renderDir: render.renderDir, render: compactCommandResult(render.result), }; } const apply = applyDefaultRuntimeLaneControlPlaneFiles(render.renderDir, options.dryRun, options.timeoutSeconds); const cleanupRenderDir = isCommandSuccess(apply) ? cleanupDefaultRuntimeLaneRenderDir(render.renderDir) : null; return { ok: isCommandSuccess(apply), command: "hwlab g14 control-plane apply --lane v02", lane: "v02", mode: options.dryRun ? "dry-run" : "confirmed-apply", sourceCommit, renderDir: render.renderDir, render: compactCommandResult(render.result), apply: compactCommandResult(apply), cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir), cleanupObsoleteCronJobs: compactCommandResult(options.dryRun ? deleteDefaultRuntimeLaneObsoleteCronJobs(true) : deleteDefaultRuntimeLaneObsoleteCronJobs(false)), status: defaultRuntimeLaneControlPlaneStatus({ sourceCommit, mode: "latest-source-head" }), next: options.dryRun ? { apply: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm" } : { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" }, }; } let before = snapshot !== null && snapshot.sourceCommit === sourceCommit && snapshot.before !== null ? snapshot.before : getPipelineRunCompact(defaultRuntimeLanePipelineRunName(sourceCommit)); if (options.dryRun) { const gitMirrorPreSync = preSyncDefaultRuntimeLaneGitMirror(sourceCommit, options); return { ok: true, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "dry-run", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), before, gitMirrorPreSync, controlPlaneRefresh: { mode: "skipped-dry-run", wouldRefreshBeforeCreate: true, resources: ["tekton-v02/rbac.yaml", "tekton-v02/pipeline.yaml", "argocd/project.yaml", "argocd/application-v02.yaml"], }, manifest: defaultRuntimeLanePipelineRunManifest(sourceCommit), next: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" }, }; } if (before.exists === true && before.status !== null) { const reuseHead = latestHead ?? resolveDefaultRuntimeLaneLatestRemoteHead(); const reuseDecision = defaultRuntimeLaneExistingPipelineRunReuseDecision({ sourceCommit, before, latestSourceCommit: reuseHead.sourceCommit }); if (reuseDecision.reusable === true) { const alreadyUsable = reuseDecision.alreadyUsable === true; return { ok: alreadyUsable, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), before, skipped: true, reuseDecision, reason: reuseDecision.reason, degradedReason: alreadyUsable ? undefined : "existing-pipelinerun-terminal-failed", latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default; source head is rechecked before reuse", }; } if (reuseHead.sourceCommit !== null && reuseHead.sourceCommit !== sourceCommit) { printProgressEvent("hwlab.v02.trigger.progress", { stage: "source-head-recheck", status: "advanced", sourceCommit, latestSourceCommit: reuseHead.sourceCommit, previousPipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), latestPipelineRun: defaultRuntimeLanePipelineRunName(reuseHead.sourceCommit), }); sourceCommit = reuseHead.sourceCommit; before = getPipelineRunCompact(defaultRuntimeLanePipelineRunName(sourceCommit)); if (before.exists === true && before.status !== null) { const advancedAlreadyUsable = before.status === "True" || before.status === "Unknown"; return { ok: advancedAlreadyUsable, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), before, skipped: true, reuseDecision: { ...reuseDecision, advancedSourceCommit: sourceCommit, advancedPipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), advancedPipelineRunStatus: before.status ?? null, }, reason: advancedAlreadyUsable ? "advanced-existing-pipelinerun-reused" : "advanced-existing-pipelinerun-terminal-failed", degradedReason: advancedAlreadyUsable ? undefined : "advanced-existing-pipelinerun-terminal-failed", latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default; source head is rechecked before reuse", }; } } else { return { ok: false, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), before, reuseDecision, degradedReason: "existing-pipelinerun-reuse-rejected", latestSourceHeadProbe: { sourceCommit: reuseHead.sourceCommit, exitCode: reuseHead.result.exitCode, stderr: reuseHead.result.stderr.trim().slice(0, 2000), }, }; } } if (latestHead !== null && latestHead.sourceCommit !== sourceCommit) { return { ok: false, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", phase: "source-head-recheck", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), before, latestSourceHeadProbe: { sourceCommit: latestHead.sourceCommit, exitCode: latestHead.result.exitCode, stderr: latestHead.result.stderr.trim().slice(0, 2000), }, degradedReason: latestHead.sourceCommit === null ? "source-head-recheck-unresolved-before-pipelinerun-create" : "source-head-advanced-before-pipelinerun-create", }; } printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit) }); const controlPlaneRefresh = refreshDefaultRuntimeLaneControlPlaneBeforeTrigger(sourceCommit, options.timeoutSeconds); printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: controlPlaneRefresh.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), mode: record(controlPlaneRefresh).mode ?? null, waitedMs: record(controlPlaneRefresh).waitedMs ?? null, renderDir: record(controlPlaneRefresh).renderDir ?? null, degradedReason: record(controlPlaneRefresh).degradedReason ?? null, }); if (controlPlaneRefresh.ok !== true) { return { ok: false, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", phase: "control-plane-refresh", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), before, controlPlaneRefresh, degradedReason: record(controlPlaneRefresh).degradedReason ?? "control-plane-refresh-failed", }; } printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync", status: "started", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), reason: "ensure local git mirror has source commit before creating PipelineRun", }); const gitMirrorPreSync = preSyncDefaultRuntimeLaneGitMirror(sourceCommit, options); printProgressEvent("hwlab.v02.trigger.progress", { stage: "git-mirror-pre-sync", status: gitMirrorPreSync.ok === true ? "succeeded" : "failed", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), mode: record(gitMirrorPreSync).mode ?? null, required: nested(gitMirrorPreSync, ["before", "required"]) ?? null, localDefaultRuntimeLane: nested(gitMirrorPreSync, ["before", "localDefaultRuntimeLane"]) ?? null, finalLocalDefaultRuntimeLane: nested(gitMirrorPreSync, ["final", "localDefaultRuntimeLane"]) ?? nested(gitMirrorPreSync, ["marker", "localDefaultRuntimeLane"]) ?? nested(gitMirrorPreSync, ["after", "localDefaultRuntimeLane"]) ?? null, pendingFlush: nested(gitMirrorPreSync, ["before", "pendingFlush"]) ?? null, reason: nested(gitMirrorPreSync, ["before", "reason"]) ?? null, syncElapsedMs: nested(gitMirrorPreSync, ["sync", "elapsedMs"]) ?? null, waitElapsedMs: nested(gitMirrorPreSync, ["wait", "elapsedMs"]) ?? null, waitAttempts: nested(gitMirrorPreSync, ["wait", "attempts"]) ?? null, lockWaitedMs: record(gitMirrorPreSync).waitedMs ?? null, degradedReason: record(gitMirrorPreSync).degradedReason ?? null, }); if (gitMirrorPreSync.ok !== true) { return { ok: false, command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", phase: "git-mirror-pre-sync", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), before, controlPlaneRefresh, gitMirrorPreSync, degradedReason: "git-mirror-pre-sync-failed", }; } printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit) }); const createPipelineRun = createDefaultRuntimeLanePipelineRun(sourceCommit, options.timeoutSeconds); printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: isCommandSuccess(createPipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), exitCode: createPipelineRun.exitCode }); return { ok: isCommandSuccess(createPipelineRun), command: "hwlab g14 control-plane trigger-current --lane v02", lane: "v02", mode: "confirmed-trigger", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), before, controlPlaneRefresh: compactControlPlaneRefresh(controlPlaneRefresh), gitMirrorPreSync, createPipelineRun: compactTriggerCommandResult(compactCommandResult(createPipelineRun)), after: getPipelineRunCompact(defaultRuntimeLanePipelineRunName(sourceCommit)), latestOnlyPolicy: "no old PipelineRun is canceled; stale commits self-supersede before GitOps writeback", disclosure: { fullTriggerOutput: "Use the async job stdout/stderr files from job status for full command details.", }, }; }