Codex
1341 lines
66 KiB
TypeScript
1341 lines
66 KiB
TypeScript
// SPEC: PJ2026-01060307 控制面模块化 draft-2026-06-25-p0. control-plane module for scripts/src/hwlab-legacy-runtime.ts.
|
|
|
|
// Moved mechanically from scripts/src/hwlab-legacy-runtime.ts:3737-5054 for #903.
|
|
|
|
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { createHash, randomBytes } from "node:crypto";
|
|
import { repoRoot, rootPath, type Config } from "../config";
|
|
import { runCommand } from "../command";
|
|
import { cancelJob, listJobs, readJob, startJob } from "../jobs";
|
|
import { hwlabRequiredNoProxyEntries, hwlabRuntimeLaneConfigPath, hwlabRuntimeLaneIds, hwlabRuntimeLaneSpec, isHwlabRuntimeLane, type HwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "../hwlab-node-lanes";
|
|
|
|
import type { CommandJsonResult, LegacyRuntimeControlPlaneOptions, DefaultRuntimeLaneControlPlaneStatusTarget, DefaultRuntimeLaneStatusTargetMode } from "./types";
|
|
import { runtimeLaneCdPassedWithMirror, runtimeLaneStatusWithGitMirror } from "./cd-trigger";
|
|
import { runControlPlaneCleanup, runControlPlaneReleasedPvCleanup, runDefaultRuntimeLaneRuntimeMigration } from "./cleanup";
|
|
import { compactGitMirrorSync, gitMirrorStatusSummary, preSyncDefaultRuntimeLaneGitMirror, runGitMirrorStatus, runGitMirrorSync, runtimeLaneGitMirrorSourceInSync } from "./git-mirror";
|
|
import { statusText, summarizeRuntimeLaneCdStatus, defaultRuntimeLaneControlPlaneCloseout } from "./pr-monitor";
|
|
import { compactCommandResult, compactControlPlaneRefresh, compactTriggerCommandResult, legacyRuntimeK3s, isCommandSuccess, nested, printProgressEvent, printRuntimeLaneTriggerProgress, record, shellQuote, stringOrNull } from "./remote";
|
|
import { applyRuntimeLaneControlPlaneFiles, applyDefaultRuntimeLaneControlPlaneFiles, cleanupRuntimeLaneRenderDir, cleanupDefaultRuntimeLaneRenderDir, deleteDefaultRuntimeLaneObsoleteCronJobs, refreshDefaultRuntimeLaneControlPlaneBeforeTrigger, runRuntimeLaneRenderToTemp, runDefaultRuntimeLaneRenderToTemp } from "./render";
|
|
import { resolveRuntimeLaneHead, resolveRuntimeLaneLatestRemoteHead, resolveDefaultRuntimeLaneHead, resolveDefaultRuntimeLaneLatestRemoteHead, runtimeLanePipelineRunName, runtimeLaneRerunPipelineRunName, defaultRuntimeLanePipelineRunName } from "./source";
|
|
import { ARGO_NAMESPACE, CI_NAMESPACE, DEFAULT_RUNTIME_LANE_APP, DEFAULT_RUNTIME_LANE_CICD_REPO, DEFAULT_RUNTIME_LANE_LANE_SPEC, DEFAULT_RUNTIME_LANE_PIPELINE, DEFAULT_RUNTIME_LANE_SOURCE_BRANCH, DEFAULT_RUNTIME_LANE_WORKSPACE } from "./types";
|
|
import { getPipelineRunCompact, getDefaultRuntimeLaneTriggerSnapshot, keyValueLinesFromText, listDefaultRuntimeLanePipelineRunsCompactFromText, numericField, parsePipelineRunRows, parseShellSections, pipelineRunCompactFromText, pipelineRunRowsJsonPath, runtimeLaneTaskRunDiagnosticsFromText, runtimeLaneTaskRunDiagnosticsScript, shellSectionOk, taskRunsCompactFromText, timestampMs, defaultRuntimeLaneCloseoutVerdict, defaultRuntimeLaneCommitAlignment, defaultRuntimeLaneControlPlaneStatusBundle, defaultRuntimeLaneExistingPipelineRunReuseDecision, defaultRuntimeLaneFalseGreenGuard, defaultRuntimeLaneLatestOnlyTargetValidation, defaultRuntimeLanePlanArtifactsFromText, defaultRuntimeLaneRuntimeWorkloadsFromText, defaultRuntimeLaneSourceHeadsFromText, defaultRuntimeLaneTargetValidation, defaultRuntimeLaneWebAssetsFromText } from "./runtime-status";
|
|
|
|
export function runtimeLanePipelineRunManifest(spec: HwlabRuntimeLaneSpec, sourceCommit: string, pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit)): Record<string, unknown> {
|
|
return {
|
|
apiVersion: "tekton.dev/v1",
|
|
kind: "PipelineRun",
|
|
metadata: {
|
|
name: pipelineRun,
|
|
namespace: CI_NAMESPACE,
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
"hwlab.pikastech.local/gitops-target": spec.lane,
|
|
"hwlab.pikastech.local/source-commit": sourceCommit,
|
|
"hwlab.pikastech.local/trigger": "manual-cli",
|
|
},
|
|
annotations: {
|
|
"hwlab.pikastech.local/source-branch": spec.sourceBranch,
|
|
"hwlab.pikastech.local/gitops-branch": spec.gitopsBranch,
|
|
"hwlab.pikastech.local/node": spec.nodeId,
|
|
"hwlab.pikastech.local/network-profile": spec.networkProfileId,
|
|
"hwlab.pikastech.local/download-profile": spec.downloadProfileId,
|
|
"hwlab.pikastech.local/no-proxy-required": hwlabRequiredNoProxyEntries().join(","),
|
|
"hwlab.pikastech.local/triggered-by": "unidesk-cli",
|
|
},
|
|
},
|
|
spec: {
|
|
pipelineRef: { name: spec.pipeline },
|
|
taskRunTemplate: {
|
|
serviceAccountName: spec.serviceAccountName,
|
|
podTemplate: {
|
|
hostNetwork: true,
|
|
dnsPolicy: "ClusterFirstWithHostNet",
|
|
securityContext: { fsGroup: 1000 },
|
|
},
|
|
},
|
|
params: [
|
|
{ name: "git-url", value: spec.gitUrl },
|
|
{ name: "git-read-url", value: spec.gitReadUrl },
|
|
{ name: "source-branch", value: spec.sourceBranch },
|
|
{ name: "gitops-branch", value: spec.gitopsBranch },
|
|
{ name: "lane", value: spec.lane },
|
|
{ name: "catalog-path", value: spec.catalogPath },
|
|
{ name: "image-tag-mode", value: "full" },
|
|
{ name: "runtime-path", value: spec.runtimePath },
|
|
{ name: "revision", value: sourceCommit },
|
|
{ name: "registry-prefix", value: spec.registryPrefix },
|
|
{ name: "services", value: spec.serviceIds.join(",") },
|
|
{ name: "base-image", value: spec.baseImage },
|
|
],
|
|
workspaces: [
|
|
{ name: "source", volumeClaimTemplate: { spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "8Gi" } } } } },
|
|
{ name: "git-ssh", secret: { secretName: "hwlab-git-ssh" } },
|
|
],
|
|
},
|
|
};
|
|
}
|
|
|
|
export function defaultRuntimeLanePipelineRunManifest(sourceCommit: string): Record<string, unknown> {
|
|
return runtimeLanePipelineRunManifest(DEFAULT_RUNTIME_LANE_LANE_SPEC, sourceCommit);
|
|
}
|
|
|
|
export function createRuntimeLanePipelineRun(spec: HwlabRuntimeLaneSpec, sourceCommit: string, timeoutSeconds: number, pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit)): CommandJsonResult {
|
|
const manifest = JSON.stringify(runtimeLanePipelineRunManifest(spec, sourceCommit, pipelineRun));
|
|
const manifestB64 = Buffer.from(manifest, "utf8").toString("base64");
|
|
const script = [
|
|
"set -eu",
|
|
`manifest_b64=${shellQuote(manifestB64)}`,
|
|
`manifest_path=/tmp/${pipelineRun}.json`,
|
|
"printf '%s' \"$manifest_b64\" | base64 -d > \"$manifest_path\"",
|
|
"if kubectl create -f \"$manifest_path\"; then",
|
|
" :",
|
|
"else",
|
|
" code=$?",
|
|
` if kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} >/dev/null 2>&1; then`,
|
|
` printf 'PipelineRun %s already exists; reusing existing object\\n' ${shellQuote(pipelineRun)} >&2`,
|
|
" else",
|
|
" exit \"$code\"",
|
|
" fi",
|
|
"fi",
|
|
`kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(pipelineRun)} -o jsonpath='{.metadata.name}{\"\\n\"}{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}{\"\\n\"}{.status.conditions[0].status}{\"\\n\"}{.status.conditions[0].reason}{\"\\n\"}'`,
|
|
].join("\n");
|
|
return legacyRuntimeK3s(["sh", "--", script], timeoutSeconds * 1000);
|
|
}
|
|
|
|
export function createDefaultRuntimeLanePipelineRun(sourceCommit: string, timeoutSeconds: number): CommandJsonResult {
|
|
return createRuntimeLanePipelineRun(DEFAULT_RUNTIME_LANE_LANE_SPEC, sourceCommit, timeoutSeconds);
|
|
}
|
|
|
|
export function defaultRuntimeLaneControlPlaneStatus(target: DefaultRuntimeLaneControlPlaneStatusTarget = {}): Record<string, unknown> {
|
|
const targetMode: DefaultRuntimeLaneStatusTargetMode = target.mode
|
|
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
|
|
const includeHistory = target.includeHistory === true;
|
|
const strictHeadAlignment = targetMode === "latest-source-head";
|
|
const bundle = defaultRuntimeLaneControlPlaneStatusBundle({ ...target, mode: targetMode });
|
|
const sections = parseShellSections(statusText(bundle));
|
|
const sourceCommit = stringOrNull(sections.sourceCommit?.stdout) ?? null;
|
|
const pipelineRun = stringOrNull(sections.pipelineRunName?.stdout) ?? (sourceCommit === null ? null : defaultRuntimeLanePipelineRunName(sourceCommit));
|
|
const statusTargetFields = keyValueLinesFromText(sections.statusTarget?.stdout ?? "");
|
|
const queryNowMs = timestampMs(sections.queryNow?.stdout) ?? Date.now();
|
|
const sourceHeadsSection = sections.sourceHeads;
|
|
const controlPlane = sections.controlPlane;
|
|
const obsoleteCronJobs = sections.obsoleteCronJobs;
|
|
const argo = sections.argo;
|
|
const pipelineRunSection = sections.pipelineRun;
|
|
const taskRunsSection = sections.taskRuns;
|
|
const planArtifactsSection = sections.planArtifacts;
|
|
const runtimeWorkloadsSection = sections.runtimeWorkloads;
|
|
const gitMirrorCacheSection = sections.gitMirrorCache;
|
|
const webAssetsSection = sections.webAssets;
|
|
const recentPipelineRuns = listDefaultRuntimeLanePipelineRunsCompactFromText(
|
|
sections.recentPipelineRuns?.stdout ?? "",
|
|
shellSectionOk(sections.recentPipelineRuns),
|
|
"kubectl get pipelinerun -n hwlab-ci -l hwlab.pikastech.local/gitops-target=v02 -o jsonpath=<summary-fields>",
|
|
sections.recentPipelineRuns?.exitCode ?? null,
|
|
bundle.stderr,
|
|
8,
|
|
queryNowMs,
|
|
);
|
|
const activePipelineRuns = Array.isArray(recentPipelineRuns.activeItems) ? recentPipelineRuns.activeItems : [];
|
|
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(argo?.stdout ?? "").split(/\r?\n/u);
|
|
const sourceHeads = defaultRuntimeLaneSourceHeadsFromText(
|
|
sourceHeadsSection?.stdout ?? "",
|
|
shellSectionOk(sourceHeadsSection),
|
|
sourceHeadsSection?.exitCode ?? null,
|
|
bundle.stderr,
|
|
);
|
|
const pipelineRunInfo = pipelineRun === null
|
|
? null
|
|
: pipelineRunCompactFromText(
|
|
pipelineRun,
|
|
pipelineRunSection?.stdout ?? "",
|
|
shellSectionOk(pipelineRunSection),
|
|
`kubectl get pipelinerun -n hwlab-ci ${pipelineRun}`,
|
|
pipelineRunSection?.exitCode ?? null,
|
|
bundle.stderr,
|
|
);
|
|
const taskRuns = taskRunsCompactFromText(
|
|
taskRunsSection?.stdout ?? "",
|
|
shellSectionOk(taskRunsSection),
|
|
pipelineRun,
|
|
taskRunsSection?.exitCode ?? null,
|
|
bundle.stderr,
|
|
);
|
|
const planArtifacts = defaultRuntimeLanePlanArtifactsFromText(
|
|
planArtifactsSection?.stdout ?? "",
|
|
shellSectionOk(planArtifactsSection),
|
|
pipelineRun,
|
|
planArtifactsSection?.exitCode ?? null,
|
|
bundle.stderr,
|
|
);
|
|
const runtimeWorkloads = defaultRuntimeLaneRuntimeWorkloadsFromText(
|
|
runtimeWorkloadsSection?.stdout ?? "",
|
|
shellSectionOk(runtimeWorkloadsSection),
|
|
runtimeWorkloadsSection?.exitCode ?? null,
|
|
bundle.stderr,
|
|
);
|
|
const webAssets = defaultRuntimeLaneWebAssetsFromText(
|
|
webAssetsSection?.stdout ?? "",
|
|
shellSectionOk(webAssetsSection),
|
|
sourceCommit,
|
|
syncRevision,
|
|
webAssetsSection?.exitCode ?? null,
|
|
bundle.stderr,
|
|
activePipelineRuns,
|
|
);
|
|
const gitMirror = {
|
|
ok: shellSectionOk(gitMirrorCacheSection),
|
|
summary: gitMirrorStatusSummary(String(gitMirrorCacheSection?.stdout ?? "").trim()),
|
|
raw: String(gitMirrorCacheSection?.stdout ?? "").trim(),
|
|
exitCode: gitMirrorCacheSection?.exitCode ?? null,
|
|
stderr: shellSectionOk(gitMirrorCacheSection) ? "" : bundle.stderr.trim().slice(0, 2000),
|
|
};
|
|
const commitAlignment = defaultRuntimeLaneCommitAlignment({
|
|
expectedSourceHead: sourceCommit,
|
|
sourceHeads,
|
|
gitMirrorSummary: record(gitMirror.summary),
|
|
pipelineRun: pipelineRunInfo,
|
|
recentPipelineRuns,
|
|
planArtifacts,
|
|
runtimeWorkloads,
|
|
webAssets,
|
|
});
|
|
const targetValidationBase = defaultRuntimeLaneTargetValidation({
|
|
targetMode,
|
|
sourceCommit,
|
|
pipelineRun: pipelineRunInfo,
|
|
argo: {
|
|
ok: shellSectionOk(argo),
|
|
fields: { targetRevision, path, syncRevision, syncStatus, health },
|
|
},
|
|
planArtifacts,
|
|
runtimeWorkloads,
|
|
webAssets,
|
|
gitMirror,
|
|
recentPipelineRuns,
|
|
});
|
|
const targetValidation = defaultRuntimeLaneLatestOnlyTargetValidation({
|
|
targetMode,
|
|
sourceCommit,
|
|
pipelineRun: pipelineRunInfo,
|
|
commitAlignment,
|
|
targetValidation: targetValidationBase,
|
|
});
|
|
const sourceHeadsTarget = record(sourceHeads.target);
|
|
const falseGreenGuard = targetValidation.state === "superseded"
|
|
? {
|
|
ok: null,
|
|
state: "superseded",
|
|
summary: "target PipelineRun succeeded but runtime now reflects a newer v0.2 PipelineRun; current-runtime false-green guard is not applicable to this historical target",
|
|
sourceCommit,
|
|
targetValidationState: targetValidation.state,
|
|
newerSucceededRuns: targetValidation.newerSucceededRuns ?? [],
|
|
supersededRuntimeServices: targetValidation.supersededRuntimeServices ?? [],
|
|
}
|
|
: defaultRuntimeLaneFalseGreenGuard({ sourceCommit, pipelineRun: pipelineRunInfo, taskRuns, planArtifacts, runtimeWorkloads });
|
|
const baseOk = sourceCommit !== null && isCommandSuccess(bundle) && shellSectionOk(controlPlane) && shellSectionOk(argo);
|
|
const targetPipelineRunOk = strictHeadAlignment
|
|
? true
|
|
: pipelineRunInfo !== null && pipelineRunInfo.ok === true && pipelineRunInfo.exists !== false;
|
|
const targetDegradedReason = !strictHeadAlignment && !targetPipelineRunOk ? "target-pipelinerun-not-found-or-unreadable" : undefined;
|
|
const statusCommand = targetMode === "pipeline-run" && pipelineRun !== null
|
|
? `hwlab g14 control-plane status --lane v02 --pipeline-run ${pipelineRun}`
|
|
: targetMode === "source-commit" && sourceCommit !== null
|
|
? `hwlab g14 control-plane status --lane v02 --source-commit ${sourceCommit}`
|
|
: "hwlab g14 control-plane status --lane v02";
|
|
const state = strictHeadAlignment ? commitAlignment.state : `target-${targetMode}`;
|
|
const sourceCommitSource = targetMode === "pipeline-run"
|
|
? "PipelineRun metadata label hwlab.pikastech.local/source-commit"
|
|
: targetMode === "source-commit"
|
|
? "explicit --source-commit"
|
|
: "G14 CI/CD dedicated bare repo refs/remotes/origin/v0.2; workspace checkout is observable but isolated";
|
|
const status: Record<string, unknown> = {
|
|
ok: baseOk && targetPipelineRunOk && (!strictHeadAlignment || commitAlignment.aligned !== false),
|
|
command: statusCommand,
|
|
lane: "v02",
|
|
state,
|
|
degradedReason: strictHeadAlignment && commitAlignment.aligned === false ? commitAlignment.state : targetDegradedReason,
|
|
statusTarget: {
|
|
mode: statusTargetFields.mode || targetMode,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
strictHeadAlignment,
|
|
ancestorOfLatest: sourceHeadsTarget.ancestorOfOriginHead ?? null,
|
|
note: strictHeadAlignment
|
|
? "default status validates the latest v0.2 source head alignment"
|
|
: "targeted status inspects the requested PipelineRun/source commit without failing merely because origin/v0.2 advanced later",
|
|
},
|
|
sourceCommit,
|
|
sourceCommitSource,
|
|
expected: {
|
|
sourceRepo: DEFAULT_RUNTIME_LANE_CICD_REPO,
|
|
workspace: DEFAULT_RUNTIME_LANE_WORKSPACE,
|
|
workspaceIsolation: "workspace dirty/stale state must not select CI/CD source commits",
|
|
branch: DEFAULT_RUNTIME_LANE_SOURCE_BRANCH,
|
|
namespace: CI_NAMESPACE,
|
|
runtimeNamespace: "hwlab-v02",
|
|
pipeline: DEFAULT_RUNTIME_LANE_PIPELINE,
|
|
trigger: "manual UniDesk CLI PipelineRun creation",
|
|
argoApplication: DEFAULT_RUNTIME_LANE_APP,
|
|
},
|
|
commitAlignment,
|
|
targetValidation,
|
|
sourceHeads,
|
|
sourceHeadsTargetAncestorOfOriginHead: sourceHeadsTarget.ancestorOfOriginHead ?? null,
|
|
gitMirror,
|
|
controlPlane: {
|
|
ok: shellSectionOk(controlPlane),
|
|
names: String(controlPlane?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
|
exitCode: controlPlane?.exitCode ?? null,
|
|
},
|
|
obsoleteCronJobs: {
|
|
ok: shellSectionOk(obsoleteCronJobs),
|
|
names: String(obsoleteCronJobs?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean),
|
|
exitCode: obsoleteCronJobs?.exitCode ?? null,
|
|
cleanupCommand: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm",
|
|
},
|
|
argo: {
|
|
ok: shellSectionOk(argo),
|
|
raw: argo?.stdout ?? "",
|
|
fields: { targetRevision, path, syncRevision, syncStatus, health },
|
|
exitCode: argo?.exitCode ?? null,
|
|
},
|
|
pipelineRun: pipelineRunInfo,
|
|
taskRuns,
|
|
planArtifacts,
|
|
runtimeWorkloads,
|
|
falseGreenGuard,
|
|
webAssets,
|
|
activePipelineRuns,
|
|
history: {
|
|
included: includeHistory,
|
|
activePipelineRunCount: activePipelineRuns.length,
|
|
recentPipelineRunCount: Array.isArray(recentPipelineRuns.items) ? recentPipelineRuns.items.length : 0,
|
|
note: includeHistory
|
|
? "recentPipelineRuns is included because --history was requested; historical runs are diagnostic evidence, not the default current-target verdict"
|
|
: "recent PipelineRun history is hidden by default so old manifest/service-id failures do not pollute the current-target verdict; rerun with --history to inspect it",
|
|
command: statusCommand.includes("--history") ? statusCommand : `${statusCommand} --history`,
|
|
},
|
|
...(includeHistory ? { recentPipelineRuns } : {}),
|
|
query: {
|
|
ok: isCommandSuccess(bundle),
|
|
exitCode: bundle.exitCode,
|
|
stderr: bundle.stderr.trim().slice(0, 2000),
|
|
},
|
|
visibilityHint: activePipelineRuns.length > 0
|
|
? "activePipelineRuns shows running v02 CI even when origin/v0.2 advanced after a previous trigger"
|
|
: "no active v02 PipelineRun observed",
|
|
};
|
|
return {
|
|
...status,
|
|
closeout: defaultRuntimeLaneCloseoutVerdict(status),
|
|
};
|
|
}
|
|
|
|
export function runtimeLaneControlPlaneStatusBundle(spec: HwlabRuntimeLaneSpec, target: DefaultRuntimeLaneControlPlaneStatusTarget = {}): CommandJsonResult {
|
|
const targetMode: DefaultRuntimeLaneStatusTargetMode = target.mode
|
|
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
|
|
const targetInit = target.pipelineRun !== undefined && target.pipelineRun !== null
|
|
? [
|
|
`pipeline_run=${shellQuote(target.pipelineRun)}`,
|
|
`source_commit=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.metadata.labels.hwlab\\.pikastech\\.local/source-commit}' 2>/dev/null || true)`,
|
|
`if [ -z "$source_commit" ]; then source_commit=$(printf '%s' "$pipeline_run" | sed 's/^${spec.pipelineRunPrefix}-//'); fi`,
|
|
].join("\n")
|
|
: [
|
|
target.sourceCommit === undefined
|
|
? `source_commit=$(git --git-dir=${shellQuote(spec.cicdRepo)} rev-parse refs/remotes/origin/${spec.sourceBranch} 2>/dev/null || true)`
|
|
: `source_commit=${shellQuote(target.sourceCommit ?? "")}`,
|
|
"pipeline_run=",
|
|
].join("\n");
|
|
const controlPlaneProbe = [
|
|
`kubectl get serviceaccount -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(spec.serviceAccountName)} -o name`,
|
|
`kubectl get pipeline -n ${shellQuote(CI_NAMESPACE)} ${shellQuote(spec.pipeline)} -o name`,
|
|
].join("; ");
|
|
const script = [
|
|
"set +e",
|
|
targetInit,
|
|
`target_mode=${shellQuote(targetMode)}`,
|
|
`if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run=$(kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} -l "hwlab.pikastech.local/gitops-target=${spec.lane},hwlab.pikastech.local/source-commit=$source_commit" --sort-by=.metadata.creationTimestamp -o name 2>/dev/null | sed 's#.*/##' | tail -n 1 || true); fi`,
|
|
`if [ -z "$pipeline_run" ] && [ -n "$source_commit" ]; then pipeline_run="${spec.pipelineRunPrefix}-$(printf '%s' "$source_commit" | cut -c1-12)"; fi`,
|
|
"section() {",
|
|
" name=\"$1\"",
|
|
" shift",
|
|
" printf '__UNIDESK_SECTION_BEGIN__ %s\\n' \"$name\"",
|
|
" \"$@\"",
|
|
" code=$?",
|
|
" printf '\\n__UNIDESK_SECTION_END__ %s exit=%s\\n' \"$name\" \"$code\"",
|
|
"}",
|
|
"printf '__UNIDESK_SECTION_BEGIN__ statusTarget\\n'",
|
|
"printf 'mode\\t%s\\n' \"$target_mode\"",
|
|
"printf 'sourceCommit\\t%s\\n' \"$source_commit\"",
|
|
"printf 'pipelineRun\\t%s\\n' \"$pipeline_run\"",
|
|
"printf '__UNIDESK_SECTION_END__ statusTarget exit=0\\n'",
|
|
`section sourceHead git --git-dir=${shellQuote(spec.cicdRepo)} rev-parse refs/remotes/origin/${spec.sourceBranch}`,
|
|
`section controlPlane sh -lc ${shellQuote(controlPlaneProbe)}`,
|
|
`section argo kubectl get application -n ${shellQuote(ARGO_NAMESPACE)} ${shellQuote(spec.app)} -o 'jsonpath={.spec.source.targetRevision}{"\\n"}{.spec.source.path}{"\\n"}{.status.sync.revision}{"\\n"}{.status.sync.status}{"\\n"}{.status.health.status}{"\\n"}'`,
|
|
`section pipelineRun kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)} "$pipeline_run" -o 'jsonpath={.status.conditions[0].status}{"\\n"}{.status.conditions[0].reason}{"\\n"}{.status.conditions[0].message}{"\\n"}'`,
|
|
`section taskRunDiagnostics env PIPELINE_RUN="$pipeline_run" CI_NAMESPACE=${shellQuote(CI_NAMESPACE)} TASKRUN_LOG_TAIL_LINES=60 TASKRUN_LOG_MAX_CHARS=4000 sh -lc ${shellQuote(runtimeLaneTaskRunDiagnosticsScript())}`,
|
|
`section runtimeWorkloads kubectl get deploy,statefulset,svc,ingress,configmap -n ${shellQuote(spec.runtimeNamespace)} -l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)} -o name`,
|
|
`section publicProbes sh -lc ${shellQuote(runtimeLanePublicProbeScript(spec))}`,
|
|
[
|
|
`section recentPipelineRuns kubectl get pipelinerun -n ${shellQuote(CI_NAMESPACE)}`,
|
|
`-l hwlab.pikastech.local/gitops-target=${shellQuote(spec.lane)}`,
|
|
"--sort-by=.metadata.creationTimestamp",
|
|
"-o",
|
|
shellQuote(pipelineRunRowsJsonPath()),
|
|
].join(" "),
|
|
].join("\n");
|
|
return legacyRuntimeK3s(["sh", "--", script], 120_000);
|
|
}
|
|
|
|
export function runtimeLanePublicProbeScript(spec: HwlabRuntimeLaneSpec): string {
|
|
const apiUrl = `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`;
|
|
const webUrl = spec.publicWebUrl.replace(/\/+$/u, "/");
|
|
return [
|
|
"set +e",
|
|
`api_url=${shellQuote(apiUrl)}`,
|
|
`web_url=${shellQuote(webUrl)}`,
|
|
"api_output=$(curl -fsS --connect-timeout 3 --max-time 15 \"$api_url\" 2>&1)",
|
|
"api_exit=$?",
|
|
"api_status_ok=no",
|
|
"if [ \"$api_exit\" = 0 ] && printf '%s' \"$api_output\" | grep -Eq '\"status\"[[:space:]]*:[[:space:]]*\"ok\"'; then api_status_ok=yes; fi",
|
|
"web_tmp=$(mktemp /tmp/hwlab-lane-web-probe.XXXXXX)",
|
|
"web_output=$(curl -fsS --connect-timeout 3 --max-time 15 -o \"$web_tmp\" \"$web_url\" 2>&1)",
|
|
"web_exit=$?",
|
|
"web_bytes=0",
|
|
"web_has_html=no",
|
|
"if [ -f \"$web_tmp\" ]; then",
|
|
" web_bytes=$(wc -c < \"$web_tmp\" | tr -d ' ')",
|
|
" if grep -Eiq '<html|<script|<div' \"$web_tmp\"; then web_has_html=yes; fi",
|
|
"fi",
|
|
"printf 'apiUrl\\t%s\\n' \"$api_url\"",
|
|
"printf 'apiExitCode\\t%s\\n' \"$api_exit\"",
|
|
"printf 'apiStatusOk\\t%s\\n' \"$api_status_ok\"",
|
|
"printf 'apiPreview\\t%s\\n' \"$(printf '%s' \"$api_output\" | tr '\\n\\t' ' ' | cut -c1-240)\"",
|
|
"printf 'webUrl\\t%s\\n' \"$web_url\"",
|
|
"printf 'webExitCode\\t%s\\n' \"$web_exit\"",
|
|
"printf 'webBytes\\t%s\\n' \"$web_bytes\"",
|
|
"printf 'webHasHtml\\t%s\\n' \"$web_has_html\"",
|
|
"printf 'webErrorPreview\\t%s\\n' \"$(printf '%s' \"$web_output\" | tr '\\n\\t' ' ' | cut -c1-240)\"",
|
|
"rm -f \"$web_tmp\"",
|
|
"exit 0",
|
|
].join("\n");
|
|
}
|
|
|
|
export function runtimeLaneControlPlaneStatus(spec: HwlabRuntimeLaneSpec, target: DefaultRuntimeLaneControlPlaneStatusTarget = {}): Record<string, unknown> {
|
|
const targetMode: DefaultRuntimeLaneStatusTargetMode = target.mode
|
|
?? (target.pipelineRun !== undefined && target.pipelineRun !== null ? "pipeline-run" : target.sourceCommit !== undefined ? "source-commit" : "latest-source-head");
|
|
const bundle = runtimeLaneControlPlaneStatusBundle(spec, target);
|
|
const sections = parseShellSections(statusText(bundle));
|
|
const statusTargetFields = keyValueLinesFromText(sections.statusTarget?.stdout ?? "");
|
|
const sourceCommit = stringOrNull(statusTargetFields.sourceCommit) ?? stringOrNull(sections.sourceHead?.stdout) ?? null;
|
|
const pipelineRun = stringOrNull(statusTargetFields.pipelineRun) ?? (sourceCommit === null ? null : runtimeLanePipelineRunName(spec, sourceCommit));
|
|
const [targetRevision = "", path = "", syncRevision = "", syncStatus = "", health = ""] = String(sections.argo?.stdout ?? "").split(/\r?\n/u);
|
|
const pipelineRunInfo = pipelineRun === null
|
|
? null
|
|
: pipelineRunCompactFromText(
|
|
pipelineRun,
|
|
sections.pipelineRun?.stdout ?? "",
|
|
shellSectionOk(sections.pipelineRun),
|
|
`kubectl get pipelinerun -n ${CI_NAMESPACE} ${pipelineRun}`,
|
|
sections.pipelineRun?.exitCode ?? null,
|
|
bundle.stderr,
|
|
);
|
|
const taskRuns = pipelineRun === null ? null : runtimeLaneTaskRunDiagnosticsFromText(pipelineRun, sections.taskRunDiagnostics, bundle.stderr);
|
|
const runtimeWorkloadNames = String(sections.runtimeWorkloads?.stdout ?? "").split(/\r?\n/u).map((line) => line.trim()).filter(Boolean);
|
|
const publicProbeFields = keyValueLinesFromText(sections.publicProbes?.stdout ?? "");
|
|
const publicProbesOk = shellSectionOk(sections.publicProbes)
|
|
&& publicProbeFields.apiExitCode === "0"
|
|
&& publicProbeFields.apiStatusOk === "yes"
|
|
&& publicProbeFields.webExitCode === "0"
|
|
&& Number(publicProbeFields.webBytes ?? 0) > 0;
|
|
const recentPipelineRuns = parsePipelineRunRows(sections.recentPipelineRuns?.stdout ?? "", 8, Date.now(), spec.pipelineRunPrefix);
|
|
const pipelineRunOk = pipelineRunInfo !== null && pipelineRunInfo.status === "True";
|
|
const argoOk = shellSectionOk(sections.argo) && syncStatus === "Synced" && health === "Healthy";
|
|
const runtimeWorkloadsOk = shellSectionOk(sections.runtimeWorkloads);
|
|
const baseOk = isCommandSuccess(bundle) && sourceCommit !== null && shellSectionOk(sections.controlPlane) && pipelineRunOk && argoOk && runtimeWorkloadsOk && publicProbesOk;
|
|
return {
|
|
ok: baseOk,
|
|
command: `hwlab g14 control-plane status --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "runtime-lane-status",
|
|
statusTarget: {
|
|
mode: statusTargetFields.mode || targetMode,
|
|
sourceCommit,
|
|
pipelineRun,
|
|
note: `${spec.lane} status is a bootstrap visibility check; v0.2 closeout/false-green verdicts are not reused for new lanes`,
|
|
},
|
|
sourceCommit,
|
|
expected: {
|
|
sourceRepo: spec.cicdRepo,
|
|
configPath: hwlabRuntimeLaneConfigPath(),
|
|
node: spec.nodeId,
|
|
nodeRoute: spec.nodeRoute,
|
|
workspace: spec.workspace,
|
|
branch: spec.sourceBranch,
|
|
namespace: CI_NAMESPACE,
|
|
runtimeNamespace: spec.runtimeNamespace,
|
|
pipeline: spec.pipeline,
|
|
serviceAccount: spec.serviceAccountName,
|
|
argoApplication: spec.app,
|
|
gitopsBranch: spec.gitopsBranch,
|
|
runtimePath: spec.runtimePath,
|
|
networkProfile: {
|
|
id: spec.networkProfileId,
|
|
httpProxy: spec.networkProfile.proxy.http,
|
|
noProxy: spec.networkProfile.proxy.noProxy.join(","),
|
|
dockerBuildHttpProxy: spec.networkProfile.dockerBuildProxy.http,
|
|
dockerBuildNoProxy: spec.networkProfile.dockerBuildProxy.noProxy.join(","),
|
|
},
|
|
downloadProfile: spec.downloadProfile,
|
|
},
|
|
sourceHead: {
|
|
ok: shellSectionOk(sections.sourceHead),
|
|
value: stringOrNull(sections.sourceHead?.stdout),
|
|
exitCode: sections.sourceHead?.exitCode ?? null,
|
|
},
|
|
controlPlane: {
|
|
ok: shellSectionOk(sections.controlPlane),
|
|
raw: sections.controlPlane?.stdout ?? "",
|
|
exitCode: sections.controlPlane?.exitCode ?? null,
|
|
},
|
|
argo: {
|
|
ok: shellSectionOk(sections.argo),
|
|
raw: sections.argo?.stdout ?? "",
|
|
fields: { targetRevision, path, syncRevision, syncStatus, health },
|
|
exitCode: sections.argo?.exitCode ?? null,
|
|
},
|
|
pipelineRun: pipelineRunInfo,
|
|
taskRuns,
|
|
runtimeWorkloads: {
|
|
ok: shellSectionOk(sections.runtimeWorkloads),
|
|
namespace: spec.runtimeNamespace,
|
|
names: runtimeWorkloadNames,
|
|
count: runtimeWorkloadNames.length,
|
|
exitCode: sections.runtimeWorkloads?.exitCode ?? null,
|
|
},
|
|
publicProbes: {
|
|
ok: publicProbesOk,
|
|
apiUrl: publicProbeFields.apiUrl || `${spec.publicApiUrl.replace(/\/+$/u, "")}/health/live`,
|
|
apiExitCode: numericField(publicProbeFields.apiExitCode),
|
|
apiStatusOk: publicProbeFields.apiStatusOk === "yes",
|
|
apiPreview: publicProbeFields.apiPreview || null,
|
|
webUrl: publicProbeFields.webUrl || spec.publicWebUrl,
|
|
webExitCode: numericField(publicProbeFields.webExitCode),
|
|
webBytes: numericField(publicProbeFields.webBytes),
|
|
webHasHtml: publicProbeFields.webHasHtml === "yes",
|
|
webErrorPreview: publicProbeFields.webErrorPreview || null,
|
|
exitCode: sections.publicProbes?.exitCode ?? null,
|
|
},
|
|
recentPipelineRuns,
|
|
query: {
|
|
ok: isCommandSuccess(bundle),
|
|
exitCode: bundle.exitCode,
|
|
timedOut: bundle.timedOut ?? false,
|
|
durationMs: bundle.durationMs ?? null,
|
|
stdoutBytes: bundle.stdoutBytes ?? Buffer.byteLength(bundle.stdout, "utf8"),
|
|
stderrBytes: bundle.stderrBytes ?? Buffer.byteLength(bundle.stderr, "utf8"),
|
|
stderr: bundle.stderr.trim().slice(0, 2000),
|
|
},
|
|
next: {
|
|
apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm`,
|
|
refresh: `bun scripts/cli.ts hwlab g14 control-plane refresh --lane ${spec.lane} --confirm`,
|
|
triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function runtimeLaneArgoRefreshScript(spec: HwlabRuntimeLaneSpec, dryRun: boolean): string {
|
|
const operationPatch = JSON.stringify({ operation: null });
|
|
return [
|
|
"set +e",
|
|
`app=${shellQuote(spec.app)}`,
|
|
`argo_namespace=${shellQuote(ARGO_NAMESPACE)}`,
|
|
`runtime_namespace=${shellQuote(spec.runtimeNamespace)}`,
|
|
`lane=${shellQuote(spec.lane)}`,
|
|
`dry_run=${shellQuote(dryRun ? "true" : "false")}`,
|
|
`operation_patch=${shellQuote(operationPatch)}`,
|
|
"app_jsonpath() { kubectl -n \"$argo_namespace\" get application \"$app\" -o \"jsonpath=$1\" 2>/dev/null || true; }",
|
|
"job_jsonpath() { kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o \"jsonpath=$1\" 2>/dev/null || true; }",
|
|
"before_operation_revision=$(app_jsonpath '{.operation.sync.revision}')",
|
|
"before_operation_phase=$(app_jsonpath '{.status.operationState.phase}')",
|
|
"before_operation_message=$(app_jsonpath '{.status.operationState.message}')",
|
|
"before_sync_revision=$(app_jsonpath '{.status.sync.revision}')",
|
|
"before_sync_status=$(app_jsonpath '{.status.sync.status}')",
|
|
"before_health=$(app_jsonpath '{.status.health.status}')",
|
|
"before_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')",
|
|
"before_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)",
|
|
"before_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')",
|
|
"before_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')",
|
|
"before_hook_active=$(job_jsonpath '{.status.active}')",
|
|
"before_operation_present=$([ -n \"$before_operation_revision$before_operation_phase$before_operation_message\" ] && printf yes || printf no)",
|
|
"patch_exit=",
|
|
"patch_output=skipped-no-live-operation",
|
|
"annotate_exit=",
|
|
"annotate_output=",
|
|
"if [ \"$before_operation_present\" = yes ]; then",
|
|
" if [ \"$dry_run\" = true ]; then",
|
|
" patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" --dry-run=server -o name 2>&1)",
|
|
" else",
|
|
" patch_output=$(kubectl -n \"$argo_namespace\" patch application \"$app\" --type=merge -p \"$operation_patch\" -o name 2>&1)",
|
|
" fi",
|
|
" patch_exit=$?",
|
|
"fi",
|
|
"if [ \"$dry_run\" = true ]; then",
|
|
" annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite --dry-run=server -o name 2>&1)",
|
|
"else",
|
|
" annotate_output=$(kubectl -n \"$argo_namespace\" annotate application \"$app\" argocd.argoproj.io/refresh=hard --overwrite -o name 2>&1)",
|
|
"fi",
|
|
"annotate_exit=$?",
|
|
"if [ \"$dry_run\" != true ]; then sleep 2; fi",
|
|
"after_operation_revision=$(app_jsonpath '{.operation.sync.revision}')",
|
|
"after_operation_phase=$(app_jsonpath '{.status.operationState.phase}')",
|
|
"after_operation_message=$(app_jsonpath '{.status.operationState.message}')",
|
|
"after_sync_revision=$(app_jsonpath '{.status.sync.revision}')",
|
|
"after_sync_status=$(app_jsonpath '{.status.sync.status}')",
|
|
"after_health=$(app_jsonpath '{.status.health.status}')",
|
|
"after_resource_version=$(app_jsonpath '{.metadata.resourceVersion}')",
|
|
"after_hook_job=$(kubectl -n \"$runtime_namespace\" get job hwlab-openfga-migrate -o name 2>/dev/null || true)",
|
|
"after_hook_source=$(job_jsonpath '{.metadata.labels.hwlab\\.pikastech\\.local/source-commit}')",
|
|
"after_hook_phase=$(job_jsonpath '{.status.conditions[0].type}')",
|
|
"after_hook_active=$(job_jsonpath '{.status.active}')",
|
|
"after_operation_present=$([ -n \"$after_operation_revision$after_operation_phase$after_operation_message\" ] && printf yes || printf no)",
|
|
"printf 'app\\t%s\\n' \"$app\"",
|
|
"printf 'lane\\t%s\\n' \"$lane\"",
|
|
"printf 'runtimeNamespace\\t%s\\n' \"$runtime_namespace\"",
|
|
"printf 'dryRun\\t%s\\n' \"$dry_run\"",
|
|
"printf 'beforeOperationPresent\\t%s\\n' \"$before_operation_present\"",
|
|
"printf 'beforeOperationRevision\\t%s\\n' \"$before_operation_revision\"",
|
|
"printf 'beforeOperationPhase\\t%s\\n' \"$before_operation_phase\"",
|
|
"printf 'beforeOperationMessage\\t%s\\n' \"$before_operation_message\"",
|
|
"printf 'beforeSyncRevision\\t%s\\n' \"$before_sync_revision\"",
|
|
"printf 'beforeSyncStatus\\t%s\\n' \"$before_sync_status\"",
|
|
"printf 'beforeHealth\\t%s\\n' \"$before_health\"",
|
|
"printf 'beforeResourceVersion\\t%s\\n' \"$before_resource_version\"",
|
|
"printf 'beforeHookJob\\t%s\\n' \"$before_hook_job\"",
|
|
"printf 'beforeHookSourceCommit\\t%s\\n' \"$before_hook_source\"",
|
|
"printf 'beforeHookPhase\\t%s\\n' \"$before_hook_phase\"",
|
|
"printf 'beforeHookActive\\t%s\\n' \"$before_hook_active\"",
|
|
"printf 'patchExitCode\\t%s\\n' \"$patch_exit\"",
|
|
"printf 'patchOutput\\t%s\\n' \"$patch_output\"",
|
|
"printf 'annotateExitCode\\t%s\\n' \"$annotate_exit\"",
|
|
"printf 'annotateOutput\\t%s\\n' \"$annotate_output\"",
|
|
"printf 'afterOperationPresent\\t%s\\n' \"$after_operation_present\"",
|
|
"printf 'afterOperationRevision\\t%s\\n' \"$after_operation_revision\"",
|
|
"printf 'afterOperationPhase\\t%s\\n' \"$after_operation_phase\"",
|
|
"printf 'afterOperationMessage\\t%s\\n' \"$after_operation_message\"",
|
|
"printf 'afterSyncRevision\\t%s\\n' \"$after_sync_revision\"",
|
|
"printf 'afterSyncStatus\\t%s\\n' \"$after_sync_status\"",
|
|
"printf 'afterHealth\\t%s\\n' \"$after_health\"",
|
|
"printf 'afterResourceVersion\\t%s\\n' \"$after_resource_version\"",
|
|
"printf 'afterHookJob\\t%s\\n' \"$after_hook_job\"",
|
|
"printf 'afterHookSourceCommit\\t%s\\n' \"$after_hook_source\"",
|
|
"printf 'afterHookPhase\\t%s\\n' \"$after_hook_phase\"",
|
|
"printf 'afterHookActive\\t%s\\n' \"$after_hook_active\"",
|
|
"if [ -n \"$patch_exit\" ] && [ \"$patch_exit\" != 0 ]; then exit \"$patch_exit\"; fi",
|
|
"if [ -n \"$annotate_exit\" ] && [ \"$annotate_exit\" != 0 ]; then exit \"$annotate_exit\"; fi",
|
|
].join("\n");
|
|
}
|
|
|
|
export function refreshRuntimeLaneArgoApplication(spec: HwlabRuntimeLaneSpec, options: LegacyRuntimeControlPlaneOptions): Record<string, unknown> {
|
|
const result = legacyRuntimeK3s(["sh", "--", runtimeLaneArgoRefreshScript(spec, options.dryRun)], options.timeoutSeconds * 1000);
|
|
const fields = keyValueLinesFromText(statusText(result));
|
|
const ok = isCommandSuccess(result);
|
|
return {
|
|
ok,
|
|
command: `hwlab g14 control-plane refresh --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-refresh",
|
|
argoApplication: spec.app,
|
|
runtimeNamespace: spec.runtimeNamespace,
|
|
dryRun: options.dryRun,
|
|
action: {
|
|
terminatedOperation: fields.beforeOperationPresent === "yes",
|
|
hardRefreshRequested: numericField(fields.annotateExitCode) === 0,
|
|
mutation: !options.dryRun && ok,
|
|
},
|
|
before: {
|
|
operationPresent: fields.beforeOperationPresent === "yes",
|
|
operationRevision: fields.beforeOperationRevision || null,
|
|
operationPhase: fields.beforeOperationPhase || null,
|
|
operationMessage: fields.beforeOperationMessage || null,
|
|
syncRevision: fields.beforeSyncRevision || null,
|
|
syncStatus: fields.beforeSyncStatus || null,
|
|
health: fields.beforeHealth || null,
|
|
resourceVersion: fields.beforeResourceVersion || null,
|
|
hookJob: fields.beforeHookJob || null,
|
|
hookSourceCommit: fields.beforeHookSourceCommit || null,
|
|
hookPhase: fields.beforeHookPhase || null,
|
|
hookActive: fields.beforeHookActive || null,
|
|
},
|
|
after: {
|
|
operationPresent: fields.afterOperationPresent === "yes",
|
|
operationRevision: fields.afterOperationRevision || null,
|
|
operationPhase: fields.afterOperationPhase || null,
|
|
operationMessage: fields.afterOperationMessage || null,
|
|
syncRevision: fields.afterSyncRevision || null,
|
|
syncStatus: fields.afterSyncStatus || null,
|
|
health: fields.afterHealth || null,
|
|
resourceVersion: fields.afterResourceVersion || null,
|
|
hookJob: fields.afterHookJob || null,
|
|
hookSourceCommit: fields.afterHookSourceCommit || null,
|
|
hookPhase: fields.afterHookPhase || null,
|
|
hookActive: fields.afterHookActive || null,
|
|
},
|
|
patchExitCode: numericField(fields.patchExitCode),
|
|
annotateExitCode: numericField(fields.annotateExitCode),
|
|
result: compactCommandResult(result),
|
|
next: {
|
|
status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane}`,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function runRuntimeLaneControlPlane(spec: HwlabRuntimeLaneSpec, options: LegacyRuntimeControlPlaneOptions): Record<string, unknown> {
|
|
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
|
|
if (options.action === "closeout" || options.action === "runtime-migration" || options.action === "cleanup-released-pvs") {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
degradedReason: "unsupported-runtime-lane-action",
|
|
message: `${options.action} is still v0.2-specific; v0.3+ currently supports status/apply/trigger-current/refresh/cleanup-runs`,
|
|
};
|
|
}
|
|
if (options.action === "status" && options.pipelineRun !== undefined) {
|
|
return runtimeLaneControlPlaneStatus(spec, { pipelineRun: options.pipelineRun, mode: "pipeline-run", includeHistory: options.history });
|
|
}
|
|
if (options.action === "status" && options.sourceCommit !== undefined) {
|
|
return runtimeLaneControlPlaneStatus(spec, { sourceCommit: options.sourceCommit, mode: "source-commit", includeHistory: options.history });
|
|
}
|
|
const latestHead = options.action === "trigger-current" ? resolveRuntimeLaneLatestRemoteHead(spec) : null;
|
|
const head = resolveRuntimeLaneHead(spec);
|
|
let sourceCommit = head.sourceCommit;
|
|
if (options.action === "trigger-current" && latestHead !== null && latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) {
|
|
const refreshedHead = resolveRuntimeLaneHead(spec);
|
|
sourceCommit = refreshedHead.sourceCommit;
|
|
if (sourceCommit !== latestHead.sourceCommit) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "confirmed-trigger",
|
|
phase: "source-head-refresh",
|
|
sourceCommit,
|
|
latestSourceHeadProbe: {
|
|
sourceCommit: latestHead.sourceCommit,
|
|
result: compactCommandResult(latestHead.result),
|
|
},
|
|
refreshedHeadProbe: compactCommandResult(refreshedHead.result),
|
|
degradedReason: `${spec.lane}-head-refresh-did-not-reach-latest-source`,
|
|
};
|
|
}
|
|
}
|
|
if (sourceCommit === null) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane ${options.action} --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
degradedReason: `${spec.lane}-head-unresolved`,
|
|
sourceRepo: spec.cicdRepo,
|
|
workspace: spec.workspace,
|
|
headProbe: compactCommandResult(head.result),
|
|
};
|
|
}
|
|
if (options.action === "status") {
|
|
return runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history });
|
|
}
|
|
if (options.action === "refresh") {
|
|
return refreshRuntimeLaneArgoApplication(spec, options);
|
|
}
|
|
if (options.action === "apply") {
|
|
const render = runRuntimeLaneRenderToTemp(spec, sourceCommit);
|
|
if (!isCommandSuccess(render.result)) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane apply --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
phase: "source-render",
|
|
sourceCommit,
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
};
|
|
}
|
|
const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, options.dryRun, options.timeoutSeconds);
|
|
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null;
|
|
return {
|
|
ok: isCommandSuccess(apply),
|
|
command: `hwlab g14 control-plane apply --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
|
sourceCommit,
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
apply: compactCommandResult(apply),
|
|
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
|
status: runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }),
|
|
next: options.dryRun
|
|
? { apply: `bun scripts/cli.ts hwlab g14 control-plane apply --lane ${spec.lane} --confirm` }
|
|
: { triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm` },
|
|
};
|
|
}
|
|
let pipelineRun = runtimeLanePipelineRunName(spec, sourceCommit);
|
|
let before = getPipelineRunCompact(pipelineRun);
|
|
const originalPipelineRun = pipelineRun;
|
|
if (options.rerun && before.exists === true) {
|
|
pipelineRun = runtimeLaneRerunPipelineRunName(spec, sourceCommit);
|
|
before = getPipelineRunCompact(pipelineRun);
|
|
}
|
|
if (options.dryRun) {
|
|
return {
|
|
ok: true,
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "dry-run",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
rerun: options.rerun,
|
|
originalPipelineRun: options.rerun ? originalPipelineRun : undefined,
|
|
before,
|
|
gitMirrorSync: {
|
|
mode: "skipped-dry-run",
|
|
wouldSyncBeforeTrigger: true,
|
|
command: `bun scripts/cli.ts hwlab g14 git-mirror sync --lane ${spec.lane} --confirm`,
|
|
},
|
|
controlPlaneRefresh: {
|
|
mode: "skipped-dry-run",
|
|
wouldRefreshBeforeCreate: true,
|
|
resources: [`${spec.runtimeRenderDir}/namespace.yaml`, `${spec.tektonDir}/rbac.yaml`, `${spec.tektonDir}/pipeline.yaml`, "argocd/project.yaml", `argocd/${spec.argoApplicationFile}`],
|
|
},
|
|
manifest: runtimeLanePipelineRunManifest(spec, sourceCommit),
|
|
next: { triggerCurrent: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm` },
|
|
};
|
|
}
|
|
const beforeStatus = stringOrNull(before.status);
|
|
if (before.exists === true && beforeStatus === "True") {
|
|
const currentCd = runtimeLaneStatusWithGitMirror(spec, sourceCommit);
|
|
if (runtimeLaneCdPassedWithMirror(spec, currentCd)) {
|
|
return {
|
|
ok: true,
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
skipped: true,
|
|
reason: "existing-pipelinerun-reused",
|
|
cd: summarizeRuntimeLaneCdStatus(spec, currentCd),
|
|
latestOnlyPolicy: "same source commit is idempotent only after PipelineRun, Argo/runtime, public probes, and Git mirror are already aligned",
|
|
};
|
|
}
|
|
pipelineRun = runtimeLaneRerunPipelineRunName(spec, sourceCommit);
|
|
before = getPipelineRunCompact(pipelineRun);
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "existing-pipelinerun",
|
|
status: "rerun-required",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
previousPipelineRun: runtimeLanePipelineRunName(spec, sourceCommit),
|
|
previousCd: summarizeRuntimeLaneCdStatus(spec, currentCd),
|
|
});
|
|
}
|
|
const effectiveBeforeStatus = stringOrNull(before.status);
|
|
if (before.exists === true && (effectiveBeforeStatus === "True" || effectiveBeforeStatus === "Unknown")) {
|
|
return {
|
|
ok: true,
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
skipped: true,
|
|
reason: "existing-rerun-pipelinerun-reused",
|
|
latestOnlyPolicy: "same source commit reuses an active or already successful rerun PipelineRun after the deterministic run failed CD alignment",
|
|
};
|
|
}
|
|
if (before.exists === true && effectiveBeforeStatus !== null) {
|
|
if (options.rerun === true) {
|
|
pipelineRun = runtimeLaneRerunPipelineRunName(spec, sourceCommit);
|
|
before = getPipelineRunCompact(pipelineRun);
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "existing-pipelinerun",
|
|
status: "explicit-rerun-required",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
previousPipelineRun: originalPipelineRun,
|
|
previousStatus: effectiveBeforeStatus,
|
|
});
|
|
} else {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
skipped: true,
|
|
reason: "existing-pipelinerun-terminal-failed",
|
|
degradedReason: "existing-pipelinerun-terminal-failed",
|
|
next: {
|
|
rerun: `bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane ${spec.lane} --confirm --rerun`,
|
|
status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --pipeline-run ${pipelineRun}`,
|
|
},
|
|
};
|
|
}
|
|
}
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "git-mirror-sync",
|
|
status: "started",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
});
|
|
const gitMirrorPreSyncStatus = runGitMirrorStatus(spec.lane);
|
|
const gitMirrorSourceAlreadyCurrent = runtimeLaneGitMirrorSourceInSync(spec, sourceCommit, gitMirrorPreSyncStatus);
|
|
const gitMirrorSync = gitMirrorSourceAlreadyCurrent
|
|
? {
|
|
ok: true,
|
|
command: "hwlab nodes git-mirror sync",
|
|
mode: "skipped-source-already-current",
|
|
skipped: true,
|
|
reason: "local-mirror-source-current",
|
|
status: gitMirrorPreSyncStatus,
|
|
}
|
|
: runGitMirrorSync({
|
|
action: "sync",
|
|
lane: spec.lane,
|
|
confirm: true,
|
|
dryRun: false,
|
|
wait: true,
|
|
timeoutSeconds: options.timeoutSeconds,
|
|
});
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "git-mirror-sync",
|
|
status: gitMirrorSync.ok === true ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
jobName: gitMirrorSync.jobName ?? null,
|
|
elapsedMs: gitMirrorSync.elapsedMs ?? null,
|
|
});
|
|
if (gitMirrorSync.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
gitMirrorSync: compactGitMirrorSync(gitMirrorSync),
|
|
degradedReason: "git-mirror-sync-before-trigger-failed",
|
|
};
|
|
}
|
|
const gitMirrorPostSyncStatus = gitMirrorSourceAlreadyCurrent ? gitMirrorPreSyncStatus : record(gitMirrorSync.status);
|
|
if (!runtimeLaneGitMirrorSourceInSync(spec, sourceCommit, gitMirrorPostSyncStatus)) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
gitMirrorSync: compactGitMirrorSync(gitMirrorSync),
|
|
gitMirror: {
|
|
ok: gitMirrorPostSyncStatus.ok === true,
|
|
summary: record(nested(gitMirrorPostSyncStatus, ["cache", "summary"])),
|
|
},
|
|
degradedReason: `${spec.lane}-git-mirror-source-still-stale-after-sync`,
|
|
next: {
|
|
apply: `bun scripts/cli.ts hwlab nodes git-mirror apply --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
sync: `bun scripts/cli.ts hwlab nodes git-mirror sync --node ${spec.nodeId} --lane ${spec.lane} --confirm`,
|
|
},
|
|
};
|
|
}
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "source-render",
|
|
status: "started",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
});
|
|
const render = runRuntimeLaneRenderToTemp(spec, sourceCommit);
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "source-render",
|
|
status: isCommandSuccess(render.result) ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
renderDir: render.renderDir,
|
|
exitCode: render.result.exitCode,
|
|
});
|
|
if (!isCommandSuccess(render.result)) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "confirmed-trigger",
|
|
phase: "source-render",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
gitMirrorSync: compactGitMirrorSync(gitMirrorSync),
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
degradedReason: "control-plane-render-failed",
|
|
};
|
|
}
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "control-plane-apply",
|
|
status: "started",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
renderDir: render.renderDir,
|
|
});
|
|
const apply = applyRuntimeLaneControlPlaneFiles(spec, render.renderDir, false, options.timeoutSeconds);
|
|
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupRuntimeLaneRenderDir(spec, render.renderDir) : null;
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "control-plane-apply",
|
|
status: isCommandSuccess(apply) ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
exitCode: apply.exitCode,
|
|
});
|
|
if (!isCommandSuccess(apply)) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "confirmed-trigger",
|
|
phase: "control-plane-apply",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
gitMirrorSync: compactGitMirrorSync(gitMirrorSync),
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
apply: compactCommandResult(apply),
|
|
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
|
degradedReason: "control-plane-apply-before-trigger-failed",
|
|
};
|
|
}
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "create-pipelinerun",
|
|
status: "started",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
});
|
|
const create = createRuntimeLanePipelineRun(spec, sourceCommit, options.timeoutSeconds, pipelineRun);
|
|
printRuntimeLaneTriggerProgress(spec, {
|
|
stage: "create-pipelinerun",
|
|
status: isCommandSuccess(create) ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
exitCode: create.exitCode,
|
|
});
|
|
return {
|
|
ok: isCommandSuccess(create),
|
|
command: `hwlab g14 control-plane trigger-current --lane ${spec.lane}`,
|
|
lane: spec.lane,
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun,
|
|
before,
|
|
gitMirrorSync: compactGitMirrorSync(gitMirrorSync),
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
apply: compactCommandResult(apply),
|
|
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
|
create: compactCommandResult(create),
|
|
status: runtimeLaneControlPlaneStatus(spec, { sourceCommit, mode: "latest-source-head", includeHistory: options.history }),
|
|
degradedReason: isCommandSuccess(create) ? undefined : "pipelinerun-create-failed",
|
|
next: { status: `bun scripts/cli.ts hwlab g14 control-plane status --lane ${spec.lane} --source-commit ${sourceCommit}` },
|
|
};
|
|
}
|
|
|
|
export function runDefaultRuntimeLaneControlPlane(options: LegacyRuntimeControlPlaneOptions): Record<string, unknown> {
|
|
if (options.action === "cleanup-runs") return runControlPlaneCleanup(options);
|
|
if (options.action === "cleanup-released-pvs") return runControlPlaneReleasedPvCleanup(options);
|
|
if ((options.action === "status" || options.action === "closeout") && options.pipelineRun !== undefined) {
|
|
const status = defaultRuntimeLaneControlPlaneStatus({ pipelineRun: options.pipelineRun, mode: "pipeline-run", includeHistory: options.history });
|
|
if (options.action === "closeout") return defaultRuntimeLaneControlPlaneCloseout(status);
|
|
return status;
|
|
}
|
|
if ((options.action === "status" || options.action === "closeout") && options.sourceCommit !== undefined) {
|
|
const status = defaultRuntimeLaneControlPlaneStatus({ sourceCommit: options.sourceCommit, mode: "source-commit", includeHistory: options.history });
|
|
if (options.action === "closeout") return defaultRuntimeLaneControlPlaneCloseout(status);
|
|
return status;
|
|
}
|
|
if (options.action === "closeout") {
|
|
const status = defaultRuntimeLaneControlPlaneStatus({ includeHistory: options.history });
|
|
return defaultRuntimeLaneControlPlaneCloseout(status);
|
|
}
|
|
const snapshot = options.action === "trigger-current" ? getDefaultRuntimeLaneTriggerSnapshot() : null;
|
|
const latestHead = options.action === "trigger-current" ? resolveDefaultRuntimeLaneLatestRemoteHead() : null;
|
|
const head = snapshot === null
|
|
? resolveDefaultRuntimeLaneHead()
|
|
: snapshot.sourceCommit === null
|
|
? resolveDefaultRuntimeLaneHead()
|
|
: { sourceCommit: snapshot.sourceCommit, result: snapshot.result };
|
|
let sourceCommit = head.sourceCommit;
|
|
if (options.action === "trigger-current" && latestHead !== null && latestHead.sourceCommit !== null && latestHead.sourceCommit !== sourceCommit) {
|
|
const refreshedHead = resolveDefaultRuntimeLaneHead();
|
|
sourceCommit = refreshedHead.sourceCommit;
|
|
if (sourceCommit !== latestHead.sourceCommit) {
|
|
return {
|
|
ok: false,
|
|
command: "hwlab g14 control-plane trigger-current --lane v02",
|
|
lane: "v02",
|
|
mode: "confirmed-trigger",
|
|
phase: "source-head-refresh",
|
|
sourceCommit,
|
|
latestSourceHeadProbe: {
|
|
sourceCommit: latestHead.sourceCommit,
|
|
result: compactCommandResult(latestHead.result),
|
|
},
|
|
refreshedHeadProbe: compactCommandResult(refreshedHead.result),
|
|
degradedReason: "v02-head-refresh-did-not-reach-latest-source",
|
|
};
|
|
}
|
|
}
|
|
if (sourceCommit === null) {
|
|
return { ok: false, command: `hwlab g14 control-plane ${options.action} --lane v02`, degradedReason: "v02-head-unresolved", sourceRepo: DEFAULT_RUNTIME_LANE_CICD_REPO, workspace: DEFAULT_RUNTIME_LANE_WORKSPACE, headProbe: compactCommandResult(head.result) };
|
|
}
|
|
if (options.action === "runtime-migration") return runDefaultRuntimeLaneRuntimeMigration(options, sourceCommit);
|
|
if (options.action === "status") return defaultRuntimeLaneControlPlaneStatus({ sourceCommit, mode: "latest-source-head", includeHistory: options.history });
|
|
if (options.action === "refresh") return refreshRuntimeLaneArgoApplication(DEFAULT_RUNTIME_LANE_LANE_SPEC, options);
|
|
if (options.action === "apply") {
|
|
const render = runDefaultRuntimeLaneRenderToTemp(sourceCommit);
|
|
if (!isCommandSuccess(render.result)) {
|
|
return {
|
|
ok: false,
|
|
command: `hwlab g14 control-plane ${options.action} --lane v02`,
|
|
phase: "source-render",
|
|
sourceCommit,
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
};
|
|
}
|
|
const apply = applyDefaultRuntimeLaneControlPlaneFiles(render.renderDir, options.dryRun, options.timeoutSeconds);
|
|
const cleanupRenderDir = isCommandSuccess(apply) ? cleanupDefaultRuntimeLaneRenderDir(render.renderDir) : null;
|
|
return {
|
|
ok: isCommandSuccess(apply),
|
|
command: "hwlab g14 control-plane apply --lane v02",
|
|
lane: "v02",
|
|
mode: options.dryRun ? "dry-run" : "confirmed-apply",
|
|
sourceCommit,
|
|
renderDir: render.renderDir,
|
|
render: compactCommandResult(render.result),
|
|
apply: compactCommandResult(apply),
|
|
cleanupRenderDir: cleanupRenderDir === null ? null : compactCommandResult(cleanupRenderDir),
|
|
cleanupObsoleteCronJobs: compactCommandResult(options.dryRun ? deleteDefaultRuntimeLaneObsoleteCronJobs(true) : deleteDefaultRuntimeLaneObsoleteCronJobs(false)),
|
|
status: defaultRuntimeLaneControlPlaneStatus({ sourceCommit, mode: "latest-source-head" }),
|
|
next: options.dryRun
|
|
? { apply: "bun scripts/cli.ts hwlab g14 control-plane apply --lane v02 --confirm" }
|
|
: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
|
};
|
|
}
|
|
let before = snapshot !== null && snapshot.sourceCommit === sourceCommit && snapshot.before !== null
|
|
? snapshot.before
|
|
: getPipelineRunCompact(defaultRuntimeLanePipelineRunName(sourceCommit));
|
|
if (options.dryRun) {
|
|
const gitMirrorPreSync = preSyncDefaultRuntimeLaneGitMirror(sourceCommit, options);
|
|
return {
|
|
ok: true,
|
|
command: "hwlab g14 control-plane trigger-current --lane v02",
|
|
lane: "v02",
|
|
mode: "dry-run",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
before,
|
|
gitMirrorPreSync,
|
|
controlPlaneRefresh: {
|
|
mode: "skipped-dry-run",
|
|
wouldRefreshBeforeCreate: true,
|
|
resources: ["tekton-v02/rbac.yaml", "tekton-v02/pipeline.yaml", "argocd/project.yaml", "argocd/application-v02.yaml"],
|
|
},
|
|
manifest: defaultRuntimeLanePipelineRunManifest(sourceCommit),
|
|
next: { triggerCurrent: "bun scripts/cli.ts hwlab g14 control-plane trigger-current --lane v02 --confirm" },
|
|
};
|
|
}
|
|
if (before.exists === true && before.status !== null) {
|
|
const reuseHead = latestHead ?? resolveDefaultRuntimeLaneLatestRemoteHead();
|
|
const reuseDecision = defaultRuntimeLaneExistingPipelineRunReuseDecision({ sourceCommit, before, latestSourceCommit: reuseHead.sourceCommit });
|
|
if (reuseDecision.reusable === true) {
|
|
const alreadyUsable = reuseDecision.alreadyUsable === true;
|
|
return {
|
|
ok: alreadyUsable,
|
|
command: "hwlab g14 control-plane trigger-current --lane v02",
|
|
lane: "v02",
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
before,
|
|
skipped: true,
|
|
reuseDecision,
|
|
reason: reuseDecision.reason,
|
|
degradedReason: alreadyUsable ? undefined : "existing-pipelinerun-terminal-failed",
|
|
latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default; source head is rechecked before reuse",
|
|
};
|
|
}
|
|
if (reuseHead.sourceCommit !== null && reuseHead.sourceCommit !== sourceCommit) {
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "source-head-recheck",
|
|
status: "advanced",
|
|
sourceCommit,
|
|
latestSourceCommit: reuseHead.sourceCommit,
|
|
previousPipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
latestPipelineRun: defaultRuntimeLanePipelineRunName(reuseHead.sourceCommit),
|
|
});
|
|
sourceCommit = reuseHead.sourceCommit;
|
|
before = getPipelineRunCompact(defaultRuntimeLanePipelineRunName(sourceCommit));
|
|
if (before.exists === true && before.status !== null) {
|
|
const advancedAlreadyUsable = before.status === "True" || before.status === "Unknown";
|
|
return {
|
|
ok: advancedAlreadyUsable,
|
|
command: "hwlab g14 control-plane trigger-current --lane v02",
|
|
lane: "v02",
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
before,
|
|
skipped: true,
|
|
reuseDecision: {
|
|
...reuseDecision,
|
|
advancedSourceCommit: sourceCommit,
|
|
advancedPipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
advancedPipelineRunStatus: before.status ?? null,
|
|
},
|
|
reason: advancedAlreadyUsable ? "advanced-existing-pipelinerun-reused" : "advanced-existing-pipelinerun-terminal-failed",
|
|
degradedReason: advancedAlreadyUsable ? undefined : "advanced-existing-pipelinerun-terminal-failed",
|
|
latestOnlyPolicy: "same source commit is idempotent; existing PipelineRun is never deleted or recreated by default; source head is rechecked before reuse",
|
|
};
|
|
}
|
|
} else {
|
|
return {
|
|
ok: false,
|
|
command: "hwlab g14 control-plane trigger-current --lane v02",
|
|
lane: "v02",
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
before,
|
|
reuseDecision,
|
|
degradedReason: "existing-pipelinerun-reuse-rejected",
|
|
latestSourceHeadProbe: {
|
|
sourceCommit: reuseHead.sourceCommit,
|
|
exitCode: reuseHead.result.exitCode,
|
|
stderr: reuseHead.result.stderr.trim().slice(0, 2000),
|
|
},
|
|
};
|
|
}
|
|
}
|
|
if (latestHead !== null && latestHead.sourceCommit !== sourceCommit) {
|
|
return {
|
|
ok: false,
|
|
command: "hwlab g14 control-plane trigger-current --lane v02",
|
|
lane: "v02",
|
|
mode: "confirmed-trigger",
|
|
phase: "source-head-recheck",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
before,
|
|
latestSourceHeadProbe: {
|
|
sourceCommit: latestHead.sourceCommit,
|
|
exitCode: latestHead.result.exitCode,
|
|
stderr: latestHead.result.stderr.trim().slice(0, 2000),
|
|
},
|
|
degradedReason: latestHead.sourceCommit === null
|
|
? "source-head-recheck-unresolved-before-pipelinerun-create"
|
|
: "source-head-advanced-before-pipelinerun-create",
|
|
};
|
|
}
|
|
printProgressEvent("hwlab.v02.trigger.progress", { stage: "control-plane-refresh", status: "started", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit) });
|
|
const controlPlaneRefresh = refreshDefaultRuntimeLaneControlPlaneBeforeTrigger(sourceCommit, options.timeoutSeconds);
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "control-plane-refresh",
|
|
status: controlPlaneRefresh.ok === true ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
mode: record(controlPlaneRefresh).mode ?? null,
|
|
waitedMs: record(controlPlaneRefresh).waitedMs ?? null,
|
|
renderDir: record(controlPlaneRefresh).renderDir ?? null,
|
|
degradedReason: record(controlPlaneRefresh).degradedReason ?? null,
|
|
});
|
|
if (controlPlaneRefresh.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
command: "hwlab g14 control-plane trigger-current --lane v02",
|
|
lane: "v02",
|
|
mode: "confirmed-trigger",
|
|
phase: "control-plane-refresh",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
before,
|
|
controlPlaneRefresh,
|
|
degradedReason: record(controlPlaneRefresh).degradedReason ?? "control-plane-refresh-failed",
|
|
};
|
|
}
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "git-mirror-pre-sync",
|
|
status: "started",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
reason: "ensure local git mirror has source commit before creating PipelineRun",
|
|
});
|
|
const gitMirrorPreSync = preSyncDefaultRuntimeLaneGitMirror(sourceCommit, options);
|
|
printProgressEvent("hwlab.v02.trigger.progress", {
|
|
stage: "git-mirror-pre-sync",
|
|
status: gitMirrorPreSync.ok === true ? "succeeded" : "failed",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
mode: record(gitMirrorPreSync).mode ?? null,
|
|
required: nested(gitMirrorPreSync, ["before", "required"]) ?? null,
|
|
localDefaultRuntimeLane: nested(gitMirrorPreSync, ["before", "localDefaultRuntimeLane"]) ?? null,
|
|
finalLocalDefaultRuntimeLane: nested(gitMirrorPreSync, ["final", "localDefaultRuntimeLane"]) ?? nested(gitMirrorPreSync, ["marker", "localDefaultRuntimeLane"]) ?? nested(gitMirrorPreSync, ["after", "localDefaultRuntimeLane"]) ?? null,
|
|
pendingFlush: nested(gitMirrorPreSync, ["before", "pendingFlush"]) ?? null,
|
|
reason: nested(gitMirrorPreSync, ["before", "reason"]) ?? null,
|
|
syncElapsedMs: nested(gitMirrorPreSync, ["sync", "elapsedMs"]) ?? null,
|
|
waitElapsedMs: nested(gitMirrorPreSync, ["wait", "elapsedMs"]) ?? null,
|
|
waitAttempts: nested(gitMirrorPreSync, ["wait", "attempts"]) ?? null,
|
|
lockWaitedMs: record(gitMirrorPreSync).waitedMs ?? null,
|
|
degradedReason: record(gitMirrorPreSync).degradedReason ?? null,
|
|
});
|
|
if (gitMirrorPreSync.ok !== true) {
|
|
return {
|
|
ok: false,
|
|
command: "hwlab g14 control-plane trigger-current --lane v02",
|
|
lane: "v02",
|
|
mode: "confirmed-trigger",
|
|
phase: "git-mirror-pre-sync",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
before,
|
|
controlPlaneRefresh,
|
|
gitMirrorPreSync,
|
|
degradedReason: "git-mirror-pre-sync-failed",
|
|
};
|
|
}
|
|
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: "started", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit) });
|
|
const createPipelineRun = createDefaultRuntimeLanePipelineRun(sourceCommit, options.timeoutSeconds);
|
|
printProgressEvent("hwlab.v02.trigger.progress", { stage: "create-pipelinerun", status: isCommandSuccess(createPipelineRun) ? "succeeded" : "failed", sourceCommit, pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit), exitCode: createPipelineRun.exitCode });
|
|
return {
|
|
ok: isCommandSuccess(createPipelineRun),
|
|
command: "hwlab g14 control-plane trigger-current --lane v02",
|
|
lane: "v02",
|
|
mode: "confirmed-trigger",
|
|
sourceCommit,
|
|
pipelineRun: defaultRuntimeLanePipelineRunName(sourceCommit),
|
|
before,
|
|
controlPlaneRefresh: compactControlPlaneRefresh(controlPlaneRefresh),
|
|
gitMirrorPreSync,
|
|
createPipelineRun: compactTriggerCommandResult(compactCommandResult(createPipelineRun)),
|
|
after: getPipelineRunCompact(defaultRuntimeLanePipelineRunName(sourceCommit)),
|
|
latestOnlyPolicy: "no old PipelineRun is canceled; stale commits self-supersede before GitOps writeback",
|
|
disclosure: {
|
|
fullTriggerOutput: "Use the async job stdout/stderr files from job status for full command details.",
|
|
},
|
|
};
|
|
}
|