fix: refresh hwlab follower control plane natively
This commit is contained in:
@@ -38,6 +38,8 @@ It must not copy runtime/GitOps/Secret details from owning configs:
|
||||
|
||||
Use configRef summaries in plan/status; do not create a `full.md` or super Markdown index.
|
||||
|
||||
Timeout, TTL, retry/backoff, reconcile interval and end-to-end budget values must be declared in YAML/source-of-truth fields. Do not introduce hidden numeric defaults in TypeScript, shell, native helper scripts, or controller manifests; helper code should read the configured values and fail structurally when required timing policy is missing.
|
||||
|
||||
## First Followers
|
||||
|
||||
- `hwlab-jd01-v03`: follows `pikasTech/HWLAB@v0.3`, adapter `hwlab-node-runtime`, native trigger `Tekton PipelineRun -> Argo Application closeout -> runtime Deployment sourceCommit readiness`.
|
||||
|
||||
@@ -44,10 +44,15 @@ controller:
|
||||
loop:
|
||||
intervalSeconds: 30
|
||||
reconcileTimeoutSeconds: 110
|
||||
leaseDurationSeconds: 140
|
||||
budgets:
|
||||
applyWaitSeconds: 120
|
||||
statusSeconds: 35
|
||||
runOnceSeconds: 120
|
||||
reconcileJobTtlSeconds: 600
|
||||
reconcileJobBackoffLimit: 0
|
||||
reconcileJobDeadlineGraceSeconds: 30
|
||||
reconcileTransportGraceSeconds: 20
|
||||
|
||||
followers:
|
||||
- id: hwlab-jd01-v03
|
||||
@@ -76,6 +81,9 @@ followers:
|
||||
statusSeconds: 35
|
||||
triggerSeconds: 120
|
||||
sourceSyncSeconds: 20
|
||||
controlPlaneRefreshSeconds: 45
|
||||
capabilityJobTtlSeconds: 600
|
||||
capabilityJobBackoffLimit: 0
|
||||
commands:
|
||||
plan:
|
||||
argv: ["bun", "scripts/cli.ts", "hwlab", "nodes", "control-plane", "trigger-current", "--node", "JD01", "--lane", "v03", "--dry-run", "--raw"]
|
||||
@@ -145,6 +153,9 @@ followers:
|
||||
statusSeconds: 35
|
||||
triggerSeconds: 120
|
||||
sourceSyncSeconds: 20
|
||||
controlPlaneRefreshSeconds: 20
|
||||
capabilityJobTtlSeconds: 600
|
||||
capabilityJobBackoffLimit: 0
|
||||
commands:
|
||||
plan:
|
||||
argv: ["bun", "scripts/cli.ts", "agentrun", "control-plane", "trigger-current", "--node", "JD01", "--lane", "jd01-v02", "--dry-run", "--raw"]
|
||||
@@ -210,6 +221,9 @@ followers:
|
||||
statusSeconds: 35
|
||||
triggerSeconds: 120
|
||||
sourceSyncSeconds: 20
|
||||
controlPlaneRefreshSeconds: 20
|
||||
capabilityJobTtlSeconds: 600
|
||||
capabilityJobBackoffLimit: 0
|
||||
commands:
|
||||
plan:
|
||||
argv: ["bun", "scripts/cli.ts", "web-probe", "sentinel", "publish-current", "--node", "JD01", "--lane", "v03", "--sentinel", "jd01-web-probe-sentinel", "--dry-run", "--raw"]
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env node
|
||||
// SPEC: PJ2026-01060703 CI/CD branch follower native HWLAB control-plane refresh.
|
||||
// Responsibility: render and apply the HWLAB node Tekton Pipeline from a k8s git-mirror snapshot.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const startedAt = Date.now();
|
||||
const sourceCommit = requiredEnv("SOURCE_COMMIT");
|
||||
const sourceStageRef = requiredEnv("SOURCE_STAGE_REF");
|
||||
const gitReadUrl = requiredEnv("GIT_READ_URL");
|
||||
const fieldManager = requiredEnv("FIELD_MANAGER");
|
||||
const tektonNamespace = requiredEnv("TEKTON_NAMESPACE");
|
||||
const overlay = JSON.parse(Buffer.from(requiredEnv("HWLAB_RENDER_OVERLAY_B64"), "base64").toString("utf8"));
|
||||
const workDir = mkdtempSync(path.join(tmpdir(), `hwlab-control-plane-${sourceCommit.slice(0, 12)}-`));
|
||||
const repoDir = path.join(workDir, "repo");
|
||||
const renderDir = path.join(workDir, "render");
|
||||
|
||||
try {
|
||||
checkoutSnapshot();
|
||||
prepareYamlDependency();
|
||||
applyDeployOverlay();
|
||||
renderControlPlane();
|
||||
applyPipeline();
|
||||
emit({ ok: true, status: "applied" });
|
||||
} finally {
|
||||
rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function checkoutSnapshot() {
|
||||
run("git", ["init", repoDir], { cwd: "/" });
|
||||
run("git", ["-C", repoDir, "remote", "add", "origin", gitReadUrl]);
|
||||
run("git", ["-C", repoDir, "fetch", "--depth", "1", "origin", sourceStageRef]);
|
||||
run("git", ["-C", repoDir, "checkout", "--detach", "FETCH_HEAD"]);
|
||||
const head = capture("git", ["-C", repoDir, "rev-parse", "HEAD"]).trim();
|
||||
if (head !== sourceCommit) throw new Error(`snapshot checkout mismatch: expected ${sourceCommit}, got ${head}`);
|
||||
}
|
||||
|
||||
function prepareYamlDependency() {
|
||||
if (canResolveYaml()) return;
|
||||
const registry = requiredOverlayString("npmRegistry");
|
||||
const timeoutMs = String(requiredPositiveNumber("npmFetchTimeoutMs"));
|
||||
const retries = String(requiredNonNegativeNumber("npmRetries"));
|
||||
const yamlSpec = yamlDependencySpec();
|
||||
const env = renderEnv({
|
||||
npm_config_registry: registry,
|
||||
BUN_CONFIG_REGISTRY: registry,
|
||||
npm_config_fetch_timeout: timeoutMs,
|
||||
npm_config_fetch_retries: retries,
|
||||
});
|
||||
if (commandExists("bun")) {
|
||||
const result = runOptional("bun", ["add", "--no-save", "--ignore-scripts", "--registry", registry, yamlSpec], { cwd: repoDir, env });
|
||||
if (result.status === 0 && canResolveYaml()) return;
|
||||
}
|
||||
run("npm", ["install", "--package-lock=false", "--no-save", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev", "--registry", registry, yamlSpec], { cwd: repoDir, env });
|
||||
if (!canResolveYaml()) throw new Error("yaml dependency remains unresolved after install");
|
||||
}
|
||||
|
||||
function yamlDependencySpec() {
|
||||
const pkg = JSON.parse(readFileSync(path.join(repoDir, "package.json"), "utf8"));
|
||||
const version = pkg.dependencies?.yaml || pkg.devDependencies?.yaml;
|
||||
if (typeof version !== "string" || version.length === 0) throw new Error("package.json must declare yaml dependency for gitops render");
|
||||
return `yaml@${version}`;
|
||||
}
|
||||
|
||||
function canResolveYaml() {
|
||||
const result = runOptional("node", ["-e", "require.resolve('yaml')"], { cwd: repoDir, stdio: "ignore" });
|
||||
return result.status === 0;
|
||||
}
|
||||
|
||||
function applyDeployOverlay() {
|
||||
const requireFromRepo = createRequire(path.join(repoDir, "package.json"));
|
||||
const YAML = requireFromRepo("yaml");
|
||||
const deployPath = path.join(repoDir, "deploy/deploy.yaml");
|
||||
const doc = YAML.parse(readFileSync(deployPath, "utf8"));
|
||||
doc.nodes = doc.nodes || {};
|
||||
doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };
|
||||
doc.lanes = doc.lanes || {};
|
||||
const lane = doc.lanes[overlay.lane] || {};
|
||||
const downloadStack = {
|
||||
...(lane.envRecipe?.downloadStack || {}),
|
||||
httpProxy: overlay.dockerProxyHttp,
|
||||
httpsProxy: overlay.dockerProxyHttps,
|
||||
noProxy: overlay.dockerNoProxyList,
|
||||
};
|
||||
doc.lanes[overlay.lane] = {
|
||||
...lane,
|
||||
node: overlay.nodeId,
|
||||
sourceBranch: overlay.sourceBranch,
|
||||
gitopsBranch: overlay.gitopsBranch,
|
||||
namespace: overlay.runtimeNamespace,
|
||||
endpoint: overlay.publicApiUrl,
|
||||
publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },
|
||||
artifactCatalog: overlay.catalogPath,
|
||||
runtimePath: overlay.runtimePath,
|
||||
imageTagMode: "full",
|
||||
sourceRepo: overlay.gitUrl,
|
||||
externalPostgres: overlay.externalPostgres,
|
||||
observability: overlay.observability,
|
||||
envRecipe: { ...(lane.envRecipe || {}), downloadStack },
|
||||
};
|
||||
if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;
|
||||
if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;
|
||||
if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;
|
||||
writeFileSync(deployPath, YAML.stringify(doc), "utf8");
|
||||
}
|
||||
|
||||
function renderControlPlane() {
|
||||
run("node", [
|
||||
"scripts/run-bun.mjs",
|
||||
"scripts/gitops-render.mjs",
|
||||
"--lane", overlay.lane,
|
||||
"--node", overlay.nodeId,
|
||||
"--gitops-root", overlay.gitopsRoot,
|
||||
"--catalog-path", overlay.catalogPath,
|
||||
"--image-tag-mode", "full",
|
||||
"--source-revision", sourceCommit,
|
||||
"--source-repo", overlay.gitUrl,
|
||||
"--source-branch", overlay.sourceBranch,
|
||||
"--gitops-branch", overlay.gitopsBranch,
|
||||
"--git-read-url", overlay.gitReadUrl,
|
||||
"--git-write-url", overlay.gitWriteUrl,
|
||||
"--registry-prefix", overlay.registryPrefix,
|
||||
"--runtime-endpoint", overlay.publicApiUrl,
|
||||
"--web-endpoint", overlay.publicWebUrl,
|
||||
"--out", renderDir,
|
||||
], { cwd: repoDir, env: renderEnv() });
|
||||
}
|
||||
|
||||
function applyPipeline() {
|
||||
const pipelinePath = path.join(renderDir, overlay.tektonDir, "pipeline.yaml");
|
||||
if (!existsSync(pipelinePath)) throw new Error(`rendered Pipeline missing: ${pipelinePath}`);
|
||||
run("kubectl", ["apply", "--server-side", "--force-conflicts", `--field-manager=${fieldManager}`, "-f", pipelinePath], { cwd: repoDir, env: renderEnv() });
|
||||
}
|
||||
|
||||
function renderEnv(extra = {}) {
|
||||
const noProxy = overlay.noProxy || process.env.NO_PROXY || process.env.no_proxy || "";
|
||||
return {
|
||||
...process.env,
|
||||
HTTP_PROXY: overlay.proxyHttp || process.env.HTTP_PROXY || "",
|
||||
HTTPS_PROXY: overlay.proxyHttps || process.env.HTTPS_PROXY || "",
|
||||
ALL_PROXY: overlay.proxyAll || process.env.ALL_PROXY || "",
|
||||
NO_PROXY: noProxy,
|
||||
http_proxy: overlay.proxyHttp || process.env.http_proxy || "",
|
||||
https_proxy: overlay.proxyHttps || process.env.https_proxy || "",
|
||||
all_proxy: overlay.proxyAll || process.env.all_proxy || "",
|
||||
no_proxy: noProxy,
|
||||
HWLAB_NODE_CI_TOOLS_IMAGE: overlay.toolsImage || process.env.HWLAB_NODE_CI_TOOLS_IMAGE || "",
|
||||
HWLAB_NODE_BUILDKIT_IMAGE: overlay.buildkitSidecarImage || process.env.HWLAB_NODE_BUILDKIT_IMAGE || "",
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function run(command, args, options = {}) {
|
||||
const result = runOptional(command, args, options);
|
||||
if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function capture(command, args) {
|
||||
const result = runOptional(command, args, { cwd: repoDir, encoding: "utf8", stdio: ["ignore", "pipe", "inherit"] });
|
||||
if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`);
|
||||
return result.stdout || "";
|
||||
}
|
||||
|
||||
function runOptional(command, args, options = {}) {
|
||||
return spawnSync(command, args, {
|
||||
cwd: options.cwd || repoDir,
|
||||
env: options.env || process.env,
|
||||
encoding: options.encoding || "utf8",
|
||||
stdio: options.stdio || "inherit",
|
||||
});
|
||||
}
|
||||
|
||||
function commandExists(command) {
|
||||
return runOptional("sh", ["-lc", `command -v ${command} >/dev/null 2>&1`], { cwd: repoDir, stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
function requiredEnv(name) {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(`${name} is required`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredOverlayString(name) {
|
||||
const value = overlay[name];
|
||||
if (typeof value !== "string" || value.length === 0) throw new Error(`overlay.${name} is required`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredPositiveNumber(name) {
|
||||
const value = Number(overlay[name]);
|
||||
if (!Number.isFinite(value) || value <= 0) throw new Error(`overlay.${name} must be a positive number`);
|
||||
return Math.floor(value);
|
||||
}
|
||||
|
||||
function requiredNonNegativeNumber(name) {
|
||||
const value = Number(overlay[name]);
|
||||
if (!Number.isFinite(value) || value < 0) throw new Error(`overlay.${name} must be a non-negative number`);
|
||||
return Math.floor(value);
|
||||
}
|
||||
|
||||
function emit(extra) {
|
||||
process.stdout.write(`${JSON.stringify({
|
||||
...extra,
|
||||
sourceCommit,
|
||||
sourceStageRef,
|
||||
pipeline: overlay.pipelineName,
|
||||
namespace: tektonNamespace,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
sourceAuthority: "k8s-git-mirror-snapshot",
|
||||
statusAuthority: "kubernetes-api-serviceaccount",
|
||||
parsedDownstreamCliOutput: false,
|
||||
})}\n`);
|
||||
}
|
||||
@@ -30,9 +30,9 @@ export function renderControllerReconcileJob(registry: BranchFollowerRegistry, o
|
||||
kind: "Job",
|
||||
metadata: { name: jobName, namespace: registry.controller.namespace, labels },
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
ttlSecondsAfterFinished: 600,
|
||||
activeDeadlineSeconds: timeoutSeconds + 30,
|
||||
backoffLimit: registry.controller.budgets.reconcileJobBackoffLimit,
|
||||
ttlSecondsAfterFinished: registry.controller.budgets.reconcileJobTtlSeconds,
|
||||
activeDeadlineSeconds: timeoutSeconds + registry.controller.budgets.reconcileJobDeadlineGraceSeconds,
|
||||
template: {
|
||||
metadata: { labels },
|
||||
spec: {
|
||||
@@ -125,6 +125,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec
|
||||
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "delete"] },
|
||||
{ apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "watch", "create", "patch", "delete"] },
|
||||
{ apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "list", "watch", "create", "update", "patch"] },
|
||||
{ apiGroups: ["tekton.dev"], resources: ["taskruns"], verbs: ["get", "list", "watch"] },
|
||||
{ apiGroups: ["argoproj.io"], resources: ["applications"], verbs: ["get", "list", "watch", "patch"] },
|
||||
],
|
||||
@@ -145,6 +146,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec
|
||||
"sync-source.sh": nativeCicdScript("sync-source.sh"),
|
||||
"controller-one-shot.sh": nativeCicdScript("controller-one-shot.sh"),
|
||||
"controller-loop.sh": nativeCicdScript("controller-loop.sh"),
|
||||
"hwlab-node-control-plane-refresh.mjs": nativeCicdScript("hwlab-node-control-plane-refresh.mjs"),
|
||||
"github-proxy-connect.mjs": nativeCicdScript("github-proxy-connect.mjs"),
|
||||
"git-ssh-proxy.sh": nativeCicdScript("git-ssh-proxy.sh"),
|
||||
},
|
||||
@@ -163,7 +165,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec
|
||||
apiVersion: "coordination.k8s.io/v1",
|
||||
kind: "Lease",
|
||||
metadata: { name: registry.controller.leaseName, namespace: registry.controller.namespace, labels },
|
||||
spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: Math.max(30, registry.controller.loop.reconcileTimeoutSeconds + 30) },
|
||||
spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: registry.controller.loop.leaseDurationSeconds },
|
||||
},
|
||||
{
|
||||
apiVersion: "apps/v1",
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
// SPEC: PJ2026-01060703 CI/CD branch follower HWLAB native refresh.
|
||||
// Responsibility: create the Kubernetes Job that refreshes HWLAB node Tekton Pipeline from a git-mirror snapshot.
|
||||
import { runNativeK8sJob } from "./cicd-native";
|
||||
import type { BranchFollowerRegistry, FollowerSpec, NativeK8sJobResult } from "./cicd-types";
|
||||
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
||||
import { nodeRuntimeGitMirrorTarget, nodeRuntimeRenderOverlay } from "./hwlab-node/web-probe";
|
||||
|
||||
export function runNativeHwlabControlPlaneRefresh(
|
||||
registry: BranchFollowerRegistry,
|
||||
follower: FollowerSpec,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
observedSha: string,
|
||||
timeoutSeconds: number,
|
||||
jobName: string,
|
||||
): { jobName: string; namespace: string; result: NativeK8sJobResult } {
|
||||
const namespace = registry.controller.namespace;
|
||||
const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh");
|
||||
return { jobName, namespace, result };
|
||||
}
|
||||
|
||||
function nativeHwlabControlPlaneRefreshJobManifest(
|
||||
registry: BranchFollowerRegistry,
|
||||
follower: FollowerSpec,
|
||||
spec: HwlabRuntimeLaneSpec,
|
||||
observedSha: string,
|
||||
jobName: string,
|
||||
): Record<string, unknown> {
|
||||
const mirror = nodeRuntimeGitMirrorTarget(spec);
|
||||
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
|
||||
const sourceStageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
|
||||
const tektonNamespace = follower.nativeStatus.tekton?.namespace;
|
||||
if (tektonNamespace === undefined) throw new Error(`follower ${follower.id} nativeStatus.tekton.namespace is required for HWLAB control-plane refresh`);
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: {
|
||||
name: jobName,
|
||||
namespace: registry.controller.namespace,
|
||||
labels: {
|
||||
"app.kubernetes.io/name": "hwlab-node-control-plane-refresh",
|
||||
"app.kubernetes.io/part-of": "unidesk-cicd-branch-follower",
|
||||
"app.kubernetes.io/component": "control-plane-refresh",
|
||||
"hwlab.pikastech.local/node": spec.nodeId,
|
||||
"hwlab.pikastech.local/lane": spec.lane,
|
||||
"hwlab.pikastech.local/source-commit": observedSha,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
backoffLimit: follower.budgets.capabilityJobBackoffLimit,
|
||||
ttlSecondsAfterFinished: follower.budgets.capabilityJobTtlSeconds,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
"app.kubernetes.io/name": "hwlab-node-control-plane-refresh",
|
||||
"app.kubernetes.io/part-of": "unidesk-cicd-branch-follower",
|
||||
"app.kubernetes.io/component": "control-plane-refresh",
|
||||
"hwlab.pikastech.local/node": spec.nodeId,
|
||||
"hwlab.pikastech.local/lane": spec.lane,
|
||||
"hwlab.pikastech.local/source-commit": observedSha,
|
||||
},
|
||||
},
|
||||
spec: {
|
||||
restartPolicy: "Never",
|
||||
serviceAccountName: registry.controller.serviceAccountName,
|
||||
hostNetwork: true,
|
||||
dnsPolicy: "ClusterFirstWithHostNet",
|
||||
volumes: [
|
||||
{ name: "native-scripts", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } },
|
||||
],
|
||||
containers: [{
|
||||
name: "control-plane-refresh",
|
||||
image: mirror.toolsImage,
|
||||
imagePullPolicy: mirror.toolsImagePullPolicy,
|
||||
command: ["node", "/etc/unidesk-cicd-branch-follower/hwlab-node-control-plane-refresh.mjs"],
|
||||
volumeMounts: [
|
||||
{ name: "native-scripts", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true },
|
||||
],
|
||||
env: [
|
||||
{ name: "SOURCE_COMMIT", value: observedSha },
|
||||
{ name: "SOURCE_STAGE_REF", value: sourceStageRef },
|
||||
{ name: "GIT_READ_URL", value: spec.gitReadUrl },
|
||||
{ name: "FIELD_MANAGER", value: spec.controlPlaneFieldManager },
|
||||
{ name: "TEKTON_NAMESPACE", value: tektonNamespace },
|
||||
{ name: "HWLAB_RENDER_OVERLAY_B64", value: overlay },
|
||||
],
|
||||
}],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -65,6 +65,9 @@ export interface FollowerSpec {
|
||||
statusSeconds: number;
|
||||
triggerSeconds: number;
|
||||
sourceSyncSeconds: number;
|
||||
controlPlaneRefreshSeconds: number;
|
||||
capabilityJobTtlSeconds: number;
|
||||
capabilityJobBackoffLimit: number;
|
||||
};
|
||||
commands: {
|
||||
plan: CommandSpec;
|
||||
@@ -148,11 +151,16 @@ export interface ControllerSpec {
|
||||
loop: {
|
||||
intervalSeconds: number;
|
||||
reconcileTimeoutSeconds: number;
|
||||
leaseDurationSeconds: number;
|
||||
};
|
||||
budgets: {
|
||||
applyWaitSeconds: number;
|
||||
statusSeconds: number;
|
||||
runOnceSeconds: number;
|
||||
reconcileJobTtlSeconds: number;
|
||||
reconcileJobBackoffLimit: number;
|
||||
reconcileJobDeadlineGraceSeconds: number;
|
||||
reconcileTransportGraceSeconds: number;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+20
-5
@@ -21,6 +21,7 @@ import { sentinelPipelineRunName } from "./hwlab-node-web-sentinel-cicd-shared";
|
||||
import { transPath } from "./hwlab-node/runtime-common";
|
||||
import { configRefGraph, resolveConfigRefString } from "./ops/config-refs";
|
||||
import { renderControllerManifests, renderControllerReconcileJob, waitForJobShell } from "./cicd-controller-render";
|
||||
import { runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh";
|
||||
import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native";
|
||||
import type { AdapterSummary, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeObjectBundle, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types";
|
||||
import {
|
||||
@@ -278,11 +279,16 @@ function parseController(root: Record<string, unknown>): ControllerSpec {
|
||||
loop: {
|
||||
intervalSeconds: integerField(loop, "intervalSeconds", "controller.loop"),
|
||||
reconcileTimeoutSeconds: integerField(loop, "reconcileTimeoutSeconds", "controller.loop"),
|
||||
leaseDurationSeconds: integerField(loop, "leaseDurationSeconds", "controller.loop"),
|
||||
},
|
||||
budgets: {
|
||||
applyWaitSeconds: integerField(budgets, "applyWaitSeconds", "controller.budgets"),
|
||||
statusSeconds: integerField(budgets, "statusSeconds", "controller.budgets"),
|
||||
runOnceSeconds: integerField(budgets, "runOnceSeconds", "controller.budgets"),
|
||||
reconcileJobTtlSeconds: integerField(budgets, "reconcileJobTtlSeconds", "controller.budgets"),
|
||||
reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"),
|
||||
reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"),
|
||||
reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"),
|
||||
},
|
||||
};
|
||||
if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) {
|
||||
@@ -334,6 +340,9 @@ function parseFollower(root: Record<string, unknown>, index: number): FollowerSp
|
||||
statusSeconds: integerField(budgets, "statusSeconds", `${label}.budgets`),
|
||||
triggerSeconds: integerField(budgets, "triggerSeconds", `${label}.budgets`),
|
||||
sourceSyncSeconds: integerField(budgets, "sourceSyncSeconds", `${label}.budgets`),
|
||||
controlPlaneRefreshSeconds: integerField(budgets, "controlPlaneRefreshSeconds", `${label}.budgets`),
|
||||
capabilityJobTtlSeconds: integerField(budgets, "capabilityJobTtlSeconds", `${label}.budgets`),
|
||||
capabilityJobBackoffLimit: integerField(budgets, "capabilityJobBackoffLimit", `${label}.budgets`),
|
||||
},
|
||||
commands: {
|
||||
plan: parseCommand(recordField(commands, "plan", `${label}.commands`), `${label}.commands.plan`),
|
||||
@@ -801,7 +810,7 @@ async function decideAndMaybeTrigger(
|
||||
}
|
||||
if (!trigger.ok) warnings.push(trigger.message);
|
||||
}
|
||||
if (options.confirm && options.wait && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) {
|
||||
if (options.confirm && (options.wait || options.inCluster) && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) {
|
||||
const closeout = await waitNativeFollowerCloseout(registry, follower, observedSha, options, options.timeoutSeconds ?? follower.budgets.endToEndSeconds);
|
||||
triggerCommand = closeoutOnlyCommand(follower, live.pipelineRun, observedSha, closeout);
|
||||
if (closeout.completed) {
|
||||
@@ -914,10 +923,14 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
|
||||
const namespace = follower.nativeStatus.tekton.namespace;
|
||||
const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun);
|
||||
const startedAt = Date.now();
|
||||
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds)));
|
||||
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
||||
if (sync !== null && !sync.result.ok) {
|
||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt);
|
||||
}
|
||||
const refresh = runNativeHwlabControlPlaneRefresh(registry, follower, spec, observedSha, Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.controlPlaneRefreshSeconds), nativeCapabilityJobName(follower.id, "control-plane-refresh", observedSha));
|
||||
if (!refresh.result.ok) {
|
||||
return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt);
|
||||
}
|
||||
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds));
|
||||
const payload = parseJsonObject(result.stdout) ?? {};
|
||||
const pipelineRunCompleted = payload.completed === true;
|
||||
@@ -944,6 +957,7 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
|
||||
...payload,
|
||||
nativeCapabilities: {
|
||||
gitMirrorSync: sync === null ? null : sync.result,
|
||||
controlPlaneRefresh: refresh.result,
|
||||
gitMirrorFlush: flush === null ? null : flush.result,
|
||||
},
|
||||
},
|
||||
@@ -963,7 +977,7 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo
|
||||
const startedAt = Date.now();
|
||||
const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
|
||||
const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
|
||||
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds)));
|
||||
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
||||
if (sync !== null && !sync.result.ok) {
|
||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt);
|
||||
}
|
||||
@@ -1580,7 +1594,7 @@ function readNativeObjectBundle(registry: BranchFollowerRegistry, follower: Foll
|
||||
"\"$tmpdir/read-native-bundle.sh\"",
|
||||
].join("\n");
|
||||
const startedAt = Date.now();
|
||||
const result = runKubeScript(registry, options, script, "", Math.max(5, timeoutSeconds) * 1000);
|
||||
const result = runKubeScript(registry, options, script, "", timeoutSeconds * 1000);
|
||||
const parsed = parseNativeBundleLines(result.stdout);
|
||||
const sourceRecord = asOptionalRecord(parsed.objects.source);
|
||||
return {
|
||||
@@ -2367,6 +2381,7 @@ function stageTimingsFromCommand(command: Record<string, unknown> | undefined):
|
||||
const capabilities = asOptionalRecord(payload.nativeCapabilities);
|
||||
for (const stage of [
|
||||
k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)),
|
||||
k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)),
|
||||
k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)),
|
||||
]) {
|
||||
if (stage !== null) stages.push(stage);
|
||||
@@ -2518,7 +2533,7 @@ function runControllerReconcileJob(registry: BranchFollowerRegistry, options: Pa
|
||||
`kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`,
|
||||
mode.wait ? waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds) : "true",
|
||||
].join("\n");
|
||||
const result = runKubeScript(registry, options, script, "", (timeoutSeconds + 20) * 1000);
|
||||
const result = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000);
|
||||
return {
|
||||
ok: result.exitCode === 0,
|
||||
name: jobName,
|
||||
|
||||
Reference in New Issue
Block a user