fix: refresh hwlab follower control plane natively

This commit is contained in:
Codex
2026-07-03 13:55:13 +00:00
parent 63040ba28c
commit 7751cc167b
7 changed files with 358 additions and 9 deletions
@@ -38,6 +38,8 @@ It must not copy runtime/GitOps/Secret details from owning configs:
Use configRef summaries in plan/status; do not create a `full.md` or super Markdown index.
Timeout, TTL, retry/backoff, reconcile interval and end-to-end budget values must be declared in YAML/source-of-truth fields. Do not introduce hidden numeric defaults in TypeScript, shell, native helper scripts, or controller manifests; helper code should read the configured values and fail structurally when required timing policy is missing.
## First Followers
- `hwlab-jd01-v03`: follows `pikasTech/HWLAB@v0.3`, adapter `hwlab-node-runtime`, native trigger `Tekton PipelineRun -> Argo Application closeout -> runtime Deployment sourceCommit readiness`.
+14
View File
@@ -44,10 +44,15 @@ controller:
loop:
intervalSeconds: 30
reconcileTimeoutSeconds: 110
leaseDurationSeconds: 140
budgets:
applyWaitSeconds: 120
statusSeconds: 35
runOnceSeconds: 120
reconcileJobTtlSeconds: 600
reconcileJobBackoffLimit: 0
reconcileJobDeadlineGraceSeconds: 30
reconcileTransportGraceSeconds: 20
followers:
- id: hwlab-jd01-v03
@@ -76,6 +81,9 @@ followers:
statusSeconds: 35
triggerSeconds: 120
sourceSyncSeconds: 20
controlPlaneRefreshSeconds: 45
capabilityJobTtlSeconds: 600
capabilityJobBackoffLimit: 0
commands:
plan:
argv: ["bun", "scripts/cli.ts", "hwlab", "nodes", "control-plane", "trigger-current", "--node", "JD01", "--lane", "v03", "--dry-run", "--raw"]
@@ -145,6 +153,9 @@ followers:
statusSeconds: 35
triggerSeconds: 120
sourceSyncSeconds: 20
controlPlaneRefreshSeconds: 20
capabilityJobTtlSeconds: 600
capabilityJobBackoffLimit: 0
commands:
plan:
argv: ["bun", "scripts/cli.ts", "agentrun", "control-plane", "trigger-current", "--node", "JD01", "--lane", "jd01-v02", "--dry-run", "--raw"]
@@ -210,6 +221,9 @@ followers:
statusSeconds: 35
triggerSeconds: 120
sourceSyncSeconds: 20
controlPlaneRefreshSeconds: 20
capabilityJobTtlSeconds: 600
capabilityJobBackoffLimit: 0
commands:
plan:
argv: ["bun", "scripts/cli.ts", "web-probe", "sentinel", "publish-current", "--node", "JD01", "--lane", "v03", "--sentinel", "jd01-web-probe-sentinel", "--dry-run", "--raw"]
@@ -0,0 +1,217 @@
#!/usr/bin/env node
// SPEC: PJ2026-01060703 CI/CD branch follower native HWLAB control-plane refresh.
// Responsibility: render and apply the HWLAB node Tekton Pipeline from a k8s git-mirror snapshot.
import { spawnSync } from "node:child_process";
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { createRequire } from "node:module";
import { tmpdir } from "node:os";
import path from "node:path";
const startedAt = Date.now();
const sourceCommit = requiredEnv("SOURCE_COMMIT");
const sourceStageRef = requiredEnv("SOURCE_STAGE_REF");
const gitReadUrl = requiredEnv("GIT_READ_URL");
const fieldManager = requiredEnv("FIELD_MANAGER");
const tektonNamespace = requiredEnv("TEKTON_NAMESPACE");
const overlay = JSON.parse(Buffer.from(requiredEnv("HWLAB_RENDER_OVERLAY_B64"), "base64").toString("utf8"));
const workDir = mkdtempSync(path.join(tmpdir(), `hwlab-control-plane-${sourceCommit.slice(0, 12)}-`));
const repoDir = path.join(workDir, "repo");
const renderDir = path.join(workDir, "render");
try {
checkoutSnapshot();
prepareYamlDependency();
applyDeployOverlay();
renderControlPlane();
applyPipeline();
emit({ ok: true, status: "applied" });
} finally {
rmSync(workDir, { recursive: true, force: true });
}
function checkoutSnapshot() {
run("git", ["init", repoDir], { cwd: "/" });
run("git", ["-C", repoDir, "remote", "add", "origin", gitReadUrl]);
run("git", ["-C", repoDir, "fetch", "--depth", "1", "origin", sourceStageRef]);
run("git", ["-C", repoDir, "checkout", "--detach", "FETCH_HEAD"]);
const head = capture("git", ["-C", repoDir, "rev-parse", "HEAD"]).trim();
if (head !== sourceCommit) throw new Error(`snapshot checkout mismatch: expected ${sourceCommit}, got ${head}`);
}
function prepareYamlDependency() {
if (canResolveYaml()) return;
const registry = requiredOverlayString("npmRegistry");
const timeoutMs = String(requiredPositiveNumber("npmFetchTimeoutMs"));
const retries = String(requiredNonNegativeNumber("npmRetries"));
const yamlSpec = yamlDependencySpec();
const env = renderEnv({
npm_config_registry: registry,
BUN_CONFIG_REGISTRY: registry,
npm_config_fetch_timeout: timeoutMs,
npm_config_fetch_retries: retries,
});
if (commandExists("bun")) {
const result = runOptional("bun", ["add", "--no-save", "--ignore-scripts", "--registry", registry, yamlSpec], { cwd: repoDir, env });
if (result.status === 0 && canResolveYaml()) return;
}
run("npm", ["install", "--package-lock=false", "--no-save", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev", "--registry", registry, yamlSpec], { cwd: repoDir, env });
if (!canResolveYaml()) throw new Error("yaml dependency remains unresolved after install");
}
function yamlDependencySpec() {
const pkg = JSON.parse(readFileSync(path.join(repoDir, "package.json"), "utf8"));
const version = pkg.dependencies?.yaml || pkg.devDependencies?.yaml;
if (typeof version !== "string" || version.length === 0) throw new Error("package.json must declare yaml dependency for gitops render");
return `yaml@${version}`;
}
function canResolveYaml() {
const result = runOptional("node", ["-e", "require.resolve('yaml')"], { cwd: repoDir, stdio: "ignore" });
return result.status === 0;
}
function applyDeployOverlay() {
const requireFromRepo = createRequire(path.join(repoDir, "package.json"));
const YAML = requireFromRepo("yaml");
const deployPath = path.join(repoDir, "deploy/deploy.yaml");
const doc = YAML.parse(readFileSync(deployPath, "utf8"));
doc.nodes = doc.nodes || {};
doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl };
doc.lanes = doc.lanes || {};
const lane = doc.lanes[overlay.lane] || {};
const downloadStack = {
...(lane.envRecipe?.downloadStack || {}),
httpProxy: overlay.dockerProxyHttp,
httpsProxy: overlay.dockerProxyHttps,
noProxy: overlay.dockerNoProxyList,
};
doc.lanes[overlay.lane] = {
...lane,
node: overlay.nodeId,
sourceBranch: overlay.sourceBranch,
gitopsBranch: overlay.gitopsBranch,
namespace: overlay.runtimeNamespace,
endpoint: overlay.publicApiUrl,
publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl },
artifactCatalog: overlay.catalogPath,
runtimePath: overlay.runtimePath,
imageTagMode: "full",
sourceRepo: overlay.gitUrl,
externalPostgres: overlay.externalPostgres,
observability: overlay.observability,
envRecipe: { ...(lane.envRecipe || {}), downloadStack },
};
if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore;
if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime;
if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror;
writeFileSync(deployPath, YAML.stringify(doc), "utf8");
}
function renderControlPlane() {
run("node", [
"scripts/run-bun.mjs",
"scripts/gitops-render.mjs",
"--lane", overlay.lane,
"--node", overlay.nodeId,
"--gitops-root", overlay.gitopsRoot,
"--catalog-path", overlay.catalogPath,
"--image-tag-mode", "full",
"--source-revision", sourceCommit,
"--source-repo", overlay.gitUrl,
"--source-branch", overlay.sourceBranch,
"--gitops-branch", overlay.gitopsBranch,
"--git-read-url", overlay.gitReadUrl,
"--git-write-url", overlay.gitWriteUrl,
"--registry-prefix", overlay.registryPrefix,
"--runtime-endpoint", overlay.publicApiUrl,
"--web-endpoint", overlay.publicWebUrl,
"--out", renderDir,
], { cwd: repoDir, env: renderEnv() });
}
function applyPipeline() {
const pipelinePath = path.join(renderDir, overlay.tektonDir, "pipeline.yaml");
if (!existsSync(pipelinePath)) throw new Error(`rendered Pipeline missing: ${pipelinePath}`);
run("kubectl", ["apply", "--server-side", "--force-conflicts", `--field-manager=${fieldManager}`, "-f", pipelinePath], { cwd: repoDir, env: renderEnv() });
}
function renderEnv(extra = {}) {
const noProxy = overlay.noProxy || process.env.NO_PROXY || process.env.no_proxy || "";
return {
...process.env,
HTTP_PROXY: overlay.proxyHttp || process.env.HTTP_PROXY || "",
HTTPS_PROXY: overlay.proxyHttps || process.env.HTTPS_PROXY || "",
ALL_PROXY: overlay.proxyAll || process.env.ALL_PROXY || "",
NO_PROXY: noProxy,
http_proxy: overlay.proxyHttp || process.env.http_proxy || "",
https_proxy: overlay.proxyHttps || process.env.https_proxy || "",
all_proxy: overlay.proxyAll || process.env.all_proxy || "",
no_proxy: noProxy,
HWLAB_NODE_CI_TOOLS_IMAGE: overlay.toolsImage || process.env.HWLAB_NODE_CI_TOOLS_IMAGE || "",
HWLAB_NODE_BUILDKIT_IMAGE: overlay.buildkitSidecarImage || process.env.HWLAB_NODE_BUILDKIT_IMAGE || "",
...extra,
};
}
function run(command, args, options = {}) {
const result = runOptional(command, args, options);
if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`);
return result;
}
function capture(command, args) {
const result = runOptional(command, args, { cwd: repoDir, encoding: "utf8", stdio: ["ignore", "pipe", "inherit"] });
if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`);
return result.stdout || "";
}
function runOptional(command, args, options = {}) {
return spawnSync(command, args, {
cwd: options.cwd || repoDir,
env: options.env || process.env,
encoding: options.encoding || "utf8",
stdio: options.stdio || "inherit",
});
}
function commandExists(command) {
return runOptional("sh", ["-lc", `command -v ${command} >/dev/null 2>&1`], { cwd: repoDir, stdio: "ignore" }).status === 0;
}
function requiredEnv(name) {
const value = process.env[name];
if (!value) throw new Error(`${name} is required`);
return value;
}
function requiredOverlayString(name) {
const value = overlay[name];
if (typeof value !== "string" || value.length === 0) throw new Error(`overlay.${name} is required`);
return value;
}
function requiredPositiveNumber(name) {
const value = Number(overlay[name]);
if (!Number.isFinite(value) || value <= 0) throw new Error(`overlay.${name} must be a positive number`);
return Math.floor(value);
}
function requiredNonNegativeNumber(name) {
const value = Number(overlay[name]);
if (!Number.isFinite(value) || value < 0) throw new Error(`overlay.${name} must be a non-negative number`);
return Math.floor(value);
}
function emit(extra) {
process.stdout.write(`${JSON.stringify({
...extra,
sourceCommit,
sourceStageRef,
pipeline: overlay.pipelineName,
namespace: tektonNamespace,
elapsedMs: Date.now() - startedAt,
sourceAuthority: "k8s-git-mirror-snapshot",
statusAuthority: "kubernetes-api-serviceaccount",
parsedDownstreamCliOutput: false,
})}\n`);
}
+6 -4
View File
@@ -30,9 +30,9 @@ export function renderControllerReconcileJob(registry: BranchFollowerRegistry, o
kind: "Job",
metadata: { name: jobName, namespace: registry.controller.namespace, labels },
spec: {
backoffLimit: 0,
ttlSecondsAfterFinished: 600,
activeDeadlineSeconds: timeoutSeconds + 30,
backoffLimit: registry.controller.budgets.reconcileJobBackoffLimit,
ttlSecondsAfterFinished: registry.controller.budgets.reconcileJobTtlSeconds,
activeDeadlineSeconds: timeoutSeconds + registry.controller.budgets.reconcileJobDeadlineGraceSeconds,
template: {
metadata: { labels },
spec: {
@@ -125,6 +125,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "delete"] },
{ apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] },
{ apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "watch", "create", "patch", "delete"] },
{ apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "list", "watch", "create", "update", "patch"] },
{ apiGroups: ["tekton.dev"], resources: ["taskruns"], verbs: ["get", "list", "watch"] },
{ apiGroups: ["argoproj.io"], resources: ["applications"], verbs: ["get", "list", "watch", "patch"] },
],
@@ -145,6 +146,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec
"sync-source.sh": nativeCicdScript("sync-source.sh"),
"controller-one-shot.sh": nativeCicdScript("controller-one-shot.sh"),
"controller-loop.sh": nativeCicdScript("controller-loop.sh"),
"hwlab-node-control-plane-refresh.mjs": nativeCicdScript("hwlab-node-control-plane-refresh.mjs"),
"github-proxy-connect.mjs": nativeCicdScript("github-proxy-connect.mjs"),
"git-ssh-proxy.sh": nativeCicdScript("git-ssh-proxy.sh"),
},
@@ -163,7 +165,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec
apiVersion: "coordination.k8s.io/v1",
kind: "Lease",
metadata: { name: registry.controller.leaseName, namespace: registry.controller.namespace, labels },
spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: Math.max(30, registry.controller.loop.reconcileTimeoutSeconds + 30) },
spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: registry.controller.loop.leaseDurationSeconds },
},
{
apiVersion: "apps/v1",
+91
View File
@@ -0,0 +1,91 @@
// SPEC: PJ2026-01060703 CI/CD branch follower HWLAB native refresh.
// Responsibility: create the Kubernetes Job that refreshes HWLAB node Tekton Pipeline from a git-mirror snapshot.
import { runNativeK8sJob } from "./cicd-native";
import type { BranchFollowerRegistry, FollowerSpec, NativeK8sJobResult } from "./cicd-types";
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
import { nodeRuntimeGitMirrorTarget, nodeRuntimeRenderOverlay } from "./hwlab-node/web-probe";
export function runNativeHwlabControlPlaneRefresh(
registry: BranchFollowerRegistry,
follower: FollowerSpec,
spec: HwlabRuntimeLaneSpec,
observedSha: string,
timeoutSeconds: number,
jobName: string,
): { jobName: string; namespace: string; result: NativeK8sJobResult } {
const namespace = registry.controller.namespace;
const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh");
return { jobName, namespace, result };
}
function nativeHwlabControlPlaneRefreshJobManifest(
registry: BranchFollowerRegistry,
follower: FollowerSpec,
spec: HwlabRuntimeLaneSpec,
observedSha: string,
jobName: string,
): Record<string, unknown> {
const mirror = nodeRuntimeGitMirrorTarget(spec);
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
const sourceStageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
const tektonNamespace = follower.nativeStatus.tekton?.namespace;
if (tektonNamespace === undefined) throw new Error(`follower ${follower.id} nativeStatus.tekton.namespace is required for HWLAB control-plane refresh`);
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: {
name: jobName,
namespace: registry.controller.namespace,
labels: {
"app.kubernetes.io/name": "hwlab-node-control-plane-refresh",
"app.kubernetes.io/part-of": "unidesk-cicd-branch-follower",
"app.kubernetes.io/component": "control-plane-refresh",
"hwlab.pikastech.local/node": spec.nodeId,
"hwlab.pikastech.local/lane": spec.lane,
"hwlab.pikastech.local/source-commit": observedSha,
},
},
spec: {
backoffLimit: follower.budgets.capabilityJobBackoffLimit,
ttlSecondsAfterFinished: follower.budgets.capabilityJobTtlSeconds,
template: {
metadata: {
labels: {
"app.kubernetes.io/name": "hwlab-node-control-plane-refresh",
"app.kubernetes.io/part-of": "unidesk-cicd-branch-follower",
"app.kubernetes.io/component": "control-plane-refresh",
"hwlab.pikastech.local/node": spec.nodeId,
"hwlab.pikastech.local/lane": spec.lane,
"hwlab.pikastech.local/source-commit": observedSha,
},
},
spec: {
restartPolicy: "Never",
serviceAccountName: registry.controller.serviceAccountName,
hostNetwork: true,
dnsPolicy: "ClusterFirstWithHostNet",
volumes: [
{ name: "native-scripts", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } },
],
containers: [{
name: "control-plane-refresh",
image: mirror.toolsImage,
imagePullPolicy: mirror.toolsImagePullPolicy,
command: ["node", "/etc/unidesk-cicd-branch-follower/hwlab-node-control-plane-refresh.mjs"],
volumeMounts: [
{ name: "native-scripts", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true },
],
env: [
{ name: "SOURCE_COMMIT", value: observedSha },
{ name: "SOURCE_STAGE_REF", value: sourceStageRef },
{ name: "GIT_READ_URL", value: spec.gitReadUrl },
{ name: "FIELD_MANAGER", value: spec.controlPlaneFieldManager },
{ name: "TEKTON_NAMESPACE", value: tektonNamespace },
{ name: "HWLAB_RENDER_OVERLAY_B64", value: overlay },
],
}],
},
},
},
};
}
+8
View File
@@ -65,6 +65,9 @@ export interface FollowerSpec {
statusSeconds: number;
triggerSeconds: number;
sourceSyncSeconds: number;
controlPlaneRefreshSeconds: number;
capabilityJobTtlSeconds: number;
capabilityJobBackoffLimit: number;
};
commands: {
plan: CommandSpec;
@@ -148,11 +151,16 @@ export interface ControllerSpec {
loop: {
intervalSeconds: number;
reconcileTimeoutSeconds: number;
leaseDurationSeconds: number;
};
budgets: {
applyWaitSeconds: number;
statusSeconds: number;
runOnceSeconds: number;
reconcileJobTtlSeconds: number;
reconcileJobBackoffLimit: number;
reconcileJobDeadlineGraceSeconds: number;
reconcileTransportGraceSeconds: number;
};
}
+20 -5
View File
@@ -21,6 +21,7 @@ import { sentinelPipelineRunName } from "./hwlab-node-web-sentinel-cicd-shared";
import { transPath } from "./hwlab-node/runtime-common";
import { configRefGraph, resolveConfigRefString } from "./ops/config-refs";
import { renderControllerManifests, renderControllerReconcileJob, waitForJobShell } from "./cicd-controller-render";
import { runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh";
import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native";
import type { AdapterSummary, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeObjectBundle, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types";
import {
@@ -278,11 +279,16 @@ function parseController(root: Record<string, unknown>): ControllerSpec {
loop: {
intervalSeconds: integerField(loop, "intervalSeconds", "controller.loop"),
reconcileTimeoutSeconds: integerField(loop, "reconcileTimeoutSeconds", "controller.loop"),
leaseDurationSeconds: integerField(loop, "leaseDurationSeconds", "controller.loop"),
},
budgets: {
applyWaitSeconds: integerField(budgets, "applyWaitSeconds", "controller.budgets"),
statusSeconds: integerField(budgets, "statusSeconds", "controller.budgets"),
runOnceSeconds: integerField(budgets, "runOnceSeconds", "controller.budgets"),
reconcileJobTtlSeconds: integerField(budgets, "reconcileJobTtlSeconds", "controller.budgets"),
reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"),
reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"),
reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"),
},
};
if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) {
@@ -334,6 +340,9 @@ function parseFollower(root: Record<string, unknown>, index: number): FollowerSp
statusSeconds: integerField(budgets, "statusSeconds", `${label}.budgets`),
triggerSeconds: integerField(budgets, "triggerSeconds", `${label}.budgets`),
sourceSyncSeconds: integerField(budgets, "sourceSyncSeconds", `${label}.budgets`),
controlPlaneRefreshSeconds: integerField(budgets, "controlPlaneRefreshSeconds", `${label}.budgets`),
capabilityJobTtlSeconds: integerField(budgets, "capabilityJobTtlSeconds", `${label}.budgets`),
capabilityJobBackoffLimit: integerField(budgets, "capabilityJobBackoffLimit", `${label}.budgets`),
},
commands: {
plan: parseCommand(recordField(commands, "plan", `${label}.commands`), `${label}.commands.plan`),
@@ -801,7 +810,7 @@ async function decideAndMaybeTrigger(
}
if (!trigger.ok) warnings.push(trigger.message);
}
if (options.confirm && options.wait && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) {
if (options.confirm && (options.wait || options.inCluster) && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) {
const closeout = await waitNativeFollowerCloseout(registry, follower, observedSha, options, options.timeoutSeconds ?? follower.budgets.endToEndSeconds);
triggerCommand = closeoutOnlyCommand(follower, live.pipelineRun, observedSha, closeout);
if (closeout.completed) {
@@ -914,10 +923,14 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
const namespace = follower.nativeStatus.tekton.namespace;
const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun);
const startedAt = Date.now();
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds)));
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
if (sync !== null && !sync.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt);
}
const refresh = runNativeHwlabControlPlaneRefresh(registry, follower, spec, observedSha, Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.controlPlaneRefreshSeconds), nativeCapabilityJobName(follower.id, "control-plane-refresh", observedSha));
if (!refresh.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt);
}
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds));
const payload = parseJsonObject(result.stdout) ?? {};
const pipelineRunCompleted = payload.completed === true;
@@ -944,6 +957,7 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
...payload,
nativeCapabilities: {
gitMirrorSync: sync === null ? null : sync.result,
controlPlaneRefresh: refresh.result,
gitMirrorFlush: flush === null ? null : flush.result,
},
},
@@ -963,7 +977,7 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo
const startedAt = Date.now();
const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds)));
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
if (sync !== null && !sync.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt);
}
@@ -1580,7 +1594,7 @@ function readNativeObjectBundle(registry: BranchFollowerRegistry, follower: Foll
"\"$tmpdir/read-native-bundle.sh\"",
].join("\n");
const startedAt = Date.now();
const result = runKubeScript(registry, options, script, "", Math.max(5, timeoutSeconds) * 1000);
const result = runKubeScript(registry, options, script, "", timeoutSeconds * 1000);
const parsed = parseNativeBundleLines(result.stdout);
const sourceRecord = asOptionalRecord(parsed.objects.source);
return {
@@ -2367,6 +2381,7 @@ function stageTimingsFromCommand(command: Record<string, unknown> | undefined):
const capabilities = asOptionalRecord(payload.nativeCapabilities);
for (const stage of [
k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)),
k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)),
k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)),
]) {
if (stage !== null) stages.push(stage);
@@ -2518,7 +2533,7 @@ function runControllerReconcileJob(registry: BranchFollowerRegistry, options: Pa
`kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`,
mode.wait ? waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds) : "true",
].join("\n");
const result = runKubeScript(registry, options, script, "", (timeoutSeconds + 20) * 1000);
const result = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000);
return {
ok: result.exitCode === 0,
name: jobName,