diff --git a/.agents/skills/unidesk-cicd/references/branch-follower.md b/.agents/skills/unidesk-cicd/references/branch-follower.md index a6e3c137..eebc4215 100644 --- a/.agents/skills/unidesk-cicd/references/branch-follower.md +++ b/.agents/skills/unidesk-cicd/references/branch-follower.md @@ -38,6 +38,8 @@ It must not copy runtime/GitOps/Secret details from owning configs: Use configRef summaries in plan/status; do not create a `full.md` or super Markdown index. +Timeout, TTL, retry/backoff, reconcile interval and end-to-end budget values must be declared in YAML/source-of-truth fields. Do not introduce hidden numeric defaults in TypeScript, shell, native helper scripts, or controller manifests; helper code should read the configured values and fail structurally when required timing policy is missing. + ## First Followers - `hwlab-jd01-v03`: follows `pikasTech/HWLAB@v0.3`, adapter `hwlab-node-runtime`, native trigger `Tekton PipelineRun -> Argo Application closeout -> runtime Deployment sourceCommit readiness`. diff --git a/config/cicd-branch-followers.yaml b/config/cicd-branch-followers.yaml index bee1c341..3ebfe42f 100644 --- a/config/cicd-branch-followers.yaml +++ b/config/cicd-branch-followers.yaml @@ -44,10 +44,15 @@ controller: loop: intervalSeconds: 30 reconcileTimeoutSeconds: 110 + leaseDurationSeconds: 140 budgets: applyWaitSeconds: 120 statusSeconds: 35 runOnceSeconds: 120 + reconcileJobTtlSeconds: 600 + reconcileJobBackoffLimit: 0 + reconcileJobDeadlineGraceSeconds: 30 + reconcileTransportGraceSeconds: 20 followers: - id: hwlab-jd01-v03 @@ -76,6 +81,9 @@ followers: statusSeconds: 35 triggerSeconds: 120 sourceSyncSeconds: 20 + controlPlaneRefreshSeconds: 45 + capabilityJobTtlSeconds: 600 + capabilityJobBackoffLimit: 0 commands: plan: argv: ["bun", "scripts/cli.ts", "hwlab", "nodes", "control-plane", "trigger-current", "--node", "JD01", "--lane", "v03", "--dry-run", "--raw"] @@ -145,6 +153,9 @@ followers: statusSeconds: 35 triggerSeconds: 120 sourceSyncSeconds: 20 + controlPlaneRefreshSeconds: 20 + capabilityJobTtlSeconds: 600 + capabilityJobBackoffLimit: 0 commands: plan: argv: ["bun", "scripts/cli.ts", "agentrun", "control-plane", "trigger-current", "--node", "JD01", "--lane", "jd01-v02", "--dry-run", "--raw"] @@ -210,6 +221,9 @@ followers: statusSeconds: 35 triggerSeconds: 120 sourceSyncSeconds: 20 + controlPlaneRefreshSeconds: 20 + capabilityJobTtlSeconds: 600 + capabilityJobBackoffLimit: 0 commands: plan: argv: ["bun", "scripts/cli.ts", "web-probe", "sentinel", "publish-current", "--node", "JD01", "--lane", "v03", "--sentinel", "jd01-web-probe-sentinel", "--dry-run", "--raw"] diff --git a/scripts/native/cicd/hwlab-node-control-plane-refresh.mjs b/scripts/native/cicd/hwlab-node-control-plane-refresh.mjs new file mode 100644 index 00000000..21cfe540 --- /dev/null +++ b/scripts/native/cicd/hwlab-node-control-plane-refresh.mjs @@ -0,0 +1,217 @@ +#!/usr/bin/env node +// SPEC: PJ2026-01060703 CI/CD branch follower native HWLAB control-plane refresh. +// Responsibility: render and apply the HWLAB node Tekton Pipeline from a k8s git-mirror snapshot. +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +const startedAt = Date.now(); +const sourceCommit = requiredEnv("SOURCE_COMMIT"); +const sourceStageRef = requiredEnv("SOURCE_STAGE_REF"); +const gitReadUrl = requiredEnv("GIT_READ_URL"); +const fieldManager = requiredEnv("FIELD_MANAGER"); +const tektonNamespace = requiredEnv("TEKTON_NAMESPACE"); +const overlay = JSON.parse(Buffer.from(requiredEnv("HWLAB_RENDER_OVERLAY_B64"), "base64").toString("utf8")); +const workDir = mkdtempSync(path.join(tmpdir(), `hwlab-control-plane-${sourceCommit.slice(0, 12)}-`)); +const repoDir = path.join(workDir, "repo"); +const renderDir = path.join(workDir, "render"); + +try { + checkoutSnapshot(); + prepareYamlDependency(); + applyDeployOverlay(); + renderControlPlane(); + applyPipeline(); + emit({ ok: true, status: "applied" }); +} finally { + rmSync(workDir, { recursive: true, force: true }); +} + +function checkoutSnapshot() { + run("git", ["init", repoDir], { cwd: "/" }); + run("git", ["-C", repoDir, "remote", "add", "origin", gitReadUrl]); + run("git", ["-C", repoDir, "fetch", "--depth", "1", "origin", sourceStageRef]); + run("git", ["-C", repoDir, "checkout", "--detach", "FETCH_HEAD"]); + const head = capture("git", ["-C", repoDir, "rev-parse", "HEAD"]).trim(); + if (head !== sourceCommit) throw new Error(`snapshot checkout mismatch: expected ${sourceCommit}, got ${head}`); +} + +function prepareYamlDependency() { + if (canResolveYaml()) return; + const registry = requiredOverlayString("npmRegistry"); + const timeoutMs = String(requiredPositiveNumber("npmFetchTimeoutMs")); + const retries = String(requiredNonNegativeNumber("npmRetries")); + const yamlSpec = yamlDependencySpec(); + const env = renderEnv({ + npm_config_registry: registry, + BUN_CONFIG_REGISTRY: registry, + npm_config_fetch_timeout: timeoutMs, + npm_config_fetch_retries: retries, + }); + if (commandExists("bun")) { + const result = runOptional("bun", ["add", "--no-save", "--ignore-scripts", "--registry", registry, yamlSpec], { cwd: repoDir, env }); + if (result.status === 0 && canResolveYaml()) return; + } + run("npm", ["install", "--package-lock=false", "--no-save", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev", "--registry", registry, yamlSpec], { cwd: repoDir, env }); + if (!canResolveYaml()) throw new Error("yaml dependency remains unresolved after install"); +} + +function yamlDependencySpec() { + const pkg = JSON.parse(readFileSync(path.join(repoDir, "package.json"), "utf8")); + const version = pkg.dependencies?.yaml || pkg.devDependencies?.yaml; + if (typeof version !== "string" || version.length === 0) throw new Error("package.json must declare yaml dependency for gitops render"); + return `yaml@${version}`; +} + +function canResolveYaml() { + const result = runOptional("node", ["-e", "require.resolve('yaml')"], { cwd: repoDir, stdio: "ignore" }); + return result.status === 0; +} + +function applyDeployOverlay() { + const requireFromRepo = createRequire(path.join(repoDir, "package.json")); + const YAML = requireFromRepo("yaml"); + const deployPath = path.join(repoDir, "deploy/deploy.yaml"); + const doc = YAML.parse(readFileSync(deployPath, "utf8")); + doc.nodes = doc.nodes || {}; + doc.nodes[overlay.nodeId] = { ...(doc.nodes[overlay.nodeId] || {}), gitopsRoot: overlay.gitopsRoot, sourceRepo: overlay.gitUrl }; + doc.lanes = doc.lanes || {}; + const lane = doc.lanes[overlay.lane] || {}; + const downloadStack = { + ...(lane.envRecipe?.downloadStack || {}), + httpProxy: overlay.dockerProxyHttp, + httpsProxy: overlay.dockerProxyHttps, + noProxy: overlay.dockerNoProxyList, + }; + doc.lanes[overlay.lane] = { + ...lane, + node: overlay.nodeId, + sourceBranch: overlay.sourceBranch, + gitopsBranch: overlay.gitopsBranch, + namespace: overlay.runtimeNamespace, + endpoint: overlay.publicApiUrl, + publicEndpoints: { frontend: overlay.publicWebUrl, api: overlay.publicApiUrl }, + artifactCatalog: overlay.catalogPath, + runtimePath: overlay.runtimePath, + imageTagMode: "full", + sourceRepo: overlay.gitUrl, + externalPostgres: overlay.externalPostgres, + observability: overlay.observability, + envRecipe: { ...(lane.envRecipe || {}), downloadStack }, + }; + if (overlay.runtimeStore !== undefined) doc.lanes[overlay.lane].runtimeStore = overlay.runtimeStore; + if (overlay.codeAgentRuntime !== undefined) doc.lanes[overlay.lane].codeAgentRuntime = overlay.codeAgentRuntime; + if (overlay.deployYamlGitMirror !== undefined) doc.lanes[overlay.lane].gitMirror = overlay.deployYamlGitMirror; + writeFileSync(deployPath, YAML.stringify(doc), "utf8"); +} + +function renderControlPlane() { + run("node", [ + "scripts/run-bun.mjs", + "scripts/gitops-render.mjs", + "--lane", overlay.lane, + "--node", overlay.nodeId, + "--gitops-root", overlay.gitopsRoot, + "--catalog-path", overlay.catalogPath, + "--image-tag-mode", "full", + "--source-revision", sourceCommit, + "--source-repo", overlay.gitUrl, + "--source-branch", overlay.sourceBranch, + "--gitops-branch", overlay.gitopsBranch, + "--git-read-url", overlay.gitReadUrl, + "--git-write-url", overlay.gitWriteUrl, + "--registry-prefix", overlay.registryPrefix, + "--runtime-endpoint", overlay.publicApiUrl, + "--web-endpoint", overlay.publicWebUrl, + "--out", renderDir, + ], { cwd: repoDir, env: renderEnv() }); +} + +function applyPipeline() { + const pipelinePath = path.join(renderDir, overlay.tektonDir, "pipeline.yaml"); + if (!existsSync(pipelinePath)) throw new Error(`rendered Pipeline missing: ${pipelinePath}`); + run("kubectl", ["apply", "--server-side", "--force-conflicts", `--field-manager=${fieldManager}`, "-f", pipelinePath], { cwd: repoDir, env: renderEnv() }); +} + +function renderEnv(extra = {}) { + const noProxy = overlay.noProxy || process.env.NO_PROXY || process.env.no_proxy || ""; + return { + ...process.env, + HTTP_PROXY: overlay.proxyHttp || process.env.HTTP_PROXY || "", + HTTPS_PROXY: overlay.proxyHttps || process.env.HTTPS_PROXY || "", + ALL_PROXY: overlay.proxyAll || process.env.ALL_PROXY || "", + NO_PROXY: noProxy, + http_proxy: overlay.proxyHttp || process.env.http_proxy || "", + https_proxy: overlay.proxyHttps || process.env.https_proxy || "", + all_proxy: overlay.proxyAll || process.env.all_proxy || "", + no_proxy: noProxy, + HWLAB_NODE_CI_TOOLS_IMAGE: overlay.toolsImage || process.env.HWLAB_NODE_CI_TOOLS_IMAGE || "", + HWLAB_NODE_BUILDKIT_IMAGE: overlay.buildkitSidecarImage || process.env.HWLAB_NODE_BUILDKIT_IMAGE || "", + ...extra, + }; +} + +function run(command, args, options = {}) { + const result = runOptional(command, args, options); + if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`); + return result; +} + +function capture(command, args) { + const result = runOptional(command, args, { cwd: repoDir, encoding: "utf8", stdio: ["ignore", "pipe", "inherit"] }); + if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`); + return result.stdout || ""; +} + +function runOptional(command, args, options = {}) { + return spawnSync(command, args, { + cwd: options.cwd || repoDir, + env: options.env || process.env, + encoding: options.encoding || "utf8", + stdio: options.stdio || "inherit", + }); +} + +function commandExists(command) { + return runOptional("sh", ["-lc", `command -v ${command} >/dev/null 2>&1`], { cwd: repoDir, stdio: "ignore" }).status === 0; +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function requiredOverlayString(name) { + const value = overlay[name]; + if (typeof value !== "string" || value.length === 0) throw new Error(`overlay.${name} is required`); + return value; +} + +function requiredPositiveNumber(name) { + const value = Number(overlay[name]); + if (!Number.isFinite(value) || value <= 0) throw new Error(`overlay.${name} must be a positive number`); + return Math.floor(value); +} + +function requiredNonNegativeNumber(name) { + const value = Number(overlay[name]); + if (!Number.isFinite(value) || value < 0) throw new Error(`overlay.${name} must be a non-negative number`); + return Math.floor(value); +} + +function emit(extra) { + process.stdout.write(`${JSON.stringify({ + ...extra, + sourceCommit, + sourceStageRef, + pipeline: overlay.pipelineName, + namespace: tektonNamespace, + elapsedMs: Date.now() - startedAt, + sourceAuthority: "k8s-git-mirror-snapshot", + statusAuthority: "kubernetes-api-serviceaccount", + parsedDownstreamCliOutput: false, + })}\n`); +} diff --git a/scripts/src/cicd-controller-render.ts b/scripts/src/cicd-controller-render.ts index 37f48465..e351af29 100644 --- a/scripts/src/cicd-controller-render.ts +++ b/scripts/src/cicd-controller-render.ts @@ -30,9 +30,9 @@ export function renderControllerReconcileJob(registry: BranchFollowerRegistry, o kind: "Job", metadata: { name: jobName, namespace: registry.controller.namespace, labels }, spec: { - backoffLimit: 0, - ttlSecondsAfterFinished: 600, - activeDeadlineSeconds: timeoutSeconds + 30, + backoffLimit: registry.controller.budgets.reconcileJobBackoffLimit, + ttlSecondsAfterFinished: registry.controller.budgets.reconcileJobTtlSeconds, + activeDeadlineSeconds: timeoutSeconds + registry.controller.budgets.reconcileJobDeadlineGraceSeconds, template: { metadata: { labels }, spec: { @@ -125,6 +125,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec { apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "delete"] }, { apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] }, { apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "watch", "create", "patch", "delete"] }, + { apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "list", "watch", "create", "update", "patch"] }, { apiGroups: ["tekton.dev"], resources: ["taskruns"], verbs: ["get", "list", "watch"] }, { apiGroups: ["argoproj.io"], resources: ["applications"], verbs: ["get", "list", "watch", "patch"] }, ], @@ -145,6 +146,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec "sync-source.sh": nativeCicdScript("sync-source.sh"), "controller-one-shot.sh": nativeCicdScript("controller-one-shot.sh"), "controller-loop.sh": nativeCicdScript("controller-loop.sh"), + "hwlab-node-control-plane-refresh.mjs": nativeCicdScript("hwlab-node-control-plane-refresh.mjs"), "github-proxy-connect.mjs": nativeCicdScript("github-proxy-connect.mjs"), "git-ssh-proxy.sh": nativeCicdScript("git-ssh-proxy.sh"), }, @@ -163,7 +165,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec apiVersion: "coordination.k8s.io/v1", kind: "Lease", metadata: { name: registry.controller.leaseName, namespace: registry.controller.namespace, labels }, - spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: Math.max(30, registry.controller.loop.reconcileTimeoutSeconds + 30) }, + spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: registry.controller.loop.leaseDurationSeconds }, }, { apiVersion: "apps/v1", diff --git a/scripts/src/cicd-hwlab-refresh.ts b/scripts/src/cicd-hwlab-refresh.ts new file mode 100644 index 00000000..9ec41cad --- /dev/null +++ b/scripts/src/cicd-hwlab-refresh.ts @@ -0,0 +1,91 @@ +// SPEC: PJ2026-01060703 CI/CD branch follower HWLAB native refresh. +// Responsibility: create the Kubernetes Job that refreshes HWLAB node Tekton Pipeline from a git-mirror snapshot. +import { runNativeK8sJob } from "./cicd-native"; +import type { BranchFollowerRegistry, FollowerSpec, NativeK8sJobResult } from "./cicd-types"; +import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; +import { nodeRuntimeGitMirrorTarget, nodeRuntimeRenderOverlay } from "./hwlab-node/web-probe"; + +export function runNativeHwlabControlPlaneRefresh( + registry: BranchFollowerRegistry, + follower: FollowerSpec, + spec: HwlabRuntimeLaneSpec, + observedSha: string, + timeoutSeconds: number, + jobName: string, +): { jobName: string; namespace: string; result: NativeK8sJobResult } { + const namespace = registry.controller.namespace; + const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh"); + return { jobName, namespace, result }; +} + +function nativeHwlabControlPlaneRefreshJobManifest( + registry: BranchFollowerRegistry, + follower: FollowerSpec, + spec: HwlabRuntimeLaneSpec, + observedSha: string, + jobName: string, +): Record { + const mirror = nodeRuntimeGitMirrorTarget(spec); + const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64"); + const sourceStageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`; + const tektonNamespace = follower.nativeStatus.tekton?.namespace; + if (tektonNamespace === undefined) throw new Error(`follower ${follower.id} nativeStatus.tekton.namespace is required for HWLAB control-plane refresh`); + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: jobName, + namespace: registry.controller.namespace, + labels: { + "app.kubernetes.io/name": "hwlab-node-control-plane-refresh", + "app.kubernetes.io/part-of": "unidesk-cicd-branch-follower", + "app.kubernetes.io/component": "control-plane-refresh", + "hwlab.pikastech.local/node": spec.nodeId, + "hwlab.pikastech.local/lane": spec.lane, + "hwlab.pikastech.local/source-commit": observedSha, + }, + }, + spec: { + backoffLimit: follower.budgets.capabilityJobBackoffLimit, + ttlSecondsAfterFinished: follower.budgets.capabilityJobTtlSeconds, + template: { + metadata: { + labels: { + "app.kubernetes.io/name": "hwlab-node-control-plane-refresh", + "app.kubernetes.io/part-of": "unidesk-cicd-branch-follower", + "app.kubernetes.io/component": "control-plane-refresh", + "hwlab.pikastech.local/node": spec.nodeId, + "hwlab.pikastech.local/lane": spec.lane, + "hwlab.pikastech.local/source-commit": observedSha, + }, + }, + spec: { + restartPolicy: "Never", + serviceAccountName: registry.controller.serviceAccountName, + hostNetwork: true, + dnsPolicy: "ClusterFirstWithHostNet", + volumes: [ + { name: "native-scripts", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } }, + ], + containers: [{ + name: "control-plane-refresh", + image: mirror.toolsImage, + imagePullPolicy: mirror.toolsImagePullPolicy, + command: ["node", "/etc/unidesk-cicd-branch-follower/hwlab-node-control-plane-refresh.mjs"], + volumeMounts: [ + { name: "native-scripts", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true }, + ], + env: [ + { name: "SOURCE_COMMIT", value: observedSha }, + { name: "SOURCE_STAGE_REF", value: sourceStageRef }, + { name: "GIT_READ_URL", value: spec.gitReadUrl }, + { name: "FIELD_MANAGER", value: spec.controlPlaneFieldManager }, + { name: "TEKTON_NAMESPACE", value: tektonNamespace }, + { name: "HWLAB_RENDER_OVERLAY_B64", value: overlay }, + ], + }], + }, + }, + }, + }; +} diff --git a/scripts/src/cicd-types.ts b/scripts/src/cicd-types.ts index 7e1a10bf..7ad0bd47 100644 --- a/scripts/src/cicd-types.ts +++ b/scripts/src/cicd-types.ts @@ -65,6 +65,9 @@ export interface FollowerSpec { statusSeconds: number; triggerSeconds: number; sourceSyncSeconds: number; + controlPlaneRefreshSeconds: number; + capabilityJobTtlSeconds: number; + capabilityJobBackoffLimit: number; }; commands: { plan: CommandSpec; @@ -148,11 +151,16 @@ export interface ControllerSpec { loop: { intervalSeconds: number; reconcileTimeoutSeconds: number; + leaseDurationSeconds: number; }; budgets: { applyWaitSeconds: number; statusSeconds: number; runOnceSeconds: number; + reconcileJobTtlSeconds: number; + reconcileJobBackoffLimit: number; + reconcileJobDeadlineGraceSeconds: number; + reconcileTransportGraceSeconds: number; }; } diff --git a/scripts/src/cicd.ts b/scripts/src/cicd.ts index 379762ba..dced4d5a 100644 --- a/scripts/src/cicd.ts +++ b/scripts/src/cicd.ts @@ -21,6 +21,7 @@ import { sentinelPipelineRunName } from "./hwlab-node-web-sentinel-cicd-shared"; import { transPath } from "./hwlab-node/runtime-common"; import { configRefGraph, resolveConfigRefString } from "./ops/config-refs"; import { renderControllerManifests, renderControllerReconcileJob, waitForJobShell } from "./cicd-controller-render"; +import { runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh"; import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native"; import type { AdapterSummary, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeObjectBundle, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types"; import { @@ -278,11 +279,16 @@ function parseController(root: Record): ControllerSpec { loop: { intervalSeconds: integerField(loop, "intervalSeconds", "controller.loop"), reconcileTimeoutSeconds: integerField(loop, "reconcileTimeoutSeconds", "controller.loop"), + leaseDurationSeconds: integerField(loop, "leaseDurationSeconds", "controller.loop"), }, budgets: { applyWaitSeconds: integerField(budgets, "applyWaitSeconds", "controller.budgets"), statusSeconds: integerField(budgets, "statusSeconds", "controller.budgets"), runOnceSeconds: integerField(budgets, "runOnceSeconds", "controller.budgets"), + reconcileJobTtlSeconds: integerField(budgets, "reconcileJobTtlSeconds", "controller.budgets"), + reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"), + reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"), + reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"), }, }; if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) { @@ -334,6 +340,9 @@ function parseFollower(root: Record, index: number): FollowerSp statusSeconds: integerField(budgets, "statusSeconds", `${label}.budgets`), triggerSeconds: integerField(budgets, "triggerSeconds", `${label}.budgets`), sourceSyncSeconds: integerField(budgets, "sourceSyncSeconds", `${label}.budgets`), + controlPlaneRefreshSeconds: integerField(budgets, "controlPlaneRefreshSeconds", `${label}.budgets`), + capabilityJobTtlSeconds: integerField(budgets, "capabilityJobTtlSeconds", `${label}.budgets`), + capabilityJobBackoffLimit: integerField(budgets, "capabilityJobBackoffLimit", `${label}.budgets`), }, commands: { plan: parseCommand(recordField(commands, "plan", `${label}.commands`), `${label}.commands.plan`), @@ -801,7 +810,7 @@ async function decideAndMaybeTrigger( } if (!trigger.ok) warnings.push(trigger.message); } - if (options.confirm && options.wait && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) { + if (options.confirm && (options.wait || options.inCluster) && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) { const closeout = await waitNativeFollowerCloseout(registry, follower, observedSha, options, options.timeoutSeconds ?? follower.budgets.endToEndSeconds); triggerCommand = closeoutOnlyCommand(follower, live.pipelineRun, observedSha, closeout); if (closeout.completed) { @@ -914,10 +923,14 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f const namespace = follower.nativeStatus.tekton.namespace; const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun); const startedAt = Date.now(); - const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds))); + const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds)); if (sync !== null && !sync.result.ok) { return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt); } + const refresh = runNativeHwlabControlPlaneRefresh(registry, follower, spec, observedSha, Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.controlPlaneRefreshSeconds), nativeCapabilityJobName(follower.id, "control-plane-refresh", observedSha)); + if (!refresh.result.ok) { + return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt); + } const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds)); const payload = parseJsonObject(result.stdout) ?? {}; const pipelineRunCompleted = payload.completed === true; @@ -944,6 +957,7 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f ...payload, nativeCapabilities: { gitMirrorSync: sync === null ? null : sync.result, + controlPlaneRefresh: refresh.result, gitMirrorFlush: flush === null ? null : flush.result, }, }, @@ -963,7 +977,7 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo const startedAt = Date.now(); const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`; const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`; - const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds))); + const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds)); if (sync !== null && !sync.result.ok) { return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt); } @@ -1580,7 +1594,7 @@ function readNativeObjectBundle(registry: BranchFollowerRegistry, follower: Foll "\"$tmpdir/read-native-bundle.sh\"", ].join("\n"); const startedAt = Date.now(); - const result = runKubeScript(registry, options, script, "", Math.max(5, timeoutSeconds) * 1000); + const result = runKubeScript(registry, options, script, "", timeoutSeconds * 1000); const parsed = parseNativeBundleLines(result.stdout); const sourceRecord = asOptionalRecord(parsed.objects.source); return { @@ -2367,6 +2381,7 @@ function stageTimingsFromCommand(command: Record | undefined): const capabilities = asOptionalRecord(payload.nativeCapabilities); for (const stage of [ k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)), + k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)), k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)), ]) { if (stage !== null) stages.push(stage); @@ -2518,7 +2533,7 @@ function runControllerReconcileJob(registry: BranchFollowerRegistry, options: Pa `kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`, mode.wait ? waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds) : "true", ].join("\n"); - const result = runKubeScript(registry, options, script, "", (timeoutSeconds + 20) * 1000); + const result = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000); return { ok: result.exitCode === 0, name: jobName,