mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: refresh hwlab follower control plane natively

Browse Source
This commit is contained in:
Codex
2026-07-03 13:55:13 +00:00
parent 63040ba28c
commit 7751cc167b
7 changed files with 358 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/cicd-controller-render.ts
+6 -4
View File
@@ -30,9 +30,9 @@ export function renderControllerReconcileJob(registry: BranchFollowerRegistry, o
kind: "Job",
metadata: { name: jobName, namespace: registry.controller.namespace, labels },
spec: {
backoffLimit: 0,
ttlSecondsAfterFinished: 600,
activeDeadlineSeconds: timeoutSeconds + 30,
backoffLimit: registry.controller.budgets.reconcileJobBackoffLimit,
ttlSecondsAfterFinished: registry.controller.budgets.reconcileJobTtlSeconds,
activeDeadlineSeconds: timeoutSeconds + registry.controller.budgets.reconcileJobDeadlineGraceSeconds,
template: {
metadata: { labels },
spec: {
@@ -125,6 +125,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec
{ apiGroups: ["batch"], resources: ["jobs"], verbs: ["get", "list", "watch", "create", "delete"] },
{ apiGroups: ["apps"], resources: ["deployments", "statefulsets"], verbs: ["get", "list", "watch"] },
{ apiGroups: ["tekton.dev"], resources: ["pipelineruns"], verbs: ["get", "list", "watch", "create", "patch", "delete"] },
{ apiGroups: ["tekton.dev"], resources: ["pipelines"], verbs: ["get", "list", "watch", "create", "update", "patch"] },
{ apiGroups: ["tekton.dev"], resources: ["taskruns"], verbs: ["get", "list", "watch"] },
{ apiGroups: ["argoproj.io"], resources: ["applications"], verbs: ["get", "list", "watch", "patch"] },
],
@@ -145,6 +146,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec
"sync-source.sh": nativeCicdScript("sync-source.sh"),
"controller-one-shot.sh": nativeCicdScript("controller-one-shot.sh"),
"controller-loop.sh": nativeCicdScript("controller-loop.sh"),
"hwlab-node-control-plane-refresh.mjs": nativeCicdScript("hwlab-node-control-plane-refresh.mjs"),
"github-proxy-connect.mjs": nativeCicdScript("github-proxy-connect.mjs"),
"git-ssh-proxy.sh": nativeCicdScript("git-ssh-proxy.sh"),
},
@@ -163,7 +165,7 @@ export function renderControllerManifests(registry: BranchFollowerRegistry): Rec
apiVersion: "coordination.k8s.io/v1",
kind: "Lease",
metadata: { name: registry.controller.leaseName, namespace: registry.controller.namespace, labels },
spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: Math.max(30, registry.controller.loop.reconcileTimeoutSeconds + 30) },
spec: { holderIdentity: "unidesk-cicd-branch-follower", leaseDurationSeconds: registry.controller.loop.leaseDurationSeconds },
},
{
apiVersion: "apps/v1",
scripts/src/cicd-hwlab-refresh.ts
+91
View File
@@ -0,0 +1,91 @@
// SPEC: PJ2026-01060703 CI/CD branch follower HWLAB native refresh.
// Responsibility: create the Kubernetes Job that refreshes HWLAB node Tekton Pipeline from a git-mirror snapshot.
import { runNativeK8sJob } from "./cicd-native";
import type { BranchFollowerRegistry, FollowerSpec, NativeK8sJobResult } from "./cicd-types";
import type { HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
import { nodeRuntimeGitMirrorTarget, nodeRuntimeRenderOverlay } from "./hwlab-node/web-probe";
export function runNativeHwlabControlPlaneRefresh(
registry: BranchFollowerRegistry,
follower: FollowerSpec,
spec: HwlabRuntimeLaneSpec,
observedSha: string,
timeoutSeconds: number,
jobName: string,
): { jobName: string; namespace: string; result: NativeK8sJobResult } {
const namespace = registry.controller.namespace;
const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh");
return { jobName, namespace, result };
}
function nativeHwlabControlPlaneRefreshJobManifest(
registry: BranchFollowerRegistry,
follower: FollowerSpec,
spec: HwlabRuntimeLaneSpec,
observedSha: string,
jobName: string,
): Record<string, unknown> {
const mirror = nodeRuntimeGitMirrorTarget(spec);
const overlay = Buffer.from(JSON.stringify(nodeRuntimeRenderOverlay(spec)), "utf8").toString("base64");
const sourceStageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
const tektonNamespace = follower.nativeStatus.tekton?.namespace;
if (tektonNamespace === undefined) throw new Error(`follower ${follower.id} nativeStatus.tekton.namespace is required for HWLAB control-plane refresh`);
return {
apiVersion: "batch/v1",
kind: "Job",
metadata: {
name: jobName,
namespace: registry.controller.namespace,
labels: {
"app.kubernetes.io/name": "hwlab-node-control-plane-refresh",
"app.kubernetes.io/part-of": "unidesk-cicd-branch-follower",
"app.kubernetes.io/component": "control-plane-refresh",
"hwlab.pikastech.local/node": spec.nodeId,
"hwlab.pikastech.local/lane": spec.lane,
"hwlab.pikastech.local/source-commit": observedSha,
},
},
spec: {
backoffLimit: follower.budgets.capabilityJobBackoffLimit,
ttlSecondsAfterFinished: follower.budgets.capabilityJobTtlSeconds,
template: {
metadata: {
labels: {
"app.kubernetes.io/name": "hwlab-node-control-plane-refresh",
"app.kubernetes.io/part-of": "unidesk-cicd-branch-follower",
"app.kubernetes.io/component": "control-plane-refresh",
"hwlab.pikastech.local/node": spec.nodeId,
"hwlab.pikastech.local/lane": spec.lane,
"hwlab.pikastech.local/source-commit": observedSha,
},
},
spec: {
restartPolicy: "Never",
serviceAccountName: registry.controller.serviceAccountName,
hostNetwork: true,
dnsPolicy: "ClusterFirstWithHostNet",
volumes: [
{ name: "native-scripts", configMap: { name: registry.controller.configMapName, defaultMode: 0o755 } },
],
containers: [{
name: "control-plane-refresh",
image: mirror.toolsImage,
imagePullPolicy: mirror.toolsImagePullPolicy,
command: ["node", "/etc/unidesk-cicd-branch-follower/hwlab-node-control-plane-refresh.mjs"],
volumeMounts: [
{ name: "native-scripts", mountPath: "/etc/unidesk-cicd-branch-follower", readOnly: true },
],
env: [
{ name: "SOURCE_COMMIT", value: observedSha },
{ name: "SOURCE_STAGE_REF", value: sourceStageRef },
{ name: "GIT_READ_URL", value: spec.gitReadUrl },
{ name: "FIELD_MANAGER", value: spec.controlPlaneFieldManager },
{ name: "TEKTON_NAMESPACE", value: tektonNamespace },
{ name: "HWLAB_RENDER_OVERLAY_B64", value: overlay },
],
}],
},
},
},
};
}
scripts/src/cicd-types.ts
+8
View File
@@ -65,6 +65,9 @@ export interface FollowerSpec {
statusSeconds: number;
triggerSeconds: number;
sourceSyncSeconds: number;
controlPlaneRefreshSeconds: number;
capabilityJobTtlSeconds: number;
capabilityJobBackoffLimit: number;
};
commands: {
plan: CommandSpec;
@@ -148,11 +151,16 @@ export interface ControllerSpec {
loop: {
intervalSeconds: number;
reconcileTimeoutSeconds: number;
leaseDurationSeconds: number;
};
budgets: {
applyWaitSeconds: number;
statusSeconds: number;
runOnceSeconds: number;
reconcileJobTtlSeconds: number;
reconcileJobBackoffLimit: number;
reconcileJobDeadlineGraceSeconds: number;
reconcileTransportGraceSeconds: number;
};
}
scripts/src/cicd.ts
+20 -5
View File
@@ -21,6 +21,7 @@ import { sentinelPipelineRunName } from "./hwlab-node-web-sentinel-cicd-shared";
import { transPath } from "./hwlab-node/runtime-common";
import { configRefGraph, resolveConfigRefString } from "./ops/config-refs";
import { renderControllerManifests, renderControllerReconcileJob, waitForJobShell } from "./cicd-controller-render";
import { runNativeHwlabControlPlaneRefresh } from "./cicd-hwlab-refresh";
import { runNativeK8sJob, runNativeTektonPipelineRun } from "./cicd-native";
import type { AdapterSummary, BranchFollowerPhase, BranchFollowerRegistry, ControllerSpec, FollowerSpec, FollowerState, K8sFollowerStateRead, K8sStateRead, NativeCloseoutWaitResult, NativeK8sJobResult, NativeObjectBundle, NativeStatusSpec, NativeWorkloadSpec, OutputMode, ParsedOptions, StageTiming, TriggerResult } from "./cicd-types";
import {
@@ -278,11 +279,16 @@ function parseController(root: Record<string, unknown>): ControllerSpec {
loop: {
intervalSeconds: integerField(loop, "intervalSeconds", "controller.loop"),
reconcileTimeoutSeconds: integerField(loop, "reconcileTimeoutSeconds", "controller.loop"),
leaseDurationSeconds: integerField(loop, "leaseDurationSeconds", "controller.loop"),
},
budgets: {
applyWaitSeconds: integerField(budgets, "applyWaitSeconds", "controller.budgets"),
statusSeconds: integerField(budgets, "statusSeconds", "controller.budgets"),
runOnceSeconds: integerField(budgets, "runOnceSeconds", "controller.budgets"),
reconcileJobTtlSeconds: integerField(budgets, "reconcileJobTtlSeconds", "controller.budgets"),
reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"),
reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"),
reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"),
},
};
if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) {
@@ -334,6 +340,9 @@ function parseFollower(root: Record<string, unknown>, index: number): FollowerSp
statusSeconds: integerField(budgets, "statusSeconds", `${label}.budgets`),
triggerSeconds: integerField(budgets, "triggerSeconds", `${label}.budgets`),
sourceSyncSeconds: integerField(budgets, "sourceSyncSeconds", `${label}.budgets`),
controlPlaneRefreshSeconds: integerField(budgets, "controlPlaneRefreshSeconds", `${label}.budgets`),
capabilityJobTtlSeconds: integerField(budgets, "capabilityJobTtlSeconds", `${label}.budgets`),
capabilityJobBackoffLimit: integerField(budgets, "capabilityJobBackoffLimit", `${label}.budgets`),
},
commands: {
plan: parseCommand(recordField(commands, "plan", `${label}.commands`), `${label}.commands.plan`),
@@ -801,7 +810,7 @@ async function decideAndMaybeTrigger(
}
if (!trigger.ok) warnings.push(trigger.message);
}
if (options.confirm && options.wait && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) {
if (options.confirm && (options.wait || options.inCluster) && phase === "ClosingOut" && observedSha !== null && triggerCommand === undefined) {
const closeout = await waitNativeFollowerCloseout(registry, follower, observedSha, options, options.timeoutSeconds ?? follower.budgets.endToEndSeconds);
triggerCommand = closeoutOnlyCommand(follower, live.pipelineRun, observedSha, closeout);
if (closeout.completed) {
@@ -914,10 +923,14 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
const namespace = follower.nativeStatus.tekton.namespace;
const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun);
const startedAt = Date.now();
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds)));
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
if (sync !== null && !sync.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt);
}
const refresh = runNativeHwlabControlPlaneRefresh(registry, follower, spec, observedSha, Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.controlPlaneRefreshSeconds), nativeCapabilityJobName(follower.id, "control-plane-refresh", observedSha));
if (!refresh.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt);
}
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds));
const payload = parseJsonObject(result.stdout) ?? {};
const pipelineRunCompleted = payload.completed === true;
@@ -944,6 +957,7 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
...payload,
nativeCapabilities: {
gitMirrorSync: sync === null ? null : sync.result,
controlPlaneRefresh: refresh.result,
gitMirrorFlush: flush === null ? null : flush.result,
},
},
@@ -963,7 +977,7 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo
const startedAt = Date.now();
const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(5, follower.budgets.sourceSyncSeconds)));
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
if (sync !== null && !sync.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt);
}
@@ -1580,7 +1594,7 @@ function readNativeObjectBundle(registry: BranchFollowerRegistry, follower: Foll
"\"$tmpdir/read-native-bundle.sh\"",
].join("\n");
const startedAt = Date.now();
const result = runKubeScript(registry, options, script, "", Math.max(5, timeoutSeconds) * 1000);
const result = runKubeScript(registry, options, script, "", timeoutSeconds * 1000);
const parsed = parseNativeBundleLines(result.stdout);
const sourceRecord = asOptionalRecord(parsed.objects.source);
return {
@@ -2367,6 +2381,7 @@ function stageTimingsFromCommand(command: Record<string, unknown> | undefined):
const capabilities = asOptionalRecord(payload.nativeCapabilities);
for (const stage of [
k8sJobTiming("git-mirror-sync", asOptionalRecord(capabilities?.gitMirrorSync)),
k8sJobTiming("control-plane-refresh", asOptionalRecord(capabilities?.controlPlaneRefresh)),
k8sJobTiming("git-mirror-flush", asOptionalRecord(capabilities?.gitMirrorFlush)),
]) {
if (stage !== null) stages.push(stage);
@@ -2518,7 +2533,7 @@ function runControllerReconcileJob(registry: BranchFollowerRegistry, options: Pa
`kubectl apply --server-side --force-conflicts --field-manager=${shQuote(registry.controller.fieldManager)} -f "$tmp" >/dev/null`,
mode.wait ? waitForJobShell(registry.controller.namespace, jobName, timeoutSeconds) : "true",
].join("\n");
const result = runKubeScript(registry, options, script, "", (timeoutSeconds + 20) * 1000);
const result = runKubeScript(registry, options, script, "", (timeoutSeconds + registry.controller.budgets.reconcileTransportGraceSeconds) * 1000);
return {
ok: result.exitCode === 0,
name: jobName,