fix: speed up native branch follower refresh
This commit is contained in:
@@ -53,6 +53,8 @@ controller:
|
|||||||
reconcileJobBackoffLimit: 0
|
reconcileJobBackoffLimit: 0
|
||||||
reconcileJobDeadlineGraceSeconds: 30
|
reconcileJobDeadlineGraceSeconds: 30
|
||||||
reconcileTransportGraceSeconds: 20
|
reconcileTransportGraceSeconds: 20
|
||||||
|
nativeTransportGraceSeconds: 10
|
||||||
|
nativePollIntervalSeconds: 2
|
||||||
|
|
||||||
followers:
|
followers:
|
||||||
- id: hwlab-jd01-v03
|
- id: hwlab-jd01-v03
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
// SPEC: PJ2026-01060703 CI/CD branch follower native HWLAB control-plane refresh.
|
// SPEC: PJ2026-01060703 CI/CD branch follower native HWLAB control-plane refresh.
|
||||||
// Responsibility: render and apply the HWLAB node Tekton Pipeline from a k8s git-mirror snapshot.
|
// Responsibility: render and apply the HWLAB node Tekton Pipeline from a k8s git-mirror snapshot.
|
||||||
import { spawnSync } from "node:child_process";
|
import { spawnSync } from "node:child_process";
|
||||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||||
import { createRequire } from "node:module";
|
import { createRequire } from "node:module";
|
||||||
import { tmpdir } from "node:os";
|
import { tmpdir } from "node:os";
|
||||||
import path from "node:path";
|
import path from "node:path";
|
||||||
@@ -17,6 +17,7 @@ const overlay = JSON.parse(Buffer.from(requiredEnv("HWLAB_RENDER_OVERLAY_B64"),
|
|||||||
const workDir = mkdtempSync(path.join(tmpdir(), `hwlab-control-plane-${sourceCommit.slice(0, 12)}-`));
|
const workDir = mkdtempSync(path.join(tmpdir(), `hwlab-control-plane-${sourceCommit.slice(0, 12)}-`));
|
||||||
const repoDir = path.join(workDir, "repo");
|
const repoDir = path.join(workDir, "repo");
|
||||||
const renderDir = path.join(workDir, "render");
|
const renderDir = path.join(workDir, "render");
|
||||||
|
const depsDir = path.join(workDir, "deps");
|
||||||
|
|
||||||
try {
|
try {
|
||||||
checkoutSnapshot();
|
checkoutSnapshot();
|
||||||
@@ -39,6 +40,8 @@ function checkoutSnapshot() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function prepareYamlDependency() {
|
function prepareYamlDependency() {
|
||||||
|
mkdirSync(depsDir, { recursive: true });
|
||||||
|
writeFileSync(path.join(depsDir, "package.json"), JSON.stringify({ private: true, dependencies: {} }), "utf8");
|
||||||
if (canResolveYaml()) return;
|
if (canResolveYaml()) return;
|
||||||
const registry = requiredOverlayString("npmRegistry");
|
const registry = requiredOverlayString("npmRegistry");
|
||||||
const timeoutMs = String(requiredPositiveNumber("npmFetchTimeoutMs"));
|
const timeoutMs = String(requiredPositiveNumber("npmFetchTimeoutMs"));
|
||||||
@@ -51,10 +54,10 @@ function prepareYamlDependency() {
|
|||||||
npm_config_fetch_retries: retries,
|
npm_config_fetch_retries: retries,
|
||||||
});
|
});
|
||||||
if (commandExists("bun")) {
|
if (commandExists("bun")) {
|
||||||
const result = runOptional("bun", ["add", "--no-save", "--ignore-scripts", "--registry", registry, yamlSpec], { cwd: repoDir, env });
|
const result = runOptional("bun", ["add", "--no-save", "--ignore-scripts", "--registry", registry, yamlSpec], { cwd: depsDir, env });
|
||||||
if (result.status === 0 && canResolveYaml()) return;
|
if (result.status === 0 && canResolveYaml()) return;
|
||||||
}
|
}
|
||||||
run("npm", ["install", "--package-lock=false", "--no-save", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev", "--registry", registry, yamlSpec], { cwd: repoDir, env });
|
run("npm", ["install", "--package-lock=false", "--no-save", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev", "--registry", registry, yamlSpec], { cwd: depsDir, env });
|
||||||
if (!canResolveYaml()) throw new Error("yaml dependency remains unresolved after install");
|
if (!canResolveYaml()) throw new Error("yaml dependency remains unresolved after install");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -66,13 +69,13 @@ function yamlDependencySpec() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function canResolveYaml() {
|
function canResolveYaml() {
|
||||||
const result = runOptional("node", ["-e", "require.resolve('yaml')"], { cwd: repoDir, stdio: "ignore" });
|
const result = runOptional("node", ["-e", "require.resolve('yaml')"], { cwd: depsDir, stdio: "ignore" });
|
||||||
return result.status === 0;
|
return result.status === 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyDeployOverlay() {
|
function applyDeployOverlay() {
|
||||||
const requireFromRepo = createRequire(path.join(repoDir, "package.json"));
|
const requireFromDeps = createRequire(path.join(depsDir, "package.json"));
|
||||||
const YAML = requireFromRepo("yaml");
|
const YAML = requireFromDeps("yaml");
|
||||||
const deployPath = path.join(repoDir, "deploy/deploy.yaml");
|
const deployPath = path.join(repoDir, "deploy/deploy.yaml");
|
||||||
const doc = YAML.parse(readFileSync(deployPath, "utf8"));
|
const doc = YAML.parse(readFileSync(deployPath, "utf8"));
|
||||||
doc.nodes = doc.nodes || {};
|
doc.nodes = doc.nodes || {};
|
||||||
@@ -155,13 +158,13 @@ function renderEnv(extra = {}) {
|
|||||||
|
|
||||||
function run(command, args, options = {}) {
|
function run(command, args, options = {}) {
|
||||||
const result = runOptional(command, args, options);
|
const result = runOptional(command, args, options);
|
||||||
if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`);
|
if (result.status !== 0) throw new Error(failedCommandMessage(command, args, result));
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
function capture(command, args) {
|
function capture(command, args) {
|
||||||
const result = runOptional(command, args, { cwd: repoDir, encoding: "utf8", stdio: ["ignore", "pipe", "inherit"] });
|
const result = runOptional(command, args, { cwd: repoDir, encoding: "utf8", stdio: ["ignore", "pipe", "inherit"] });
|
||||||
if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`);
|
if (result.status !== 0) throw new Error(failedCommandMessage(command, args, result));
|
||||||
return result.stdout || "";
|
return result.stdout || "";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -178,6 +181,17 @@ function commandExists(command) {
|
|||||||
return runOptional("sh", ["-lc", `command -v ${command} >/dev/null 2>&1`], { cwd: repoDir, stdio: "ignore" }).status === 0;
|
return runOptional("sh", ["-lc", `command -v ${command} >/dev/null 2>&1`], { cwd: repoDir, stdio: "ignore" }).status === 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function failedCommandMessage(command, args, result) {
|
||||||
|
const signal = result.signal ? ` signal=${result.signal}` : "";
|
||||||
|
const error = result.error ? ` error=${result.error.message}` : "";
|
||||||
|
const stderr = typeof result.stderr === "string" && result.stderr.length > 0 ? ` stderr=${tail(result.stderr, 1000)}` : "";
|
||||||
|
return `${command} ${args.join(" ")} failed with exit ${result.status}${signal}${error}${stderr}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tail(value, maxChars) {
|
||||||
|
return value.length <= maxChars ? value : value.slice(value.length - maxChars);
|
||||||
|
}
|
||||||
|
|
||||||
function requiredEnv(name) {
|
function requiredEnv(name) {
|
||||||
const value = process.env[name];
|
const value = process.env[name];
|
||||||
if (!value) throw new Error(`${name} is required`);
|
if (!value) throw new Error(`${name} is required`);
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ import https from "node:https";
|
|||||||
const namespace = process.env.NAMESPACE || "";
|
const namespace = process.env.NAMESPACE || "";
|
||||||
const jobName = process.env.JOB_NAME || "";
|
const jobName = process.env.JOB_NAME || "";
|
||||||
const logContainer = process.env.LOG_CONTAINER || "";
|
const logContainer = process.env.LOG_CONTAINER || "";
|
||||||
const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS || "60");
|
const timeoutSeconds = requiredPositiveNumber("TIMEOUT_SECONDS");
|
||||||
|
const pollIntervalSeconds = requiredPositiveNumber("POLL_INTERVAL_SECONDS");
|
||||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||||
const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443");
|
const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443");
|
||||||
const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim();
|
const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim();
|
||||||
@@ -75,7 +76,23 @@ async function logsTail() {
|
|||||||
|
|
||||||
let created = false;
|
let created = false;
|
||||||
let reused = false;
|
let reused = false;
|
||||||
const existing = await getJob();
|
let replacedFailed = false;
|
||||||
|
let existing = await getJob();
|
||||||
|
if (existing && condition(existing, "Failed")) {
|
||||||
|
await deleteJob();
|
||||||
|
replacedFailed = true;
|
||||||
|
existing = null;
|
||||||
|
const deleteDeadline = Date.now() + timeoutSeconds * 1000;
|
||||||
|
let deleted = false;
|
||||||
|
while (Date.now() <= deleteDeadline) {
|
||||||
|
if (await getJob() === null) {
|
||||||
|
deleted = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
await delay(pollIntervalSeconds * 1000);
|
||||||
|
}
|
||||||
|
if (!deleted) throw new Error(`timed out deleting failed job ${jobName}`);
|
||||||
|
}
|
||||||
if (existing) {
|
if (existing) {
|
||||||
reused = true;
|
reused = true;
|
||||||
} else {
|
} else {
|
||||||
@@ -89,7 +106,7 @@ if (existing) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000;
|
const deadline = startedAt + timeoutSeconds * 1000;
|
||||||
let polls = 0;
|
let polls = 0;
|
||||||
let latest = await getJob();
|
let latest = await getJob();
|
||||||
while (Date.now() <= deadline) {
|
while (Date.now() <= deadline) {
|
||||||
@@ -97,7 +114,7 @@ while (Date.now() <= deadline) {
|
|||||||
const failed = condition(latest, "Failed");
|
const failed = condition(latest, "Failed");
|
||||||
if (complete || failed) break;
|
if (complete || failed) break;
|
||||||
polls += 1;
|
polls += 1;
|
||||||
await delay(2000);
|
await delay(pollIntervalSeconds * 1000);
|
||||||
latest = await getJob();
|
latest = await getJob();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -112,6 +129,7 @@ const output = {
|
|||||||
timedOut,
|
timedOut,
|
||||||
created,
|
created,
|
||||||
reused,
|
reused,
|
||||||
|
replacedFailed,
|
||||||
jobName,
|
jobName,
|
||||||
namespace,
|
namespace,
|
||||||
polls,
|
polls,
|
||||||
@@ -125,3 +143,15 @@ const output = {
|
|||||||
};
|
};
|
||||||
process.stdout.write(JSON.stringify(output));
|
process.stdout.write(JSON.stringify(output));
|
||||||
if (!output.ok) process.exit(1);
|
if (!output.ok) process.exit(1);
|
||||||
|
|
||||||
|
async function deleteJob() {
|
||||||
|
const result = await request("DELETE", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs/${encodeURIComponent(jobName)}?propagationPolicy=Background`);
|
||||||
|
if (result.status === 404) return;
|
||||||
|
if (result.status < 200 || result.status >= 300) throw new Error(result.text || `kube api DELETE job status ${result.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function requiredPositiveNumber(name) {
|
||||||
|
const value = Number(process.env[name]);
|
||||||
|
if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive number`);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,7 +4,8 @@ import https from "node:https";
|
|||||||
const namespace = process.env.NAMESPACE || "";
|
const namespace = process.env.NAMESPACE || "";
|
||||||
const pipelineRun = process.env.PIPELINERUN || "";
|
const pipelineRun = process.env.PIPELINERUN || "";
|
||||||
const shouldWait = process.env.WAIT === "true";
|
const shouldWait = process.env.WAIT === "true";
|
||||||
const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS || "60");
|
const timeoutSeconds = requiredPositiveNumber("TIMEOUT_SECONDS");
|
||||||
|
const pollIntervalSeconds = requiredPositiveNumber("POLL_INTERVAL_SECONDS");
|
||||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||||
const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443");
|
const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443");
|
||||||
const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim();
|
const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim();
|
||||||
@@ -89,7 +90,7 @@ if (latest.found) {
|
|||||||
latest = await getPipelineRun();
|
latest = await getPipelineRun();
|
||||||
}
|
}
|
||||||
|
|
||||||
const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000;
|
const deadline = Date.now() + timeoutSeconds * 1000;
|
||||||
let polls = 0;
|
let polls = 0;
|
||||||
while (shouldWait) {
|
while (shouldWait) {
|
||||||
const condition = succeededCondition(latest.object);
|
const condition = succeededCondition(latest.object);
|
||||||
@@ -97,7 +98,7 @@ while (shouldWait) {
|
|||||||
if (Date.now() >= deadline) break;
|
if (Date.now() >= deadline) break;
|
||||||
polls += 1;
|
polls += 1;
|
||||||
process.stderr.write(JSON.stringify({ event: "cicd.branch-follower.native-tekton.wait", pipelineRun, namespace, polls, conditionStatus: condition?.status || null, valuesRedacted: true }) + "\n");
|
process.stderr.write(JSON.stringify({ event: "cicd.branch-follower.native-tekton.wait", pipelineRun, namespace, polls, conditionStatus: condition?.status || null, valuesRedacted: true }) + "\n");
|
||||||
await delay(2000);
|
await delay(pollIntervalSeconds * 1000);
|
||||||
latest = await getPipelineRun();
|
latest = await getPipelineRun();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,3 +126,9 @@ const output = {
|
|||||||
};
|
};
|
||||||
process.stdout.write(JSON.stringify(output));
|
process.stdout.write(JSON.stringify(output));
|
||||||
if (failed) process.exit(1);
|
if (failed) process.exit(1);
|
||||||
|
|
||||||
|
function requiredPositiveNumber(name) {
|
||||||
|
const value = Number(process.env[name]);
|
||||||
|
if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive number`);
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ export function runNativeHwlabControlPlaneRefresh(
|
|||||||
jobName: string,
|
jobName: string,
|
||||||
): { jobName: string; namespace: string; result: NativeK8sJobResult } {
|
): { jobName: string; namespace: string; result: NativeK8sJobResult } {
|
||||||
const namespace = registry.controller.namespace;
|
const namespace = registry.controller.namespace;
|
||||||
const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh");
|
const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh", registry.controller.budgets);
|
||||||
return { jobName, namespace, result };
|
return { jobName, namespace, result };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,33 +6,40 @@ import type { NativeK8sJobResult } from "./cicd-types";
|
|||||||
|
|
||||||
const NATIVE_SCRIPT_DIR = "scripts/native/cicd";
|
const NATIVE_SCRIPT_DIR = "scripts/native/cicd";
|
||||||
|
|
||||||
export function runNativeTektonPipelineRun(namespace: string, pipelineRun: string, manifest: Record<string, unknown>, wait: boolean, timeoutSeconds: number): CommandResult {
|
interface NativeTimingOptions {
|
||||||
|
nativeTransportGraceSeconds: number;
|
||||||
|
nativePollIntervalSeconds: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runNativeTektonPipelineRun(namespace: string, pipelineRun: string, manifest: Record<string, unknown>, wait: boolean, timeoutSeconds: number, timing: NativeTimingOptions): CommandResult {
|
||||||
return runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "submit-pipelinerun.mjs")], repoRoot, {
|
return runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "submit-pipelinerun.mjs")], repoRoot, {
|
||||||
input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"),
|
input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"),
|
||||||
timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000,
|
timeoutMs: (timeoutSeconds + timing.nativeTransportGraceSeconds) * 1000,
|
||||||
env: {
|
env: {
|
||||||
...process.env,
|
...process.env,
|
||||||
NAMESPACE: namespace,
|
NAMESPACE: namespace,
|
||||||
PIPELINERUN: pipelineRun,
|
PIPELINERUN: pipelineRun,
|
||||||
WAIT: wait ? "true" : "false",
|
WAIT: wait ? "true" : "false",
|
||||||
TIMEOUT_SECONDS: String(timeoutSeconds),
|
TIMEOUT_SECONDS: String(timeoutSeconds),
|
||||||
|
POLL_INTERVAL_SECONDS: String(timing.nativePollIntervalSeconds),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export function runNativeK8sJob(namespace: string, jobName: string, manifest: Record<string, unknown>, timeoutSeconds: number, logContainer: string): NativeK8sJobResult {
|
export function runNativeK8sJob(namespace: string, jobName: string, manifest: Record<string, unknown>, timeoutSeconds: number, logContainer: string, timing: NativeTimingOptions): NativeK8sJobResult {
|
||||||
const result = runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "native-job.mjs")], repoRoot, {
|
const result = runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "native-job.mjs")], repoRoot, {
|
||||||
input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"),
|
input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"),
|
||||||
timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000,
|
timeoutMs: (timeoutSeconds + timing.nativeTransportGraceSeconds) * 1000,
|
||||||
env: {
|
env: {
|
||||||
...process.env,
|
...process.env,
|
||||||
NAMESPACE: namespace,
|
NAMESPACE: namespace,
|
||||||
JOB_NAME: jobName,
|
JOB_NAME: jobName,
|
||||||
LOG_CONTAINER: logContainer,
|
LOG_CONTAINER: logContainer,
|
||||||
TIMEOUT_SECONDS: String(timeoutSeconds),
|
TIMEOUT_SECONDS: String(timeoutSeconds),
|
||||||
|
POLL_INTERVAL_SECONDS: String(timing.nativePollIntervalSeconds),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null;
|
const parsed = parseJsonObject(result.stdout);
|
||||||
return {
|
return {
|
||||||
ok: result.exitCode === 0 && parsed?.ok === true,
|
ok: result.exitCode === 0 && parsed?.ok === true,
|
||||||
completed: parsed?.completed === true,
|
completed: parsed?.completed === true,
|
||||||
|
|||||||
@@ -161,6 +161,8 @@ export interface ControllerSpec {
|
|||||||
reconcileJobBackoffLimit: number;
|
reconcileJobBackoffLimit: number;
|
||||||
reconcileJobDeadlineGraceSeconds: number;
|
reconcileJobDeadlineGraceSeconds: number;
|
||||||
reconcileTransportGraceSeconds: number;
|
reconcileTransportGraceSeconds: number;
|
||||||
|
nativeTransportGraceSeconds: number;
|
||||||
|
nativePollIntervalSeconds: number;
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+16
-19
@@ -289,6 +289,8 @@ function parseController(root: Record<string, unknown>): ControllerSpec {
|
|||||||
reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"),
|
reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"),
|
||||||
reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"),
|
reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"),
|
||||||
reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"),
|
reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"),
|
||||||
|
nativeTransportGraceSeconds: integerField(budgets, "nativeTransportGraceSeconds", "controller.budgets"),
|
||||||
|
nativePollIntervalSeconds: integerField(budgets, "nativePollIntervalSeconds", "controller.budgets"),
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) {
|
if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) {
|
||||||
@@ -798,7 +800,7 @@ async function decideAndMaybeTrigger(
|
|||||||
const trigger = await executeTrigger(registry, follower, observedSha, options);
|
const trigger = await executeTrigger(registry, follower, observedSha, options);
|
||||||
triggerCommand = trigger.command;
|
triggerCommand = trigger.command;
|
||||||
phase = trigger.ok ? (options.wait || options.inCluster ? "ClosingOut" : "Triggering") : "Failed";
|
phase = trigger.ok ? (options.wait || options.inCluster ? "ClosingOut" : "Triggering") : "Failed";
|
||||||
decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}`;
|
decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}: ${redactText(trigger.message).slice(0, 220)}`;
|
||||||
inFlightJob = trigger.jobId ?? live.inFlightJob;
|
inFlightJob = trigger.jobId ?? live.inFlightJob;
|
||||||
lastTriggeredSha = observedSha;
|
lastTriggeredSha = observedSha;
|
||||||
if (trigger.ok && options.wait && trigger.completed) {
|
if (trigger.ok && options.wait && trigger.completed) {
|
||||||
@@ -923,7 +925,7 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
|
|||||||
const namespace = follower.nativeStatus.tekton.namespace;
|
const namespace = follower.nativeStatus.tekton.namespace;
|
||||||
const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun);
|
const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun);
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
||||||
if (sync !== null && !sync.result.ok) {
|
if (sync !== null && !sync.result.ok) {
|
||||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt);
|
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt);
|
||||||
}
|
}
|
||||||
@@ -931,12 +933,12 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
|
|||||||
if (!refresh.result.ok) {
|
if (!refresh.result.ok) {
|
||||||
return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt);
|
return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt);
|
||||||
}
|
}
|
||||||
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds));
|
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets);
|
||||||
const payload = parseJsonObject(result.stdout) ?? {};
|
const payload = parseJsonObject(result.stdout) ?? {};
|
||||||
const pipelineRunCompleted = payload.completed === true;
|
const pipelineRunCompleted = payload.completed === true;
|
||||||
const failed = payload.failed === true || result.exitCode !== 0;
|
const failed = payload.failed === true || result.exitCode !== 0;
|
||||||
const flush = !failed && options.wait && pipelineRunCompleted
|
const flush = !failed && options.wait && pipelineRunCompleted
|
||||||
? runNativeGitMirrorStage(follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds))
|
? runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds))
|
||||||
: null;
|
: null;
|
||||||
if (flush !== null && !flush.result.ok) {
|
if (flush !== null && !flush.result.ok) {
|
||||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native git-mirror flush failed", startedAt);
|
return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native git-mirror flush failed", startedAt);
|
||||||
@@ -977,12 +979,12 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo
|
|||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
|
const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
|
||||||
const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
|
const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
|
||||||
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
||||||
if (sync !== null && !sync.result.ok) {
|
if (sync !== null && !sync.result.ok) {
|
||||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt);
|
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt);
|
||||||
}
|
}
|
||||||
const buildJob = `${jobPrefix}-build-${observedSha.slice(0, 12)}`.slice(0, 63);
|
const buildJob = `${jobPrefix}-build-${observedSha.slice(0, 12)}`.slice(0, 63);
|
||||||
const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds)), "buildkit");
|
const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), spec.deployment.manager.imageBuild.timeoutSeconds), "buildkit", registry.controller.budgets);
|
||||||
const buildPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(build.logsTail) ?? "" });
|
const buildPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(build.logsTail) ?? "" });
|
||||||
const digest = stringOrNull(buildPayload.digest);
|
const digest = stringOrNull(buildPayload.digest);
|
||||||
const envIdentity = stringOrNull(buildPayload.envIdentity);
|
const envIdentity = stringOrNull(buildPayload.envIdentity);
|
||||||
@@ -992,17 +994,17 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo
|
|||||||
const image = agentRunImageArtifact(spec, { sourceCommit: observedSha, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" });
|
const image = agentRunImageArtifact(spec, { sourceCommit: observedSha, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" });
|
||||||
const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: observedSha, image });
|
const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: observedSha, image });
|
||||||
const publishJob = `${jobPrefix}-gitops-${observedSha.slice(0, 12)}`.slice(0, 63);
|
const publishJob = `${jobPrefix}-gitops-${observedSha.slice(0, 12)}`.slice(0, 63);
|
||||||
const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish");
|
const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish", registry.controller.budgets);
|
||||||
const publishPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(publish.logsTail) ?? "" });
|
const publishPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(publish.logsTail) ?? "" });
|
||||||
if (!publish.ok || publishPayload.ok === false || stringOrNull(publishPayload.gitopsCommit) === null) {
|
if (!publish.ok || publishPayload.ok === false || stringOrNull(publishPayload.gitopsCommit) === null) {
|
||||||
return nativeK8sStageFailure(follower, observedSha, "gitops-publish", publishJob, publish, publishPayload, "native AgentRun GitOps publish failed", startedAt);
|
return nativeK8sStageFailure(follower, observedSha, "gitops-publish", publishJob, publish, publishPayload, "native AgentRun GitOps publish failed", startedAt);
|
||||||
}
|
}
|
||||||
const flush = runNativeGitMirrorStage(follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds));
|
const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds));
|
||||||
if (flush !== null && !flush.result.ok) {
|
if (flush !== null && !flush.result.ok) {
|
||||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native AgentRun git-mirror flush failed", startedAt);
|
return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native AgentRun git-mirror flush failed", startedAt);
|
||||||
}
|
}
|
||||||
const pipelineRun = agentRunPipelineRunName(spec, observedSha);
|
const pipelineRun = agentRunPipelineRunName(spec, observedSha);
|
||||||
const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds));
|
const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets);
|
||||||
const tektonPayload = parseJsonObject(tektonResult.stdout) ?? {};
|
const tektonPayload = parseJsonObject(tektonResult.stdout) ?? {};
|
||||||
const pipelineRunCompleted = tektonPayload.completed === true;
|
const pipelineRunCompleted = tektonPayload.completed === true;
|
||||||
const failed = tektonPayload.failed === true || tektonResult.exitCode !== 0;
|
const failed = tektonPayload.failed === true || tektonResult.exitCode !== 0;
|
||||||
@@ -1084,15 +1086,10 @@ function nativeK8sStageFailure(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function runNativeGitMirrorStage(
|
function runNativeGitMirrorStage(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string, action: "sync" | "flush", timeoutSeconds: number): { jobName: string; namespace: string; result: NativeK8sJobResult } | null {
|
||||||
follower: FollowerSpec,
|
|
||||||
observedSha: string,
|
|
||||||
action: "sync" | "flush",
|
|
||||||
timeoutSeconds: number,
|
|
||||||
): { jobName: string; namespace: string; result: NativeK8sJobResult } | null {
|
|
||||||
const job = nativeGitMirrorJobForFollower(follower, observedSha, action);
|
const job = nativeGitMirrorJobForFollower(follower, observedSha, action);
|
||||||
if (job === null) return null;
|
if (job === null) return null;
|
||||||
const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action);
|
const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action, registry.controller.budgets);
|
||||||
return { jobName: job.jobName, namespace: job.namespace, result };
|
return { jobName: job.jobName, namespace: job.namespace, result };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1228,7 +1225,7 @@ async function executeNativeSentinelTrigger(registry: BranchFollowerRegistry, fo
|
|||||||
const namespace = stringField(recordField(state.cicd, "builder", `${follower.id}.sentinel.cicd`), "namespace", `${follower.id}.sentinel.cicd.builder`);
|
const namespace = stringField(recordField(state.cicd, "builder", `${follower.id}.sentinel.cicd`), "namespace", `${follower.id}.sentinel.cicd.builder`);
|
||||||
const manifest = sentinelPublishPipelineRunManifest(state, pipelineRun, true);
|
const manifest = sentinelPublishPipelineRunManifest(state, pipelineRun, true);
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds);
|
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds, registry.controller.budgets);
|
||||||
const payload = parseJsonObject(result.stdout) ?? {};
|
const payload = parseJsonObject(result.stdout) ?? {};
|
||||||
const pipelineRunCompleted = payload.completed === true;
|
const pipelineRunCompleted = payload.completed === true;
|
||||||
const failed = payload.failed === true || result.exitCode !== 0;
|
const failed = payload.failed === true || result.exitCode !== 0;
|
||||||
@@ -1301,7 +1298,7 @@ async function waitNativeSentinelCloseout(
|
|||||||
const latestPayload = asOptionalRecord(latest.payload);
|
const latestPayload = asOptionalRecord(latest.payload);
|
||||||
const latestGitMirror = asOptionalRecord(latestPayload?.gitMirror);
|
const latestGitMirror = asOptionalRecord(latestPayload?.gitMirror);
|
||||||
if (latest.observedSha === observedSha && gitMirrorFlush === null && shouldFlushNativeGitMirrorDuringCloseout(follower, latestGitMirror)) {
|
if (latest.observedSha === observedSha && gitMirrorFlush === null && shouldFlushNativeGitMirrorDuringCloseout(follower, latestGitMirror)) {
|
||||||
const flush = runNativeGitMirrorStage(follower, observedSha, "flush", Math.max(1, Math.min(remainingSeconds, follower.budgets.sourceSyncSeconds)));
|
const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", Math.min(remainingSeconds, follower.budgets.sourceSyncSeconds));
|
||||||
gitMirrorFlush = flush === null ? null : {
|
gitMirrorFlush = flush === null ? null : {
|
||||||
jobName: flush.jobName,
|
jobName: flush.jobName,
|
||||||
namespace: flush.namespace,
|
namespace: flush.namespace,
|
||||||
@@ -2123,7 +2120,7 @@ function compactCloseoutGitMirrorFlush(value: Record<string, unknown> | null): R
|
|||||||
}
|
}
|
||||||
|
|
||||||
function compactStateWarnings(warnings: string[]): string[] {
|
function compactStateWarnings(warnings: string[]): string[] {
|
||||||
return warnings.slice(0, 4).map((item) => redactText(tailText(item, 400)));
|
return warnings.slice(0, 4).map((item) => redactText(item.length <= 800 ? item : `${item.slice(0, 400)} ... ${item.slice(-400)}`));
|
||||||
}
|
}
|
||||||
|
|
||||||
function compactNativePayload(payload: Record<string, unknown> | null): Record<string, unknown> | null {
|
function compactNativePayload(payload: Record<string, unknown> | null): Record<string, unknown> | null {
|
||||||
|
|||||||
Reference in New Issue
Block a user