fix: speed up native branch follower refresh
This commit is contained in:
@@ -53,6 +53,8 @@ controller:
|
||||
reconcileJobBackoffLimit: 0
|
||||
reconcileJobDeadlineGraceSeconds: 30
|
||||
reconcileTransportGraceSeconds: 20
|
||||
nativeTransportGraceSeconds: 10
|
||||
nativePollIntervalSeconds: 2
|
||||
|
||||
followers:
|
||||
- id: hwlab-jd01-v03
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// SPEC: PJ2026-01060703 CI/CD branch follower native HWLAB control-plane refresh.
|
||||
// Responsibility: render and apply the HWLAB node Tekton Pipeline from a k8s git-mirror snapshot.
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -17,6 +17,7 @@ const overlay = JSON.parse(Buffer.from(requiredEnv("HWLAB_RENDER_OVERLAY_B64"),
|
||||
const workDir = mkdtempSync(path.join(tmpdir(), `hwlab-control-plane-${sourceCommit.slice(0, 12)}-`));
|
||||
const repoDir = path.join(workDir, "repo");
|
||||
const renderDir = path.join(workDir, "render");
|
||||
const depsDir = path.join(workDir, "deps");
|
||||
|
||||
try {
|
||||
checkoutSnapshot();
|
||||
@@ -39,6 +40,8 @@ function checkoutSnapshot() {
|
||||
}
|
||||
|
||||
function prepareYamlDependency() {
|
||||
mkdirSync(depsDir, { recursive: true });
|
||||
writeFileSync(path.join(depsDir, "package.json"), JSON.stringify({ private: true, dependencies: {} }), "utf8");
|
||||
if (canResolveYaml()) return;
|
||||
const registry = requiredOverlayString("npmRegistry");
|
||||
const timeoutMs = String(requiredPositiveNumber("npmFetchTimeoutMs"));
|
||||
@@ -51,10 +54,10 @@ function prepareYamlDependency() {
|
||||
npm_config_fetch_retries: retries,
|
||||
});
|
||||
if (commandExists("bun")) {
|
||||
const result = runOptional("bun", ["add", "--no-save", "--ignore-scripts", "--registry", registry, yamlSpec], { cwd: repoDir, env });
|
||||
const result = runOptional("bun", ["add", "--no-save", "--ignore-scripts", "--registry", registry, yamlSpec], { cwd: depsDir, env });
|
||||
if (result.status === 0 && canResolveYaml()) return;
|
||||
}
|
||||
run("npm", ["install", "--package-lock=false", "--no-save", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev", "--registry", registry, yamlSpec], { cwd: repoDir, env });
|
||||
run("npm", ["install", "--package-lock=false", "--no-save", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev", "--registry", registry, yamlSpec], { cwd: depsDir, env });
|
||||
if (!canResolveYaml()) throw new Error("yaml dependency remains unresolved after install");
|
||||
}
|
||||
|
||||
@@ -66,13 +69,13 @@ function yamlDependencySpec() {
|
||||
}
|
||||
|
||||
function canResolveYaml() {
|
||||
const result = runOptional("node", ["-e", "require.resolve('yaml')"], { cwd: repoDir, stdio: "ignore" });
|
||||
const result = runOptional("node", ["-e", "require.resolve('yaml')"], { cwd: depsDir, stdio: "ignore" });
|
||||
return result.status === 0;
|
||||
}
|
||||
|
||||
function applyDeployOverlay() {
|
||||
const requireFromRepo = createRequire(path.join(repoDir, "package.json"));
|
||||
const YAML = requireFromRepo("yaml");
|
||||
const requireFromDeps = createRequire(path.join(depsDir, "package.json"));
|
||||
const YAML = requireFromDeps("yaml");
|
||||
const deployPath = path.join(repoDir, "deploy/deploy.yaml");
|
||||
const doc = YAML.parse(readFileSync(deployPath, "utf8"));
|
||||
doc.nodes = doc.nodes || {};
|
||||
@@ -155,13 +158,13 @@ function renderEnv(extra = {}) {
|
||||
|
||||
function run(command, args, options = {}) {
|
||||
const result = runOptional(command, args, options);
|
||||
if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`);
|
||||
if (result.status !== 0) throw new Error(failedCommandMessage(command, args, result));
|
||||
return result;
|
||||
}
|
||||
|
||||
function capture(command, args) {
|
||||
const result = runOptional(command, args, { cwd: repoDir, encoding: "utf8", stdio: ["ignore", "pipe", "inherit"] });
|
||||
if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status}`);
|
||||
if (result.status !== 0) throw new Error(failedCommandMessage(command, args, result));
|
||||
return result.stdout || "";
|
||||
}
|
||||
|
||||
@@ -178,6 +181,17 @@ function commandExists(command) {
|
||||
return runOptional("sh", ["-lc", `command -v ${command} >/dev/null 2>&1`], { cwd: repoDir, stdio: "ignore" }).status === 0;
|
||||
}
|
||||
|
||||
function failedCommandMessage(command, args, result) {
|
||||
const signal = result.signal ? ` signal=${result.signal}` : "";
|
||||
const error = result.error ? ` error=${result.error.message}` : "";
|
||||
const stderr = typeof result.stderr === "string" && result.stderr.length > 0 ? ` stderr=${tail(result.stderr, 1000)}` : "";
|
||||
return `${command} ${args.join(" ")} failed with exit ${result.status}${signal}${error}${stderr}`;
|
||||
}
|
||||
|
||||
function tail(value, maxChars) {
|
||||
return value.length <= maxChars ? value : value.slice(value.length - maxChars);
|
||||
}
|
||||
|
||||
function requiredEnv(name) {
|
||||
const value = process.env[name];
|
||||
if (!value) throw new Error(`${name} is required`);
|
||||
|
||||
@@ -4,7 +4,8 @@ import https from "node:https";
|
||||
const namespace = process.env.NAMESPACE || "";
|
||||
const jobName = process.env.JOB_NAME || "";
|
||||
const logContainer = process.env.LOG_CONTAINER || "";
|
||||
const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS || "60");
|
||||
const timeoutSeconds = requiredPositiveNumber("TIMEOUT_SECONDS");
|
||||
const pollIntervalSeconds = requiredPositiveNumber("POLL_INTERVAL_SECONDS");
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443");
|
||||
const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim();
|
||||
@@ -75,7 +76,23 @@ async function logsTail() {
|
||||
|
||||
let created = false;
|
||||
let reused = false;
|
||||
const existing = await getJob();
|
||||
let replacedFailed = false;
|
||||
let existing = await getJob();
|
||||
if (existing && condition(existing, "Failed")) {
|
||||
await deleteJob();
|
||||
replacedFailed = true;
|
||||
existing = null;
|
||||
const deleteDeadline = Date.now() + timeoutSeconds * 1000;
|
||||
let deleted = false;
|
||||
while (Date.now() <= deleteDeadline) {
|
||||
if (await getJob() === null) {
|
||||
deleted = true;
|
||||
break;
|
||||
}
|
||||
await delay(pollIntervalSeconds * 1000);
|
||||
}
|
||||
if (!deleted) throw new Error(`timed out deleting failed job ${jobName}`);
|
||||
}
|
||||
if (existing) {
|
||||
reused = true;
|
||||
} else {
|
||||
@@ -89,7 +106,7 @@ if (existing) {
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
const deadline = startedAt + Math.max(1, timeoutSeconds) * 1000;
|
||||
const deadline = startedAt + timeoutSeconds * 1000;
|
||||
let polls = 0;
|
||||
let latest = await getJob();
|
||||
while (Date.now() <= deadline) {
|
||||
@@ -97,7 +114,7 @@ while (Date.now() <= deadline) {
|
||||
const failed = condition(latest, "Failed");
|
||||
if (complete || failed) break;
|
||||
polls += 1;
|
||||
await delay(2000);
|
||||
await delay(pollIntervalSeconds * 1000);
|
||||
latest = await getJob();
|
||||
}
|
||||
|
||||
@@ -112,6 +129,7 @@ const output = {
|
||||
timedOut,
|
||||
created,
|
||||
reused,
|
||||
replacedFailed,
|
||||
jobName,
|
||||
namespace,
|
||||
polls,
|
||||
@@ -125,3 +143,15 @@ const output = {
|
||||
};
|
||||
process.stdout.write(JSON.stringify(output));
|
||||
if (!output.ok) process.exit(1);
|
||||
|
||||
async function deleteJob() {
|
||||
const result = await request("DELETE", `/apis/batch/v1/namespaces/${encodeURIComponent(namespace)}/jobs/${encodeURIComponent(jobName)}?propagationPolicy=Background`);
|
||||
if (result.status === 404) return;
|
||||
if (result.status < 200 || result.status >= 300) throw new Error(result.text || `kube api DELETE job status ${result.status}`);
|
||||
}
|
||||
|
||||
function requiredPositiveNumber(name) {
|
||||
const value = Number(process.env[name]);
|
||||
if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive number`);
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -4,7 +4,8 @@ import https from "node:https";
|
||||
const namespace = process.env.NAMESPACE || "";
|
||||
const pipelineRun = process.env.PIPELINERUN || "";
|
||||
const shouldWait = process.env.WAIT === "true";
|
||||
const timeoutSeconds = Number(process.env.TIMEOUT_SECONDS || "60");
|
||||
const timeoutSeconds = requiredPositiveNumber("TIMEOUT_SECONDS");
|
||||
const pollIntervalSeconds = requiredPositiveNumber("POLL_INTERVAL_SECONDS");
|
||||
const host = process.env.KUBERNETES_SERVICE_HOST;
|
||||
const port = Number(process.env.KUBERNETES_SERVICE_PORT || "443");
|
||||
const token = readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8").trim();
|
||||
@@ -89,7 +90,7 @@ if (latest.found) {
|
||||
latest = await getPipelineRun();
|
||||
}
|
||||
|
||||
const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000;
|
||||
const deadline = Date.now() + timeoutSeconds * 1000;
|
||||
let polls = 0;
|
||||
while (shouldWait) {
|
||||
const condition = succeededCondition(latest.object);
|
||||
@@ -97,7 +98,7 @@ while (shouldWait) {
|
||||
if (Date.now() >= deadline) break;
|
||||
polls += 1;
|
||||
process.stderr.write(JSON.stringify({ event: "cicd.branch-follower.native-tekton.wait", pipelineRun, namespace, polls, conditionStatus: condition?.status || null, valuesRedacted: true }) + "\n");
|
||||
await delay(2000);
|
||||
await delay(pollIntervalSeconds * 1000);
|
||||
latest = await getPipelineRun();
|
||||
}
|
||||
|
||||
@@ -125,3 +126,9 @@ const output = {
|
||||
};
|
||||
process.stdout.write(JSON.stringify(output));
|
||||
if (failed) process.exit(1);
|
||||
|
||||
function requiredPositiveNumber(name) {
|
||||
const value = Number(process.env[name]);
|
||||
if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive number`);
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ export function runNativeHwlabControlPlaneRefresh(
|
||||
jobName: string,
|
||||
): { jobName: string; namespace: string; result: NativeK8sJobResult } {
|
||||
const namespace = registry.controller.namespace;
|
||||
const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh");
|
||||
const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh", registry.controller.budgets);
|
||||
return { jobName, namespace, result };
|
||||
}
|
||||
|
||||
|
||||
@@ -6,33 +6,40 @@ import type { NativeK8sJobResult } from "./cicd-types";
|
||||
|
||||
const NATIVE_SCRIPT_DIR = "scripts/native/cicd";
|
||||
|
||||
export function runNativeTektonPipelineRun(namespace: string, pipelineRun: string, manifest: Record<string, unknown>, wait: boolean, timeoutSeconds: number): CommandResult {
|
||||
interface NativeTimingOptions {
|
||||
nativeTransportGraceSeconds: number;
|
||||
nativePollIntervalSeconds: number;
|
||||
}
|
||||
|
||||
export function runNativeTektonPipelineRun(namespace: string, pipelineRun: string, manifest: Record<string, unknown>, wait: boolean, timeoutSeconds: number, timing: NativeTimingOptions): CommandResult {
|
||||
return runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "submit-pipelinerun.mjs")], repoRoot, {
|
||||
input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"),
|
||||
timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000,
|
||||
timeoutMs: (timeoutSeconds + timing.nativeTransportGraceSeconds) * 1000,
|
||||
env: {
|
||||
...process.env,
|
||||
NAMESPACE: namespace,
|
||||
PIPELINERUN: pipelineRun,
|
||||
WAIT: wait ? "true" : "false",
|
||||
TIMEOUT_SECONDS: String(timeoutSeconds),
|
||||
POLL_INTERVAL_SECONDS: String(timing.nativePollIntervalSeconds),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function runNativeK8sJob(namespace: string, jobName: string, manifest: Record<string, unknown>, timeoutSeconds: number, logContainer: string): NativeK8sJobResult {
|
||||
export function runNativeK8sJob(namespace: string, jobName: string, manifest: Record<string, unknown>, timeoutSeconds: number, logContainer: string, timing: NativeTimingOptions): NativeK8sJobResult {
|
||||
const result = runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "native-job.mjs")], repoRoot, {
|
||||
input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"),
|
||||
timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000,
|
||||
timeoutMs: (timeoutSeconds + timing.nativeTransportGraceSeconds) * 1000,
|
||||
env: {
|
||||
...process.env,
|
||||
NAMESPACE: namespace,
|
||||
JOB_NAME: jobName,
|
||||
LOG_CONTAINER: logContainer,
|
||||
TIMEOUT_SECONDS: String(timeoutSeconds),
|
||||
POLL_INTERVAL_SECONDS: String(timing.nativePollIntervalSeconds),
|
||||
},
|
||||
});
|
||||
const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null;
|
||||
const parsed = parseJsonObject(result.stdout);
|
||||
return {
|
||||
ok: result.exitCode === 0 && parsed?.ok === true,
|
||||
completed: parsed?.completed === true,
|
||||
|
||||
@@ -161,6 +161,8 @@ export interface ControllerSpec {
|
||||
reconcileJobBackoffLimit: number;
|
||||
reconcileJobDeadlineGraceSeconds: number;
|
||||
reconcileTransportGraceSeconds: number;
|
||||
nativeTransportGraceSeconds: number;
|
||||
nativePollIntervalSeconds: number;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
+16
-19
@@ -289,6 +289,8 @@ function parseController(root: Record<string, unknown>): ControllerSpec {
|
||||
reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"),
|
||||
reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"),
|
||||
reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"),
|
||||
nativeTransportGraceSeconds: integerField(budgets, "nativeTransportGraceSeconds", "controller.budgets"),
|
||||
nativePollIntervalSeconds: integerField(budgets, "nativePollIntervalSeconds", "controller.budgets"),
|
||||
},
|
||||
};
|
||||
if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) {
|
||||
@@ -798,7 +800,7 @@ async function decideAndMaybeTrigger(
|
||||
const trigger = await executeTrigger(registry, follower, observedSha, options);
|
||||
triggerCommand = trigger.command;
|
||||
phase = trigger.ok ? (options.wait || options.inCluster ? "ClosingOut" : "Triggering") : "Failed";
|
||||
decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}`;
|
||||
decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}: ${redactText(trigger.message).slice(0, 220)}`;
|
||||
inFlightJob = trigger.jobId ?? live.inFlightJob;
|
||||
lastTriggeredSha = observedSha;
|
||||
if (trigger.ok && options.wait && trigger.completed) {
|
||||
@@ -923,7 +925,7 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
|
||||
const namespace = follower.nativeStatus.tekton.namespace;
|
||||
const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun);
|
||||
const startedAt = Date.now();
|
||||
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
||||
const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
||||
if (sync !== null && !sync.result.ok) {
|
||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt);
|
||||
}
|
||||
@@ -931,12 +933,12 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
|
||||
if (!refresh.result.ok) {
|
||||
return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt);
|
||||
}
|
||||
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds));
|
||||
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets);
|
||||
const payload = parseJsonObject(result.stdout) ?? {};
|
||||
const pipelineRunCompleted = payload.completed === true;
|
||||
const failed = payload.failed === true || result.exitCode !== 0;
|
||||
const flush = !failed && options.wait && pipelineRunCompleted
|
||||
? runNativeGitMirrorStage(follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds))
|
||||
? runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds))
|
||||
: null;
|
||||
if (flush !== null && !flush.result.ok) {
|
||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native git-mirror flush failed", startedAt);
|
||||
@@ -977,12 +979,12 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo
|
||||
const startedAt = Date.now();
|
||||
const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
|
||||
const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
|
||||
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
||||
const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
|
||||
if (sync !== null && !sync.result.ok) {
|
||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt);
|
||||
}
|
||||
const buildJob = `${jobPrefix}-build-${observedSha.slice(0, 12)}`.slice(0, 63);
|
||||
const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds)), "buildkit");
|
||||
const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), spec.deployment.manager.imageBuild.timeoutSeconds), "buildkit", registry.controller.budgets);
|
||||
const buildPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(build.logsTail) ?? "" });
|
||||
const digest = stringOrNull(buildPayload.digest);
|
||||
const envIdentity = stringOrNull(buildPayload.envIdentity);
|
||||
@@ -992,17 +994,17 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo
|
||||
const image = agentRunImageArtifact(spec, { sourceCommit: observedSha, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" });
|
||||
const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: observedSha, image });
|
||||
const publishJob = `${jobPrefix}-gitops-${observedSha.slice(0, 12)}`.slice(0, 63);
|
||||
const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish");
|
||||
const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish", registry.controller.budgets);
|
||||
const publishPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(publish.logsTail) ?? "" });
|
||||
if (!publish.ok || publishPayload.ok === false || stringOrNull(publishPayload.gitopsCommit) === null) {
|
||||
return nativeK8sStageFailure(follower, observedSha, "gitops-publish", publishJob, publish, publishPayload, "native AgentRun GitOps publish failed", startedAt);
|
||||
}
|
||||
const flush = runNativeGitMirrorStage(follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds));
|
||||
const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds));
|
||||
if (flush !== null && !flush.result.ok) {
|
||||
return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native AgentRun git-mirror flush failed", startedAt);
|
||||
}
|
||||
const pipelineRun = agentRunPipelineRunName(spec, observedSha);
|
||||
const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds));
|
||||
const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets);
|
||||
const tektonPayload = parseJsonObject(tektonResult.stdout) ?? {};
|
||||
const pipelineRunCompleted = tektonPayload.completed === true;
|
||||
const failed = tektonPayload.failed === true || tektonResult.exitCode !== 0;
|
||||
@@ -1084,15 +1086,10 @@ function nativeK8sStageFailure(
|
||||
};
|
||||
}
|
||||
|
||||
function runNativeGitMirrorStage(
|
||||
follower: FollowerSpec,
|
||||
observedSha: string,
|
||||
action: "sync" | "flush",
|
||||
timeoutSeconds: number,
|
||||
): { jobName: string; namespace: string; result: NativeK8sJobResult } | null {
|
||||
function runNativeGitMirrorStage(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string, action: "sync" | "flush", timeoutSeconds: number): { jobName: string; namespace: string; result: NativeK8sJobResult } | null {
|
||||
const job = nativeGitMirrorJobForFollower(follower, observedSha, action);
|
||||
if (job === null) return null;
|
||||
const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action);
|
||||
const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action, registry.controller.budgets);
|
||||
return { jobName: job.jobName, namespace: job.namespace, result };
|
||||
}
|
||||
|
||||
@@ -1228,7 +1225,7 @@ async function executeNativeSentinelTrigger(registry: BranchFollowerRegistry, fo
|
||||
const namespace = stringField(recordField(state.cicd, "builder", `${follower.id}.sentinel.cicd`), "namespace", `${follower.id}.sentinel.cicd.builder`);
|
||||
const manifest = sentinelPublishPipelineRunManifest(state, pipelineRun, true);
|
||||
const startedAt = Date.now();
|
||||
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds);
|
||||
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds, registry.controller.budgets);
|
||||
const payload = parseJsonObject(result.stdout) ?? {};
|
||||
const pipelineRunCompleted = payload.completed === true;
|
||||
const failed = payload.failed === true || result.exitCode !== 0;
|
||||
@@ -1301,7 +1298,7 @@ async function waitNativeSentinelCloseout(
|
||||
const latestPayload = asOptionalRecord(latest.payload);
|
||||
const latestGitMirror = asOptionalRecord(latestPayload?.gitMirror);
|
||||
if (latest.observedSha === observedSha && gitMirrorFlush === null && shouldFlushNativeGitMirrorDuringCloseout(follower, latestGitMirror)) {
|
||||
const flush = runNativeGitMirrorStage(follower, observedSha, "flush", Math.max(1, Math.min(remainingSeconds, follower.budgets.sourceSyncSeconds)));
|
||||
const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", Math.min(remainingSeconds, follower.budgets.sourceSyncSeconds));
|
||||
gitMirrorFlush = flush === null ? null : {
|
||||
jobName: flush.jobName,
|
||||
namespace: flush.namespace,
|
||||
@@ -2123,7 +2120,7 @@ function compactCloseoutGitMirrorFlush(value: Record<string, unknown> | null): R
|
||||
}
|
||||
|
||||
function compactStateWarnings(warnings: string[]): string[] {
|
||||
return warnings.slice(0, 4).map((item) => redactText(tailText(item, 400)));
|
||||
return warnings.slice(0, 4).map((item) => redactText(item.length <= 800 ? item : `${item.slice(0, 400)} ... ${item.slice(-400)}`));
|
||||
}
|
||||
|
||||
function compactNativePayload(payload: Record<string, unknown> | null): Record<string, unknown> | null {
|
||||
|
||||
Reference in New Issue
Block a user