mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: speed up native branch follower refresh

Browse Source
This commit is contained in:
Codex
2026-07-03 14:21:39 +00:00
parent 7751cc167b
commit f3fccb8f35
8 changed files with 99 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/cicd-hwlab-refresh.ts
+1 -1
View File
@@ -14,7 +14,7 @@ export function runNativeHwlabControlPlaneRefresh(
jobName: string,
): { jobName: string; namespace: string; result: NativeK8sJobResult } {
const namespace = registry.controller.namespace;
const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh");
const result = runNativeK8sJob(namespace, jobName, nativeHwlabControlPlaneRefreshJobManifest(registry, follower, spec, observedSha, jobName), timeoutSeconds, "control-plane-refresh", registry.controller.budgets);
return { jobName, namespace, result };
}
scripts/src/cicd-native.ts
+12 -5
View File
@@ -6,33 +6,40 @@ import type { NativeK8sJobResult } from "./cicd-types";
const NATIVE_SCRIPT_DIR = "scripts/native/cicd";
export function runNativeTektonPipelineRun(namespace: string, pipelineRun: string, manifest: Record<string, unknown>, wait: boolean, timeoutSeconds: number): CommandResult {
interface NativeTimingOptions {
nativeTransportGraceSeconds: number;
nativePollIntervalSeconds: number;
}
export function runNativeTektonPipelineRun(namespace: string, pipelineRun: string, manifest: Record<string, unknown>, wait: boolean, timeoutSeconds: number, timing: NativeTimingOptions): CommandResult {
return runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "submit-pipelinerun.mjs")], repoRoot, {
input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"),
timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000,
timeoutMs: (timeoutSeconds + timing.nativeTransportGraceSeconds) * 1000,
env: {
...process.env,
NAMESPACE: namespace,
PIPELINERUN: pipelineRun,
WAIT: wait ? "true" : "false",
TIMEOUT_SECONDS: String(timeoutSeconds),
POLL_INTERVAL_SECONDS: String(timing.nativePollIntervalSeconds),
},
});
}
export function runNativeK8sJob(namespace: string, jobName: string, manifest: Record<string, unknown>, timeoutSeconds: number, logContainer: string): NativeK8sJobResult {
export function runNativeK8sJob(namespace: string, jobName: string, manifest: Record<string, unknown>, timeoutSeconds: number, logContainer: string, timing: NativeTimingOptions): NativeK8sJobResult {
const result = runCommand(["node", rootPath(NATIVE_SCRIPT_DIR, "native-job.mjs")], repoRoot, {
input: Buffer.from(JSON.stringify(manifest), "utf8").toString("base64"),
timeoutMs: Math.max(5, timeoutSeconds + 10) * 1000,
timeoutMs: (timeoutSeconds + timing.nativeTransportGraceSeconds) * 1000,
env: {
...process.env,
NAMESPACE: namespace,
JOB_NAME: jobName,
LOG_CONTAINER: logContainer,
TIMEOUT_SECONDS: String(timeoutSeconds),
POLL_INTERVAL_SECONDS: String(timing.nativePollIntervalSeconds),
},
});
const parsed = result.exitCode === 0 ? parseJsonObject(result.stdout) : null;
const parsed = parseJsonObject(result.stdout);
return {
ok: result.exitCode === 0 && parsed?.ok === true,
completed: parsed?.completed === true,
scripts/src/cicd-types.ts
+2
View File
@@ -161,6 +161,8 @@ export interface ControllerSpec {
reconcileJobBackoffLimit: number;
reconcileJobDeadlineGraceSeconds: number;
reconcileTransportGraceSeconds: number;
nativeTransportGraceSeconds: number;
nativePollIntervalSeconds: number;
};
}
scripts/src/cicd.ts
+16 -19
View File
@@ -289,6 +289,8 @@ function parseController(root: Record<string, unknown>): ControllerSpec {
reconcileJobBackoffLimit: integerField(budgets, "reconcileJobBackoffLimit", "controller.budgets"),
reconcileJobDeadlineGraceSeconds: integerField(budgets, "reconcileJobDeadlineGraceSeconds", "controller.budgets"),
reconcileTransportGraceSeconds: integerField(budgets, "reconcileTransportGraceSeconds", "controller.budgets"),
nativeTransportGraceSeconds: integerField(budgets, "nativeTransportGraceSeconds", "controller.budgets"),
nativePollIntervalSeconds: integerField(budgets, "nativePollIntervalSeconds", "controller.budgets"),
},
};
if (result.source.sourceAuthority.allowHostGit || result.source.sourceAuthority.allowHostWorkspace || result.source.sourceAuthority.allowGithubDirectInPipeline) {
@@ -798,7 +800,7 @@ async function decideAndMaybeTrigger(
const trigger = await executeTrigger(registry, follower, observedSha, options);
triggerCommand = trigger.command;
phase = trigger.ok ? (options.wait || options.inCluster ? "ClosingOut" : "Triggering") : "Failed";
decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}`;
decision = trigger.ok ? `trigger submitted for ${shortSha(observedSha)}` : `trigger failed for ${shortSha(observedSha)}: ${redactText(trigger.message).slice(0, 220)}`;
inFlightJob = trigger.jobId ?? live.inFlightJob;
lastTriggeredSha = observedSha;
if (trigger.ok && options.wait && trigger.completed) {
@@ -923,7 +925,7 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
const namespace = follower.nativeStatus.tekton.namespace;
const manifest = nodeRuntimePipelineRunManifest(spec, observedSha, pipelineRun);
const startedAt = Date.now();
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
if (sync !== null && !sync.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native git-mirror sync failed", startedAt);
}
@@ -931,12 +933,12 @@ async function executeNativeHwlabNodeTrigger(registry: BranchFollowerRegistry, f
if (!refresh.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "control-plane-refresh", refresh.jobName, refresh.result, { action: "control-plane-refresh" }, "native HWLAB control-plane refresh failed", startedAt);
}
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds));
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets);
const payload = parseJsonObject(result.stdout) ?? {};
const pipelineRunCompleted = payload.completed === true;
const failed = payload.failed === true || result.exitCode !== 0;
const flush = !failed && options.wait && pipelineRunCompleted
? runNativeGitMirrorStage(follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds))
? runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds))
: null;
if (flush !== null && !flush.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native git-mirror flush failed", startedAt);
@@ -977,12 +979,12 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo
const startedAt = Date.now();
const stageRef = `${follower.source.snapshotPrefix.replace(/\/+$/u, "")}/${observedSha}`;
const jobPrefix = `agentrun-bf-${spec.nodeId.toLowerCase()}-${spec.lane}`;
const sync = runNativeGitMirrorStage(follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
const sync = runNativeGitMirrorStage(registry, follower, observedSha, "sync", Math.min(remainingSeconds(startedAt, timeoutSeconds), follower.budgets.sourceSyncSeconds));
if (sync !== null && !sync.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "git-mirror-sync", sync.jobName, sync.result, { action: "sync" }, "native AgentRun git-mirror sync failed", startedAt);
}
const buildJob = `${jobPrefix}-build-${observedSha.slice(0, 12)}`.slice(0, 63);
const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), Math.max(60, spec.deployment.manager.imageBuild.timeoutSeconds)), "buildkit");
const build = runNativeK8sJob(spec.ci.namespace, buildJob, yamlLaneK3sBuildImageJobManifest(spec, observedSha, buildJob), Math.min(remainingSeconds(startedAt, timeoutSeconds), spec.deployment.manager.imageBuild.timeoutSeconds), "buildkit", registry.controller.budgets);
const buildPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(build.logsTail) ?? "" });
const digest = stringOrNull(buildPayload.digest);
const envIdentity = stringOrNull(buildPayload.envIdentity);
@@ -992,17 +994,17 @@ async function executeNativeAgentRunTrigger(registry: BranchFollowerRegistry, fo
const image = agentRunImageArtifact(spec, { sourceCommit: observedSha, envIdentity, digest, status: stringOrNull(buildPayload.status) ?? "built" });
const renderedFiles = renderAgentRunGitopsFiles(spec, { sourceCommit: observedSha, image });
const publishJob = `${jobPrefix}-gitops-${observedSha.slice(0, 12)}`.slice(0, 63);
const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish");
const publish = runNativeK8sJob(spec.gitMirror.namespace, publishJob, yamlLaneGitopsPublishJobManifest(spec, renderedFiles, publishJob), remainingSeconds(startedAt, timeoutSeconds), "publish", registry.controller.budgets);
const publishPayload = yamlLaneGitopsPublishPayloadFromProbe({ logsTail: stringOrNull(publish.logsTail) ?? "" });
if (!publish.ok || publishPayload.ok === false || stringOrNull(publishPayload.gitopsCommit) === null) {
return nativeK8sStageFailure(follower, observedSha, "gitops-publish", publishJob, publish, publishPayload, "native AgentRun GitOps publish failed", startedAt);
}
const flush = runNativeGitMirrorStage(follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds));
const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", remainingSeconds(startedAt, timeoutSeconds));
if (flush !== null && !flush.result.ok) {
return nativeK8sStageFailure(follower, observedSha, "git-mirror-flush", flush.jobName, flush.result, { action: "flush" }, "native AgentRun git-mirror flush failed", startedAt);
}
const pipelineRun = agentRunPipelineRunName(spec, observedSha);
const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds));
const tektonResult = runNativeTektonPipelineRun(follower.nativeStatus.tekton.namespace, pipelineRun, yamlLanePipelineRunManifest(spec, observedSha, pipelineRun), options.wait, remainingSeconds(startedAt, timeoutSeconds), registry.controller.budgets);
const tektonPayload = parseJsonObject(tektonResult.stdout) ?? {};
const pipelineRunCompleted = tektonPayload.completed === true;
const failed = tektonPayload.failed === true || tektonResult.exitCode !== 0;
@@ -1084,15 +1086,10 @@ function nativeK8sStageFailure(
};
}
function runNativeGitMirrorStage(
follower: FollowerSpec,
observedSha: string,
action: "sync" | "flush",
timeoutSeconds: number,
): { jobName: string; namespace: string; result: NativeK8sJobResult } | null {
function runNativeGitMirrorStage(registry: BranchFollowerRegistry, follower: FollowerSpec, observedSha: string, action: "sync" | "flush", timeoutSeconds: number): { jobName: string; namespace: string; result: NativeK8sJobResult } | null {
const job = nativeGitMirrorJobForFollower(follower, observedSha, action);
if (job === null) return null;
const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action);
const result = runNativeK8sJob(job.namespace, job.jobName, job.manifest, timeoutSeconds, action, registry.controller.budgets);
return { jobName: job.jobName, namespace: job.namespace, result };
}
@@ -1228,7 +1225,7 @@ async function executeNativeSentinelTrigger(registry: BranchFollowerRegistry, fo
const namespace = stringField(recordField(state.cicd, "builder", `${follower.id}.sentinel.cicd`), "namespace", `${follower.id}.sentinel.cicd.builder`);
const manifest = sentinelPublishPipelineRunManifest(state, pipelineRun, true);
const startedAt = Date.now();
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds);
const result = runNativeTektonPipelineRun(namespace, pipelineRun, manifest, options.wait, timeoutSeconds, registry.controller.budgets);
const payload = parseJsonObject(result.stdout) ?? {};
const pipelineRunCompleted = payload.completed === true;
const failed = payload.failed === true || result.exitCode !== 0;
@@ -1301,7 +1298,7 @@ async function waitNativeSentinelCloseout(
const latestPayload = asOptionalRecord(latest.payload);
const latestGitMirror = asOptionalRecord(latestPayload?.gitMirror);
if (latest.observedSha === observedSha && gitMirrorFlush === null && shouldFlushNativeGitMirrorDuringCloseout(follower, latestGitMirror)) {
const flush = runNativeGitMirrorStage(follower, observedSha, "flush", Math.max(1, Math.min(remainingSeconds, follower.budgets.sourceSyncSeconds)));
const flush = runNativeGitMirrorStage(registry, follower, observedSha, "flush", Math.min(remainingSeconds, follower.budgets.sourceSyncSeconds));
gitMirrorFlush = flush === null ? null : {
jobName: flush.jobName,
namespace: flush.namespace,
@@ -2123,7 +2120,7 @@ function compactCloseoutGitMirrorFlush(value: Record<string, unknown> | null): R
}
function compactStateWarnings(warnings: string[]): string[] {
return warnings.slice(0, 4).map((item) => redactText(tailText(item, 400)));
return warnings.slice(0, 4).map((item) => redactText(item.length <= 800 ? item : `${item.slice(0, 400)} ... ${item.slice(-400)}`));
}
function compactNativePayload(payload: Record<string, unknown> | null): Record<string, unknown> | null {