fix: retry agentrun git mirror transients
This commit is contained in:
Codex
+231
-43
@@ -3888,34 +3888,135 @@ function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
const AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS = 5;
|
||||
const AGENTRUN_GIT_MIRROR_RETRYABLE_PATTERN =
|
||||
/(kex_exchange_identification|connection closed by remote host|connection closed by unknown|could not read from remote repository|ssh\.github\.com|github\.com|fetch-pack|early eof|econnreset|read econnreset|proxy-connect|tunnel socket error)/i;
|
||||
|
||||
function agentRunGitMirrorRetryDelayMs(attempt: number): number {
|
||||
return Math.min(15_000, 1_000 * Math.pow(2, Math.max(0, attempt - 1)));
|
||||
}
|
||||
|
||||
function agentRunGitMirrorFailureText(value: unknown): string {
|
||||
try {
|
||||
return JSON.stringify(value);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
function agentRunGitMirrorRetryableFailure(
|
||||
spec: AgentRunLaneSpec,
|
||||
action: "sync" | "flush",
|
||||
attempt: number,
|
||||
value: unknown,
|
||||
): Record<string, unknown> | null {
|
||||
const text = agentRunGitMirrorFailureText(value);
|
||||
if (!AGENTRUN_GIT_MIRROR_RETRYABLE_PATTERN.test(text)) return null;
|
||||
const retryExhausted = attempt >= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS;
|
||||
const retryBackoffMs = retryExhausted ? 0 : agentRunGitMirrorRetryDelayMs(attempt);
|
||||
return {
|
||||
retryable: true,
|
||||
retryAttempt: attempt,
|
||||
retryMaxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS,
|
||||
retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`,
|
||||
retryExhausted,
|
||||
retryBackoffMs,
|
||||
backoffPolicy: "exponential",
|
||||
failureKind: "git-mirror-upstream-transient",
|
||||
upstream: "git-mirror-ssh",
|
||||
action,
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
message: retryExhausted
|
||||
? "AgentRun git-mirror upstream transient failure exhausted retries; stopping instead of continuing silently."
|
||||
: "AgentRun git-mirror upstream transient failure; retrying with exponential backoff.",
|
||||
next: retryExhausted
|
||||
? { stopped: true, reason: "retry-exhausted" }
|
||||
: {
|
||||
retry: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`,
|
||||
delayMs: retryBackoffMs,
|
||||
},
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function agentRunGitMirrorRetrySummary(attempts: Record<string, unknown>[]): Record<string, unknown> {
|
||||
const last = attempts.length > 0 ? record(attempts[attempts.length - 1]) : {};
|
||||
return {
|
||||
policy: "exponential",
|
||||
maxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS,
|
||||
attempts,
|
||||
exhausted: last.retryExhausted === true,
|
||||
stopped: last.retryExhausted === true,
|
||||
valuesPrinted: false,
|
||||
};
|
||||
}
|
||||
|
||||
async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise<Record<string, unknown>> {
|
||||
const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}`.slice(0, 63);
|
||||
const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName);
|
||||
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
|
||||
if (created.exitCode !== 0) {
|
||||
return { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false };
|
||||
const attempts: Record<string, unknown>[] = [];
|
||||
for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
||||
const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63);
|
||||
const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName);
|
||||
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
|
||||
if (created.exitCode !== 0) {
|
||||
const result = { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false };
|
||||
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
|
||||
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
||||
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
||||
progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
|
||||
await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt));
|
||||
continue;
|
||||
}
|
||||
return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) };
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
let polls = 0;
|
||||
let lastProbe: SshCaptureResult | null = null;
|
||||
while (Date.now() - startedAt < 300_000) {
|
||||
polls += 1;
|
||||
lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]);
|
||||
const payload = captureJsonPayload(lastProbe);
|
||||
progressEvent("agentrun.yaml-lane.git-mirror.progress", {
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
jobName,
|
||||
polls,
|
||||
retryAttempt: attempt,
|
||||
retryMaxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS,
|
||||
retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`,
|
||||
succeeded: payload.succeeded === true,
|
||||
failed: payload.failed === true,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
});
|
||||
if (payload.succeeded === true) {
|
||||
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: true, polls, elapsedMs: Date.now() - startedAt });
|
||||
return { ok: true, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false };
|
||||
}
|
||||
if (payload.failed === true) {
|
||||
const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
|
||||
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
|
||||
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
||||
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
||||
progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
|
||||
await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt));
|
||||
break;
|
||||
}
|
||||
return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) };
|
||||
}
|
||||
await sleep(5_000);
|
||||
}
|
||||
if (attempts.length > 0 && record(attempts[attempts.length - 1]).jobName === jobName) continue;
|
||||
const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
|
||||
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result);
|
||||
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
||||
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
||||
progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure);
|
||||
await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt));
|
||||
continue;
|
||||
}
|
||||
return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) };
|
||||
}
|
||||
const startedAt = Date.now();
|
||||
let polls = 0;
|
||||
let lastProbe: SshCaptureResult | null = null;
|
||||
while (Date.now() - startedAt < 300_000) {
|
||||
polls += 1;
|
||||
lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]);
|
||||
const payload = captureJsonPayload(lastProbe);
|
||||
progressEvent("agentrun.yaml-lane.git-mirror.progress", {
|
||||
node: spec.nodeId,
|
||||
lane: spec.lane,
|
||||
jobName,
|
||||
polls,
|
||||
succeeded: payload.succeeded === true,
|
||||
failed: payload.failed === true,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
});
|
||||
if (payload.succeeded === true) return { ok: true, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, valuesPrinted: false };
|
||||
if (payload.failed === true) return { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
|
||||
await sleep(5_000);
|
||||
}
|
||||
return { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false };
|
||||
return { ok: false, degradedReason: "git-mirror-sync-retry-loop-exhausted", retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false };
|
||||
}
|
||||
|
||||
function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCommit: string, pipelineRun: string): string {
|
||||
@@ -5252,8 +5353,8 @@ async function readGitMirrorStatus(config: UniDeskConfig, target: { configPath:
|
||||
|
||||
async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", options: GitMirrorOptions): Promise<Record<string, unknown>> {
|
||||
const { configPath, spec } = resolveAgentRunLaneTarget(options);
|
||||
const jobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63);
|
||||
const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName);
|
||||
const previewJobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63);
|
||||
const previewManifest = yamlLaneGitMirrorJobManifest(spec, action, previewJobName);
|
||||
const command = `agentrun git-mirror ${action}`;
|
||||
if (options.dryRun || !options.confirm) {
|
||||
return {
|
||||
@@ -5264,18 +5365,20 @@ async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush",
|
||||
target: agentRunLaneSummary(spec),
|
||||
dryRun: true,
|
||||
namespace: spec.gitMirror.namespace,
|
||||
jobName,
|
||||
manifest,
|
||||
jobName: previewJobName,
|
||||
manifest: previewManifest,
|
||||
next: { confirm: `bun scripts/cli.ts ${command} --node ${spec.nodeId} --lane ${spec.lane} --confirm` },
|
||||
};
|
||||
}
|
||||
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
|
||||
if (created.exitCode !== 0 || !options.wait) {
|
||||
if (!options.wait) {
|
||||
const jobName = previewJobName;
|
||||
const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName);
|
||||
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
|
||||
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
||||
return {
|
||||
ok: created.exitCode === 0,
|
||||
command,
|
||||
mode: options.wait ? "create-failed" : "submitted",
|
||||
mode: created.exitCode === 0 ? "submitted" : "create-failed",
|
||||
configPath,
|
||||
target: agentRunLaneSummary(spec),
|
||||
dryRun: false,
|
||||
@@ -5289,24 +5392,109 @@ async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush",
|
||||
},
|
||||
};
|
||||
}
|
||||
const wait = await waitForGitMirrorJob(config, spec, action, jobName, options.timeoutSeconds);
|
||||
const attempts: Record<string, unknown>[] = [];
|
||||
let lastCreated: SshCaptureResult | null = null;
|
||||
let lastWait: Record<string, unknown> | null = null;
|
||||
for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
||||
const jobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63);
|
||||
const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName);
|
||||
const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]);
|
||||
lastCreated = created;
|
||||
if (created.exitCode !== 0) {
|
||||
const result = { phase: "create-job", capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }) };
|
||||
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result);
|
||||
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
||||
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
||||
progressEvent(`agentrun.git-mirror.${action}.retry`, retryableFailure);
|
||||
await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt));
|
||||
continue;
|
||||
}
|
||||
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
||||
return {
|
||||
ok: false,
|
||||
command,
|
||||
mode: "create-failed",
|
||||
configPath,
|
||||
target: agentRunLaneSummary(spec),
|
||||
dryRun: false,
|
||||
namespace: spec.gitMirror.namespace,
|
||||
jobName,
|
||||
result: compactCapture(created),
|
||||
retryableFailure,
|
||||
retry: agentRunGitMirrorRetrySummary(attempts),
|
||||
status,
|
||||
next: {
|
||||
status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
||||
wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`,
|
||||
},
|
||||
};
|
||||
}
|
||||
const wait = await waitForGitMirrorJob(config, spec, action, jobName, options.timeoutSeconds);
|
||||
lastWait = wait;
|
||||
if (wait.ok === true) {
|
||||
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: true, wait });
|
||||
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
||||
return {
|
||||
ok: true,
|
||||
command,
|
||||
mode: "waited",
|
||||
configPath,
|
||||
target: agentRunLaneSummary(spec),
|
||||
dryRun: false,
|
||||
namespace: spec.gitMirror.namespace,
|
||||
jobName,
|
||||
result: compactCapture(created),
|
||||
wait,
|
||||
retry: agentRunGitMirrorRetrySummary(attempts),
|
||||
status,
|
||||
next: {
|
||||
status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
||||
flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null,
|
||||
},
|
||||
};
|
||||
}
|
||||
const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, wait);
|
||||
attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, wait, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true });
|
||||
if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) {
|
||||
progressEvent(`agentrun.git-mirror.${action}.retry`, retryableFailure);
|
||||
await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt));
|
||||
continue;
|
||||
}
|
||||
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
||||
return {
|
||||
ok: false,
|
||||
command,
|
||||
mode: "waited",
|
||||
configPath,
|
||||
target: agentRunLaneSummary(spec),
|
||||
dryRun: false,
|
||||
namespace: spec.gitMirror.namespace,
|
||||
jobName,
|
||||
result: compactCapture(created),
|
||||
wait,
|
||||
retryableFailure,
|
||||
retry: agentRunGitMirrorRetrySummary(attempts),
|
||||
status,
|
||||
next: {
|
||||
status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
||||
wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`,
|
||||
},
|
||||
};
|
||||
}
|
||||
const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane });
|
||||
return {
|
||||
ok: created.exitCode === 0 && wait.ok === true,
|
||||
ok: false,
|
||||
command,
|
||||
mode: "waited",
|
||||
mode: "retry-exhausted",
|
||||
configPath,
|
||||
target: agentRunLaneSummary(spec),
|
||||
dryRun: false,
|
||||
namespace: spec.gitMirror.namespace,
|
||||
jobName,
|
||||
result: compactCapture(created),
|
||||
wait,
|
||||
result: lastCreated === null ? null : compactCapture(lastCreated),
|
||||
wait: lastWait,
|
||||
retry: agentRunGitMirrorRetrySummary(attempts),
|
||||
status,
|
||||
next: {
|
||||
status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`,
|
||||
flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null,
|
||||
},
|
||||
next: { status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` },
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user