diff --git a/scripts/src/agentrun.ts b/scripts/src/agentrun.ts index 65ecd6cd..0ddf90dc 100644 --- a/scripts/src/agentrun.ts +++ b/scripts/src/agentrun.ts @@ -3888,34 +3888,135 @@ function yamlLaneGitopsPublishStatusScript(spec: AgentRunLaneSpec, jobId: string ].join("\n"); } +const AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS = 5; +const AGENTRUN_GIT_MIRROR_RETRYABLE_PATTERN = + /(kex_exchange_identification|connection closed by remote host|connection closed by unknown|could not read from remote repository|ssh\.github\.com|github\.com|fetch-pack|early eof|econnreset|read econnreset|proxy-connect|tunnel socket error)/i; + +function agentRunGitMirrorRetryDelayMs(attempt: number): number { + return Math.min(15_000, 1_000 * Math.pow(2, Math.max(0, attempt - 1))); +} + +function agentRunGitMirrorFailureText(value: unknown): string { + try { + return JSON.stringify(value); + } catch { + return String(value); + } +} + +function agentRunGitMirrorRetryableFailure( + spec: AgentRunLaneSpec, + action: "sync" | "flush", + attempt: number, + value: unknown, +): Record | null { + const text = agentRunGitMirrorFailureText(value); + if (!AGENTRUN_GIT_MIRROR_RETRYABLE_PATTERN.test(text)) return null; + const retryExhausted = attempt >= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; + const retryBackoffMs = retryExhausted ? 0 : agentRunGitMirrorRetryDelayMs(attempt); + return { + retryable: true, + retryAttempt: attempt, + retryMaxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS, + retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, + retryExhausted, + retryBackoffMs, + backoffPolicy: "exponential", + failureKind: "git-mirror-upstream-transient", + upstream: "git-mirror-ssh", + action, + node: spec.nodeId, + lane: spec.lane, + message: retryExhausted + ? "AgentRun git-mirror upstream transient failure exhausted retries; stopping instead of continuing silently." + : "AgentRun git-mirror upstream transient failure; retrying with exponential backoff.", + next: retryExhausted + ? { stopped: true, reason: "retry-exhausted" } + : { + retry: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`, + delayMs: retryBackoffMs, + }, + valuesPrinted: false, + }; +} + +function agentRunGitMirrorRetrySummary(attempts: Record[]): Record { + const last = attempts.length > 0 ? record(attempts[attempts.length - 1]) : {}; + return { + policy: "exponential", + maxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS, + attempts, + exhausted: last.retryExhausted === true, + stopped: last.retryExhausted === true, + valuesPrinted: false, + }; +} + async function runYamlLaneGitMirrorSyncJob(config: UniDeskConfig, spec: AgentRunLaneSpec): Promise> { - const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}`.slice(0, 63); - const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName); - const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); - if (created.exitCode !== 0) { - return { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false }; + const attempts: Record[] = []; + for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) { + const jobName = `${spec.gitMirror.syncJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63); + const manifest = yamlLaneGitMirrorJobManifest(spec, "sync", jobName); + const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); + if (created.exitCode !== 0) { + const result = { ok: false, phase: "create-job", jobName, capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }), valuesPrinted: false }; + const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result); + attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); + if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { + progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure); + await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt)); + continue; + } + return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) }; + } + const startedAt = Date.now(); + let polls = 0; + let lastProbe: SshCaptureResult | null = null; + while (Date.now() - startedAt < 300_000) { + polls += 1; + lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]); + const payload = captureJsonPayload(lastProbe); + progressEvent("agentrun.yaml-lane.git-mirror.progress", { + node: spec.nodeId, + lane: spec.lane, + jobName, + polls, + retryAttempt: attempt, + retryMaxAttempts: AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS, + retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, + succeeded: payload.succeeded === true, + failed: payload.failed === true, + elapsedMs: Date.now() - startedAt, + }); + if (payload.succeeded === true) { + attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: true, polls, elapsedMs: Date.now() - startedAt }); + return { ok: true, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false }; + } + if (payload.failed === true) { + const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; + const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result); + attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); + if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { + progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure); + await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt)); + break; + } + return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) }; + } + await sleep(5_000); + } + if (attempts.length > 0 && record(attempts[attempts.length - 1]).jobName === jobName) continue; + const result = { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; + const retryableFailure = agentRunGitMirrorRetryableFailure(spec, "sync", attempt, result); + attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); + if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { + progressEvent("agentrun.yaml-lane.git-mirror.retry", retryableFailure); + await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt)); + continue; + } + return { ...result, retryableFailure, retry: agentRunGitMirrorRetrySummary(attempts) }; } - const startedAt = Date.now(); - let polls = 0; - let lastProbe: SshCaptureResult | null = null; - while (Date.now() - startedAt < 300_000) { - polls += 1; - lastProbe = await capture(config, spec.nodeKubeRoute, ["sh", "--", yamlLaneJobProbeScript(spec.gitMirror.namespace, jobName)]); - const payload = captureJsonPayload(lastProbe); - progressEvent("agentrun.yaml-lane.git-mirror.progress", { - node: spec.nodeId, - lane: spec.lane, - jobName, - polls, - succeeded: payload.succeeded === true, - failed: payload.failed === true, - elapsedMs: Date.now() - startedAt, - }); - if (payload.succeeded === true) return { ok: true, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, valuesPrinted: false }; - if (payload.failed === true) return { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, result: payload, capture: compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; - await sleep(5_000); - } - return { ok: false, jobName, polls, elapsedMs: Date.now() - startedAt, degradedReason: "git-mirror-sync-job-timeout", capture: lastProbe === null ? null : compactCapture(lastProbe, { full: true, stdoutTailChars: 5000, stderrTailChars: 3000 }), valuesPrinted: false }; + return { ok: false, degradedReason: "git-mirror-sync-retry-loop-exhausted", retry: agentRunGitMirrorRetrySummary(attempts), valuesPrinted: false }; } function yamlLanePipelineRunCreateScript(spec: AgentRunLaneSpec, sourceCommit: string, pipelineRun: string): string { @@ -5252,8 +5353,8 @@ async function readGitMirrorStatus(config: UniDeskConfig, target: { configPath: async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", options: GitMirrorOptions): Promise> { const { configPath, spec } = resolveAgentRunLaneTarget(options); - const jobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63); - const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName); + const previewJobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}`.slice(0, 63); + const previewManifest = yamlLaneGitMirrorJobManifest(spec, action, previewJobName); const command = `agentrun git-mirror ${action}`; if (options.dryRun || !options.confirm) { return { @@ -5264,18 +5365,20 @@ async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", target: agentRunLaneSummary(spec), dryRun: true, namespace: spec.gitMirror.namespace, - jobName, - manifest, + jobName: previewJobName, + manifest: previewManifest, next: { confirm: `bun scripts/cli.ts ${command} --node ${spec.nodeId} --lane ${spec.lane} --confirm` }, }; } - const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); - if (created.exitCode !== 0 || !options.wait) { + if (!options.wait) { + const jobName = previewJobName; + const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName); + const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); return { ok: created.exitCode === 0, command, - mode: options.wait ? "create-failed" : "submitted", + mode: created.exitCode === 0 ? "submitted" : "create-failed", configPath, target: agentRunLaneSummary(spec), dryRun: false, @@ -5289,24 +5392,109 @@ async function runGitMirrorJob(config: UniDeskConfig, action: "sync" | "flush", }, }; } - const wait = await waitForGitMirrorJob(config, spec, action, jobName, options.timeoutSeconds); + const attempts: Record[] = []; + let lastCreated: SshCaptureResult | null = null; + let lastWait: Record | null = null; + for (let attempt = 1; attempt <= AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS; attempt += 1) { + const jobName = `${action === "sync" ? spec.gitMirror.syncJobPrefix : spec.gitMirror.flushJobPrefix}-${Date.now().toString(36)}-${attempt}`.slice(0, 63); + const manifest = yamlLaneGitMirrorJobManifest(spec, action, jobName); + const created = await capture(config, spec.nodeKubeRoute, ["sh", "--", createYamlLaneJobScript(spec.gitMirror.namespace, jobName, manifest)]); + lastCreated = created; + if (created.exitCode !== 0) { + const result = { phase: "create-job", capture: compactCapture(created, { full: true, stdoutTailChars: 4000, stderrTailChars: 4000 }) }; + const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, result); + attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, phase: "create-job", retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); + if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { + progressEvent(`agentrun.git-mirror.${action}.retry`, retryableFailure); + await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt)); + continue; + } + const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); + return { + ok: false, + command, + mode: "create-failed", + configPath, + target: agentRunLaneSummary(spec), + dryRun: false, + namespace: spec.gitMirror.namespace, + jobName, + result: compactCapture(created), + retryableFailure, + retry: agentRunGitMirrorRetrySummary(attempts), + status, + next: { + status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, + wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`, + }, + }; + } + const wait = await waitForGitMirrorJob(config, spec, action, jobName, options.timeoutSeconds); + lastWait = wait; + if (wait.ok === true) { + attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: true, wait }); + const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); + return { + ok: true, + command, + mode: "waited", + configPath, + target: agentRunLaneSummary(spec), + dryRun: false, + namespace: spec.gitMirror.namespace, + jobName, + result: compactCapture(created), + wait, + retry: agentRunGitMirrorRetrySummary(attempts), + status, + next: { + status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, + flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null, + }, + }; + } + const retryableFailure = agentRunGitMirrorRetryableFailure(spec, action, attempt, wait); + attempts.push({ attempt, retryLabel: `${attempt}/${AGENTRUN_GIT_MIRROR_RETRY_MAX_ATTEMPTS}`, jobName, ok: false, wait, retryableFailure, retryExhausted: retryableFailure !== null && record(retryableFailure).retryExhausted === true }); + if (retryableFailure !== null && record(retryableFailure).retryExhausted !== true) { + progressEvent(`agentrun.git-mirror.${action}.retry`, retryableFailure); + await sleep(numberOrNull(record(retryableFailure).retryBackoffMs) ?? agentRunGitMirrorRetryDelayMs(attempt)); + continue; + } + const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); + return { + ok: false, + command, + mode: "waited", + configPath, + target: agentRunLaneSummary(spec), + dryRun: false, + namespace: spec.gitMirror.namespace, + jobName, + result: compactCapture(created), + wait, + retryableFailure, + retry: agentRunGitMirrorRetrySummary(attempts), + status, + next: { + status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, + wait: `bun scripts/cli.ts agentrun git-mirror ${action} --node ${spec.nodeId} --lane ${spec.lane} --confirm --wait`, + }, + }; + } const status = await gitMirrorStatus(config, { full: false, raw: false, node: spec.nodeId, lane: spec.lane }); return { - ok: created.exitCode === 0 && wait.ok === true, + ok: false, command, - mode: "waited", + mode: "retry-exhausted", configPath, target: agentRunLaneSummary(spec), dryRun: false, namespace: spec.gitMirror.namespace, - jobName, - result: compactCapture(created), - wait, + result: lastCreated === null ? null : compactCapture(lastCreated), + wait: lastWait, + retry: agentRunGitMirrorRetrySummary(attempts), status, - next: { - status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}`, - flush: action === "sync" && status.summary && record(status.summary).pendingFlush === true ? `bun scripts/cli.ts agentrun git-mirror flush --node ${spec.nodeId} --lane ${spec.lane} --confirm` : null, - }, + next: { status: `bun scripts/cli.ts agentrun git-mirror status --node ${spec.nodeId} --lane ${spec.lane}` }, }; }