fix: retry Gitea PaC webhook mirror sync

scripts/src/platform-infra-gitea-github-sync-server.mjs

@@ -13,6 +13,9 @@ const giteaUsername = requiredEnv("GITEA_USERNAME");
 const giteaPassword = requiredEnv("GITEA_PASSWORD");
 const webhookSecret = requiredEnv("GITHUB_WEBHOOK_SECRET");
 const giteaBaseUrl = requiredEnv("UNIDESK_GITEA_SERVICE_BASE_URL").replace(/\/+$/u, "");
+const retryMaxAttempts = positiveIntegerEnv("UNIDESK_GITEA_WEBHOOK_RETRY_MAX_ATTEMPTS");
+const retryInitialDelayMs = positiveIntegerEnv("UNIDESK_GITEA_WEBHOOK_RETRY_INITIAL_DELAY_MS");
+const retryMaxDelayMs = positiveIntegerEnv("UNIDESK_GITEA_WEBHOOK_RETRY_MAX_DELAY_MS");
 const githubAuthHeader = `Authorization: Basic ${Buffer.from(`x-access-token:${githubToken}`).toString("base64")}`;
 const giteaAuthHeader = `Authorization: Basic ${Buffer.from(`${giteaUsername}:${giteaPassword}`).toString("base64")}`;
 
@@ -60,7 +63,7 @@ createServer(async (req, res) => {
     state.pending = false;
     writeJson(res, 202, { ok: true, event, repository, ref, repo: repo.key, disposition: "accepted", valuesPrinted: false });
     setImmediate(() => {
-      runSyncLoop(repo, state, startedAt);
+      void runSyncLoop(repo, state, startedAt);
     });
   } catch (error) {
     log({ event: "github-to-gitea-sync-error", ok: false, error: String(error?.message || error), valuesPrinted: false });
@@ -79,14 +82,33 @@ function syncState(key) {
   return state;
 }
 
-function runSyncLoop(repo, state, startedAt) {
+async function runSyncLoop(repo, state, startedAt) {
   let attempt = 0;
   try {
     do {
       state.pending = false;
       attempt += 1;
-      const sync = syncRepository(repo);
-      log({ event: "github-to-gitea-sync", repo: repo.key, ok: sync.ok, sourceCommit: sync.sourceCommit, snapshotRef: sync.snapshotRef, error: sync.error ? redact(sync.error) : null, attempt, pendingRerun: state.pending, elapsedMs: Date.now() - startedAt, valuesPrinted: false });
+      for (let syncAttempt = 1; syncAttempt <= retryMaxAttempts; syncAttempt += 1) {
+        const sync = syncRepository(repo);
+        const willRetry = !sync.ok && syncAttempt < retryMaxAttempts;
+        log({
+          event: "github-to-gitea-sync",
+          repo: repo.key,
+          ok: sync.ok,
+          sourceCommit: sync.sourceCommit,
+          snapshotRef: sync.snapshotRef,
+          error: sync.error ? redact(sync.error) : null,
+          attempt,
+          syncAttempt,
+          maxSyncAttempts: retryMaxAttempts,
+          willRetry,
+          pendingRerun: state.pending,
+          elapsedMs: Date.now() - startedAt,
+          valuesPrinted: false,
+        });
+        if (sync.ok || !willRetry) break;
+        await sleep(retryDelayMs(syncAttempt));
+      }
     } while (state.pending);
   } catch (error) {
     log({ event: "github-to-gitea-sync-error", repo: repo.key, ok: false, error: redact(String(error?.message || error)), attempt, elapsedMs: Date.now() - startedAt, valuesPrinted: false });
@@ -95,6 +117,14 @@ function runSyncLoop(repo, state, startedAt) {
   }
 }
 
+function retryDelayMs(syncAttempt) {
+  return Math.min(retryMaxDelayMs, retryInitialDelayMs * (2 ** Math.max(0, syncAttempt - 1)));
+}
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
 function syncRepository(repo) {
   const work = mkdtempSync(join(tmpdir(), `gitea-gh-sync-${repo.key}-`));
   try {
@@ -174,3 +204,9 @@ function requiredEnv(name) {
   if (!value) throw new Error(`missing env ${name}`);
   return value;
 }
+
+function positiveIntegerEnv(name) {
+  const value = Number.parseInt(requiredEnv(name), 10);
+  if (!Number.isFinite(value) || value < 1) throw new Error(`invalid positive integer env ${name}`);
+  return value;
+}

scripts/src/platform-infra-gitea.ts

@@ -183,6 +183,11 @@ interface GiteaWebhookSync {
     replicas: number;
     httpPort: number;
     serviceAccountName: string;
+    retry: {
+      maxAttempts: number;
+      initialDelayMs: number;
+      maxDelayMs: number;
+    };
   };
   frpc: {
     proxyName: string;
@@ -485,6 +490,7 @@ function parseWebhookSync(record: Record<string, unknown>, path: string): GiteaW
       replicas: positiveInteger(bridge, "replicas", `${path}.bridge`),
       httpPort: y.portField(bridge, "httpPort", `${path}.bridge`),
       serviceAccountName: y.kubernetesNameField(bridge, "serviceAccountName", `${path}.bridge`),
+      retry: parseWebhookRetry(y.objectField(bridge, "retry", `${path}.bridge`), `${path}.bridge.retry`),
     },
     frpc: {
       proxyName: y.stringField(frpc, "proxyName", `${path}.frpc`),
@@ -493,6 +499,14 @@ function parseWebhookSync(record: Record<string, unknown>, path: string): GiteaW
   };
 }
 
+function parseWebhookRetry(record: Record<string, unknown>, path: string): GiteaWebhookSync["bridge"]["retry"] {
+  return {
+    maxAttempts: positiveInteger(record, "maxAttempts", path),
+    initialDelayMs: positiveInteger(record, "initialDelayMs", path),
+    maxDelayMs: positiveInteger(record, "maxDelayMs", path),
+  };
+}
+
 function parsePublicExposure(record: Record<string, unknown>, path: string): PublicServiceExposure & { secretRoot: string } {
   const dns = y.objectField(record, "dns", path);
   const frpc = y.objectField(record, "frpc", path);
@@ -1219,6 +1233,12 @@ spec:
               value: /etc/gitea-github-sync/repos.json
             - name: UNIDESK_GITEA_SERVICE_BASE_URL
               value: ${yamlQuote(`http://${gitea.app.serviceName}.${target.namespace}.svc.cluster.local:${gitea.app.service.httpPort}`)}
+            - name: UNIDESK_GITEA_WEBHOOK_RETRY_MAX_ATTEMPTS
+              value: ${yamlQuote(String(sync.bridge.retry.maxAttempts))}
+            - name: UNIDESK_GITEA_WEBHOOK_RETRY_INITIAL_DELAY_MS
+              value: ${yamlQuote(String(sync.bridge.retry.initialDelayMs))}
+            - name: UNIDESK_GITEA_WEBHOOK_RETRY_MAX_DELAY_MS
+              value: ${yamlQuote(String(sync.bridge.retry.maxDelayMs))}
             - name: GITHUB_TOKEN
               valueFrom:
                 secretKeyRef:
@@ -1948,10 +1968,12 @@ function renderMirrorWebhookStatus(result: Record<string, unknown>): RenderedCli
     .map((repo) => [stringValue(repo.key), stringValue(repo.repairCommand)]);
   const bridge = record(result.bridge);
   const bridgeLogs = record(result.bridgeLogs);
+  const retry = record(record(record(result.webhook).bridge).retry);
+  const retryText = `${stringValue(retry.maxAttempts)}x/${stringValue(retry.initialDelayMs)}-${stringValue(retry.maxDelayMs)}ms`;
   const next = record(result.next);
   return rendered(result, "platform-infra gitea mirror webhook status", [
     "PLATFORM-INFRA GITEA MIRROR WEBHOOK STATUS",
-    ...table(["TARGET", "BRIDGE", "READY", "URL", "LOGS"], [[stringValue(record(result.target).id), stringValue(bridge.deployment), boolText(bridge.ready), stringValue(record(result.webhook).publicUrl), `rc=${stringValue(bridgeLogs.exitCode)}`]]),
+    ...table(["TARGET", "BRIDGE", "READY", "RETRY", "URL", "LOGS"], [[stringValue(record(result.target).id), stringValue(bridge.deployment), boolText(bridge.ready), retryText, stringValue(record(result.webhook).publicUrl), `rc=${stringValue(bridgeLogs.exitCode)}`]]),
     "",
     "GITHUB HOOKS",
     ...(repos.length === 0 ? ["-"] : table(["KEY", "HOOK", "GH_HEAD", "GITEA", "SNAPSHOT", "MATCH", "STALE", "DELIVERY", "BRIDGE_EVENT", "ERROR"], repos)),