fix: handle stale Gitea webhook bridge events

scripts/src/platform-infra-gitea-github-sync-server.mjs

@@ -16,7 +16,7 @@ const giteaBaseUrl = requiredEnv("UNIDESK_GITEA_SERVICE_BASE_URL").replace(/\/+$
 const githubAuthHeader = `Authorization: Basic ${Buffer.from(`x-access-token:${githubToken}`).toString("base64")}`;
 const giteaAuthHeader = `Authorization: Basic ${Buffer.from(`${giteaUsername}:${giteaPassword}`).toString("base64")}`;
 
-const running = new Set();
+const syncStates = new Map();
 
 createServer(async (req, res) => {
   const startedAt = Date.now();
@@ -49,21 +49,18 @@ createServer(async (req, res) => {
       writeJson(res, 202, { ok: true, event, repository, ref, disposition: "ignored-repo-or-ref", valuesPrinted: false });
       return;
     }
-    if (running.has(repo.key)) {
-      writeJson(res, 202, { ok: true, event, repository, ref, disposition: "already-running", repo: repo.key, valuesPrinted: false });
+    const state = syncState(repo.key);
+    if (state.running) {
+      state.pending = true;
+      log({ event: "github-to-gitea-sync-queued", repo: repo.key, repository, ref, disposition: "pending-rerun", valuesPrinted: false });
+      writeJson(res, 202, { ok: true, event, repository, ref, disposition: "queued-after-running", repo: repo.key, valuesPrinted: false });
       return;
     }
-    running.add(repo.key);
+    state.running = true;
+    state.pending = false;
     writeJson(res, 202, { ok: true, event, repository, ref, repo: repo.key, disposition: "accepted", valuesPrinted: false });
     setImmediate(() => {
-      try {
-        const sync = syncRepository(repo);
-        log({ event: "github-to-gitea-sync", repo: repo.key, ok: sync.ok, sourceCommit: sync.sourceCommit, snapshotRef: sync.snapshotRef, error: sync.error ? redact(sync.error) : null, elapsedMs: Date.now() - startedAt, valuesPrinted: false });
-      } catch (error) {
-        log({ event: "github-to-gitea-sync-error", repo: repo.key, ok: false, error: redact(String(error?.message || error)), elapsedMs: Date.now() - startedAt, valuesPrinted: false });
-      } finally {
-        running.delete(repo.key);
-      }
+      runSyncLoop(repo, state, startedAt);
     });
   } catch (error) {
     log({ event: "github-to-gitea-sync-error", ok: false, error: String(error?.message || error), valuesPrinted: false });
@@ -73,6 +70,31 @@ createServer(async (req, res) => {
   log({ event: "github-to-gitea-sync-listening", port, path, repos: repos.map((repo) => repo.key), valuesPrinted: false });
 });
 
+function syncState(key) {
+  let state = syncStates.get(key);
+  if (!state) {
+    state = { running: false, pending: false };
+    syncStates.set(key, state);
+  }
+  return state;
+}
+
+function runSyncLoop(repo, state, startedAt) {
+  let attempt = 0;
+  try {
+    do {
+      state.pending = false;
+      attempt += 1;
+      const sync = syncRepository(repo);
+      log({ event: "github-to-gitea-sync", repo: repo.key, ok: sync.ok, sourceCommit: sync.sourceCommit, snapshotRef: sync.snapshotRef, error: sync.error ? redact(sync.error) : null, attempt, pendingRerun: state.pending, elapsedMs: Date.now() - startedAt, valuesPrinted: false });
+    } while (state.pending);
+  } catch (error) {
+    log({ event: "github-to-gitea-sync-error", repo: repo.key, ok: false, error: redact(String(error?.message || error)), attempt, elapsedMs: Date.now() - startedAt, valuesPrinted: false });
+  } finally {
+    state.running = false;
+  }
+}
+
 function syncRepository(repo) {
   const work = mkdtempSync(join(tmpdir(), `gitea-gh-sync-${repo.key}-`));
   try {

scripts/src/platform-infra-gitea-remote.sh

@@ -806,6 +806,24 @@ for repo in repos:
     snapshot_matches_branch = bool(branch_commit and snapshot_commit == branch_commit)
     repo_bridge_events = [event for event in bridge_events if event.get("repo") == repo["key"]]
     last_bridge_event = repo_bridge_events[-1] if repo_bridge_events else None
+    latest_push_delivery = delivery.get("latestPush")
+    delivery_accepted = isinstance(latest_push_delivery, dict) and latest_push_delivery.get("statusCode") in (200, 202)
+    bridge_event_stale = bool(delivery_accepted and head.get("sha") and (not branch_matches_github or not snapshot_matches_branch))
+    stale_reason = None
+    if bridge_event_stale:
+        if not last_bridge_event:
+            stale_reason = "no-bridge-event-in-log-tail"
+        elif last_bridge_event.get("ok") is not True:
+            stale_reason = "last-bridge-event-failed"
+        elif last_bridge_event.get("sourceCommit") != head.get("sha"):
+            stale_reason = "last-bridge-event-behind-github-head"
+        elif not branch_matches_github:
+            stale_reason = "gitea-branch-behind-github"
+        elif not snapshot_matches_branch:
+            stale_reason = "gitea-snapshot-behind-branch"
+    repair_command = None
+    if not branch_matches_github or not snapshot_matches_branch:
+        repair_command = f"bun scripts/cli.ts platform-infra gitea mirror sync --target {os.environ['UNIDESK_GITEA_TARGET_ID']} --repo {repo['key']} --confirm"
     error = None
     if not result.get("ok"):
         error = result.get("body", "")[:500]
@@ -813,6 +831,8 @@ for repo in repos:
         error = head.get("error")
     elif not delivery.get("ok"):
         error = delivery.get("error")
+    elif bridge_event_stale:
+        error = "bridge event stale: " + (stale_reason or "delivery-accepted-but-mirror-not-current")
     elif not branch_matches_github:
         error = "gitea branch does not match GitHub head"
     elif not snapshot_matches_branch:
@@ -831,8 +851,12 @@ for repo in repos:
         "branchMatchesGitHub": branch_matches_github,
         "snapshotMatchesBranch": snapshot_matches_branch,
         "latestDelivery": delivery.get("latest"),
-        "latestPushDelivery": delivery.get("latestPush"),
+        "latestPushDelivery": latest_push_delivery,
         "lastBridgeEvent": last_bridge_event,
+        "deliveryAccepted": delivery_accepted,
+        "bridgeEventStale": bridge_event_stale,
+        "staleReason": stale_reason,
+        "repairCommand": repair_command,
         "error": error,
         "valuesPrinted": False,
     })
@@ -842,6 +866,7 @@ payload = {
     "bridge": bridge,
     "repositories": rows,
     "url": url,
+    "repairCommands": [row["repairCommand"] for row in rows if row.get("repairCommand")],
     "mirrorStatus": {"exitCode": mirror_status_rc, "errorTail": text(mirror_status_err_path)},
     "bridgeLogs": {"exitCode": bridge_logs_rc, "events": bridge_events[-12:], "errorTail": text(bridge_log_err_path)},
     "valuesPrinted": False,

scripts/src/platform-infra-gitea.ts

@@ -931,6 +931,7 @@ async function mirrorWebhookStatus(config: UniDeskConfig, options: MirrorOptions
   const result = await capture(config, target.route, ["sh"], remoteScript("mirror-webhook-status", gitea, target, "", { ...options, confirm: false, dryRun: true, wait: false }, { repos, secrets }));
   const parsed = parseJsonOutput(result.stdout);
   const repositories = arrayRecords(record(parsed).repositories);
+  const repairCommands = repositories.map((repo) => stringValue(repo.repairCommand, "")).filter((command) => command.length > 0);
   return {
     ok: result.exitCode === 0 && parsed?.ok === true,
     action: "platform-infra-gitea-mirror-webhook-status",
@@ -941,8 +942,9 @@ async function mirrorWebhookStatus(config: UniDeskConfig, options: MirrorOptions
     bridge: record(parsed).bridge,
     bridgeLogs: record(parsed).bridgeLogs,
     mirrorStatus: record(parsed).mirrorStatus,
+    repairCommands,
     remote: parsed ?? compactCapture(result, { full: true }),
-    next: mirrorNextCommands(target.id),
+    next: { ...mirrorNextCommands(target.id), repair: repairCommands[0] ?? null },
   };
 }
 
@@ -1935,11 +1937,15 @@ function renderMirrorWebhookStatus(result: Record<string, unknown>): RenderedCli
       stringValue(repo.branchCommit).slice(0, 12),
       stringValue(repo.snapshotCommit).slice(0, 12),
       boolText(repo.branchMatchesGitHub),
+      boolText(repo.bridgeEventStale),
       `${stringValue(delivery.event)}:${stringValue(delivery.statusCode)}:${stringValue(delivery.status)}`,
       `${stringValue(bridge.event)}:${boolText(bridge.ok)}:${stringValue(bridge.sourceCommit).slice(0, 12)}`,
       compactTail(stringValue(repo.error)),
     ];
   });
+  const repairs = arrayRecords(result.repositories)
+    .filter((repo) => stringValue(repo.repairCommand, "") !== "")
+    .map((repo) => [stringValue(repo.key), stringValue(repo.repairCommand)]);
   const bridge = record(result.bridge);
   const bridgeLogs = record(result.bridgeLogs);
   const next = record(result.next);
@@ -1948,10 +1954,13 @@ function renderMirrorWebhookStatus(result: Record<string, unknown>): RenderedCli
     ...table(["TARGET", "BRIDGE", "READY", "URL", "LOGS"], [[stringValue(record(result.target).id), stringValue(bridge.deployment), boolText(bridge.ready), stringValue(record(result.webhook).publicUrl), `rc=${stringValue(bridgeLogs.exitCode)}`]]),
     "",
     "GITHUB HOOKS",
-    ...(repos.length === 0 ? ["-"] : table(["KEY", "HOOK", "GH_HEAD", "GITEA", "SNAPSHOT", "MATCH", "DELIVERY", "BRIDGE_EVENT", "ERROR"], repos)),
+    ...(repos.length === 0 ? ["-"] : table(["KEY", "HOOK", "GH_HEAD", "GITEA", "SNAPSHOT", "MATCH", "STALE", "DELIVERY", "BRIDGE_EVENT", "ERROR"], repos)),
+    "",
+    "REPAIR",
+    ...(repairs.length === 0 ? ["-"] : table(["KEY", "COMMAND"], repairs)),
     ...remoteErrorLines(result),
     "",
-    `NEXT  ${stringValue(next.webhookApply)}`,
+    `NEXT  ${repairs.length === 0 ? stringValue(next.webhookApply) : stringValue(next.repair)}`,
   ]);
 }