From 99b394691b16bfb627291cfa39c154abe6c445ae Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 9 Jul 2026 15:15:55 +0200 Subject: [PATCH] fix: retry Gitea PaC webhook mirror sync --- .../unidesk-cicd/references/gitea-pac.md | 2 + config/platform-infra/gitea.yaml | 4 ++ ...latform-infra-gitea-github-sync-server.mjs | 44 +++++++++++++++++-- scripts/src/platform-infra-gitea.ts | 24 +++++++++- 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/.agents/skills/unidesk-cicd/references/gitea-pac.md b/.agents/skills/unidesk-cicd/references/gitea-pac.md index a3de47eb..09034b06 100644 --- a/.agents/skills/unidesk-cicd/references/gitea-pac.md +++ b/.agents/skills/unidesk-cicd/references/gitea-pac.md @@ -45,6 +45,8 @@ Use `bun scripts/cli.ts platform-infra pipelines-as-code history --target GitHub webhook delivery `202` only means the bridge accepted the event. If `platform-infra gitea mirror webhook status --target ` shows `STALE=true`, treat the bridge event as stale and run the row's `REPAIR` command; the repair path is still the controlled `platform-infra gitea mirror sync --target --repo --confirm` entry and must not be replaced with a direct Gitea push. +The GitHub -> Gitea bridge must use YAML-declared bounded retry for its asynchronous `git fetch` and Gitea `push` worker. Retry ownership is `config/platform-infra/gitea.yaml#sourceAuthority.webhookSync.bridge.retry`; `webhook status` should show the active retry shape in the default summary. A GitHub delivery returning 202 is not enough to close a rollout or issue unless Gitea branch, snapshot ref and the latest bridge event are also aligned to GitHub head. + Do not use `cicd branch-follower status` to decide whether the three migrated JD01 consumers are current. It is historical/migration-only and may contain stale state from before PaC cutover. For migrated sentinel CI/CD half-state triage, the PaC `status` command owns the source/registry/GitOps/Argo/runtime diagnosis. It must classify registry-missing, GitOps-missing, Argo-pending and runtime-mismatch states from target-side short probes, using YAML-declared probe endpoints such as `registry_probe_base`; do not reintroduce old control-plane status as the primary architecture or use registry manifest `HEAD` as the readiness check. diff --git a/config/platform-infra/gitea.yaml b/config/platform-infra/gitea.yaml index 06b39552..1388ac59 100644 --- a/config/platform-infra/gitea.yaml +++ b/config/platform-infra/gitea.yaml @@ -77,6 +77,10 @@ sourceAuthority: replicas: 1 httpPort: 8080 serviceAccountName: gitea-github-sync + retry: + maxAttempts: 4 + initialDelayMs: 2000 + maxDelayMs: 15000 frpc: proxyName: platform-infra-gitea-jd01-github-webhook remotePort: 22081 diff --git a/scripts/src/platform-infra-gitea-github-sync-server.mjs b/scripts/src/platform-infra-gitea-github-sync-server.mjs index b189889e..76e9fcd9 100644 --- a/scripts/src/platform-infra-gitea-github-sync-server.mjs +++ b/scripts/src/platform-infra-gitea-github-sync-server.mjs @@ -13,6 +13,9 @@ const giteaUsername = requiredEnv("GITEA_USERNAME"); const giteaPassword = requiredEnv("GITEA_PASSWORD"); const webhookSecret = requiredEnv("GITHUB_WEBHOOK_SECRET"); const giteaBaseUrl = requiredEnv("UNIDESK_GITEA_SERVICE_BASE_URL").replace(/\/+$/u, ""); +const retryMaxAttempts = positiveIntegerEnv("UNIDESK_GITEA_WEBHOOK_RETRY_MAX_ATTEMPTS"); +const retryInitialDelayMs = positiveIntegerEnv("UNIDESK_GITEA_WEBHOOK_RETRY_INITIAL_DELAY_MS"); +const retryMaxDelayMs = positiveIntegerEnv("UNIDESK_GITEA_WEBHOOK_RETRY_MAX_DELAY_MS"); const githubAuthHeader = `Authorization: Basic ${Buffer.from(`x-access-token:${githubToken}`).toString("base64")}`; const giteaAuthHeader = `Authorization: Basic ${Buffer.from(`${giteaUsername}:${giteaPassword}`).toString("base64")}`; @@ -60,7 +63,7 @@ createServer(async (req, res) => { state.pending = false; writeJson(res, 202, { ok: true, event, repository, ref, repo: repo.key, disposition: "accepted", valuesPrinted: false }); setImmediate(() => { - runSyncLoop(repo, state, startedAt); + void runSyncLoop(repo, state, startedAt); }); } catch (error) { log({ event: "github-to-gitea-sync-error", ok: false, error: String(error?.message || error), valuesPrinted: false }); @@ -79,14 +82,33 @@ function syncState(key) { return state; } -function runSyncLoop(repo, state, startedAt) { +async function runSyncLoop(repo, state, startedAt) { let attempt = 0; try { do { state.pending = false; attempt += 1; - const sync = syncRepository(repo); - log({ event: "github-to-gitea-sync", repo: repo.key, ok: sync.ok, sourceCommit: sync.sourceCommit, snapshotRef: sync.snapshotRef, error: sync.error ? redact(sync.error) : null, attempt, pendingRerun: state.pending, elapsedMs: Date.now() - startedAt, valuesPrinted: false }); + for (let syncAttempt = 1; syncAttempt <= retryMaxAttempts; syncAttempt += 1) { + const sync = syncRepository(repo); + const willRetry = !sync.ok && syncAttempt < retryMaxAttempts; + log({ + event: "github-to-gitea-sync", + repo: repo.key, + ok: sync.ok, + sourceCommit: sync.sourceCommit, + snapshotRef: sync.snapshotRef, + error: sync.error ? redact(sync.error) : null, + attempt, + syncAttempt, + maxSyncAttempts: retryMaxAttempts, + willRetry, + pendingRerun: state.pending, + elapsedMs: Date.now() - startedAt, + valuesPrinted: false, + }); + if (sync.ok || !willRetry) break; + await sleep(retryDelayMs(syncAttempt)); + } } while (state.pending); } catch (error) { log({ event: "github-to-gitea-sync-error", repo: repo.key, ok: false, error: redact(String(error?.message || error)), attempt, elapsedMs: Date.now() - startedAt, valuesPrinted: false }); @@ -95,6 +117,14 @@ function runSyncLoop(repo, state, startedAt) { } } +function retryDelayMs(syncAttempt) { + return Math.min(retryMaxDelayMs, retryInitialDelayMs * (2 ** Math.max(0, syncAttempt - 1))); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + function syncRepository(repo) { const work = mkdtempSync(join(tmpdir(), `gitea-gh-sync-${repo.key}-`)); try { @@ -174,3 +204,9 @@ function requiredEnv(name) { if (!value) throw new Error(`missing env ${name}`); return value; } + +function positiveIntegerEnv(name) { + const value = Number.parseInt(requiredEnv(name), 10); + if (!Number.isFinite(value) || value < 1) throw new Error(`invalid positive integer env ${name}`); + return value; +} diff --git a/scripts/src/platform-infra-gitea.ts b/scripts/src/platform-infra-gitea.ts index 71b78c0d..62cfd390 100644 --- a/scripts/src/platform-infra-gitea.ts +++ b/scripts/src/platform-infra-gitea.ts @@ -183,6 +183,11 @@ interface GiteaWebhookSync { replicas: number; httpPort: number; serviceAccountName: string; + retry: { + maxAttempts: number; + initialDelayMs: number; + maxDelayMs: number; + }; }; frpc: { proxyName: string; @@ -485,6 +490,7 @@ function parseWebhookSync(record: Record, path: string): GiteaW replicas: positiveInteger(bridge, "replicas", `${path}.bridge`), httpPort: y.portField(bridge, "httpPort", `${path}.bridge`), serviceAccountName: y.kubernetesNameField(bridge, "serviceAccountName", `${path}.bridge`), + retry: parseWebhookRetry(y.objectField(bridge, "retry", `${path}.bridge`), `${path}.bridge.retry`), }, frpc: { proxyName: y.stringField(frpc, "proxyName", `${path}.frpc`), @@ -493,6 +499,14 @@ function parseWebhookSync(record: Record, path: string): GiteaW }; } +function parseWebhookRetry(record: Record, path: string): GiteaWebhookSync["bridge"]["retry"] { + return { + maxAttempts: positiveInteger(record, "maxAttempts", path), + initialDelayMs: positiveInteger(record, "initialDelayMs", path), + maxDelayMs: positiveInteger(record, "maxDelayMs", path), + }; +} + function parsePublicExposure(record: Record, path: string): PublicServiceExposure & { secretRoot: string } { const dns = y.objectField(record, "dns", path); const frpc = y.objectField(record, "frpc", path); @@ -1219,6 +1233,12 @@ spec: value: /etc/gitea-github-sync/repos.json - name: UNIDESK_GITEA_SERVICE_BASE_URL value: ${yamlQuote(`http://${gitea.app.serviceName}.${target.namespace}.svc.cluster.local:${gitea.app.service.httpPort}`)} + - name: UNIDESK_GITEA_WEBHOOK_RETRY_MAX_ATTEMPTS + value: ${yamlQuote(String(sync.bridge.retry.maxAttempts))} + - name: UNIDESK_GITEA_WEBHOOK_RETRY_INITIAL_DELAY_MS + value: ${yamlQuote(String(sync.bridge.retry.initialDelayMs))} + - name: UNIDESK_GITEA_WEBHOOK_RETRY_MAX_DELAY_MS + value: ${yamlQuote(String(sync.bridge.retry.maxDelayMs))} - name: GITHUB_TOKEN valueFrom: secretKeyRef: @@ -1948,10 +1968,12 @@ function renderMirrorWebhookStatus(result: Record): RenderedCli .map((repo) => [stringValue(repo.key), stringValue(repo.repairCommand)]); const bridge = record(result.bridge); const bridgeLogs = record(result.bridgeLogs); + const retry = record(record(record(result.webhook).bridge).retry); + const retryText = `${stringValue(retry.maxAttempts)}x/${stringValue(retry.initialDelayMs)}-${stringValue(retry.maxDelayMs)}ms`; const next = record(result.next); return rendered(result, "platform-infra gitea mirror webhook status", [ "PLATFORM-INFRA GITEA MIRROR WEBHOOK STATUS", - ...table(["TARGET", "BRIDGE", "READY", "URL", "LOGS"], [[stringValue(record(result.target).id), stringValue(bridge.deployment), boolText(bridge.ready), stringValue(record(result.webhook).publicUrl), `rc=${stringValue(bridgeLogs.exitCode)}`]]), + ...table(["TARGET", "BRIDGE", "READY", "RETRY", "URL", "LOGS"], [[stringValue(record(result.target).id), stringValue(bridge.deployment), boolText(bridge.ready), retryText, stringValue(record(result.webhook).publicUrl), `rc=${stringValue(bridgeLogs.exitCode)}`]]), "", "GITHUB HOOKS", ...(repos.length === 0 ? ["-"] : table(["KEY", "HOOK", "GH_HEAD", "GITEA", "SNAPSHOT", "MATCH", "STALE", "DELIVERY", "BRIDGE_EVENT", "ERROR"], repos)),