diff --git a/config/platform-infra/gitea.yaml b/config/platform-infra/gitea.yaml index df1a99ed..0f912ce9 100644 --- a/config/platform-infra/gitea.yaml +++ b/config/platform-infra/gitea.yaml @@ -53,6 +53,28 @@ sourceAuthority: - localhost - .svc - .svc.cluster.local + webhookSync: + enabled: true + direction: github-to-gitea + publicPath: /_unidesk/github-to-gitea + events: + - push + secret: + sourceRef: .env/gitea.github-webhook.env + sourceKey: GITHUB_WEBHOOK_SECRET + createIfMissing: true + bridge: + deploymentName: gitea-github-sync + serviceName: gitea-github-sync + secretName: gitea-github-sync-secrets + configMapName: gitea-github-sync-config + image: 127.0.0.1:5000/hwlab/hwlab-ci-node-tools:node22-alpine-bun-v1 + replicas: 1 + httpPort: 8080 + serviceAccountName: gitea-github-sync + frpc: + proxyName: platform-infra-gitea-jd01-github-webhook + remotePort: 22081 responsibilities: - name: source-read current: legacy-git-mirror diff --git a/scripts/src/pk01-caddy.ts b/scripts/src/pk01-caddy.ts index 6c0c9078..ef029168 100644 --- a/scripts/src/pk01-caddy.ts +++ b/scripts/src/pk01-caddy.ts @@ -64,16 +64,23 @@ export async function applyPk01CaddyManagedBlock( serviceId: string, exposure: Pk01CaddyBlockExposure, markers: Pk01CaddyManagedBlockMarkers, +): Promise> { + return await applyPk01CaddySiteBlock(config, serviceId, exposure, renderSimpleReverseProxyCaddySiteBlock({ + hostname: exposure.dns.hostname, + upstream: `127.0.0.1:${exposure.frpc.remotePort}`, + responseHeaderTimeoutSeconds: exposure.pk01.responseHeaderTimeoutSeconds, + }), markers); +} + +export async function applyPk01CaddySiteBlock( + config: UniDeskConfig, + serviceId: string, + exposure: Pk01CaddyBlockExposure, + siteBlock: string, + markers: Pk01CaddyManagedBlockMarkers, ): Promise> { if (!exposure.enabled) return { ok: true, action: "not-enabled" }; - const block = renderCaddyManagedBlock( - markerNameFromMarkers(markers), - renderSimpleReverseProxyCaddySiteBlock({ - hostname: exposure.dns.hostname, - upstream: `127.0.0.1:${exposure.frpc.remotePort}`, - responseHeaderTimeoutSeconds: exposure.pk01.responseHeaderTimeoutSeconds, - }), - ); + const block = renderCaddyManagedBlock(markerNameFromMarkers(markers), siteBlock); const blockB64 = Buffer.from(block, "utf8").toString("base64"); const script = ` set -u diff --git a/scripts/src/platform-infra-gitea-github-sync-server.mjs b/scripts/src/platform-infra-gitea-github-sync-server.mjs new file mode 100644 index 00000000..189ca3b7 --- /dev/null +++ b/scripts/src/platform-infra-gitea-github-sync-server.mjs @@ -0,0 +1,143 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { createServer } from "node:http"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const port = Number.parseInt(process.env.UNIDESK_GITEA_WEBHOOK_PORT || "8080", 10); +const path = process.env.UNIDESK_GITEA_WEBHOOK_PATH || "/_unidesk/github-to-gitea"; +const repos = JSON.parse(readFileSync(process.env.UNIDESK_GITEA_REPOS_PATH || "/etc/gitea-github-sync/repos.json", "utf8")); +const githubToken = requiredEnv("GITHUB_TOKEN"); +const giteaUsername = requiredEnv("GITEA_USERNAME"); +const giteaPassword = requiredEnv("GITEA_PASSWORD"); +const webhookSecret = requiredEnv("GITHUB_WEBHOOK_SECRET"); +const giteaBaseUrl = requiredEnv("UNIDESK_GITEA_SERVICE_BASE_URL").replace(/\/+$/u, ""); +const githubAuthHeader = `Authorization: Basic ${Buffer.from(`x-access-token:${githubToken}`).toString("base64")}`; +const giteaAuthHeader = `Authorization: Basic ${Buffer.from(`${giteaUsername}:${giteaPassword}`).toString("base64")}`; + +const running = new Set(); + +createServer(async (req, res) => { + const startedAt = Date.now(); + try { + const url = new URL(req.url || "/", "http://gitea-github-sync.local"); + if (req.method === "GET" && url.pathname === "/healthz") { + writeJson(res, 200, { ok: true, repos: repos.length, valuesPrinted: false }); + return; + } + if (req.method !== "POST" || url.pathname !== path) { + writeJson(res, 404, { ok: false, error: "not-found", valuesPrinted: false }); + return; + } + const body = await readBody(req); + verifySignature(req.headers["x-hub-signature-256"], body); + const event = String(req.headers["x-github-event"] || ""); + if (event === "ping") { + writeJson(res, 200, { ok: true, event, disposition: "pong", valuesPrinted: false }); + return; + } + if (event !== "push") { + writeJson(res, 202, { ok: true, event, disposition: "ignored-event", valuesPrinted: false }); + return; + } + const payload = JSON.parse(body.toString("utf8") || "{}"); + const repository = payload?.repository?.full_name; + const ref = payload?.ref; + const repo = repos.find((item) => item?.upstream?.repository === repository && ref === `refs/heads/${item?.upstream?.branch}`); + if (!repo) { + writeJson(res, 202, { ok: true, event, repository, ref, disposition: "ignored-repo-or-ref", valuesPrinted: false }); + return; + } + if (running.has(repo.key)) { + writeJson(res, 202, { ok: true, event, repository, ref, disposition: "already-running", repo: repo.key, valuesPrinted: false }); + return; + } + running.add(repo.key); + try { + const sync = syncRepository(repo); + log({ event: "github-to-gitea-sync", repo: repo.key, ok: sync.ok, sourceCommit: sync.sourceCommit, snapshotRef: sync.snapshotRef, elapsedMs: Date.now() - startedAt, valuesPrinted: false }); + writeJson(res, sync.ok ? 200 : 500, { ok: sync.ok, event, repository, ref, repo: repo.key, ...sync, valuesPrinted: false }); + } finally { + running.delete(repo.key); + } + } catch (error) { + log({ event: "github-to-gitea-sync-error", ok: false, error: String(error?.message || error), valuesPrinted: false }); + writeJson(res, 500, { ok: false, error: String(error?.message || error), valuesPrinted: false }); + } +}).listen(port, "0.0.0.0", () => { + log({ event: "github-to-gitea-sync-listening", port, path, repos: repos.map((repo) => repo.key), valuesPrinted: false }); +}); + +function syncRepository(repo) { + const work = mkdtempSync(join(tmpdir(), `gitea-gh-sync-${repo.key}-`)); + try { + run(["git", "init", "--bare", work]); + run(["git", "-C", work, "-c", `http.extraHeader=${githubAuthHeader}`, "fetch", "--prune", repo.upstream.cloneUrl, `+refs/heads/${repo.upstream.branch}:refs/remotes/github/${repo.upstream.branch}`]); + const sourceCommit = run(["git", "-C", work, "rev-parse", `refs/remotes/github/${repo.upstream.branch}`]).stdout.trim(); + const snapshotRef = `${repo.snapshot.prefix.replace(/\/+$/u, "")}/${sourceCommit}`; + const giteaUrl = giteaBaseUrl + new URL(repo.gitea.readUrl).pathname; + run(["git", "-C", work, "-c", `http.extraHeader=${giteaAuthHeader}`, "push", "--force", giteaUrl, `${sourceCommit}:refs/heads/${repo.upstream.branch}`, `${sourceCommit}:${snapshotRef}`]); + return { ok: true, sourceCommit, snapshotRef }; + } catch (error) { + return { ok: false, error: String(error?.message || error) }; + } finally { + rmSync(work, { recursive: true, force: true }); + } +} + +function run(args) { + const result = spawnSync(args[0], args.slice(1), { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout: 120000, + env: { + ...process.env, + GIT_TERMINAL_PROMPT: "0", + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + }, + }); + if (result.error || result.status !== 0) { + const stderr = String(result.stderr || result.error?.message || "").slice(-1600); + throw new Error(`${args[0]} failed status=${result.status ?? "error"} stderr=${stderr}`); + } + return { stdout: result.stdout || "", stderr: result.stderr || "" }; +} + +function verifySignature(signatureHeader, body) { + const signature = Array.isArray(signatureHeader) ? signatureHeader[0] : signatureHeader; + if (!signature || !signature.startsWith("sha256=")) throw new Error("missing-github-signature"); + const expected = `sha256=${createHmac("sha256", webhookSecret).update(body).digest("hex")}`; + if (!timingSafeEqualString(signature, expected)) throw new Error("invalid-github-signature"); +} + +function timingSafeEqualString(a, b) { + const left = Buffer.from(a); + const right = Buffer.from(b); + return left.length === right.length && timingSafeEqual(left, right); +} + +function readBody(req) { + return new Promise((resolve, reject) => { + const chunks = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => resolve(Buffer.concat(chunks))); + req.on("error", reject); + }); +} + +function writeJson(res, status, payload) { + res.writeHead(status, { "Content-Type": "application/json" }); + res.end(`${JSON.stringify(payload)}\n`); +} + +function log(payload) { + console.log(JSON.stringify(payload)); +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`missing env ${name}`); + return value; +} diff --git a/scripts/src/platform-infra-gitea-remote.sh b/scripts/src/platform-infra-gitea-remote.sh index a27d76ec..117842f1 100644 --- a/scripts/src/platform-infra-gitea-remote.sh +++ b/scripts/src/platform-infra-gitea-remote.sh @@ -45,17 +45,34 @@ run_apply() { printf '%s\n' "frpc disabled" >"$tmp/frpc-secret.err" fi fi + if [ "$UNIDESK_GITEA_WEBHOOK_SYNC_ENABLED" = "1" ] && [ "$UNIDESK_GITEA_DRY_RUN" != "1" ]; then + kubectl -n "$UNIDESK_GITEA_NAMESPACE" create secret generic "$UNIDESK_GITEA_WEBHOOK_SECRET_NAME" \ + --from-literal=github-token="$UNIDESK_GITEA_GITHUB_TOKEN" \ + --from-literal=gitea-username="$UNIDESK_GITEA_ADMIN_USERNAME" \ + --from-literal=gitea-password="$UNIDESK_GITEA_ADMIN_PASSWORD" \ + --from-literal=github-webhook-secret="$UNIDESK_GITEA_GITHUB_WEBHOOK_SECRET" \ + --dry-run=client -o yaml | kubectl apply --server-side --force-conflicts --field-manager="$UNIDESK_GITEA_FIELD_MANAGER" -f - >"$tmp/webhook-secret.out" 2>"$tmp/webhook-secret.err" + webhook_secret_rc=$? + else + webhook_secret_rc=0 + : >"$tmp/webhook-secret.out" + if [ "$UNIDESK_GITEA_WEBHOOK_SYNC_ENABLED" = "1" ]; then + printf '%s\n' "dry-run: webhook sync secret apply skipped" >"$tmp/webhook-secret.err" + else + printf '%s\n' "webhook sync disabled" >"$tmp/webhook-secret.err" + fi + fi dry_arg="" if [ "$UNIDESK_GITEA_DRY_RUN" = "1" ]; then dry_arg="--dry-run=server" fi - if [ "$frpc_rc" -eq 0 ]; then + if [ "$frpc_rc" -eq 0 ] && [ "$webhook_secret_rc" -eq 0 ]; then timeout "$UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS" kubectl apply --server-side $dry_arg --force-conflicts --field-manager="$UNIDESK_GITEA_FIELD_MANAGER" -f "$manifest" >"$tmp/apply.out" 2>"$tmp/apply.err" apply_rc=$? else apply_rc=1 : >"$tmp/apply.out" - printf '%s\n' "frpc secret apply failed; manifest apply skipped" >"$tmp/apply.err" + printf '%s\n' "frpc or webhook secret apply failed; manifest apply skipped" >"$tmp/apply.err" fi if [ "$apply_rc" -eq 0 ] && [ "$UNIDESK_GITEA_WAIT" = "1" ] && [ "$UNIDESK_GITEA_DRY_RUN" != "1" ]; then timeout "$UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS" kubectl -n "$UNIDESK_GITEA_NAMESPACE" rollout status "statefulset/$UNIDESK_GITEA_STATEFULSET_NAME" --timeout="${UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS}s" >"$tmp/rollout.out" 2>"$tmp/rollout.err" @@ -69,9 +86,45 @@ run_apply() { printf '%s\n' "rollout wait not requested" >"$tmp/rollout.err" fi fi - python3 - "$frpc_rc" "$apply_rc" "$rollout_rc" "$tmp/frpc-secret.out" "$tmp/frpc-secret.err" "$tmp/apply.out" "$tmp/apply.err" "$tmp/rollout.out" "$tmp/rollout.err" <<'PY' + if [ "$apply_rc" -eq 0 ] && [ "$UNIDESK_GITEA_FRPC_ENABLED" = "1" ] && [ "$UNIDESK_GITEA_DRY_RUN" != "1" ]; then + kubectl -n "$UNIDESK_GITEA_NAMESPACE" rollout restart "deployment/$UNIDESK_GITEA_FRPC_DEPLOYMENT_NAME" >"$tmp/frpc-rollout.out" 2>"$tmp/frpc-rollout.err" + frpc_rollout_rc=$? + if [ "$frpc_rollout_rc" -eq 0 ] && [ "$UNIDESK_GITEA_WAIT" = "1" ]; then + timeout "$UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS" kubectl -n "$UNIDESK_GITEA_NAMESPACE" rollout status "deployment/$UNIDESK_GITEA_FRPC_DEPLOYMENT_NAME" --timeout="${UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS}s" >>"$tmp/frpc-rollout.out" 2>>"$tmp/frpc-rollout.err" + frpc_rollout_rc=$? + fi + else + frpc_rollout_rc=0 + : >"$tmp/frpc-rollout.out" + if [ "$UNIDESK_GITEA_DRY_RUN" = "1" ]; then + printf '%s\n' "dry-run: frpc rollout skipped" >"$tmp/frpc-rollout.err" + elif [ "$UNIDESK_GITEA_FRPC_ENABLED" = "1" ]; then + printf '%s\n' "manifest apply failed; frpc rollout skipped" >"$tmp/frpc-rollout.err" + else + printf '%s\n' "frpc disabled" >"$tmp/frpc-rollout.err" + fi + fi + if [ "$apply_rc" -eq 0 ] && [ "$UNIDESK_GITEA_WEBHOOK_SYNC_ENABLED" = "1" ] && [ "$UNIDESK_GITEA_DRY_RUN" != "1" ]; then + kubectl -n "$UNIDESK_GITEA_NAMESPACE" rollout restart "deployment/$UNIDESK_GITEA_WEBHOOK_DEPLOYMENT" >"$tmp/webhook-rollout.out" 2>"$tmp/webhook-rollout.err" + webhook_rollout_rc=$? + if [ "$webhook_rollout_rc" -eq 0 ] && [ "$UNIDESK_GITEA_WAIT" = "1" ]; then + timeout "$UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS" kubectl -n "$UNIDESK_GITEA_NAMESPACE" rollout status "deployment/$UNIDESK_GITEA_WEBHOOK_DEPLOYMENT" --timeout="${UNIDESK_GITEA_WAIT_TIMEOUT_SECONDS}s" >>"$tmp/webhook-rollout.out" 2>>"$tmp/webhook-rollout.err" + webhook_rollout_rc=$? + fi + else + webhook_rollout_rc=0 + : >"$tmp/webhook-rollout.out" + if [ "$UNIDESK_GITEA_DRY_RUN" = "1" ]; then + printf '%s\n' "dry-run: webhook sync rollout skipped" >"$tmp/webhook-rollout.err" + elif [ "$UNIDESK_GITEA_WEBHOOK_SYNC_ENABLED" = "1" ]; then + printf '%s\n' "manifest apply failed; webhook sync rollout skipped" >"$tmp/webhook-rollout.err" + else + printf '%s\n' "webhook sync disabled" >"$tmp/webhook-rollout.err" + fi + fi + python3 - "$frpc_rc" "$webhook_secret_rc" "$apply_rc" "$rollout_rc" "$frpc_rollout_rc" "$webhook_rollout_rc" "$tmp/frpc-secret.out" "$tmp/frpc-secret.err" "$tmp/webhook-secret.out" "$tmp/webhook-secret.err" "$tmp/apply.out" "$tmp/apply.err" "$tmp/rollout.out" "$tmp/rollout.err" "$tmp/frpc-rollout.out" "$tmp/frpc-rollout.err" "$tmp/webhook-rollout.out" "$tmp/webhook-rollout.err" <<'PY' import json, os, sys -frpc_rc, apply_rc, rollout_rc = int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]) +frpc_rc, webhook_secret_rc, apply_rc, rollout_rc, frpc_rollout_rc, webhook_rollout_rc = [int(value) for value in sys.argv[1:7]] def text(path, limit=3000): try: return open(path, encoding="utf-8", errors="replace").read()[-limit:] @@ -79,7 +132,7 @@ def text(path, limit=3000): return "" dry = os.environ.get("UNIDESK_GITEA_DRY_RUN") == "1" payload = { - "ok": frpc_rc == 0 and apply_rc == 0 and rollout_rc == 0, + "ok": all(value == 0 for value in [frpc_rc, webhook_secret_rc, apply_rc, rollout_rc, frpc_rollout_rc, webhook_rollout_rc]), "target": os.environ.get("UNIDESK_GITEA_TARGET_ID"), "route": os.environ.get("UNIDESK_GITEA_ROUTE"), "namespace": os.environ.get("UNIDESK_GITEA_NAMESPACE"), @@ -92,9 +145,12 @@ payload = { "networkPolicy": "allow-all", }, "steps": { - "frpcSecret": {"exitCode": frpc_rc, "stdoutTail": text(sys.argv[4]), "stderrTail": text(sys.argv[5])}, - "apply": {"exitCode": apply_rc, "stdoutTail": text(sys.argv[6]), "stderrTail": text(sys.argv[7])}, - "rollout": {"exitCode": rollout_rc, "stdoutTail": text(sys.argv[8]), "stderrTail": text(sys.argv[9])}, + "frpcSecret": {"exitCode": frpc_rc, "stdoutTail": text(sys.argv[7]), "stderrTail": text(sys.argv[8])}, + "webhookSyncSecret": {"exitCode": webhook_secret_rc, "stdoutTail": text(sys.argv[9]), "stderrTail": text(sys.argv[10])}, + "apply": {"exitCode": apply_rc, "stdoutTail": text(sys.argv[11]), "stderrTail": text(sys.argv[12])}, + "rollout": {"exitCode": rollout_rc, "stdoutTail": text(sys.argv[13]), "stderrTail": text(sys.argv[14])}, + "frpcRollout": {"exitCode": frpc_rollout_rc, "stdoutTail": text(sys.argv[15]), "stderrTail": text(sys.argv[16])}, + "webhookSyncRollout": {"exitCode": webhook_rollout_rc, "stdoutTail": text(sys.argv[17]), "stderrTail": text(sys.argv[18])}, }, "valuesPrinted": False, } @@ -573,6 +629,217 @@ sys.exit(0 if payload["ok"] else 1) PY } +github_api_env() { + if [ "$UNIDESK_GITEA_GITHUB_PROXY_ENABLED" = "1" ]; then + export https_proxy="$UNIDESK_GITEA_GITHUB_PROXY_URL" + export http_proxy="$UNIDESK_GITEA_GITHUB_PROXY_URL" + export no_proxy="$UNIDESK_GITEA_NO_PROXY" + fi +} + +run_mirror_webhook_apply() { + mirror_repos_json + github_api_env + python3 - "$tmp/repos.json" <<'PY' +import json, os, sys, urllib.error, urllib.request +repos = json.load(open(sys.argv[1], encoding="utf-8")) +token = os.environ["UNIDESK_GITEA_GITHUB_TOKEN"] +url = os.environ["UNIDESK_GITEA_WEBHOOK_PUBLIC_URL"] +secret = os.environ["UNIDESK_GITEA_GITHUB_WEBHOOK_SECRET"] +events = [item for item in os.environ.get("UNIDESK_GITEA_WEBHOOK_EVENTS", "push").split(",") if item] +def request(method, api_path, payload=None, expected=(200, 201, 204), tolerate=()): + data = None if payload is None else json.dumps(payload).encode() + req = urllib.request.Request("https://api.github.com" + api_path, data=data, method=method) + req.add_header("Authorization", "Bearer " + token) + req.add_header("Accept", "application/vnd.github+json") + req.add_header("X-GitHub-Api-Version", "2022-11-28") + if payload is not None: + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + body = resp.read().decode("utf-8", errors="replace") + return {"ok": resp.status in expected, "status": resp.status, "body": body} + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + return {"ok": exc.code in tolerate, "status": exc.code, "body": body[:1000]} +rows = [] +for repo in repos: + repository = repo["upstream"]["repository"] + list_result = request("GET", f"/repos/{repository}/hooks") + hooks = [] + try: + hooks = json.loads(list_result.get("body") or "[]") + except Exception: + hooks = [] + matches = [item for item in hooks if (item.get("config") or {}).get("url") == url] + body = {"name": "web", "active": True, "events": events, "config": {"url": url, "content_type": "json", "secret": secret, "insecure_ssl": "0"}} + if matches: + hook_id = matches[0].get("id") + result = request("PATCH", f"/repos/{repository}/hooks/{hook_id}", body) + for duplicate in matches[1:]: + duplicate_id = duplicate.get("id") + if duplicate_id: + request("DELETE", f"/repos/{repository}/hooks/{duplicate_id}", expected=(204,), tolerate=(404,)) + else: + result = request("POST", f"/repos/{repository}/hooks", body, expected=(201,)) + try: + hook_id = json.loads(result.get("body") or "{}").get("id") + except Exception: + hook_id = None + rows.append({"key": repo["key"], "repository": repository, "hookId": hook_id, "hookReady": result.get("ok") is True, "status": result.get("status"), "error": None if result.get("ok") else result.get("body", "")[:500], "valuesPrinted": False}) +payload = {"ok": all(row["hookReady"] for row in rows), "repositories": rows, "url": url, "valuesPrinted": False} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if payload["ok"] else 1) +PY +} + +run_mirror_webhook_status() { + mirror_repos_json + github_api_env + bridge_ready=$(kubectl -n "$UNIDESK_GITEA_NAMESPACE" get deploy "$UNIDESK_GITEA_WEBHOOK_DEPLOYMENT" -o jsonpath='{.status.readyReplicas}/{.spec.replicas}' 2>/dev/null || echo "0/0") + service_exists=$(kubectl -n "$UNIDESK_GITEA_NAMESPACE" get service "$UNIDESK_GITEA_WEBHOOK_SERVICE" -o name 2>/dev/null || true) + python3 - "$tmp/repos.json" "$bridge_ready" "$service_exists" <<'PY' +import json, os, sys, urllib.error, urllib.request +repos = json.load(open(sys.argv[1], encoding="utf-8")) +bridge_ready = sys.argv[2] +service_exists = bool(sys.argv[3]) +token = os.environ["UNIDESK_GITEA_GITHUB_TOKEN"] +url = os.environ["UNIDESK_GITEA_WEBHOOK_PUBLIC_URL"] +def request(api_path): + req = urllib.request.Request("https://api.github.com" + api_path, method="GET") + req.add_header("Authorization", "Bearer " + token) + req.add_header("Accept", "application/vnd.github+json") + req.add_header("X-GitHub-Api-Version", "2022-11-28") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return {"ok": True, "status": resp.status, "body": resp.read().decode("utf-8", errors="replace")} + except urllib.error.HTTPError as exc: + return {"ok": False, "status": exc.code, "body": exc.read().decode("utf-8", errors="replace")[:1000]} +rows = [] +for repo in repos: + repository = repo["upstream"]["repository"] + result = request(f"/repos/{repository}/hooks") + hooks = [] + try: + hooks = json.loads(result.get("body") or "[]") + except Exception: + hooks = [] + match = next((item for item in hooks if (item.get("config") or {}).get("url") == url), None) + rows.append({"key": repo["key"], "repository": repository, "hookId": match.get("id") if match else None, "hookReady": bool(match and match.get("active") is True), "active": match.get("active") if match else None, "status": result.get("status"), "error": None if result.get("ok") else result.get("body", "")[:500], "valuesPrinted": False}) +bridge = {"deployment": os.environ["UNIDESK_GITEA_WEBHOOK_DEPLOYMENT"], "ready": bridge_ready not in ("0/0", "/") and bridge_ready.split("/")[0] == bridge_ready.split("/")[-1], "readyReplicas": bridge_ready, "serviceExists": service_exists} +payload = {"ok": bridge["ready"] and service_exists and all(row["hookReady"] for row in rows), "bridge": bridge, "repositories": rows, "url": url, "valuesPrinted": False} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if payload["ok"] else 1) +PY +} + +run_mirror_webhook_test() { + mirror_repos_json + repo_count=$(python3 - "$tmp/repos.json" <<'PY' +import json, sys +print(len(json.load(open(sys.argv[1], encoding="utf-8")))) +PY +) + if [ "$repo_count" != "1" ]; then + printf '{"ok":false,"error":"webhook test requires exactly one repo","valuesPrinted":false}\n' + exit 2 + fi + repo_key=$(python3 - "$tmp/repos.json" <<'PY' +import json, sys +print(json.load(open(sys.argv[1], encoding="utf-8"))[0]["key"]) +PY +) + github_api_env + python3 - "$tmp/repos.json" >"$tmp/webhook-ping.json" 2>"$tmp/webhook-ping.err" <<'PY' +import json, os, sys, time, urllib.error, urllib.request +repo = json.load(open(sys.argv[1], encoding="utf-8"))[0] +repository = repo["upstream"]["repository"] +token = os.environ["UNIDESK_GITEA_GITHUB_TOKEN"] +url = os.environ["UNIDESK_GITEA_WEBHOOK_PUBLIC_URL"] +def request(method, path, payload=None, expected=(200, 201, 204)): + data = None if payload is None else json.dumps(payload).encode() + req = urllib.request.Request("https://api.github.com" + path, data=data, method=method) + req.add_header("Authorization", "Bearer " + token) + req.add_header("Accept", "application/vnd.github+json") + req.add_header("X-GitHub-Api-Version", "2022-11-28") + if payload is not None: + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return {"ok": resp.status in expected, "status": resp.status, "body": resp.read().decode("utf-8", errors="replace")} + except urllib.error.HTTPError as exc: + return {"ok": False, "status": exc.code, "body": exc.read().decode("utf-8", errors="replace")[:1000]} +hooks_result = request("GET", f"/repos/{repository}/hooks") +hooks = json.loads(hooks_result.get("body") or "[]") if hooks_result.get("ok") else [] +hook = next((item for item in hooks if (item.get("config") or {}).get("url") == url), None) +hook_id = hook.get("id") if hook else None +ping = request("POST", f"/repos/{repository}/hooks/{hook_id}/pings", expected=(204,)) if hook_id else {"ok": False, "status": None, "body": "hook not found"} +delivery = None +if hook_id and ping.get("ok"): + time.sleep(3) + deliveries = request("GET", f"/repos/{repository}/hooks/{hook_id}/deliveries") + try: + rows = json.loads(deliveries.get("body") or "[]") + except Exception: + rows = [] + delivery = next((item for item in rows if item.get("event") == "ping"), rows[0] if rows else None) +payload = {"ok": bool(ping.get("ok") and (delivery is None or delivery.get("status_code") in (200, 202))), "repository": repository, "hookId": hook_id, "pingStatus": ping.get("status"), "delivery": {"id": delivery.get("id"), "event": delivery.get("event"), "statusCode": delivery.get("status_code"), "status": delivery.get("status"), "deliveredAt": delivery.get("delivered_at")} if isinstance(delivery, dict) else None, "error": None if ping.get("ok") else ping.get("body"), "valuesPrinted": False} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if payload["ok"] else 1) +PY + ping_rc=$? + repos_inline=$(cat "$tmp/repos.json") + kubectl -n "$UNIDESK_GITEA_NAMESPACE" exec deploy/"$UNIDESK_GITEA_WEBHOOK_DEPLOYMENT" -- node - "$UNIDESK_GITEA_WEBHOOK_PATH" "$repos_inline" >"$tmp/webhook-test.out" 2>"$tmp/webhook-test.err" <<'NODE' +const http = require('node:http'); +const crypto = require('node:crypto'); +const path = process.argv[2]; +const repos = JSON.parse(process.argv[3]); +const repo = repos[0]; +const payload = Buffer.from(JSON.stringify({ ref: `refs/heads/${repo.upstream.branch}`, repository: { full_name: repo.upstream.repository } })); +const signature = `sha256=${crypto.createHmac('sha256', process.env.GITHUB_WEBHOOK_SECRET).update(payload).digest('hex')}`; +const req = http.request({ hostname: '127.0.0.1', port: Number(process.env.UNIDESK_GITEA_WEBHOOK_PORT || '8080'), path, method: 'POST', headers: { 'content-type': 'application/json', 'content-length': payload.length, 'x-github-event': 'push', 'x-hub-signature-256': signature } }, (res) => { + const chunks = []; + res.on('data', (chunk) => chunks.push(chunk)); + res.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + process.stdout.write(JSON.stringify({ statusCode: res.statusCode, body }) + '\n'); + process.exit(res.statusCode >= 200 && res.statusCode < 300 ? 0 : 1); + }); +}); +req.on('error', (error) => { console.error(error.stack || error.message); process.exit(1); }); +req.write(payload); +req.end(); +NODE + test_rc=$? + run_mirror_status >"$tmp/status-after-test.json" 2>"$tmp/status-after-test.err" + status_rc=$? + python3 - "$ping_rc" "$test_rc" "$status_rc" "$repo_key" "$tmp/webhook-ping.json" "$tmp/webhook-ping.err" "$tmp/webhook-test.out" "$tmp/webhook-test.err" "$tmp/status-after-test.json" "$tmp/status-after-test.err" <<'PY' +import json, sys +ping_rc, test_rc, status_rc = int(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]) +repo_key = sys.argv[4] +def text(path, limit=2000): + try: + return open(path, encoding="utf-8", errors="replace").read()[-limit:] + except FileNotFoundError: + return "" +try: + ping = json.loads(text(sys.argv[5], 100000)) +except Exception: + ping = {"ok": False, "error": text(sys.argv[6])} +try: + status = json.loads(text(sys.argv[9], 100000)) +except Exception: + status = {} +rows = [] +for repo in status.get("repositories", []): + if repo.get("key") == repo_key: + rows.append({**repo, "pingOk": ping_rc == 0 and ping.get("ok") is True, "testOk": test_rc == 0, "error": None if ping_rc == 0 and test_rc == 0 and repo.get("sourceBundleReady") else (ping.get("error") or text(sys.argv[8]) or text(sys.argv[10]))[:500]}) +payload = {"ok": ping_rc == 0 and test_rc == 0 and status_rc == 0 and all(row.get("sourceBundleReady") for row in rows), "repositories": rows, "githubPing": ping, "test": {"exitCode": test_rc, "stdoutTail": text(sys.argv[7]), "stderrTail": text(sys.argv[8])}, "valuesPrinted": False} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +sys.exit(0 if payload["ok"] else 1) +PY +} + case "$UNIDESK_GITEA_ACTION" in apply) run_apply ;; status) run_status ;; @@ -580,5 +847,8 @@ case "$UNIDESK_GITEA_ACTION" in mirror-bootstrap) run_mirror_bootstrap ;; mirror-sync) run_mirror_sync ;; mirror-status) run_mirror_status ;; + mirror-webhook-apply) run_mirror_webhook_apply ;; + mirror-webhook-status) run_mirror_webhook_status ;; + mirror-webhook-test) run_mirror_webhook_test ;; *) printf '{"ok":false,"error":"unsupported-gitea-remote-action"}\n'; exit 2 ;; esac diff --git a/scripts/src/platform-infra-gitea.ts b/scripts/src/platform-infra-gitea.ts index ed3c7917..cdd1e081 100644 --- a/scripts/src/platform-infra-gitea.ts +++ b/scripts/src/platform-infra-gitea.ts @@ -8,17 +8,20 @@ import { capture, compactCapture, createYamlFieldReader, + fingerprintValues, parseJsonOutput, readYamlRecord, shQuote, } from "./platform-infra-ops-library"; -import { applyPk01CaddyBlock, prepareFrpcSecret, renderFrpcManifest } from "./platform-infra-public-service"; +import { applyPk01CaddyBlock, escapeTomlString, prepareFrpcSecret, renderFrpcManifest } from "./platform-infra-public-service"; import type { PublicServiceExposure, PublicServiceTarget, FrpcSecretMaterial } from "./platform-infra-public-service"; +import { applyPk01CaddySiteBlock, caddyManagedBlockMarkers } from "./pk01-caddy"; import { parseEnvFile, readTextFile, redactRepoPath, requiredEnvValue } from "./secrets"; const configFile = rootPath("config", "platform-infra", "gitea.yaml"); const configLabel = "config/platform-infra/gitea.yaml"; const remoteScriptFile = rootPath("scripts", "src", "platform-infra-gitea-remote.sh"); +const githubSyncServerFile = rootPath("scripts", "src", "platform-infra-gitea-github-sync-server.mjs"); const fieldManager = "unidesk-platform-infra-gitea"; const y = createYamlFieldReader(configLabel); @@ -67,6 +70,7 @@ interface GiteaConfig { url: string; noProxy: string[]; }; + webhookSync: GiteaWebhookSync; responsibilities: GiteaSourceResponsibility[]; repositories: GiteaMirrorRepository[]; }; @@ -149,6 +153,32 @@ interface GiteaGithubCredential { requiredFor: string[]; } +interface GiteaWebhookSync { + enabled: boolean; + direction: "github-to-gitea"; + publicPath: string; + events: string[]; + secret: { + sourceRef: string; + sourceKey: string; + createIfMissing: boolean; + }; + bridge: { + deploymentName: string; + serviceName: string; + secretName: string; + configMapName: string; + image: string; + replicas: number; + httpPort: number; + serviceAccountName: string; + }; + frpc: { + proxyName: string; + remotePort: number; + }; +} + interface GiteaSourceResponsibility { name: string; current: string; @@ -206,6 +236,7 @@ interface MirrorSecrets { adminUsername: string; adminPassword: string; githubToken: string; + webhookSecret: string; } interface MirrorRemoteParams { @@ -275,6 +306,7 @@ function readGiteaConfig(): GiteaConfig { const sourceAuthority = y.objectField(root, "sourceAuthority", ""); const sourceCredentials = y.objectField(sourceAuthority, "credentials", "sourceAuthority"); const githubProxy = y.objectField(sourceAuthority, "githubProxy", "sourceAuthority"); + const webhookSync = y.objectField(sourceAuthority, "webhookSync", "sourceAuthority"); const app = y.objectField(root, "app", ""); const image = y.objectField(app, "image", "app"); const service = y.objectField(app, "service", "app"); @@ -328,6 +360,7 @@ function readGiteaConfig(): GiteaConfig { url: urlField(githubProxy, "url", "sourceAuthority.githubProxy"), noProxy: y.stringArrayField(githubProxy, "noProxy", "sourceAuthority.githubProxy"), }, + webhookSync: parseWebhookSync(webhookSync, "sourceAuthority.webhookSync"), responsibilities: y.arrayOfRecords(sourceAuthority.responsibilities, "sourceAuthority.responsibilities").map(parseResponsibility), repositories: y.arrayOfRecords(sourceAuthority.repositories, "sourceAuthority.repositories").map(parseMirrorRepository), }, @@ -417,6 +450,37 @@ function parseGithubCredential(record: Record, path: string): G }; } +function parseWebhookSync(record: Record, path: string): GiteaWebhookSync { + const secret = y.objectField(record, "secret", path); + const bridge = y.objectField(record, "bridge", path); + const frpc = y.objectField(record, "frpc", path); + return { + enabled: y.booleanField(record, "enabled", path), + direction: y.enumField(record, "direction", path, ["github-to-gitea"] as const), + publicPath: y.apiPathField(record, "publicPath", path), + events: y.stringArrayField(record, "events", path), + secret: { + sourceRef: secretSourceRefField(secret, "sourceRef", `${path}.secret`), + sourceKey: y.envKeyField(secret, "sourceKey", `${path}.secret`), + createIfMissing: y.booleanField(secret, "createIfMissing", `${path}.secret`), + }, + bridge: { + deploymentName: y.kubernetesNameField(bridge, "deploymentName", `${path}.bridge`), + serviceName: y.kubernetesNameField(bridge, "serviceName", `${path}.bridge`), + secretName: y.kubernetesNameField(bridge, "secretName", `${path}.bridge`), + configMapName: y.kubernetesNameField(bridge, "configMapName", `${path}.bridge`), + image: y.stringField(bridge, "image", `${path}.bridge`), + replicas: positiveInteger(bridge, "replicas", `${path}.bridge`), + httpPort: y.portField(bridge, "httpPort", `${path}.bridge`), + serviceAccountName: y.kubernetesNameField(bridge, "serviceAccountName", `${path}.bridge`), + }, + frpc: { + proxyName: y.stringField(frpc, "proxyName", `${path}.frpc`), + remotePort: y.portField(frpc, "remotePort", `${path}.frpc`), + }, + }; +} + function parsePublicExposure(record: Record, path: string): PublicServiceExposure & { secretRoot: string } { const dns = y.objectField(record, "dns", path); const frpc = y.objectField(record, "frpc", path); @@ -527,6 +591,11 @@ function validateConfig(gitea: GiteaConfig): void { if (!gitea.app.registration.disabled) throw new Error(`${configLabel}.app.registration.disabled must be true for the internal POC service`); if (gitea.app.probes.healthPath !== gitea.validation.healthPath) throw new Error(`${configLabel}.app.probes.healthPath must match validation.healthPath`); if (gitea.app.server.rootUrl !== gitea.app.publicExposure.publicBaseUrl.replace(/\/?$/u, "/")) throw new Error(`${configLabel}.app.server.rootUrl must match app.publicExposure.publicBaseUrl for the Web UI`); + if (gitea.sourceAuthority.webhookSync.enabled) { + const events = new Set(gitea.sourceAuthority.webhookSync.events); + if (!events.has("push")) throw new Error(`${configLabel}.sourceAuthority.webhookSync.events must include push for GitHub -> Gitea source sync`); + if (gitea.sourceAuthority.webhookSync.frpc.remotePort === gitea.app.publicExposure.frpc.remotePort) throw new Error(`${configLabel}.sourceAuthority.webhookSync.frpc.remotePort must differ from app.publicExposure.frpc.remotePort`); + } const repoKeys = new Set(); for (const repo of gitea.sourceAuthority.repositories) { if (repoKeys.has(repo.key)) throw new Error(`${configLabel}.sourceAuthority.repositories contains duplicate key ${repo.key}`); @@ -571,11 +640,12 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise check.ok)) return { ok: false, action: "platform-infra-gitea-apply", mode: "policy-blocked", mutation: false, policy }; - const frpcSecret = prepareGiteaFrpcSecret(gitea); - const result = await capture(config, target.route, ["sh"], remoteScript("apply", gitea, target, manifest, options, { frpcSecret })); + const frpcSecret = prepareGiteaFrpcSecret(gitea, target); + const secrets = gitea.sourceAuthority.webhookSync.enabled && !options.dryRun ? ensureMirrorSecrets(gitea, true, true) : undefined; + const result = await capture(config, target.route, ["sh"], remoteScript("apply", gitea, target, manifest, options, { frpcSecret, secrets })); const parsed = parseJsonOutput(result.stdout); const caddy = !options.dryRun && result.exitCode === 0 && parsed?.ok === true && gitea.app.publicExposure.enabled - ? await applyPk01CaddyBlock(config, "gitea", gitea.app.publicExposure) + ? await applyGiteaCaddyBlock(config, gitea) : { ok: true, skipped: true, reason: options.dryRun ? "dry-run" : "apply-not-ready" }; return { ok: result.exitCode === 0 && parsed?.ok === true && record(caddy).ok !== false, @@ -586,7 +656,7 @@ async function apply(config: UniDeskConfig, options: ApplyOptions): Promise --confirm", ], }, }; } +async function mirrorWebhookCommand(config: UniDeskConfig, args: string[]): Promise | RenderedCliResult> { + const [action = "status"] = args; + if (action === "apply") { + const options = parseMirrorOptions(args.slice(1)); + const result = await mirrorWebhookApply(config, options); + return options.full || options.raw ? result : renderMirrorWebhookApply(result); + } + if (action === "status") { + const options = parseMirrorOptions(args.slice(1)); + const result = await mirrorWebhookStatus(config, options); + return options.full || options.raw ? result : renderMirrorWebhookStatus(result); + } + if (action === "test") { + const options = parseMirrorOptions(args.slice(1)); + const result = await mirrorWebhookTest(config, options); + return options.full || options.raw ? result : renderMirrorWebhookTest(result); + } + return { + ok: false, + error: "unsupported-platform-infra-gitea-mirror-webhook-command", + usage: "platform-infra gitea mirror webhook apply|status|test --target [--repo ] [--confirm]", + }; +} + function mirrorPlan(options: CommonOptions): Record { const gitea = readGiteaConfig(); const target = resolveTarget(gitea, options.targetId); @@ -689,7 +789,7 @@ async function mirrorStatus(config: UniDeskConfig, options: CommonOptions & { re const health = await validate(config, options); const healthValidation = record(health.validation); const credentials = credentialSummaries(gitea); - const secrets = ensureMirrorSecrets(gitea, false); + const secrets = ensureMirrorSecrets(gitea, false, false); const selectedRepos = selectedRepositories(gitea, target, options.repoKey ?? null); const result = await capture(config, target.route, ["sh"], remoteScript("mirror-status", gitea, target, "", { ...options, confirm: false, dryRun: true, wait: false }, { repos: selectedRepos, secrets })); const parsed = parseJsonOutput(result.stdout); @@ -724,7 +824,7 @@ async function mirrorBootstrap(config: UniDeskConfig, options: MirrorOptions): P const target = resolveTarget(gitea, options.targetId); if (!options.confirm) return { ok: false, action: "platform-infra-gitea-mirror-bootstrap", mutation: false, mode: "missing-confirm", error: "mirror bootstrap requires --confirm" }; const repos = selectedRepositories(gitea, target, options.repoKey); - const secrets = ensureMirrorSecrets(gitea, true); + const secrets = ensureMirrorSecrets(gitea, true, true); const result = await capture(config, target.route, ["sh"], remoteScript("mirror-bootstrap", gitea, target, "", { ...options, dryRun: false, wait: false }, { repos, secrets })); const parsed = parseJsonOutput(result.stdout); return { @@ -744,7 +844,7 @@ async function mirrorSync(config: UniDeskConfig, options: MirrorOptions): Promis const target = resolveTarget(gitea, options.targetId); if (!options.confirm) return { ok: false, action: "platform-infra-gitea-mirror-sync", mutation: false, mode: "missing-confirm", error: "mirror sync requires --confirm" }; const repos = selectedRepositories(gitea, target, options.repoKey); - const secrets = ensureMirrorSecrets(gitea, false); + const secrets = ensureMirrorSecrets(gitea, false, false); const result = await capture(config, target.route, ["sh"], remoteScript("mirror-sync", gitea, target, "", { ...options, dryRun: false, wait: false }, { repos, secrets })); const parsed = parseJsonOutput(result.stdout); return { @@ -759,6 +859,69 @@ async function mirrorSync(config: UniDeskConfig, options: MirrorOptions): Promis }; } +async function mirrorWebhookApply(config: UniDeskConfig, options: MirrorOptions): Promise> { + const gitea = readGiteaConfig(); + const target = resolveTarget(gitea, options.targetId); + if (!gitea.sourceAuthority.webhookSync.enabled) return { ok: false, action: "platform-infra-gitea-mirror-webhook-apply", mutation: false, error: "sourceAuthority.webhookSync.enabled is false" }; + if (!options.confirm) return { ok: false, action: "platform-infra-gitea-mirror-webhook-apply", mutation: false, mode: "missing-confirm", error: "mirror webhook apply requires --confirm" }; + const repos = selectedRepositories(gitea, target, options.repoKey); + const secrets = ensureMirrorSecrets(gitea, false, false); + const result = await capture(config, target.route, ["sh"], remoteScript("mirror-webhook-apply", gitea, target, "", { ...options, dryRun: false, wait: false }, { repos, secrets })); + const parsed = parseJsonOutput(result.stdout); + return { + ok: result.exitCode === 0 && parsed?.ok === true, + action: "platform-infra-gitea-mirror-webhook-apply", + mutation: true, + target: targetSummary(target), + webhook: webhookSyncSummary(gitea), + repositories: arrayRecords(record(parsed).repositories), + remote: parsed ?? compactCapture(result, { full: true }), + next: mirrorNextCommands(target.id), + }; +} + +async function mirrorWebhookStatus(config: UniDeskConfig, options: MirrorOptions): Promise> { + const gitea = readGiteaConfig(); + const target = resolveTarget(gitea, options.targetId); + const repos = selectedRepositories(gitea, target, options.repoKey); + const secrets = ensureMirrorSecrets(gitea, false, false); + const result = await capture(config, target.route, ["sh"], remoteScript("mirror-webhook-status", gitea, target, "", { ...options, confirm: false, dryRun: true, wait: false }, { repos, secrets })); + const parsed = parseJsonOutput(result.stdout); + const repositories = arrayRecords(record(parsed).repositories); + return { + ok: result.exitCode === 0 && parsed?.ok === true, + action: "platform-infra-gitea-mirror-webhook-status", + mutation: false, + target: targetSummary(target), + webhook: webhookSyncSummary(gitea), + repositories, + bridge: record(parsed).bridge, + remote: parsed ?? compactCapture(result, { full: true }), + next: mirrorNextCommands(target.id), + }; +} + +async function mirrorWebhookTest(config: UniDeskConfig, options: MirrorOptions): Promise> { + const gitea = readGiteaConfig(); + const target = resolveTarget(gitea, options.targetId); + if (!options.confirm) return { ok: false, action: "platform-infra-gitea-mirror-webhook-test", mutation: false, mode: "missing-confirm", error: "mirror webhook test requires --confirm" }; + const repos = selectedRepositories(gitea, target, options.repoKey); + if (repos.length !== 1) throw new Error("mirror webhook test requires exactly one --repo "); + const secrets = ensureMirrorSecrets(gitea, false, false); + const result = await capture(config, target.route, ["sh"], remoteScript("mirror-webhook-test", gitea, target, "", { ...options, dryRun: false, wait: false }, { repos, secrets })); + const parsed = parseJsonOutput(result.stdout); + return { + ok: result.exitCode === 0 && parsed?.ok === true, + action: "platform-infra-gitea-mirror-webhook-test", + mutation: true, + target: targetSummary(target), + webhook: webhookSyncSummary(gitea), + repositories: arrayRecords(record(parsed).repositories), + remote: parsed ?? compactCapture(result, { full: true }), + next: mirrorNextCommands(target.id), + }; +} + function renderManifest(gitea: GiteaConfig, target: GiteaTarget): string { const app = gitea.app; const image = `${app.image.repository}:${app.image.tag}`; @@ -918,7 +1081,139 @@ ${envVars(gitea, target)} resources: requests: storage: ${app.storage.config.size} -${renderFrpcManifest(publicServiceTarget(gitea, target))}`; +${renderFrpcManifest(publicServiceTarget(gitea, target))} +${renderGithubSyncManifest(gitea, target)}`; +} + +function renderGithubSyncManifest(gitea: GiteaConfig, target: GiteaTarget): string { + const sync = gitea.sourceAuthority.webhookSync; + if (!sync.enabled) return ""; + const labels = ` app.kubernetes.io/name: ${sync.bridge.deploymentName} + app.kubernetes.io/component: github-to-gitea-sync + app.kubernetes.io/part-of: devops-infra + app.kubernetes.io/managed-by: unidesk`; + const reposJson = JSON.stringify(repositoriesForTarget(gitea, target).map(remoteRepoSpec), null, 2); + const serverSource = readFileSync(githubSyncServerFile, "utf8"); + const configHash = createHash("sha256").update(reposJson).update("\0").update(serverSource).digest("hex").slice(0, 16); + return `--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ${sync.bridge.configMapName} + namespace: ${target.namespace} + labels: +${labels} +data: + repos.json: | +${indentBlock(reposJson, 4)} + server.mjs: | +${indentBlock(serverSource, 4)} +--- +apiVersion: v1 +kind: Service +metadata: + name: ${sync.bridge.serviceName} + namespace: ${target.namespace} + labels: +${labels} +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: ${sync.bridge.deploymentName} + app.kubernetes.io/component: github-to-gitea-sync + ports: + - name: http + port: ${sync.bridge.httpPort} + targetPort: http + protocol: TCP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${sync.bridge.deploymentName} + namespace: ${target.namespace} + labels: +${labels} + annotations: + unidesk.ai/sync-direction: ${yamlQuote(sync.direction)} + unidesk.ai/public-path: ${yamlQuote(sync.publicPath)} +spec: + replicas: ${sync.bridge.replicas} + selector: + matchLabels: + app.kubernetes.io/name: ${sync.bridge.deploymentName} + app.kubernetes.io/component: github-to-gitea-sync + template: + metadata: + labels: + app.kubernetes.io/name: ${sync.bridge.deploymentName} + app.kubernetes.io/component: github-to-gitea-sync + app.kubernetes.io/part-of: devops-infra + app.kubernetes.io/managed-by: unidesk + annotations: + unidesk.ai/github-sync-config-sha: ${yamlQuote(configHash)} + spec: + containers: + - name: github-to-gitea-sync + image: ${sync.bridge.image} + imagePullPolicy: IfNotPresent + command: + - node + - /etc/gitea-github-sync/server.mjs + ports: + - name: http + containerPort: ${sync.bridge.httpPort} + env: + - name: UNIDESK_GITEA_WEBHOOK_PORT + value: ${yamlQuote(String(sync.bridge.httpPort))} + - name: UNIDESK_GITEA_WEBHOOK_PATH + value: ${yamlQuote(sync.publicPath)} + - name: UNIDESK_GITEA_REPOS_PATH + value: /etc/gitea-github-sync/repos.json + - name: UNIDESK_GITEA_SERVICE_BASE_URL + value: ${yamlQuote(`http://${gitea.app.serviceName}.${target.namespace}.svc.cluster.local:${gitea.app.service.httpPort}`)} + - name: GITHUB_TOKEN + valueFrom: + secretKeyRef: + name: ${sync.bridge.secretName} + key: github-token + - name: GITEA_USERNAME + valueFrom: + secretKeyRef: + name: ${sync.bridge.secretName} + key: gitea-username + - name: GITEA_PASSWORD + valueFrom: + secretKeyRef: + name: ${sync.bridge.secretName} + key: gitea-password + - name: GITHUB_WEBHOOK_SECRET + valueFrom: + secretKeyRef: + name: ${sync.bridge.secretName} + key: github-webhook-secret + - name: HTTPS_PROXY + value: ${yamlQuote(gitea.sourceAuthority.githubProxy.enabled ? gitea.sourceAuthority.githubProxy.url : "")} + - name: HTTP_PROXY + value: ${yamlQuote(gitea.sourceAuthority.githubProxy.enabled ? gitea.sourceAuthority.githubProxy.url : "")} + - name: NO_PROXY + value: ${yamlQuote(gitea.sourceAuthority.githubProxy.noProxy.join(","))} + readinessProbe: + httpGet: + path: /healthz + port: http + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 6 + volumeMounts: + - name: config + mountPath: /etc/gitea-github-sync + readOnly: true + volumes: + - name: config + configMap: + name: ${sync.bridge.configMapName} +`; } function envVars(gitea: GiteaConfig, target: GiteaTarget): string { @@ -947,7 +1242,7 @@ function envVars(gitea: GiteaConfig, target: GiteaTarget): string { value: ${yamlQuote(value)}`).join("\n"); } -function remoteScript(action: "apply" | "status" | "validate" | "mirror-bootstrap" | "mirror-sync" | "mirror-status", gitea: GiteaConfig, target: GiteaTarget, manifest: string, options: ApplyOptions | MirrorOptions, params: MirrorRemoteParams = {}): string { +function remoteScript(action: "apply" | "status" | "validate" | "mirror-bootstrap" | "mirror-sync" | "mirror-status" | "mirror-webhook-apply" | "mirror-webhook-status" | "mirror-webhook-test", gitea: GiteaConfig, target: GiteaTarget, manifest: string, options: ApplyOptions | MirrorOptions, params: MirrorRemoteParams = {}): string { const env: Record = { UNIDESK_GITEA_ACTION: action, UNIDESK_GITEA_TARGET_ID: target.id, @@ -971,6 +1266,15 @@ function remoteScript(action: "apply" | "status" | "validate" | "mirror-bootstra UNIDESK_GITEA_GITHUB_PROXY_URL: gitea.sourceAuthority.githubProxy.url, UNIDESK_GITEA_NO_PROXY: gitea.sourceAuthority.githubProxy.noProxy.join(","), UNIDESK_GITEA_FRPC_ENABLED: gitea.app.publicExposure.enabled ? "1" : "0", + UNIDESK_GITEA_FRPC_DEPLOYMENT_NAME: gitea.app.publicExposure.frpc.deploymentName, + UNIDESK_GITEA_WEBHOOK_SYNC_ENABLED: gitea.sourceAuthority.webhookSync.enabled ? "1" : "0", + UNIDESK_GITEA_WEBHOOK_PUBLIC_URL: githubWebhookPublicUrl(gitea), + UNIDESK_GITEA_WEBHOOK_PATH: gitea.sourceAuthority.webhookSync.publicPath, + UNIDESK_GITEA_WEBHOOK_EVENTS: gitea.sourceAuthority.webhookSync.events.join(","), + UNIDESK_GITEA_WEBHOOK_DEPLOYMENT: gitea.sourceAuthority.webhookSync.bridge.deploymentName, + UNIDESK_GITEA_WEBHOOK_SERVICE: gitea.sourceAuthority.webhookSync.bridge.serviceName, + UNIDESK_GITEA_WEBHOOK_SERVICE_PORT: String(gitea.sourceAuthority.webhookSync.bridge.httpPort), + UNIDESK_GITEA_WEBHOOK_SECRET_NAME: gitea.sourceAuthority.webhookSync.bridge.secretName, }; if (params.frpcSecret !== undefined) { env.UNIDESK_GITEA_FRPC_SECRET_NAME = params.frpcSecret.secretName; @@ -981,6 +1285,7 @@ function remoteScript(action: "apply" | "status" | "validate" | "mirror-bootstra env.UNIDESK_GITEA_ADMIN_USERNAME = params.secrets.adminUsername; env.UNIDESK_GITEA_ADMIN_PASSWORD = params.secrets.adminPassword; env.UNIDESK_GITEA_GITHUB_TOKEN = params.secrets.githubToken; + env.UNIDESK_GITEA_GITHUB_WEBHOOK_SECRET = params.secrets.webhookSecret; } const exports = Object.entries(env).map(([key, value]) => `export ${key}=${shQuote(value)}`).join("\n"); return `${exports}\n${readFileSync(remoteScriptFile, "utf8")}`; @@ -1062,8 +1367,8 @@ function publicServiceTarget(gitea: GiteaConfig, target: GiteaTarget): PublicSer }; } -function prepareGiteaFrpcSecret(gitea: GiteaConfig): FrpcSecretMaterial { - return prepareFrpcSecret({ +function prepareGiteaFrpcSecret(gitea: GiteaConfig, target: GiteaTarget): FrpcSecretMaterial { + const base = prepareFrpcSecret({ secretRoot: gitea.app.publicExposure.secretRoot, exposure: gitea.app.publicExposure, sourcePathRedactor: redactRepoPath, @@ -1071,6 +1376,47 @@ function prepareGiteaFrpcSecret(gitea: GiteaConfig): FrpcSecretMaterial { requiredEnvValue, readTextFile, }); + const sync = gitea.sourceAuthority.webhookSync; + if (!sync.enabled) return base; + const extraProxy = [ + "[[proxies]]", + `name = "${escapeTomlString(sync.frpc.proxyName)}"`, + 'type = "tcp"', + `localIP = "${escapeTomlString(sync.bridge.serviceName)}.${escapeTomlString(target.namespace)}.svc.cluster.local"`, + `localPort = ${sync.bridge.httpPort}`, + `remotePort = ${sync.frpc.remotePort}`, + "", + ].join("\n"); + const frpcToml = `${base.frpcToml.trim()}\n\n${extraProxy}`; + return { + ...base, + frpcToml, + fingerprint: fingerprintValues({ frpcToml }, ["frpcToml"]), + }; +} + +async function applyGiteaCaddyBlock(config: UniDeskConfig, gitea: GiteaConfig): Promise> { + const exposure = gitea.app.publicExposure; + const sync = gitea.sourceAuthority.webhookSync; + if (!sync.enabled) return await applyPk01CaddyBlock(config, "gitea", exposure); + const siteBlock = `${exposure.dns.hostname} { + handle ${sync.publicPath}* { + reverse_proxy 127.0.0.1:${sync.frpc.remotePort} { + transport http { + response_header_timeout ${exposure.pk01.responseHeaderTimeoutSeconds}s + } + } + } + + handle { + reverse_proxy 127.0.0.1:${exposure.frpc.remotePort} { + transport http { + response_header_timeout ${exposure.pk01.responseHeaderTimeoutSeconds}s + } + } + } +}`; + return await applyPk01CaddySiteBlock(config, "gitea", exposure, siteBlock, caddyManagedBlockMarkers("gitea")); } function publicExposureSummary(gitea: GiteaConfig): Record { @@ -1103,6 +1449,24 @@ function frpcSecretSummary(secret: FrpcSecretMaterial): Record }; } +function webhookSecretSummary(gitea: GiteaConfig, secrets: MirrorSecrets): Record { + const sync = gitea.sourceAuthority.webhookSync; + return { + enabled: sync.enabled, + publicUrl: githubWebhookPublicUrl(gitea), + sourceRef: sync.secret.sourceRef, + targetSecret: sync.bridge.secretName, + keys: ["github-token", "gitea-username", "gitea-password", "github-webhook-secret"], + fingerprint: fingerprintSecretParts([secrets.webhookSecret]), + valuesPrinted: false, + }; +} + +function githubWebhookPublicUrl(gitea: GiteaConfig): string { + const base = gitea.app.publicExposure.publicBaseUrl.replace(/\/+$/u, ""); + return `${base}${gitea.sourceAuthority.webhookSync.publicPath}`; +} + function statusSummary(payload: Record): Record { return { ready: payload.ready === true, @@ -1183,10 +1547,25 @@ function mirrorNextCommands(targetId: string): Record { bootstrap: `bun scripts/cli.ts platform-infra gitea mirror bootstrap --target ${targetId} --confirm`, sync: `bun scripts/cli.ts platform-infra gitea mirror sync --target ${targetId} --confirm`, status: `bun scripts/cli.ts platform-infra gitea mirror status --target ${targetId}`, + webhookApply: `bun scripts/cli.ts platform-infra gitea mirror webhook apply --target ${targetId} --confirm`, + webhookStatus: `bun scripts/cli.ts platform-infra gitea mirror webhook status --target ${targetId}`, full: `bun scripts/cli.ts platform-infra gitea mirror status --target ${targetId} --full`, }; } +function webhookSyncSummary(gitea: GiteaConfig): Record { + const sync = gitea.sourceAuthority.webhookSync; + return { + enabled: sync.enabled, + direction: sync.direction, + publicUrl: githubWebhookPublicUrl(gitea), + events: sync.events, + bridge: sync.bridge, + frpc: sync.frpc, + valuesPrinted: false, + }; +} + function credentialSummaries(gitea: GiteaConfig): Array> { const adminPath = credentialPath(gitea, gitea.sourceAuthority.credentials.admin.sourceRef); const githubPath = credentialPath(gitea, gitea.sourceAuthority.credentials.github.sourceRef); @@ -1220,7 +1599,7 @@ function credentialBlockers(credentials: Array>): string return credentials.filter((item) => item.present !== true || item.requiredKeysPresent !== true).map((item) => String(item.id)); } -function ensureMirrorSecrets(gitea: GiteaConfig, createAdmin: boolean): MirrorSecrets { +function ensureMirrorSecrets(gitea: GiteaConfig, createAdmin: boolean, createWebhookSecret: boolean): MirrorSecrets { const admin = gitea.sourceAuthority.credentials.admin; const adminPath = credentialPath(gitea, admin.sourceRef); if (!existsSync(adminPath)) { @@ -1236,12 +1615,29 @@ function ensureMirrorSecrets(gitea: GiteaConfig, createAdmin: boolean): MirrorSe const githubPath = credentialPath(gitea, github.sourceRef); if (!existsSync(githubPath)) throw new Error(`${github.sourceRef} is missing; cannot sync GitHub upstream`); const githubEnv = parseEnvFileSafe(githubPath); + const webhookSecret = ensureWebhookSecret(gitea, createWebhookSecret); const adminUsername = adminLinePair.username; const adminPassword = adminLinePair.password; const githubToken = githubEnv[github.sourceKey]; if (!adminUsername || !adminPassword) throw new Error(`${admin.sourceRef} must contain username on line ${admin.usernameLine} and password on line ${admin.passwordLine}`); if (!githubToken) throw new Error(`${github.sourceRef} must contain ${github.sourceKey}`); - return { adminUsername, adminPassword, githubToken }; + return { adminUsername, adminPassword, githubToken, webhookSecret }; +} + +function ensureWebhookSecret(gitea: GiteaConfig, createIfMissing: boolean): string { + const sync = gitea.sourceAuthority.webhookSync; + if (!sync.enabled) return ""; + const path = credentialPath(gitea, sync.secret.sourceRef); + if (!existsSync(path)) { + if (!createIfMissing || !sync.secret.createIfMissing) throw new Error(`${sync.secret.sourceRef} is missing; run platform-infra gitea apply --target --confirm first`); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `${sync.secret.sourceKey}=ghs_${randomBytes(32).toString("base64url")}\n`, { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); + } + const values = parseEnvFileSafe(path); + const value = values[sync.secret.sourceKey]; + if (!value) throw new Error(`${sync.secret.sourceRef} must contain ${sync.secret.sourceKey}`); + return value; } function credentialPath(gitea: GiteaConfig, sourceRef: string): string { @@ -1382,6 +1778,52 @@ function renderMirrorStatus(result: Record): RenderedCliResult ]); } +function renderMirrorWebhookApply(result: Record): RenderedCliResult { + const repos = arrayRecords(result.repositories).map((repo) => [stringValue(repo.key), stringValue(repo.repository), boolText(repo.hookReady), stringValue(repo.hookId), compactTail(stringValue(repo.error))]); + const next = record(result.next); + return rendered(result, "platform-infra gitea mirror webhook apply", [ + "PLATFORM-INFRA GITEA MIRROR WEBHOOK APPLY", + ...table(["OK", "MUTATION", "TARGET", "URL"], [[boolText(result.ok), boolText(result.mutation), stringValue(record(result.target).id), stringValue(record(result.webhook).publicUrl)]]), + "", + "GITHUB HOOKS", + ...(repos.length === 0 ? ["-"] : table(["KEY", "REPOSITORY", "READY", "HOOK_ID", "ERROR"], repos)), + ...remoteErrorLines(result), + "", + `NEXT ${stringValue(next.webhookStatus)}`, + ]); +} + +function renderMirrorWebhookStatus(result: Record): RenderedCliResult { + const repos = arrayRecords(result.repositories).map((repo) => [stringValue(repo.key), stringValue(repo.repository), boolText(repo.hookReady), stringValue(repo.hookId), stringValue(repo.active), compactTail(stringValue(repo.error))]); + const bridge = record(result.bridge); + const next = record(result.next); + return rendered(result, "platform-infra gitea mirror webhook status", [ + "PLATFORM-INFRA GITEA MIRROR WEBHOOK STATUS", + ...table(["TARGET", "BRIDGE", "READY", "URL"], [[stringValue(record(result.target).id), stringValue(bridge.deployment), boolText(bridge.ready), stringValue(record(result.webhook).publicUrl)]]), + "", + "GITHUB HOOKS", + ...(repos.length === 0 ? ["-"] : table(["KEY", "REPOSITORY", "READY", "HOOK_ID", "ACTIVE", "ERROR"], repos)), + ...remoteErrorLines(result), + "", + `NEXT ${stringValue(next.webhookApply)}`, + ]); +} + +function renderMirrorWebhookTest(result: Record): RenderedCliResult { + const repos = arrayRecords(result.repositories).map((repo) => [stringValue(repo.key), boolText(repo.pingOk), boolText(repo.testOk), boolText(repo.sourceBundleReady), stringValue(repo.sourceCommit ?? repo.branchCommit).slice(0, 12), stringValue(repo.snapshotCommit).slice(0, 12), compactTail(stringValue(repo.error))]); + const next = record(result.next); + return rendered(result, "platform-infra gitea mirror webhook test", [ + "PLATFORM-INFRA GITEA MIRROR WEBHOOK TEST", + ...table(["OK", "MUTATION", "TARGET"], [[boolText(result.ok), boolText(result.mutation), stringValue(record(result.target).id)]]), + "", + "TEST", + ...(repos.length === 0 ? ["-"] : table(["KEY", "GH_PING", "POST_OK", "SNAPSHOT_READY", "SOURCE", "SNAPSHOT", "ERROR"], repos)), + ...remoteErrorLines(result), + "", + `NEXT ${stringValue(next.webhookStatus)}`, + ]); +} + function renderPlan(result: Record): RenderedCliResult { const config = record(result.config); @@ -1420,8 +1862,11 @@ function renderApply(result: Record): RenderedCliResult { const remote = record(result.remote); const steps = record(remote.steps); const frpcStep = record(steps.frpcSecret); + const webhookSecretStep = record(steps.webhookSyncSecret); const applyStep = record(steps.apply); const rolloutStep = record(steps.rollout); + const frpcRolloutStep = record(steps.frpcRollout); + const webhookRolloutStep = record(steps.webhookSyncRollout); const caddy = record(result.pk01Caddy); return rendered(result, "platform-infra gitea apply", [ "PLATFORM-INFRA GITEA APPLY", @@ -1430,8 +1875,11 @@ function renderApply(result: Record): RenderedCliResult { "STEPS", ...table(["STEP", "EXIT", "DETAIL"], [ ["frpc-secret", stringValue(frpcStep.exitCode), compactLine(stringValue(frpcStep.stderrTail, stringValue(frpcStep.stdoutTail)))], + ["webhook-secret", stringValue(webhookSecretStep.exitCode), compactLine(stringValue(webhookSecretStep.stderrTail, stringValue(webhookSecretStep.stdoutTail)))], ["apply", stringValue(applyStep.exitCode), compactLine(stringValue(applyStep.stderrTail, stringValue(applyStep.stdoutTail)))], ["rollout", stringValue(rolloutStep.exitCode), compactLine(stringValue(rolloutStep.stderrTail, stringValue(rolloutStep.stdoutTail)))], + ["frpc-rollout", stringValue(frpcRolloutStep.exitCode), compactLine(stringValue(frpcRolloutStep.stderrTail, stringValue(frpcRolloutStep.stdoutTail)))], + ["webhook-rollout", stringValue(webhookRolloutStep.exitCode), compactLine(stringValue(webhookRolloutStep.stderrTail, stringValue(webhookRolloutStep.stdoutTail)))], ["pk01-caddy", stringValue(caddy.exitCode ?? (caddy.ok === false ? 1 : 0)), compactLine(stringValue(caddy.error, stringValue(caddy.status, stringValue(caddy.reason))))], ]), ...remoteErrorLines(result), @@ -1660,6 +2108,11 @@ function yamlQuote(value: string): string { return JSON.stringify(value); } +function indentBlock(value: string, spaces: number): string { + const prefix = " ".repeat(spaces); + return value.split(/\r?\n/u).map((line) => `${prefix}${line}`).join("\n"); +} + function record(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; }