pikasTech-unidesk/scripts/src/platform-infra-gitea-github-sync-server.test.mjs

import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { createHash, createHmac } from "node:crypto";
import { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
import { createServer } from "node:http";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import {
  createGithubSyncHandler,
  createDurableInboxStore,
  createDurableSyncController,
  createSyncCoordinator,
  readFailedDeliveryRecoveryStatus,
  run,
  runResult,
  syncRepository,
} from "./platform-infra-gitea-github-sync-server.mjs";

const zeroDelay = async () => {};

test("primary status exposes a bounded redacted failed-delivery recovery ledger", async () => {
  const root = mkdtempSync(join(tmpdir(), "gitea-hook-recovery-status-"));
  const statePath = join(root, "ledger.json");
  try {
    const records = Object.fromEntries(Array.from({ length: 55 }, (_, index) => [`record-${index}`, {
      repository: "pikasTech/agentrun",
      hookId: 77,
      deliveryId: `guid-${index}`,
      apiDeliveryId: String(1000 + index),
      state: index % 2 === 0 ? "attempted" : "pending",
      attempts: index % 3,
      cooldownMs: 300_000,
      nextAt: "2026-07-12T07:11:40Z",
      updatedAt: `2026-07-12T07:${String(index).padStart(2, "0")}:00Z`,
      lastStatusCode: 502,
      secret: "must-not-leak",
      url: "https://must-not-leak.invalid/hook",
    }]));
    writeFileSync(statePath, JSON.stringify({ version: 1, kind: "GiteaGithubHookRecoveryLedger", records }));
    const status = await readFailedDeliveryRecoveryStatus(statePath);
    assert.equal(status.ready, true);
    assert.equal(status.entries.length, 50);
    assert.equal(status.counts.attempted + status.counts.pending, 55);
    assert.equal(JSON.stringify(status).includes("must-not-leak"), false);
    writeFileSync(statePath, JSON.stringify({
      version: 1,
      kind: "GiteaGithubHookRecoveryLedger",
      records: { corrupt: { ...records["record-0"], state: "unknown-hidden-state" } },
    }));
    const invalid = await readFailedDeliveryRecoveryStatus(statePath);
    assert.equal(invalid.ready, false);
    assert.equal(invalid.state, "invalid-or-unreadable");
    assert.equal(invalid.errorType, "hook-recovery-ledger-record-invalid");
    assert.deepEqual(invalid.entries, []);
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
});

test("explicit entrypoint starts through Kubernetes ConfigMap symlinks", () => {
  const root = mkdtempSync(join(tmpdir(), "gitea-sync-main-"));
  try {
    const dataDirectory = join(root, "..data");
    const serverSource = join(dataDirectory, "server.mjs");
    const entrypointSource = join(dataDirectory, "entrypoint.mjs");
    const marker = join(root, "started");
    mkdirSync(dataDirectory);
    writeFileSync(serverSource, `import { writeFileSync } from "node:fs";\nexport function startServer() { writeFileSync(${JSON.stringify(marker)}, "started\\n"); }\n`, { flag: "wx" });
    writeFileSync(entrypointSource, 'import { startServer } from "./server.mjs";\nstartServer();\n', { flag: "wx" });
    symlinkSync(join("..data", "server.mjs"), join(root, "server.mjs"));
    symlinkSync(join("..data", "entrypoint.mjs"), join(root, "entrypoint.mjs"));

    execFileSync(process.execPath, [join(root, "entrypoint.mjs")], { stdio: "pipe" });
    assert.equal(existsSync(marker), true);
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
});

test("HTTP 202 waits for durable acceptance but does not claim refs are committed", async () => {
  const repo = fakeRepo("one");
  const logs = [];
  let release;
  const gate = new Promise((resolve) => { release = resolve; });
  let mode = "accept";
  const controller = {
    readiness: () => ({ ready: true }),
    status: async () => ({ ready: true, counts: {}, deliveries: [] }),
    async accept() {
      if (mode === "accept") {
        await gate;
        return { state: "accepted", duplicate: false };
      }
      throw new Error("Authorization: Basic c2VjcmV0");
    },
  };
  const server = createServer(createGithubSyncHandler({
    path: "/hook",
    responseBudgetMs: 1000,
    maxBodyBytes: 4096,
    webhookSecret: "test-secret",
    repos: [repo],
    controller,
    log: (event) => logs.push(event),
  }));
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
  const port = server.address().port;
  try {
    const pending = postWebhook(port, repo, "http-success", "a".repeat(40), "test-secret");
    const beforeCommit = await Promise.race([pending.then(() => "responded"), new Promise((resolve) => setTimeout(() => resolve("waiting"), 30))]);
    assert.equal(beforeCommit, "waiting");
    release();
    const accepted = await pending;
    assert.equal(accepted.status, 202);
    assert.deepEqual(await accepted.json(), {
      ok: true,
      event: "push",
      deliveryId: "http-success",
      repository: repo.upstream.repository,
      ref: "refs/heads/main",
      repo: repo.key,
      requestedCommit: "a".repeat(40),
      state: "accepted",
      disposition: "durable-inbox-accepted",
      refsCommitted: false,
      valuesPrinted: false,
    });

    mode = "throw";
    const unexpected = await postWebhook(port, repo, "http-throw", "d".repeat(40), "test-secret");
    assert.equal(unexpected.status, 503);
    assert.equal((await unexpected.json()).error, "internal-sync-error");
    const rejected = logs.at(-1);
    assert.deepEqual(
      [rejected.deliveryId, rejected.repository, rejected.ref, rejected.requestedCommit],
      ["http-throw", repo.upstream.repository, "refs/heads/main", "d".repeat(40)],
    );
    assert.match(rejected.error, /<redacted>/);
  } finally {
    server.close();
  }
});

test("duplicate HTTP 202 reports the durable accepted, processing, failed or committed state truthfully", async () => {
  const repo = fakeRepo("duplicate-state");
  const committedResult = {
    sourceCommit: "c".repeat(40),
    authorityCommit: "c".repeat(40),
    snapshotRef: `refs/snapshots/source/${"c".repeat(40)}`,
    disposition: "atomic-source-authority-committed",
  };
  let durable = { state: "accepted", duplicate: true, result: null };
  const controller = {
    readiness: () => ({ ready: true }),
    status: async () => ({ ready: true, counts: {}, deliveries: [] }),
    async accept() { return durable; },
  };
  const server = createServer(createGithubSyncHandler({
    path: "/hook",
    responseBudgetMs: 1000,
    maxBodyBytes: 4096,
    webhookSecret: "test-secret",
    repos: [repo],
    controller,
    log: () => {},
  }));
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
  const port = server.address().port;
  try {
    for (const state of ["accepted", "processing", "failed", "committed"]) {
      durable = { state, duplicate: true, result: state === "committed" ? committedResult : null };
      const response = await postWebhook(port, repo, `duplicate-${state}`, "c".repeat(40), "test-secret");
      assert.equal(response.status, 202);
      const body = await response.json();
      assert.equal(body.state, state);
      assert.equal(body.disposition, "duplicate-durable-inbox-record");
      assert.equal(body.refsCommitted, state === "committed");
      if (state === "committed") assert.deepEqual(body.result, committedResult);
      else assert.equal(Object.hasOwn(body, "result"), false);
    }
  } finally {
    server.close();
  }
});

test("durable acceptance response budget returns typed 503 while a late write continues and remains idempotent", async () => {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-acceptance-budget-"));
  const repo = fakeRepo("acceptance-budget");
  let releaseFirstAccept;
  const firstAcceptGate = new Promise((resolve) => { releaseFirstAccept = resolve; });
  let releaseSync;
  const syncGate = new Promise((resolve) => { releaseSync = resolve; });
  const logs = [];
  const durableController = createDurableSyncController(durableControllerOptions(root, [repo], {
    sync: async (_repo, delivery) => {
      await syncGate;
      return {
        ok: true,
        sourceCommit: delivery.requestedCommit,
        authorityCommit: delivery.requestedCommit,
        snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`,
        disposition: "atomic-source-authority-committed",
      };
    },
  }));
  let acceptCalls = 0;
  const delayedController = {
    readiness: () => durableController.readiness(),
    status: () => durableController.status(),
    async accept(repoValue, delivery) {
      acceptCalls += 1;
      if (acceptCalls === 1) await firstAcceptGate;
      return await durableController.accept(repoValue, delivery);
    },
  };
  const server = createServer(createGithubSyncHandler({
    path: "/hook",
    responseBudgetMs: 40,
    maxBodyBytes: 4096,
    webhookSecret: "test-secret",
    repos: [repo],
    controller: delayedController,
    log: (event) => logs.push(event),
  }));
  try {
    await durableController.start();
    await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
    const port = server.address().port;
    const commit = "7".repeat(40);
    const startedAt = performance.now();
    const timedOut = await postWebhook(port, repo, "late-durable-write", commit, "test-secret", { marker: "late" });
    const elapsedMs = performance.now() - startedAt;
    assert.equal(timedOut.status, 503);
    assert.equal((await timedOut.json()).error, "durable-acceptance-response-deadline-exhausted");
    assert.ok(elapsedMs < 200, `durable acceptance deadline responded after ${elapsedMs}ms`);
    assert.equal((await durableController.status()).deliveries.length, 0);

    releaseFirstAccept();
    await waitFor(async () => (await durableController.status()).deliveries.length === 1);
    const duplicate = await postWebhook(port, repo, "late-durable-write", commit, "test-secret", { marker: "late" });
    assert.equal(duplicate.status, 202);
    const duplicateBody = await duplicate.json();
    assert.equal(duplicateBody.disposition, "duplicate-durable-inbox-record");
    assert.equal((await durableController.status()).deliveries.length, 1);
    assert.equal(logs.some((event) => event.event === "github-to-gitea-delivery-accepted-after-response-deadline"), true);

    releaseSync();
    await waitFor(async () => (await durableController.status()).counts.committed === 1);
    const committedDuplicate = await postWebhook(port, repo, "late-durable-write", commit, "test-secret", { marker: "late" });
    const committedBody = await committedDuplicate.json();
    assert.equal(committedBody.state, "committed");
    assert.equal(committedBody.refsCommitted, true);
    assert.equal(committedBody.result.sourceCommit, commit);
  } finally {
    server.close();
    await durableController.stop();
    rmSync(root, { recursive: true, force: true });
  }
});

test("durable inbox fsync acceptance is body-sensitive, fast, and later becomes committed", async () => {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-inbox-http-"));
  const repo = fakeRepo("durable");
  let releaseSync;
  const syncGate = new Promise((resolve) => { releaseSync = resolve; });
  let syncStarted = false;
  const controller = createDurableSyncController(durableControllerOptions(root, [repo], {
    sync: async (_repo, delivery) => {
      syncStarted = true;
      await syncGate;
      return {
        ok: true,
        sourceCommit: delivery.requestedCommit,
        authorityCommit: delivery.requestedCommit,
        snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`,
        disposition: "atomic-source-authority-committed",
      };
    },
  }));
  const server = createServer(createGithubSyncHandler({
    path: "/hook",
    responseBudgetMs: 1000,
    maxBodyBytes: 4096,
    webhookSecret: "test-secret",
    repos: [repo],
    controller,
    log: () => {},
  }));
  try {
    await controller.start();
    await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
    const port = server.address().port;
    const commit = "a".repeat(40);
    const startedAt = performance.now();
    const accepted = await postWebhook(port, repo, "durable-delivery", commit, "test-secret", { marker: "one" });
    const elapsedMs = performance.now() - startedAt;
    assert.equal(accepted.status, 202);
    assert.ok(elapsedMs < 300, `fsync acceptance took ${elapsedMs}ms`);
    assert.equal((await accepted.json()).refsCommitted, false);
    await waitFor(() => syncStarted);

    const duplicate = await postWebhook(port, repo, "durable-delivery", commit, "test-secret", { marker: "one" });
    assert.equal(duplicate.status, 202);
    assert.match((await duplicate.json()).disposition, /duplicate-durable-inbox-record/);

    const bodyConflict = await postWebhook(port, repo, "durable-delivery", commit, "test-secret", { marker: "two" });
    assert.equal(bodyConflict.status, 409);

    releaseSync();
    await waitFor(async () => (await controller.status()).deliveries.some((record) => record.deliveryId === "durable-delivery" && record.state === "committed"));
    const committed = (await controller.status()).deliveries.find((record) => record.deliveryId === "durable-delivery");
    assert.equal(committed.result.sourceCommit, commit);
    assert.equal(committed.result.authorityCommit, commit);
    assert.equal(committed.result.snapshotRef, `refs/snapshots/source/${commit}`);
  } finally {
    server.close();
    await controller.stop();
    rmSync(root, { recursive: true, force: true });
  }
});

test("startup rejects corrupt journal records and recovers a crash-left processing record", async () => {
  const corruptRoot = mkdtempSync(join(tmpdir(), "unidesk-gitea-inbox-corrupt-"));
  const corruptInbox = join(corruptRoot, "inbox");
  const journalCreatedAt = Date.parse("2026-07-11T07:00:00.000Z");
  const corruptStore = createDurableInboxStore({ path: corruptInbox, maxBytes: 1024 * 1024, now: () => journalCreatedAt });
  await corruptStore.initialize();
  assert.equal(corruptStore.journalMetadata().createdAt, new Date(journalCreatedAt).toISOString());
  const restartedStore = createDurableInboxStore({ path: corruptInbox, maxBytes: 1024 * 1024, now: () => journalCreatedAt + 60_000 });
  await restartedStore.initialize();
  assert.equal(restartedStore.journalMetadata().createdAt, new Date(journalCreatedAt).toISOString());
  writeFileSync(join(corruptInbox, `${"a".repeat(64)}.json`), JSON.stringify({ version: 1, deliveryId: "corrupt", fingerprint: "b".repeat(64), state: "accepted" }));
  const corruptController = createDurableSyncController(durableControllerOptions(corruptRoot, [fakeRepo("corrupt")]));
  try {
    await corruptController.start();
    assert.equal(corruptController.readiness().ready, false);
    assert.equal(corruptController.readiness().errorType, "durable-inbox-record-invalid");
  } finally {
    await corruptController.stop();
    rmSync(corruptRoot, { recursive: true, force: true });
  }

  const recoveryRoot = mkdtempSync(join(tmpdir(), "unidesk-gitea-inbox-recovery-"));
  const repo = fakeRepo("recovery");
  const store = createDurableInboxStore({ path: join(recoveryRoot, "inbox"), maxBytes: 1024 * 1024 });
  try {
    await store.initialize();
    const delivery = fakeDelivery("processing-before-crash", "b".repeat(40), repo);
    await store.accept(repo, delivery);
    await assert.rejects(
      () => store.update(delivery.deliveryId, (record) => ({ ...record, acceptedSequence: record.acceptedSequence + 1 })),
      /invalid-record-update/,
    );
    await store.update(delivery.deliveryId, (record) => ({
      ...record,
      state: "processing",
      processingAt: new Date().toISOString(),
      updatedAt: new Date().toISOString(),
      totalAttempts: 1,
      cycleAttempt: 1,
    }));
    const controller = createDurableSyncController(durableControllerOptions(recoveryRoot, [repo], {
      sync: async () => ({
        ok: true,
        sourceCommit: delivery.requestedCommit,
        authorityCommit: delivery.requestedCommit,
        snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`,
        disposition: "atomic-source-authority-committed",
      }),
    }));
    await controller.start();
    await waitFor(async () => (await controller.status()).deliveries.some((record) => record.deliveryId === delivery.deliveryId && record.state === "committed"));
    await controller.stop();
  } finally {
    rmSync(recoveryRoot, { recursive: true, force: true });
  }
});

test("unavailable or full durable inbox is not ready and fails closed with 503", async () => {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-inbox-unavailable-"));
  const blockingFile = join(root, "not-a-directory");
  writeFileSync(blockingFile, "blocked");
  const repo = fakeRepo("capacity");
  const unavailable = createDurableSyncController(durableControllerOptions(root, [repo], {
    inbox: { path: join(blockingFile, "inbox"), maxBytes: 1024 },
  }));
  await unavailable.start();
  assert.equal(unavailable.readiness().ready, false);
  await assert.rejects(() => unavailable.accept(repo, fakeDelivery("unavailable", "a".repeat(40), repo)), /durable-inbox/);
  await unavailable.stop();

  const full = createDurableSyncController(durableControllerOptions(root, [repo], {
    inbox: { path: join(root, "full-inbox"), maxBytes: 1 },
  }));
  try {
    await full.start();
    await assert.rejects(() => full.accept(repo, fakeDelivery("full", "b".repeat(40), repo)), /capacity-exhausted/);
    assert.equal(full.readiness().ready, false);
    assert.equal(full.readiness().errorType, "durable-inbox-capacity-exhausted");
  } finally {
    await full.stop();
    rmSync(root, { recursive: true, force: true });
  }
});

test("retention keeps each repository latest committed watermark and status keeps every repo latest record", async () => {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-inbox-retention-"));
  const store = createDurableInboxStore({ path: join(root, "inbox"), maxBytes: 4 * 1024 * 1024 });
  try {
    await store.initialize();
    const old = new Date(Date.now() - 10_000).toISOString();
    for (let index = 0; index < 105; index += 1) {
      const repo = fakeRepo(index < 2 ? "quiet" : `noise-${index}`);
      const delivery = { ...fakeDelivery(`retained-${index}`, index.toString(16).padStart(40, "0"), repo), receivedAt: old };
      await store.accept(repo, delivery);
      await store.update(delivery.deliveryId, (record) => ({
        ...record,
        state: "committed",
        updatedAt: old,
        committedAt: old,
        nextAttemptAt: null,
        result: {
          sourceCommit: delivery.requestedCommit,
          authorityCommit: delivery.requestedCommit,
          snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`,
          disposition: "atomic-source-authority-committed",
        },
      }));
    }
    const removed = await store.cleanupCommitted(Date.now());
    assert.equal(removed, 1);
    const retained = await store.list();
    assert.equal(retained.filter((record) => record.repoKey === "quiet").length, 1);
    assert.equal(retained.find((record) => record.repoKey === "quiet").deliveryId, "retained-1");

    const quietRepo = fakeRepo("quiet");
    const failedDelivery = fakeDelivery("quiet-failed", "f".repeat(40), quietRepo);
    await store.accept(quietRepo, failedDelivery);
    await store.update(failedDelivery.deliveryId, (record) => ({
      ...record,
      state: "failed",
      updatedAt: new Date().toISOString(),
      failedAt: new Date().toISOString(),
      nextAttemptAt: "2099-01-01T00:00:00.000Z",
      lastError: { errorType: "test-failure", retryable: true, terminal: true },
    }));

    const controller = createDurableSyncController(durableControllerOptions(root, [...retained.map((record) => fakeRepo(record.repoKey)), quietRepo], { store }));
    await controller.start();
    const status = await controller.status();
    assert.ok(status.deliveries.some((record) => record.repo === "quiet" && record.deliveryId === "retained-1"));
    assert.ok(status.deliveries.some((record) => record.repo === "quiet" && record.deliveryId === "quiet-failed"));
    await controller.stop();
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
});

test("same-millisecond records are listed and scheduled by immutable acceptedSequence", async () => {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-same-millisecond-order-"));
  const repo = fakeRepo("same-millisecond");
  const store = createDurableInboxStore({ path: join(root, "inbox"), maxBytes: 1024 * 1024 });
  const receivedAt = "2026-07-11T08:00:00.000Z";
  const first = { ...fakeDelivery("same-ms-first", "1".repeat(40), repo), receivedAt };
  const second = { ...fakeDelivery("same-ms-second", "2".repeat(40), repo), receivedAt };
  const executionOrder = [];
  let controller;
  try {
    await store.initialize();
    await store.accept(repo, first);
    await store.accept(repo, second);
    const listed = await store.list();
    assert.deepEqual(listed.map((record) => [record.deliveryId, record.acceptedSequence]), [
      [first.deliveryId, 1],
      [second.deliveryId, 2],
    ]);
    controller = createDurableSyncController(durableControllerOptions(root, [repo], {
      sync: async (_repo, delivery) => {
        executionOrder.push(delivery.deliveryId);
        return {
          ok: true,
          sourceCommit: delivery.requestedCommit,
          authorityCommit: delivery.requestedCommit,
          snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`,
          disposition: "atomic-source-authority-committed",
        };
      },
    }));
    await controller.start();
    await waitFor(async () => (await controller.status()).counts.committed === 2);
    assert.deepEqual(executionOrder, [first.deliveryId, second.deliveryId]);
  } finally {
    if (controller) await controller.stop();
    rmSync(root, { recursive: true, force: true });
  }
});

test("concurrent duplicate delivery shares one promise and payload conflict fails closed", async () => {
  let release;
  const gate = new Promise((resolve) => { release = resolve; });
  let calls = 0;
  const coordinator = createSyncCoordinator({
    responseBudgetMs: 1000,
    retryMaxAttempts: 2,
    retryInitialDelayMs: 1,
    retryMaxDelayMs: 1,
    sleep: zeroDelay,
    log: () => {},
    sync: async (_repo, delivery) => {
      calls += 1;
      await gate;
      return { ok: true, sourceCommit: delivery.requestedCommit, authorityCommit: delivery.requestedCommit, snapshotRef: `refs/snapshots/${delivery.requestedCommit}`, disposition: "atomic-source-authority-committed" };
    },
  });
  const repo = fakeRepo("one");
  const delivery = fakeDelivery("delivery-1", "a".repeat(40), repo);
  const first = coordinator.execute(repo, delivery);
  const duplicate = coordinator.execute(repo, { ...delivery });
  assert.throws(
    () => coordinator.execute(repo, { ...delivery, requestedCommit: "b".repeat(40) }),
    /github-delivery-id-payload-conflict/,
  );
  release();
  const [firstResult, duplicateResult] = await Promise.all([first, duplicate]);
  assert.equal(calls, 1);
  assert.equal(firstResult.ok, true);
  assert.equal(duplicateResult.duplicate, true);
});

test("different repositories are isolated while each repository stays serialized", async () => {
  let releaseOne;
  const gateOne = new Promise((resolve) => { releaseOne = resolve; });
  const order = [];
  const coordinator = createSyncCoordinator({
    responseBudgetMs: 1000,
    retryMaxAttempts: 1,
    retryInitialDelayMs: 1,
    retryMaxDelayMs: 1,
    sleep: zeroDelay,
    log: () => {},
    sync: async (repo, delivery) => {
      order.push(`start:${repo.key}:${delivery.deliveryId}`);
      if (repo.key === "one") await gateOne;
      order.push(`end:${repo.key}:${delivery.deliveryId}`);
      return { ok: true, sourceCommit: delivery.requestedCommit, authorityCommit: delivery.requestedCommit, snapshotRef: `refs/snapshots/${delivery.requestedCommit}`, disposition: "atomic-source-authority-committed" };
    },
  });
  const repoOne = fakeRepo("one");
  const repoTwo = fakeRepo("two");
  const oneA = coordinator.execute(repoOne, fakeDelivery("one-a", "a".repeat(40), repoOne));
  const oneB = coordinator.execute(repoOne, fakeDelivery("one-b", "b".repeat(40), repoOne));
  const two = coordinator.execute(repoTwo, fakeDelivery("two-a", "c".repeat(40), repoTwo));
  await two;
  assert.deepEqual(order.slice(0, 3), ["start:one:one-a", "start:two:two-a", "end:two:two-a"]);
  releaseOne();
  await Promise.all([oneA, oneB]);
  assert.ok(order.indexOf("start:one:one-b") > order.indexOf("end:one:one-a"));
});

test("retry exhaustion is terminal and observable", async () => {
  const logs = [];
  let calls = 0;
  const coordinator = createSyncCoordinator({
    responseBudgetMs: 1000,
    retryMaxAttempts: 3,
    retryInitialDelayMs: 1,
    retryMaxDelayMs: 1,
    sleep: zeroDelay,
    log: (event) => logs.push(event),
    sync: async () => {
      calls += 1;
      return { ok: false, retryable: true, errorType: "git-source-sync-failed" };
    },
  });
  const repo = fakeRepo("one");
  const result = await coordinator.execute(repo, fakeDelivery("delivery-fail", "a".repeat(40), repo));
  assert.equal(result.ok, false);
  assert.equal(calls, 3);
  assert.deepEqual(logs.filter((event) => event.event === "github-to-gitea-sync").map((event) => [event.syncAttempt, event.willRetry, event.terminal]), [
    [1, true, false],
    [2, true, false],
    [3, false, true],
  ]);
});

test("durable worker automatically starts a new cycle after terminal retry exhaustion", async () => {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-terminal-retry-"));
  const repo = fakeRepo("terminal-retry");
  const logs = [];
  let calls = 0;
  const controller = createDurableSyncController(durableControllerOptions(root, [repo], {
    log: (event) => logs.push(event),
    workerRetry: {
      maxAttempts: 2,
      initialDelayMs: 1,
      maxDelayMs: 1,
      terminalRetryDelayMs: 5,
      scanIntervalMs: 1,
    },
    sync: async (_repo, delivery) => {
      calls += 1;
      if (calls < 3) return { ok: false, retryable: true, errorType: "transient-source-sync-failed" };
      return {
        ok: true,
        sourceCommit: delivery.requestedCommit,
        authorityCommit: delivery.requestedCommit,
        snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`,
        disposition: "atomic-source-authority-committed",
      };
    },
  }));
  try {
    await controller.start();
    const delivery = fakeDelivery("terminal-retry-delivery", "a".repeat(40), repo);
    await controller.accept(repo, delivery);
    await waitFor(async () => (await controller.status()).deliveries.some((record) => record.deliveryId === delivery.deliveryId && record.state === "committed"));
    const committed = (await controller.status()).deliveries.find((record) => record.deliveryId === delivery.deliveryId);
    assert.equal(calls, 3);
    assert.equal(committed.totalAttempts, 3);
    assert.equal(logs.some((event) => event.event === "github-to-gitea-delivery-failed" && event.terminal === true && event.automaticRetryScheduled === true), true);
  } finally {
    await controller.stop();
    rmSync(root, { recursive: true, force: true });
  }
});

test("unexpected executor rejection is durably failed and automatically retries without restart", async () => {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-executor-rejection-"));
  const repo = fakeRepo("executor-rejection");
  const logs = [];
  let calls = 0;
  const controller = createDurableSyncController(durableControllerOptions(root, [repo], {
    log: (event) => logs.push(event),
    workerRetry: { initialDelayMs: 1, maxDelayMs: 1, scanIntervalMs: 1 },
    executor: {
      async execute(_repo, delivery) {
        calls += 1;
        if (calls === 1) throw new Error("unexpected executor rejection");
        return {
          ok: true,
          sourceCommit: delivery.requestedCommit,
          authorityCommit: delivery.requestedCommit,
          snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`,
          disposition: "atomic-source-authority-committed",
        };
      },
    },
  }));
  try {
    await controller.start();
    const delivery = fakeDelivery("executor-rejection-delivery", "8".repeat(40), repo);
    await controller.accept(repo, delivery);
    await waitFor(async () => (await controller.status()).counts.committed === 1);
    const committed = (await controller.status()).deliveries[0];
    assert.equal(calls, 2);
    assert.equal(committed.state, "committed");
    assert.equal(committed.totalAttempts, 2);
    assert.equal(logs.some((event) => event.event === "github-to-gitea-delivery-failed" && event.errorType === "source-sync-executor-threw" && event.automaticRetryScheduled === true), true);
  } finally {
    await controller.stop();
    rmSync(root, { recursive: true, force: true });
  }
});

test("undefined and incomplete successful executor results are durably failed before retry", async () => {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-malformed-executor-result-"));
  const repo = fakeRepo("malformed-executor-result");
  const logs = [];
  let calls = 0;
  const controller = createDurableSyncController(durableControllerOptions(root, [repo], {
    log: (event) => logs.push(event),
    workerRetry: { maxAttempts: 3, initialDelayMs: 1, maxDelayMs: 1, scanIntervalMs: 1 },
    executor: {
      async execute(_repo, delivery) {
        calls += 1;
        if (calls === 1) return undefined;
        if (calls === 2) return { ok: true };
        return {
          ok: true,
          sourceCommit: delivery.requestedCommit,
          authorityCommit: delivery.requestedCommit,
          snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`,
          disposition: "atomic-source-authority-committed",
        };
      },
    },
  }));
  try {
    await controller.start();
    const delivery = fakeDelivery("malformed-executor-result-delivery", "9".repeat(40), repo);
    await controller.accept(repo, delivery);
    await waitFor(async () => (await controller.status()).counts.committed === 1);
    const committed = (await controller.status()).deliveries[0];
    assert.equal(calls, 3);
    assert.equal(committed.totalAttempts, 3);
    assert.equal(logs.filter((event) => event.event === "github-to-gitea-delivery-failed" && event.errorType === "source-sync-executor-invalid-result").length, 2);
  } finally {
    await controller.stop();
    rmSync(root, { recursive: true, force: true });
  }
});

test("slow sync cannot return success after the absolute response deadline", async () => {
  const coordinator = createSyncCoordinator({
    responseBudgetMs: 20,
    retryMaxAttempts: 1,
    retryInitialDelayMs: 1,
    retryMaxDelayMs: 1,
    sleep: zeroDelay,
    log: () => {},
    sync: async (_repo, delivery) => {
      await new Promise((resolve) => setTimeout(resolve, 35));
      return { ok: true, sourceCommit: delivery.requestedCommit, authorityCommit: delivery.requestedCommit, snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`, disposition: "atomic-source-authority-committed" };
    },
  });
  const repo = fakeRepo("deadline");
  const result = await coordinator.execute(repo, fakeDelivery("slow", "e".repeat(40), repo));
  assert.equal(result.ok, false);
  assert.equal(result.retryable, true);
  assert.equal(result.errorType, "response-deadline-exhausted");
  assert.equal(result.committedBeforeDeadlineUnknown, true);
});

test("git subprocesses fail closed before launch when the shared deadline is exhausted", async () => {
  let commands = 0;
  const repo = {
    ...fakeRepo("expired"),
    upstream: { ...fakeRepo("expired").upstream, cloneUrl: "/does/not/matter" },
    gitea: { readUrl: "file:///does/not/matter" },
  };
  const result = await syncRepository(repo, fakeDelivery("expired", "f".repeat(40), repo), {
    deadlineAt: 10,
    now: () => 10,
    runResult: () => {
      commands += 1;
      return { status: 0, stdout: "", stderr: "" };
    },
    githubAuthHeader: "",
    giteaAuthHeader: "",
    giteaBaseUrl: "file://",
  });
  assert.equal(result.ok, false);
  assert.equal(result.retryable, true);
  assert.equal(commands, 0);
  assert.match(result.error, /response deadline exhausted/);
});

test("async process runner handles spawn errors and kills the whole subprocess group at deadline", async () => {
  const missing = await runResult(["unidesk-command-that-does-not-exist"], 100);
  assert.equal(missing.status, null);
  assert.match(missing.stderr, /ENOENT|Executable not found/u);

  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-process-tree-"));
  const marker = join(root, "child-survived");
  try {
    const timedOut = await runResult(["sh", "-c", '(sleep 0.25; : > "$1") & wait', "sh", marker], 40);
    assert.equal(timedOut.status, null);
    assert.match(timedOut.stderr, /deadline exceeded/);
    await new Promise((resolve) => setTimeout(resolve, 350));
    assert.equal(existsSync(marker), false);
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
});

test("controller shutdown aborts the active process tree and the durable record resumes after restart", async () => {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-controller-shutdown-"));
  const startedMarker = join(root, "worker-started");
  const survivorMarker = join(root, "child-survived");
  const repo = fakeRepo("shutdown");
  const delivery = fakeDelivery("shutdown-delivery", "c".repeat(40), repo);
  const controller = createDurableSyncController(durableControllerOptions(root, [repo], {
    shutdownGraceMs: 500,
    sync: async (_repo, _delivery, context) => {
      const result = await runResult([
        "sh", "-c", ': > "$1"; (sleep 0.25; : > "$2") & wait', "sh", startedMarker, survivorMarker,
      ], 10_000, context.signal);
      return { ok: false, retryable: true, errorType: result.stderr.includes("aborted") ? "worker-aborted" : "worker-exited" };
    },
  }));
  try {
    await controller.start();
    await controller.accept(repo, delivery);
    await waitFor(() => existsSync(startedMarker));
    await controller.stop();
    await new Promise((resolve) => setTimeout(resolve, 320));
    assert.equal(existsSync(survivorMarker), false);

    const resumed = createDurableSyncController(durableControllerOptions(root, [repo], {
      sync: async () => ({
        ok: true,
        sourceCommit: delivery.requestedCommit,
        authorityCommit: delivery.requestedCommit,
        snapshotRef: `refs/snapshots/source/${delivery.requestedCommit}`,
        disposition: "atomic-source-authority-committed",
      }),
    }));
    await resumed.start();
    await waitFor(async () => (await resumed.status()).deliveries.some((record) => record.deliveryId === delivery.deliveryId && record.state === "committed"));
    await resumed.stop();
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
});

test("atomic mirror handles partial refs, immutable snapshots, out-of-order delivery and lease race", async () => {
  const fixture = gitFixture();
  try {
    const { repo, runtime, commits, gitea } = fixture;
    const first = await syncRepository(repo, fakeDelivery("a", commits.a, repo), runtime);
    assert.equal(first.ok, true);
    assert.equal(remoteRef(gitea, "refs/heads/main"), commits.a);
    assert.equal(remoteRef(gitea, `refs/snapshots/source/${commits.a}`), commits.a);

    git("--git-dir", gitea, "update-ref", `refs/snapshots/source/${commits.a}`, "-d");
    const partial = await syncRepository(repo, fakeDelivery("a-partial", commits.a, repo), runtime);
    assert.equal(partial.ok, true);
    assert.equal(remoteRef(gitea, `refs/snapshots/source/${commits.a}`), commits.a);

    const forward = await syncRepository(repo, fakeDelivery("b", commits.b, repo), runtime);
    assert.equal(forward.ok, true);
    assert.equal(remoteRef(gitea, "refs/heads/main"), commits.b);
    assert.equal(remoteRef(gitea, `refs/snapshots/source/${commits.b}`), commits.b);

    const old = await syncRepository(repo, fakeDelivery("a-late", commits.a, repo), runtime);
    assert.equal(old.ok, true);
    assert.equal(old.disposition, "superseded-with-immutable-snapshot");
    assert.equal(remoteRef(gitea, "refs/heads/main"), commits.b);

    const sameSnapshot = await syncRepository(repo, fakeDelivery("b-again", commits.b, repo), runtime);
    assert.equal(sameSnapshot.ok, true);
    assert.equal(remoteRef(gitea, "refs/heads/main"), commits.b);

    git("--git-dir", gitea, "update-ref", `refs/snapshots/source/${commits.c}`, commits.a);
    const conflict = await syncRepository(repo, fakeDelivery("c-conflict", commits.c, repo), runtime);
    assert.equal(conflict.ok, false);
    assert.equal(conflict.errorType, "immutable-snapshot-conflict");
    assert.equal(remoteRef(gitea, "refs/heads/main"), commits.b);
    git("--git-dir", gitea, "update-ref", `refs/snapshots/source/${commits.c}`, "-d");
    git("--git-dir", gitea, "fetch", "-q", fixture.source, `${commits.d}:refs/hidden/race-d`);

    let raced = false;
    const racingRuntime = {
      ...runtime,
      async runResult(args) {
        if (!raced && args.includes("push") && args.includes("--atomic")) {
          raced = true;
          git("--git-dir", gitea, "update-ref", "refs/heads/main", commits.d);
        }
        return await runResult(args);
      },
    };
    const leaseRace = await syncRepository(repo, fakeDelivery("c-race", commits.c, repo), racingRuntime);
    assert.equal(leaseRace.ok, false);
    assert.equal(leaseRace.errorType, "atomic-push-race-or-failure");
    assert.equal(remoteRef(gitea, "refs/heads/main"), commits.d);
    assert.equal(remoteRef(gitea, `refs/snapshots/source/${commits.c}`), null);

    const retry = await syncRepository(repo, fakeDelivery("c-retry", commits.c, repo), runtime);
    assert.equal(retry.ok, true);
    assert.equal(retry.disposition, "superseded-with-immutable-snapshot");
    assert.equal(remoteRef(gitea, "refs/heads/main"), commits.d);
    assert.equal(remoteRef(gitea, `refs/snapshots/source/${commits.c}`), commits.c);

    const unavailable = await syncRepository(repo, fakeDelivery("force-push-unavailable", `${"0".repeat(39)}1`, repo), runtime);
    assert.equal(unavailable.ok, false);
    assert.equal(unavailable.retryable, true);
    assert.equal(unavailable.errorType, "requested-commit-temporarily-unavailable");
  } finally {
    fixture.cleanup();
  }
});

test("production async Git runner preserves cross-repository concurrency", async () => {
  const slow = gitFixture("slow");
  const fast = gitFixture("fast");
  const marker = join(slow.root, "pre-receive-entered");
  writeFileSync(join(slow.gitea, "hooks", "pre-receive"), `#!/bin/sh\n: > "${marker}"\nsleep 1\nexit 0\n`, { mode: 0o755 });
  const coordinator = createSyncCoordinator({
    responseBudgetMs: 5000,
    retryMaxAttempts: 1,
    retryInitialDelayMs: 1,
    retryMaxDelayMs: 1,
    log: () => {},
    run,
    runResult,
    githubAuthHeader: "",
    giteaAuthHeader: "",
    giteaBaseUrl: "file://",
  });
  try {
    const slowResult = coordinator.execute(slow.repo, fakeDelivery("slow-a", slow.commits.a, slow.repo));
    await waitFor(() => existsSync(marker));
    const fastResult = coordinator.execute(fast.repo, fakeDelivery("fast-a", fast.commits.a, fast.repo));
    const first = await Promise.race([
      slowResult.then(() => "slow"),
      fastResult.then(() => "fast"),
    ]);
    assert.equal(first, "fast");
    assert.equal((await fastResult).ok, true);
    assert.equal((await slowResult).ok, true);
  } finally {
    slow.cleanup();
    fast.cleanup();
  }
});

test("atomic push rejection leaves branch and snapshot unchanged", async () => {
  const fixture = gitFixture();
  try {
    const { repo, runtime, commits, gitea } = fixture;
    assert.equal((await syncRepository(repo, fakeDelivery("a", commits.a, repo), runtime)).ok, true);
    const hook = join(gitea, "hooks", "pre-receive");
    writeFileSync(hook, "#!/bin/sh\nexit 1\n", { mode: 0o755 });
    const rejected = await syncRepository(repo, fakeDelivery("b", commits.b, repo), runtime);
    assert.equal(rejected.ok, false);
    assert.equal(rejected.errorType, "atomic-push-race-or-failure");
    assert.equal(remoteRef(gitea, "refs/heads/main"), commits.a);
    assert.equal(remoteRef(gitea, `refs/snapshots/source/${commits.b}`), null);
  } finally {
    fixture.cleanup();
  }
});

function gitFixture(key = "fixture") {
  const root = mkdtempSync(join(tmpdir(), "unidesk-gitea-sync-test-"));
  const work = join(root, "work");
  const source = join(root, "source.git");
  const gitea = join(root, "gitea.git");
  git("init", "-q", "-b", "main", work);
  git("-C", work, "config", "user.name", "UniDesk Test");
  git("-C", work, "config", "user.email", "test@unidesk.local");
  const commits = {};
  for (const name of ["a", "b", "c", "d"]) {
    writeFileSync(join(work, "source.txt"), `${name}\n`);
    git("-C", work, "add", "source.txt");
    git("-C", work, "commit", "-q", "-m", name);
    commits[name] = git("-C", work, "rev-parse", "HEAD").trim();
  }
  git("clone", "-q", "--bare", work, source);
  git("init", "-q", "--bare", gitea);
  const repo = {
    key,
    upstream: { repository: `pikasTech/${key}`, branch: "main", cloneUrl: source },
    gitea: { readUrl: `file://${gitea}` },
    snapshot: { prefix: "refs/snapshots/source" },
  };
  return {
    root,
    work,
    source,
    gitea,
    commits,
    repo,
    runtime: {
      run,
      runResult,
      githubAuthHeader: "",
      giteaAuthHeader: "",
      giteaBaseUrl: "file://",
      deadlineAt: Date.now() + 60000,
      now: Date.now,
    },
    cleanup: () => rmSync(root, { recursive: true, force: true }),
  };
}

function fakeRepo(key) {
  return {
    key,
    upstream: { repository: `pikasTech/${key}`, branch: "main" },
    snapshot: { prefix: "refs/snapshots/source" },
  };
}

function fakeDelivery(deliveryId, requestedCommit, repo) {
  return {
    deliveryId,
    repository: repo.upstream.repository,
    ref: `refs/heads/${repo.upstream.branch}`,
    requestedCommit,
    payloadSha: createHash("sha256").update(`${deliveryId}:${requestedCommit}`).digest("hex"),
    receivedAt: new Date().toISOString(),
  };
}

function remoteRef(repo, ref) {
  try {
    return git("--git-dir", repo, "rev-parse", "--verify", ref).trim();
  } catch {
    return null;
  }
}

async function postWebhook(port, repo, deliveryId, requestedCommit, secret, extra = {}) {
  const body = JSON.stringify({
    ref: `refs/heads/${repo.upstream.branch}`,
    after: requestedCommit,
    repository: { full_name: repo.upstream.repository },
    ...extra,
  });
  const signature = `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`;
  return await fetch(`http://127.0.0.1:${port}/hook`, {
    method: "POST",
    headers: {
      "content-type": "application/json",
      "x-github-event": "push",
      "x-github-delivery": deliveryId,
      "x-hub-signature-256": signature,
    },
    body,
  });
}

function durableControllerOptions(root, repos, overrides = {}) {
  return {
    repos,
    inbox: {
      path: join(root, "inbox"),
      maxBytes: 4 * 1024 * 1024,
      committedRetentionMs: 60_000,
      cleanupIntervalMs: 60_000,
      ...overrides.inbox,
    },
    workerRetry: {
      maxAttempts: 2,
      initialDelayMs: 10,
      maxDelayMs: 20,
      terminalRetryDelayMs: 50,
      attemptTimeoutMs: 2000,
      scanIntervalMs: 10,
      ...overrides.workerRetry,
    },
    shutdownGraceMs: 50,
    githubAuthHeader: "",
    giteaAuthHeader: "",
    giteaBaseUrl: "file://",
    run,
    runResult,
    log: () => {},
    ...overrides,
    inbox: {
      path: join(root, "inbox"),
      maxBytes: 4 * 1024 * 1024,
      committedRetentionMs: 60_000,
      cleanupIntervalMs: 60_000,
      ...overrides.inbox,
    },
    workerRetry: {
      maxAttempts: 2,
      initialDelayMs: 10,
      maxDelayMs: 20,
      terminalRetryDelayMs: 50,
      attemptTimeoutMs: 2000,
      scanIntervalMs: 10,
      ...overrides.workerRetry,
    },
  };
}

async function waitFor(predicate, timeoutMs = 3000) {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    if (await predicate()) return;
    await new Promise((resolve) => setTimeout(resolve, 10));
  }
  throw new Error("condition not reached before timeout");
}

function git(...args) {
  return execFileSync("git", args, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] });
}