fix: wait for stale runner lease before replacement claim

src/runner/run-once.ts

@@ -19,6 +19,8 @@ export interface RunnerOnceOptions extends BackendAdapterOptions {
   logPath?: string;
   idleTimeoutMs?: number;
   pollIntervalMs?: number;
+  claimRetryTimeoutMs?: number;
+  claimRetryIntervalMs?: number;
   oneShot?: boolean;
 }
 
@@ -50,7 +52,7 @@ export async function runOnce(options: RunnerOnceOptions): Promise<JsonRecord> {
   }) as RunnerRecord;
   let claimed: RunRecord;
   try {
-    claimed = await api.claim(options.runId, runner.id, leaseMs);
+    claimed = await claimRunWithLeaseRecovery(api, options, runner, attemptId, leaseMs);
     await api.heartbeat(options.runId, runner.id, leaseMs);
   } catch (error) {
     const failureKind = failureKindFromError(error);
@@ -355,6 +357,77 @@ async function appendBestEffort(api: RunnerManagerApi, runId: string, event: Bac
   }
 }
 
+async function claimRunWithLeaseRecovery(api: RunnerManagerApi, options: RunnerOnceOptions, runner: RunnerRecord, attemptId: string, leaseMs: number): Promise<RunRecord> {
+  const startedAt = Date.now();
+  const timeoutMs = normalizeClaimRetryTimeoutMs(options.claimRetryTimeoutMs, leaseMs);
+  const intervalMs = normalizeClaimRetryIntervalMs(options.claimRetryIntervalMs);
+  let waitingEventWritten = false;
+  let lastError: unknown = null;
+
+  while (Date.now() - startedAt <= timeoutMs) {
+    try {
+      const claimed = await api.claim(options.runId, runner.id, leaseMs);
+      if (waitingEventWritten) {
+        await appendBestEffort(api, options.runId, {
+          type: "backend_status",
+          payload: { phase: "runner-claim-recovered", attemptId, runnerId: runner.id, waitedMs: Date.now() - startedAt },
+        });
+      }
+      return claimed;
+    } catch (error) {
+      if (failureKindFromError(error) !== "runner-lease-conflict") throw error;
+      lastError = error;
+      const run = await getRunBestEffort(api, options.runId);
+      if (!waitingEventWritten) {
+        waitingEventWritten = true;
+        await appendBestEffort(api, options.runId, {
+          type: "backend_status",
+          payload: {
+            phase: "runner-claim-waiting-for-stale-lease",
+            attemptId,
+            runnerId: runner.id,
+            claimedBy: run?.claimedBy ?? null,
+            leaseExpiresAt: run?.leaseExpiresAt ?? null,
+            retryTimeoutMs: timeoutMs,
+          },
+        });
+      }
+      const remainingMs = timeoutMs - (Date.now() - startedAt);
+      if (remainingMs <= 0) break;
+      await sleep(Math.min(remainingMs, claimRetryDelayMs(run, intervalMs)));
+    }
+  }
+
+  throw lastError ?? new AgentRunError("runner-lease-conflict", `run ${options.runId} could not be claimed before retry timeout`, { httpStatus: 409 });
+}
+
+async function getRunBestEffort(api: RunnerManagerApi, runId: string): Promise<RunRecord | null> {
+  try {
+    return await api.getRun(runId);
+  } catch {
+    return null;
+  }
+}
+
+function claimRetryDelayMs(run: RunRecord | null, intervalMs: number): number {
+  const leaseExpiresAt = run?.leaseExpiresAt ? Date.parse(run.leaseExpiresAt) : NaN;
+  if (Number.isFinite(leaseExpiresAt)) {
+    const untilExpiryMs = leaseExpiresAt - Date.now() + 100;
+    if (untilExpiryMs > 0) return Math.max(25, Math.min(intervalMs, untilExpiryMs));
+  }
+  return intervalMs;
+}
+
+function normalizeClaimRetryTimeoutMs(value: number | undefined, leaseMs: number): number {
+  if (Number.isFinite(value ?? NaN)) return Math.max(0, Math.floor(value!));
+  return Math.max(leaseMs + Math.max(5_000, Math.floor(leaseMs / 3)), leaseMs);
+}
+
+function normalizeClaimRetryIntervalMs(value: number | undefined): number {
+  if (!Number.isFinite(value ?? NaN)) return 5_000;
+  return Math.max(25, Math.min(30_000, Math.floor(value!)));
+}
+
 function annotateCommandEvent(event: BackendEvent, commandId: string, attemptId: string, runnerId: string): BackendEvent {
   return { ...event, payload: { ...event.payload, commandId, attemptId, runnerId } };
 }