fix: wait for stale runner lease before replacement claim

src/selftest/cases/30-codex-stdio.ts

@@ -27,6 +27,8 @@ const selfTest: SelfTestCase = async (context) => {
     const finalCommand = await client.get(`/api/v1/runs/${happy.runId}/commands/${happy.commandId}`) as { state?: string };
     assert.equal(finalCommand.state, "completed");
 
+    await runLeaseConflictRecoveryCase({ client, managerUrl: server.baseUrl, context });
+
     const projectedHome = path.join(context.tmp, "runtime-codex-home");
     const projected = await createRunWithCommand(client, { workspace: context.workspace, codexHome: projectedHome }, "hello projected", "selftest-projected-codex-home", 15_000);
     const projectedResult = await runOnce({ managerUrl: server.baseUrl, runId: projected.runId, codexCommand: context.fakeCodexCommand, codexArgs: context.fakeCodexArgs, codexHome: projectedHome, env: { CODEX_HOME: projectedHome, AGENTRUN_CODEX_SECRET_HOME: context.codexHome }, oneShot: true });
@@ -215,12 +217,47 @@ const selfTest: SelfTestCase = async (context) => {
     await runSessionStorageSubdirCase({ client, managerUrl: server.baseUrl, context });
     await runSessionStorageNoSecretLeakCase({ client, managerUrl: server.baseUrl, context });
 
-    return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "codex-stdio-fake-turn", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-minimax-m3-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-minimax-m3-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-final-agent-message-only", "codex-stdio-web-search-progress", "codex-stdio-stale-thread-resume-failed", "codex-stdio-live-tool-events", "codex-stdio-noisy-reasoning-suppression", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-invalid-tool-call", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-503-retry-event", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-idle-timeout-progress-refresh", "codex-stdio-command-failure-keeps-run-open", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
+    return { name: "codex-stdio", tests: ["runner-lease-heartbeat", "runner-lease-conflict-recovery", "codex-stdio-fake-turn", "codex-stdio-projected-writable-home", "codex-stdio-deepseek-profile-fake-turn", "codex-stdio-minimax-m3-profile-fake-turn", "codex-stdio-deepseek-missing-secret-no-fallback", "codex-stdio-minimax-m3-missing-secret-no-fallback", "codex-stdio-config-model-authoritative", "codex-stdio-explicit-model-forwarded", "codex-stdio-final-agent-message-only", "codex-stdio-web-search-progress", "codex-stdio-stale-thread-resume-failed", "codex-stdio-live-tool-events", "codex-stdio-noisy-reasoning-suppression", "codex-stdio-missing-turn-result", "codex-stdio-provider-auth-failed", "codex-stdio-provider-rate-limited", "codex-stdio-provider-invalid-tool-call", "codex-stdio-provider-503-rpc-error", "codex-stdio-provider-503-terminal", "codex-stdio-provider-503-retry-event", "codex-stdio-invalid-json", "codex-stdio-timeout", "codex-stdio-idle-timeout-progress-refresh", "codex-stdio-command-failure-keeps-run-open", "codex-stdio-secret-unavailable", "codex-stdio-spawn-failure"] };
   } finally {
     await new Promise<void>((resolve) => server.server.close(() => resolve()));
   }
 };
 
+async function runLeaseConflictRecoveryCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
+  const item = await createRunWithCommand(options.client, options.context, "claim after stale lease", "selftest-runner-lease-conflict-recovery", 15_000);
+  const staleRunner = await options.client.post("/api/v1/runners/register", {
+    runId: item.runId,
+    attemptId: "attempt_stale_claim",
+    backendProfile: "codex",
+    placement: "kubernetes-job",
+    sourceCommit: "self-test",
+    id: "runner_stale_claim",
+  }) as JsonRecord;
+  await options.client.post(`/api/v1/runs/${item.runId}/claim`, { runnerId: staleRunner.id, leaseMs: 300 });
+  const result = await runOnce({
+    managerUrl: options.managerUrl,
+    runId: item.runId,
+    commandId: item.commandId,
+    runnerId: "runner_recovered_claim",
+    attemptId: "attempt_recovered_claim",
+    leaseMs: 1_000,
+    claimRetryTimeoutMs: 2_000,
+    claimRetryIntervalMs: 50,
+    codexCommand: options.context.fakeCodexCommand,
+    codexArgs: options.context.fakeCodexArgs,
+    codexHome: options.context.codexHome,
+    env: { CODEX_HOME: options.context.codexHome },
+    oneShot: true,
+  }) as JsonRecord;
+  assert.equal(result.terminalStatus, "completed", "replacement runner should claim after stale lease expiry and finish the pending command");
+  const run = await options.client.get(`/api/v1/runs/${item.runId}`) as JsonRecord;
+  assert.equal(run.claimedBy, "runner_recovered_claim");
+  const events = await options.client.get(`/api/v1/runs/${item.runId}/events?afterSeq=0&limit=100`) as { items?: Array<{ type: string; payload: unknown }> };
+  assert.equal(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "runner-claim-waiting-for-stale-lease"), true, "lease conflict recovery should be visible while waiting");
+  assert.equal(events.items?.some((event) => event.type === "backend_status" && eventPayload(event).phase === "runner-claim-recovered"), true, "lease conflict recovery should be visible after claim succeeds");
+  assertNoSecretLeak({ result, run, events });
+}
+
 async function runSlowProgressIdleCase(options: { client: ManagerClient; managerUrl: string; context: SelfTestContext }): Promise<void> {
   const item = await createRunWithCommand(options.client, options.context, "slow progress before terminal", "selftest-slow-progress-idle-refresh", 60);
   const result = await runOnce({