fix(code-queue): forward-port stale-active recovery

Supersedes stale PR #79 with a current-master forward-port of safe stale-active recovery diagnostics and explicit recovery controls.

src/components/microservices/code-queue/src/execution-diagnostics.ts

@@ -4,6 +4,7 @@ import type { ActiveRun } from "./code-agent/common";
 import type { CodeQueueExecutionDiagnostics, JsonValue, QueueTask, SchedulerActiveRunHeartbeat } from "./types";
 
 export const schedulerHeartbeatStaleMs = 5 * 60 * 1000;
+export const schedulerHeartbeatStaleMsMax = 24 * 60 * 60_000;
 
 interface ExecutionDiagnosticsInput {
   now?: string;
@@ -59,6 +60,19 @@ function heartbeatFresh(heartbeat: SchedulerActiveRunHeartbeat | null, nowMs: nu
   return atMs !== null && nowMs - atMs < staleMs;
 }
 
+export function normalizeSchedulerHeartbeatStaleMs(
+  value: number | null | undefined,
+  options: { min?: number; max?: number } = {},
+): number {
+  const minimum = Math.max(1, Math.floor(options.min ?? schedulerHeartbeatStaleMs));
+  const maximum = Math.max(minimum, Math.floor(options.max ?? Number.MAX_SAFE_INTEGER));
+  const parsed = value ?? schedulerHeartbeatStaleMs;
+  if (!Number.isFinite(parsed)) return minimum;
+  const floored = Math.floor(parsed);
+  if (floored < minimum) return minimum;
+  return Math.min(maximum, floored);
+}
+
 function outputMaxSeq(task: QueueTask): number {
   const values = [
     Number(task.outputMaxSeq ?? 0),
@@ -77,7 +91,8 @@ export function staleRecoveryCandidate(input: StaleRecoveryInput): { allowed: bo
   const heartbeatMs = timestampMs(heartbeat.lastLocalHeartbeatAt);
   if (heartbeatMs === null) return { allowed: false, reason: "owner-heartbeat-missing", heartbeatFresh: false, heartbeatAgeMs: null };
   const heartbeatAgeMs = Math.max(0, nowMs - heartbeatMs);
-  const fresh = heartbeatAgeMs < (input.heartbeatStaleMs ?? schedulerHeartbeatStaleMs);
+  const staleMs = normalizeSchedulerHeartbeatStaleMs(input.heartbeatStaleMs);
+  const fresh = heartbeatAgeMs < staleMs;
   if (fresh) return { allowed: false, reason: "owner-heartbeat-fresh", heartbeatFresh: true, heartbeatAgeMs };
   return { allowed: true, reason: "owner-heartbeat-expired", heartbeatFresh: false, heartbeatAgeMs };
 }
@@ -85,10 +100,11 @@ export function staleRecoveryCandidate(input: StaleRecoveryInput): { allowed: bo
 export function taskHasTraceGapButFreshHeartbeat(task: QueueTask, now: string, heartbeatStaleMs = schedulerHeartbeatStaleMs): boolean {
   if (!activeTask(task)) return false;
   const heartbeat = taskHeartbeat(task);
-  if (!heartbeatFresh(heartbeat, timestampMs(now) ?? Date.now(), heartbeatStaleMs)) return false;
+  const staleMs = normalizeSchedulerHeartbeatStaleMs(heartbeatStaleMs);
+  if (!heartbeatFresh(heartbeat, timestampMs(now) ?? Date.now(), staleMs)) return false;
   const traceAt = timestampMs(heartbeat?.lastPersistedTraceAt);
   if (traceAt === null) return false;
-  return (timestampMs(now) ?? Date.now()) - traceAt >= heartbeatStaleMs;
+  return (timestampMs(now) ?? Date.now()) - traceAt >= staleMs;
 }
 
 export function buildSchedulerHeartbeat(task: QueueTask, run: ActiveRun | null, options: {
@@ -122,7 +138,7 @@ export function buildSchedulerHeartbeat(task: QueueTask, run: ActiveRun | null,
 export function buildExecutionDiagnostics(input: ExecutionDiagnosticsInput): CodeQueueExecutionDiagnostics {
   const now = input.now ?? new Date().toISOString();
   const nowMs = timestampMs(now) ?? Date.now();
-  const staleMs = input.heartbeatStaleMs ?? schedulerHeartbeatStaleMs;
+  const staleMs = normalizeSchedulerHeartbeatStaleMs(input.heartbeatStaleMs);
   const databaseActiveTaskIds = input.tasks.filter(activeTask).map((task) => task.id).sort();
   const activeRunRows = Array.from(input.activeRuns ?? []);
   const schedulerActiveTaskIds = Array.from(new Set(activeRunRows.map((run) => run.taskId).filter(Boolean))).sort();

src/components/microservices/code-queue/src/index.ts

@@ -116,7 +116,7 @@ import {
   taskForListResponse,
   transcriptChunkResponse,
 } from "./queue-api";
-import { buildExecutionDiagnostics, buildSchedulerHeartbeat, schedulerHeartbeatStaleMs, staleRecoveryCandidate } from "./execution-diagnostics";
+import { buildExecutionDiagnostics, buildSchedulerHeartbeat, normalizeSchedulerHeartbeatStaleMs, schedulerHeartbeatStaleMs, schedulerHeartbeatStaleMsMax, staleRecoveryCandidate } from "./execution-diagnostics";
 import { ReferenceTaskLookupError, configureReferences, injectReferencedTaskContext, taskReferenceIds } from "./references";
 import {
   applyOaTraceStatsToTaskJson,
@@ -135,7 +135,7 @@ import {
 } from "./oa-events";
 import { collectRuntimePreflight, runtimePreflightJson } from "./runtime-preflight";
 import { collectSkillAvailability, collectSkillSyncPreflight, skillAvailabilityJson, skillSyncPreflightJson } from "./skill-availability";
-import { configureSelfTests, runJudgeInfraSelfTest, runQueueClaimMoveSelfTest, runQueueOrderingSelfTest, runReferenceInjectionSelfTest, runTracePortSelfTest, runTraceSummaryContractSelfTest } from "./self-tests";
+import { configureSelfTests, runJudgeInfraSelfTest, runQueueClaimMoveSelfTest, runQueueOrderingSelfTest, runReferenceInjectionSelfTest, runStaleActiveRecoverySelfTest, runTracePortSelfTest, runTraceSummaryContractSelfTest } from "./self-tests";
 import {
   codexToolLifecycleStartedBeforeIn,
   configureTaskView,
@@ -177,7 +177,7 @@ const maxTaskAttempts = 99;
 const referenceInjectionMaxRounds: number | null = null;
 const retryBackoffBaseMs = 1000;
 const retryBackoffMaxMs = 10 * 60 * 1000;
-const orphanedActiveTaskRecoveryStaleMs = 5 * 60 * 1000;
+const orphanedActiveTaskRecoveryStaleMs = normalizeSchedulerHeartbeatStaleMs(5 * 60 * 1000);
 const queueIdPattern = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u;
 const queueNameMaxLength = 80;
 const workdirMaxLength = 512;
@@ -2722,6 +2722,12 @@ function truthyParam(url: URL, name: string): boolean {
   return value === "1" || value === "true" || value === "yes";
 }
 
+function boundedStringSetParam(url: URL, name: string, maxItems = 50): Set<string> | undefined {
+  const rawValues = url.searchParams.getAll(name).flatMap((value) => value.split(","));
+  const values = Array.from(new Set(rawValues.map((value) => value.trim()).filter(Boolean))).slice(0, maxItems);
+  return values.length === 0 ? undefined : new Set(values);
+}
+
 function parseSeqParam(url: URL, name: string, defaultValue: number | null): number | null {
   const raw = url.searchParams.get(name);
   if (raw === null) return defaultValue;
@@ -2965,6 +2971,7 @@ configureSelfTests({
   removeActiveRunSlotWaiter,
   resolveReasoningEffort,
   runDatabaseClaimMoveSelfTest,
+  staleActiveRecoverySelfTest: runSchedulerStaleActiveRecoverySelfTest,
   taskIsRecoverableOrphanedActive,
   tasks: () => state.tasks,
   updateProcessingFlag,
@@ -3608,10 +3615,39 @@ function orphanedActiveTaskAgeMs(task: QueueTask): number {
 function taskIsRecoverableOrphanedActive(task: QueueTask, staleMs = orphanedActiveTaskRecoveryStaleMs): boolean {
   if (task.status !== "running" && task.status !== "judging") return false;
   if (activeTaskHasLocalRunSlotOrWaiter(task)) return false;
-  const decision = staleRecoveryCandidate({ task, localActive: false, heartbeatStaleMs: staleMs, now: nowIso() });
+  const effectiveStaleMs = normalizeSchedulerHeartbeatStaleMs(staleMs, {
+    min: orphanedActiveTaskRecoveryStaleMs,
+    max: schedulerHeartbeatStaleMsMax,
+  });
+  const decision = staleRecoveryCandidate({ task, localActive: false, heartbeatStaleMs: effectiveStaleMs, now: nowIso() });
   return decision.allowed;
 }
 
+function recoverableOrphanedActiveTaskIds(tasks: QueueTask[] = state.tasks, staleMs = orphanedActiveTaskRecoveryStaleMs, filterIds?: Set<string>): string[] {
+  return tasks
+    .filter((task) => filterIds === undefined || filterIds.has(task.id))
+    .filter((task) => taskIsRecoverableOrphanedActive(task, staleMs))
+    .map((task) => task.id)
+    .sort();
+}
+
+function schedulerReconcileCandidateTasks(databaseTasks: QueueTask[]): QueueTask[] {
+  const mergedById = new Map<string, QueueTask>();
+  for (const task of state.tasks) mergedById.set(task.id, task);
+  for (const task of databaseTasks) {
+    if (task.status !== "queued" && task.status !== "retry_wait" && task.status !== "running" && task.status !== "judging") continue;
+    const existing = mergedById.get(task.id);
+    if (existing === undefined) {
+      mergedById.set(task.id, task);
+      continue;
+    }
+    const existingUpdatedAt = timestampMs(existing.updatedAt) ?? 0;
+    const taskUpdatedAt = timestampMs(task.updatedAt) ?? 0;
+    if (taskUpdatedAt > existingUpdatedAt && !activeRunForTask(existing)) mergedById.set(task.id, task);
+  }
+  return Array.from(mergedById.values());
+}
+
 function schedulerReconcileStatus(tasks: QueueTask[] = state.tasks): JsonValue {
   const orphanedActiveTasks = tasks
     .filter((task) => (task.status === "running" || task.status === "judging") && !activeTaskHasLocalRunSlotOrWaiter(task))
@@ -3694,13 +3730,11 @@ function queueActiveTasksForRestartRetry(reason: string, method: string, options
 async function recoverOrphanedActiveTasks(reason: string, method: string, options: { taskIds?: Set<string>; staleMs?: number } = {}): Promise<number> {
   schedulerReconcileAudit.lastRunAt = nowIso();
   schedulerReconcileAudit.lastReason = reason;
-  const staleMs = options.staleMs ?? orphanedActiveTaskRecoveryStaleMs;
-  const staleOrphanTaskIds = new Set(
-    state.tasks
-      .filter((task) => options.taskIds === undefined || options.taskIds.has(task.id))
-      .filter((task) => taskIsRecoverableOrphanedActive(task, staleMs))
-      .map((task) => task.id),
-  );
+  const staleMs = normalizeSchedulerHeartbeatStaleMs(options.staleMs, {
+    min: orphanedActiveTaskRecoveryStaleMs,
+    max: schedulerHeartbeatStaleMsMax,
+  });
+  const staleOrphanTaskIds = new Set(recoverableOrphanedActiveTaskIds(state.tasks, staleMs, options.taskIds));
   if (staleOrphanTaskIds.size === 0) {
     schedulerReconcileAudit.lastRecovered = 0;
     schedulerReconcileAudit.lastError = null;
@@ -3722,6 +3756,178 @@ async function recoverOrphanedActiveTasks(reason: string, method: string, option
   return recovered;
 }
 
+function schedulerRecoveryStaleMsFromUrl(url: URL): {
+  staleMs: number;
+  requestedStaleMs: number | null;
+  requestedStaleMsRaw: string | null;
+  staleMsAdjusted: boolean;
+  staleMsAdjustmentReason: "default" | "accepted" | "invalid" | "below-safe-minimum" | "above-maximum";
+  minStaleMs: number;
+  maxStaleMs: number;
+} {
+  const requestedStaleMsRaw = url.searchParams.get("staleMs");
+  const minStaleMs = orphanedActiveTaskRecoveryStaleMs;
+  const maxStaleMs = schedulerHeartbeatStaleMsMax;
+  if (requestedStaleMsRaw === null || requestedStaleMsRaw.trim().length === 0) {
+    return {
+      staleMs: minStaleMs,
+      requestedStaleMs: null,
+      requestedStaleMsRaw,
+      staleMsAdjusted: false,
+      staleMsAdjustmentReason: "default",
+      minStaleMs,
+      maxStaleMs,
+    };
+  }
+  const requestedStaleMs = Number(requestedStaleMsRaw);
+  const staleMs = normalizeSchedulerHeartbeatStaleMs(requestedStaleMs, { min: minStaleMs, max: maxStaleMs });
+  const requestedFloor = Math.floor(requestedStaleMs);
+  const staleMsAdjustmentReason = !Number.isFinite(requestedStaleMs)
+    ? "invalid"
+    : requestedFloor < minStaleMs
+      ? "below-safe-minimum"
+      : requestedFloor > maxStaleMs
+        ? "above-maximum"
+        : "accepted";
+  return {
+    staleMs,
+    requestedStaleMs: Number.isFinite(requestedStaleMs) ? requestedStaleMs : null,
+    requestedStaleMsRaw,
+    staleMsAdjusted: staleMsAdjustmentReason !== "accepted",
+    staleMsAdjustmentReason,
+    minStaleMs,
+    maxStaleMs,
+  };
+}
+
+async function reconcileStaleActiveTasksFromApi(url: URL, req: Request): Promise<Response> {
+  if (!serviceRoleAllowsScheduler(config.serviceRole)) return schedulerOnlyRejectResponse(req.method, "/api/scheduler/reconcile");
+  const notReady = requireDatabaseReadyForWrite(req.method, "/api/scheduler/reconcile");
+  if (notReady !== null) return notReady;
+  const recoverRequested = req.method === "POST" && truthyParam(url, "recover");
+  const dryRun = !recoverRequested || truthyParam(url, "dryRun");
+  const taskIds = boundedStringSetParam(url, "taskId");
+  const staleThreshold = schedulerRecoveryStaleMsFromUrl(url);
+  const staleMs = staleThreshold.staleMs;
+  let loaded = 0;
+  let candidateTasks = state.tasks;
+  if (dryRun) {
+    const databaseTasks = await loadTasksFromDatabase("hot");
+    loaded = databaseTasks.length;
+    candidateTasks = schedulerReconcileCandidateTasks(databaseTasks);
+  } else {
+    await refreshSchedulerTasksFromDatabase("api-reconcile", { recover: false });
+    loaded = schedulerReconcileAudit.lastLoaded;
+  }
+  const candidateTaskIds = recoverableOrphanedActiveTaskIds(candidateTasks, staleMs, taskIds);
+  let recovered = 0;
+  if (!dryRun && candidateTaskIds.length > 0) {
+    recovered = await recoverOrphanedActiveTasks("Scheduler API recovered expired database active task without local run", "scheduler/api-reconcile-recovery", { taskIds: new Set(candidateTaskIds), staleMs });
+  }
+  return jsonResponse({
+    ok: true,
+    dryRun,
+    recoverRequested,
+    loaded,
+    staleMs,
+    requestedStaleMs: staleThreshold.requestedStaleMs,
+    requestedStaleMsRaw: staleThreshold.requestedStaleMsRaw,
+    staleMsAdjusted: staleThreshold.staleMsAdjusted,
+    staleMsAdjustmentReason: staleThreshold.staleMsAdjustmentReason,
+    minStaleMs: staleThreshold.minStaleMs,
+    maxStaleMs: staleThreshold.maxStaleMs,
+    candidateTaskIds,
+    candidateCount: candidateTaskIds.length,
+    recovered,
+    reconcile: schedulerReconcileStatus(dryRun ? candidateTasks : state.tasks),
+    executionDiagnostics: executionDiagnosticsForTasks(dryRun ? candidateTasks : state.tasks),
+    oaEventPublisher: oaEventPublisherStatus(),
+  });
+}
+
+async function runSchedulerStaleActiveRecoverySelfTest(): Promise<JsonValue> {
+  const staleAt = new Date(Date.now() - orphanedActiveTaskRecoveryStaleMs - 60_000).toISOString();
+  const createdAt = new Date(Date.now() - orphanedActiveTaskRecoveryStaleMs - 120_000).toISOString();
+  const task = normalizeTask({
+    id: "codex_stale_active_recovery_self_test",
+    queueId: "stale_active_recovery_self_test",
+    queueEnteredAt: createdAt,
+    prompt: "stale active recovery self-test",
+    basePrompt: "stale active recovery self-test",
+    referenceTaskIds: [],
+    referenceInjection: null,
+    providerId: config.mainProviderId,
+    cwd: config.defaultWorkdir,
+    model: config.defaultModel,
+    reasoningEffort: resolveReasoningEffort(config.defaultModel, config.defaultReasoningEffort),
+    executionMode: "default",
+    maxAttempts: 1,
+    status: "running",
+    createdAt,
+    updatedAt: staleAt,
+    startedAt: createdAt,
+    finishedAt: null,
+    readAt: null,
+    currentAttempt: 1,
+    currentMode: "initial",
+    codexThreadId: "thread_stale_active_recovery_self_test",
+    activeTurnId: "turn_stale_active_recovery_self_test",
+    finalResponse: "",
+    lastError: null,
+    lastJudge: null,
+    judgeFailCount: 0,
+    promptHistory: [],
+    output: [],
+    events: [],
+    attempts: [],
+    cancelRequested: false,
+    nextPrompt: null,
+    nextMode: null,
+  });
+  task.schedulerHeartbeat = {
+    taskId: task.id,
+    queueId: task.queueId,
+    attempt: 1,
+    activeTurnId: task.activeTurnId,
+    codexThreadId: task.codexThreadId,
+    owner: "self-test",
+    schedulerInstance: "self-test",
+    executionPlane: "scheduler-execution-plane",
+    agentPort: "codex",
+    status: task.status,
+    lastLocalHeartbeatAt: staleAt,
+    lastObservedAgentEventAt: staleAt,
+    lastPersistedTraceAt: staleAt,
+    outputMaxSeq: 0,
+    source: "scheduler",
+  };
+  const before = state.tasks.slice();
+  try {
+    state.tasks.push(task);
+    const candidatesBefore = recoverableOrphanedActiveTaskIds(state.tasks, 0, new Set([task.id]));
+    const recovered = queueActiveTasksForRestartRetry("stale active recovery self-test", "self-test/stale-active-recovery", { taskIds: new Set(candidatesBefore) });
+    const candidatesAfter = recoverableOrphanedActiveTaskIds(state.tasks, 0, new Set([task.id]));
+    if (!candidatesBefore.includes(task.id)) throw new Error("expired active self-test task was not recovery-eligible");
+    if (recovered !== 1) throw new Error(`expected one recovered stale active task, got ${recovered}`);
+    if (task.status !== "retry_wait") throw new Error(`expected retry_wait after recovery, got ${task.status}`);
+    if (task.activeTurnId !== null) throw new Error("recovery should clear activeTurnId");
+    if (candidatesAfter.length !== 0) throw new Error("recovered self-test task should no longer be stale-active eligible");
+    return {
+      ok: true,
+      taskId: task.id,
+      recovered,
+      status: task.status,
+      candidatesBefore,
+      candidatesAfter,
+      lastError: task.lastError,
+      nextMode: task.nextMode,
+    } as unknown as JsonValue;
+  } finally {
+    state.tasks.splice(0, state.tasks.length, ...before);
+    dirtyDatabaseTaskIds.delete(task.id);
+  }
+}
+
 function failTaskForFallbackRetryLimit(task: QueueTask, judge: JudgeResult | null): void {
   const count = fallbackJudgeRetryCount(task);
   const reason = `Fallback/non-LLM judge retry limit reached (${count}/${fallbackJudgeRetryLimit}). ${judge?.reason ?? task.lastJudge?.reason ?? "MiniMax judge was unavailable."}`;
@@ -4245,7 +4451,7 @@ function hasRunnableTask(): boolean {
 }
 
 function shouldPollSchedulerDatabase(): boolean {
-  return config.schedulerEnabled && config.serviceRole === "scheduler";
+  return config.schedulerEnabled && serviceRoleAllowsScheduler(config.serviceRole);
 }
 
 function mergeSchedulerDatabaseTasks(tasks: QueueTask[]): number {
@@ -4272,14 +4478,14 @@ function mergeSchedulerDatabaseTasks(tasks: QueueTask[]): number {
   return changed;
 }
 
-async function refreshSchedulerTasksFromDatabase(reason: string): Promise<number> {
+async function refreshSchedulerTasksFromDatabase(reason: string, options: { recover?: boolean } = {}): Promise<number> {
   if (!databaseReady || !config.schedulerEnabled) return 0;
   schedulerReconcileAudit.lastRunAt = nowIso();
   schedulerReconcileAudit.lastReason = reason;
   try {
     const tasks = await loadTasksFromDatabase("hot");
     const changed = mergeSchedulerDatabaseTasks(tasks);
-    const recovered = await recoverOrphanedActiveTasks("Scheduler recovered database active task without local run", `scheduler/${reason}-recovery`);
+    const recovered = options.recover === false ? 0 : await recoverOrphanedActiveTasks("Scheduler recovered database active task without local run", `scheduler/${reason}-recovery`);
     schedulerReconcileAudit.lastLoaded = tasks.length;
     schedulerReconcileAudit.lastChanged = changed;
     schedulerReconcileAudit.lastRecovered = recovered;
@@ -5372,6 +5578,8 @@ async function route(req: Request): Promise<Response> {
     if (url.pathname === "/api/reference-injection/self-test" && (req.method === "GET" || req.method === "POST")) return jsonResponse(await runReferenceInjectionSelfTest());
     if (url.pathname === "/api/trace-port/self-test" && (req.method === "GET" || req.method === "POST")) return jsonResponse(runTracePortSelfTest());
     if (url.pathname === "/api/trace-summary-contract/self-test" && (req.method === "GET" || req.method === "POST")) return jsonResponse(runTraceSummaryContractSelfTest());
+    if (url.pathname === "/api/stale-active-recovery/self-test" && (req.method === "GET" || req.method === "POST")) return jsonResponse(await runStaleActiveRecoverySelfTest());
+    if (url.pathname === "/api/scheduler/reconcile" && (req.method === "GET" || req.method === "POST")) return await reconcileStaleActiveTasksFromApi(url, req);
     if (url.pathname === "/api/oa/backfill" && (req.method === "GET" || req.method === "POST")) return jsonResponse(await backfillOaTraceStats(url));
     if (url.pathname === "/api/notifications/claudeqq" && req.method === "GET") {
       await loadClaudeQqNotificationOutboxFromDatabase();

src/components/microservices/code-queue/src/oa-events.ts

@@ -72,9 +72,13 @@ let totalFailed = 0;
 let totalRetried = 0;
 let totalAbandoned = 0;
 let totalDropped = 0;
+let totalOverflowDropped = 0;
 let lastEnqueuedAt: string | null = null;
 let lastSucceededAt: string | null = null;
+let lastDroppedAt: string | null = null;
+let lastOverflowAt: string | null = null;
 let lastError: JsonRecord | null = null;
+let nextOverflowLogAtMs = 0;
 
 export function configureOaEvents(runtimeContext: OaEventContext): void {
   const baseUrl = runtimeContext.baseUrl.replace(/\/+$/u, "");
@@ -122,8 +126,11 @@ export function oaEventPublisherStatus(): JsonRecord {
     totalRetried,
     totalAbandoned,
     totalDropped,
+    totalOverflowDropped,
     lastEnqueuedAt,
     lastSucceededAt,
+    lastDroppedAt,
+    lastOverflowAt,
     lastError,
   };
 }
@@ -187,10 +194,23 @@ function postOaEvent(event: OaEventEnvelope): void {
   totalEnqueued += 1;
   lastEnqueuedAt = runtime.nowIso();
   if (queuedEvents.length >= maxQueuedEvents) {
-    const dropped = queuedEvents.shift();
     totalDropped += 1;
-    lastError = { eventId: dropped?.event.eventId ?? "", type: dropped?.event.type ?? "", error: "oa event publish queue overflow" };
-    runtime.logger("error", "oa_event_publish_queue_overflow", { ...lastError, pending: queuedEvents.length, maxQueuedEvents });
+    totalOverflowDropped += 1;
+    lastDroppedAt = lastEnqueuedAt;
+    lastOverflowAt = lastEnqueuedAt;
+    lastError = { eventId: event.eventId, type: event.type, error: "oa event publish queue overflow; incoming observability event dropped" };
+    const nowMs = Date.now();
+    if (nowMs >= nextOverflowLogAtMs) {
+      nextOverflowLogAtMs = nowMs + 30_000;
+      runtime.logger("error", "oa_event_publish_queue_overflow", {
+        ...lastError,
+        pending: queuedEvents.length,
+        maxQueuedEvents,
+        totalOverflowDropped,
+        isolated: true,
+      });
+    }
+    return;
   }
   queuedEvents.push({ event, attempts: 1 });
   pumpOaEventQueue();

src/components/microservices/code-queue/src/self-tests.ts

@@ -7,7 +7,7 @@ import { codeQueueEnvironmentHintTitle, injectCodeQueueEnvironmentHint, promptWi
 import { buildTaskTranscript, safePreview, taskTraceSummaryFixtureResponse, transcriptLineSummaryLines } from "./task-view";
 import type { ActiveRunSlotWaiter } from "./code-agent/common";
 import type { OaTraceStepSummary } from "./oa-events";
-import type { JsonValue, LiveOutput, QueueTask, QueuedStatusReason, QueueTaskRequest, RuntimeConfig, TaskStatus } from "./types";
+import type { JsonValue, LiveOutput, QueueTask, QueuedStatusReason, QueueTaskRequest, RuntimeConfig, SchedulerActiveRunHeartbeat, TaskStatus } from "./types";
 
 export interface SelfTestsContext {
   config: Pick<RuntimeConfig, "defaultModel" | "defaultReasoningEffort" | "defaultWorkdir" | "mainProviderId" | "maxActiveQueues">;
@@ -30,6 +30,7 @@ export interface SelfTestsContext {
   resolveReasoningEffort: (model: string, explicit?: string | null) => string | null;
   runDatabaseClaimMoveSelfTest?: () => Promise<JsonValue | null>;
   runTraceSummaryContractSelfTest?: () => JsonValue;
+  staleActiveRecoverySelfTest: () => Promise<JsonValue>;
   taskIsRecoverableOrphanedActive: (task: QueueTask, staleMs?: number) => boolean;
   tasks: () => QueueTask[];
   updateProcessingFlag: () => void;
@@ -311,6 +312,30 @@ function queueOrderTestTask(id: string, status: TaskStatus, createdAt: string, q
   return ctx().normalizeTask(task);
 }
 
+function selfTestSchedulerHeartbeat(task: QueueTask, lastLocalHeartbeatAt: string): SchedulerActiveRunHeartbeat {
+  return {
+    taskId: task.id,
+    queueId: task.queueId,
+    attempt: 1,
+    activeTurnId: task.activeTurnId,
+    codexThreadId: task.codexThreadId,
+    owner: "self-test",
+    schedulerInstance: "self-test",
+    executionPlane: "scheduler-execution-plane",
+    agentPort: "codex",
+    status: task.status,
+    lastLocalHeartbeatAt,
+    lastObservedAgentEventAt: lastLocalHeartbeatAt,
+    lastPersistedTraceAt: lastLocalHeartbeatAt,
+    outputMaxSeq: 0,
+    source: "scheduler",
+  };
+}
+
+async function runStaleActiveRecoverySelfTest(): Promise<JsonValue> {
+  return await ctx().staleActiveRecoverySelfTest();
+}
+
 async function runQueueClaimMoveSelfTest(): Promise<JsonValue> {
   const at = "2026-05-17T06:09:46.702Z";
   const task = queueOrderTestTask("codex_claim_move_self_test", "running", at, at);
@@ -370,23 +395,7 @@ function runQueueOrderingSelfTest(): JsonValue {
   const orphanRunning = queueOrderTestTask("codex_4400_orphan_running", "running", "2026-05-11T13:00:00.000Z", "2026-05-11T13:00:00.000Z");
   orphanRunning.queueId = "queue_orphan_recovery";
   orphanRunning.activeTurnId = null;
-  orphanRunning.schedulerHeartbeat = {
-    taskId: orphanRunning.id,
-    queueId: orphanRunning.queueId,
-    attempt: 1,
-    activeTurnId: null,
-    codexThreadId: null,
-    owner: "self-test",
-    schedulerInstance: "self-test",
-    executionPlane: "scheduler-execution-plane",
-    agentPort: "codex",
-    status: "running",
-    lastLocalHeartbeatAt: "2026-05-11T13:00:00.000Z",
-    lastObservedAgentEventAt: null,
-    lastPersistedTraceAt: null,
-    outputMaxSeq: 0,
-    source: "scheduler",
-  };
+  orphanRunning.schedulerHeartbeat = selfTestSchedulerHeartbeat(orphanRunning, "2026-05-11T13:00:00.000Z");
   const queuedBehindOrphan = queueOrderTestTask("codex_4401_queued", "queued", "2026-05-11T13:01:00.000Z", "2026-05-11T13:01:00.000Z");
   queuedBehindOrphan.queueId = "queue_orphan_recovery";
   const originalMaxActiveQueues = ctx().config.maxActiveQueues;
@@ -676,4 +685,4 @@ function runJudgeInfraSelfTest(): JsonValue {
   };
 }
 
-export { runJudgeInfraSelfTest, runQueueClaimMoveSelfTest, runQueueOrderingSelfTest, runReferenceInjectionSelfTest, runTracePortSelfTest, runTraceSummaryContractSelfTest };
+export { runJudgeInfraSelfTest, runQueueClaimMoveSelfTest, runQueueOrderingSelfTest, runReferenceInjectionSelfTest, runStaleActiveRecoverySelfTest, runTracePortSelfTest, runTraceSummaryContractSelfTest };