mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(code-queue): forward-port stale-active recovery

Browse Source 
Supersedes stale PR #79 with a current-master forward-port of safe stale-active recovery diagnostics and explicit recovery controls.
This commit is contained in:
Lyon
2026-05-23 16:22:41 +08:00
committed by GitHub
parent d88eb460a5
commit 4ce5d2fd97
6 changed files with 379 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/code-queue-liveness-fixtures.ts
+82 -2
View File
@@ -1,11 +1,13 @@
import { buildExecutionDiagnostics, buildSchedulerHeartbeat, staleRecoveryCandidate, taskHasTraceGapButFreshHeartbeat } from "../../src/components/microservices/code-queue/src/execution-diagnostics";
import { buildExecutionDiagnostics, buildSchedulerHeartbeat, schedulerHeartbeatStaleMs, staleRecoveryCandidate, taskHasTraceGapButFreshHeartbeat } from "../../src/components/microservices/code-queue/src/execution-diagnostics";
import type { ActiveRun } from "../../src/components/microservices/code-queue/src/code-agent/common";
import type { CodeQueueExecutionDiagnostics, QueueTask, SchedulerActiveRunHeartbeat, TaskStatus } from "../../src/components/microservices/code-queue/src/types";
export const CODE_QUEUE_LIVENESS_CHECK_NAMES = [
"code-queue:active-run-heartbeat-visible",
"code-queue:trace-gap-not-stale",
"code-queue:heartbeat-threshold-normalized",
"code-queue:stale-active-owner-expired",
"code-queue:stale-active-recovery-transition",
"code-queue:control-plane-split-brain-diagnostics",
"code-queue:oa-publisher-degraded-visible",
] as const;
@@ -104,9 +106,10 @@ function activeRun(taskId: string, queueId = "default"): ActiveRun {
};
}
function schedulerDiagnostics(tasks: QueueTask[], activeRuns: ActiveRun[] = [], oaPublisher: unknown = null): CodeQueueExecutionDiagnostics {
function schedulerDiagnostics(tasks: QueueTask[], activeRuns: ActiveRun[] = [], oaPublisher: unknown = null, heartbeatStaleMs?: number): CodeQueueExecutionDiagnostics {
return buildExecutionDiagnostics({
now,
heartbeatStaleMs,
controlPlane: "D601-code-queue-scheduler",
executionStateSource: "scheduler-execution-plane",
tasks,
@@ -119,6 +122,37 @@ function schedulerDiagnostics(tasks: QueueTask[], activeRuns: ActiveRun[] = [],
});
}
function checkHeartbeatThresholdNormalized(): FixtureCheck {
const task = fixtureTask("codex_fixture_threshold_fresh_1", "running", heartbeat("codex_fixture_threshold_fresh_1", freshAt));
const run = activeRun(task.id);
const zeroDecision = staleRecoveryCandidate({ task, localActive: false, now, heartbeatStaleMs: 0 });
const tooLowDecision = staleRecoveryCandidate({ task, localActive: false, now, heartbeatStaleMs: 1 });
const zeroDiagnostics = schedulerDiagnostics([task], [run], null, 0);
const tooLowDiagnostics = schedulerDiagnostics([task], [run], null, 1);
assertCondition(zeroDecision.allowed === false && zeroDecision.reason === "owner-heartbeat-fresh", "zero heartbeat threshold must normalize before stale recovery", { zeroDecision });
assertCondition(tooLowDecision.allowed === false && tooLowDecision.reason === "owner-heartbeat-fresh", "too-low heartbeat threshold must normalize before stale recovery", { tooLowDecision });
assertCondition(zeroDiagnostics.schedulerHeartbeatStaleMs === schedulerHeartbeatStaleMs, "zero diagnostics threshold should normalize to default stale window", zeroDiagnostics as unknown as Record<string, unknown>);
assertCondition(tooLowDiagnostics.schedulerHeartbeatStaleMs === schedulerHeartbeatStaleMs, "too-low diagnostics threshold should normalize to default stale window", tooLowDiagnostics as unknown as Record<string, unknown>);
assertCondition(zeroDiagnostics.state === "healthy" && zeroDiagnostics.effectiveLiveness === "healthy", "fresh heartbeat with zero requested threshold must stay healthy", zeroDiagnostics as unknown as Record<string, unknown>);
assertCondition(tooLowDiagnostics.state === "healthy" && tooLowDiagnostics.effectiveLiveness === "healthy", "fresh heartbeat with too-low requested threshold must stay healthy", tooLowDiagnostics as unknown as Record<string, unknown>);
assertCondition(zeroDiagnostics.heartbeatFreshTaskIds.includes(task.id), "fresh heartbeat must remain in fresh task ids", zeroDiagnostics as unknown as Record<string, unknown>);
assertCondition(zeroDiagnostics.heartbeatExpiredTaskIds.length === 0 && zeroDiagnostics.staleRecoveryCandidateTaskIds.length === 0 && zeroDiagnostics.heartbeatRiskTaskIds.length === 0, "fresh heartbeat must not be reported stale or risky", zeroDiagnostics as unknown as Record<string, unknown>);
return {
name: "code-queue:heartbeat-threshold-normalized",
ok: true,
detail: {
zeroDecision,
tooLowDecision,
normalizedStaleMs: zeroDiagnostics.schedulerHeartbeatStaleMs,
state: zeroDiagnostics.state,
effectiveLiveness: zeroDiagnostics.effectiveLiveness,
heartbeatFreshTaskIds: zeroDiagnostics.heartbeatFreshTaskIds,
heartbeatExpiredTaskIds: zeroDiagnostics.heartbeatExpiredTaskIds,
staleRecoveryCandidateTaskIds: zeroDiagnostics.staleRecoveryCandidateTaskIds,
},
};
}
function checkActiveRunHeartbeatVisible(): FixtureCheck {
const task = fixtureTask("codex_fixture_active_1", "running");
const run = activeRun(task.id);
@@ -194,6 +228,50 @@ function checkStaleActiveOwnerExpired(): FixtureCheck {
};
}
function recoverStaleActiveTaskForFixture(task: QueueTask): number {
const decision = staleRecoveryCandidate({ task, localActive: false, now });
if (!decision.allowed) return 0;
task.status = "retry_wait";
task.finishedAt = null;
task.readAt = null;
task.activeTurnId = null;
task.lastError = "fixture stale-active recovery";
task.nextMode = "retry";
task.nextPrompt = "fixture recovery prompt";
task.updatedAt = now;
return 1;
}
function checkStaleActiveRecoveryTransition(): FixtureCheck {
const task = fixtureTask("codex_fixture_stale_recovery_1", "running", heartbeat("codex_fixture_stale_recovery_1", expiredAt, {
lastObservedAgentEventAt: expiredAt,
lastPersistedTraceAt: expiredAt,
}));
const freshTask = fixtureTask("codex_fixture_fresh_no_recovery_1", "running", heartbeat("codex_fixture_fresh_no_recovery_1", freshAt));
const diagnosticsBefore = schedulerDiagnostics([task], []);
const recovered = recoverStaleActiveTaskForFixture(task);
const freshRecovered = recoverStaleActiveTaskForFixture(freshTask);
const diagnosticsAfter = schedulerDiagnostics([task], []);
assertCondition(recovered === 1, "expired active task should be recovered exactly once", { recovered, task });
assertCondition(freshRecovered === 0 && freshTask.status === "running", "fresh owner heartbeat must block recovery", { freshRecovered, freshStatus: freshTask.status });
assertCondition(task.status === "retry_wait" && task.activeTurnId === null && task.nextMode === "retry", "recovered task should re-enter retry_wait with active turn cleared", { task });
assertCondition(diagnosticsBefore.staleRecoveryCandidateTaskIds.includes("codex_fixture_stale_recovery_1"), "before recovery should show stale candidate", diagnosticsBefore as unknown as Record<string, unknown>);
assertCondition(!diagnosticsAfter.staleRecoveryCandidateTaskIds.includes("codex_fixture_stale_recovery_1"), "after recovery should remove stale candidate", diagnosticsAfter as unknown as Record<string, unknown>);
return {
name: "code-queue:stale-active-recovery-transition",
ok: true,
detail: {
recovered,
freshRecovered,
status: task.status,
activeTurnId: task.activeTurnId,
nextMode: task.nextMode,
candidatesBefore: diagnosticsBefore.staleRecoveryCandidateTaskIds,
candidatesAfter: diagnosticsAfter.staleRecoveryCandidateTaskIds,
},
};
}
function checkControlPlaneSplitBrainDiagnostics(): FixtureCheck {
const task = fixtureTask("codex_fixture_split_1", "running", heartbeat("codex_fixture_split_1", freshAt));
const diagnostics = buildExecutionDiagnostics({
@@ -247,7 +325,9 @@ export function runCodeQueueLivenessFixtureChecks(only: string[] = []): { ok: bo
const runners: Array<[CodeQueueLivenessCheckName, () => FixtureCheck]> = [
["code-queue:active-run-heartbeat-visible", checkActiveRunHeartbeatVisible],
["code-queue:trace-gap-not-stale", checkTraceGapNotStale],
["code-queue:heartbeat-threshold-normalized", checkHeartbeatThresholdNormalized],
["code-queue:stale-active-owner-expired", checkStaleActiveOwnerExpired],
["code-queue:stale-active-recovery-transition", checkStaleActiveRecoveryTransition],
["code-queue:control-plane-split-brain-diagnostics", checkControlPlaneSplitBrainDiagnostics],
["code-queue:oa-publisher-degraded-visible", checkOaPublisherDegradedVisible],
];