mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: foreground commander queue concurrency

Browse Source 
Foreground codex queues commander concurrency summary and fixtures.
This commit is contained in:
Lyon
2026-05-23 21:33:49 +08:00
committed by GitHub
parent ed513628e6
commit d90c9d10e3
5 changed files with 558 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/code-queue-queues-shape-contract-test.ts
+293
View File
@@ -140,6 +140,224 @@ function splitBrainLiveResponse(): JsonRecord {
};
}
function heartbeatRiskResponse(): JsonRecord {
const staleTaskIds = manyIds("task-stale", 4);
return {
ok: true,
status: 200,
body: {
ok: true,
queue: {
total: 6,
queueCount: 3,
activeQueueIds: [],
activeTaskIds: [],
queuedTaskIds: ["task-queued-risk"],
counts: { running: 4, queued: 1, retry_wait: 1 },
unreadTerminal: 0,
executionDiagnostics: {
state: "stale-active",
effectiveLiveness: "at-risk",
recommendedAction: "investigate-heartbeat-risk",
now: "2026-05-20T00:30:00.000Z",
databaseActiveTaskCount: 4,
databaseActiveTaskIds: staleTaskIds,
schedulerActiveRunSlotCount: 0,
schedulerActiveTaskIds: [],
activeHeartbeatCount: 4,
activeHeartbeatTaskIds: staleTaskIds,
heartbeatFreshTaskIds: [],
heartbeatExpiredTaskIds: staleTaskIds,
heartbeatMissingTaskIds: [],
staleRecoveryCandidateTaskIds: staleTaskIds,
heartbeatRiskTaskIds: staleTaskIds,
lastSchedulerHeartbeatAt: "2026-05-20T00:15:00.000Z",
lastObservedAgentEventAt: "2026-05-20T00:14:30.000Z",
reasons: ["heartbeat expired for database-active tasks"],
},
},
queues: [
{
id: "risk-a",
name: "Risk A",
total: 3,
counts: { running: 3 },
unreadTerminal: 0,
activeTaskId: null,
runnableTaskId: null,
updatedAt: "2026-05-20T00:20:00.000Z",
},
{
id: "risk-b",
name: "Risk B",
total: 1,
counts: { running: 1 },
unreadTerminal: 0,
activeTaskId: null,
runnableTaskId: null,
updatedAt: "2026-05-20T00:19:00.000Z",
},
{
id: "risk-queued",
name: "Risk Queued",
total: 2,
counts: { queued: 1, retry_wait: 1 },
unreadTerminal: 0,
activeTaskId: null,
runnableTaskId: "task-queued-risk",
updatedAt: "2026-05-20T00:18:00.000Z",
},
],
},
};
}
function noActiveResponse(): JsonRecord {
return {
ok: true,
status: 200,
body: {
ok: true,
queue: {
total: 2,
queueCount: 2,
activeQueueIds: [],
activeTaskIds: [],
queuedTaskIds: [],
counts: { succeeded: 2 },
unreadTerminal: 1,
executionDiagnostics: {
state: "healthy",
effectiveLiveness: "healthy",
recommendedAction: "none",
databaseActiveTaskCount: 0,
databaseActiveTaskIds: [],
schedulerActiveRunSlotCount: 0,
schedulerActiveTaskIds: [],
activeHeartbeatCount: 0,
activeHeartbeatTaskIds: [],
heartbeatFreshTaskIds: [],
heartbeatExpiredTaskIds: [],
heartbeatMissingTaskIds: [],
staleRecoveryCandidateTaskIds: [],
heartbeatRiskTaskIds: [],
},
},
queues: [
{
id: "done-a",
name: "Done A",
total: 1,
counts: { succeeded: 1 },
unreadTerminal: 1,
activeTaskId: null,
runnableTaskId: null,
updatedAt: "2026-05-20T00:05:00.000Z",
},
{
id: "done-b",
name: "Done B",
total: 1,
counts: { succeeded: 1 },
unreadTerminal: 0,
activeTaskId: null,
runnableTaskId: null,
updatedAt: "2026-05-20T00:04:00.000Z",
},
],
},
};
}
function activeBelowTargetResponse(): JsonRecord {
return {
ok: true,
status: 200,
body: {
ok: true,
queue: {
total: 8,
queueCount: 3,
activeQueueIds: ["feature-a", "feature-b"],
activeTaskIds: ["task-active-a", "task-active-b"],
queuedTaskIds: ["task-queued-a", "task-queued-b", "task-queued-c"],
counts: { running: 2, queued: 3, retry_wait: 1, succeeded: 2 },
unreadTerminal: 0,
executionDiagnostics: {
state: "healthy",
effectiveLiveness: "healthy",
recommendedAction: "none",
databaseActiveTaskCount: 2,
databaseActiveTaskIds: ["task-active-a", "task-active-b"],
schedulerActiveRunSlotCount: 2,
schedulerActiveTaskIds: ["task-active-a", "task-active-b"],
activeHeartbeatCount: 2,
activeHeartbeatTaskIds: ["task-active-a", "task-active-b"],
heartbeatFreshTaskIds: ["task-active-a", "task-active-b"],
heartbeatExpiredTaskIds: [],
heartbeatMissingTaskIds: [],
staleRecoveryCandidateTaskIds: [],
heartbeatRiskTaskIds: [],
lastSchedulerHeartbeatAt: "2026-05-20T01:00:00.000Z",
lastObservedAgentEventAt: "2026-05-20T01:00:05.000Z",
},
},
queues: [
{
id: "feature-a",
name: "Feature A",
total: 3,
counts: { running: 1, queued: 1, succeeded: 1 },
unreadTerminal: 0,
activeTaskId: "task-active-a",
runnableTaskId: null,
updatedAt: "2026-05-20T01:01:00.000Z",
},
{
id: "feature-b",
name: "Feature B",
total: 3,
counts: { running: 1, retry_wait: 1, succeeded: 1 },
unreadTerminal: 0,
activeTaskId: "task-active-b",
runnableTaskId: "task-queued-b",
updatedAt: "2026-05-20T01:00:30.000Z",
},
{
id: "feature-c",
name: "Feature C",
total: 2,
counts: { queued: 2 },
unreadTerminal: 0,
activeTaskId: null,
runnableTaskId: "task-queued-c",
updatedAt: "2026-05-20T01:00:10.000Z",
},
],
tasks: [
{
id: "task-active-a",
queueId: "feature-a",
status: "running",
displayPrompt: "Implement feature A polish and contract tests",
},
{
id: "task-active-b",
queueId: "feature-b",
status: "running",
displayPrompt: "Fix feature B queue handling",
},
{
id: "task-queued-a",
queueId: "feature-a",
status: "queued",
displayPrompt: "Queued feature A follow-up",
},
],
},
};
}
function assertQueuesShape(label: string, result: unknown, expectedView: string): void {
const data = asRecord(result);
const queues = asRecord(data.queues);
@@ -175,6 +393,12 @@ function assertQueuesShape(label: string, result: unknown, expectedView: string)
assertCondition(activity.activeQueueIdsScope === "scheduler-local-active-run-slots", `${label} activity should label activeQueueIds scope`, activity);
assertCondition(Array.isArray(queues.activeTaskIds), `${label} activeTaskIds should be present`, queues);
assertCondition(Array.isArray(queues.queuedTaskIds), `${label} queuedTaskIds should be present`, queues);
const commander = asRecord(queues.commander);
assertCondition(commander.activeRunnerCount === 1, `${label} commander block should be near the front and expose active runner count`, commander);
assertCondition(commander.target === 15 && commander.slotDeficit === 14, `${label} commander block should expose 15 target slot deficit`, commander);
assertCondition(commander.queuedCount === 2, `${label} commander block should expose queued count`, commander);
assertCondition(asArray(asRecord(commander.runningTasks).items).length >= 1, `${label} commander runningTasks should expose active task ids`, commander);
assertCondition(asArray(asRecord(asRecord(commander.heartbeat).fresh).items).includes("task-running"), `${label} commander heartbeat should foreground fresh ids`, commander);
}
function assertSplitBrainLiveActivity(label: string, result: unknown): void {
@@ -209,6 +433,50 @@ function assertSplitBrainLiveActivity(label: string, result: unknown): void {
assertCondition(String(commanderConcurrency.decisionRule ?? "").includes("15 - activeRunnerCount"), `${label} should give 15-concurrency arithmetic`, commanderConcurrency);
assertCondition(String(activity.activeQueueIdsNote ?? "").includes("zero local queue ids does not mean zero active runners"), `${label} activity note should prevent zero-active misread`, activity);
assertCondition(String(activity.interpretation ?? "").includes("continue supervision"), `${label} activity interpretation should keep supervision action`, activity);
const commander = asRecord(queues.commander);
assertCondition(commander.activeRunnerCount === 8, `${label} commander should foreground heartbeat-effective active count`, commander);
assertCondition(commander.slotDeficit === 7 && commander.target === 15, `${label} commander should compute below-target slot deficit`, commander);
assertCondition(asRecord(commander.runningTasks).count === 8, `${label} commander runningTasks should preserve live active count`, commander);
assertCondition(asRecord(asRecord(commander.heartbeat).fresh).count === 8, `${label} commander heartbeat should preserve fresh heartbeat count`, commander);
assertCondition(asRecord(asRecord(commander.heartbeat).risk).count === 0, `${label} commander heartbeat risk should be zero for live split-brain`, commander);
}
function assertHeartbeatRiskCommander(label: string, result: unknown): void {
const queues = asRecord(asRecord(result).queues);
const commander = asRecord(queues.commander);
const heartbeat = asRecord(commander.heartbeat);
const risk = asRecord(heartbeat.risk);
const stale = asRecord(heartbeat.staleRecoveryCandidates);
assertCondition(commander.activeRunnerCount === 4, `${label} should count database-active stale tasks as active before recovery`, commander);
assertCondition(commander.slotDeficit === 11 && commander.target === 15, `${label} should compute slot deficit from target 15`, commander);
assertCondition(commander.queuedCount === 2, `${label} should expose queued + retry_wait count`, commander);
assertCondition(commander.attentionRequired === true && commander.interventionRequired === false, `${label} should require attention but not high-risk intervention from one snapshot`, commander);
assertCondition(heartbeat.effectiveLiveness === "at-risk", `${label} heartbeat liveness should be at-risk`, heartbeat);
assertCondition(risk.count === 4 && asArray(risk.items).length > 0, `${label} should expose bounded heartbeat risk ids`, risk);
assertCondition(stale.count === 4 && String(stale.command ?? "").includes("dryRun=1"), `${label} should expose stale recovery candidates and dry-run command`, stale);
}
function assertNoActiveCommander(label: string, result: unknown): void {
const commander = asRecord(asRecord(asRecord(result).queues).commander);
assertCondition(commander.activeRunnerCount === 0, `${label} should expose zero active runners`, commander);
assertCondition(commander.slotDeficit === 15 && commander.slotDeficitState === "below-target", `${label} should expose full 15-slot deficit`, commander);
assertCondition(commander.queuedCount === 0, `${label} queued count should be zero`, commander);
assertCondition(asRecord(commander.runningTasks).count === 0, `${label} running task list should be empty`, commander);
}
function assertBelowTargetCommander(label: string, result: unknown): void {
const commander = asRecord(asRecord(asRecord(result).queues).commander);
const runningTasks = asRecord(commander.runningTasks);
const runningItems = asArray(runningTasks.items).map(asRecord);
const activeQueues = asRecord(commander.activeQueues);
const runnableQueues = asRecord(commander.runnableQueues);
assertCondition(commander.activeRunnerCount === 2, `${label} should foreground active runner count`, commander);
assertCondition(commander.slotDeficit === 13 && commander.target === 15, `${label} should expose deficit below target`, commander);
assertCondition(commander.queuedCount === 4, `${label} should include queued and retry_wait`, commander);
assertCondition(runningItems.some((item) => item.id === "task-active-a" && item.name === "Implement feature A polish and contract tests"), `${label} should include active task names when upstream task rows are available`, runningTasks);
assertCondition(runningItems.some((item) => asArray(item.queueIds).includes("feature-a") && asArray(item.queueNames).includes("Feature A")), `${label} should include active task queues`, runningTasks);
assertCondition(activeQueues.count === 2 && runnableQueues.count === 3, `${label} should keep active and queued queue rows separate`, commander);
assertCondition(String(asRecord(commander.commands).running ?? "").includes("--status running,judging"), `${label} should expose running drill-down command`, commander);
}
export function runCodeQueueQueuesShapeContract(): JsonRecord {
@@ -220,11 +488,28 @@ export function runCodeQueueQueuesShapeContract(): JsonRecord {
assertCondition(path === "/api/microservices/code-queue/proxy/api/queues", "codex queues should use stable proxy path", { path });
return splitBrainLiveResponse();
};
const heartbeatRiskFetcher = (path: string): JsonRecord => {
assertCondition(path === "/api/microservices/code-queue/proxy/api/queues", "codex queues should use stable proxy path", { path });
return heartbeatRiskResponse();
};
const noActiveFetcher = (path: string): JsonRecord => {
assertCondition(path === "/api/microservices/code-queue/proxy/api/queues", "codex queues should use stable proxy path", { path });
return noActiveResponse();
};
const activeBelowTargetFetcher = (path: string): JsonRecord => {
assertCondition(path === "/api/microservices/code-queue/proxy/api/queues", "codex queues should use stable proxy path", { path });
return activeBelowTargetResponse();
};
const summary = codexQueuesQueryForTest([], fetcher);
assertQueuesShape("summary", summary, "summary");
const summaryQueues = asRecord(asRecord(summary).queues);
assertCondition(summaryQueues.deprecatedFullArray === undefined, "summary should not expose deprecated full array compatibility field", summaryQueues);
assertCondition(summaryQueues.summaryMode === "commander-first", "summary should default to commander-first mode", summaryQueues);
const explicitCommander = codexQueuesQueryForTest(["--commander"], fetcher);
const explicitCommanderQueues = asRecord(asRecord(explicitCommander).queues);
assertCondition(explicitCommanderQueues.summaryMode === "commander-first", "--commander should keep commander-first mode", explicitCommanderQueues);
const full = codexQueuesQueryForTest(["--full"], fetcher);
assertQueuesShape("full", full, "full");
@@ -255,6 +540,12 @@ export function runCodeQueueQueuesShapeContract(): JsonRecord {
assertSplitBrainLiveActivity("split-brain summary", splitSummary);
const splitFull = codexQueuesQueryForTest(["--full"], splitBrainFetcher);
assertSplitBrainLiveActivity("split-brain full", splitFull);
const heartbeatRisk = codexQueuesQueryForTest([], heartbeatRiskFetcher);
assertHeartbeatRiskCommander("heartbeat-risk summary", heartbeatRisk);
const noActive = codexQueuesQueryForTest([], noActiveFetcher);
assertNoActiveCommander("no-active summary", noActive);
const belowTarget = codexQueuesQueryForTest([], activeBelowTargetFetcher);
assertBelowTargetCommander("active-below-target summary", belowTarget);
return {
ok: true,
@@ -268,6 +559,8 @@ export function runCodeQueueQueuesShapeContract(): JsonRecord {
"offset pagination",
"split-brain live activity counts distinguish scheduler-local queues, DB running tasks, and heartbeat-fresh runners",
"commander concurrency block names the active runner count and 15-concurrency rule",
"queues commander-first summary foregrounds active runner count, target 15 deficit, running task ids/names/queues, heartbeat risk, stale recovery candidates, and queued count",
"queues commander fixtures cover split-brain live, heartbeat risk, no active, and active below target",
],
};
}