From 031c3fda39c2b2656f35cee9c05a61d22d02f7a8 Mon Sep 17 00:00:00 2001 From: UniDesk Code Queue Date: Fri, 22 May 2026 23:42:39 +0000 Subject: [PATCH] fix: clarify code queue activity counts --- docs/reference/cli.md | 2 +- docs/reference/code-queue-supervision.md | 6 +- .../code-queue-queues-shape-contract-test.ts | 106 ++++++++++++++++ ...eue-supervisor-disclosure-contract-test.ts | 71 +++++++++++ scripts/src/code-queue.ts | 114 ++++++++++++++++++ scripts/src/help.ts | 12 +- 6 files changed, 305 insertions(+), 6 deletions(-) diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 7d6e7101..b466c637 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -56,7 +56,7 @@ CLI 可以从 `master` 快速演进，但必须兼容 `deploy.json` 固定的 CI - `codex steer [prompt|--prompt-file path|--prompt-stdin] [--dry-run] [--no-retry|--retry-attempts N]` 通过 Code Queue 私有代理向正在运行的 task 注入纠偏提示，正式替代底层 `microservice proxy code-queue /api/tasks//steer` 调用。prompt 必须且只能来自位置参数、文件或 stdin 之一；`--dry-run` 只输出 `method`、`path`、`stableProxyPath`、retry policy、prompt 字符数、截断预览和 raw proxy 等价命令，不触碰运行中 session，也不得泄露超长 prompt 全文。真实执行是写入操作，成功只返回 `accepted=true`、task id、prompt 字符数、`promptOmitted=true`、有界 task/queue 确认、attempt summary 和后续查看命令，不回显 prompt 或完整 task state；路径固定为 `/api/microservices/code-queue/proxy/api/tasks//steer`，只能作用于 D601 scheduler 上存在 active steerable turn 的 running task。默认对 `stable-proxy-failed` 和 `backend-core-unreachable` 这类 retryable control-plane failures 做一次有界重试；`--retry-attempts N` 最大为 3，`--retry-delay-ms N` 最大为 5000，`--no-retry` 用于复现单次失败。 - `codex steer` 非 dry-run 失败仍输出 JSON 且退出非零；`.data.diagnostics.reason` 用于 runner 分流，当前包括 `backend-core-unreachable`、`code-queue-microservice-unregistered`、`proxy-unauthorized`、`proxy-404`、`steer-endpoint-404`、`upstream-runtime-rejected`、`stable-proxy-failed` 和 `invalid-proxy-response`。`scope` 区分 `backend-core`、`stable-proxy`、`code-queue-runtime` 或 `unknown`，并带 `status`、`exitCode`、`retryable`、有界 `upstreamBodyPreview`、`attempts`、`retryPolicy` 和推荐交叉验证命令；若任务不在 running/active-turn 状态，通常归类为 `upstream-runtime-rejected`，不得静默成功。`502 provider HTTP tunnel failed`、`provider-gateway-http-fetch`、`The operation was aborted` 或约 30 秒 tunnel wait abort 会归类为 `stable-proxy-failed`，CLI 会先按 retry policy 重试；如果仍失败，`.data.diagnostics.operatorGuidance.rawProxyEquivalentIsFallback=false` 表示 raw proxy 等价命令走同一条 tunnel，只能用于对照诊断，不应被当作更低噪声 fallback。此时 `.data.steer.deliveryUnconfirmed=true`，指挥官应先看 `codex tasks --view supervisor`、`codex task ` 和 `microservice health code-queue`，再从主 server CLI 或显式 SSH transport 重试同一个 `codex steer`。 - `codex interrupt|cancel ` 通过 Code Queue 私有代理请求中断；running/judging 任务会请求 D601 当前 agent run 停止，queued/retry_wait 任务的取消也必须保持与 WebUI 相同代理路径，返回有界 task 摘要和后续查询命令。任何需要接触 active run 的动作仍属于 D601 执行面。 -- Code Queue 多队列 lane 由 `codex` 命令命名空间管理：`queues [--full|--all] [--limit N] [--page N|--offset N]` 列表、`queue create ` 创建、`queue merge --into ` 合并、`move --queue ` 迁移；这些队列管理入口默认由主 server `code-queue-mgr` 直管 PostgreSQL，仍通过稳定 `code-queue` 用户服务代理路径访问。`codex queues` 默认只返回 active/nonempty/unread/runnable queue 摘要、全局 counts 和 execution diagnostics；`--full` 或 `--all` 只切换为完整队列行视图的一页，仍受 `--limit`/`--page`/`--offset` 分页约束，不再默认携带 deprecated full array。summary 和 full 的稳定机读路径都是 `.data.queues.items[]`，全局元数据固定在 `.data.queues.counts`、`.data.queues.executionDiagnostics`、`.data.queues.activeTaskIds` 和 `.data.queues.queuedTaskIds`；需要完整 upstream 时使用输出中的 raw command。旧 full 顶层数组语义已作为 deprecated 兼容信息记录，不再作为 `.data.queues` 主形态。同一个 queue 内部串行执行，不同 queue 之间并行执行。迁移只允许尚未被 scheduler claim 的 `queued`/`retry_wait` 任务，必须满足 `startedAt=null`、`currentAttempt=0` 且没有 active thread/turn；已进入 `running`/`judging` 或已有 claim 标记的任务返回 409，不得被 move/merge 回写成 queued。合并会移动可迁移任务归属并自动删除源 queue 记录，只保留合并后的目标 queue；若 source 或 target queue 存在 active/claimed 任务，合并整体返回 409。合并后的目标 queue 按任务原 `queueEnteredAt`/`createdAt` 时间顺序串行，成功迁移 queued/retry_wait 任务后由 D601 scheduler 轮询推进。 +- Code Queue 多队列 lane 由 `codex` 命令命名空间管理：`queues [--full|--all] [--limit N] [--page N|--offset N]` 列表、`queue create ` 创建、`queue merge --into ` 合并、`move --queue ` 迁移；这些队列管理入口默认由主 server `code-queue-mgr` 直管 PostgreSQL，仍通过稳定 `code-queue` 用户服务代理路径访问。`codex queues` 默认只返回 active/nonempty/unread/runnable queue 摘要、activity、全局 counts 和 execution diagnostics；`--full` 或 `--all` 只切换为完整队列行视图的一页，仍受 `--limit`/`--page`/`--offset` 分页约束，不再默认携带 deprecated full array。summary 和 full 的稳定机读路径都是 `.data.queues.items[]`，全局元数据固定在 `.data.queues.activity`、`.data.queues.counts`、`.data.queues.executionDiagnostics`、`.data.queues.activeTaskIds` 和 `.data.queues.queuedTaskIds`；需要完整 upstream 时使用输出中的 raw command。`activity.effectiveActiveTaskCount` 是指挥官并发判断的有效活跃数，`schedulerLocalActiveQueueCount`/`activeQueueIds` 只描述本地 scheduler active-run slots，不能覆盖数据库 running 计数或 heartbeat-fresh runner 计数。旧 full 顶层数组语义已作为 deprecated 兼容信息记录，不再作为 `.data.queues` 主形态。同一个 queue 内部串行执行，不同 queue 之间并行执行。迁移只允许尚未被 scheduler claim 的 `queued`/`retry_wait` 任务，必须满足 `startedAt=null`、`currentAttempt=0` 且没有 active thread/turn；已进入 `running`/`judging` 或已有 claim 标记的任务返回 409，不得被 move/merge 回写成 queued。合并会移动可迁移任务归属并自动删除源 queue 记录，只保留合并后的目标 queue；若 source 或 target queue 存在 active/claimed 任务，合并整体返回 409。合并后的目标 queue 按任务原 `queueEnteredAt`/`createdAt` 时间顺序串行，成功迁移 queued/retry_wait 任务后由 D601 scheduler 轮询推进。 - 所有 `codex` 查询和管理命令必须走与 WebUI 相同的 backend-core 私有代理路径 `/api/microservices/code-queue/proxy/...`；CLI 不得为了提交、移动、中断、取消或队列管理直接调用 D601 内部 Service、数据库、pod curl 或 k3sctl scheduler 子服务。若该路径失败，应先修复 CLI/backend/provider tunnel 链路，而不是绕过控制面。 - `job list [--limit N] [--include-command]` 与 `job status [--tail-bytes N]` 查询 `.state/jobs/` 文件系统状态，是异步命令的可观测入口。`job list` 默认只返回最新 50 条摘要；`job status` 默认只返回 stdout/stderr 末尾 12000 字节，并带 `tailPolicy` 与完整日志路径。 - `debug health`、`debug dispatch` 与 `debug task` 走真实内部 core、WebSocket、数据库、provider、系统指标、Docker 状态和 Host SSH 维护桥流程，只用于开发调试，不写入 `TEST.md` 的正式验收步骤。 diff --git a/docs/reference/code-queue-supervision.md b/docs/reference/code-queue-supervision.md index 207a63f8..b6973d82 100644 --- a/docs/reference/code-queue-supervision.md +++ b/docs/reference/code-queue-supervision.md @@ -220,8 +220,8 @@ bun scripts/cli.ts codex pr-preflight --remote --issue 常用入口： -- `bun scripts/cli.ts codex tasks --view supervisor --limit N`：查看默认低噪声监督视图，包括 running、完成未读、少量最近完成、queued/runnable、execution diagnostics、任务分类和下一步 drill-down 命令。默认行只保留 task id、队列、短 prompt/body 预览和原始字符数；`--limit` 是扫描/分页预算，不是返回几十条肥行的开关；`show/detail/trace/output/full/read` 放在 section template 中，避免每条任务重复刷屏，需要更多内容再按 taskId 展开。 -- `bun scripts/cli.ts codex queues`：查看低噪声队列计数、active task id、完成未读队列、runnable 队列和控制面诊断；需要完整队列行视图时加 `--full`，但 `--full` 仍默认分页，继续用 `--limit N`、`--page N` 或 `--offset N` 渐进展开。summary 和 full 都使用稳定 JSON path `.data.queues.items[]` 读取队列行，并从 `.data.queues.counts` 与 `.data.queues.executionDiagnostics` 读取全局计数和执行诊断；完整 upstream 只通过输出中的 raw command 显式获取。 +- `bun scripts/cli.ts codex tasks --view supervisor --limit N`：查看默认低噪声监督视图，包括 running、完成未读、少量最近完成、queued/runnable、activity、execution diagnostics、任务分类和下一步 drill-down 命令。默认行只保留 task id、队列、短 prompt/body 预览和原始字符数；`--limit` 是扫描/分页预算，不是返回几十条肥行的开关；`show/detail/trace/output/full/read` 放在 section template 中，避免每条任务重复刷屏，需要更多内容再按 taskId 展开。 +- `bun scripts/cli.ts codex queues`：查看低噪声队列计数、activity、active task id、完成未读队列、runnable 队列和控制面诊断；需要完整队列行视图时加 `--full`，但 `--full` 仍默认分页，继续用 `--limit N`、`--page N` 或 `--offset N` 渐进展开。summary 和 full 都使用稳定 JSON path `.data.queues.items[]` 读取队列行，并从 `.data.queues.activity`、`.data.queues.counts` 与 `.data.queues.executionDiagnostics` 读取全局活跃计数和执行诊断；完整 upstream 只通过输出中的 raw command 显式获取。 - `bun scripts/cli.ts codex tasks --unread --limit N`：查看完成未读审阅积压；`--unread` 与 `--unread-only` 等价，不能被静默忽略。 - `bun scripts/cli.ts codex tasks --status succeeded --unread --limit N`：按具体终态过滤监督结果；不支持的 status filter 必须显式失败，不能扩大为未过滤结果。 - `bun scripts/cli.ts codex task `：默认只查看原始 prompt、最终 response、最后错误和 drill-down 命令，这是完成未读任务审阅的第一步。 @@ -238,6 +238,8 @@ bun scripts/cli.ts codex pr-preflight --remote --issue 队列诊断中的 `split-brain` 表示控制面/执行面观测分裂，不自动证明任务已经死亡。只要任务 heartbeat 还在刷新、trace 仍在推进，就不能把它判成服务中断或要求立刻 stop；应把它视为 `splitBrainLive=true` 的 live 任务，继续监督并推进 #20 里的已排任务，而不是 interrupt、替换或把 backend 当成已经挂掉。队列摘要应显示 `effectiveLiveness=live`、`splitBrainLive=true` 和 `recommendedAction=continue-supervision`；compact 输出还应在 `executionDiagnostics.liveness` 中重复这些低噪声字段，并突出 `activeHeartbeatCount`、有界 `heartbeatFreshTaskIds`、`databaseActiveTaskCount` 和 `schedulerActiveRunSlotCount`。当 master/control-plane 的 `schedulerActiveRunSlotCount=0` 但 `heartbeatFreshTaskIds` 非空时，active 数应优先按 scheduler heartbeat 摘要解释为 live，而不是按 master 本地 slot 0 解释为执行停摆。只有 heartbeat expired/missing 或满足 stale-recovery 条件时，才应显示 `effectiveLiveness=at-risk` 并进入恢复判断。 +`codex queues` 和默认 supervisor 视图的 `activity` 是指挥官并发治理的主读数。`effectiveActiveTaskCount` 表示用于调度判断的有效活跃任务数；`databaseRunningTaskCount` 来自 PostgreSQL 中 `running` 状态计数；`databaseActiveTaskCount` 覆盖 running/judging 等数据库活跃任务；`heartbeatFreshActiveTaskCount` 表示 heartbeat-fresh 的有效 runner 数；`schedulerLocalActiveQueueCount` 和 `schedulerLocalActiveRunSlotCount` 只表示当前控制面本地可见 active run slots。`activeQueueIds` 与 `activeQueueCount` 是 scheduler-local 字段，可能在 `counts.running>0` 且 heartbeat 新鲜时为 0；看到这种组合时应按 `activity.effectiveActiveTaskCount`、`activity.heartbeatFreshActiveTaskCount` 和 `splitBrainLive` 决策，不得把空 `activeQueueIds` 当作零并发或停摆证据。 + 单次 `provider is not online`、SSH 超时、proxy 超时或 registry 请求失败只能证明“当前观察路径失败”，不能单独升级为 D601 全局离线、CI/CD 全局阻塞或业务任务不可推进。指挥官和 runner 必须用多信号裁决运行面状态，至少区分以下观察面： - backend-core 节点视图和 provider heartbeat，例如 `debug health` 中的 D601 `status`、`lastHeartbeat` 和 gateway 版本； diff --git a/scripts/code-queue-queues-shape-contract-test.ts b/scripts/code-queue-queues-shape-contract-test.ts index a414c971..42e8a1c1 100644 --- a/scripts/code-queue-queues-shape-contract-test.ts +++ b/scripts/code-queue-queues-shape-contract-test.ts @@ -16,6 +16,10 @@ function asArray(value: unknown): unknown[] { return value as unknown[]; } +function manyIds(prefix: string, count: number): string[] { + return Array.from({ length: count }, (_, index) => `${prefix}-${String(index + 1).padStart(2, "0")}`); +} + function fixtureResponse(): JsonRecord { return { ok: true, @@ -76,6 +80,66 @@ function fixtureResponse(): JsonRecord { }; } +function splitBrainLiveResponse(): JsonRecord { + const liveTaskIds = manyIds("task-running", 8); + return { + ok: true, + status: 200, + body: { + ok: true, + queue: { + total: 8, + queueCount: 2, + activeQueueIds: [], + activeTaskIds: [], + queuedTaskIds: [], + counts: { running: 8 }, + unreadTerminal: 0, + executionDiagnostics: { + state: "split-brain", + splitBrain: true, + splitBrainLive: true, + effectiveLiveness: "live", + recommendedAction: "continue-supervision", + databaseActiveTaskCount: 8, + databaseActiveTaskIds: liveTaskIds, + schedulerActiveRunSlotCount: 0, + schedulerActiveTaskIds: [], + activeHeartbeatCount: 8, + activeHeartbeatTaskIds: liveTaskIds, + heartbeatFreshTaskIds: liveTaskIds, + heartbeatExpiredTaskIds: [], + heartbeatMissingTaskIds: [], + staleRecoveryCandidateTaskIds: [], + heartbeatRiskTaskIds: [], + }, + }, + queues: [ + { + id: "alpha", + name: "Alpha", + total: 4, + counts: { running: 4 }, + unreadTerminal: 0, + activeTaskId: null, + runnableTaskId: null, + updatedAt: "2026-05-20T00:00:00.000Z", + }, + { + id: "beta", + name: "Beta", + total: 4, + counts: { running: 4 }, + unreadTerminal: 0, + activeTaskId: null, + runnableTaskId: null, + updatedAt: "2026-05-20T00:01:00.000Z", + }, + ], + }, + }; +} + function assertQueuesShape(label: string, result: unknown, expectedView: string): void { const data = asRecord(result); const queues = asRecord(data.queues); @@ -100,15 +164,51 @@ function assertQueuesShape(label: string, result: unknown, expectedView: string) assertCondition(liveness.schedulerActiveRunSlotCount === 0, `${label} liveness summary should keep master active slot zero visible`, liveness); assertCondition(asArray(liveness.heartbeatFreshTaskIds).length === 1, `${label} liveness summary should include bounded fresh heartbeat task ids`, liveness); assertCondition(String(liveness.interpretation ?? "").includes("heartbeat is fresh"), `${label} liveness interpretation should explain slot-zero split-brain`, liveness); + const activity = asRecord(queues.activity); + assertCondition(activity.effectiveActiveTaskCount === 1, `${label} activity should foreground effective active task count`, activity); + assertCondition(activity.databaseRunningTaskCount === 1, `${label} activity should distinguish database running tasks`, activity); + assertCondition(activity.heartbeatFreshActiveTaskCount === 1, `${label} activity should distinguish heartbeat-fresh active runners`, activity); + assertCondition(activity.schedulerLocalActiveQueueCount === 1, `${label} activity should distinguish scheduler-local active queues`, activity); + assertCondition(activity.activeQueueIdsScope === "scheduler-local-active-run-slots", `${label} activity should label activeQueueIds scope`, activity); assertCondition(Array.isArray(queues.activeTaskIds), `${label} activeTaskIds should be present`, queues); assertCondition(Array.isArray(queues.queuedTaskIds), `${label} queuedTaskIds should be present`, queues); } +function assertSplitBrainLiveActivity(label: string, result: unknown): void { + const queues = asRecord(asRecord(result).queues); + const totals = asRecord(queues.totals); + assertCondition(totals.activeQueueCount === 0, `${label} scheduler-local active queue count should be zero`, totals); + assertCondition(totals.schedulerLocalActiveQueueCount === 0, `${label} should preserve zero scheduler-local active queues`, totals); + assertCondition(totals.runnableQueueCount === 0, `${label} runnable queue count should be zero`, totals); + assertCondition(totals.databaseRunningTaskCount === 8, `${label} should foreground DB running task count`, totals); + assertCondition(totals.databaseActiveTaskCount === 8, `${label} should foreground DB active task count`, totals); + assertCondition(totals.heartbeatFreshActiveTaskCount === 8, `${label} should foreground heartbeat-effective active runners`, totals); + assertCondition(totals.effectiveActiveTaskCount === 8, `${label} should foreground effective active task count`, totals); + assertCondition(asArray(queues.activeQueueIds).length === 0, `${label} activeQueueIds should remain the scheduler-local list`, queues); + assertCondition(queues.activeQueueIdsScope === "scheduler-local-active-run-slots", `${label} activeQueueIds should be scoped`, queues); + assertCondition(String(queues.activeQueueIdsNote ?? "").includes("scheduler-local only"), `${label} activeQueueIds note should explain local-only semantics`, queues); + const activity = asRecord(queues.activity); + assertCondition(activity.effectiveActiveTaskCount === 8, `${label} activity should expose effective active count`, activity); + assertCondition(activity.effectiveActiveSource === "heartbeat-fresh", `${label} activity should choose heartbeat-fresh source`, activity); + assertCondition(activity.databaseRunningTaskCount === 8, `${label} activity should expose DB running count`, activity); + assertCondition(activity.heartbeatFreshActiveTaskCount === 8, `${label} activity should expose heartbeat-effective active count`, activity); + assertCondition(activity.schedulerLocalActiveQueueCount === 0, `${label} activity should expose scheduler-local queue count`, activity); + assertCondition(activity.schedulerLocalActiveRunSlotCount === 0, `${label} activity should expose scheduler-local slot count`, activity); + assertCondition(activity.runnableQueueCount === 0, `${label} activity should expose runnable queue count`, activity); + assertCondition(activity.splitBrainLive === true, `${label} activity should preserve split-brain live`, activity); + assertCondition(String(activity.activeQueueIdsNote ?? "").includes("zero local queue ids does not mean zero active runners"), `${label} activity note should prevent zero-active misread`, activity); + assertCondition(String(activity.interpretation ?? "").includes("continue supervision"), `${label} activity interpretation should keep supervision action`, activity); +} + export function runCodeQueueQueuesShapeContract(): JsonRecord { const fetcher = (path: string): JsonRecord => { assertCondition(path === "/api/microservices/code-queue/proxy/api/queues", "codex queues should use stable proxy path", { path }); return fixtureResponse(); }; + const splitBrainFetcher = (path: string): JsonRecord => { + assertCondition(path === "/api/microservices/code-queue/proxy/api/queues", "codex queues should use stable proxy path", { path }); + return splitBrainLiveResponse(); + }; const summary = codexQueuesQueryForTest([], fetcher); assertQueuesShape("summary", summary, "summary"); @@ -140,6 +240,11 @@ export function runCodeQueueQueuesShapeContract(): JsonRecord { assertCondition(offsetFullQueues.hasPrevious === true, "offset full should expose previous page", offsetFullQueues); assertCondition(asRecord(asArray(offsetFullQueues.items)[0]).id === "gamma", "offset full should return second page rows", offsetFullQueues); + const splitSummary = codexQueuesQueryForTest([], splitBrainFetcher); + assertSplitBrainLiveActivity("split-brain summary", splitSummary); + const splitFull = codexQueuesQueryForTest(["--full"], splitBrainFetcher); + assertSplitBrainLiveActivity("split-brain full", splitFull); + return { ok: true, checks: [ @@ -150,6 +255,7 @@ export function runCodeQueueQueuesShapeContract(): JsonRecord { "deprecated full array omitted from default output", "full explicit limit remains bounded and paged", "offset pagination", + "split-brain live activity counts distinguish scheduler-local queues, DB running tasks, and heartbeat-fresh runners", ], }; } diff --git a/scripts/code-queue-supervisor-disclosure-contract-test.ts b/scripts/code-queue-supervisor-disclosure-contract-test.ts index e6f87b14..25384eb5 100644 --- a/scripts/code-queue-supervisor-disclosure-contract-test.ts +++ b/scripts/code-queue-supervisor-disclosure-contract-test.ts @@ -174,11 +174,64 @@ function manyRunningFixtureResponse(path: string): JsonRecord { }; } +function splitBrainLiveSupervisorFixtureResponse(path: string): JsonRecord { + if (path.includes("/summary")) return fixtureResponse(path); + assertCondition(path.startsWith("/api/microservices/code-queue/proxy/api/tasks/overview"), "unexpected path", { path }); + const liveTaskIds = manyIds("split-live", 8); + const tasks = liveTaskIds.map((taskId, index) => task( + taskId, + "running", + `2026-05-22T01:${String(50 - index).padStart(2, "0")}:00.000Z`, + )); + return { + ok: true, + status: 200, + body: { + ok: true, + queue: { + counts: { running: 8 }, + activeQueueIds: [], + activeTaskIds: [], + activeRunSlotCount: 0, + databaseActiveTaskCount: 8, + executionDiagnostics: { + state: "split-brain", + splitBrain: true, + splitBrainLive: true, + effectiveLiveness: "live", + recommendedAction: "continue-supervision", + databaseActiveTaskCount: 8, + databaseActiveTaskIds: liveTaskIds, + schedulerActiveRunSlotCount: 0, + schedulerActiveTaskIds: [], + activeHeartbeatCount: 8, + activeHeartbeatTaskIds: liveTaskIds, + heartbeatFreshTaskIds: liveTaskIds, + heartbeatExpiredTaskIds: [], + heartbeatMissingTaskIds: [], + staleRecoveryCandidateTaskIds: [], + heartbeatRiskTaskIds: [], + }, + }, + pagination: { + limit: 200, + returned: 8, + total: 8, + hasMore: false, + nextBeforeId: null, + includeActive: true, + }, + tasks, + }, + }; +} + export function runCodeQueueSupervisorDisclosureContract(): JsonRecord { const supervisor = codexTasksQueryForTest(["--view", "supervisor", "--limit", "20"], fixtureResponse); const full = codexTasksQueryForTest(["--view", "full", "--limit", "20"], fixtureResponse); const runningFiltered = codexTasksQueryForTest(["--status", "running", "--limit", "40"], manyRunningFixtureResponse); const unreadFiltered = codexTasksQueryForTest(["--unread", "--limit", "20"], fixtureResponse); + const splitBrainLive = codexTasksQueryForTest(["--view", "supervisor", "--limit", "20"], splitBrainLiveSupervisorFixtureResponse); const supervisorBody = JSON.stringify(supervisor); const fullBody = JSON.stringify(full); @@ -200,6 +253,9 @@ export function runCodeQueueSupervisorDisclosureContract(): JsonRecord { const diagnostics = asRecord(supervisorView.executionDiagnostics); const listBudget = asRecord(diagnostics.listBudget); const omittedCounts = asRecord(listBudget.omittedCounts); + const splitBrainLiveView = asRecord(asRecord(splitBrainLive).supervisor); + const splitBrainLiveActivity = asRecord(splitBrainLiveView.activity); + const splitBrainLiveCounts = asRecord(splitBrainLiveView.counts); assertCondition(supervisorBody.length < fullBody.length * 0.55, "supervisor output should be materially smaller than full output", { supervisorChars: supervisorBody.length, fullChars: fullBody.length }); assertCondition(supervisorBody.length < 45_000, "supervisor output should remain bounded even with large diagnostics", { supervisorChars: supervisorBody.length }); @@ -237,6 +293,20 @@ export function runCodeQueueSupervisorDisclosureContract(): JsonRecord { assertCondition(runningFilteredBody.length < 14_000, "running status filter output should remain bounded", { chars: runningFilteredBody.length }); assertCondition(asArray(unreadFilteredSection.items).length <= 3, "unread list should be locally paged below --limit", unreadFilteredSection); assertCondition(unreadFilteredBody.length < 14_000, "unread output should remain bounded", { chars: unreadFilteredBody.length }); + assertCondition(splitBrainLiveCounts.running === 8, "split-brain supervisor should preserve DB running task count", splitBrainLiveCounts); + assertCondition(splitBrainLiveCounts.effectiveActive === 8, "split-brain supervisor should foreground effective active count", splitBrainLiveCounts); + assertCondition(splitBrainLiveCounts.databaseRunning === 8, "split-brain supervisor should distinguish database running tasks", splitBrainLiveCounts); + assertCondition(splitBrainLiveCounts.heartbeatFreshActive === 8, "split-brain supervisor should distinguish heartbeat-effective active runners", splitBrainLiveCounts); + assertCondition(splitBrainLiveCounts.schedulerLocalActiveQueues === 0, "split-brain supervisor should preserve zero scheduler-local active queues", splitBrainLiveCounts); + assertCondition(splitBrainLiveActivity.effectiveActiveTaskCount === 8, "split-brain supervisor activity should expose effective active count", splitBrainLiveActivity); + assertCondition(splitBrainLiveActivity.effectiveActiveSource === "heartbeat-fresh", "split-brain supervisor activity should prefer heartbeat-fresh source", splitBrainLiveActivity); + assertCondition(splitBrainLiveActivity.databaseRunningTaskCount === 8, "split-brain supervisor activity should expose DB running count", splitBrainLiveActivity); + assertCondition(splitBrainLiveActivity.heartbeatFreshActiveTaskCount === 8, "split-brain supervisor activity should expose heartbeat-fresh active count", splitBrainLiveActivity); + assertCondition(splitBrainLiveActivity.schedulerLocalActiveQueueCount === 0, "split-brain supervisor activity should expose scheduler-local queue count", splitBrainLiveActivity); + assertCondition(splitBrainLiveActivity.schedulerLocalActiveRunSlotCount === 0, "split-brain supervisor activity should expose scheduler-local slot count", splitBrainLiveActivity); + assertCondition(splitBrainLiveActivity.splitBrainLive === true, "split-brain supervisor activity should mark live split-brain", splitBrainLiveActivity); + assertCondition(String(splitBrainLiveActivity.activeQueueIdsNote ?? "").includes("zero local queue ids does not mean zero active runners"), "split-brain supervisor activity should explain activeQueueIds are local-only", splitBrainLiveActivity); + assertCondition(String(splitBrainLiveActivity.interpretation ?? "").includes("continue supervision"), "split-brain supervisor activity should not imply scheduler stoppage", splitBrainLiveActivity); return { ok: true, @@ -249,6 +319,7 @@ export function runCodeQueueSupervisorDisclosureContract(): JsonRecord { "running finalResponse rows labeled awaiting terminal/judge", "drill-down commands preserved", "full view remains detailed", + "split-brain live supervisor activity distinguishes scheduler-local, database, and heartbeat counts", ], supervisorChars: supervisorBody.length, fullChars: fullBody.length, diff --git a/scripts/src/code-queue.ts b/scripts/src/code-queue.ts index abbff91d..1226694e 100644 --- a/scripts/src/code-queue.ts +++ b/scripts/src/code-queue.ts @@ -433,6 +433,10 @@ function asNumber(value: unknown, fallback = 0): number { return typeof value === "number" && Number.isFinite(value) ? value : fallback; } +function finiteNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + function asBoolean(value: unknown): boolean { return value === true || value === 1 || value === "1" || value === "true"; } @@ -1304,6 +1308,100 @@ function compactQueueExecutionDiagnostics(value: unknown): Record, + diagnostics: Record | null, + options: { schedulerLocalActiveQueueIds?: string[]; runnableQueueCount?: number | null } = {}, +): Record { + const rawDiagnostics = asRecord(queue.executionDiagnostics) ?? {}; + const compactDiagnostics = diagnostics ?? {}; + const rawLiveness = asRecord(rawDiagnostics.liveness) ?? {}; + const compactLiveness = asRecord(compactDiagnostics.liveness) ?? {}; + const counts = asRecord(queue.counts) ?? {}; + const schedulerLocalActiveQueueIds = options.schedulerLocalActiveQueueIds ?? stringList(queue.activeQueueIds); + const databaseRunningTaskCount = firstFiniteNumber(counts.running) ?? 0; + const databaseJudgingTaskCount = firstFiniteNumber(counts.judging) ?? 0; + const databaseActiveTaskCount = firstFiniteNumber( + rawDiagnostics.databaseActiveTaskCount, + queue.databaseActiveTaskCount, + compactDiagnostics.databaseActiveTaskCount, + stringListCount(rawDiagnostics.databaseActiveTaskIds), + stringListCount(queue.databaseActiveTaskIds), + databaseRunningTaskCount + databaseJudgingTaskCount, + ) ?? 0; + const heartbeatFreshActiveTaskCount = firstFiniteNumber( + rawLiveness.heartbeatFreshTaskCount, + compactLiveness.heartbeatFreshTaskCount, + stringListCount(rawDiagnostics.heartbeatFreshTaskIds), + stringListCount(compactDiagnostics.heartbeatFreshTaskIds), + ) ?? 0; + const activeHeartbeatTaskCount = firstFiniteNumber( + rawDiagnostics.activeHeartbeatCount, + compactDiagnostics.activeHeartbeatCount, + stringListCount(rawDiagnostics.activeHeartbeatTaskIds), + heartbeatFreshActiveTaskCount, + ) ?? 0; + const heartbeatRiskTaskCount = firstFiniteNumber( + rawLiveness.heartbeatRiskTaskCount, + compactLiveness.heartbeatRiskTaskCount, + stringListCount(rawDiagnostics.heartbeatRiskTaskIds), + stringListCount(compactDiagnostics.heartbeatRiskTaskIds), + ) ?? 0; + const schedulerLocalActiveRunSlotCount = firstFiniteNumber( + rawDiagnostics.schedulerActiveRunSlotCount, + queue.schedulerActiveRunSlotCount, + queue.activeRunSlotCount, + compactDiagnostics.schedulerActiveRunSlotCount, + ); + const runnableQueueCount = firstFiniteNumber(options.runnableQueueCount, queue.runnableQueueCount); + const effectiveActiveTaskCount = Math.max(databaseActiveTaskCount, databaseRunningTaskCount, heartbeatFreshActiveTaskCount); + const splitBrainLive = splitBrainLiveFromDiagnostics(rawDiagnostics) || splitBrainLiveFromDiagnostics(compactDiagnostics); + const effectiveActiveSource = heartbeatFreshActiveTaskCount > 0 && heartbeatFreshActiveTaskCount >= databaseActiveTaskCount + ? "heartbeat-fresh" + : databaseActiveTaskCount > 0 + ? "database-active" + : schedulerLocalActiveQueueIds.length > 0 || (schedulerLocalActiveRunSlotCount ?? 0) > 0 + ? "scheduler-local" + : "none"; + const activeQueueIdsNote = schedulerLocalActiveQueueIds.length === 0 && effectiveActiveTaskCount > 0 + ? "activeQueueIds are scheduler-local only; zero local queue ids does not mean zero active runners when database or heartbeat counts are nonzero." + : "activeQueueIds are scheduler-local active-run slots; use effectiveActiveTaskCount for commander concurrency decisions."; + return { + effectiveActiveTaskCount, + effectiveActiveSource, + databaseRunningTaskCount, + databaseActiveTaskCount, + heartbeatFreshActiveTaskCount, + activeHeartbeatTaskCount, + heartbeatRiskTaskCount, + schedulerLocalActiveQueueCount: schedulerLocalActiveQueueIds.length, + schedulerLocalActiveRunSlotCount, + runnableQueueCount, + splitBrainLive, + activeQueueIdsScope: "scheduler-local-active-run-slots", + activeQueueIdsNote, + interpretation: splitBrainLive + ? "split-brain live: database-active tasks have fresh scheduler heartbeat; continue supervision." + : heartbeatRiskTaskCount > 0 + ? "heartbeat risk is present; investigate before retry or recovery." + : effectiveActiveTaskCount > 0 + ? "active work is present; compare database, heartbeat, and scheduler-local counts before changing concurrency." + : "no active work observed by database, heartbeat, or scheduler-local signals.", + }; +} + function supervisorExecutionDiagnostics(value: unknown): Record | null { const diagnostics = compactExecutionDiagnostics(value); if (diagnostics === null) return null; @@ -2355,6 +2453,7 @@ function codexTasksOverviewResult( const queuedSection = buildSupervisorTaskSection(queuedTasks, summaries, sectionLimit, sectionNextCommand(queuedTasks, sectionLimit, options, nextCommand), fullCommand); const pagination = taskPage.pagination; const diagnostics = supervisorExecutionDiagnostics(asRecord(taskPage.queue)?.executionDiagnostics); + const activity = compactCodeQueueActivity(asRecord(taskPage.queue) ?? {}, diagnostics); const visibleSupervisorItems = [...runningSection.items, ...unreadSection.items, ...recentSection.items, ...queuedSection.items]; const classifierCounts = visibleSupervisorItems.reduce((counts, item) => { const key = item.kind; @@ -2405,8 +2504,13 @@ function codexTasksOverviewResult( completedUnread: unreadSection.count, recentCompleted: recentSection.count, queued: queuedSection.count, + effectiveActive: asNumber(activity.effectiveActiveTaskCount, 0), + databaseRunning: asNumber(activity.databaseRunningTaskCount, 0), + heartbeatFreshActive: asNumber(activity.heartbeatFreshActiveTaskCount, 0), + schedulerLocalActiveQueues: asNumber(activity.schedulerLocalActiveQueueCount, 0), }, classifierCounts, + activity, executionDiagnostics: diagnostics, degraded, commands: { @@ -2698,6 +2802,7 @@ function compactQueuesResponse(body: Record, options: CodexQueu const selected = options.full ? queues : Array.from(new Map([...activeQueues, ...unreadQueues, ...runnableQueues, ...nonemptyQueues].map((row) => [String(row.id), row])).values()); const visible = selected.slice(options.offset, options.offset + options.limit); const diagnostics = compactQueueExecutionDiagnostics(queue.executionDiagnostics); + const activity = compactCodeQueueActivity(queue, diagnostics, { schedulerLocalActiveQueueIds: activeIds, runnableQueueCount: runnableQueues.length }); const activeTaskIds = boundedUniqueStringList(queue.activeTaskIds, Math.min(options.limit, maxTasksLimit)); const queuedTaskIds = boundedUniqueStringList(queue.queuedTaskIds, Math.min(options.limit, maxTasksLimit)); const nextOffset = options.offset + visible.length; @@ -2725,11 +2830,20 @@ function compactQueuesResponse(body: Record, options: CodexQueu totalTasks: queue.total ?? null, queueCount: queue.queueCount ?? queues.length, activeQueueCount: activeIds.length, + activeQueueCountScope: "scheduler-local-active-run-slots", + schedulerLocalActiveQueueCount: activeIds.length, + effectiveActiveTaskCount: activity.effectiveActiveTaskCount, + databaseRunningTaskCount: activity.databaseRunningTaskCount, + databaseActiveTaskCount: activity.databaseActiveTaskCount, + heartbeatFreshActiveTaskCount: activity.heartbeatFreshActiveTaskCount, nonemptyQueueCount: nonemptyQueues.length, unreadQueueCount: unreadQueues.length, runnableQueueCount: runnableQueues.length, }, + activity, activeQueueIds: queue.activeQueueIds ?? [], + activeQueueIdsScope: "scheduler-local-active-run-slots", + activeQueueIdsNote: activity.activeQueueIdsNote, activeTaskIds: activeTaskIds.items, activeTaskIdsCount: activeTaskIds.count, activeTaskIdsTruncated: activeTaskIds.truncated, diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 7cbc163e..c18247b7 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -56,14 +56,14 @@ export function rootHelp(): unknown { { command: "codex skills-sync --dry-run [--full]", description: "Inspect the controlled runner skills hostPath lifecycle contract without copying files, restarting services, reading secrets, or mutating live runner paths." }, { command: "codex pr-preflight [--remote] [--push-dry-run --push-dry-run-ref refs/heads/probe/] [--pr-create-dry-run --pr-create-dry-run-head ] [--issue N]", description: "Read-only PR admission check against the D601 scheduler/runner token, GitHub egress, repo visibility, skills lifecycle health, optional push dry-run, and PR body/create dry-run guard." }, { command: "codex task [--detail] [--trace --tail|--from-start|--after-seq N|--before-seq N --limit N] [--full]", description: "Fetch the bounded review view by default; --detail is still capped, while --full/trace/output explicitly expand evidence." }, - { command: "codex tasks [--view supervisor|full] [--queue id] [--status status[,status]] [--unread|--unread-only] [--limit N] [--before-id id]", description: "Show the low-noise supervisor view by default: compact task rows, tiny local sections, diagnostics, and drill-down commands; use --view full for detailed rows." }, + { command: "codex tasks [--view supervisor|full] [--queue id] [--status status[,status]] [--unread|--unread-only] [--limit N] [--before-id id]", description: "Show the low-noise supervisor view by default: compact task rows, tiny local sections, activity counts, diagnostics, and drill-down commands; use --view full for detailed rows." }, { command: "codex output [--tail|--from-start|--after-seq N|--before-seq N --limit N] [--full-text]", description: "Fetch paged raw Code Queue output records; default caps large limits/text previews, --full-text explicitly expands one seq window." }, { command: "codex read ", description: "Mark one reviewed terminal task read; never run automatically as part of listing." }, { command: "codex dev-ready", description: "Fetch execution-container readiness, including sanitized skill injection status from /api/dev-ready." }, { command: "codex judge --attempt N [--dry-run] [--include-prompt]", description: "Replay one stored Code Queue attempt through the same judge context builder and MiniMax judge call path used by the live queue worker." }, { command: "codex steer [prompt|--prompt-file path|--prompt-stdin] [--dry-run] [--no-retry|--retry-attempts N]", description: "Push a corrective prompt into a running Code Queue task; retryable tunnel aborts get bounded retry diagnostics, and real success does not echo prompt text." }, { command: "codex interrupt|cancel ", description: "Request interrupt for a running Code Queue task, or cancel a queued/retry_wait task, through the same private proxy." }, - { command: "codex (queues [--full|--all] | queue create | queue merge --into | move --queue )", description: "List low-noise queue summaries by default; full queue rows require --full/--all." }, + { command: "codex (queues [--full|--all] | queue create | queue merge --into | move --queue )", description: "List low-noise queue summaries by default, including effective activity counts that distinguish scheduler-local queues, DB running tasks, and heartbeat-fresh runners; full queue rows require --full/--all." }, { command: "job list [--limit N] [--include-command]", description: "List async jobs from .state/jobs with a bounded default page." }, { command: "job status [--tail-bytes N]", description: "Show job state with bounded stdout/stderr tails." }, { command: "debug health", description: "Probe internal core, nodes, system/Docker status, frontend, provider ingress, and public boundary." }, @@ -284,7 +284,13 @@ function codexHelp(): unknown { defaultPolicy: "low-noise JSON by default; write commands confirm persistence, list/detail/output commands return bounded summaries with drill-down commands", expand: ["codex task --full", "codex task --trace --limit N", "codex output --after-seq N --limit N --full-text", "codex tasks --view full --limit N", "codex skills-sync --dry-run --full"], }, - description: "Operate Code Queue through the stable backend-core private proxy path. Real submit/steer success is a low-noise write confirmation and does not echo prompt text.", + activityFields: { + path: "data.queues.activity and data.supervisor.activity", + effectiveActiveTaskCount: "Commander-facing active count derived from database active/running tasks and heartbeat-fresh runners.", + schedulerLocalActiveQueueCount: "Only queues currently visible in this scheduler-local active-run slot view; zero does not override DB or heartbeat activity.", + heartbeatFreshActiveTaskCount: "Heartbeat-effective active runner count used to avoid split-brain zero-active mistakes.", + }, + description: "Operate Code Queue through the stable backend-core private proxy path with bounded activity summaries for queue and supervisor views. Real submit/steer success is a low-noise write confirmation and does not echo prompt text.", }; }