Merge pull request #80 from pikasTech/code-queue/issue-20-host-commander-governance-docs

fix: compact code queue health output
This commit is contained in:
Lyon
2026-05-23 15:18:10 +08:00
committed by GitHub
7 changed files with 431 additions and 12 deletions
+2 -2
View File
@@ -23,7 +23,7 @@ CLI 可以从 `master` 快速演进,但必须兼容 `deploy.json` 固定的 CI
- `ssh <providerId> apply-patch [tool args...] < patch.diff` 直接调用远端注入的 `apply_patch` 工具,并把本地 stdin 中的标准 `*** Begin Patch` / `*** End Patch` patch 流透传给目标节点。
- `ssh <providerId> py [script-args...] < script.py` 把本地 stdin 落到远端临时 `.py` 文件后再以 `python3 -u` 执行并自动清理,避免再手写 `'python3 -'`、heredoc 或多层引号;`script-args` 会按 argv 安全透传给远端脚本。
- `ssh <providerId> skills [--scope all|wsl|windows] [--limit N]` 发现目标节点上的 WSL/Linux skill 根目录;当 provider 是 WSL 时同一次调用还会扫描 Windows 用户目录下的 `.agents/skills``.codex/skills`
- `microservice list/status/health/diagnostics/tunnel-self-test/proxy` 通过 backend-core 内网 API 管理挂载在计算节点 Docker 或 k3s 控制面中的用户服务(底层命令名仍为 microservice);`health``status``diagnostics` 默认返回 compact summary、body 字节数和 `--full|--raw` 展开命令,只有小 body 或无法抽取 summary 时才带有界 body preview,避免 Code Queue/k3s 诊断一次性输出爆炸;`tunnel-self-test``proxy` 会走真实 backend-core -> provider-gateway 或 k3sctl-adapter -> 节点服务链路`proxy` 支持受控 JSON 请求体并对超大响应 body 默认输出有界预览,规则见 `docs/reference/microservices.md`
- `microservice list/status/health/diagnostics/tunnel-self-test/proxy` 通过 backend-core 内网 API 管理挂载在计算节点 Docker 或 k3s 控制面中的用户服务(底层命令名仍为 microservice);`health``status``diagnostics` 默认返回 compact summary、body 字节数和 `--full|--raw` 展开命令,只有小 body 或无法抽取 summary 时才带有界 body preview,避免 Code Queue/k3s 诊断一次性输出爆炸;`tunnel-self-test``proxy` 会走真实 backend-core -> provider-gateway 或 k3sctl-adapter -> 节点服务链路`microservice health code-queue` 使用 commander-safe 专用摘要,必须保留 ok/status、service id、running count、queue count、heartbeat freshness/risk、split-brain/live/degraded 解释和 raw drill-down 命令;需要完整健康 JSON 时显式加 `--raw``--full`,等价深挖路径是 `microservice proxy code-queue /health --raw --full``proxy` 支持受控 JSON 请求体并对超大响应 body 默认输出有界预览,规则见 `docs/reference/microservices.md`
- `decision upload/list/show/health` 通过 backend-core 用户服务代理访问 D601 k3s Decision Center,用于上传会议记录/决议 Markdown、列出权威记录、查看详情和健康检查;`decision list` 默认只返回摘要并省略完整 Markdown body,需要排查大正文时显式加 `--include-body`。正式文书字段通过 records 模型一等字段返回和查询:`--doc-no DC-...``--doc-type DCSN|GOAL|PLAN|RPRT|ACTN|ISSU|RETR|RQST|RESP|MINS``--doc-priority P0|P1|P2|P3``--year YYYY``--signer``--issued-at``--effective-scope``--supersedes``--superseded-by``show``requirement update` 可使用 `id``docNo``decision requirement list/create/upsert/update/show` 在同一 records 模型上管理 `goal|decision|blocker|debt|experiment` 需求记录,`docNo` 唯一,未传 `--doc-no` 但提供 `--doc-type/--doc-priority/--year` 时由服务分配下一个序号。它们不得直连 D601 Service、NodePort 或 provider-gateway 业务 HTTP。
- `decision diary import <markdown-file>` 将带 `# YYYY年M月D日``# YYYY-MM-DD``# YYYY/M/D` 标题的工作日志拆成每天一篇 Markdown 日记,按 `YYYY-MM/YYYY-MM-DD.md` 虚拟路径写入 Decision Center PostgreSQL`decision diary list/history` 默认只返回摘要,需要完整 Markdown 时显式加 `--include-body``decision diary show <YYYY-MM-DD|id> [--source-file path]` 查看单日正文,`--source-file` 用于同一天存在多个导入来源时精确选择;`decision diary edit|upsert <YYYY-MM-DD|id> --body-file <path> [--title text] [--source-file path] [--tag tag]` 通过 `PUT /api/diary/entries/:idOrDate` 创建当天或历史条目并编辑既有条目。
- `deploy check/plan/apply` 默认从根目录 `deploy.json` 读取服务 repo 与 commit 期望状态,join `config.json` 和现有 manifest 后使用 target-side build 或 reviewed artifact consumer 校验/更新已支持目标;`deploy plan --env dev|prod` 只从 `origin/master:deploy.json#environments.<env>` 读取 manifest 并输出 dry-run 环境计划,不使用本地 dirty worktree;当前 `deploy apply --env dev` 支持 `backend-core``frontend``baidu-netdisk``decision-center``mdtodo``claudeqq` 和 dev-only `code-queue` artifact consumers`findjob`/`pipeline`/`met-nonlinear` 为 D601 direct Compose artifact consumers`k3sctl-adapter` 只提供 plan/dry-rundev desired-state smoke 使用 `ci run-dev-e2e`;规则见 `docs/reference/deploy.md``docs/reference/dev-environment.md``docs/reference/dev-ci-runner.md``deploy apply --env prod` 同时覆盖 `findjob``pipeline` 的 pull-only Compose CD,但 `met-nonlinear` 仍然只允许 dry-run/plan`k3sctl-adapter` 只允许 plan/dry-run。
@@ -74,7 +74,7 @@ CLI 可以从 `master` 快速演进,但必须兼容 `deploy.json` 固定的 CI
每条命令的最外层 JSON 包含 `ok``command``data``error`。失败时 CLI 设置非零退出码,但仍然输出 JSON 错误对象;错误对象应包含 `name``message` 和可用的 `stack`
诊断命令默认采用渐进披露:`server logs``job list/status``codex task/trace/output``microservice proxy` 都必须有默认条数、字节数或文本预览上限;用户显式传 `--limit``--tail-bytes``--full-text``--full` 才扩大单次输出。CLI stdout 遇到下游 pipe 关闭的 `EPIPE` 必须安静退出,不得打印 Bun stack trace。
诊断命令默认采用渐进披露:`server logs``job list/status``codex task/trace/output``microservice health code-queue``microservice proxy` 都必须有默认条数、字节数或文本预览上限;用户显式传 `--limit``--tail-bytes``--full-text``--raw``--full` 才扩大单次输出。CLI stdout 遇到下游 pipe 关闭的 `EPIPE` 必须安静退出,不得打印 Bun stack trace。
`microservice proxy` 是面向人工验证和受控调试的私有后端入口。默认 method 为 GET;使用 `--body-json JSON``--body-file path``--body-stdin` 时默认 method 切换为 POST,也可显式加 `--method POST|PUT|PATCH|DELETE`,但 GET/HEAD 不允许携带请求体。所有请求仍受 config 中的 `allowedMethods``allowedPathPrefixes` 限制。为了避免 Pipeline snapshot 这类超大业务 JSON 造成 CLI 输出爆炸,响应 body 超过默认阈值时会返回 `bodyOmitted=true``bodyPreview``bodyBytes``rawHint``--raw` 仍受默认硬限额保护,需要完整 body 时显式添加 `--raw --full`,或用 `--max-body-bytes <N>` 调整预览阈值。正式 frontend 展示仍应优先使用业务控件和 `__unideskArrayLimit` 这类展示级裁剪参数,而不是默认倾倒完整 JSON。
+4
View File
@@ -30,6 +30,8 @@
当 HWLAB 是当前优先级最高的推进对象时,所有派单 prompt 的开头都必须显式引用 `DC-DCSN-P0-2026-003`,并说明任务究竟服务于 M3 虚拟硬件可信闭环的哪一环。如果任务只是在做发布前置、诊断、证据收口或文档治理,也必须明确写出它不抢占 M3 P0。
HWLAB 业务目标、验收和实现优先级归 `pikasTech/HWLAB#7`;UniDesk 指挥治理、队列监督、并发窗口、PR handoff 和 runner 能力归 `pikasTech/unidesk#20`#20 可以记录 HWLAB #7 的调度状态和证据链接,但不能替代 HWLAB 业务 issue。
审阅 HWLAB runner 输出时,不能把 `SOURCE``LOCAL``DRY-RUN`、fixture 或只读报告误当成 `DEV-LIVE`。除非输出真的证明了 `res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1` 的真链路,并且带有 operation / audit / evidence 关联,否则只能归类为 support、diagnostics 或 contract。
`split-brain live` 且 heartbeat/trace 新鲜时,指挥官必须继续监督,不把它当作服务中断。此类状态的优先动作是继续轮询、继续审阅、继续派单,而不是默认 interrupt 或 cancel。
@@ -140,6 +142,8 @@ CLI 是短 shout 的需求原语,不是长驻服务器进程。CLI 功能不
PR 是审查型交付入口,不是所有 Code Queue 任务的默认出口。默认 master-only 交付仍按项目 Git 规则执行;当变更风险高、跨模块、需要人工审查、或任务目标明确要求 PR 交付时,worker 可以创建 PR。PR 型任务必须报告源分支、目标分支、PR URL、关联 issue、测试证据和未完成风险。禁止把 PR 当成隐藏分支仓库;PR 分支必须来自最新目标线,保持小而可审查,并在合并后确认目标分支远端 commit 可 fetch。
PR handoff 的职责固定分开:runner 实现、测试、提交、push head branch 并创建 PR;指挥官监督并发、steer、审阅、确认 checks 和合并裁决。runner 不合并自己的 PRhost commander 也不把直接编辑业务代码当成常规 PR 替代路径。
PR 支持本身是 Code Queue 能力的一部分。当前 UniDesk CLI 支持 `gh pr list|view|create|update|comment create|comment delete|close|reopen`,其中 create 需要显式 `--title``--base``--head` 和正文来源,update 需要显式 PR number、正文来源和 `--mode replace|append`comment create 需要显式 PR number 和正文来源,且推荐使用 `--body-file`。PR 收口观察应使用 `gh pr view <number> --json headRefName,baseRefName,mergeable,mergeStateStatus,statusCheckRollup` 获取目标分支、源分支、mergeability 和检查汇总;该路径仍是只读元数据,不执行 merge。`pr create --dry-run``pr update --dry-run``pr comment create --dry-run` 只返回 planned operation,不创建 PR、不更新正文、不写评论;非 dry-run 创建前会校验 repo、base、head 和 compare ahead 状态,append 更新会先读取当前 PR body。`gh pr delete``gh pr merge` 仍是后续范围,本阶段没有 `--confirm` 可以启用真实 merge,也不能伪造硬删除;需要移除活跃 PR 时使用 `gh pr close`。普通 worker 不应隐式依赖未实现的 PR 合并能力;需要 PR 交付时,prompt 必须明确允许的人工或后续工具路径,并报告未覆盖范围。
### PR 驱动派单模板
+8
View File
@@ -46,6 +46,14 @@ ls -t /root/unidesk/logs/commander.log.*.jsonl | head -n 1
该运行形态可以用于指挥 Code Queue、审阅任务、创建/更新 issue、PR 收口和必要的热修复例外,但不得被解释为普通 worker。涉及实现、测试和 PR 创建的常规工作仍应优先通过 Code Queue 派单;host commander 直接执行只适用于授权的 PR 收口、基础设施热修复或 runner 权限缺口,并且必须留下复盘和长期改进 issue。
## 指挥职责
host commander 的工作是调度、监督、steer、审阅和 PR 收口;它维护并发窗口、阻塞分类、issue/#20/#24 记录,并在 checks 和审阅通过后负责合并裁决或走已批准的合并路径。Code Queue runner 的工作是实现、验证、提交、push head branch 和创建 PRrunner 不合并自己的 PR。
HWLAB 业务目标、验收和优先级属于 `pikasTech/HWLAB#7`;UniDesk 指挥治理、队列监督、并发窗口和 runner/PR handoff 属于 `pikasTech/unidesk#20`#20 可以链接 HWLAB #7 的证据和状态,但不能替代 HWLAB 业务 issue,也不能把 HWLAB 代码实现决策写成 UniDesk 指挥规则。
host commander 不直接编辑 HWLAB 业务代码,不以本地热修绕过 HWLAB runner PR。只有用户明确授权的 PR 收口、基础设施热修或 runner 权限缺口可以例外;例外必须最小化变更,在相关 issue/PR 中留下命令、diff、验证证据和影响范围,并创建或更新 follow-up issue 消除该例外原因。
## Dev 验证计划
`commander smoke --dry-run` 是无 daemon smoke contract。它只输出验证计划,不启动 HTTP daemon、不打开 SSH/PTY/stdio bridge、不发送 ClaudeQQ、不重启服务、不 interrupt/cancel 任务、不部署、不跑全量 check/e2e。
@@ -0,0 +1,153 @@
import { summarizeMicroserviceHealthResponse } from "./src/microservices";
type JsonRecord = Record<string, unknown>;
function assertCondition(condition: unknown, message: string, detail: JsonRecord = {}): void {
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
}
function asRecord(value: unknown): JsonRecord {
assertCondition(typeof value === "object" && value !== null && !Array.isArray(value), "expected JSON object", { value });
return value as JsonRecord;
}
function manyIds(prefix: string, count: number): string[] {
return Array.from({ length: count }, (_, index) => `${prefix}-${String(index + 1).padStart(3, "0")}`);
}
function longText(prefix: string, count: number): string {
return Array.from({ length: count }, (_, index) => `${prefix} diagnostic row ${index + 1} with queue detail and commander-noise payload`).join("\n");
}
function largeCodeQueueHealth(): JsonRecord {
return {
ok: true,
status: 200,
body: {
ok: true,
service: "code-queue",
instanceId: "D601-code-queue-scheduler",
role: "scheduler",
deploy: {
commit: "abcdef1234567890",
requestedCommit: "abcdef1234567890",
},
databaseReady: true,
schedulerEnabled: true,
queue: {
total: 640,
counts: {
running: 3,
judging: 2,
queued: 41,
retry_wait: 4,
succeeded: 560,
failed: 30,
},
queues: manyIds("queue", 90).map((id) => ({
id,
name: id,
total: 12,
counts: { queued: 5, succeeded: 7 },
diagnostics: longText(id, 18),
})),
queueCount: 90,
activeRunSlotCount: 5,
activeTaskIds: manyIds("active-task", 12),
activeTaskId: "active-task-001",
activeQueueIds: manyIds("active-queue", 12),
processingQueueIds: manyIds("processing-queue", 8),
processing: true,
orphanedActiveTaskCount: 0,
unreadTerminal: 7,
defaultQueueId: "default",
defaultModel: "gpt-5.5",
defaultReasoningEffort: "xhigh",
maxActiveQueues: 0,
schedulerHeartbeatStaleMs: 120000,
},
executionDiagnostics: {
state: "split-brain",
degraded: true,
splitBrain: true,
splitBrainLive: true,
effectiveLiveness: "live",
recommendedAction: "continue-supervision",
executionStateSource: "postgres-control-plane",
controlPlane: "D601-code-queue-scheduler",
databaseActiveTaskCount: 12,
schedulerActiveRunSlotCount: 5,
activeHeartbeatCount: 12,
activeHeartbeatTaskIds: manyIds("active-heartbeat", 12),
heartbeatFreshTaskIds: manyIds("fresh-heartbeat", 12),
heartbeatExpiredTaskIds: [],
heartbeatMissingTaskIds: [],
staleRecoveryCandidateTaskIds: [],
heartbeatRiskTaskIds: [],
traceGapTaskIds: manyIds("trace-gap", 15),
lastSchedulerHeartbeatAt: "2026-05-22T00:00:12.000Z",
lastObservedAgentEventAt: "2026-05-22T00:00:11.000Z",
lastPersistedTraceAt: "2026-05-22T00:00:10.000Z",
reasons: Array.from({ length: 12 }, (_, index) => longText(`reason-${index + 1}`, 8)),
},
modelProviderConfig: {
source: "fixture",
largeNoise: longText("model-provider", 200),
},
},
};
}
const fixture = largeCodeQueueHealth();
const compact = asRecord(summarizeMicroserviceHealthResponse(fixture, ["health", "code-queue"], "code-queue"));
const raw = summarizeMicroserviceHealthResponse(fixture, ["health", "code-queue", "--raw"], "code-queue");
const full = summarizeMicroserviceHealthResponse(fixture, ["health", "code-queue", "--full"], "code-queue");
const unhealthyWrapper = asRecord(summarizeMicroserviceHealthResponse({
ok: false,
status: 503,
body: {
ok: false,
status: "unhealthy",
serviceId: "code-queue",
reason: "health body ok=false",
upstream: {
status: 503,
contentType: "application/json",
body: fixture.body,
},
},
}, ["health", "code-queue"], "code-queue"));
const compactBody = JSON.stringify(compact);
const rawBody = JSON.stringify(fixture);
const queue = asRecord(compact.queue);
const heartbeat = asRecord(compact.heartbeat);
const liveness = asRecord(compact.liveness);
const outputPolicy = asRecord(compact.outputPolicy);
const unhealthyQueue = asRecord(unhealthyWrapper.queue);
const unhealthyLiveness = asRecord(unhealthyWrapper.liveness);
assertCondition(compactBody.length < rawBody.length * 0.25, "default code-queue health output should be materially compact", { compactChars: compactBody.length, rawChars: rawBody.length });
assertCondition(compactBody.length < 12000, "default code-queue health output should stay commander-safe", { compactChars: compactBody.length });
assertCondition(!("body" in compact), "compact health must not retain the raw body", compact);
assertCondition(!("queues" in queue), "compact queue summary must not include full queue dumps", queue);
assertCondition(compact.ok === true && compact.status === 200 && compact.serviceId === "code-queue", "compact health keeps ok/status/service identity", compact);
assertCondition(queue.runningCount === 5 && queue.queueCount === 90, "compact queue keeps running count and queue count", queue);
assertCondition(heartbeat.heartbeatFreshTaskCount === 12 && heartbeat.heartbeatRiskTaskCount === 0, "compact heartbeat keeps freshness and risk counts", heartbeat);
assertCondition(liveness.effectiveLiveness === "live" && liveness.splitBrainLive === true, "compact liveness keeps split-brain live interpretation", liveness);
assertCondition(String(liveness.interpretation).includes("continue supervision"), "compact liveness includes commander interpretation", liveness);
assertCondition(typeof outputPolicy.rawCommand === "string" && String(outputPolicy.rawCommand).includes("--raw"), "compact output exposes raw drill-down command", outputPolicy);
assertCondition(raw === fixture && full === fixture, "--raw/--full must preserve full health access", { rawSame: raw === fixture, fullSame: full === fixture });
assertCondition(unhealthyWrapper.ok === false && unhealthyWrapper.reason === "health body ok=false", "unhealthy wrapper keeps probe failure", unhealthyWrapper);
assertCondition(unhealthyQueue.runningCount === 5 && unhealthyLiveness.effectiveLiveness === "live", "unhealthy wrapper still surfaces upstream queue liveness", { unhealthyQueue, unhealthyLiveness });
console.log(JSON.stringify({
ok: true,
checks: [
"default code-queue health output is compact and omits raw body/queue dumps",
"critical ok/status/service/running/heartbeat/liveness fields remain visible",
"unhealthy backend health wrapper still surfaces upstream code-queue liveness",
"--raw and --full return the original health response for diagnostics",
],
compactChars: compactBody.length,
rawChars: rawBody.length,
}, null, 2));
+1
View File
@@ -34,6 +34,7 @@ const syntaxFiles = [
"scripts/code-queue-cli-steer-test.ts",
"scripts/code-queue-cli-submit-prompt-contract-test.ts",
"scripts/code-queue-gh-auth-redaction-contract-test.ts",
"scripts/microservice-health-output-contract-test.ts",
"scripts/code-queue-supervisor-disclosure-contract-test.ts",
"src/components/frontend/src/index.ts",
"src/components/frontend/src/app.tsx",
+2 -2
View File
@@ -27,7 +27,7 @@ export function rootHelp(): unknown {
{ command: "ssh <providerId> argv <command> [args...]", description: "Run a remote command with each argv token shell-quoted by UniDesk before SSH passthrough." },
{ command: "microservice list", description: "List UniDesk-managed user services and their provider/runtime mapping." },
{ command: "microservice status <id>", description: "Show one user service config, repository reference, backend mapping, and runtime status." },
{ command: "microservice health <id> [--full|--raw]", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy; default output is a compact health summary." },
{ command: "microservice health <id> [--compact|--raw|--full]", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy; default output is compact, and code-queue uses a commander-safe liveness summary unless raw/full is requested." },
{ command: "microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--body-json JSON|--body-file path|--body-stdin] [--raw] [--max-body-bytes N]", description: "Access a private user-service backend path through the same frontend-only proxy used by WebUI; JSON request bodies are supported for controlled write/debug endpoints." },
{ command: "microservice diagnostics <id> [--full|--raw]", description: "Split k3sctl-managed proxy health into a compact summary by default; use --full/--raw for complete evidence." },
{ command: "microservice tunnel-self-test <id>", description: "Trigger an expected provider HTTP tunnel failure and verify requestId/stage diagnostics are returned." },
@@ -167,7 +167,7 @@ function microserviceHelp(): unknown {
usage: [
"bun scripts/cli.ts microservice list",
"bun scripts/cli.ts microservice status <id> [--full|--raw]",
"bun scripts/cli.ts microservice health <id> [--full|--raw]",
"bun scripts/cli.ts microservice health <id> [--compact|--full|--raw]",
"bun scripts/cli.ts microservice diagnostics <id> [--full|--raw]",
"bun scripts/cli.ts microservice tunnel-self-test <id>",
"bun scripts/cli.ts microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--body-json JSON|--body-file path|--body-stdin] [--raw] [--full] [--max-body-bytes N]",
+261 -8
View File
@@ -199,6 +199,263 @@ function parseJsonRecord(text: string): Record<string, unknown> | null {
}
}
function asRecord(value: unknown): Record<string, unknown> | null {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
}
function asNumber(value: unknown, fallback = 0): number {
const number = Number(value);
return Number.isFinite(number) ? number : fallback;
}
function stringList(value: unknown): string[] {
if (!Array.isArray(value)) return [];
return value.map((item) => String(item)).filter((item) => item.length > 0);
}
function boundedUniqueStringList(value: unknown, limit = 5): Record<string, unknown> {
const all = Array.from(new Set(stringList(value))).sort();
const items = all.slice(0, limit);
return {
items,
count: all.length,
omitted: Math.max(0, all.length - items.length),
truncated: all.length > items.length,
};
}
function limitedStrings(value: unknown, limit: number, maxLength: number): string[] {
return stringList(value).slice(0, limit).map((item) => item.length > maxLength ? item.slice(0, maxLength) : item);
}
function recordValue(record: Record<string, unknown> | null, key: string): Record<string, unknown> | null {
return record === null ? null : asRecord(record[key]);
}
function isCodeQueueHealthBody(record: Record<string, unknown> | null): boolean {
if (record === null) return false;
return record.service === "code-queue"
|| record.serviceId === "code-queue"
|| asRecord(record.queue) !== null
|| asRecord(record.executionDiagnostics) !== null;
}
function hasCodeQueueHealthDetails(record: Record<string, unknown> | null): boolean {
if (record === null) return false;
return record.service === "code-queue"
|| asRecord(record.queue) !== null
|| asRecord(record.executionDiagnostics) !== null;
}
function splitBrainLiveFromDiagnostics(record: Record<string, unknown>): boolean {
if (typeof record.splitBrainLive === "boolean") return record.splitBrainLive;
const state = String(record.state ?? record.health ?? "").toLowerCase();
const riskTaskIds = Array.from(new Set([
...stringList(record.heartbeatRiskTaskIds),
...stringList(record.heartbeatExpiredTaskIds),
...stringList(record.heartbeatMissingTaskIds),
...stringList(record.staleRecoveryCandidateTaskIds),
]));
return state === "split-brain" && stringList(record.heartbeatFreshTaskIds).length > 0 && riskTaskIds.length === 0;
}
function effectiveLivenessFromDiagnostics(record: Record<string, unknown>): string {
if (typeof record.effectiveLiveness === "string" && record.effectiveLiveness.length > 0) return record.effectiveLiveness;
if (stringList(record.heartbeatRiskTaskIds).length > 0
|| stringList(record.heartbeatExpiredTaskIds).length > 0
|| stringList(record.heartbeatMissingTaskIds).length > 0
|| stringList(record.staleRecoveryCandidateTaskIds).length > 0) {
return "at-risk";
}
if (splitBrainLiveFromDiagnostics(record)) return "live";
return String(record.state ?? record.health ?? "unknown") === "healthy" ? "healthy" : "degraded";
}
function recommendedActionFromDiagnostics(record: Record<string, unknown>, effectiveLiveness: string): string {
if (typeof record.recommendedAction === "string" && record.recommendedAction.length > 0) return record.recommendedAction;
if (effectiveLiveness === "at-risk") return "investigate-heartbeat-risk";
if (effectiveLiveness === "live") return "continue-supervision";
if (effectiveLiveness === "degraded") return "observe-degraded";
return "none";
}
function livenessInterpretation(effectiveLiveness: string, splitBrainLive: boolean): string {
if (splitBrainLive) return "scheduler heartbeat is fresh; treat active task count from heartbeat as live and continue supervision";
if (effectiveLiveness === "at-risk") return "heartbeat risk is present; investigate heartbeat freshness before recovery";
if (effectiveLiveness === "degraded") return "diagnostics are degraded; cross-check heartbeat, trace and control-plane sources";
return "diagnostics indicate healthy liveness";
}
function compactCodeQueueDiagnostics(value: unknown): Record<string, unknown> | null {
const diagnostics = asRecord(value);
if (diagnostics === null) return null;
const heartbeatRiskIds = Array.from(new Set([
...stringList(diagnostics.heartbeatRiskTaskIds),
...stringList(diagnostics.heartbeatExpiredTaskIds),
...stringList(diagnostics.heartbeatMissingTaskIds),
...stringList(diagnostics.staleRecoveryCandidateTaskIds),
])).sort();
const splitBrainLive = splitBrainLiveFromDiagnostics(diagnostics);
const effectiveLiveness = effectiveLivenessFromDiagnostics(diagnostics);
const recommendedAction = recommendedActionFromDiagnostics(diagnostics, effectiveLiveness);
const heartbeatFreshTaskIds = boundedUniqueStringList(diagnostics.heartbeatFreshTaskIds);
const activeHeartbeatTaskIds = boundedUniqueStringList(diagnostics.activeHeartbeatTaskIds);
return {
state: diagnostics.state ?? diagnostics.health ?? null,
degraded: diagnostics.degraded ?? null,
splitBrain: diagnostics.splitBrain ?? null,
splitBrainLive,
effectiveLiveness,
recommendedAction,
interpretation: livenessInterpretation(effectiveLiveness, splitBrainLive),
executionStateSource: diagnostics.executionStateSource ?? null,
controlPlane: diagnostics.controlPlane ?? null,
databaseActiveTaskCount: diagnostics.databaseActiveTaskCount ?? stringList(diagnostics.databaseActiveTaskIds).length,
schedulerActiveRunSlotCount: diagnostics.schedulerActiveRunSlotCount ?? null,
activeHeartbeatCount: diagnostics.activeHeartbeatCount ?? Math.max(asNumber(activeHeartbeatTaskIds.count), asNumber(heartbeatFreshTaskIds.count)),
activeHeartbeatTaskIds,
heartbeatFreshTaskCount: heartbeatFreshTaskIds.count,
heartbeatFreshTaskIds,
heartbeatRiskTaskCount: heartbeatRiskIds.length,
heartbeatRiskTaskIds: boundedUniqueStringList(heartbeatRiskIds),
heartbeatExpiredTaskCount: stringList(diagnostics.heartbeatExpiredTaskIds).length,
heartbeatMissingTaskCount: stringList(diagnostics.heartbeatMissingTaskIds).length,
staleRecoveryCandidateTaskCount: stringList(diagnostics.staleRecoveryCandidateTaskIds).length,
traceGapTaskCount: stringList(diagnostics.traceGapTaskIds).length,
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt ?? null,
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt ?? null,
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt ?? null,
reasons: limitedStrings(diagnostics.reasons, 3, 240),
};
}
function compactCodeQueueQueueSummary(value: unknown, diagnostics: Record<string, unknown> | null): Record<string, unknown> {
const queue = asRecord(value);
const counts = asRecord(queue?.counts) ?? {};
const running = asNumber(counts.running);
const judging = asNumber(counts.judging);
const activeTaskIds = boundedUniqueStringList(queue?.activeTaskIds);
const activeQueueIds = boundedUniqueStringList(queue?.activeQueueIds);
const processingQueueIds = boundedUniqueStringList(queue?.processingQueueIds);
return {
total: queue?.total ?? null,
counts,
runningCount: running + judging,
running,
judging,
queued: asNumber(counts.queued),
retryWait: asNumber(counts.retry_wait),
queueCount: queue?.queueCount ?? null,
unreadTerminal: queue?.unreadTerminal ?? null,
activeRunSlotCount: queue?.activeRunSlotCount ?? diagnostics?.schedulerActiveRunSlotCount ?? null,
activeTaskCount: activeTaskIds.count,
activeTaskIds,
activeTaskId: queue?.activeTaskId ?? null,
activeQueueIds,
processingQueueIds,
processing: queue?.processing ?? null,
orphanedActiveTaskCount: queue?.orphanedActiveTaskCount ?? null,
defaultQueueId: queue?.defaultQueueId ?? null,
defaultModel: queue?.defaultModel ?? null,
defaultReasoningEffort: queue?.defaultReasoningEffort ?? null,
maxActiveQueues: queue?.maxActiveQueues ?? null,
};
}
function compactCodeQueueComponents(body: Record<string, unknown>, queue: Record<string, unknown> | null): Record<string, unknown> {
const storage = asRecord(queue?.storage);
const egressProxy = asRecord(body.egressProxy);
const oaPublisher = asRecord(body.oaEventPublisher);
return {
databaseReady: body.databaseReady ?? storage?.postgresReady ?? null,
serviceReady: body.serviceReady ?? null,
schedulerEnabled: body.schedulerEnabled ?? null,
egressProxyConnected: egressProxy?.connected ?? null,
oaEventPublisher: oaPublisher === null ? null : {
enabled: oaPublisher.enabled ?? null,
pendingCount: oaPublisher.pendingCount ?? null,
lastError: oaPublisher.lastError ?? null,
},
};
}
export function summarizeMicroserviceHealthResponse(response: unknown, args: string[], serviceId: string): unknown {
if (serviceId !== "code-queue") return summarizeMicroserviceObservation("health", serviceId, response, args);
if (hasFlag(args, "--raw") || hasFlag(args, "--full")) return response;
const responseRecord = asRecord(response);
if (responseRecord === null) return response;
const responseBodySource = "body" in responseRecord ? responseRecord.body : responseRecord;
const responseBody = asRecord(responseBodySource);
if (responseBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
const upstream = recordValue(responseBody, "upstream");
const upstreamBody = recordValue(upstream, "body");
const healthBody = hasCodeQueueHealthDetails(responseBody)
? responseBody
: isCodeQueueHealthBody(upstreamBody)
? upstreamBody
: isCodeQueueHealthBody(responseBody)
? responseBody
: null;
if (healthBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
const queue = asRecord(healthBody.queue);
const diagnostics = compactCodeQueueDiagnostics(healthBody.executionDiagnostics ?? queue?.executionDiagnostics);
const bodyBytes = jsonByteLength(responseBodySource);
return {
ok: responseRecord.ok ?? responseBody.ok ?? healthBody.ok ?? null,
status: responseRecord.status ?? responseBody.status ?? healthBody.status ?? null,
serviceId: "code-queue",
service: healthBody.service ?? "code-queue",
name: responseBody.name ?? null,
providerId: responseBody.providerId ?? null,
healthPath: responseBody.healthPath ?? "/health",
reason: responseBody.reason ?? null,
checkedAt: responseBody.checkedAt ?? null,
instanceId: healthBody.instanceId ?? null,
role: healthBody.role ?? null,
deploy: previewJson(healthBody.deploy ?? null, { maxDepth: 2, maxArrayItems: 2, maxObjectKeys: 8, maxStringLength: 160 }),
components: compactCodeQueueComponents(healthBody, queue),
queue: compactCodeQueueQueueSummary(queue, diagnostics),
heartbeat: diagnostics === null ? null : {
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt,
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt,
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt,
activeHeartbeatCount: diagnostics.activeHeartbeatCount,
activeHeartbeatTaskIds: diagnostics.activeHeartbeatTaskIds,
heartbeatFreshTaskCount: diagnostics.heartbeatFreshTaskCount,
heartbeatFreshTaskIds: diagnostics.heartbeatFreshTaskIds,
heartbeatRiskTaskCount: diagnostics.heartbeatRiskTaskCount,
heartbeatRiskTaskIds: diagnostics.heartbeatRiskTaskIds,
heartbeatExpiredTaskCount: diagnostics.heartbeatExpiredTaskCount,
heartbeatMissingTaskCount: diagnostics.heartbeatMissingTaskCount,
staleRecoveryCandidateTaskCount: diagnostics.staleRecoveryCandidateTaskCount,
traceGapTaskCount: diagnostics.traceGapTaskCount,
schedulerHeartbeatStaleMs: queue?.schedulerHeartbeatStaleMs ?? null,
},
liveness: diagnostics === null ? null : {
state: diagnostics.state,
degraded: diagnostics.degraded,
splitBrain: diagnostics.splitBrain,
splitBrainLive: diagnostics.splitBrainLive,
effectiveLiveness: diagnostics.effectiveLiveness,
recommendedAction: diagnostics.recommendedAction,
interpretation: diagnostics.interpretation,
reasons: diagnostics.reasons,
},
outputPolicy: {
mode: "compact",
defaultCompact: true,
omittedFullHealthBody: true,
originalBodyBytes: bodyBytes,
idPreviewLimit: 5,
rawCommand: "bun scripts/cli.ts microservice health code-queue --raw",
fullCommand: "bun scripts/cli.ts microservice health code-queue --raw --full",
proxyFullCommand: "bun scripts/cli.ts microservice proxy code-queue /health --raw --full",
},
};
}
function requireId(value: string | undefined, command: string): string {
if (value === undefined || value.length === 0) throw new Error(`${command} requires microservice id`);
return value;
@@ -240,10 +497,6 @@ function hasFlag(args: string[], name: string): boolean {
return args.includes(name);
}
function asRecord(value: unknown): Record<string, unknown> | null {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
}
function asArray(value: unknown): unknown[] {
return Array.isArray(value) ? value : [];
}
@@ -272,8 +525,8 @@ function requestBodyOption(args: string[]): unknown | undefined {
return undefined;
}
function assertKnownObservationOptions(args: string[], command: string): void {
const flags = new Set(["--full", "--raw"]);
function assertKnownObservationOptions(args: string[], command: string, extraFlags: string[] = []): void {
const flags = new Set(["--full", "--raw", ...extraFlags]);
for (const arg of args) {
if (!arg.startsWith("--")) continue;
if (!flags.has(arg)) throw new Error(`unsupported ${command} option: ${arg}`);
@@ -547,8 +800,8 @@ export async function runMicroserviceCommand(_config: UniDeskConfig, args: strin
if (action === "health") {
const id = requireId(idArg, "microservice health");
const optionArgs = args.slice(2);
assertKnownObservationOptions(optionArgs, "microservice health");
return summarizeMicroserviceObservation(action, id, coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs);
assertKnownObservationOptions(optionArgs, "microservice health", ["--compact"]);
return summarizeMicroserviceHealthResponse(coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs, id);
}
if (action === "diagnostics") {
const id = requireId(idArg, "microservice diagnostics");