Merge pull request #80 from pikasTech/code-queue/issue-20-host-commander-governance-docs
fix: compact code queue health output
This commit is contained in:
@@ -23,7 +23,7 @@ CLI 可以从 `master` 快速演进,但必须兼容 `deploy.json` 固定的 CI
|
||||
- `ssh <providerId> apply-patch [tool args...] < patch.diff` 直接调用远端注入的 `apply_patch` 工具,并把本地 stdin 中的标准 `*** Begin Patch` / `*** End Patch` patch 流透传给目标节点。
|
||||
- `ssh <providerId> py [script-args...] < script.py` 把本地 stdin 落到远端临时 `.py` 文件后再以 `python3 -u` 执行并自动清理,避免再手写 `'python3 -'`、heredoc 或多层引号;`script-args` 会按 argv 安全透传给远端脚本。
|
||||
- `ssh <providerId> skills [--scope all|wsl|windows] [--limit N]` 发现目标节点上的 WSL/Linux skill 根目录;当 provider 是 WSL 时同一次调用还会扫描 Windows 用户目录下的 `.agents/skills` 与 `.codex/skills`。
|
||||
- `microservice list/status/health/diagnostics/tunnel-self-test/proxy` 通过 backend-core 内网 API 管理挂载在计算节点 Docker 或 k3s 控制面中的用户服务(底层命令名仍为 microservice);`health`、`status` 和 `diagnostics` 默认返回 compact summary、body 字节数和 `--full|--raw` 展开命令,只有小 body 或无法抽取 summary 时才带有界 body preview,避免 Code Queue/k3s 诊断一次性输出爆炸;`tunnel-self-test` 和 `proxy` 会走真实 backend-core -> provider-gateway 或 k3sctl-adapter -> 节点服务链路,`proxy` 支持受控 JSON 请求体并对超大响应 body 默认输出有界预览,规则见 `docs/reference/microservices.md`。
|
||||
- `microservice list/status/health/diagnostics/tunnel-self-test/proxy` 通过 backend-core 内网 API 管理挂载在计算节点 Docker 或 k3s 控制面中的用户服务(底层命令名仍为 microservice);`health`、`status` 和 `diagnostics` 默认返回 compact summary、body 字节数和 `--full|--raw` 展开命令,只有小 body 或无法抽取 summary 时才带有界 body preview,避免 Code Queue/k3s 诊断一次性输出爆炸;`tunnel-self-test` 和 `proxy` 会走真实 backend-core -> provider-gateway 或 k3sctl-adapter -> 节点服务链路。`microservice health code-queue` 使用 commander-safe 专用摘要,必须保留 ok/status、service id、running count、queue count、heartbeat freshness/risk、split-brain/live/degraded 解释和 raw drill-down 命令;需要完整健康 JSON 时显式加 `--raw` 或 `--full`,等价深挖路径是 `microservice proxy code-queue /health --raw --full`。`proxy` 支持受控 JSON 请求体并对超大响应 body 默认输出有界预览,规则见 `docs/reference/microservices.md`。
|
||||
- `decision upload/list/show/health` 通过 backend-core 用户服务代理访问 D601 k3s Decision Center,用于上传会议记录/决议 Markdown、列出权威记录、查看详情和健康检查;`decision list` 默认只返回摘要并省略完整 Markdown body,需要排查大正文时显式加 `--include-body`。正式文书字段通过 records 模型一等字段返回和查询:`--doc-no DC-...`、`--doc-type DCSN|GOAL|PLAN|RPRT|ACTN|ISSU|RETR|RQST|RESP|MINS`、`--doc-priority P0|P1|P2|P3`、`--year YYYY`、`--signer`、`--issued-at`、`--effective-scope`、`--supersedes`、`--superseded-by`;`show` 和 `requirement update` 可使用 `id` 或 `docNo`。`decision requirement list/create/upsert/update/show` 在同一 records 模型上管理 `goal|decision|blocker|debt|experiment` 需求记录,`docNo` 唯一,未传 `--doc-no` 但提供 `--doc-type/--doc-priority/--year` 时由服务分配下一个序号。它们不得直连 D601 Service、NodePort 或 provider-gateway 业务 HTTP。
|
||||
- `decision diary import <markdown-file>` 将带 `# YYYY年M月D日`、`# YYYY-MM-DD` 或 `# YYYY/M/D` 标题的工作日志拆成每天一篇 Markdown 日记,按 `YYYY-MM/YYYY-MM-DD.md` 虚拟路径写入 Decision Center PostgreSQL;`decision diary list/history` 默认只返回摘要,需要完整 Markdown 时显式加 `--include-body`;`decision diary show <YYYY-MM-DD|id> [--source-file path]` 查看单日正文,`--source-file` 用于同一天存在多个导入来源时精确选择;`decision diary edit|upsert <YYYY-MM-DD|id> --body-file <path> [--title text] [--source-file path] [--tag tag]` 通过 `PUT /api/diary/entries/:idOrDate` 创建当天或历史条目并编辑既有条目。
|
||||
- `deploy check/plan/apply` 默认从根目录 `deploy.json` 读取服务 repo 与 commit 期望状态,join `config.json` 和现有 manifest 后使用 target-side build 或 reviewed artifact consumer 校验/更新已支持目标;`deploy plan --env dev|prod` 只从 `origin/master:deploy.json#environments.<env>` 读取 manifest 并输出 dry-run 环境计划,不使用本地 dirty worktree;当前 `deploy apply --env dev` 支持 `backend-core`、`frontend`、`baidu-netdisk`、`decision-center`、`mdtodo`、`claudeqq` 和 dev-only `code-queue` artifact consumers,`findjob`/`pipeline`/`met-nonlinear` 为 D601 direct Compose artifact consumers,`k3sctl-adapter` 只提供 plan/dry-run;dev desired-state smoke 使用 `ci run-dev-e2e`;规则见 `docs/reference/deploy.md`、`docs/reference/dev-environment.md` 和 `docs/reference/dev-ci-runner.md`。`deploy apply --env prod` 同时覆盖 `findjob` 和 `pipeline` 的 pull-only Compose CD,但 `met-nonlinear` 仍然只允许 dry-run/plan,`k3sctl-adapter` 只允许 plan/dry-run。
|
||||
@@ -74,7 +74,7 @@ CLI 可以从 `master` 快速演进,但必须兼容 `deploy.json` 固定的 CI
|
||||
|
||||
每条命令的最外层 JSON 包含 `ok`、`command` 和 `data` 或 `error`。失败时 CLI 设置非零退出码,但仍然输出 JSON 错误对象;错误对象应包含 `name`、`message` 和可用的 `stack`。
|
||||
|
||||
诊断命令默认采用渐进披露:`server logs`、`job list/status`、`codex task/trace/output` 和 `microservice proxy` 都必须有默认条数、字节数或文本预览上限;用户显式传 `--limit`、`--tail-bytes`、`--full-text` 或 `--full` 才扩大单次输出。CLI stdout 遇到下游 pipe 关闭的 `EPIPE` 必须安静退出,不得打印 Bun stack trace。
|
||||
诊断命令默认采用渐进披露:`server logs`、`job list/status`、`codex task/trace/output`、`microservice health code-queue` 和 `microservice proxy` 都必须有默认条数、字节数或文本预览上限;用户显式传 `--limit`、`--tail-bytes`、`--full-text`、`--raw` 或 `--full` 才扩大单次输出。CLI stdout 遇到下游 pipe 关闭的 `EPIPE` 必须安静退出,不得打印 Bun stack trace。
|
||||
|
||||
`microservice proxy` 是面向人工验证和受控调试的私有后端入口。默认 method 为 GET;使用 `--body-json JSON`、`--body-file path` 或 `--body-stdin` 时默认 method 切换为 POST,也可显式加 `--method POST|PUT|PATCH|DELETE`,但 GET/HEAD 不允许携带请求体。所有请求仍受 config 中的 `allowedMethods` 和 `allowedPathPrefixes` 限制。为了避免 Pipeline snapshot 这类超大业务 JSON 造成 CLI 输出爆炸,响应 body 超过默认阈值时会返回 `bodyOmitted=true`、`bodyPreview`、`bodyBytes` 和 `rawHint`;`--raw` 仍受默认硬限额保护,需要完整 body 时显式添加 `--raw --full`,或用 `--max-body-bytes <N>` 调整预览阈值。正式 frontend 展示仍应优先使用业务控件和 `__unideskArrayLimit` 这类展示级裁剪参数,而不是默认倾倒完整 JSON。
|
||||
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
|
||||
当 HWLAB 是当前优先级最高的推进对象时,所有派单 prompt 的开头都必须显式引用 `DC-DCSN-P0-2026-003`,并说明任务究竟服务于 M3 虚拟硬件可信闭环的哪一环。如果任务只是在做发布前置、诊断、证据收口或文档治理,也必须明确写出它不抢占 M3 P0。
|
||||
|
||||
HWLAB 业务目标、验收和实现优先级归 `pikasTech/HWLAB#7`;UniDesk 指挥治理、队列监督、并发窗口、PR handoff 和 runner 能力归 `pikasTech/unidesk#20`。#20 可以记录 HWLAB #7 的调度状态和证据链接,但不能替代 HWLAB 业务 issue。
|
||||
|
||||
审阅 HWLAB runner 输出时,不能把 `SOURCE`、`LOCAL`、`DRY-RUN`、fixture 或只读报告误当成 `DEV-LIVE`。除非输出真的证明了 `res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1` 的真链路,并且带有 operation / audit / evidence 关联,否则只能归类为 support、diagnostics 或 contract。
|
||||
|
||||
`split-brain live` 且 heartbeat/trace 新鲜时,指挥官必须继续监督,不把它当作服务中断。此类状态的优先动作是继续轮询、继续审阅、继续派单,而不是默认 interrupt 或 cancel。
|
||||
@@ -140,6 +142,8 @@ CLI 是短 shout 的需求原语,不是长驻服务器进程。CLI 功能不
|
||||
|
||||
PR 是审查型交付入口,不是所有 Code Queue 任务的默认出口。默认 master-only 交付仍按项目 Git 规则执行;当变更风险高、跨模块、需要人工审查、或任务目标明确要求 PR 交付时,worker 可以创建 PR。PR 型任务必须报告源分支、目标分支、PR URL、关联 issue、测试证据和未完成风险。禁止把 PR 当成隐藏分支仓库;PR 分支必须来自最新目标线,保持小而可审查,并在合并后确认目标分支远端 commit 可 fetch。
|
||||
|
||||
PR handoff 的职责固定分开:runner 实现、测试、提交、push head branch 并创建 PR;指挥官监督并发、steer、审阅、确认 checks 和合并裁决。runner 不合并自己的 PR,host commander 也不把直接编辑业务代码当成常规 PR 替代路径。
|
||||
|
||||
PR 支持本身是 Code Queue 能力的一部分。当前 UniDesk CLI 支持 `gh pr list|view|create|update|comment create|comment delete|close|reopen`,其中 create 需要显式 `--title`、`--base`、`--head` 和正文来源,update 需要显式 PR number、正文来源和 `--mode replace|append`,comment create 需要显式 PR number 和正文来源,且推荐使用 `--body-file`。PR 收口观察应使用 `gh pr view <number> --json headRefName,baseRefName,mergeable,mergeStateStatus,statusCheckRollup` 获取目标分支、源分支、mergeability 和检查汇总;该路径仍是只读元数据,不执行 merge。`pr create --dry-run`、`pr update --dry-run` 与 `pr comment create --dry-run` 只返回 planned operation,不创建 PR、不更新正文、不写评论;非 dry-run 创建前会校验 repo、base、head 和 compare ahead 状态,append 更新会先读取当前 PR body。`gh pr delete` 和 `gh pr merge` 仍是后续范围,本阶段没有 `--confirm` 可以启用真实 merge,也不能伪造硬删除;需要移除活跃 PR 时使用 `gh pr close`。普通 worker 不应隐式依赖未实现的 PR 合并能力;需要 PR 交付时,prompt 必须明确允许的人工或后续工具路径,并报告未覆盖范围。
|
||||
|
||||
### PR 驱动派单模板
|
||||
|
||||
@@ -46,6 +46,14 @@ ls -t /root/unidesk/logs/commander.log.*.jsonl | head -n 1
|
||||
|
||||
该运行形态可以用于指挥 Code Queue、审阅任务、创建/更新 issue、PR 收口和必要的热修复例外,但不得被解释为普通 worker。涉及实现、测试和 PR 创建的常规工作仍应优先通过 Code Queue 派单;host commander 直接执行只适用于授权的 PR 收口、基础设施热修复或 runner 权限缺口,并且必须留下复盘和长期改进 issue。
|
||||
|
||||
## 指挥职责
|
||||
|
||||
host commander 的工作是调度、监督、steer、审阅和 PR 收口;它维护并发窗口、阻塞分类、issue/#20/#24 记录,并在 checks 和审阅通过后负责合并裁决或走已批准的合并路径。Code Queue runner 的工作是实现、验证、提交、push head branch 和创建 PR;runner 不合并自己的 PR。
|
||||
|
||||
HWLAB 业务目标、验收和优先级属于 `pikasTech/HWLAB#7`;UniDesk 指挥治理、队列监督、并发窗口和 runner/PR handoff 属于 `pikasTech/unidesk#20`。#20 可以链接 HWLAB #7 的证据和状态,但不能替代 HWLAB 业务 issue,也不能把 HWLAB 代码实现决策写成 UniDesk 指挥规则。
|
||||
|
||||
host commander 不直接编辑 HWLAB 业务代码,不以本地热修绕过 HWLAB runner PR。只有用户明确授权的 PR 收口、基础设施热修或 runner 权限缺口可以例外;例外必须最小化变更,在相关 issue/PR 中留下命令、diff、验证证据和影响范围,并创建或更新 follow-up issue 消除该例外原因。
|
||||
|
||||
## Dev 验证计划
|
||||
|
||||
`commander smoke --dry-run` 是无 daemon smoke contract。它只输出验证计划,不启动 HTTP daemon、不打开 SSH/PTY/stdio bridge、不发送 ClaudeQQ、不重启服务、不 interrupt/cancel 任务、不部署、不跑全量 check/e2e。
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import { summarizeMicroserviceHealthResponse } from "./src/microservices";
|
||||
|
||||
type JsonRecord = Record<string, unknown>;
|
||||
|
||||
function assertCondition(condition: unknown, message: string, detail: JsonRecord = {}): void {
|
||||
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): JsonRecord {
|
||||
assertCondition(typeof value === "object" && value !== null && !Array.isArray(value), "expected JSON object", { value });
|
||||
return value as JsonRecord;
|
||||
}
|
||||
|
||||
function manyIds(prefix: string, count: number): string[] {
|
||||
return Array.from({ length: count }, (_, index) => `${prefix}-${String(index + 1).padStart(3, "0")}`);
|
||||
}
|
||||
|
||||
function longText(prefix: string, count: number): string {
|
||||
return Array.from({ length: count }, (_, index) => `${prefix} diagnostic row ${index + 1} with queue detail and commander-noise payload`).join("\n");
|
||||
}
|
||||
|
||||
function largeCodeQueueHealth(): JsonRecord {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
body: {
|
||||
ok: true,
|
||||
service: "code-queue",
|
||||
instanceId: "D601-code-queue-scheduler",
|
||||
role: "scheduler",
|
||||
deploy: {
|
||||
commit: "abcdef1234567890",
|
||||
requestedCommit: "abcdef1234567890",
|
||||
},
|
||||
databaseReady: true,
|
||||
schedulerEnabled: true,
|
||||
queue: {
|
||||
total: 640,
|
||||
counts: {
|
||||
running: 3,
|
||||
judging: 2,
|
||||
queued: 41,
|
||||
retry_wait: 4,
|
||||
succeeded: 560,
|
||||
failed: 30,
|
||||
},
|
||||
queues: manyIds("queue", 90).map((id) => ({
|
||||
id,
|
||||
name: id,
|
||||
total: 12,
|
||||
counts: { queued: 5, succeeded: 7 },
|
||||
diagnostics: longText(id, 18),
|
||||
})),
|
||||
queueCount: 90,
|
||||
activeRunSlotCount: 5,
|
||||
activeTaskIds: manyIds("active-task", 12),
|
||||
activeTaskId: "active-task-001",
|
||||
activeQueueIds: manyIds("active-queue", 12),
|
||||
processingQueueIds: manyIds("processing-queue", 8),
|
||||
processing: true,
|
||||
orphanedActiveTaskCount: 0,
|
||||
unreadTerminal: 7,
|
||||
defaultQueueId: "default",
|
||||
defaultModel: "gpt-5.5",
|
||||
defaultReasoningEffort: "xhigh",
|
||||
maxActiveQueues: 0,
|
||||
schedulerHeartbeatStaleMs: 120000,
|
||||
},
|
||||
executionDiagnostics: {
|
||||
state: "split-brain",
|
||||
degraded: true,
|
||||
splitBrain: true,
|
||||
splitBrainLive: true,
|
||||
effectiveLiveness: "live",
|
||||
recommendedAction: "continue-supervision",
|
||||
executionStateSource: "postgres-control-plane",
|
||||
controlPlane: "D601-code-queue-scheduler",
|
||||
databaseActiveTaskCount: 12,
|
||||
schedulerActiveRunSlotCount: 5,
|
||||
activeHeartbeatCount: 12,
|
||||
activeHeartbeatTaskIds: manyIds("active-heartbeat", 12),
|
||||
heartbeatFreshTaskIds: manyIds("fresh-heartbeat", 12),
|
||||
heartbeatExpiredTaskIds: [],
|
||||
heartbeatMissingTaskIds: [],
|
||||
staleRecoveryCandidateTaskIds: [],
|
||||
heartbeatRiskTaskIds: [],
|
||||
traceGapTaskIds: manyIds("trace-gap", 15),
|
||||
lastSchedulerHeartbeatAt: "2026-05-22T00:00:12.000Z",
|
||||
lastObservedAgentEventAt: "2026-05-22T00:00:11.000Z",
|
||||
lastPersistedTraceAt: "2026-05-22T00:00:10.000Z",
|
||||
reasons: Array.from({ length: 12 }, (_, index) => longText(`reason-${index + 1}`, 8)),
|
||||
},
|
||||
modelProviderConfig: {
|
||||
source: "fixture",
|
||||
largeNoise: longText("model-provider", 200),
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const fixture = largeCodeQueueHealth();
|
||||
const compact = asRecord(summarizeMicroserviceHealthResponse(fixture, ["health", "code-queue"], "code-queue"));
|
||||
const raw = summarizeMicroserviceHealthResponse(fixture, ["health", "code-queue", "--raw"], "code-queue");
|
||||
const full = summarizeMicroserviceHealthResponse(fixture, ["health", "code-queue", "--full"], "code-queue");
|
||||
const unhealthyWrapper = asRecord(summarizeMicroserviceHealthResponse({
|
||||
ok: false,
|
||||
status: 503,
|
||||
body: {
|
||||
ok: false,
|
||||
status: "unhealthy",
|
||||
serviceId: "code-queue",
|
||||
reason: "health body ok=false",
|
||||
upstream: {
|
||||
status: 503,
|
||||
contentType: "application/json",
|
||||
body: fixture.body,
|
||||
},
|
||||
},
|
||||
}, ["health", "code-queue"], "code-queue"));
|
||||
const compactBody = JSON.stringify(compact);
|
||||
const rawBody = JSON.stringify(fixture);
|
||||
const queue = asRecord(compact.queue);
|
||||
const heartbeat = asRecord(compact.heartbeat);
|
||||
const liveness = asRecord(compact.liveness);
|
||||
const outputPolicy = asRecord(compact.outputPolicy);
|
||||
const unhealthyQueue = asRecord(unhealthyWrapper.queue);
|
||||
const unhealthyLiveness = asRecord(unhealthyWrapper.liveness);
|
||||
|
||||
assertCondition(compactBody.length < rawBody.length * 0.25, "default code-queue health output should be materially compact", { compactChars: compactBody.length, rawChars: rawBody.length });
|
||||
assertCondition(compactBody.length < 12000, "default code-queue health output should stay commander-safe", { compactChars: compactBody.length });
|
||||
assertCondition(!("body" in compact), "compact health must not retain the raw body", compact);
|
||||
assertCondition(!("queues" in queue), "compact queue summary must not include full queue dumps", queue);
|
||||
assertCondition(compact.ok === true && compact.status === 200 && compact.serviceId === "code-queue", "compact health keeps ok/status/service identity", compact);
|
||||
assertCondition(queue.runningCount === 5 && queue.queueCount === 90, "compact queue keeps running count and queue count", queue);
|
||||
assertCondition(heartbeat.heartbeatFreshTaskCount === 12 && heartbeat.heartbeatRiskTaskCount === 0, "compact heartbeat keeps freshness and risk counts", heartbeat);
|
||||
assertCondition(liveness.effectiveLiveness === "live" && liveness.splitBrainLive === true, "compact liveness keeps split-brain live interpretation", liveness);
|
||||
assertCondition(String(liveness.interpretation).includes("continue supervision"), "compact liveness includes commander interpretation", liveness);
|
||||
assertCondition(typeof outputPolicy.rawCommand === "string" && String(outputPolicy.rawCommand).includes("--raw"), "compact output exposes raw drill-down command", outputPolicy);
|
||||
assertCondition(raw === fixture && full === fixture, "--raw/--full must preserve full health access", { rawSame: raw === fixture, fullSame: full === fixture });
|
||||
assertCondition(unhealthyWrapper.ok === false && unhealthyWrapper.reason === "health body ok=false", "unhealthy wrapper keeps probe failure", unhealthyWrapper);
|
||||
assertCondition(unhealthyQueue.runningCount === 5 && unhealthyLiveness.effectiveLiveness === "live", "unhealthy wrapper still surfaces upstream queue liveness", { unhealthyQueue, unhealthyLiveness });
|
||||
|
||||
console.log(JSON.stringify({
|
||||
ok: true,
|
||||
checks: [
|
||||
"default code-queue health output is compact and omits raw body/queue dumps",
|
||||
"critical ok/status/service/running/heartbeat/liveness fields remain visible",
|
||||
"unhealthy backend health wrapper still surfaces upstream code-queue liveness",
|
||||
"--raw and --full return the original health response for diagnostics",
|
||||
],
|
||||
compactChars: compactBody.length,
|
||||
rawChars: rawBody.length,
|
||||
}, null, 2));
|
||||
@@ -34,6 +34,7 @@ const syntaxFiles = [
|
||||
"scripts/code-queue-cli-steer-test.ts",
|
||||
"scripts/code-queue-cli-submit-prompt-contract-test.ts",
|
||||
"scripts/code-queue-gh-auth-redaction-contract-test.ts",
|
||||
"scripts/microservice-health-output-contract-test.ts",
|
||||
"scripts/code-queue-supervisor-disclosure-contract-test.ts",
|
||||
"src/components/frontend/src/index.ts",
|
||||
"src/components/frontend/src/app.tsx",
|
||||
|
||||
+2
-2
@@ -27,7 +27,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "ssh <providerId> argv <command> [args...]", description: "Run a remote command with each argv token shell-quoted by UniDesk before SSH passthrough." },
|
||||
{ command: "microservice list", description: "List UniDesk-managed user services and their provider/runtime mapping." },
|
||||
{ command: "microservice status <id>", description: "Show one user service config, repository reference, backend mapping, and runtime status." },
|
||||
{ command: "microservice health <id> [--full|--raw]", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy; default output is a compact health summary." },
|
||||
{ command: "microservice health <id> [--compact|--raw|--full]", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy; default output is compact, and code-queue uses a commander-safe liveness summary unless raw/full is requested." },
|
||||
{ command: "microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--body-json JSON|--body-file path|--body-stdin] [--raw] [--max-body-bytes N]", description: "Access a private user-service backend path through the same frontend-only proxy used by WebUI; JSON request bodies are supported for controlled write/debug endpoints." },
|
||||
{ command: "microservice diagnostics <id> [--full|--raw]", description: "Split k3sctl-managed proxy health into a compact summary by default; use --full/--raw for complete evidence." },
|
||||
{ command: "microservice tunnel-self-test <id>", description: "Trigger an expected provider HTTP tunnel failure and verify requestId/stage diagnostics are returned." },
|
||||
@@ -167,7 +167,7 @@ function microserviceHelp(): unknown {
|
||||
usage: [
|
||||
"bun scripts/cli.ts microservice list",
|
||||
"bun scripts/cli.ts microservice status <id> [--full|--raw]",
|
||||
"bun scripts/cli.ts microservice health <id> [--full|--raw]",
|
||||
"bun scripts/cli.ts microservice health <id> [--compact|--full|--raw]",
|
||||
"bun scripts/cli.ts microservice diagnostics <id> [--full|--raw]",
|
||||
"bun scripts/cli.ts microservice tunnel-self-test <id>",
|
||||
"bun scripts/cli.ts microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--body-json JSON|--body-file path|--body-stdin] [--raw] [--full] [--max-body-bytes N]",
|
||||
|
||||
@@ -199,6 +199,263 @@ function parseJsonRecord(text: string): Record<string, unknown> | null {
|
||||
}
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
function asNumber(value: unknown, fallback = 0): number {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? number : fallback;
|
||||
}
|
||||
|
||||
function stringList(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.map((item) => String(item)).filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function boundedUniqueStringList(value: unknown, limit = 5): Record<string, unknown> {
|
||||
const all = Array.from(new Set(stringList(value))).sort();
|
||||
const items = all.slice(0, limit);
|
||||
return {
|
||||
items,
|
||||
count: all.length,
|
||||
omitted: Math.max(0, all.length - items.length),
|
||||
truncated: all.length > items.length,
|
||||
};
|
||||
}
|
||||
|
||||
function limitedStrings(value: unknown, limit: number, maxLength: number): string[] {
|
||||
return stringList(value).slice(0, limit).map((item) => item.length > maxLength ? item.slice(0, maxLength) : item);
|
||||
}
|
||||
|
||||
function recordValue(record: Record<string, unknown> | null, key: string): Record<string, unknown> | null {
|
||||
return record === null ? null : asRecord(record[key]);
|
||||
}
|
||||
|
||||
function isCodeQueueHealthBody(record: Record<string, unknown> | null): boolean {
|
||||
if (record === null) return false;
|
||||
return record.service === "code-queue"
|
||||
|| record.serviceId === "code-queue"
|
||||
|| asRecord(record.queue) !== null
|
||||
|| asRecord(record.executionDiagnostics) !== null;
|
||||
}
|
||||
|
||||
function hasCodeQueueHealthDetails(record: Record<string, unknown> | null): boolean {
|
||||
if (record === null) return false;
|
||||
return record.service === "code-queue"
|
||||
|| asRecord(record.queue) !== null
|
||||
|| asRecord(record.executionDiagnostics) !== null;
|
||||
}
|
||||
|
||||
function splitBrainLiveFromDiagnostics(record: Record<string, unknown>): boolean {
|
||||
if (typeof record.splitBrainLive === "boolean") return record.splitBrainLive;
|
||||
const state = String(record.state ?? record.health ?? "").toLowerCase();
|
||||
const riskTaskIds = Array.from(new Set([
|
||||
...stringList(record.heartbeatRiskTaskIds),
|
||||
...stringList(record.heartbeatExpiredTaskIds),
|
||||
...stringList(record.heartbeatMissingTaskIds),
|
||||
...stringList(record.staleRecoveryCandidateTaskIds),
|
||||
]));
|
||||
return state === "split-brain" && stringList(record.heartbeatFreshTaskIds).length > 0 && riskTaskIds.length === 0;
|
||||
}
|
||||
|
||||
function effectiveLivenessFromDiagnostics(record: Record<string, unknown>): string {
|
||||
if (typeof record.effectiveLiveness === "string" && record.effectiveLiveness.length > 0) return record.effectiveLiveness;
|
||||
if (stringList(record.heartbeatRiskTaskIds).length > 0
|
||||
|| stringList(record.heartbeatExpiredTaskIds).length > 0
|
||||
|| stringList(record.heartbeatMissingTaskIds).length > 0
|
||||
|| stringList(record.staleRecoveryCandidateTaskIds).length > 0) {
|
||||
return "at-risk";
|
||||
}
|
||||
if (splitBrainLiveFromDiagnostics(record)) return "live";
|
||||
return String(record.state ?? record.health ?? "unknown") === "healthy" ? "healthy" : "degraded";
|
||||
}
|
||||
|
||||
function recommendedActionFromDiagnostics(record: Record<string, unknown>, effectiveLiveness: string): string {
|
||||
if (typeof record.recommendedAction === "string" && record.recommendedAction.length > 0) return record.recommendedAction;
|
||||
if (effectiveLiveness === "at-risk") return "investigate-heartbeat-risk";
|
||||
if (effectiveLiveness === "live") return "continue-supervision";
|
||||
if (effectiveLiveness === "degraded") return "observe-degraded";
|
||||
return "none";
|
||||
}
|
||||
|
||||
function livenessInterpretation(effectiveLiveness: string, splitBrainLive: boolean): string {
|
||||
if (splitBrainLive) return "scheduler heartbeat is fresh; treat active task count from heartbeat as live and continue supervision";
|
||||
if (effectiveLiveness === "at-risk") return "heartbeat risk is present; investigate heartbeat freshness before recovery";
|
||||
if (effectiveLiveness === "degraded") return "diagnostics are degraded; cross-check heartbeat, trace and control-plane sources";
|
||||
return "diagnostics indicate healthy liveness";
|
||||
}
|
||||
|
||||
function compactCodeQueueDiagnostics(value: unknown): Record<string, unknown> | null {
|
||||
const diagnostics = asRecord(value);
|
||||
if (diagnostics === null) return null;
|
||||
const heartbeatRiskIds = Array.from(new Set([
|
||||
...stringList(diagnostics.heartbeatRiskTaskIds),
|
||||
...stringList(diagnostics.heartbeatExpiredTaskIds),
|
||||
...stringList(diagnostics.heartbeatMissingTaskIds),
|
||||
...stringList(diagnostics.staleRecoveryCandidateTaskIds),
|
||||
])).sort();
|
||||
const splitBrainLive = splitBrainLiveFromDiagnostics(diagnostics);
|
||||
const effectiveLiveness = effectiveLivenessFromDiagnostics(diagnostics);
|
||||
const recommendedAction = recommendedActionFromDiagnostics(diagnostics, effectiveLiveness);
|
||||
const heartbeatFreshTaskIds = boundedUniqueStringList(diagnostics.heartbeatFreshTaskIds);
|
||||
const activeHeartbeatTaskIds = boundedUniqueStringList(diagnostics.activeHeartbeatTaskIds);
|
||||
return {
|
||||
state: diagnostics.state ?? diagnostics.health ?? null,
|
||||
degraded: diagnostics.degraded ?? null,
|
||||
splitBrain: diagnostics.splitBrain ?? null,
|
||||
splitBrainLive,
|
||||
effectiveLiveness,
|
||||
recommendedAction,
|
||||
interpretation: livenessInterpretation(effectiveLiveness, splitBrainLive),
|
||||
executionStateSource: diagnostics.executionStateSource ?? null,
|
||||
controlPlane: diagnostics.controlPlane ?? null,
|
||||
databaseActiveTaskCount: diagnostics.databaseActiveTaskCount ?? stringList(diagnostics.databaseActiveTaskIds).length,
|
||||
schedulerActiveRunSlotCount: diagnostics.schedulerActiveRunSlotCount ?? null,
|
||||
activeHeartbeatCount: diagnostics.activeHeartbeatCount ?? Math.max(asNumber(activeHeartbeatTaskIds.count), asNumber(heartbeatFreshTaskIds.count)),
|
||||
activeHeartbeatTaskIds,
|
||||
heartbeatFreshTaskCount: heartbeatFreshTaskIds.count,
|
||||
heartbeatFreshTaskIds,
|
||||
heartbeatRiskTaskCount: heartbeatRiskIds.length,
|
||||
heartbeatRiskTaskIds: boundedUniqueStringList(heartbeatRiskIds),
|
||||
heartbeatExpiredTaskCount: stringList(diagnostics.heartbeatExpiredTaskIds).length,
|
||||
heartbeatMissingTaskCount: stringList(diagnostics.heartbeatMissingTaskIds).length,
|
||||
staleRecoveryCandidateTaskCount: stringList(diagnostics.staleRecoveryCandidateTaskIds).length,
|
||||
traceGapTaskCount: stringList(diagnostics.traceGapTaskIds).length,
|
||||
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt ?? null,
|
||||
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt ?? null,
|
||||
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt ?? null,
|
||||
reasons: limitedStrings(diagnostics.reasons, 3, 240),
|
||||
};
|
||||
}
|
||||
|
||||
function compactCodeQueueQueueSummary(value: unknown, diagnostics: Record<string, unknown> | null): Record<string, unknown> {
|
||||
const queue = asRecord(value);
|
||||
const counts = asRecord(queue?.counts) ?? {};
|
||||
const running = asNumber(counts.running);
|
||||
const judging = asNumber(counts.judging);
|
||||
const activeTaskIds = boundedUniqueStringList(queue?.activeTaskIds);
|
||||
const activeQueueIds = boundedUniqueStringList(queue?.activeQueueIds);
|
||||
const processingQueueIds = boundedUniqueStringList(queue?.processingQueueIds);
|
||||
return {
|
||||
total: queue?.total ?? null,
|
||||
counts,
|
||||
runningCount: running + judging,
|
||||
running,
|
||||
judging,
|
||||
queued: asNumber(counts.queued),
|
||||
retryWait: asNumber(counts.retry_wait),
|
||||
queueCount: queue?.queueCount ?? null,
|
||||
unreadTerminal: queue?.unreadTerminal ?? null,
|
||||
activeRunSlotCount: queue?.activeRunSlotCount ?? diagnostics?.schedulerActiveRunSlotCount ?? null,
|
||||
activeTaskCount: activeTaskIds.count,
|
||||
activeTaskIds,
|
||||
activeTaskId: queue?.activeTaskId ?? null,
|
||||
activeQueueIds,
|
||||
processingQueueIds,
|
||||
processing: queue?.processing ?? null,
|
||||
orphanedActiveTaskCount: queue?.orphanedActiveTaskCount ?? null,
|
||||
defaultQueueId: queue?.defaultQueueId ?? null,
|
||||
defaultModel: queue?.defaultModel ?? null,
|
||||
defaultReasoningEffort: queue?.defaultReasoningEffort ?? null,
|
||||
maxActiveQueues: queue?.maxActiveQueues ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function compactCodeQueueComponents(body: Record<string, unknown>, queue: Record<string, unknown> | null): Record<string, unknown> {
|
||||
const storage = asRecord(queue?.storage);
|
||||
const egressProxy = asRecord(body.egressProxy);
|
||||
const oaPublisher = asRecord(body.oaEventPublisher);
|
||||
return {
|
||||
databaseReady: body.databaseReady ?? storage?.postgresReady ?? null,
|
||||
serviceReady: body.serviceReady ?? null,
|
||||
schedulerEnabled: body.schedulerEnabled ?? null,
|
||||
egressProxyConnected: egressProxy?.connected ?? null,
|
||||
oaEventPublisher: oaPublisher === null ? null : {
|
||||
enabled: oaPublisher.enabled ?? null,
|
||||
pendingCount: oaPublisher.pendingCount ?? null,
|
||||
lastError: oaPublisher.lastError ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeMicroserviceHealthResponse(response: unknown, args: string[], serviceId: string): unknown {
|
||||
if (serviceId !== "code-queue") return summarizeMicroserviceObservation("health", serviceId, response, args);
|
||||
if (hasFlag(args, "--raw") || hasFlag(args, "--full")) return response;
|
||||
|
||||
const responseRecord = asRecord(response);
|
||||
if (responseRecord === null) return response;
|
||||
const responseBodySource = "body" in responseRecord ? responseRecord.body : responseRecord;
|
||||
const responseBody = asRecord(responseBodySource);
|
||||
if (responseBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
|
||||
const upstream = recordValue(responseBody, "upstream");
|
||||
const upstreamBody = recordValue(upstream, "body");
|
||||
const healthBody = hasCodeQueueHealthDetails(responseBody)
|
||||
? responseBody
|
||||
: isCodeQueueHealthBody(upstreamBody)
|
||||
? upstreamBody
|
||||
: isCodeQueueHealthBody(responseBody)
|
||||
? responseBody
|
||||
: null;
|
||||
if (healthBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
|
||||
|
||||
const queue = asRecord(healthBody.queue);
|
||||
const diagnostics = compactCodeQueueDiagnostics(healthBody.executionDiagnostics ?? queue?.executionDiagnostics);
|
||||
const bodyBytes = jsonByteLength(responseBodySource);
|
||||
return {
|
||||
ok: responseRecord.ok ?? responseBody.ok ?? healthBody.ok ?? null,
|
||||
status: responseRecord.status ?? responseBody.status ?? healthBody.status ?? null,
|
||||
serviceId: "code-queue",
|
||||
service: healthBody.service ?? "code-queue",
|
||||
name: responseBody.name ?? null,
|
||||
providerId: responseBody.providerId ?? null,
|
||||
healthPath: responseBody.healthPath ?? "/health",
|
||||
reason: responseBody.reason ?? null,
|
||||
checkedAt: responseBody.checkedAt ?? null,
|
||||
instanceId: healthBody.instanceId ?? null,
|
||||
role: healthBody.role ?? null,
|
||||
deploy: previewJson(healthBody.deploy ?? null, { maxDepth: 2, maxArrayItems: 2, maxObjectKeys: 8, maxStringLength: 160 }),
|
||||
components: compactCodeQueueComponents(healthBody, queue),
|
||||
queue: compactCodeQueueQueueSummary(queue, diagnostics),
|
||||
heartbeat: diagnostics === null ? null : {
|
||||
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt,
|
||||
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt,
|
||||
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt,
|
||||
activeHeartbeatCount: diagnostics.activeHeartbeatCount,
|
||||
activeHeartbeatTaskIds: diagnostics.activeHeartbeatTaskIds,
|
||||
heartbeatFreshTaskCount: diagnostics.heartbeatFreshTaskCount,
|
||||
heartbeatFreshTaskIds: diagnostics.heartbeatFreshTaskIds,
|
||||
heartbeatRiskTaskCount: diagnostics.heartbeatRiskTaskCount,
|
||||
heartbeatRiskTaskIds: diagnostics.heartbeatRiskTaskIds,
|
||||
heartbeatExpiredTaskCount: diagnostics.heartbeatExpiredTaskCount,
|
||||
heartbeatMissingTaskCount: diagnostics.heartbeatMissingTaskCount,
|
||||
staleRecoveryCandidateTaskCount: diagnostics.staleRecoveryCandidateTaskCount,
|
||||
traceGapTaskCount: diagnostics.traceGapTaskCount,
|
||||
schedulerHeartbeatStaleMs: queue?.schedulerHeartbeatStaleMs ?? null,
|
||||
},
|
||||
liveness: diagnostics === null ? null : {
|
||||
state: diagnostics.state,
|
||||
degraded: diagnostics.degraded,
|
||||
splitBrain: diagnostics.splitBrain,
|
||||
splitBrainLive: diagnostics.splitBrainLive,
|
||||
effectiveLiveness: diagnostics.effectiveLiveness,
|
||||
recommendedAction: diagnostics.recommendedAction,
|
||||
interpretation: diagnostics.interpretation,
|
||||
reasons: diagnostics.reasons,
|
||||
},
|
||||
outputPolicy: {
|
||||
mode: "compact",
|
||||
defaultCompact: true,
|
||||
omittedFullHealthBody: true,
|
||||
originalBodyBytes: bodyBytes,
|
||||
idPreviewLimit: 5,
|
||||
rawCommand: "bun scripts/cli.ts microservice health code-queue --raw",
|
||||
fullCommand: "bun scripts/cli.ts microservice health code-queue --raw --full",
|
||||
proxyFullCommand: "bun scripts/cli.ts microservice proxy code-queue /health --raw --full",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function requireId(value: string | undefined, command: string): string {
|
||||
if (value === undefined || value.length === 0) throw new Error(`${command} requires microservice id`);
|
||||
return value;
|
||||
@@ -240,10 +497,6 @@ function hasFlag(args: string[], name: string): boolean {
|
||||
return args.includes(name);
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
function asArray(value: unknown): unknown[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
@@ -272,8 +525,8 @@ function requestBodyOption(args: string[]): unknown | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function assertKnownObservationOptions(args: string[], command: string): void {
|
||||
const flags = new Set(["--full", "--raw"]);
|
||||
function assertKnownObservationOptions(args: string[], command: string, extraFlags: string[] = []): void {
|
||||
const flags = new Set(["--full", "--raw", ...extraFlags]);
|
||||
for (const arg of args) {
|
||||
if (!arg.startsWith("--")) continue;
|
||||
if (!flags.has(arg)) throw new Error(`unsupported ${command} option: ${arg}`);
|
||||
@@ -547,8 +800,8 @@ export async function runMicroserviceCommand(_config: UniDeskConfig, args: strin
|
||||
if (action === "health") {
|
||||
const id = requireId(idArg, "microservice health");
|
||||
const optionArgs = args.slice(2);
|
||||
assertKnownObservationOptions(optionArgs, "microservice health");
|
||||
return summarizeMicroserviceObservation(action, id, coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs);
|
||||
assertKnownObservationOptions(optionArgs, "microservice health", ["--compact"]);
|
||||
return summarizeMicroserviceHealthResponse(coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs, id);
|
||||
}
|
||||
if (action === "diagnostics") {
|
||||
const id = requireId(idArg, "microservice diagnostics");
|
||||
|
||||
Reference in New Issue
Block a user