Merge pull request #80 from pikasTech/code-queue/issue-20-host-commander-governance-docs

fix: compact code queue health output
This commit is contained in:
Lyon
2026-05-23 15:18:10 +08:00
committed by GitHub
7 changed files with 431 additions and 12 deletions
@@ -0,0 +1,153 @@
import { summarizeMicroserviceHealthResponse } from "./src/microservices";
type JsonRecord = Record<string, unknown>;
function assertCondition(condition: unknown, message: string, detail: JsonRecord = {}): void {
if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
}
function asRecord(value: unknown): JsonRecord {
assertCondition(typeof value === "object" && value !== null && !Array.isArray(value), "expected JSON object", { value });
return value as JsonRecord;
}
function manyIds(prefix: string, count: number): string[] {
return Array.from({ length: count }, (_, index) => `${prefix}-${String(index + 1).padStart(3, "0")}`);
}
function longText(prefix: string, count: number): string {
return Array.from({ length: count }, (_, index) => `${prefix} diagnostic row ${index + 1} with queue detail and commander-noise payload`).join("\n");
}
function largeCodeQueueHealth(): JsonRecord {
return {
ok: true,
status: 200,
body: {
ok: true,
service: "code-queue",
instanceId: "D601-code-queue-scheduler",
role: "scheduler",
deploy: {
commit: "abcdef1234567890",
requestedCommit: "abcdef1234567890",
},
databaseReady: true,
schedulerEnabled: true,
queue: {
total: 640,
counts: {
running: 3,
judging: 2,
queued: 41,
retry_wait: 4,
succeeded: 560,
failed: 30,
},
queues: manyIds("queue", 90).map((id) => ({
id,
name: id,
total: 12,
counts: { queued: 5, succeeded: 7 },
diagnostics: longText(id, 18),
})),
queueCount: 90,
activeRunSlotCount: 5,
activeTaskIds: manyIds("active-task", 12),
activeTaskId: "active-task-001",
activeQueueIds: manyIds("active-queue", 12),
processingQueueIds: manyIds("processing-queue", 8),
processing: true,
orphanedActiveTaskCount: 0,
unreadTerminal: 7,
defaultQueueId: "default",
defaultModel: "gpt-5.5",
defaultReasoningEffort: "xhigh",
maxActiveQueues: 0,
schedulerHeartbeatStaleMs: 120000,
},
executionDiagnostics: {
state: "split-brain",
degraded: true,
splitBrain: true,
splitBrainLive: true,
effectiveLiveness: "live",
recommendedAction: "continue-supervision",
executionStateSource: "postgres-control-plane",
controlPlane: "D601-code-queue-scheduler",
databaseActiveTaskCount: 12,
schedulerActiveRunSlotCount: 5,
activeHeartbeatCount: 12,
activeHeartbeatTaskIds: manyIds("active-heartbeat", 12),
heartbeatFreshTaskIds: manyIds("fresh-heartbeat", 12),
heartbeatExpiredTaskIds: [],
heartbeatMissingTaskIds: [],
staleRecoveryCandidateTaskIds: [],
heartbeatRiskTaskIds: [],
traceGapTaskIds: manyIds("trace-gap", 15),
lastSchedulerHeartbeatAt: "2026-05-22T00:00:12.000Z",
lastObservedAgentEventAt: "2026-05-22T00:00:11.000Z",
lastPersistedTraceAt: "2026-05-22T00:00:10.000Z",
reasons: Array.from({ length: 12 }, (_, index) => longText(`reason-${index + 1}`, 8)),
},
modelProviderConfig: {
source: "fixture",
largeNoise: longText("model-provider", 200),
},
},
};
}
const fixture = largeCodeQueueHealth();
const compact = asRecord(summarizeMicroserviceHealthResponse(fixture, ["health", "code-queue"], "code-queue"));
const raw = summarizeMicroserviceHealthResponse(fixture, ["health", "code-queue", "--raw"], "code-queue");
const full = summarizeMicroserviceHealthResponse(fixture, ["health", "code-queue", "--full"], "code-queue");
const unhealthyWrapper = asRecord(summarizeMicroserviceHealthResponse({
ok: false,
status: 503,
body: {
ok: false,
status: "unhealthy",
serviceId: "code-queue",
reason: "health body ok=false",
upstream: {
status: 503,
contentType: "application/json",
body: fixture.body,
},
},
}, ["health", "code-queue"], "code-queue"));
const compactBody = JSON.stringify(compact);
const rawBody = JSON.stringify(fixture);
const queue = asRecord(compact.queue);
const heartbeat = asRecord(compact.heartbeat);
const liveness = asRecord(compact.liveness);
const outputPolicy = asRecord(compact.outputPolicy);
const unhealthyQueue = asRecord(unhealthyWrapper.queue);
const unhealthyLiveness = asRecord(unhealthyWrapper.liveness);
assertCondition(compactBody.length < rawBody.length * 0.25, "default code-queue health output should be materially compact", { compactChars: compactBody.length, rawChars: rawBody.length });
assertCondition(compactBody.length < 12000, "default code-queue health output should stay commander-safe", { compactChars: compactBody.length });
assertCondition(!("body" in compact), "compact health must not retain the raw body", compact);
assertCondition(!("queues" in queue), "compact queue summary must not include full queue dumps", queue);
assertCondition(compact.ok === true && compact.status === 200 && compact.serviceId === "code-queue", "compact health keeps ok/status/service identity", compact);
assertCondition(queue.runningCount === 5 && queue.queueCount === 90, "compact queue keeps running count and queue count", queue);
assertCondition(heartbeat.heartbeatFreshTaskCount === 12 && heartbeat.heartbeatRiskTaskCount === 0, "compact heartbeat keeps freshness and risk counts", heartbeat);
assertCondition(liveness.effectiveLiveness === "live" && liveness.splitBrainLive === true, "compact liveness keeps split-brain live interpretation", liveness);
assertCondition(String(liveness.interpretation).includes("continue supervision"), "compact liveness includes commander interpretation", liveness);
assertCondition(typeof outputPolicy.rawCommand === "string" && String(outputPolicy.rawCommand).includes("--raw"), "compact output exposes raw drill-down command", outputPolicy);
assertCondition(raw === fixture && full === fixture, "--raw/--full must preserve full health access", { rawSame: raw === fixture, fullSame: full === fixture });
assertCondition(unhealthyWrapper.ok === false && unhealthyWrapper.reason === "health body ok=false", "unhealthy wrapper keeps probe failure", unhealthyWrapper);
assertCondition(unhealthyQueue.runningCount === 5 && unhealthyLiveness.effectiveLiveness === "live", "unhealthy wrapper still surfaces upstream queue liveness", { unhealthyQueue, unhealthyLiveness });
console.log(JSON.stringify({
ok: true,
checks: [
"default code-queue health output is compact and omits raw body/queue dumps",
"critical ok/status/service/running/heartbeat/liveness fields remain visible",
"unhealthy backend health wrapper still surfaces upstream code-queue liveness",
"--raw and --full return the original health response for diagnostics",
],
compactChars: compactBody.length,
rawChars: rawBody.length,
}, null, 2));
+1
View File
@@ -34,6 +34,7 @@ const syntaxFiles = [
"scripts/code-queue-cli-steer-test.ts",
"scripts/code-queue-cli-submit-prompt-contract-test.ts",
"scripts/code-queue-gh-auth-redaction-contract-test.ts",
"scripts/microservice-health-output-contract-test.ts",
"scripts/code-queue-supervisor-disclosure-contract-test.ts",
"src/components/frontend/src/index.ts",
"src/components/frontend/src/app.tsx",
+2 -2
View File
@@ -27,7 +27,7 @@ export function rootHelp(): unknown {
{ command: "ssh <providerId> argv <command> [args...]", description: "Run a remote command with each argv token shell-quoted by UniDesk before SSH passthrough." },
{ command: "microservice list", description: "List UniDesk-managed user services and their provider/runtime mapping." },
{ command: "microservice status <id>", description: "Show one user service config, repository reference, backend mapping, and runtime status." },
{ command: "microservice health <id> [--full|--raw]", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy; default output is a compact health summary." },
{ command: "microservice health <id> [--compact|--raw|--full]", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy; default output is compact, and code-queue uses a commander-safe liveness summary unless raw/full is requested." },
{ command: "microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--body-json JSON|--body-file path|--body-stdin] [--raw] [--max-body-bytes N]", description: "Access a private user-service backend path through the same frontend-only proxy used by WebUI; JSON request bodies are supported for controlled write/debug endpoints." },
{ command: "microservice diagnostics <id> [--full|--raw]", description: "Split k3sctl-managed proxy health into a compact summary by default; use --full/--raw for complete evidence." },
{ command: "microservice tunnel-self-test <id>", description: "Trigger an expected provider HTTP tunnel failure and verify requestId/stage diagnostics are returned." },
@@ -167,7 +167,7 @@ function microserviceHelp(): unknown {
usage: [
"bun scripts/cli.ts microservice list",
"bun scripts/cli.ts microservice status <id> [--full|--raw]",
"bun scripts/cli.ts microservice health <id> [--full|--raw]",
"bun scripts/cli.ts microservice health <id> [--compact|--full|--raw]",
"bun scripts/cli.ts microservice diagnostics <id> [--full|--raw]",
"bun scripts/cli.ts microservice tunnel-self-test <id>",
"bun scripts/cli.ts microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--body-json JSON|--body-file path|--body-stdin] [--raw] [--full] [--max-body-bytes N]",
+261 -8
View File
@@ -199,6 +199,263 @@ function parseJsonRecord(text: string): Record<string, unknown> | null {
}
}
function asRecord(value: unknown): Record<string, unknown> | null {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
}
function asNumber(value: unknown, fallback = 0): number {
const number = Number(value);
return Number.isFinite(number) ? number : fallback;
}
function stringList(value: unknown): string[] {
if (!Array.isArray(value)) return [];
return value.map((item) => String(item)).filter((item) => item.length > 0);
}
function boundedUniqueStringList(value: unknown, limit = 5): Record<string, unknown> {
const all = Array.from(new Set(stringList(value))).sort();
const items = all.slice(0, limit);
return {
items,
count: all.length,
omitted: Math.max(0, all.length - items.length),
truncated: all.length > items.length,
};
}
function limitedStrings(value: unknown, limit: number, maxLength: number): string[] {
return stringList(value).slice(0, limit).map((item) => item.length > maxLength ? item.slice(0, maxLength) : item);
}
function recordValue(record: Record<string, unknown> | null, key: string): Record<string, unknown> | null {
return record === null ? null : asRecord(record[key]);
}
function isCodeQueueHealthBody(record: Record<string, unknown> | null): boolean {
if (record === null) return false;
return record.service === "code-queue"
|| record.serviceId === "code-queue"
|| asRecord(record.queue) !== null
|| asRecord(record.executionDiagnostics) !== null;
}
function hasCodeQueueHealthDetails(record: Record<string, unknown> | null): boolean {
if (record === null) return false;
return record.service === "code-queue"
|| asRecord(record.queue) !== null
|| asRecord(record.executionDiagnostics) !== null;
}
function splitBrainLiveFromDiagnostics(record: Record<string, unknown>): boolean {
if (typeof record.splitBrainLive === "boolean") return record.splitBrainLive;
const state = String(record.state ?? record.health ?? "").toLowerCase();
const riskTaskIds = Array.from(new Set([
...stringList(record.heartbeatRiskTaskIds),
...stringList(record.heartbeatExpiredTaskIds),
...stringList(record.heartbeatMissingTaskIds),
...stringList(record.staleRecoveryCandidateTaskIds),
]));
return state === "split-brain" && stringList(record.heartbeatFreshTaskIds).length > 0 && riskTaskIds.length === 0;
}
function effectiveLivenessFromDiagnostics(record: Record<string, unknown>): string {
if (typeof record.effectiveLiveness === "string" && record.effectiveLiveness.length > 0) return record.effectiveLiveness;
if (stringList(record.heartbeatRiskTaskIds).length > 0
|| stringList(record.heartbeatExpiredTaskIds).length > 0
|| stringList(record.heartbeatMissingTaskIds).length > 0
|| stringList(record.staleRecoveryCandidateTaskIds).length > 0) {
return "at-risk";
}
if (splitBrainLiveFromDiagnostics(record)) return "live";
return String(record.state ?? record.health ?? "unknown") === "healthy" ? "healthy" : "degraded";
}
function recommendedActionFromDiagnostics(record: Record<string, unknown>, effectiveLiveness: string): string {
if (typeof record.recommendedAction === "string" && record.recommendedAction.length > 0) return record.recommendedAction;
if (effectiveLiveness === "at-risk") return "investigate-heartbeat-risk";
if (effectiveLiveness === "live") return "continue-supervision";
if (effectiveLiveness === "degraded") return "observe-degraded";
return "none";
}
function livenessInterpretation(effectiveLiveness: string, splitBrainLive: boolean): string {
if (splitBrainLive) return "scheduler heartbeat is fresh; treat active task count from heartbeat as live and continue supervision";
if (effectiveLiveness === "at-risk") return "heartbeat risk is present; investigate heartbeat freshness before recovery";
if (effectiveLiveness === "degraded") return "diagnostics are degraded; cross-check heartbeat, trace and control-plane sources";
return "diagnostics indicate healthy liveness";
}
function compactCodeQueueDiagnostics(value: unknown): Record<string, unknown> | null {
const diagnostics = asRecord(value);
if (diagnostics === null) return null;
const heartbeatRiskIds = Array.from(new Set([
...stringList(diagnostics.heartbeatRiskTaskIds),
...stringList(diagnostics.heartbeatExpiredTaskIds),
...stringList(diagnostics.heartbeatMissingTaskIds),
...stringList(diagnostics.staleRecoveryCandidateTaskIds),
])).sort();
const splitBrainLive = splitBrainLiveFromDiagnostics(diagnostics);
const effectiveLiveness = effectiveLivenessFromDiagnostics(diagnostics);
const recommendedAction = recommendedActionFromDiagnostics(diagnostics, effectiveLiveness);
const heartbeatFreshTaskIds = boundedUniqueStringList(diagnostics.heartbeatFreshTaskIds);
const activeHeartbeatTaskIds = boundedUniqueStringList(diagnostics.activeHeartbeatTaskIds);
return {
state: diagnostics.state ?? diagnostics.health ?? null,
degraded: diagnostics.degraded ?? null,
splitBrain: diagnostics.splitBrain ?? null,
splitBrainLive,
effectiveLiveness,
recommendedAction,
interpretation: livenessInterpretation(effectiveLiveness, splitBrainLive),
executionStateSource: diagnostics.executionStateSource ?? null,
controlPlane: diagnostics.controlPlane ?? null,
databaseActiveTaskCount: diagnostics.databaseActiveTaskCount ?? stringList(diagnostics.databaseActiveTaskIds).length,
schedulerActiveRunSlotCount: diagnostics.schedulerActiveRunSlotCount ?? null,
activeHeartbeatCount: diagnostics.activeHeartbeatCount ?? Math.max(asNumber(activeHeartbeatTaskIds.count), asNumber(heartbeatFreshTaskIds.count)),
activeHeartbeatTaskIds,
heartbeatFreshTaskCount: heartbeatFreshTaskIds.count,
heartbeatFreshTaskIds,
heartbeatRiskTaskCount: heartbeatRiskIds.length,
heartbeatRiskTaskIds: boundedUniqueStringList(heartbeatRiskIds),
heartbeatExpiredTaskCount: stringList(diagnostics.heartbeatExpiredTaskIds).length,
heartbeatMissingTaskCount: stringList(diagnostics.heartbeatMissingTaskIds).length,
staleRecoveryCandidateTaskCount: stringList(diagnostics.staleRecoveryCandidateTaskIds).length,
traceGapTaskCount: stringList(diagnostics.traceGapTaskIds).length,
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt ?? null,
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt ?? null,
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt ?? null,
reasons: limitedStrings(diagnostics.reasons, 3, 240),
};
}
function compactCodeQueueQueueSummary(value: unknown, diagnostics: Record<string, unknown> | null): Record<string, unknown> {
const queue = asRecord(value);
const counts = asRecord(queue?.counts) ?? {};
const running = asNumber(counts.running);
const judging = asNumber(counts.judging);
const activeTaskIds = boundedUniqueStringList(queue?.activeTaskIds);
const activeQueueIds = boundedUniqueStringList(queue?.activeQueueIds);
const processingQueueIds = boundedUniqueStringList(queue?.processingQueueIds);
return {
total: queue?.total ?? null,
counts,
runningCount: running + judging,
running,
judging,
queued: asNumber(counts.queued),
retryWait: asNumber(counts.retry_wait),
queueCount: queue?.queueCount ?? null,
unreadTerminal: queue?.unreadTerminal ?? null,
activeRunSlotCount: queue?.activeRunSlotCount ?? diagnostics?.schedulerActiveRunSlotCount ?? null,
activeTaskCount: activeTaskIds.count,
activeTaskIds,
activeTaskId: queue?.activeTaskId ?? null,
activeQueueIds,
processingQueueIds,
processing: queue?.processing ?? null,
orphanedActiveTaskCount: queue?.orphanedActiveTaskCount ?? null,
defaultQueueId: queue?.defaultQueueId ?? null,
defaultModel: queue?.defaultModel ?? null,
defaultReasoningEffort: queue?.defaultReasoningEffort ?? null,
maxActiveQueues: queue?.maxActiveQueues ?? null,
};
}
function compactCodeQueueComponents(body: Record<string, unknown>, queue: Record<string, unknown> | null): Record<string, unknown> {
const storage = asRecord(queue?.storage);
const egressProxy = asRecord(body.egressProxy);
const oaPublisher = asRecord(body.oaEventPublisher);
return {
databaseReady: body.databaseReady ?? storage?.postgresReady ?? null,
serviceReady: body.serviceReady ?? null,
schedulerEnabled: body.schedulerEnabled ?? null,
egressProxyConnected: egressProxy?.connected ?? null,
oaEventPublisher: oaPublisher === null ? null : {
enabled: oaPublisher.enabled ?? null,
pendingCount: oaPublisher.pendingCount ?? null,
lastError: oaPublisher.lastError ?? null,
},
};
}
export function summarizeMicroserviceHealthResponse(response: unknown, args: string[], serviceId: string): unknown {
if (serviceId !== "code-queue") return summarizeMicroserviceObservation("health", serviceId, response, args);
if (hasFlag(args, "--raw") || hasFlag(args, "--full")) return response;
const responseRecord = asRecord(response);
if (responseRecord === null) return response;
const responseBodySource = "body" in responseRecord ? responseRecord.body : responseRecord;
const responseBody = asRecord(responseBodySource);
if (responseBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
const upstream = recordValue(responseBody, "upstream");
const upstreamBody = recordValue(upstream, "body");
const healthBody = hasCodeQueueHealthDetails(responseBody)
? responseBody
: isCodeQueueHealthBody(upstreamBody)
? upstreamBody
: isCodeQueueHealthBody(responseBody)
? responseBody
: null;
if (healthBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
const queue = asRecord(healthBody.queue);
const diagnostics = compactCodeQueueDiagnostics(healthBody.executionDiagnostics ?? queue?.executionDiagnostics);
const bodyBytes = jsonByteLength(responseBodySource);
return {
ok: responseRecord.ok ?? responseBody.ok ?? healthBody.ok ?? null,
status: responseRecord.status ?? responseBody.status ?? healthBody.status ?? null,
serviceId: "code-queue",
service: healthBody.service ?? "code-queue",
name: responseBody.name ?? null,
providerId: responseBody.providerId ?? null,
healthPath: responseBody.healthPath ?? "/health",
reason: responseBody.reason ?? null,
checkedAt: responseBody.checkedAt ?? null,
instanceId: healthBody.instanceId ?? null,
role: healthBody.role ?? null,
deploy: previewJson(healthBody.deploy ?? null, { maxDepth: 2, maxArrayItems: 2, maxObjectKeys: 8, maxStringLength: 160 }),
components: compactCodeQueueComponents(healthBody, queue),
queue: compactCodeQueueQueueSummary(queue, diagnostics),
heartbeat: diagnostics === null ? null : {
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt,
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt,
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt,
activeHeartbeatCount: diagnostics.activeHeartbeatCount,
activeHeartbeatTaskIds: diagnostics.activeHeartbeatTaskIds,
heartbeatFreshTaskCount: diagnostics.heartbeatFreshTaskCount,
heartbeatFreshTaskIds: diagnostics.heartbeatFreshTaskIds,
heartbeatRiskTaskCount: diagnostics.heartbeatRiskTaskCount,
heartbeatRiskTaskIds: diagnostics.heartbeatRiskTaskIds,
heartbeatExpiredTaskCount: diagnostics.heartbeatExpiredTaskCount,
heartbeatMissingTaskCount: diagnostics.heartbeatMissingTaskCount,
staleRecoveryCandidateTaskCount: diagnostics.staleRecoveryCandidateTaskCount,
traceGapTaskCount: diagnostics.traceGapTaskCount,
schedulerHeartbeatStaleMs: queue?.schedulerHeartbeatStaleMs ?? null,
},
liveness: diagnostics === null ? null : {
state: diagnostics.state,
degraded: diagnostics.degraded,
splitBrain: diagnostics.splitBrain,
splitBrainLive: diagnostics.splitBrainLive,
effectiveLiveness: diagnostics.effectiveLiveness,
recommendedAction: diagnostics.recommendedAction,
interpretation: diagnostics.interpretation,
reasons: diagnostics.reasons,
},
outputPolicy: {
mode: "compact",
defaultCompact: true,
omittedFullHealthBody: true,
originalBodyBytes: bodyBytes,
idPreviewLimit: 5,
rawCommand: "bun scripts/cli.ts microservice health code-queue --raw",
fullCommand: "bun scripts/cli.ts microservice health code-queue --raw --full",
proxyFullCommand: "bun scripts/cli.ts microservice proxy code-queue /health --raw --full",
},
};
}
function requireId(value: string | undefined, command: string): string {
if (value === undefined || value.length === 0) throw new Error(`${command} requires microservice id`);
return value;
@@ -240,10 +497,6 @@ function hasFlag(args: string[], name: string): boolean {
return args.includes(name);
}
function asRecord(value: unknown): Record<string, unknown> | null {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
}
function asArray(value: unknown): unknown[] {
return Array.isArray(value) ? value : [];
}
@@ -272,8 +525,8 @@ function requestBodyOption(args: string[]): unknown | undefined {
return undefined;
}
function assertKnownObservationOptions(args: string[], command: string): void {
const flags = new Set(["--full", "--raw"]);
function assertKnownObservationOptions(args: string[], command: string, extraFlags: string[] = []): void {
const flags = new Set(["--full", "--raw", ...extraFlags]);
for (const arg of args) {
if (!arg.startsWith("--")) continue;
if (!flags.has(arg)) throw new Error(`unsupported ${command} option: ${arg}`);
@@ -547,8 +800,8 @@ export async function runMicroserviceCommand(_config: UniDeskConfig, args: strin
if (action === "health") {
const id = requireId(idArg, "microservice health");
const optionArgs = args.slice(2);
assertKnownObservationOptions(optionArgs, "microservice health");
return summarizeMicroserviceObservation(action, id, coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs);
assertKnownObservationOptions(optionArgs, "microservice health", ["--compact"]);
return summarizeMicroserviceHealthResponse(coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs, id);
}
if (action === "diagnostics") {
const id = requireId(idArg, "microservice diagnostics");