Merge pull request #80 from pikasTech/code-queue/issue-20-host-commander-governance-docs
fix: compact code queue health output
This commit is contained in:
Lyon
@@ -34,6 +34,7 @@ const syntaxFiles = [
|
||||
"scripts/code-queue-cli-steer-test.ts",
|
||||
"scripts/code-queue-cli-submit-prompt-contract-test.ts",
|
||||
"scripts/code-queue-gh-auth-redaction-contract-test.ts",
|
||||
"scripts/microservice-health-output-contract-test.ts",
|
||||
"scripts/code-queue-supervisor-disclosure-contract-test.ts",
|
||||
"src/components/frontend/src/index.ts",
|
||||
"src/components/frontend/src/app.tsx",
|
||||
|
||||
+2
-2
@@ -27,7 +27,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "ssh <providerId> argv <command> [args...]", description: "Run a remote command with each argv token shell-quoted by UniDesk before SSH passthrough." },
|
||||
{ command: "microservice list", description: "List UniDesk-managed user services and their provider/runtime mapping." },
|
||||
{ command: "microservice status <id>", description: "Show one user service config, repository reference, backend mapping, and runtime status." },
|
||||
{ command: "microservice health <id> [--full|--raw]", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy; default output is a compact health summary." },
|
||||
{ command: "microservice health <id> [--compact|--raw|--full]", description: "Probe one user service through backend-core -> provider-gateway HTTP proxy; default output is compact, and code-queue uses a commander-safe liveness summary unless raw/full is requested." },
|
||||
{ command: "microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--body-json JSON|--body-file path|--body-stdin] [--raw] [--max-body-bytes N]", description: "Access a private user-service backend path through the same frontend-only proxy used by WebUI; JSON request bodies are supported for controlled write/debug endpoints." },
|
||||
{ command: "microservice diagnostics <id> [--full|--raw]", description: "Split k3sctl-managed proxy health into a compact summary by default; use --full/--raw for complete evidence." },
|
||||
{ command: "microservice tunnel-self-test <id>", description: "Trigger an expected provider HTTP tunnel failure and verify requestId/stage diagnostics are returned." },
|
||||
@@ -167,7 +167,7 @@ function microserviceHelp(): unknown {
|
||||
usage: [
|
||||
"bun scripts/cli.ts microservice list",
|
||||
"bun scripts/cli.ts microservice status <id> [--full|--raw]",
|
||||
"bun scripts/cli.ts microservice health <id> [--full|--raw]",
|
||||
"bun scripts/cli.ts microservice health <id> [--compact|--full|--raw]",
|
||||
"bun scripts/cli.ts microservice diagnostics <id> [--full|--raw]",
|
||||
"bun scripts/cli.ts microservice tunnel-self-test <id>",
|
||||
"bun scripts/cli.ts microservice proxy <id> <path> [--method GET|POST|PUT|PATCH|DELETE] [--body-json JSON|--body-file path|--body-stdin] [--raw] [--full] [--max-body-bytes N]",
|
||||
|
||||
@@ -199,6 +199,263 @@ function parseJsonRecord(text: string): Record<string, unknown> | null {
|
||||
}
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
function asNumber(value: unknown, fallback = 0): number {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? number : fallback;
|
||||
}
|
||||
|
||||
function stringList(value: unknown): string[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.map((item) => String(item)).filter((item) => item.length > 0);
|
||||
}
|
||||
|
||||
function boundedUniqueStringList(value: unknown, limit = 5): Record<string, unknown> {
|
||||
const all = Array.from(new Set(stringList(value))).sort();
|
||||
const items = all.slice(0, limit);
|
||||
return {
|
||||
items,
|
||||
count: all.length,
|
||||
omitted: Math.max(0, all.length - items.length),
|
||||
truncated: all.length > items.length,
|
||||
};
|
||||
}
|
||||
|
||||
function limitedStrings(value: unknown, limit: number, maxLength: number): string[] {
|
||||
return stringList(value).slice(0, limit).map((item) => item.length > maxLength ? item.slice(0, maxLength) : item);
|
||||
}
|
||||
|
||||
function recordValue(record: Record<string, unknown> | null, key: string): Record<string, unknown> | null {
|
||||
return record === null ? null : asRecord(record[key]);
|
||||
}
|
||||
|
||||
function isCodeQueueHealthBody(record: Record<string, unknown> | null): boolean {
|
||||
if (record === null) return false;
|
||||
return record.service === "code-queue"
|
||||
|| record.serviceId === "code-queue"
|
||||
|| asRecord(record.queue) !== null
|
||||
|| asRecord(record.executionDiagnostics) !== null;
|
||||
}
|
||||
|
||||
function hasCodeQueueHealthDetails(record: Record<string, unknown> | null): boolean {
|
||||
if (record === null) return false;
|
||||
return record.service === "code-queue"
|
||||
|| asRecord(record.queue) !== null
|
||||
|| asRecord(record.executionDiagnostics) !== null;
|
||||
}
|
||||
|
||||
function splitBrainLiveFromDiagnostics(record: Record<string, unknown>): boolean {
|
||||
if (typeof record.splitBrainLive === "boolean") return record.splitBrainLive;
|
||||
const state = String(record.state ?? record.health ?? "").toLowerCase();
|
||||
const riskTaskIds = Array.from(new Set([
|
||||
...stringList(record.heartbeatRiskTaskIds),
|
||||
...stringList(record.heartbeatExpiredTaskIds),
|
||||
...stringList(record.heartbeatMissingTaskIds),
|
||||
...stringList(record.staleRecoveryCandidateTaskIds),
|
||||
]));
|
||||
return state === "split-brain" && stringList(record.heartbeatFreshTaskIds).length > 0 && riskTaskIds.length === 0;
|
||||
}
|
||||
|
||||
function effectiveLivenessFromDiagnostics(record: Record<string, unknown>): string {
|
||||
if (typeof record.effectiveLiveness === "string" && record.effectiveLiveness.length > 0) return record.effectiveLiveness;
|
||||
if (stringList(record.heartbeatRiskTaskIds).length > 0
|
||||
|| stringList(record.heartbeatExpiredTaskIds).length > 0
|
||||
|| stringList(record.heartbeatMissingTaskIds).length > 0
|
||||
|| stringList(record.staleRecoveryCandidateTaskIds).length > 0) {
|
||||
return "at-risk";
|
||||
}
|
||||
if (splitBrainLiveFromDiagnostics(record)) return "live";
|
||||
return String(record.state ?? record.health ?? "unknown") === "healthy" ? "healthy" : "degraded";
|
||||
}
|
||||
|
||||
function recommendedActionFromDiagnostics(record: Record<string, unknown>, effectiveLiveness: string): string {
|
||||
if (typeof record.recommendedAction === "string" && record.recommendedAction.length > 0) return record.recommendedAction;
|
||||
if (effectiveLiveness === "at-risk") return "investigate-heartbeat-risk";
|
||||
if (effectiveLiveness === "live") return "continue-supervision";
|
||||
if (effectiveLiveness === "degraded") return "observe-degraded";
|
||||
return "none";
|
||||
}
|
||||
|
||||
function livenessInterpretation(effectiveLiveness: string, splitBrainLive: boolean): string {
|
||||
if (splitBrainLive) return "scheduler heartbeat is fresh; treat active task count from heartbeat as live and continue supervision";
|
||||
if (effectiveLiveness === "at-risk") return "heartbeat risk is present; investigate heartbeat freshness before recovery";
|
||||
if (effectiveLiveness === "degraded") return "diagnostics are degraded; cross-check heartbeat, trace and control-plane sources";
|
||||
return "diagnostics indicate healthy liveness";
|
||||
}
|
||||
|
||||
function compactCodeQueueDiagnostics(value: unknown): Record<string, unknown> | null {
|
||||
const diagnostics = asRecord(value);
|
||||
if (diagnostics === null) return null;
|
||||
const heartbeatRiskIds = Array.from(new Set([
|
||||
...stringList(diagnostics.heartbeatRiskTaskIds),
|
||||
...stringList(diagnostics.heartbeatExpiredTaskIds),
|
||||
...stringList(diagnostics.heartbeatMissingTaskIds),
|
||||
...stringList(diagnostics.staleRecoveryCandidateTaskIds),
|
||||
])).sort();
|
||||
const splitBrainLive = splitBrainLiveFromDiagnostics(diagnostics);
|
||||
const effectiveLiveness = effectiveLivenessFromDiagnostics(diagnostics);
|
||||
const recommendedAction = recommendedActionFromDiagnostics(diagnostics, effectiveLiveness);
|
||||
const heartbeatFreshTaskIds = boundedUniqueStringList(diagnostics.heartbeatFreshTaskIds);
|
||||
const activeHeartbeatTaskIds = boundedUniqueStringList(diagnostics.activeHeartbeatTaskIds);
|
||||
return {
|
||||
state: diagnostics.state ?? diagnostics.health ?? null,
|
||||
degraded: diagnostics.degraded ?? null,
|
||||
splitBrain: diagnostics.splitBrain ?? null,
|
||||
splitBrainLive,
|
||||
effectiveLiveness,
|
||||
recommendedAction,
|
||||
interpretation: livenessInterpretation(effectiveLiveness, splitBrainLive),
|
||||
executionStateSource: diagnostics.executionStateSource ?? null,
|
||||
controlPlane: diagnostics.controlPlane ?? null,
|
||||
databaseActiveTaskCount: diagnostics.databaseActiveTaskCount ?? stringList(diagnostics.databaseActiveTaskIds).length,
|
||||
schedulerActiveRunSlotCount: diagnostics.schedulerActiveRunSlotCount ?? null,
|
||||
activeHeartbeatCount: diagnostics.activeHeartbeatCount ?? Math.max(asNumber(activeHeartbeatTaskIds.count), asNumber(heartbeatFreshTaskIds.count)),
|
||||
activeHeartbeatTaskIds,
|
||||
heartbeatFreshTaskCount: heartbeatFreshTaskIds.count,
|
||||
heartbeatFreshTaskIds,
|
||||
heartbeatRiskTaskCount: heartbeatRiskIds.length,
|
||||
heartbeatRiskTaskIds: boundedUniqueStringList(heartbeatRiskIds),
|
||||
heartbeatExpiredTaskCount: stringList(diagnostics.heartbeatExpiredTaskIds).length,
|
||||
heartbeatMissingTaskCount: stringList(diagnostics.heartbeatMissingTaskIds).length,
|
||||
staleRecoveryCandidateTaskCount: stringList(diagnostics.staleRecoveryCandidateTaskIds).length,
|
||||
traceGapTaskCount: stringList(diagnostics.traceGapTaskIds).length,
|
||||
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt ?? null,
|
||||
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt ?? null,
|
||||
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt ?? null,
|
||||
reasons: limitedStrings(diagnostics.reasons, 3, 240),
|
||||
};
|
||||
}
|
||||
|
||||
function compactCodeQueueQueueSummary(value: unknown, diagnostics: Record<string, unknown> | null): Record<string, unknown> {
|
||||
const queue = asRecord(value);
|
||||
const counts = asRecord(queue?.counts) ?? {};
|
||||
const running = asNumber(counts.running);
|
||||
const judging = asNumber(counts.judging);
|
||||
const activeTaskIds = boundedUniqueStringList(queue?.activeTaskIds);
|
||||
const activeQueueIds = boundedUniqueStringList(queue?.activeQueueIds);
|
||||
const processingQueueIds = boundedUniqueStringList(queue?.processingQueueIds);
|
||||
return {
|
||||
total: queue?.total ?? null,
|
||||
counts,
|
||||
runningCount: running + judging,
|
||||
running,
|
||||
judging,
|
||||
queued: asNumber(counts.queued),
|
||||
retryWait: asNumber(counts.retry_wait),
|
||||
queueCount: queue?.queueCount ?? null,
|
||||
unreadTerminal: queue?.unreadTerminal ?? null,
|
||||
activeRunSlotCount: queue?.activeRunSlotCount ?? diagnostics?.schedulerActiveRunSlotCount ?? null,
|
||||
activeTaskCount: activeTaskIds.count,
|
||||
activeTaskIds,
|
||||
activeTaskId: queue?.activeTaskId ?? null,
|
||||
activeQueueIds,
|
||||
processingQueueIds,
|
||||
processing: queue?.processing ?? null,
|
||||
orphanedActiveTaskCount: queue?.orphanedActiveTaskCount ?? null,
|
||||
defaultQueueId: queue?.defaultQueueId ?? null,
|
||||
defaultModel: queue?.defaultModel ?? null,
|
||||
defaultReasoningEffort: queue?.defaultReasoningEffort ?? null,
|
||||
maxActiveQueues: queue?.maxActiveQueues ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function compactCodeQueueComponents(body: Record<string, unknown>, queue: Record<string, unknown> | null): Record<string, unknown> {
|
||||
const storage = asRecord(queue?.storage);
|
||||
const egressProxy = asRecord(body.egressProxy);
|
||||
const oaPublisher = asRecord(body.oaEventPublisher);
|
||||
return {
|
||||
databaseReady: body.databaseReady ?? storage?.postgresReady ?? null,
|
||||
serviceReady: body.serviceReady ?? null,
|
||||
schedulerEnabled: body.schedulerEnabled ?? null,
|
||||
egressProxyConnected: egressProxy?.connected ?? null,
|
||||
oaEventPublisher: oaPublisher === null ? null : {
|
||||
enabled: oaPublisher.enabled ?? null,
|
||||
pendingCount: oaPublisher.pendingCount ?? null,
|
||||
lastError: oaPublisher.lastError ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeMicroserviceHealthResponse(response: unknown, args: string[], serviceId: string): unknown {
|
||||
if (serviceId !== "code-queue") return summarizeMicroserviceObservation("health", serviceId, response, args);
|
||||
if (hasFlag(args, "--raw") || hasFlag(args, "--full")) return response;
|
||||
|
||||
const responseRecord = asRecord(response);
|
||||
if (responseRecord === null) return response;
|
||||
const responseBodySource = "body" in responseRecord ? responseRecord.body : responseRecord;
|
||||
const responseBody = asRecord(responseBodySource);
|
||||
if (responseBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
|
||||
const upstream = recordValue(responseBody, "upstream");
|
||||
const upstreamBody = recordValue(upstream, "body");
|
||||
const healthBody = hasCodeQueueHealthDetails(responseBody)
|
||||
? responseBody
|
||||
: isCodeQueueHealthBody(upstreamBody)
|
||||
? upstreamBody
|
||||
: isCodeQueueHealthBody(responseBody)
|
||||
? responseBody
|
||||
: null;
|
||||
if (healthBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
|
||||
|
||||
const queue = asRecord(healthBody.queue);
|
||||
const diagnostics = compactCodeQueueDiagnostics(healthBody.executionDiagnostics ?? queue?.executionDiagnostics);
|
||||
const bodyBytes = jsonByteLength(responseBodySource);
|
||||
return {
|
||||
ok: responseRecord.ok ?? responseBody.ok ?? healthBody.ok ?? null,
|
||||
status: responseRecord.status ?? responseBody.status ?? healthBody.status ?? null,
|
||||
serviceId: "code-queue",
|
||||
service: healthBody.service ?? "code-queue",
|
||||
name: responseBody.name ?? null,
|
||||
providerId: responseBody.providerId ?? null,
|
||||
healthPath: responseBody.healthPath ?? "/health",
|
||||
reason: responseBody.reason ?? null,
|
||||
checkedAt: responseBody.checkedAt ?? null,
|
||||
instanceId: healthBody.instanceId ?? null,
|
||||
role: healthBody.role ?? null,
|
||||
deploy: previewJson(healthBody.deploy ?? null, { maxDepth: 2, maxArrayItems: 2, maxObjectKeys: 8, maxStringLength: 160 }),
|
||||
components: compactCodeQueueComponents(healthBody, queue),
|
||||
queue: compactCodeQueueQueueSummary(queue, diagnostics),
|
||||
heartbeat: diagnostics === null ? null : {
|
||||
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt,
|
||||
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt,
|
||||
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt,
|
||||
activeHeartbeatCount: diagnostics.activeHeartbeatCount,
|
||||
activeHeartbeatTaskIds: diagnostics.activeHeartbeatTaskIds,
|
||||
heartbeatFreshTaskCount: diagnostics.heartbeatFreshTaskCount,
|
||||
heartbeatFreshTaskIds: diagnostics.heartbeatFreshTaskIds,
|
||||
heartbeatRiskTaskCount: diagnostics.heartbeatRiskTaskCount,
|
||||
heartbeatRiskTaskIds: diagnostics.heartbeatRiskTaskIds,
|
||||
heartbeatExpiredTaskCount: diagnostics.heartbeatExpiredTaskCount,
|
||||
heartbeatMissingTaskCount: diagnostics.heartbeatMissingTaskCount,
|
||||
staleRecoveryCandidateTaskCount: diagnostics.staleRecoveryCandidateTaskCount,
|
||||
traceGapTaskCount: diagnostics.traceGapTaskCount,
|
||||
schedulerHeartbeatStaleMs: queue?.schedulerHeartbeatStaleMs ?? null,
|
||||
},
|
||||
liveness: diagnostics === null ? null : {
|
||||
state: diagnostics.state,
|
||||
degraded: diagnostics.degraded,
|
||||
splitBrain: diagnostics.splitBrain,
|
||||
splitBrainLive: diagnostics.splitBrainLive,
|
||||
effectiveLiveness: diagnostics.effectiveLiveness,
|
||||
recommendedAction: diagnostics.recommendedAction,
|
||||
interpretation: diagnostics.interpretation,
|
||||
reasons: diagnostics.reasons,
|
||||
},
|
||||
outputPolicy: {
|
||||
mode: "compact",
|
||||
defaultCompact: true,
|
||||
omittedFullHealthBody: true,
|
||||
originalBodyBytes: bodyBytes,
|
||||
idPreviewLimit: 5,
|
||||
rawCommand: "bun scripts/cli.ts microservice health code-queue --raw",
|
||||
fullCommand: "bun scripts/cli.ts microservice health code-queue --raw --full",
|
||||
proxyFullCommand: "bun scripts/cli.ts microservice proxy code-queue /health --raw --full",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function requireId(value: string | undefined, command: string): string {
|
||||
if (value === undefined || value.length === 0) throw new Error(`${command} requires microservice id`);
|
||||
return value;
|
||||
@@ -240,10 +497,6 @@ function hasFlag(args: string[], name: string): boolean {
|
||||
return args.includes(name);
|
||||
}
|
||||
|
||||
function asRecord(value: unknown): Record<string, unknown> | null {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
function asArray(value: unknown): unknown[] {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
@@ -272,8 +525,8 @@ function requestBodyOption(args: string[]): unknown | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function assertKnownObservationOptions(args: string[], command: string): void {
|
||||
const flags = new Set(["--full", "--raw"]);
|
||||
function assertKnownObservationOptions(args: string[], command: string, extraFlags: string[] = []): void {
|
||||
const flags = new Set(["--full", "--raw", ...extraFlags]);
|
||||
for (const arg of args) {
|
||||
if (!arg.startsWith("--")) continue;
|
||||
if (!flags.has(arg)) throw new Error(`unsupported ${command} option: ${arg}`);
|
||||
@@ -547,8 +800,8 @@ export async function runMicroserviceCommand(_config: UniDeskConfig, args: strin
|
||||
if (action === "health") {
|
||||
const id = requireId(idArg, "microservice health");
|
||||
const optionArgs = args.slice(2);
|
||||
assertKnownObservationOptions(optionArgs, "microservice health");
|
||||
return summarizeMicroserviceObservation(action, id, coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs);
|
||||
assertKnownObservationOptions(optionArgs, "microservice health", ["--compact"]);
|
||||
return summarizeMicroserviceHealthResponse(coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs, id);
|
||||
}
|
||||
if (action === "diagnostics") {
|
||||
const id = requireId(idArg, "microservice diagnostics");
|
||||
|
||||
Reference in New Issue
Block a user