Codex
849 lines
37 KiB
TypeScript
849 lines
37 KiB
TypeScript
import { existsSync, readFileSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { runCommand } from "./command";
|
|
import { type UniDeskConfig, repoRoot } from "./config";
|
|
import { jsonByteLength, previewJson } from "./preview";
|
|
|
|
function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
}
|
|
|
|
function dockerCoreFetchCommand(path: string, init?: { method?: string; body?: unknown; maxResponseBytes?: number }): string[] {
|
|
const maxResponseBytes = Math.max(1024, Math.floor(init?.maxResponseBytes ?? 5_000_000));
|
|
const method = init?.method ?? "GET";
|
|
const body = init?.body === undefined ? "" : JSON.stringify(init.body);
|
|
const url = `http://127.0.0.1:8080${path}`;
|
|
const script = [
|
|
"set -euo pipefail",
|
|
"if command -v backend-core >/dev/null 2>&1; then",
|
|
` exec backend-core --fetch-json ${shellQuote(url)} --method ${shellQuote(method)} --max-response-bytes ${shellQuote(String(maxResponseBytes))}${body.length > 0 ? ` --body-json ${shellQuote(body)}` : ""}`,
|
|
"fi",
|
|
`url=${shellQuote(url)}`,
|
|
`method=${shellQuote(method)}`,
|
|
`max_bytes=${shellQuote(String(maxResponseBytes))}`,
|
|
`body=${shellQuote(body)}`,
|
|
"export url method body max_bytes",
|
|
"bun -e 'const url=process.env.url; const method=process.env.method; const body=process.env.body; const maxBytes=Number(process.env.max_bytes||\"5000000\"); fetch(url,{method,body:body?body:undefined,headers:body?{\"content-type\":\"application/json\"}:undefined}).then(async r=>{const text=await r.text(); const out={ok:r.ok,status:r.status,body:text?JSON.parse(text):null}; const json=JSON.stringify(out); if (Buffer.byteLength(json) > maxBytes) { console.error(\"response too large\"); process.exit(1); } console.log(json); process.exit(r.ok?0:1);}).catch(e=>{console.error(e); process.exit(1)})'",
|
|
].join("\n");
|
|
return ["docker", "exec", "unidesk-backend-core", "sh", "-lc", script];
|
|
}
|
|
|
|
interface RelatedContainer {
|
|
name: string;
|
|
image: string;
|
|
status: string;
|
|
}
|
|
|
|
function unquoteEnvValue(value: string): string {
|
|
return value.replace(/^['"]|['"]$/g, "");
|
|
}
|
|
|
|
function baiduSecretPresence(envPath: string): Record<string, unknown> {
|
|
const keys = [
|
|
"UNIDESK_BAIDU_NETDISK_CLIENT_ID",
|
|
"UNIDESK_BAIDU_NETDISK_CLIENT_SECRET",
|
|
"UNIDESK_BAIDU_NETDISK_TOKEN_KEY",
|
|
];
|
|
if (!existsSync(envPath)) {
|
|
return { envPath, exists: false, keys: Object.fromEntries(keys.map((key) => [key, { present: false, nonEmpty: false }])) };
|
|
}
|
|
const env = new Map<string, string>();
|
|
for (const line of readFileSync(envPath, "utf8").split(/\r?\n/u)) {
|
|
if (line.length === 0 || line.trimStart().startsWith("#")) continue;
|
|
const index = line.indexOf("=");
|
|
if (index === -1) continue;
|
|
env.set(line.slice(0, index), unquoteEnvValue(line.slice(index + 1)));
|
|
}
|
|
return {
|
|
envPath,
|
|
exists: true,
|
|
keys: Object.fromEntries(keys.map((key) => [key, { present: env.has(key), nonEmpty: Boolean((env.get(key) || "").trim()) }])),
|
|
};
|
|
}
|
|
|
|
function listRelatedStackContainers(): RelatedContainer[] {
|
|
const result = runCommand(["docker", "ps", "-a", "--format", "{{json .}}"], repoRoot);
|
|
if (result.exitCode !== 0 || result.stdout.trim().length === 0) return [];
|
|
const names = ["unidesk-backend-core", "unidesk-database", "baidu-netdisk-backend"];
|
|
return result.stdout
|
|
.split("\n")
|
|
.map((line) => line.trim())
|
|
.filter(Boolean)
|
|
.map((line) => {
|
|
try {
|
|
return JSON.parse(line) as Record<string, string>;
|
|
} catch {
|
|
return null;
|
|
}
|
|
})
|
|
.filter((row): row is Record<string, string> => row !== null)
|
|
.filter((row) => names.some((name) => row.Names === name || String(row.Names || "").startsWith(`${name}.`)))
|
|
.map((row) => ({ name: row.Names ?? "", image: row.Image ?? "", status: row.Status ?? "" }));
|
|
}
|
|
|
|
export function backendCoreUnavailableDiagnostic(detail: {
|
|
exitCode: number | null;
|
|
stdoutTail: string;
|
|
stderrTail: string;
|
|
relatedContainers: RelatedContainer[];
|
|
envPath: string;
|
|
baiduSecretPresence: Record<string, unknown>;
|
|
}): Record<string, unknown> {
|
|
const expectedContainers = ["unidesk-backend-core", "unidesk-database", "baidu-netdisk-backend"];
|
|
const presentExactNames = new Set(detail.relatedContainers.map((container) => container.name));
|
|
const missingContainers = expectedContainers.filter((name) => !presentExactNames.has(name));
|
|
const verifyOnlyObserved = detail.relatedContainers.some((container) => /\.verify-/u.test(container.name));
|
|
return {
|
|
ok: false,
|
|
failureKind: "target-stack-not-running",
|
|
degradedReason: "backend-core-container-missing",
|
|
runnerDisposition: "infra-blocked",
|
|
readOnlyReview: true,
|
|
message: verifyOnlyObserved
|
|
? "backend-core/database target containers are not running; only verify-only containers were observed."
|
|
: "backend-core target container is not running, so private backend-core API checks cannot observe schedules or microservices.",
|
|
targetStack: {
|
|
expectedContainers,
|
|
missingContainers,
|
|
relatedContainers: detail.relatedContainers,
|
|
verifyOnlyObserved,
|
|
},
|
|
observed: {
|
|
commandExitCode: detail.exitCode,
|
|
stdoutTail: detail.stdoutTail,
|
|
stderrTail: detail.stderrTail,
|
|
},
|
|
baiduNetdiskConfigPresence: detail.baiduSecretPresence,
|
|
readOnlyCommands: [
|
|
"bun scripts/cli.ts server status",
|
|
"bun scripts/cli.ts schedule list",
|
|
"bun scripts/cli.ts schedule runs --limit 20",
|
|
"bun scripts/cli.ts microservice health baidu-netdisk",
|
|
"bun scripts/cli.ts microservice proxy baidu-netdisk /api/auth/status --raw",
|
|
"bun scripts/cli.ts microservice proxy baidu-netdisk '/api/transfers?limit=20' --raw",
|
|
],
|
|
authorizationRequiredForRecovery: [
|
|
"restore non-empty Baidu Netdisk runtime secrets in the controlled Compose env source without printing secret values",
|
|
"start or deploy the production target stack containing backend-core, database, and baidu-netdisk",
|
|
"after health is green, explicitly authorize schedule retry-run or schedule run before triggering a real PGDATA backup",
|
|
],
|
|
blockedWithoutAuthorization: [
|
|
"server start",
|
|
"server rebuild backend-core",
|
|
"server rebuild baidu-netdisk",
|
|
"deploy apply --env prod --service baidu-netdisk",
|
|
"schedule run <id>",
|
|
"schedule retry-run <failedRunId>",
|
|
],
|
|
};
|
|
}
|
|
|
|
function backendCoreContainerMissing(stderr: string): boolean {
|
|
return stderr.includes("No such container: unidesk-backend-core");
|
|
}
|
|
|
|
export function coreInternalFetch(path: string, init?: { method?: string; body?: unknown; maxResponseBytes?: number; timeoutMs?: number }): unknown {
|
|
if (!path.startsWith("/")) throw new Error("core internal path must start with /");
|
|
const command = dockerCoreFetchCommand(path, init);
|
|
const result = runCommand(command, repoRoot, { timeoutMs: init?.timeoutMs });
|
|
if (result.exitCode !== 0) {
|
|
if (backendCoreContainerMissing(result.stderr)) {
|
|
const envPath = join(repoRoot, ".state", "docker-compose.env");
|
|
return backendCoreUnavailableDiagnostic({
|
|
exitCode: result.exitCode,
|
|
stdoutTail: result.stdout.slice(-1200),
|
|
stderrTail: result.stderr.slice(-1200),
|
|
relatedContainers: listRelatedStackContainers(),
|
|
envPath,
|
|
baiduSecretPresence: baiduSecretPresence(envPath),
|
|
});
|
|
}
|
|
const parsedStdout = parseJsonRecord(result.stdout.trim());
|
|
if (parsedStdout !== null) {
|
|
return {
|
|
...parsedStdout,
|
|
commandExitCode: result.exitCode,
|
|
commandStderrTail: result.stderr.slice(-1200),
|
|
};
|
|
}
|
|
const codeQueueStableProxy = path.startsWith("/api/microservices/code-queue/");
|
|
return {
|
|
ok: false,
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
stdoutTail: result.stdout.slice(-1200),
|
|
stderrTail: result.stderr.slice(-1200),
|
|
...(codeQueueStableProxy ? {
|
|
codeQueueObservationNote: "The stable /api/microservices/code-queue path could not be reached from this CLI container. This does not by itself mean Code Queue tasks are unevaluable or stopped. From the supervisor or main-server CLI environment, use the existing codex queues/tasks/task observation commands instead of treating this container-local proxy failure as liveness evidence.",
|
|
recommendedCommands: [
|
|
"bun scripts/cli.ts codex queues",
|
|
"bun scripts/cli.ts codex tasks --limit 20",
|
|
"bun scripts/cli.ts codex task <taskId>",
|
|
],
|
|
} : {}),
|
|
};
|
|
}
|
|
try {
|
|
return JSON.parse(result.stdout.trim()) as unknown;
|
|
} catch {
|
|
return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) };
|
|
}
|
|
}
|
|
|
|
function parseJsonRecord(text: string): Record<string, unknown> | null {
|
|
if (text.length === 0) return null;
|
|
try {
|
|
const parsed = JSON.parse(text) as unknown;
|
|
return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record<string, unknown> : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function asRecord(value: unknown): Record<string, unknown> | null {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : null;
|
|
}
|
|
|
|
function asNumber(value: unknown, fallback = 0): number {
|
|
const number = Number(value);
|
|
return Number.isFinite(number) ? number : fallback;
|
|
}
|
|
|
|
function stringList(value: unknown): string[] {
|
|
if (!Array.isArray(value)) return [];
|
|
return value.map((item) => String(item)).filter((item) => item.length > 0);
|
|
}
|
|
|
|
function boundedUniqueStringList(value: unknown, limit = 5): Record<string, unknown> {
|
|
const all = Array.from(new Set(stringList(value))).sort();
|
|
const items = all.slice(0, limit);
|
|
return {
|
|
items,
|
|
count: all.length,
|
|
omitted: Math.max(0, all.length - items.length),
|
|
truncated: all.length > items.length,
|
|
};
|
|
}
|
|
|
|
function limitedStrings(value: unknown, limit: number, maxLength: number): string[] {
|
|
return stringList(value).slice(0, limit).map((item) => item.length > maxLength ? item.slice(0, maxLength) : item);
|
|
}
|
|
|
|
function recordValue(record: Record<string, unknown> | null, key: string): Record<string, unknown> | null {
|
|
return record === null ? null : asRecord(record[key]);
|
|
}
|
|
|
|
function isCodeQueueHealthBody(record: Record<string, unknown> | null): boolean {
|
|
if (record === null) return false;
|
|
return record.service === "code-queue"
|
|
|| record.serviceId === "code-queue"
|
|
|| asRecord(record.queue) !== null
|
|
|| asRecord(record.executionDiagnostics) !== null;
|
|
}
|
|
|
|
function hasCodeQueueHealthDetails(record: Record<string, unknown> | null): boolean {
|
|
if (record === null) return false;
|
|
return record.service === "code-queue"
|
|
|| asRecord(record.queue) !== null
|
|
|| asRecord(record.executionDiagnostics) !== null;
|
|
}
|
|
|
|
function splitBrainLiveFromDiagnostics(record: Record<string, unknown>): boolean {
|
|
if (typeof record.splitBrainLive === "boolean") return record.splitBrainLive;
|
|
const state = String(record.state ?? record.health ?? "").toLowerCase();
|
|
const riskTaskIds = Array.from(new Set([
|
|
...stringList(record.heartbeatRiskTaskIds),
|
|
...stringList(record.heartbeatExpiredTaskIds),
|
|
...stringList(record.heartbeatMissingTaskIds),
|
|
...stringList(record.staleRecoveryCandidateTaskIds),
|
|
]));
|
|
return state === "split-brain" && stringList(record.heartbeatFreshTaskIds).length > 0 && riskTaskIds.length === 0;
|
|
}
|
|
|
|
function effectiveLivenessFromDiagnostics(record: Record<string, unknown>): string {
|
|
if (typeof record.effectiveLiveness === "string" && record.effectiveLiveness.length > 0) return record.effectiveLiveness;
|
|
if (stringList(record.heartbeatRiskTaskIds).length > 0
|
|
|| stringList(record.heartbeatExpiredTaskIds).length > 0
|
|
|| stringList(record.heartbeatMissingTaskIds).length > 0
|
|
|| stringList(record.staleRecoveryCandidateTaskIds).length > 0) {
|
|
return "at-risk";
|
|
}
|
|
if (splitBrainLiveFromDiagnostics(record)) return "live";
|
|
return String(record.state ?? record.health ?? "unknown") === "healthy" ? "healthy" : "degraded";
|
|
}
|
|
|
|
function recommendedActionFromDiagnostics(record: Record<string, unknown>, effectiveLiveness: string): string {
|
|
if (typeof record.recommendedAction === "string" && record.recommendedAction.length > 0) return record.recommendedAction;
|
|
if (effectiveLiveness === "at-risk") return "investigate-heartbeat-risk";
|
|
if (effectiveLiveness === "live") return "continue-supervision";
|
|
if (effectiveLiveness === "degraded") return "observe-degraded";
|
|
return "none";
|
|
}
|
|
|
|
function livenessInterpretation(effectiveLiveness: string, splitBrainLive: boolean): string {
|
|
if (splitBrainLive) return "scheduler heartbeat is fresh; treat active task count from heartbeat as live and continue supervision";
|
|
if (effectiveLiveness === "at-risk") return "heartbeat risk is present; investigate heartbeat freshness before recovery";
|
|
if (effectiveLiveness === "degraded") return "diagnostics are degraded; cross-check heartbeat, trace and control-plane sources";
|
|
return "diagnostics indicate healthy liveness";
|
|
}
|
|
|
|
function compactCodeQueueDiagnostics(value: unknown): Record<string, unknown> | null {
|
|
const diagnostics = asRecord(value);
|
|
if (diagnostics === null) return null;
|
|
const heartbeatRiskIds = Array.from(new Set([
|
|
...stringList(diagnostics.heartbeatRiskTaskIds),
|
|
...stringList(diagnostics.heartbeatExpiredTaskIds),
|
|
...stringList(diagnostics.heartbeatMissingTaskIds),
|
|
...stringList(diagnostics.staleRecoveryCandidateTaskIds),
|
|
])).sort();
|
|
const splitBrainLive = splitBrainLiveFromDiagnostics(diagnostics);
|
|
const effectiveLiveness = effectiveLivenessFromDiagnostics(diagnostics);
|
|
const recommendedAction = recommendedActionFromDiagnostics(diagnostics, effectiveLiveness);
|
|
const heartbeatFreshTaskIds = boundedUniqueStringList(diagnostics.heartbeatFreshTaskIds);
|
|
const activeHeartbeatTaskIds = boundedUniqueStringList(diagnostics.activeHeartbeatTaskIds);
|
|
return {
|
|
state: diagnostics.state ?? diagnostics.health ?? null,
|
|
degraded: diagnostics.degraded ?? null,
|
|
splitBrain: diagnostics.splitBrain ?? null,
|
|
splitBrainLive,
|
|
effectiveLiveness,
|
|
recommendedAction,
|
|
interpretation: livenessInterpretation(effectiveLiveness, splitBrainLive),
|
|
executionStateSource: diagnostics.executionStateSource ?? null,
|
|
controlPlane: diagnostics.controlPlane ?? null,
|
|
databaseActiveTaskCount: diagnostics.databaseActiveTaskCount ?? stringList(diagnostics.databaseActiveTaskIds).length,
|
|
schedulerActiveRunSlotCount: diagnostics.schedulerActiveRunSlotCount ?? null,
|
|
activeHeartbeatCount: diagnostics.activeHeartbeatCount ?? Math.max(asNumber(activeHeartbeatTaskIds.count), asNumber(heartbeatFreshTaskIds.count)),
|
|
activeHeartbeatTaskIds,
|
|
heartbeatFreshTaskCount: heartbeatFreshTaskIds.count,
|
|
heartbeatFreshTaskIds,
|
|
heartbeatRiskTaskCount: heartbeatRiskIds.length,
|
|
heartbeatRiskTaskIds: boundedUniqueStringList(heartbeatRiskIds),
|
|
heartbeatExpiredTaskCount: stringList(diagnostics.heartbeatExpiredTaskIds).length,
|
|
heartbeatMissingTaskCount: stringList(diagnostics.heartbeatMissingTaskIds).length,
|
|
staleRecoveryCandidateTaskCount: stringList(diagnostics.staleRecoveryCandidateTaskIds).length,
|
|
traceGapTaskCount: stringList(diagnostics.traceGapTaskIds).length,
|
|
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt ?? null,
|
|
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt ?? null,
|
|
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt ?? null,
|
|
reasons: limitedStrings(diagnostics.reasons, 3, 240),
|
|
};
|
|
}
|
|
|
|
function compactCodeQueueQueueSummary(value: unknown, diagnostics: Record<string, unknown> | null): Record<string, unknown> {
|
|
const queue = asRecord(value);
|
|
const counts = asRecord(queue?.counts) ?? {};
|
|
const running = asNumber(counts.running);
|
|
const judging = asNumber(counts.judging);
|
|
const activeTaskIds = boundedUniqueStringList(queue?.activeTaskIds);
|
|
const activeQueueIds = boundedUniqueStringList(queue?.activeQueueIds);
|
|
const processingQueueIds = boundedUniqueStringList(queue?.processingQueueIds);
|
|
return {
|
|
total: queue?.total ?? null,
|
|
counts,
|
|
runningCount: running + judging,
|
|
running,
|
|
judging,
|
|
queued: asNumber(counts.queued),
|
|
retryWait: asNumber(counts.retry_wait),
|
|
queueCount: queue?.queueCount ?? null,
|
|
unreadTerminal: queue?.unreadTerminal ?? null,
|
|
activeRunSlotCount: queue?.activeRunSlotCount ?? diagnostics?.schedulerActiveRunSlotCount ?? null,
|
|
activeTaskCount: activeTaskIds.count,
|
|
activeTaskIds,
|
|
activeTaskId: queue?.activeTaskId ?? null,
|
|
activeQueueIds,
|
|
processingQueueIds,
|
|
processing: queue?.processing ?? null,
|
|
orphanedActiveTaskCount: queue?.orphanedActiveTaskCount ?? null,
|
|
defaultQueueId: queue?.defaultQueueId ?? null,
|
|
defaultModel: queue?.defaultModel ?? null,
|
|
defaultReasoningEffort: queue?.defaultReasoningEffort ?? null,
|
|
maxActiveQueues: queue?.maxActiveQueues ?? null,
|
|
};
|
|
}
|
|
|
|
function compactCodeQueueComponents(body: Record<string, unknown>, queue: Record<string, unknown> | null): Record<string, unknown> {
|
|
const storage = asRecord(queue?.storage);
|
|
const egressProxy = asRecord(body.egressProxy);
|
|
const oaPublisher = asRecord(body.oaEventPublisher);
|
|
return {
|
|
databaseReady: body.databaseReady ?? storage?.postgresReady ?? null,
|
|
serviceReady: body.serviceReady ?? null,
|
|
schedulerEnabled: body.schedulerEnabled ?? null,
|
|
egressProxyConnected: egressProxy?.connected ?? null,
|
|
oaEventPublisher: oaPublisher === null ? null : {
|
|
enabled: oaPublisher.enabled ?? null,
|
|
pendingCount: oaPublisher.pendingCount ?? null,
|
|
lastError: oaPublisher.lastError ?? null,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function summarizeMicroserviceHealthResponse(response: unknown, args: string[], serviceId: string): unknown {
|
|
if (serviceId !== "code-queue") return summarizeMicroserviceObservation("health", serviceId, response, args);
|
|
if (hasFlag(args, "--raw") || hasFlag(args, "--full")) return response;
|
|
|
|
const responseRecord = asRecord(response);
|
|
if (responseRecord === null) return response;
|
|
const responseBodySource = "body" in responseRecord ? responseRecord.body : responseRecord;
|
|
const responseBody = asRecord(responseBodySource);
|
|
if (responseBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
|
|
const upstream = recordValue(responseBody, "upstream");
|
|
const upstreamBody = recordValue(upstream, "body");
|
|
const healthBody = hasCodeQueueHealthDetails(responseBody)
|
|
? responseBody
|
|
: isCodeQueueHealthBody(upstreamBody)
|
|
? upstreamBody
|
|
: isCodeQueueHealthBody(responseBody)
|
|
? responseBody
|
|
: null;
|
|
if (healthBody === null) return summarizeMicroserviceObservation("health", serviceId, response, args);
|
|
|
|
const queue = asRecord(healthBody.queue);
|
|
const diagnostics = compactCodeQueueDiagnostics(healthBody.executionDiagnostics ?? queue?.executionDiagnostics);
|
|
const bodyBytes = jsonByteLength(responseBodySource);
|
|
return {
|
|
ok: responseRecord.ok ?? responseBody.ok ?? healthBody.ok ?? null,
|
|
status: responseRecord.status ?? responseBody.status ?? healthBody.status ?? null,
|
|
serviceId: "code-queue",
|
|
service: healthBody.service ?? "code-queue",
|
|
name: responseBody.name ?? null,
|
|
providerId: responseBody.providerId ?? null,
|
|
healthPath: responseBody.healthPath ?? "/health",
|
|
reason: responseBody.reason ?? null,
|
|
checkedAt: responseBody.checkedAt ?? null,
|
|
instanceId: healthBody.instanceId ?? null,
|
|
role: healthBody.role ?? null,
|
|
deploy: previewJson(healthBody.deploy ?? null, { maxDepth: 2, maxArrayItems: 2, maxObjectKeys: 8, maxStringLength: 160 }),
|
|
components: compactCodeQueueComponents(healthBody, queue),
|
|
queue: compactCodeQueueQueueSummary(queue, diagnostics),
|
|
heartbeat: diagnostics === null ? null : {
|
|
lastSchedulerHeartbeatAt: diagnostics.lastSchedulerHeartbeatAt,
|
|
lastObservedAgentEventAt: diagnostics.lastObservedAgentEventAt,
|
|
lastPersistedTraceAt: diagnostics.lastPersistedTraceAt,
|
|
activeHeartbeatCount: diagnostics.activeHeartbeatCount,
|
|
activeHeartbeatTaskIds: diagnostics.activeHeartbeatTaskIds,
|
|
heartbeatFreshTaskCount: diagnostics.heartbeatFreshTaskCount,
|
|
heartbeatFreshTaskIds: diagnostics.heartbeatFreshTaskIds,
|
|
heartbeatRiskTaskCount: diagnostics.heartbeatRiskTaskCount,
|
|
heartbeatRiskTaskIds: diagnostics.heartbeatRiskTaskIds,
|
|
heartbeatExpiredTaskCount: diagnostics.heartbeatExpiredTaskCount,
|
|
heartbeatMissingTaskCount: diagnostics.heartbeatMissingTaskCount,
|
|
staleRecoveryCandidateTaskCount: diagnostics.staleRecoveryCandidateTaskCount,
|
|
traceGapTaskCount: diagnostics.traceGapTaskCount,
|
|
schedulerHeartbeatStaleMs: queue?.schedulerHeartbeatStaleMs ?? null,
|
|
},
|
|
liveness: diagnostics === null ? null : {
|
|
state: diagnostics.state,
|
|
degraded: diagnostics.degraded,
|
|
splitBrain: diagnostics.splitBrain,
|
|
splitBrainLive: diagnostics.splitBrainLive,
|
|
effectiveLiveness: diagnostics.effectiveLiveness,
|
|
recommendedAction: diagnostics.recommendedAction,
|
|
interpretation: diagnostics.interpretation,
|
|
reasons: diagnostics.reasons,
|
|
},
|
|
outputPolicy: {
|
|
mode: "compact",
|
|
defaultCompact: true,
|
|
omittedFullHealthBody: true,
|
|
originalBodyBytes: bodyBytes,
|
|
idPreviewLimit: 5,
|
|
rawCommand: "bun scripts/cli.ts microservice health code-queue --raw",
|
|
fullCommand: "bun scripts/cli.ts microservice health code-queue --raw --full",
|
|
proxyFullCommand: "bun scripts/cli.ts microservice proxy code-queue /health --raw --full",
|
|
},
|
|
};
|
|
}
|
|
|
|
function requireId(value: string | undefined, command: string): string {
|
|
if (value === undefined || value.length === 0) throw new Error(`${command} requires microservice id`);
|
|
return value;
|
|
}
|
|
|
|
function requireProxyPath(value: string | undefined): string {
|
|
if (value === undefined || value.length === 0) throw new Error("microservice proxy requires upstream path, for example /api/summary");
|
|
if (!value.startsWith("/")) throw new Error("microservice proxy upstream path must start with /");
|
|
return value;
|
|
}
|
|
|
|
function encodeId(value: string): string {
|
|
return encodeURIComponent(value);
|
|
}
|
|
|
|
function numberOption(args: string[], name: string, defaultValue: number): number {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) return defaultValue;
|
|
const raw = args[index + 1];
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`);
|
|
return value;
|
|
}
|
|
|
|
function cappedNumberOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
|
const value = numberOption(args, name, defaultValue);
|
|
return Math.min(value, maxValue);
|
|
}
|
|
|
|
function stringOption(args: string[], name: string): string | undefined {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) return undefined;
|
|
const raw = args[index + 1];
|
|
if (raw === undefined || raw.length === 0) throw new Error(`${name} requires a non-empty value`);
|
|
return raw;
|
|
}
|
|
|
|
function hasFlag(args: string[], name: string): boolean {
|
|
return args.includes(name);
|
|
}
|
|
|
|
function asArray(value: unknown): unknown[] {
|
|
return Array.isArray(value) ? value : [];
|
|
}
|
|
|
|
function parseJsonOption(raw: string, name: string): unknown {
|
|
try {
|
|
return JSON.parse(raw) as unknown;
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
throw new Error(`${name} must be valid JSON: ${message}`);
|
|
}
|
|
}
|
|
|
|
function requestBodyOption(args: string[]): unknown | undefined {
|
|
const bodyJson = stringOption(args, "--body-json");
|
|
const bodyFile = stringOption(args, "--body-file");
|
|
const bodyStdin = hasFlag(args, "--body-stdin");
|
|
const sources = [bodyJson !== undefined, bodyFile !== undefined, bodyStdin].filter(Boolean).length;
|
|
if (sources > 1) throw new Error("microservice proxy accepts only one request body source: --body-json, --body-file, or --body-stdin");
|
|
if (bodyJson !== undefined) return parseJsonOption(bodyJson, "--body-json");
|
|
if (bodyFile !== undefined) {
|
|
const text = bodyFile === "-" ? readFileSync(0, "utf8") : readFileSync(bodyFile, "utf8");
|
|
return parseJsonOption(text, "--body-file");
|
|
}
|
|
if (bodyStdin) return parseJsonOption(readFileSync(0, "utf8"), "--body-stdin");
|
|
return undefined;
|
|
}
|
|
|
|
function assertKnownObservationOptions(args: string[], command: string, extraFlags: string[] = []): void {
|
|
const flags = new Set(["--full", "--raw", ...extraFlags]);
|
|
for (const arg of args) {
|
|
if (!arg.startsWith("--")) continue;
|
|
if (!flags.has(arg)) throw new Error(`unsupported ${command} option: ${arg}`);
|
|
}
|
|
}
|
|
|
|
function methodOption(args: string[], hasBody = false): string {
|
|
const method = (stringOption(args, "--method") ?? (hasBody ? "POST" : "GET")).toUpperCase();
|
|
if (!["GET", "HEAD", "POST", "DELETE", "PUT", "PATCH"].includes(method)) throw new Error(`unsupported --method ${method}`);
|
|
if (hasBody && (method === "GET" || method === "HEAD")) throw new Error(`microservice proxy cannot send a request body with ${method}`);
|
|
return method;
|
|
}
|
|
|
|
export function summarizeMicroserviceProxyResponse(response: unknown, args: string[]): unknown {
|
|
const full = args.includes("--full");
|
|
const raw = args.includes("--raw");
|
|
const maxBodyBytes = full ? numberOption(args, "--max-body-bytes", 5_000_000) : cappedNumberOption(args, "--max-body-bytes", raw ? 120_000 : 60_000, 500_000);
|
|
if (typeof response !== "object" || response === null || Array.isArray(response)) return response;
|
|
const record = response as Record<string, unknown>;
|
|
if (!("body" in record)) return response;
|
|
if (record.responseTruncated === true) {
|
|
return {
|
|
...record,
|
|
bodyOmitted: true,
|
|
bodyMaxBytes: maxBodyBytes,
|
|
rawHint: "The upstream response exceeded the CLI collection cap before JSON parsing; re-run with --raw --full and a specific --max-body-bytes only when the full body is required.",
|
|
};
|
|
}
|
|
const bodyBytes = jsonByteLength(record.body);
|
|
if (bodyBytes <= maxBodyBytes) {
|
|
if (!raw || full) return response;
|
|
return {
|
|
...record,
|
|
outputPolicy: {
|
|
rawRequested: true,
|
|
bounded: true,
|
|
maxBodyBytes,
|
|
bodyBytes,
|
|
fullCommand: "Re-run with --raw --full to allow the complete body.",
|
|
},
|
|
};
|
|
}
|
|
const rest = { ...record };
|
|
delete rest.body;
|
|
return {
|
|
...rest,
|
|
bodyOmitted: true,
|
|
bodyBytes,
|
|
bodyMaxBytes: maxBodyBytes,
|
|
bodyPreview: previewJson(record.body, { maxDepth: 3, maxArrayItems: 3, maxObjectKeys: 16, maxStringLength: 320 }),
|
|
rawHint: raw && !full
|
|
? "The --raw response exceeded the default hard limit; re-run with --raw --full for the complete body, or add/tighten __unideskArrayLimit=<path>:<limit> in the proxied path."
|
|
: "Re-run with --raw --full for the complete body, or add/tighten __unideskArrayLimit=<path>:<limit> in the proxied path.",
|
|
};
|
|
}
|
|
|
|
function compactStringList(value: unknown, limit = 8): Record<string, unknown> {
|
|
const all = Array.from(new Set(asArray(value).map((item) => String(item ?? "")).filter(Boolean)));
|
|
return {
|
|
items: all.slice(0, limit),
|
|
count: all.length,
|
|
truncated: all.length > limit,
|
|
omitted: Math.max(0, all.length - limit),
|
|
};
|
|
}
|
|
|
|
function compactRecordFields(record: Record<string, unknown> | null, keys: string[]): Record<string, unknown> | null {
|
|
if (record === null) return null;
|
|
const selected: Record<string, unknown> = {};
|
|
for (const key of keys) {
|
|
if (record[key] !== undefined) selected[key] = record[key];
|
|
}
|
|
return Object.keys(selected).length > 0 ? selected : null;
|
|
}
|
|
|
|
function compactQueueHealth(value: unknown): Record<string, unknown> | null {
|
|
const queue = asRecord(value);
|
|
if (queue === null) return null;
|
|
return {
|
|
controlPlane: queue.controlPlane ?? null,
|
|
defaultProviderId: queue.defaultProviderId ?? null,
|
|
defaultModel: queue.defaultModel ?? null,
|
|
counts: queue.counts ?? {},
|
|
total: queue.total ?? null,
|
|
queueCount: queue.queueCount ?? null,
|
|
activeQueueIds: compactStringList(queue.activeQueueIds, 8),
|
|
activeTaskIds: compactStringList(queue.activeTaskIds ?? queue.databaseActiveTaskIds, 8),
|
|
queuedTaskIds: compactStringList(queue.queuedTaskIds, 8),
|
|
databaseActiveTaskCount: queue.databaseActiveTaskCount ?? null,
|
|
schedulerActiveRunSlotCount: queue.schedulerActiveRunSlotCount ?? queue.activeRunSlotCount ?? null,
|
|
};
|
|
}
|
|
|
|
function compactSkillAvailability(value: unknown): Record<string, unknown> | null {
|
|
const skills = asRecord(value);
|
|
if (skills === null) return null;
|
|
return {
|
|
ok: skills.ok ?? false,
|
|
runnerUsable: skills.runnerUsable ?? skills.ok ?? false,
|
|
contractOk: skills.contractOk ?? skills.ok ?? false,
|
|
path: skills.path ?? null,
|
|
resolvedPath: skills.resolvedPath ?? skills.path ?? null,
|
|
resolvedPathSource: skills.resolvedPathSource ?? null,
|
|
resolution: skills.resolution ?? null,
|
|
source: skills.source ?? null,
|
|
target: skills.target ?? null,
|
|
mountPoint: skills.mountPoint ?? null,
|
|
exists: skills.exists ?? false,
|
|
available: skills.available ?? false,
|
|
degraded: skills.degraded ?? true,
|
|
blocker: skills.blocker ?? null,
|
|
degradedReason: skills.degradedReason ?? skills.blocker ?? null,
|
|
readonly: skills.readonly ?? false,
|
|
skillCount: skills.skillCount ?? 0,
|
|
version: skills.version ?? null,
|
|
sourceSkillCount: skills.sourceSkillCount ?? null,
|
|
targetSkillCount: skills.targetSkillCount ?? null,
|
|
sourceMissingSkills: Array.isArray(skills.sourceMissingSkills) ? skills.sourceMissingSkills.map(String) : [],
|
|
targetMissingSkills: Array.isArray(skills.targetMissingSkills) ? skills.targetMissingSkills.map(String) : [],
|
|
requiredSkills: Array.isArray(skills.requiredSkills) ? skills.requiredSkills.map(String) : [],
|
|
missingSkills: Array.isArray(skills.missingSkills) ? skills.missingSkills.map(String) : [],
|
|
valuesPrinted: skills.valuesPrinted ?? false,
|
|
pathSpelling: compactRecordFields(asRecord(skills.pathSpelling), [
|
|
"expectedTarget",
|
|
"forbiddenPathChecked",
|
|
"forbiddenPathExists",
|
|
"forbiddenPathConfigured",
|
|
"forbiddenPathRoles",
|
|
"forbiddenPathMustNotBeUsed",
|
|
]),
|
|
repairHint: skills.repairHint ?? null,
|
|
};
|
|
}
|
|
|
|
function compactSkillSync(value: unknown): Record<string, unknown> | null {
|
|
const sync = asRecord(value);
|
|
if (sync === null) return null;
|
|
const source = compactRecordFields(asRecord(sync.source), [
|
|
"path",
|
|
"approved",
|
|
"exists",
|
|
"directory",
|
|
"readable",
|
|
"writable",
|
|
"readonly",
|
|
"mountPoint",
|
|
"symlink",
|
|
"realPath",
|
|
"skillCount",
|
|
"version",
|
|
"requiredSkills",
|
|
"missingSkills",
|
|
"error",
|
|
]);
|
|
const target = compactRecordFields(asRecord(sync.target), [
|
|
"path",
|
|
"approved",
|
|
"exists",
|
|
"directory",
|
|
"readable",
|
|
"writable",
|
|
"readonly",
|
|
"mountPoint",
|
|
"symlink",
|
|
"realPath",
|
|
"skillCount",
|
|
"version",
|
|
"requiredSkills",
|
|
"missingSkills",
|
|
"error",
|
|
]);
|
|
const permissionFailures = Array.isArray(sync.permissionFailures) ? sync.permissionFailures : [];
|
|
return {
|
|
ok: sync.ok ?? false,
|
|
degraded: sync.degraded ?? true,
|
|
blocker: sync.blocker ?? null,
|
|
mode: sync.mode ?? "dry-run",
|
|
dryRun: sync.dryRun ?? true,
|
|
mutation: sync.mutation ?? false,
|
|
syncMode: sync.syncMode ?? null,
|
|
source,
|
|
target,
|
|
counts: sync.counts ?? null,
|
|
version: sync.version ?? null,
|
|
missing: sync.missing ?? null,
|
|
permissionFailureCount: permissionFailures.length,
|
|
permissionFailures: permissionFailures.slice(0, 4),
|
|
plannedActions: sync.plannedActions ?? null,
|
|
commands: sync.commands ?? null,
|
|
valuesPrinted: sync.valuesPrinted ?? false,
|
|
};
|
|
}
|
|
|
|
function compactMicroserviceBody(body: unknown, serviceId: string): Record<string, unknown> | null {
|
|
const record = asRecord(body);
|
|
if (record === null) return null;
|
|
const diagnostics = asRecord(record.executionDiagnostics) ?? asRecord(record.diagnostics);
|
|
const devReady = asRecord(record.devReady);
|
|
return {
|
|
ok: record.ok ?? null,
|
|
serviceId: record.serviceId ?? record.service ?? serviceId,
|
|
service: record.service ?? null,
|
|
role: record.role ?? null,
|
|
status: record.status ?? null,
|
|
healthy: record.healthy ?? null,
|
|
mode: record.mode ?? null,
|
|
environment: record.environment ?? null,
|
|
version: record.version ?? record.gatewayVersion ?? null,
|
|
target: record.target ?? record.k3sServiceId ?? null,
|
|
updatedAt: record.updatedAt ?? record.observedAt ?? record.generatedAt ?? null,
|
|
startedAt: record.startedAt ?? null,
|
|
taskCount: record.taskCount ?? null,
|
|
schemaReady: record.schemaReady ?? null,
|
|
queue: compactQueueHealth(record.queue),
|
|
skills: compactSkillAvailability(record.skills),
|
|
skillsSync: compactSkillSync(record.skillsSync),
|
|
resourceBudget: compactRecordFields(asRecord(record.resourceBudget), [
|
|
"targetMemoryMb",
|
|
"mgrPoolMax",
|
|
"tracePoolMax",
|
|
"noRunnerDependencies",
|
|
"noDockerSocket",
|
|
"noPlaywright",
|
|
"rustRewrite",
|
|
]),
|
|
topLevelKeys: compactStringList(Object.keys(record), 16),
|
|
devReady: devReady === null ? null : {
|
|
ok: devReady.ok ?? null,
|
|
missingTools: compactStringList(devReady.missingTools, 8),
|
|
skills: compactSkillAvailability(devReady.skills),
|
|
skillsSync: compactSkillSync(devReady.skillsSync),
|
|
},
|
|
executionDiagnostics: diagnostics === null ? null : {
|
|
state: diagnostics.state ?? null,
|
|
effectiveLiveness: diagnostics.effectiveLiveness ?? null,
|
|
recommendedAction: diagnostics.recommendedAction ?? null,
|
|
splitBrainLive: diagnostics.splitBrainLive ?? null,
|
|
schedulerActiveRunSlotCount: diagnostics.schedulerActiveRunSlotCount ?? null,
|
|
databaseActiveTaskCount: diagnostics.databaseActiveTaskCount ?? null,
|
|
heartbeatFreshTaskIds: compactStringList(diagnostics.heartbeatFreshTaskIds, 8),
|
|
heartbeatRiskTaskIds: compactStringList(diagnostics.heartbeatRiskTaskIds, 8),
|
|
},
|
|
};
|
|
}
|
|
|
|
export function summarizeMicroserviceObservation(action: string, serviceId: string, response: unknown, args: string[]): unknown {
|
|
if (hasFlag(args, "--full") || hasFlag(args, "--raw")) return response;
|
|
const record = asRecord(response);
|
|
if (record === null) return response;
|
|
const body = "body" in record ? record.body : response;
|
|
const bodyBytes = jsonByteLength(body);
|
|
const summary = compactMicroserviceBody(body, serviceId);
|
|
const includePreview = summary === null || bodyBytes <= 20_000;
|
|
return {
|
|
upstream: {
|
|
ok: record.ok ?? null,
|
|
status: record.status ?? null,
|
|
exitCode: record.exitCode ?? null,
|
|
},
|
|
microservice: {
|
|
action,
|
|
id: serviceId,
|
|
summary,
|
|
bodyBytes,
|
|
bodyOmitted: true,
|
|
outputPolicy: {
|
|
default: "compact-health-summary",
|
|
full: `bun scripts/cli.ts microservice ${action} ${serviceId} --full`,
|
|
raw: `bun scripts/cli.ts microservice ${action} ${serviceId} --raw`,
|
|
proxyRaw: serviceId === "code-queue" && action === "health" ? "bun scripts/cli.ts microservice proxy code-queue /health --raw --full" : null,
|
|
},
|
|
...(includePreview
|
|
? { bodyPreview: previewJson(body, { maxDepth: 2, maxArrayItems: 2, maxObjectKeys: 8, maxStringLength: 160 }) }
|
|
: {
|
|
bodyPreviewOmitted: true,
|
|
bodyPreviewHint: "Large body preview omitted because compact summary is available; use --full or --raw for complete evidence.",
|
|
}),
|
|
},
|
|
};
|
|
}
|
|
|
|
export async function runMicroserviceCommand(_config: UniDeskConfig, args: string[]): Promise<unknown> {
|
|
const [action = "list", idArg, pathArg] = args;
|
|
if (action === "list") return coreInternalFetch("/api/microservices");
|
|
if (action === "status") {
|
|
const id = requireId(idArg, "microservice status");
|
|
const optionArgs = args.slice(2);
|
|
assertKnownObservationOptions(optionArgs, "microservice status");
|
|
return summarizeMicroserviceObservation(action, id, coreInternalFetch(`/api/microservices/${encodeId(id)}/status`), optionArgs);
|
|
}
|
|
if (action === "health") {
|
|
const id = requireId(idArg, "microservice health");
|
|
const optionArgs = args.slice(2);
|
|
assertKnownObservationOptions(optionArgs, "microservice health", ["--compact"]);
|
|
return summarizeMicroserviceHealthResponse(coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs, id);
|
|
}
|
|
if (action === "diagnostics") {
|
|
const id = requireId(idArg, "microservice diagnostics");
|
|
const optionArgs = args.slice(2);
|
|
assertKnownObservationOptions(optionArgs, "microservice diagnostics");
|
|
return summarizeMicroserviceObservation(action, id, coreInternalFetch(`/api/microservices/${encodeId(id)}/diagnostics`), optionArgs);
|
|
}
|
|
if (action === "tunnel-self-test") {
|
|
const id = requireId(idArg, "microservice tunnel-self-test");
|
|
return coreInternalFetch(`/api/microservices/${encodeId(id)}/tunnel-self-test`);
|
|
}
|
|
if (action === "proxy") {
|
|
const id = requireId(idArg, "microservice proxy");
|
|
const path = requireProxyPath(pathArg);
|
|
const body = requestBodyOption(args);
|
|
const full = hasFlag(args, "--full");
|
|
const raw = hasFlag(args, "--raw");
|
|
const maxBodyBytes = full ? numberOption(args, "--max-body-bytes", 5_000_000) : cappedNumberOption(args, "--max-body-bytes", raw ? 120_000 : 60_000, 500_000);
|
|
const maxResponseBytes = full ? Math.min(Math.max(maxBodyBytes, 120_000), 5_000_000) : Math.min(Math.max(maxBodyBytes * 3, 240_000), 1_500_000);
|
|
return summarizeMicroserviceProxyResponse(coreInternalFetch(`/api/microservices/${encodeId(id)}/proxy${path}`, { method: methodOption(args, body !== undefined), body, maxResponseBytes }), args);
|
|
}
|
|
throw new Error("microservice command must be one of: list, status, health, diagnostics, tunnel-self-test, proxy");
|
|
}
|