import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { runCommand } from "./command"; import { type UniDeskConfig, repoRoot } from "./config"; import { jsonByteLength, previewJson } from "./preview"; function shellQuote(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'`; } function dockerCoreFetchCommand(path: string, init?: { method?: string; body?: unknown; maxResponseBytes?: number }): string[] { const maxResponseBytes = Math.max(1024, Math.floor(init?.maxResponseBytes ?? 5_000_000)); const method = init?.method ?? "GET"; const body = init?.body === undefined ? "" : JSON.stringify(init.body); const url = `http://127.0.0.1:8080${path}`; const script = [ "set -euo pipefail", "if command -v backend-core >/dev/null 2>&1; then", ` exec backend-core --fetch-json ${shellQuote(url)} --method ${shellQuote(method)} --max-response-bytes ${shellQuote(String(maxResponseBytes))}${body.length > 0 ? ` --body-json ${shellQuote(body)}` : ""}`, "fi", `url=${shellQuote(url)}`, `method=${shellQuote(method)}`, `max_bytes=${shellQuote(String(maxResponseBytes))}`, `body=${shellQuote(body)}`, "export url method body max_bytes", "bun -e 'const url=process.env.url; const method=process.env.method; const body=process.env.body; const maxBytes=Number(process.env.max_bytes||\"5000000\"); fetch(url,{method,body:body?body:undefined,headers:body?{\"content-type\":\"application/json\"}:undefined}).then(async r=>{const text=await r.text(); const out={ok:r.ok,status:r.status,body:text?JSON.parse(text):null}; const json=JSON.stringify(out); if (Buffer.byteLength(json) > maxBytes) { console.error(\"response too large\"); process.exit(1); } console.log(json); process.exit(r.ok?0:1);}).catch(e=>{console.error(e); process.exit(1)})'", ].join("\n"); return ["docker", "exec", "unidesk-backend-core", "sh", "-lc", script]; } interface RelatedContainer { name: string; image: string; status: string; } function unquoteEnvValue(value: string): string { return value.replace(/^['"]|['"]$/g, ""); } function baiduSecretPresence(envPath: string): Record { const keys = [ "UNIDESK_BAIDU_NETDISK_CLIENT_ID", "UNIDESK_BAIDU_NETDISK_CLIENT_SECRET", "UNIDESK_BAIDU_NETDISK_TOKEN_KEY", ]; if (!existsSync(envPath)) { return { envPath, exists: false, keys: Object.fromEntries(keys.map((key) => [key, { present: false, nonEmpty: false }])) }; } const env = new Map(); for (const line of readFileSync(envPath, "utf8").split(/\r?\n/u)) { if (line.length === 0 || line.trimStart().startsWith("#")) continue; const index = line.indexOf("="); if (index === -1) continue; env.set(line.slice(0, index), unquoteEnvValue(line.slice(index + 1))); } return { envPath, exists: true, keys: Object.fromEntries(keys.map((key) => [key, { present: env.has(key), nonEmpty: Boolean((env.get(key) || "").trim()) }])), }; } function listRelatedStackContainers(): RelatedContainer[] { const result = runCommand(["docker", "ps", "-a", "--format", "{{json .}}"], repoRoot); if (result.exitCode !== 0 || result.stdout.trim().length === 0) return []; const names = ["unidesk-backend-core", "unidesk-database", "baidu-netdisk-backend"]; return result.stdout .split("\n") .map((line) => line.trim()) .filter(Boolean) .map((line) => { try { return JSON.parse(line) as Record; } catch { return null; } }) .filter((row): row is Record => row !== null) .filter((row) => names.some((name) => row.Names === name || String(row.Names || "").startsWith(`${name}.`))) .map((row) => ({ name: row.Names ?? "", image: row.Image ?? "", status: row.Status ?? "" })); } export function backendCoreUnavailableDiagnostic(detail: { exitCode: number | null; stdoutTail: string; stderrTail: string; relatedContainers: RelatedContainer[]; envPath: string; baiduSecretPresence: Record; }): Record { const expectedContainers = ["unidesk-backend-core", "unidesk-database", "baidu-netdisk-backend"]; const presentExactNames = new Set(detail.relatedContainers.map((container) => container.name)); const missingContainers = expectedContainers.filter((name) => !presentExactNames.has(name)); const verifyOnlyObserved = detail.relatedContainers.some((container) => /\.verify-/u.test(container.name)); return { ok: false, failureKind: "target-stack-not-running", degradedReason: "backend-core-container-missing", runnerDisposition: "infra-blocked", readOnlyReview: true, message: verifyOnlyObserved ? "backend-core/database target containers are not running; only verify-only containers were observed." : "backend-core target container is not running, so private backend-core API checks cannot observe schedules or microservices.", targetStack: { expectedContainers, missingContainers, relatedContainers: detail.relatedContainers, verifyOnlyObserved, }, observed: { commandExitCode: detail.exitCode, stdoutTail: detail.stdoutTail, stderrTail: detail.stderrTail, }, baiduNetdiskConfigPresence: detail.baiduSecretPresence, readOnlyCommands: [ "bun scripts/cli.ts server status", "bun scripts/cli.ts schedule list", "bun scripts/cli.ts schedule runs --limit 20", "bun scripts/cli.ts microservice health baidu-netdisk", "bun scripts/cli.ts microservice proxy baidu-netdisk /api/auth/status --raw", "bun scripts/cli.ts microservice proxy baidu-netdisk '/api/transfers?limit=20' --raw", ], authorizationRequiredForRecovery: [ "restore non-empty Baidu Netdisk runtime secrets in the controlled Compose env source without printing secret values", "start or deploy the production target stack containing backend-core, database, and baidu-netdisk", "after health is green, explicitly authorize schedule retry-run or schedule run before triggering a real PGDATA backup", ], blockedWithoutAuthorization: [ "server start", "server rebuild backend-core", "server rebuild baidu-netdisk", "deploy apply --env prod --service baidu-netdisk", "schedule run ", "schedule retry-run ", ], }; } function backendCoreContainerMissing(stderr: string): boolean { return stderr.includes("No such container: unidesk-backend-core"); } export function coreInternalFetch(path: string, init?: { method?: string; body?: unknown; maxResponseBytes?: number }): unknown { if (!path.startsWith("/")) throw new Error("core internal path must start with /"); const command = dockerCoreFetchCommand(path, init); const result = runCommand(command, repoRoot); if (result.exitCode !== 0) { if (backendCoreContainerMissing(result.stderr)) { const envPath = join(repoRoot, ".state", "docker-compose.env"); return backendCoreUnavailableDiagnostic({ exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200), relatedContainers: listRelatedStackContainers(), envPath, baiduSecretPresence: baiduSecretPresence(envPath), }); } const parsedStdout = parseJsonRecord(result.stdout.trim()); if (parsedStdout !== null) { return { ...parsedStdout, commandExitCode: result.exitCode, commandStderrTail: result.stderr.slice(-1200), }; } const codeQueueStableProxy = path.startsWith("/api/microservices/code-queue/"); return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200), ...(codeQueueStableProxy ? { codeQueueObservationNote: "The stable /api/microservices/code-queue path could not be reached from this CLI container. This does not by itself mean Code Queue tasks are unevaluable or stopped. From the supervisor or main-server CLI environment, use the existing codex queues/tasks/task observation commands instead of treating this container-local proxy failure as liveness evidence.", recommendedCommands: [ "bun scripts/cli.ts codex queues", "bun scripts/cli.ts codex tasks --limit 20", "bun scripts/cli.ts codex task ", ], } : {}), }; } try { return JSON.parse(result.stdout.trim()) as unknown; } catch { return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } } function parseJsonRecord(text: string): Record | null { if (text.length === 0) return null; try { const parsed = JSON.parse(text) as unknown; return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) ? parsed as Record : null; } catch { return null; } } function requireId(value: string | undefined, command: string): string { if (value === undefined || value.length === 0) throw new Error(`${command} requires microservice id`); return value; } function requireProxyPath(value: string | undefined): string { if (value === undefined || value.length === 0) throw new Error("microservice proxy requires upstream path, for example /api/summary"); if (!value.startsWith("/")) throw new Error("microservice proxy upstream path must start with /"); return value; } function encodeId(value: string): string { return encodeURIComponent(value); } function numberOption(args: string[], name: string, defaultValue: number): number { const index = args.indexOf(name); if (index === -1) return defaultValue; const raw = args[index + 1]; const value = Number(raw); if (!Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`); return value; } function cappedNumberOption(args: string[], name: string, defaultValue: number, maxValue: number): number { const value = numberOption(args, name, defaultValue); return Math.min(value, maxValue); } function stringOption(args: string[], name: string): string | undefined { const index = args.indexOf(name); if (index === -1) return undefined; const raw = args[index + 1]; if (raw === undefined || raw.length === 0) throw new Error(`${name} requires a non-empty value`); return raw; } function hasFlag(args: string[], name: string): boolean { return args.includes(name); } function asRecord(value: unknown): Record | null { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; } function asArray(value: unknown): unknown[] { return Array.isArray(value) ? value : []; } function parseJsonOption(raw: string, name: string): unknown { try { return JSON.parse(raw) as unknown; } catch (error) { const message = error instanceof Error ? error.message : String(error); throw new Error(`${name} must be valid JSON: ${message}`); } } function requestBodyOption(args: string[]): unknown | undefined { const bodyJson = stringOption(args, "--body-json"); const bodyFile = stringOption(args, "--body-file"); const bodyStdin = hasFlag(args, "--body-stdin"); const sources = [bodyJson !== undefined, bodyFile !== undefined, bodyStdin].filter(Boolean).length; if (sources > 1) throw new Error("microservice proxy accepts only one request body source: --body-json, --body-file, or --body-stdin"); if (bodyJson !== undefined) return parseJsonOption(bodyJson, "--body-json"); if (bodyFile !== undefined) { const text = bodyFile === "-" ? readFileSync(0, "utf8") : readFileSync(bodyFile, "utf8"); return parseJsonOption(text, "--body-file"); } if (bodyStdin) return parseJsonOption(readFileSync(0, "utf8"), "--body-stdin"); return undefined; } function assertKnownObservationOptions(args: string[], command: string): void { const flags = new Set(["--full", "--raw"]); for (const arg of args) { if (!arg.startsWith("--")) continue; if (!flags.has(arg)) throw new Error(`unsupported ${command} option: ${arg}`); } } function methodOption(args: string[], hasBody = false): string { const method = (stringOption(args, "--method") ?? (hasBody ? "POST" : "GET")).toUpperCase(); if (!["GET", "HEAD", "POST", "DELETE", "PUT", "PATCH"].includes(method)) throw new Error(`unsupported --method ${method}`); if (hasBody && (method === "GET" || method === "HEAD")) throw new Error(`microservice proxy cannot send a request body with ${method}`); return method; } export function summarizeMicroserviceProxyResponse(response: unknown, args: string[]): unknown { const full = args.includes("--full"); const raw = args.includes("--raw"); const maxBodyBytes = full ? numberOption(args, "--max-body-bytes", 5_000_000) : cappedNumberOption(args, "--max-body-bytes", raw ? 120_000 : 60_000, 500_000); if (typeof response !== "object" || response === null || Array.isArray(response)) return response; const record = response as Record; if (!("body" in record)) return response; if (record.responseTruncated === true) { return { ...record, bodyOmitted: true, bodyMaxBytes: maxBodyBytes, rawHint: "The upstream response exceeded the CLI collection cap before JSON parsing; re-run with --raw --full and a specific --max-body-bytes only when the full body is required.", }; } const bodyBytes = jsonByteLength(record.body); if (bodyBytes <= maxBodyBytes) { if (!raw || full) return response; return { ...record, outputPolicy: { rawRequested: true, bounded: true, maxBodyBytes, bodyBytes, fullCommand: "Re-run with --raw --full to allow the complete body.", }, }; } const rest = { ...record }; delete rest.body; return { ...rest, bodyOmitted: true, bodyBytes, bodyMaxBytes: maxBodyBytes, bodyPreview: previewJson(record.body, { maxDepth: 3, maxArrayItems: 3, maxObjectKeys: 16, maxStringLength: 320 }), rawHint: raw && !full ? "The --raw response exceeded the default hard limit; re-run with --raw --full for the complete body, or add/tighten __unideskArrayLimit=: in the proxied path." : "Re-run with --raw --full for the complete body, or add/tighten __unideskArrayLimit=: in the proxied path.", }; } function compactStringList(value: unknown, limit = 8): Record { const all = Array.from(new Set(asArray(value).map((item) => String(item ?? "")).filter(Boolean))); return { items: all.slice(0, limit), count: all.length, truncated: all.length > limit, omitted: Math.max(0, all.length - limit), }; } function compactRecordFields(record: Record | null, keys: string[]): Record | null { if (record === null) return null; const selected: Record = {}; for (const key of keys) { if (record[key] !== undefined) selected[key] = record[key]; } return Object.keys(selected).length > 0 ? selected : null; } function compactQueueHealth(value: unknown): Record | null { const queue = asRecord(value); if (queue === null) return null; return { controlPlane: queue.controlPlane ?? null, defaultProviderId: queue.defaultProviderId ?? null, defaultModel: queue.defaultModel ?? null, counts: queue.counts ?? {}, total: queue.total ?? null, queueCount: queue.queueCount ?? null, activeQueueIds: compactStringList(queue.activeQueueIds, 8), activeTaskIds: compactStringList(queue.activeTaskIds ?? queue.databaseActiveTaskIds, 8), queuedTaskIds: compactStringList(queue.queuedTaskIds, 8), databaseActiveTaskCount: queue.databaseActiveTaskCount ?? null, schedulerActiveRunSlotCount: queue.schedulerActiveRunSlotCount ?? queue.activeRunSlotCount ?? null, }; } function compactSkillAvailability(value: unknown): Record | null { const skills = asRecord(value); if (skills === null) return null; return { ok: skills.ok ?? false, path: skills.path ?? null, source: skills.source ?? null, target: skills.target ?? null, mountPoint: skills.mountPoint ?? null, exists: skills.exists ?? false, available: skills.available ?? false, degraded: skills.degraded ?? true, blocker: skills.blocker ?? null, readonly: skills.readonly ?? false, skillCount: skills.skillCount ?? 0, requiredSkills: Array.isArray(skills.requiredSkills) ? skills.requiredSkills.map(String) : [], missingSkills: Array.isArray(skills.missingSkills) ? skills.missingSkills.map(String) : [], valuesPrinted: skills.valuesPrinted ?? false, pathSpelling: compactRecordFields(asRecord(skills.pathSpelling), [ "expectedTarget", "forbiddenPathChecked", "forbiddenPathExists", "forbiddenPathMustNotBeUsed", ]), repairHint: skills.repairHint ?? null, }; } function compactSkillSync(value: unknown): Record | null { const sync = asRecord(value); if (sync === null) return null; const source = compactRecordFields(asRecord(sync.source), [ "path", "approved", "exists", "directory", "readable", "writable", "readonly", "mountPoint", "skillCount", "requiredSkills", "missingSkills", "error", ]); const target = compactRecordFields(asRecord(sync.target), [ "path", "approved", "exists", "directory", "readable", "writable", "readonly", "mountPoint", "skillCount", "requiredSkills", "missingSkills", "error", ]); const permissionFailures = Array.isArray(sync.permissionFailures) ? sync.permissionFailures : []; return { ok: sync.ok ?? false, degraded: sync.degraded ?? true, blocker: sync.blocker ?? null, mode: sync.mode ?? "dry-run", dryRun: sync.dryRun ?? true, mutation: sync.mutation ?? false, syncMode: sync.syncMode ?? null, source, target, counts: sync.counts ?? null, missing: sync.missing ?? null, permissionFailureCount: permissionFailures.length, permissionFailures: permissionFailures.slice(0, 4), plannedActions: sync.plannedActions ?? null, commands: sync.commands ?? null, valuesPrinted: sync.valuesPrinted ?? false, }; } function compactMicroserviceBody(body: unknown, serviceId: string): Record | null { const record = asRecord(body); if (record === null) return null; const diagnostics = asRecord(record.executionDiagnostics) ?? asRecord(record.diagnostics); const devReady = asRecord(record.devReady); return { ok: record.ok ?? null, serviceId: record.serviceId ?? record.service ?? serviceId, service: record.service ?? null, role: record.role ?? null, status: record.status ?? null, healthy: record.healthy ?? null, mode: record.mode ?? null, environment: record.environment ?? null, version: record.version ?? record.gatewayVersion ?? null, target: record.target ?? record.k3sServiceId ?? null, updatedAt: record.updatedAt ?? record.observedAt ?? record.generatedAt ?? null, startedAt: record.startedAt ?? null, taskCount: record.taskCount ?? null, schemaReady: record.schemaReady ?? null, queue: compactQueueHealth(record.queue), skills: compactSkillAvailability(record.skills), skillsSync: compactSkillSync(record.skillsSync), resourceBudget: compactRecordFields(asRecord(record.resourceBudget), [ "targetMemoryMb", "mgrPoolMax", "tracePoolMax", "noRunnerDependencies", "noDockerSocket", "noPlaywright", "rustRewrite", ]), topLevelKeys: compactStringList(Object.keys(record), 16), devReady: devReady === null ? null : { ok: devReady.ok ?? null, missingTools: compactStringList(devReady.missingTools, 8), skills: compactSkillAvailability(devReady.skills), skillsSync: compactSkillSync(devReady.skillsSync), }, executionDiagnostics: diagnostics === null ? null : { state: diagnostics.state ?? null, effectiveLiveness: diagnostics.effectiveLiveness ?? null, recommendedAction: diagnostics.recommendedAction ?? null, splitBrainLive: diagnostics.splitBrainLive ?? null, schedulerActiveRunSlotCount: diagnostics.schedulerActiveRunSlotCount ?? null, databaseActiveTaskCount: diagnostics.databaseActiveTaskCount ?? null, heartbeatFreshTaskIds: compactStringList(diagnostics.heartbeatFreshTaskIds, 8), heartbeatRiskTaskIds: compactStringList(diagnostics.heartbeatRiskTaskIds, 8), }, }; } export function summarizeMicroserviceObservation(action: string, serviceId: string, response: unknown, args: string[]): unknown { if (hasFlag(args, "--full") || hasFlag(args, "--raw")) return response; const record = asRecord(response); if (record === null) return response; const body = "body" in record ? record.body : response; const bodyBytes = jsonByteLength(body); const summary = compactMicroserviceBody(body, serviceId); const includePreview = summary === null || bodyBytes <= 20_000; return { upstream: { ok: record.ok ?? null, status: record.status ?? null, exitCode: record.exitCode ?? null, }, microservice: { action, id: serviceId, summary, bodyBytes, bodyOmitted: true, outputPolicy: { default: "compact-health-summary", full: `bun scripts/cli.ts microservice ${action} ${serviceId} --full`, raw: `bun scripts/cli.ts microservice ${action} ${serviceId} --raw`, proxyRaw: serviceId === "code-queue" && action === "health" ? "bun scripts/cli.ts microservice proxy code-queue /health --raw --full" : null, }, ...(includePreview ? { bodyPreview: previewJson(body, { maxDepth: 2, maxArrayItems: 2, maxObjectKeys: 8, maxStringLength: 160 }) } : { bodyPreviewOmitted: true, bodyPreviewHint: "Large body preview omitted because compact summary is available; use --full or --raw for complete evidence.", }), }, }; } export async function runMicroserviceCommand(_config: UniDeskConfig, args: string[]): Promise { const [action = "list", idArg, pathArg] = args; if (action === "list") return coreInternalFetch("/api/microservices"); if (action === "status") { const id = requireId(idArg, "microservice status"); const optionArgs = args.slice(2); assertKnownObservationOptions(optionArgs, "microservice status"); return summarizeMicroserviceObservation(action, id, coreInternalFetch(`/api/microservices/${encodeId(id)}/status`), optionArgs); } if (action === "health") { const id = requireId(idArg, "microservice health"); const optionArgs = args.slice(2); assertKnownObservationOptions(optionArgs, "microservice health"); return summarizeMicroserviceObservation(action, id, coreInternalFetch(`/api/microservices/${encodeId(id)}/health`), optionArgs); } if (action === "diagnostics") { const id = requireId(idArg, "microservice diagnostics"); const optionArgs = args.slice(2); assertKnownObservationOptions(optionArgs, "microservice diagnostics"); return summarizeMicroserviceObservation(action, id, coreInternalFetch(`/api/microservices/${encodeId(id)}/diagnostics`), optionArgs); } if (action === "tunnel-self-test") { const id = requireId(idArg, "microservice tunnel-self-test"); return coreInternalFetch(`/api/microservices/${encodeId(id)}/tunnel-self-test`); } if (action === "proxy") { const id = requireId(idArg, "microservice proxy"); const path = requireProxyPath(pathArg); const body = requestBodyOption(args); const full = hasFlag(args, "--full"); const raw = hasFlag(args, "--raw"); const maxBodyBytes = full ? numberOption(args, "--max-body-bytes", 5_000_000) : cappedNumberOption(args, "--max-body-bytes", raw ? 120_000 : 60_000, 500_000); const maxResponseBytes = full ? Math.min(Math.max(maxBodyBytes, 120_000), 5_000_000) : Math.min(Math.max(maxBodyBytes * 3, 240_000), 1_500_000); return summarizeMicroserviceProxyResponse(coreInternalFetch(`/api/microservices/${encodeId(id)}/proxy${path}`, { method: methodOption(args, body !== undefined), body, maxResponseBytes }), args); } throw new Error("microservice command must be one of: list, status, health, diagnostics, tunnel-self-test, proxy"); }