import { runCommand } from "./command"; import { type UniDeskConfig, repoRoot } from "./config"; export const dispatchCommands = ["docker.ps", "provider.upgrade", "host.ssh", "microservice.http", "echo"] as const; export type DebugDispatchCommand = typeof dispatchCommands[number]; export function isDebugDispatchCommand(value: unknown): value is DebugDispatchCommand { return dispatchCommands.includes(value as DebugDispatchCommand); } async function readJson(url: string, init?: RequestInit): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), 5000); try { const res = await fetch(url, { ...init, signal: controller.signal }); const text = await res.text(); return { ok: res.ok, status: res.status, body: text.length > 0 ? JSON.parse(text) : null }; } catch (error) { return { ok: false, error: error instanceof Error ? error.message : String(error) }; } finally { clearTimeout(timer); } } function coreFetchCommand(path: string, init?: { method?: string; body?: unknown }): string[] { const method = init?.method ?? "GET"; const url = `http://127.0.0.1:8080${path}`; const body = init?.body === undefined ? "" : JSON.stringify(init.body); const script = [ "set -eu", "if command -v backend-core >/dev/null 2>&1; then", ` exec backend-core --fetch-json ${shellQuote(url)} --method ${shellQuote(method)}${body.length > 0 ? ` --body-json ${shellQuote(body)}` : ""}`, "fi", `url=${shellQuote(url)}`, `method=${shellQuote(method)}`, `body=${shellQuote(body)}`, "export url method body", "bun -e 'const url=process.env.url; const method=process.env.method; const body=process.env.body; fetch(url,{method,body:body?body:undefined,headers:body?{\"content-type\":\"application/json\"}:undefined}).then(async r=>{const text=await r.text(); console.log(JSON.stringify({ok:r.ok,status:r.status,body:text?JSON.parse(text):null})); process.exit(r.ok?0:1);}).catch(e=>{console.error(e); process.exit(1);})'", ].join("\n"); return ["docker", "exec", "unidesk-backend-core", "sh", "-lc", script]; } function shellQuote(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'`; } function coreInternalFetch(path: string, init?: { method?: string; body?: unknown }): unknown { const command = coreFetchCommand(path, init); const result = runCommand(command, repoRoot); if (result.exitCode !== 0) { return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } try { return JSON.parse(result.stdout.trim()) as unknown; } catch { return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } } function coreDockerStatusSummary(): unknown { const result = runCommand(coreFetchCommand("/api/nodes/docker-status"), repoRoot); if (result.exitCode !== 0) { return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } try { const parsed = JSON.parse(result.stdout.trim()) as { ok?: boolean; status?: number; body?: { dockerStatuses?: Array> } }; const dockerStatuses = (parsed.body?.dockerStatuses || []).map((item) => { const status = (item.dockerStatus || {}) as Record; return { providerId: item.providerId, name: item.name, nodeStatus: item.nodeStatus, updatedAt: item.updatedAt, dockerStatus: { ok: status.ok, socketPresent: status.socketPresent, collectedAt: status.collectedAt, counts: status.counts, daemon: status.daemon, containersPreview: Array.isArray(status.containers) ? status.containers.slice(0, 8) : [], }, }; }); return { ok: parsed.ok, status: parsed.status, body: { ok: parsed.body !== undefined, dockerStatuses } }; } catch { return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } } function coreSystemStatusSummary(): unknown { const result = runCommand(coreFetchCommand("/api/nodes/system-status?limit=24"), repoRoot); if (result.exitCode !== 0) { return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } try { const parsed = JSON.parse(result.stdout.trim()) as { ok?: boolean; status?: number; body?: { systemStatuses?: Array> } }; const systemStatuses = (parsed.body?.systemStatuses || []).map((item) => { const current = (item.current || {}) as Record; const history = Array.isArray(item.history) ? item.history : []; return { providerId: item.providerId, name: item.name, nodeStatus: item.nodeStatus, updatedAt: item.updatedAt, current: item.current ? { ok: current.ok, collectedAt: current.collectedAt, cpu: current.cpu, memory: current.memory, disk: current.disk } : null, historyPreview: history.slice(-8), historyCount: history.length, }; }); return { ok: parsed.ok, status: parsed.status, body: { ok: parsed.body !== undefined, systemStatuses } }; } catch { return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } } export async function debugHealth(config: UniDeskConfig): Promise { return { coreInternal: await coreInternalFetch("/health"), overviewInternal: await coreInternalFetch("/api/overview"), nodesInternal: await coreInternalFetch("/api/nodes"), systemStatusInternal: coreSystemStatusSummary(), dockerStatusInternal: coreDockerStatusSummary(), frontendPublic: await readJson(`http://127.0.0.1:${config.network.frontend.port}/health`), providerIngressPublic: await readJson(`http://127.0.0.1:${config.network.providerIngress.port}/health`), publicExposureBoundary: { coreHostPort: { port: config.network.core.port, expected: "not-exposed" }, databaseHostPort: { port: config.network.database.port, expected: "restricted-to-code-queue-provider" }, oaEventFlowHostPort: { port: 4255, expected: "restricted-to-code-queue-provider" }, }, }; } function recordValue(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function arrayValue(value: unknown): unknown[] { return Array.isArray(value) ? value : []; } function stringValue(value: unknown): string | null { return typeof value === "string" ? value : null; } export async function debugSshPool(_config: UniDeskConfig, providerId: string): Promise { const nodesResponse = await coreInternalFetch("/api/nodes"); const body = recordValue(recordValue(nodesResponse).body); const nodes = arrayValue(body.nodes); const node = nodes .map((item) => recordValue(item)) .find((item) => item.providerId === providerId) ?? null; if (node === null) { return { ok: false, providerId, degradedReason: "provider-not-found", nodesFetch: nodesResponse, next: { fullHealth: "bun scripts/cli.ts debug health" }, }; } const labels = recordValue(node.labels); const pool = { transport: stringValue(labels.providerGatewaySshDataTransport), host: stringValue(labels.providerGatewaySshDataHost), port: labels.providerGatewaySshDataPort ?? null, desired: labels.providerGatewaySshDataPoolDesired ?? null, total: labels.providerGatewaySshDataPoolTotal ?? null, ready: labels.providerGatewaySshDataPoolReady ?? null, claimed: labels.providerGatewaySshDataPoolClaimed ?? null, connecting: labels.providerGatewaySshDataPoolConnecting ?? null, lastError: labels.providerGatewaySshDataPoolLastError ?? null, }; const ready = Number(pool.ready ?? 0); const claimed = Number(pool.claimed ?? 0); const desired = Number(pool.desired ?? 0); const ok = pool.transport === "tcp-pool" && Number.isFinite(ready) && ready > 0; return { ok, providerId, node: { providerId: node.providerId, name: node.name, status: node.status, lastHeartbeat: node.lastHeartbeat ?? null, updatedAt: node.updatedAt ?? null, }, pool, classification: ok ? "ssh-tcp-pool-ready" : pool.transport !== "tcp-pool" ? "provider-gateway-upgrade-required" : desired > 0 && claimed >= desired ? "provider-data-pool-exhausted" : "provider-data-pool-not-ready", next: { smoke: `trans ${providerId} argv true`, fullHealth: "bun scripts/cli.ts debug health", }, }; } export async function debugEgressProxy(_config: UniDeskConfig, providerId: string): Promise { const nodesResponse = await coreInternalFetch("/api/nodes"); const body = recordValue(recordValue(nodesResponse).body); const nodes = arrayValue(body.nodes); const node = nodes .map((item) => recordValue(item)) .find((item) => item.providerId === providerId) ?? null; if (node === null) { return { ok: false, providerId, degradedReason: "provider-not-found", nodesFetch: nodesResponse, next: { fullHealth: "bun scripts/cli.ts debug health" }, }; } const labels = recordValue(node.labels); const active = Number(labels.providerGatewayEgressProxyActiveTunnels ?? 0); const pending = Number(labels.providerGatewayEgressProxyPendingTunnels ?? 0); const opened = Number(labels.providerGatewayEgressProxyOpenedTunnels ?? 0); const stale = Number(labels.providerGatewayEgressProxyStaleTunnels ?? 0); const enabled = labels.providerGatewayEgressProxy === true; const connected = labels.providerGatewayEgressProxyConnected === true; const sshReady = Number(labels.providerGatewaySshDataPoolReady ?? 0); const activeTunnelDetails = arrayValue(labels.providerGatewayEgressProxyActiveTunnelDetails); const recentClosedTunnels = arrayValue(labels.providerGatewayEgressProxyRecentClosedTunnels); const ok = enabled && connected && stale === 0; return { ok, providerId, node: { providerId: node.providerId, name: node.name, status: node.status, lastHeartbeat: node.lastHeartbeat ?? null, updatedAt: node.updatedAt ?? null, }, egressProxy: { enabled, connected, port: labels.providerGatewayEgressProxyPort ?? null, activeTunnels: Number.isFinite(active) ? active : 0, pendingTunnels: Number.isFinite(pending) ? pending : 0, openedTunnels: Number.isFinite(opened) ? opened : 0, staleTunnels: Number.isFinite(stale) ? stale : 0, oldestTunnelAgeMs: labels.providerGatewayEgressProxyOldestTunnelAgeMs ?? null, policy: { openTimeoutMs: labels.providerGatewayEgressProxyOpenTimeoutMs ?? null, idleTimeoutMs: labels.providerGatewayEgressProxyIdleTimeoutMs ?? null, maxTunnelAgeMs: labels.providerGatewayEgressProxyMaxTunnelAgeMs ?? null, staleTunnelIdleMs: labels.providerGatewayEgressProxyStaleTunnelIdleMs ?? null, maxPendingBytes: labels.providerGatewayEgressProxyMaxPendingBytes ?? null, }, activeTunnelDetails, activeTunnelDetailsCount: activeTunnelDetails.length, recentClosedTunnels, recentClosedTunnelCount: recentClosedTunnels.length, }, controlPlaneSeparation: { sshDataTransport: stringValue(labels.providerGatewaySshDataTransport), sshDataPoolReady: Number.isFinite(sshReady) ? sshReady : 0, sshDataPoolDesired: labels.providerGatewaySshDataPoolDesired ?? null, sshDataPoolClaimed: labels.providerGatewaySshDataPoolClaimed ?? null, note: "SSH/control readiness is reported separately from egress download tunnel activity.", }, classification: !enabled ? "egress-proxy-disabled" : !connected ? "egress-proxy-core-channel-disconnected" : stale > 0 ? "egress-tunnel-stale" : active > 0 ? "egress-tunnel-active" : "egress-proxy-ready", outputPolicy: { redaction: "activeTunnelDetails expose target host/port and path/query-key summary only; URL credentials, headers, tokens and query values are not emitted.", boundedRecentClosedTunnels: true, }, next: { smoke: `trans ${providerId} argv true`, fullHealth: "bun scripts/cli.ts debug health", }, }; } async function waitForTask(taskId: string, timeoutMs: number): Promise { const started = Date.now(); let latest: unknown = null; while (Date.now() - started < timeoutMs) { latest = coreInternalFetch("/api/tasks?limit=100"); const tasks = (latest as { body?: { tasks?: Array<{ id?: string; status?: string; result?: unknown }> } }).body?.tasks ?? []; const task = tasks.find((item) => item.id === taskId); if (task?.status === "succeeded" || task?.status === "failed") return { ok: true, task }; await Bun.sleep(500); } return { ok: false, timeoutMs, latest }; } export async function debugTask(_config: UniDeskConfig, taskId: string): Promise { const tasksResponse = coreInternalFetch("/api/tasks?limit=100"); const tasks = (tasksResponse as { body?: { tasks?: Array<{ id?: string }> } }).body?.tasks ?? []; const task = taskId === "latest" ? tasks[0] : tasks.find((item) => item.id === taskId); return { tasksResponse, taskId, task: task ?? null }; } export async function debugDispatch( config: UniDeskConfig, providerId: string, command: DebugDispatchCommand, payload?: Record, waitMs = 0, ): Promise { const dispatchPayload = payload ?? (command === "provider.upgrade" ? { source: "cli-debug", mode: "plan" } : { source: "cli-debug" }); const dispatch = coreInternalFetch("/api/dispatch", { method: "POST", body: { providerId, command, payload: dispatchPayload }, }); const taskId = (dispatch as { body?: { taskId?: string } }).body?.taskId ?? ""; const wait = waitMs > 0 && taskId.length > 0 ? await waitForTask(taskId, waitMs) : null; return { dispatch, wait }; }