import { runCommand } from "./command"; import { type UniDeskConfig, repoRoot } from "./config"; export const dispatchCommands = ["docker.ps", "provider.upgrade", "host.ssh", "microservice.http", "echo"] as const; export type DebugDispatchCommand = typeof dispatchCommands[number]; export function isDebugDispatchCommand(value: unknown): value is DebugDispatchCommand { return dispatchCommands.includes(value as DebugDispatchCommand); } async function readJson(url: string, init?: RequestInit): Promise { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), 5000); try { const res = await fetch(url, { ...init, signal: controller.signal }); const text = await res.text(); return { ok: res.ok, status: res.status, body: text.length > 0 ? JSON.parse(text) : null }; } catch (error) { return { ok: false, error: error instanceof Error ? error.message : String(error) }; } finally { clearTimeout(timer); } } function coreFetchCommand(path: string, init?: { method?: string; body?: unknown }): string[] { const method = init?.method ?? "GET"; const url = `http://127.0.0.1:8080${path}`; const body = init?.body === undefined ? "" : JSON.stringify(init.body); const script = [ "set -eu", "if command -v backend-core >/dev/null 2>&1; then", ` exec backend-core --fetch-json ${shellQuote(url)} --method ${shellQuote(method)}${body.length > 0 ? ` --body-json ${shellQuote(body)}` : ""}`, "fi", `url=${shellQuote(url)}`, `method=${shellQuote(method)}`, `body=${shellQuote(body)}`, "export url method body", "bun -e 'const url=process.env.url; const method=process.env.method; const body=process.env.body; fetch(url,{method,body:body?body:undefined,headers:body?{\"content-type\":\"application/json\"}:undefined}).then(async r=>{const text=await r.text(); console.log(JSON.stringify({ok:r.ok,status:r.status,body:text?JSON.parse(text):null})); process.exit(r.ok?0:1);}).catch(e=>{console.error(e); process.exit(1);})'", ].join("\n"); return ["docker", "exec", "unidesk-backend-core", "sh", "-lc", script]; } function shellQuote(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'`; } function coreInternalFetch(path: string, init?: { method?: string; body?: unknown }): unknown { const command = coreFetchCommand(path, init); const result = runCommand(command, repoRoot); if (result.exitCode !== 0) { return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } try { return JSON.parse(result.stdout.trim()) as unknown; } catch { return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } } function coreDockerStatusSummary(): unknown { const result = runCommand(coreFetchCommand("/api/nodes/docker-status"), repoRoot); if (result.exitCode !== 0) { return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } try { const parsed = JSON.parse(result.stdout.trim()) as { ok?: boolean; status?: number; body?: { dockerStatuses?: Array> } }; const dockerStatuses = (parsed.body?.dockerStatuses || []).map((item) => { const status = (item.dockerStatus || {}) as Record; return { providerId: item.providerId, name: item.name, nodeStatus: item.nodeStatus, updatedAt: item.updatedAt, dockerStatus: { ok: status.ok, socketPresent: status.socketPresent, collectedAt: status.collectedAt, counts: status.counts, daemon: status.daemon, containersPreview: Array.isArray(status.containers) ? status.containers.slice(0, 8) : [], }, }; }); return { ok: parsed.ok, status: parsed.status, body: { ok: parsed.body !== undefined, dockerStatuses } }; } catch { return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } } function coreSystemStatusSummary(): unknown { const result = runCommand(coreFetchCommand("/api/nodes/system-status?limit=24"), repoRoot); if (result.exitCode !== 0) { return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } try { const parsed = JSON.parse(result.stdout.trim()) as { ok?: boolean; status?: number; body?: { systemStatuses?: Array> } }; const systemStatuses = (parsed.body?.systemStatuses || []).map((item) => { const current = (item.current || {}) as Record; const history = Array.isArray(item.history) ? item.history : []; return { providerId: item.providerId, name: item.name, nodeStatus: item.nodeStatus, updatedAt: item.updatedAt, current: item.current ? { ok: current.ok, collectedAt: current.collectedAt, cpu: current.cpu, memory: current.memory, disk: current.disk } : null, historyPreview: history.slice(-8), historyCount: history.length, }; }); return { ok: parsed.ok, status: parsed.status, body: { ok: parsed.body !== undefined, systemStatuses } }; } catch { return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) }; } } export async function debugHealth(config: UniDeskConfig): Promise { return { coreInternal: await coreInternalFetch("/health"), overviewInternal: await coreInternalFetch("/api/overview"), nodesInternal: await coreInternalFetch("/api/nodes"), systemStatusInternal: coreSystemStatusSummary(), dockerStatusInternal: coreDockerStatusSummary(), frontendPublic: await readJson(`http://127.0.0.1:${config.network.frontend.port}/health`), providerIngressPublic: await readJson(`http://127.0.0.1:${config.network.providerIngress.port}/health`), publicExposureBoundary: { coreHostPort: { port: config.network.core.port, expected: "not-exposed" }, databaseHostPort: { port: config.network.database.port, expected: "restricted-to-code-queue-provider" }, oaEventFlowHostPort: { port: 4255, expected: "restricted-to-code-queue-provider" }, }, }; } function recordValue(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; } function arrayValue(value: unknown): unknown[] { return Array.isArray(value) ? value : []; } function stringValue(value: unknown): string | null { return typeof value === "string" ? value : null; } function numberValue(value: unknown): number | null { const parsed = Number(value); return Number.isFinite(parsed) ? parsed : null; } function rounded(value: number | null): number | null { return value === null ? null : Math.round(value * 100) / 100; } function providerHostPressureSummary(providerId: string): unknown { const response = coreInternalFetch("/api/nodes/system-status?limit=24"); const body = recordValue(recordValue(response).body); const systemStatuses = arrayValue(body.systemStatuses); const item = systemStatuses .map((entry) => recordValue(entry)) .find((entry) => entry.providerId === providerId) ?? null; if (item === null) { return { ok: false, degradedReason: "system-status-not-found", systemStatusFetch: response, }; } const current = recordValue(item.current); if (Object.keys(current).length === 0) { return { ok: false, providerId, degradedReason: "system-status-current-missing", updatedAt: item.updatedAt ?? null, nodeStatus: item.nodeStatus ?? null, classification: "host-pressure-snapshot-unavailable", }; } const cpu = recordValue(current.cpu); const memory = recordValue(current.memory); const disk = recordValue(current.disk); const cores = numberValue(cpu.cores); const load1 = numberValue(cpu.load1); const load5 = numberValue(cpu.load5); const load15 = numberValue(cpu.load15); const load1PerCore = cores !== null && cores > 0 && load1 !== null ? load1 / cores : null; const memoryPercent = numberValue(memory.percent); const memoryAvailableBytes = numberValue(memory.availableBytes); const swapTotalBytes = numberValue(memory.swapTotalBytes); const diskPercent = numberValue(disk.percent); const signals: string[] = []; if (load1PerCore !== null && load1PerCore >= 4) signals.push("load1-per-core>=4"); if (memoryPercent !== null && memoryPercent >= 90) signals.push("memory-percent>=90"); if (memoryAvailableBytes !== null && memoryAvailableBytes < 512 * 1024 * 1024) signals.push("memory-available<512MiB"); if (swapTotalBytes === 0) signals.push("swap-disabled"); if (diskPercent !== null && diskPercent >= 90) signals.push("disk-percent>=90"); return { ok: current.ok ?? null, providerId, updatedAt: item.updatedAt ?? null, collectedAt: current.collectedAt ?? null, cpu: { cores, load1, load5, load15, load1PerCore: rounded(load1PerCore), }, memory: { percent: memoryPercent, availableBytes: memoryAvailableBytes, swapTotalBytes, }, disk: { percent: diskPercent, availableBytes: numberValue(disk.availableBytes), mount: disk.mount ?? null, }, signals, classification: signals.length > 0 ? "host-pressure-signals-present" : "host-pressure-signals-not-observed", }; } export async function debugSshPool(_config: UniDeskConfig, providerId: string): Promise { const nodesResponse = await coreInternalFetch("/api/nodes"); const body = recordValue(recordValue(nodesResponse).body); const nodes = arrayValue(body.nodes); const node = nodes .map((item) => recordValue(item)) .find((item) => item.providerId === providerId) ?? null; if (node === null) { return { ok: false, providerId, degradedReason: "provider-not-found", nodesFetch: nodesResponse, next: { fullHealth: "bun scripts/cli.ts debug health" }, }; } const labels = recordValue(node.labels); const nodeStatus = stringValue(node.status); const pool = { transport: stringValue(labels.providerGatewaySshDataTransport), host: stringValue(labels.providerGatewaySshDataHost), port: labels.providerGatewaySshDataPort ?? null, desired: labels.providerGatewaySshDataPoolDesired ?? null, total: labels.providerGatewaySshDataPoolTotal ?? null, ready: labels.providerGatewaySshDataPoolReady ?? null, claimed: labels.providerGatewaySshDataPoolClaimed ?? null, connecting: labels.providerGatewaySshDataPoolConnecting ?? null, lastError: labels.providerGatewaySshDataPoolLastError ?? null, }; const ready = Number(pool.ready ?? 0); const claimed = Number(pool.claimed ?? 0); const desired = Number(pool.desired ?? 0); const providerOnline = nodeStatus === "online"; const poolLabelReady = pool.transport === "tcp-pool" && Number.isFinite(ready) && ready > 0; const executionPathReady = providerOnline && poolLabelReady; return { ok: executionPathReady, providerId, providerOnline, poolLabelReady, executionPathReady, node: { providerId: node.providerId, name: node.name, status: nodeStatus, lastHeartbeat: node.lastHeartbeat ?? null, updatedAt: node.updatedAt ?? null, }, pool, hostPressure: providerHostPressureSummary(providerId), classification: !providerOnline ? (nodeStatus === "offline" ? "provider-offline" : "provider-not-online") : poolLabelReady ? "ssh-tcp-pool-ready" : pool.transport !== "tcp-pool" ? "provider-gateway-upgrade-required" : desired > 0 && claimed >= desired ? "provider-data-pool-exhausted" : "provider-data-pool-not-ready", next: { smoke: `trans ${providerId} argv true`, fullHealth: "bun scripts/cli.ts debug health", gcDegradedSnapshot: `bun scripts/cli.ts gc remote ${providerId} snapshot --no-save`, }, note: "Pool labels are provider-reported channel state; use the smoke command or gc degraded output to verify execution path.", }; } export async function debugEgressProxy(_config: UniDeskConfig, providerId: string): Promise { const nodesResponse = await coreInternalFetch("/api/nodes"); const body = recordValue(recordValue(nodesResponse).body); const nodes = arrayValue(body.nodes); const node = nodes .map((item) => recordValue(item)) .find((item) => item.providerId === providerId) ?? null; if (node === null) { return { ok: false, providerId, degradedReason: "provider-not-found", nodesFetch: nodesResponse, next: { fullHealth: "bun scripts/cli.ts debug health" }, }; } const labels = recordValue(node.labels); const active = Number(labels.providerGatewayEgressProxyActiveTunnels ?? 0); const pending = Number(labels.providerGatewayEgressProxyPendingTunnels ?? 0); const opened = Number(labels.providerGatewayEgressProxyOpenedTunnels ?? 0); const stale = Number(labels.providerGatewayEgressProxyStaleTunnels ?? 0); const enabled = labels.providerGatewayEgressProxy === true; const connected = labels.providerGatewayEgressProxyConnected === true; const sshReady = Number(labels.providerGatewaySshDataPoolReady ?? 0); const activeTunnelDetails = arrayValue(labels.providerGatewayEgressProxyActiveTunnelDetails); const recentClosedTunnels = arrayValue(labels.providerGatewayEgressProxyRecentClosedTunnels); const ok = enabled && connected && stale === 0; return { ok, providerId, node: { providerId: node.providerId, name: node.name, status: node.status, lastHeartbeat: node.lastHeartbeat ?? null, updatedAt: node.updatedAt ?? null, }, egressProxy: { enabled, connected, port: labels.providerGatewayEgressProxyPort ?? null, activeTunnels: Number.isFinite(active) ? active : 0, pendingTunnels: Number.isFinite(pending) ? pending : 0, openedTunnels: Number.isFinite(opened) ? opened : 0, staleTunnels: Number.isFinite(stale) ? stale : 0, oldestTunnelAgeMs: labels.providerGatewayEgressProxyOldestTunnelAgeMs ?? null, policy: { openTimeoutMs: labels.providerGatewayEgressProxyOpenTimeoutMs ?? null, idleTimeoutMs: labels.providerGatewayEgressProxyIdleTimeoutMs ?? null, maxTunnelAgeMs: labels.providerGatewayEgressProxyMaxTunnelAgeMs ?? null, staleTunnelIdleMs: labels.providerGatewayEgressProxyStaleTunnelIdleMs ?? null, maxPendingBytes: labels.providerGatewayEgressProxyMaxPendingBytes ?? null, }, activeTunnelDetails, activeTunnelDetailsCount: activeTunnelDetails.length, recentClosedTunnels, recentClosedTunnelCount: recentClosedTunnels.length, }, controlPlaneSeparation: { sshDataTransport: stringValue(labels.providerGatewaySshDataTransport), sshDataPoolReady: Number.isFinite(sshReady) ? sshReady : 0, sshDataPoolDesired: labels.providerGatewaySshDataPoolDesired ?? null, sshDataPoolClaimed: labels.providerGatewaySshDataPoolClaimed ?? null, note: "SSH/control readiness is reported separately from egress download tunnel activity.", }, classification: !enabled ? "egress-proxy-disabled" : !connected ? "egress-proxy-core-channel-disconnected" : stale > 0 ? "egress-tunnel-stale" : active > 0 ? "egress-tunnel-active" : "egress-proxy-ready", outputPolicy: { redaction: "activeTunnelDetails expose target host/port and path/query-key summary only; URL credentials, headers, tokens and query values are not emitted.", boundedRecentClosedTunnels: true, }, next: { smoke: `trans ${providerId} argv true`, fullHealth: "bun scripts/cli.ts debug health", }, }; } async function waitForTask(taskId: string, timeoutMs: number): Promise { const started = Date.now(); let latest: unknown = null; while (Date.now() - started < timeoutMs) { latest = coreInternalFetch("/api/tasks?limit=100"); const tasks = (latest as { body?: { tasks?: Array<{ id?: string; status?: string; result?: unknown }> } }).body?.tasks ?? []; const task = tasks.find((item) => item.id === taskId); if (task?.status === "succeeded" || task?.status === "failed") return { ok: true, task }; await Bun.sleep(500); } return { ok: false, timeoutMs, latest }; } export async function debugTask(_config: UniDeskConfig, taskId: string): Promise { const tasksResponse = coreInternalFetch("/api/tasks?limit=100"); const tasks = (tasksResponse as { body?: { tasks?: Array<{ id?: string }> } }).body?.tasks ?? []; const task = taskId === "latest" ? tasks[0] : tasks.find((item) => item.id === taskId); return { tasksResponse, taskId, task: task ?? null }; } export async function debugDispatch( config: UniDeskConfig, providerId: string, command: DebugDispatchCommand, payload?: Record, waitMs = 0, ): Promise { const dispatchPayload = payload ?? (command === "provider.upgrade" ? { source: "cli-debug", mode: "plan" } : { source: "cli-debug" }); const dispatch = coreInternalFetch("/api/dispatch", { method: "POST", body: { providerId, command, payload: dispatchPayload }, }); const taskId = (dispatch as { body?: { taskId?: string } }).body?.taskId ?? ""; const wait = waitMs > 0 && taskId.length > 0 ? await waitForTask(taskId, waitMs) : null; return { dispatch, wait }; }