From 42f605dceb72325075c58976f257b733aaf851d2 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 5 Jul 2026 18:08:17 +0000 Subject: [PATCH] fix: add provider rootfs diagnostic --- scripts/cli.ts | 11 +- scripts/src/debug.ts | 190 +++++++++++++++++++ scripts/src/help.ts | 4 +- src/components/provider-gateway/src/index.ts | 33 +++- 4 files changed, 235 insertions(+), 3 deletions(-) diff --git a/scripts/cli.ts b/scripts/cli.ts index 5fb4476c..f1b1474a 100644 --- a/scripts/cli.ts +++ b/scripts/cli.ts @@ -2,7 +2,7 @@ // SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower. // UniDesk CLI dispatcher with bounded server lifecycle and job drill-down output. import { readConfig } from "./src/config"; -import { debugDispatch, debugEgressProxy, debugHealth, debugSshPool, debugTask, isDebugDispatchCommand, type DebugDispatchCommand } from "./src/debug"; +import { debugDispatch, debugEgressProxy, debugHealth, debugProviderRootfs, debugSshPool, debugTask, isDebugDispatchCommand, type DebugDispatchCommand } from "./src/debug"; import { isRebuildableService, isRestartableService, rebuildService, restartService, stackLogs, stackStatus, startStack, stopStack, unsupportedRebuildService, unsupportedRestartService } from "./src/docker"; import { emitError, emitJson, emitText, isRenderedCliResult } from "./src/output"; import { cancelJob, jobWithTail, listJobs, listJobsSummary, readJob, renderJobLaunchSummary, renderJobStatusSummary, runJob } from "./src/jobs"; @@ -675,6 +675,15 @@ async function main(): Promise { if (!ok) process.exitCode = 1; return; } + if (sub === "provider-rootfs") { + const providerId = third ?? ""; + if (providerId.length === 0) throw new Error("debug provider-rootfs requires providerId"); + const result = await debugProviderRootfs(config, providerId); + const ok = (result as { ok?: unknown }).ok !== false; + emitJson(commandName, result, ok); + if (!ok) process.exitCode = 1; + return; + } if (sub === "dispatch") { const providerId = isDebugDispatchCommand(third) ? config.providerGateway.id : third ?? config.providerGateway.id; const commandArg = isDebugDispatchCommand(third) ? third : fourth; diff --git a/scripts/src/debug.ts b/scripts/src/debug.ts index e193d172..29deee72 100644 --- a/scripts/src/debug.ts +++ b/scripts/src/debug.ts @@ -1,5 +1,6 @@ import { runCommand } from "./command"; import { type UniDeskConfig, repoRoot } from "./config"; +import { runSshCommandCapture } from "./ssh"; export const dispatchCommands = ["docker.ps", "provider.upgrade", "host.ssh", "microservice.http", "echo"] as const; export type DebugDispatchCommand = typeof dispatchCommands[number]; @@ -374,6 +375,195 @@ export async function debugEgressProxy(_config: UniDeskConfig, providerId: strin }; } +const providerRootfsMarker = "__UNIDESK_PROVIDER_ROOTFS__"; + +function safeDockerName(value: string): string { + return value.replace(/[^a-zA-Z0-9_.-]/gu, "-").slice(0, 80); +} + +function providerGatewayContainerName(providerId: string): string { + return `unidesk-provider-gateway-${safeDockerName(providerId)}`.toLowerCase(); +} + +function providerRootfsDiagnosticScript(providerId: string): string { + const container = providerGatewayContainerName(providerId); + return [ + "set +e", + `MARK=${shellQuote(providerRootfsMarker)}`, + `container=${shellQuote(container)}`, + `provider_id=${shellQuote(providerId)}`, + 'emit() { printf "%s%s=%s\\n" "$MARK" "$1" "$2"; }', + 'emit_b64() { key="$1"; value="$2"; encoded=$(printf "%s" "$value" | base64 | tr -d "\\n"); emit "$key" "$encoded"; }', + 'emit provider_id "$provider_id"', + 'emit container_name "$container"', + "inspect_output=$(docker inspect \"$container\" --format 'id={{.Id}}\nname={{.Name}}\nimageId={{.Image}}\nimageName={{.Config.Image}}\nstatus={{.State.Status}}\nrunning={{.State.Running}}\nrestartPolicy={{.HostConfig.RestartPolicy.Name}}\npidMode={{.HostConfig.PidMode}}' 2>&1)", + "inspect_code=$?", + 'emit inspect_exit_code "$inspect_code"', + 'emit_b64 inspect_output "$inspect_output"', + "smoke_output=$(docker exec \"$container\" /bin/sh -lc 'set -eu\n test -x /bin/sh\n test -x /usr/bin/ssh\n /usr/bin/ssh -V >/tmp/unidesk-provider-ssh-version 2>&1 || { cat /tmp/unidesk-provider-ssh-version >&2; exit 41; }\n command -v docker >/dev/null\n env -u DOCKER_HOST docker version --format \"{{.Client.Version}}\" >/dev/null\n test -e /lib/libz.so.1 -o -e /usr/lib/libz.so.1\n command -v bun >/dev/null\n printf \"rootfs-smoke-ok\\n\"' 2>&1)", + "smoke_code=$?", + 'emit rootfs_smoke_exit_code "$smoke_code"', + 'emit_b64 rootfs_smoke_output "$smoke_output"', + 'system_df_output=$(docker system df 2>&1)', + "system_df_code=$?", + 'emit docker_system_df_exit_code "$system_df_code"', + 'emit_b64 docker_system_df_output "$system_df_output"', + "snapshot_summary=$(printf '%s\\n' \"$system_df_output\" | grep -Ei 'snapshot|lowerdir|no such file|not found|missing|lstat|failed' | tail -20)", + 'emit_b64 snapshot_missing_summary "$snapshot_summary"', + 'logs_output=$(docker logs --since 24h --tail 3000 "$container" 2>&1)', + "logs_code=$?", + 'emit docker_logs_exit_code "$logs_code"', + "last_started=$(printf '%s\\n' \"$logs_output\" | grep 'host_ssh_session_started' | tail -1)", + "last_success=$(printf '%s\\n' \"$logs_output\" | grep 'host_ssh_session_completed' | grep '\"exitCode\":0' | tail -1)", + "last_failure=$(printf '%s\\n' \"$logs_output\" | grep -E 'host_ssh_session_completed|host_ssh_session_failed|host_ssh_session_error' | grep -Ev '\"exitCode\":0' | tail -1)", + 'emit_b64 last_host_ssh_started "$last_started"', + 'emit_b64 last_host_ssh_success "$last_success"', + 'emit_b64 last_host_ssh_failure "$last_failure"', + "exit 0", + ].join("\n"); +} + +function parseProviderRootfsFields(stdout: string): Record { + const fields: Record = {}; + for (const line of stdout.split(/\r?\n/u)) { + if (!line.startsWith(providerRootfsMarker)) continue; + const payload = line.slice(providerRootfsMarker.length); + const index = payload.indexOf("="); + if (index <= 0) continue; + fields[payload.slice(0, index)] = payload.slice(index + 1); + } + return fields; +} + +function b64Field(fields: Record, key: string): string { + const raw = fields[key] ?? ""; + if (raw.length === 0) return ""; + try { + return Buffer.from(raw, "base64").toString("utf8"); + } catch { + return ""; + } +} + +function numberField(fields: Record, key: string): number | null { + const value = Number(fields[key]); + return Number.isFinite(value) ? value : null; +} + +function parseKeyValueBlock(text: string): Record { + const values: Record = {}; + for (const line of text.split(/\r?\n/u)) { + const index = line.indexOf("="); + if (index <= 0) continue; + values[line.slice(0, index)] = line.slice(index + 1); + } + return values; +} + +function trimForDiagnostic(text: string, maxChars = 1200): string { + const trimmed = text.trim(); + if (trimmed.length <= maxChars) return trimmed; + return trimmed.slice(trimmed.length - maxChars); +} + +function extractIsoTimeFromLogLine(line: string): string | null { + const trimmed = line.trim(); + if (trimmed.length === 0) return null; + try { + const parsed = JSON.parse(trimmed) as Record; + for (const key of ["time", "timestamp", "at"]) { + const value = parsed[key]; + if (typeof value === "string" && value.length > 0) return value; + } + } catch { + // Fall through to regex extraction for non-JSON log formats. + } + return trimmed.match(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z/u)?.[0] ?? null; +} + +export async function debugProviderRootfs(config: UniDeskConfig, providerId: string): Promise { + const script = providerRootfsDiagnosticScript(providerId); + const capture = await runSshCommandCapture(config, providerId, ["bash"], script); + const fields = parseProviderRootfsFields(capture.stdout); + const inspectOutput = b64Field(fields, "inspect_output"); + const inspect = parseKeyValueBlock(inspectOutput); + const smokeOutput = b64Field(fields, "rootfs_smoke_output"); + const systemDfOutput = b64Field(fields, "docker_system_df_output"); + const snapshotSummary = b64Field(fields, "snapshot_missing_summary"); + const lastStarted = b64Field(fields, "last_host_ssh_started"); + const lastSuccess = b64Field(fields, "last_host_ssh_success"); + const lastFailure = b64Field(fields, "last_host_ssh_failure"); + const inspectExitCode = numberField(fields, "inspect_exit_code"); + const smokeExitCode = numberField(fields, "rootfs_smoke_exit_code"); + const systemDfExitCode = numberField(fields, "docker_system_df_exit_code"); + const dockerLogsExitCode = numberField(fields, "docker_logs_exit_code"); + const transOk = capture.exitCode === 0; + const ok = transOk && inspectExitCode === 0 && smokeExitCode === 0 && systemDfExitCode === 0; + return { + ok, + providerId, + containerName: fields.container_name ?? providerGatewayContainerName(providerId), + trans: { + ok: transOk, + exitCode: capture.exitCode, + stderrTail: trimForDiagnostic(capture.stderr, 1600), + }, + container: { + inspectExitCode, + id: inspect.id ?? null, + name: inspect.name?.replace(/^\//u, "") ?? null, + imageId: inspect.imageId ?? null, + imageName: inspect.imageName ?? null, + status: inspect.status ?? null, + running: inspect.running ?? null, + restartPolicy: inspect.restartPolicy ?? null, + pidMode: inspect.pidMode ?? null, + inspectError: inspectExitCode === 0 ? null : trimForDiagnostic(inspectOutput), + }, + rootfsSmoke: { + ok: smokeExitCode === 0, + exitCode: smokeExitCode, + outputTail: trimForDiagnostic(smokeOutput), + checks: ["/bin/sh", "/usr/bin/ssh -V", "libz.so.1", "docker client", "bun"], + }, + dockerSystemDf: { + ok: systemDfExitCode === 0, + exitCode: systemDfExitCode, + outputTail: trimForDiagnostic(systemDfOutput), + snapshotMissingSummary: trimForDiagnostic(snapshotSummary), + }, + hostSsh: { + dockerLogsExitCode, + lastStarted: { + at: extractIsoTimeFromLogLine(lastStarted), + line: trimForDiagnostic(lastStarted), + }, + lastSuccess: { + at: extractIsoTimeFromLogLine(lastSuccess), + line: trimForDiagnostic(lastSuccess), + }, + lastFailure: { + at: extractIsoTimeFromLogLine(lastFailure), + line: trimForDiagnostic(lastFailure), + }, + note: "Success/failure rows require provider-gateway versions that log host_ssh_session_completed or host_ssh_session_failed; older containers may only expose lastStarted/error.", + }, + classification: !transOk + ? "trans-channel-unavailable" + : inspectExitCode !== 0 + ? "provider-container-missing" + : smokeExitCode !== 0 + ? "provider-rootfs-smoke-failed" + : systemDfExitCode !== 0 + ? "docker-system-df-failed" + : "provider-rootfs-healthy", + next: { + sshPool: `bun scripts/cli.ts debug ssh-pool ${providerId}`, + providerUpgrade: `bun scripts/cli.ts debug dispatch ${providerId} provider.upgrade --mode schedule --wait-ms 300000`, + }, + }; +} + async function waitForTask(taskId: string, timeoutMs: number): Promise { const started = Date.now(); let latest: unknown = null; diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 92ca454b..f4f98f05 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -99,6 +99,7 @@ export function rootHelp(): unknown { { command: "debug health", description: "Probe internal core, nodes, system/Docker status, frontend, provider ingress, and public boundary." }, { command: "debug ssh-pool ", description: "Show bounded host.ssh.tcp-pool labels for one provider, including ready/claimed/desired/lastError." }, { command: "debug egress-proxy ", description: "Show provider-gateway egress proxy tunnel counts, stale tunnel diagnosis, active target summaries, and recent closed tunnel lifecycle without URL credential leakage." }, + { command: "debug provider-rootfs ", description: "Run a read-only provider-gateway rootfs diagnostic over trans: image id, runtime smoke, docker system df snapshot errors, and recent host_ssh lifecycle lines." }, { command: "debug dispatch [providerId] [docker.ps|provider.upgrade|host.ssh|microservice.http|echo] [--wait-ms N]", description: "Submit a real internal-core dispatch request for CLI debugging." }, { command: "debug task ", description: "Read a dispatched task record from internal core for CLI debugging." }, { command: "network perf [--service code-queue --path /api/tasks/overview?limit=30 --count N --concurrency N --label before|after]", description: "Benchmark frontend -> backend-core -> provider/adapter user-service networking and report latency/proxy-mode distributions." }, @@ -544,12 +545,13 @@ function jobHelp(): unknown { function debugHelp(): unknown { return { - command: "debug health|ssh-pool|egress-proxy|dispatch|task", + command: "debug health|ssh-pool|egress-proxy|provider-rootfs|dispatch|task", output: "json", usage: [ "bun scripts/cli.ts debug health", "bun scripts/cli.ts debug ssh-pool ", "bun scripts/cli.ts debug egress-proxy ", + "bun scripts/cli.ts debug provider-rootfs ", "bun scripts/cli.ts debug dispatch [providerId] [docker.ps|provider.upgrade|host.ssh|microservice.http|echo] [--wait-ms N]", "bun scripts/cli.ts debug task ", ], diff --git a/src/components/provider-gateway/src/index.ts b/src/components/provider-gateway/src/index.ts index 03e58105..3023ce57 100644 --- a/src/components/provider-gateway/src/index.ts +++ b/src/components/provider-gateway/src/index.ts @@ -102,6 +102,9 @@ interface HostSshSession { openedAt: number; tty: boolean; dataChannelId?: string; + commandBytes: number; + commandDigest: string | null; + cwd: string | null; } interface PendingSshDataFrame { @@ -1983,7 +1986,15 @@ function startHostSshSession(message: CoreHostSshOpenMessage): void { stdout: "pipe", stderr: "pipe", }); - hostSshSessions.set(message.sessionId, { proc, openedAt: Date.now(), tty: allocateTty, dataChannelId: message.dataChannelId }); + hostSshSessions.set(message.sessionId, { + proc, + openedAt: Date.now(), + tty: allocateTty, + dataChannelId: message.dataChannelId, + commandBytes: command === null ? 0 : Buffer.byteLength(command), + commandDigest, + cwd: message.cwd ?? null, + }); writeSshDataFrame(dataChannel, { type: "opened", sessionId: message.sessionId, @@ -2002,10 +2013,30 @@ function startHostSshSession(message: CoreHostSshOpenMessage): void { at: new Date().toISOString(), }); rememberCompletedHostSshSession(message.sessionId, message.dataChannelId, "exit", typeof exitCode === "number" ? exitCode : null); + logger("info", "host_ssh_session_completed", { + sessionId: message.sessionId, + exitCode, + commandBytes: command === null ? 0 : Buffer.byteLength(command), + commandDigest, + tty: allocateTty, + cwd: message.cwd ?? null, + transport: "tcp-pool", + dataChannelId: message.dataChannelId, + }); hostSshSessions.delete(message.sessionId); releaseSshDataChannel(message.dataChannelId, message.sessionId); }) .catch((error) => { + logger("error", "host_ssh_session_failed", { + sessionId: message.sessionId, + error: error instanceof Error ? error.message : String(error), + commandBytes: command === null ? 0 : Buffer.byteLength(command), + commandDigest, + tty: allocateTty, + cwd: message.cwd ?? null, + transport: "tcp-pool", + dataChannelId: message.dataChannelId, + }); sendHostSshError(message.sessionId, error); rememberCompletedHostSshSession(message.sessionId, message.dataChannelId, "error"); hostSshSessions.delete(message.sessionId);