fix: add provider rootfs diagnostic
This commit is contained in:
Codex
+10
-1
@@ -2,7 +2,7 @@
|
||||
// SPEC: PJ2026-01060703 CI/CD branch follower draft-2026-07-03-p0-branch-follower.
|
||||
// UniDesk CLI dispatcher with bounded server lifecycle and job drill-down output.
|
||||
import { readConfig } from "./src/config";
|
||||
import { debugDispatch, debugEgressProxy, debugHealth, debugSshPool, debugTask, isDebugDispatchCommand, type DebugDispatchCommand } from "./src/debug";
|
||||
import { debugDispatch, debugEgressProxy, debugHealth, debugProviderRootfs, debugSshPool, debugTask, isDebugDispatchCommand, type DebugDispatchCommand } from "./src/debug";
|
||||
import { isRebuildableService, isRestartableService, rebuildService, restartService, stackLogs, stackStatus, startStack, stopStack, unsupportedRebuildService, unsupportedRestartService } from "./src/docker";
|
||||
import { emitError, emitJson, emitText, isRenderedCliResult } from "./src/output";
|
||||
import { cancelJob, jobWithTail, listJobs, listJobsSummary, readJob, renderJobLaunchSummary, renderJobStatusSummary, runJob } from "./src/jobs";
|
||||
@@ -675,6 +675,15 @@ async function main(): Promise<void> {
|
||||
if (!ok) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (sub === "provider-rootfs") {
|
||||
const providerId = third ?? "";
|
||||
if (providerId.length === 0) throw new Error("debug provider-rootfs requires providerId");
|
||||
const result = await debugProviderRootfs(config, providerId);
|
||||
const ok = (result as { ok?: unknown }).ok !== false;
|
||||
emitJson(commandName, result, ok);
|
||||
if (!ok) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
if (sub === "dispatch") {
|
||||
const providerId = isDebugDispatchCommand(third) ? config.providerGateway.id : third ?? config.providerGateway.id;
|
||||
const commandArg = isDebugDispatchCommand(third) ? third : fourth;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { runCommand } from "./command";
|
||||
import { type UniDeskConfig, repoRoot } from "./config";
|
||||
import { runSshCommandCapture } from "./ssh";
|
||||
|
||||
export const dispatchCommands = ["docker.ps", "provider.upgrade", "host.ssh", "microservice.http", "echo"] as const;
|
||||
export type DebugDispatchCommand = typeof dispatchCommands[number];
|
||||
@@ -374,6 +375,195 @@ export async function debugEgressProxy(_config: UniDeskConfig, providerId: strin
|
||||
};
|
||||
}
|
||||
|
||||
const providerRootfsMarker = "__UNIDESK_PROVIDER_ROOTFS__";
|
||||
|
||||
function safeDockerName(value: string): string {
|
||||
return value.replace(/[^a-zA-Z0-9_.-]/gu, "-").slice(0, 80);
|
||||
}
|
||||
|
||||
function providerGatewayContainerName(providerId: string): string {
|
||||
return `unidesk-provider-gateway-${safeDockerName(providerId)}`.toLowerCase();
|
||||
}
|
||||
|
||||
function providerRootfsDiagnosticScript(providerId: string): string {
|
||||
const container = providerGatewayContainerName(providerId);
|
||||
return [
|
||||
"set +e",
|
||||
`MARK=${shellQuote(providerRootfsMarker)}`,
|
||||
`container=${shellQuote(container)}`,
|
||||
`provider_id=${shellQuote(providerId)}`,
|
||||
'emit() { printf "%s%s=%s\\n" "$MARK" "$1" "$2"; }',
|
||||
'emit_b64() { key="$1"; value="$2"; encoded=$(printf "%s" "$value" | base64 | tr -d "\\n"); emit "$key" "$encoded"; }',
|
||||
'emit provider_id "$provider_id"',
|
||||
'emit container_name "$container"',
|
||||
"inspect_output=$(docker inspect \"$container\" --format 'id={{.Id}}\nname={{.Name}}\nimageId={{.Image}}\nimageName={{.Config.Image}}\nstatus={{.State.Status}}\nrunning={{.State.Running}}\nrestartPolicy={{.HostConfig.RestartPolicy.Name}}\npidMode={{.HostConfig.PidMode}}' 2>&1)",
|
||||
"inspect_code=$?",
|
||||
'emit inspect_exit_code "$inspect_code"',
|
||||
'emit_b64 inspect_output "$inspect_output"',
|
||||
"smoke_output=$(docker exec \"$container\" /bin/sh -lc 'set -eu\n test -x /bin/sh\n test -x /usr/bin/ssh\n /usr/bin/ssh -V >/tmp/unidesk-provider-ssh-version 2>&1 || { cat /tmp/unidesk-provider-ssh-version >&2; exit 41; }\n command -v docker >/dev/null\n env -u DOCKER_HOST docker version --format \"{{.Client.Version}}\" >/dev/null\n test -e /lib/libz.so.1 -o -e /usr/lib/libz.so.1\n command -v bun >/dev/null\n printf \"rootfs-smoke-ok\\n\"' 2>&1)",
|
||||
"smoke_code=$?",
|
||||
'emit rootfs_smoke_exit_code "$smoke_code"',
|
||||
'emit_b64 rootfs_smoke_output "$smoke_output"',
|
||||
'system_df_output=$(docker system df 2>&1)',
|
||||
"system_df_code=$?",
|
||||
'emit docker_system_df_exit_code "$system_df_code"',
|
||||
'emit_b64 docker_system_df_output "$system_df_output"',
|
||||
"snapshot_summary=$(printf '%s\\n' \"$system_df_output\" | grep -Ei 'snapshot|lowerdir|no such file|not found|missing|lstat|failed' | tail -20)",
|
||||
'emit_b64 snapshot_missing_summary "$snapshot_summary"',
|
||||
'logs_output=$(docker logs --since 24h --tail 3000 "$container" 2>&1)',
|
||||
"logs_code=$?",
|
||||
'emit docker_logs_exit_code "$logs_code"',
|
||||
"last_started=$(printf '%s\\n' \"$logs_output\" | grep 'host_ssh_session_started' | tail -1)",
|
||||
"last_success=$(printf '%s\\n' \"$logs_output\" | grep 'host_ssh_session_completed' | grep '\"exitCode\":0' | tail -1)",
|
||||
"last_failure=$(printf '%s\\n' \"$logs_output\" | grep -E 'host_ssh_session_completed|host_ssh_session_failed|host_ssh_session_error' | grep -Ev '\"exitCode\":0' | tail -1)",
|
||||
'emit_b64 last_host_ssh_started "$last_started"',
|
||||
'emit_b64 last_host_ssh_success "$last_success"',
|
||||
'emit_b64 last_host_ssh_failure "$last_failure"',
|
||||
"exit 0",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function parseProviderRootfsFields(stdout: string): Record<string, string> {
|
||||
const fields: Record<string, string> = {};
|
||||
for (const line of stdout.split(/\r?\n/u)) {
|
||||
if (!line.startsWith(providerRootfsMarker)) continue;
|
||||
const payload = line.slice(providerRootfsMarker.length);
|
||||
const index = payload.indexOf("=");
|
||||
if (index <= 0) continue;
|
||||
fields[payload.slice(0, index)] = payload.slice(index + 1);
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
function b64Field(fields: Record<string, string>, key: string): string {
|
||||
const raw = fields[key] ?? "";
|
||||
if (raw.length === 0) return "";
|
||||
try {
|
||||
return Buffer.from(raw, "base64").toString("utf8");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function numberField(fields: Record<string, string>, key: string): number | null {
|
||||
const value = Number(fields[key]);
|
||||
return Number.isFinite(value) ? value : null;
|
||||
}
|
||||
|
||||
function parseKeyValueBlock(text: string): Record<string, string> {
|
||||
const values: Record<string, string> = {};
|
||||
for (const line of text.split(/\r?\n/u)) {
|
||||
const index = line.indexOf("=");
|
||||
if (index <= 0) continue;
|
||||
values[line.slice(0, index)] = line.slice(index + 1);
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
function trimForDiagnostic(text: string, maxChars = 1200): string {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed.length <= maxChars) return trimmed;
|
||||
return trimmed.slice(trimmed.length - maxChars);
|
||||
}
|
||||
|
||||
function extractIsoTimeFromLogLine(line: string): string | null {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length === 0) return null;
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
|
||||
for (const key of ["time", "timestamp", "at"]) {
|
||||
const value = parsed[key];
|
||||
if (typeof value === "string" && value.length > 0) return value;
|
||||
}
|
||||
} catch {
|
||||
// Fall through to regex extraction for non-JSON log formats.
|
||||
}
|
||||
return trimmed.match(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z/u)?.[0] ?? null;
|
||||
}
|
||||
|
||||
export async function debugProviderRootfs(config: UniDeskConfig, providerId: string): Promise<unknown> {
|
||||
const script = providerRootfsDiagnosticScript(providerId);
|
||||
const capture = await runSshCommandCapture(config, providerId, ["bash"], script);
|
||||
const fields = parseProviderRootfsFields(capture.stdout);
|
||||
const inspectOutput = b64Field(fields, "inspect_output");
|
||||
const inspect = parseKeyValueBlock(inspectOutput);
|
||||
const smokeOutput = b64Field(fields, "rootfs_smoke_output");
|
||||
const systemDfOutput = b64Field(fields, "docker_system_df_output");
|
||||
const snapshotSummary = b64Field(fields, "snapshot_missing_summary");
|
||||
const lastStarted = b64Field(fields, "last_host_ssh_started");
|
||||
const lastSuccess = b64Field(fields, "last_host_ssh_success");
|
||||
const lastFailure = b64Field(fields, "last_host_ssh_failure");
|
||||
const inspectExitCode = numberField(fields, "inspect_exit_code");
|
||||
const smokeExitCode = numberField(fields, "rootfs_smoke_exit_code");
|
||||
const systemDfExitCode = numberField(fields, "docker_system_df_exit_code");
|
||||
const dockerLogsExitCode = numberField(fields, "docker_logs_exit_code");
|
||||
const transOk = capture.exitCode === 0;
|
||||
const ok = transOk && inspectExitCode === 0 && smokeExitCode === 0 && systemDfExitCode === 0;
|
||||
return {
|
||||
ok,
|
||||
providerId,
|
||||
containerName: fields.container_name ?? providerGatewayContainerName(providerId),
|
||||
trans: {
|
||||
ok: transOk,
|
||||
exitCode: capture.exitCode,
|
||||
stderrTail: trimForDiagnostic(capture.stderr, 1600),
|
||||
},
|
||||
container: {
|
||||
inspectExitCode,
|
||||
id: inspect.id ?? null,
|
||||
name: inspect.name?.replace(/^\//u, "") ?? null,
|
||||
imageId: inspect.imageId ?? null,
|
||||
imageName: inspect.imageName ?? null,
|
||||
status: inspect.status ?? null,
|
||||
running: inspect.running ?? null,
|
||||
restartPolicy: inspect.restartPolicy ?? null,
|
||||
pidMode: inspect.pidMode ?? null,
|
||||
inspectError: inspectExitCode === 0 ? null : trimForDiagnostic(inspectOutput),
|
||||
},
|
||||
rootfsSmoke: {
|
||||
ok: smokeExitCode === 0,
|
||||
exitCode: smokeExitCode,
|
||||
outputTail: trimForDiagnostic(smokeOutput),
|
||||
checks: ["/bin/sh", "/usr/bin/ssh -V", "libz.so.1", "docker client", "bun"],
|
||||
},
|
||||
dockerSystemDf: {
|
||||
ok: systemDfExitCode === 0,
|
||||
exitCode: systemDfExitCode,
|
||||
outputTail: trimForDiagnostic(systemDfOutput),
|
||||
snapshotMissingSummary: trimForDiagnostic(snapshotSummary),
|
||||
},
|
||||
hostSsh: {
|
||||
dockerLogsExitCode,
|
||||
lastStarted: {
|
||||
at: extractIsoTimeFromLogLine(lastStarted),
|
||||
line: trimForDiagnostic(lastStarted),
|
||||
},
|
||||
lastSuccess: {
|
||||
at: extractIsoTimeFromLogLine(lastSuccess),
|
||||
line: trimForDiagnostic(lastSuccess),
|
||||
},
|
||||
lastFailure: {
|
||||
at: extractIsoTimeFromLogLine(lastFailure),
|
||||
line: trimForDiagnostic(lastFailure),
|
||||
},
|
||||
note: "Success/failure rows require provider-gateway versions that log host_ssh_session_completed or host_ssh_session_failed; older containers may only expose lastStarted/error.",
|
||||
},
|
||||
classification: !transOk
|
||||
? "trans-channel-unavailable"
|
||||
: inspectExitCode !== 0
|
||||
? "provider-container-missing"
|
||||
: smokeExitCode !== 0
|
||||
? "provider-rootfs-smoke-failed"
|
||||
: systemDfExitCode !== 0
|
||||
? "docker-system-df-failed"
|
||||
: "provider-rootfs-healthy",
|
||||
next: {
|
||||
sshPool: `bun scripts/cli.ts debug ssh-pool ${providerId}`,
|
||||
providerUpgrade: `bun scripts/cli.ts debug dispatch ${providerId} provider.upgrade --mode schedule --wait-ms 300000`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForTask(taskId: string, timeoutMs: number): Promise<unknown> {
|
||||
const started = Date.now();
|
||||
let latest: unknown = null;
|
||||
|
||||
+3
-1
@@ -99,6 +99,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "debug health", description: "Probe internal core, nodes, system/Docker status, frontend, provider ingress, and public boundary." },
|
||||
{ command: "debug ssh-pool <providerId>", description: "Show bounded host.ssh.tcp-pool labels for one provider, including ready/claimed/desired/lastError." },
|
||||
{ command: "debug egress-proxy <providerId>", description: "Show provider-gateway egress proxy tunnel counts, stale tunnel diagnosis, active target summaries, and recent closed tunnel lifecycle without URL credential leakage." },
|
||||
{ command: "debug provider-rootfs <providerId>", description: "Run a read-only provider-gateway rootfs diagnostic over trans: image id, runtime smoke, docker system df snapshot errors, and recent host_ssh lifecycle lines." },
|
||||
{ command: "debug dispatch [providerId] [docker.ps|provider.upgrade|host.ssh|microservice.http|echo] [--wait-ms N]", description: "Submit a real internal-core dispatch request for CLI debugging." },
|
||||
{ command: "debug task <taskId|latest>", description: "Read a dispatched task record from internal core for CLI debugging." },
|
||||
{ command: "network perf [--service code-queue --path /api/tasks/overview?limit=30 --count N --concurrency N --label before|after]", description: "Benchmark frontend -> backend-core -> provider/adapter user-service networking and report latency/proxy-mode distributions." },
|
||||
@@ -544,12 +545,13 @@ function jobHelp(): unknown {
|
||||
|
||||
function debugHelp(): unknown {
|
||||
return {
|
||||
command: "debug health|ssh-pool|egress-proxy|dispatch|task",
|
||||
command: "debug health|ssh-pool|egress-proxy|provider-rootfs|dispatch|task",
|
||||
output: "json",
|
||||
usage: [
|
||||
"bun scripts/cli.ts debug health",
|
||||
"bun scripts/cli.ts debug ssh-pool <providerId>",
|
||||
"bun scripts/cli.ts debug egress-proxy <providerId>",
|
||||
"bun scripts/cli.ts debug provider-rootfs <providerId>",
|
||||
"bun scripts/cli.ts debug dispatch [providerId] [docker.ps|provider.upgrade|host.ssh|microservice.http|echo] [--wait-ms N]",
|
||||
"bun scripts/cli.ts debug task <taskId|latest>",
|
||||
],
|
||||
|
||||
@@ -102,6 +102,9 @@ interface HostSshSession {
|
||||
openedAt: number;
|
||||
tty: boolean;
|
||||
dataChannelId?: string;
|
||||
commandBytes: number;
|
||||
commandDigest: string | null;
|
||||
cwd: string | null;
|
||||
}
|
||||
|
||||
interface PendingSshDataFrame {
|
||||
@@ -1983,7 +1986,15 @@ function startHostSshSession(message: CoreHostSshOpenMessage): void {
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
hostSshSessions.set(message.sessionId, { proc, openedAt: Date.now(), tty: allocateTty, dataChannelId: message.dataChannelId });
|
||||
hostSshSessions.set(message.sessionId, {
|
||||
proc,
|
||||
openedAt: Date.now(),
|
||||
tty: allocateTty,
|
||||
dataChannelId: message.dataChannelId,
|
||||
commandBytes: command === null ? 0 : Buffer.byteLength(command),
|
||||
commandDigest,
|
||||
cwd: message.cwd ?? null,
|
||||
});
|
||||
writeSshDataFrame(dataChannel, {
|
||||
type: "opened",
|
||||
sessionId: message.sessionId,
|
||||
@@ -2002,10 +2013,30 @@ function startHostSshSession(message: CoreHostSshOpenMessage): void {
|
||||
at: new Date().toISOString(),
|
||||
});
|
||||
rememberCompletedHostSshSession(message.sessionId, message.dataChannelId, "exit", typeof exitCode === "number" ? exitCode : null);
|
||||
logger("info", "host_ssh_session_completed", {
|
||||
sessionId: message.sessionId,
|
||||
exitCode,
|
||||
commandBytes: command === null ? 0 : Buffer.byteLength(command),
|
||||
commandDigest,
|
||||
tty: allocateTty,
|
||||
cwd: message.cwd ?? null,
|
||||
transport: "tcp-pool",
|
||||
dataChannelId: message.dataChannelId,
|
||||
});
|
||||
hostSshSessions.delete(message.sessionId);
|
||||
releaseSshDataChannel(message.dataChannelId, message.sessionId);
|
||||
})
|
||||
.catch((error) => {
|
||||
logger("error", "host_ssh_session_failed", {
|
||||
sessionId: message.sessionId,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
commandBytes: command === null ? 0 : Buffer.byteLength(command),
|
||||
commandDigest,
|
||||
tty: allocateTty,
|
||||
cwd: message.cwd ?? null,
|
||||
transport: "tcp-pool",
|
||||
dataChannelId: message.dataChannelId,
|
||||
});
|
||||
sendHostSshError(message.sessionId, error);
|
||||
rememberCompletedHostSshSession(message.sessionId, message.dataChannelId, "error");
|
||||
hostSshSessions.delete(message.sessionId);
|
||||
|
||||
Reference in New Issue
Block a user