Codex
603 lines
26 KiB
TypeScript
603 lines
26 KiB
TypeScript
import { runCommand } from "./command";
|
|
import { type UniDeskConfig, repoRoot } from "./config";
|
|
import { runSshCommandCapture } from "./ssh";
|
|
|
|
export const dispatchCommands = ["docker.ps", "provider.upgrade", "host.ssh", "microservice.http", "echo"] as const;
|
|
export type DebugDispatchCommand = typeof dispatchCommands[number];
|
|
|
|
export function isDebugDispatchCommand(value: unknown): value is DebugDispatchCommand {
|
|
return dispatchCommands.includes(value as DebugDispatchCommand);
|
|
}
|
|
|
|
async function readJson(url: string, init?: RequestInit): Promise<unknown> {
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), 5000);
|
|
try {
|
|
const res = await fetch(url, { ...init, signal: controller.signal });
|
|
const text = await res.text();
|
|
return { ok: res.ok, status: res.status, body: text.length > 0 ? JSON.parse(text) : null };
|
|
} catch (error) {
|
|
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
function coreFetchCommand(path: string, init?: { method?: string; body?: unknown }): string[] {
|
|
const method = init?.method ?? "GET";
|
|
const url = `http://127.0.0.1:8080${path}`;
|
|
const body = init?.body === undefined ? "" : JSON.stringify(init.body);
|
|
const script = [
|
|
"set -eu",
|
|
"if command -v backend-core >/dev/null 2>&1; then",
|
|
` exec backend-core --fetch-json ${shellQuote(url)} --method ${shellQuote(method)}${body.length > 0 ? ` --body-json ${shellQuote(body)}` : ""}`,
|
|
"fi",
|
|
`url=${shellQuote(url)}`,
|
|
`method=${shellQuote(method)}`,
|
|
`body=${shellQuote(body)}`,
|
|
"export url method body",
|
|
"bun -e 'const url=process.env.url; const method=process.env.method; const body=process.env.body; fetch(url,{method,body:body?body:undefined,headers:body?{\"content-type\":\"application/json\"}:undefined}).then(async r=>{const text=await r.text(); console.log(JSON.stringify({ok:r.ok,status:r.status,body:text?JSON.parse(text):null})); process.exit(r.ok?0:1);}).catch(e=>{console.error(e); process.exit(1);})'",
|
|
].join("\n");
|
|
return ["docker", "exec", "unidesk-backend-core", "sh", "-lc", script];
|
|
}
|
|
|
|
function shellQuote(value: string): string {
|
|
return `'${value.replace(/'/g, `'\\''`)}'`;
|
|
}
|
|
|
|
function coreInternalFetch(path: string, init?: { method?: string; body?: unknown }): unknown {
|
|
const command = coreFetchCommand(path, init);
|
|
const result = runCommand(command, repoRoot);
|
|
if (result.exitCode !== 0) {
|
|
return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) };
|
|
}
|
|
try {
|
|
return JSON.parse(result.stdout.trim()) as unknown;
|
|
} catch {
|
|
return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) };
|
|
}
|
|
}
|
|
|
|
function coreDockerStatusSummary(): unknown {
|
|
const result = runCommand(coreFetchCommand("/api/nodes/docker-status"), repoRoot);
|
|
if (result.exitCode !== 0) {
|
|
return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) };
|
|
}
|
|
try {
|
|
const parsed = JSON.parse(result.stdout.trim()) as { ok?: boolean; status?: number; body?: { dockerStatuses?: Array<Record<string, unknown>> } };
|
|
const dockerStatuses = (parsed.body?.dockerStatuses || []).map((item) => {
|
|
const status = (item.dockerStatus || {}) as Record<string, unknown>;
|
|
return {
|
|
providerId: item.providerId,
|
|
name: item.name,
|
|
nodeStatus: item.nodeStatus,
|
|
updatedAt: item.updatedAt,
|
|
dockerStatus: {
|
|
ok: status.ok,
|
|
socketPresent: status.socketPresent,
|
|
collectedAt: status.collectedAt,
|
|
counts: status.counts,
|
|
daemon: status.daemon,
|
|
containersPreview: Array.isArray(status.containers) ? status.containers.slice(0, 8) : [],
|
|
},
|
|
};
|
|
});
|
|
return { ok: parsed.ok, status: parsed.status, body: { ok: parsed.body !== undefined, dockerStatuses } };
|
|
} catch {
|
|
return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) };
|
|
}
|
|
}
|
|
|
|
function coreSystemStatusSummary(): unknown {
|
|
const result = runCommand(coreFetchCommand("/api/nodes/system-status?limit=24"), repoRoot);
|
|
if (result.exitCode !== 0) {
|
|
return { ok: false, exitCode: result.exitCode, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) };
|
|
}
|
|
try {
|
|
const parsed = JSON.parse(result.stdout.trim()) as { ok?: boolean; status?: number; body?: { systemStatuses?: Array<Record<string, unknown>> } };
|
|
const systemStatuses = (parsed.body?.systemStatuses || []).map((item) => {
|
|
const current = (item.current || {}) as Record<string, unknown>;
|
|
const history = Array.isArray(item.history) ? item.history : [];
|
|
return {
|
|
providerId: item.providerId,
|
|
name: item.name,
|
|
nodeStatus: item.nodeStatus,
|
|
updatedAt: item.updatedAt,
|
|
current: item.current ? { ok: current.ok, collectedAt: current.collectedAt, cpu: current.cpu, memory: current.memory, disk: current.disk } : null,
|
|
historyPreview: history.slice(-8),
|
|
historyCount: history.length,
|
|
};
|
|
});
|
|
return { ok: parsed.ok, status: parsed.status, body: { ok: parsed.body !== undefined, systemStatuses } };
|
|
} catch {
|
|
return { ok: true, stdoutTail: result.stdout.slice(-1200), stderrTail: result.stderr.slice(-1200) };
|
|
}
|
|
}
|
|
|
|
export async function debugHealth(config: UniDeskConfig): Promise<unknown> {
|
|
return {
|
|
coreInternal: await coreInternalFetch("/health"),
|
|
overviewInternal: await coreInternalFetch("/api/overview"),
|
|
nodesInternal: await coreInternalFetch("/api/nodes"),
|
|
systemStatusInternal: coreSystemStatusSummary(),
|
|
dockerStatusInternal: coreDockerStatusSummary(),
|
|
frontendPublic: await readJson(`http://127.0.0.1:${config.network.frontend.port}/health`),
|
|
providerIngressPublic: await readJson(`http://127.0.0.1:${config.network.providerIngress.port}/health`),
|
|
publicExposureBoundary: {
|
|
coreHostPort: { port: config.network.core.port, expected: "not-exposed" },
|
|
databaseHostPort: { port: config.network.database.port, expected: "restricted-to-code-queue-provider" },
|
|
oaEventFlowHostPort: { port: 4255, expected: "restricted-to-code-queue-provider" },
|
|
},
|
|
};
|
|
}
|
|
|
|
function recordValue(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function arrayValue(value: unknown): unknown[] {
|
|
return Array.isArray(value) ? value : [];
|
|
}
|
|
|
|
function stringValue(value: unknown): string | null {
|
|
return typeof value === "string" ? value : null;
|
|
}
|
|
|
|
function numberValue(value: unknown): number | null {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function rounded(value: number | null): number | null {
|
|
return value === null ? null : Math.round(value * 100) / 100;
|
|
}
|
|
|
|
function providerHostPressureSummary(providerId: string): unknown {
|
|
const response = coreInternalFetch("/api/nodes/system-status?limit=24");
|
|
const body = recordValue(recordValue(response).body);
|
|
const systemStatuses = arrayValue(body.systemStatuses);
|
|
const item = systemStatuses
|
|
.map((entry) => recordValue(entry))
|
|
.find((entry) => entry.providerId === providerId) ?? null;
|
|
if (item === null) {
|
|
return {
|
|
ok: false,
|
|
degradedReason: "system-status-not-found",
|
|
systemStatusFetch: response,
|
|
};
|
|
}
|
|
const current = recordValue(item.current);
|
|
if (Object.keys(current).length === 0) {
|
|
return {
|
|
ok: false,
|
|
providerId,
|
|
degradedReason: "system-status-current-missing",
|
|
updatedAt: item.updatedAt ?? null,
|
|
nodeStatus: item.nodeStatus ?? null,
|
|
classification: "host-pressure-snapshot-unavailable",
|
|
};
|
|
}
|
|
const cpu = recordValue(current.cpu);
|
|
const memory = recordValue(current.memory);
|
|
const disk = recordValue(current.disk);
|
|
const cores = numberValue(cpu.cores);
|
|
const load1 = numberValue(cpu.load1);
|
|
const load5 = numberValue(cpu.load5);
|
|
const load15 = numberValue(cpu.load15);
|
|
const load1PerCore = cores !== null && cores > 0 && load1 !== null ? load1 / cores : null;
|
|
const memoryPercent = numberValue(memory.percent);
|
|
const memoryAvailableBytes = numberValue(memory.availableBytes);
|
|
const swapTotalBytes = numberValue(memory.swapTotalBytes);
|
|
const diskPercent = numberValue(disk.percent);
|
|
const signals: string[] = [];
|
|
if (load1PerCore !== null && load1PerCore >= 4) signals.push("load1-per-core>=4");
|
|
if (memoryPercent !== null && memoryPercent >= 90) signals.push("memory-percent>=90");
|
|
if (memoryAvailableBytes !== null && memoryAvailableBytes < 512 * 1024 * 1024) signals.push("memory-available<512MiB");
|
|
if (swapTotalBytes === 0) signals.push("swap-disabled");
|
|
if (diskPercent !== null && diskPercent >= 90) signals.push("disk-percent>=90");
|
|
return {
|
|
ok: current.ok ?? null,
|
|
providerId,
|
|
updatedAt: item.updatedAt ?? null,
|
|
collectedAt: current.collectedAt ?? null,
|
|
cpu: {
|
|
cores,
|
|
load1,
|
|
load5,
|
|
load15,
|
|
load1PerCore: rounded(load1PerCore),
|
|
},
|
|
memory: {
|
|
percent: memoryPercent,
|
|
availableBytes: memoryAvailableBytes,
|
|
swapTotalBytes,
|
|
},
|
|
disk: {
|
|
percent: diskPercent,
|
|
availableBytes: numberValue(disk.availableBytes),
|
|
mount: disk.mount ?? null,
|
|
},
|
|
signals,
|
|
classification: signals.length > 0 ? "host-pressure-signals-present" : "host-pressure-signals-not-observed",
|
|
};
|
|
}
|
|
|
|
export async function debugSshPool(_config: UniDeskConfig, providerId: string): Promise<unknown> {
|
|
const nodesResponse = await coreInternalFetch("/api/nodes");
|
|
const body = recordValue(recordValue(nodesResponse).body);
|
|
const nodes = arrayValue(body.nodes);
|
|
const node = nodes
|
|
.map((item) => recordValue(item))
|
|
.find((item) => item.providerId === providerId) ?? null;
|
|
if (node === null) {
|
|
return {
|
|
ok: false,
|
|
providerId,
|
|
degradedReason: "provider-not-found",
|
|
nodesFetch: nodesResponse,
|
|
next: { fullHealth: "bun scripts/cli.ts debug health" },
|
|
};
|
|
}
|
|
const labels = recordValue(node.labels);
|
|
const nodeStatus = stringValue(node.status);
|
|
const pool = {
|
|
transport: stringValue(labels.providerGatewaySshDataTransport),
|
|
host: stringValue(labels.providerGatewaySshDataHost),
|
|
port: labels.providerGatewaySshDataPort ?? null,
|
|
desired: labels.providerGatewaySshDataPoolDesired ?? null,
|
|
total: labels.providerGatewaySshDataPoolTotal ?? null,
|
|
ready: labels.providerGatewaySshDataPoolReady ?? null,
|
|
claimed: labels.providerGatewaySshDataPoolClaimed ?? null,
|
|
connecting: labels.providerGatewaySshDataPoolConnecting ?? null,
|
|
lastError: labels.providerGatewaySshDataPoolLastError ?? null,
|
|
};
|
|
const ready = Number(pool.ready ?? 0);
|
|
const claimed = Number(pool.claimed ?? 0);
|
|
const desired = Number(pool.desired ?? 0);
|
|
const providerOnline = nodeStatus === "online";
|
|
const poolLabelReady = pool.transport === "tcp-pool" && Number.isFinite(ready) && ready > 0;
|
|
const executionPathReady = providerOnline && poolLabelReady;
|
|
return {
|
|
ok: executionPathReady,
|
|
providerId,
|
|
providerOnline,
|
|
poolLabelReady,
|
|
executionPathReady,
|
|
node: {
|
|
providerId: node.providerId,
|
|
name: node.name,
|
|
status: nodeStatus,
|
|
lastHeartbeat: node.lastHeartbeat ?? null,
|
|
updatedAt: node.updatedAt ?? null,
|
|
},
|
|
pool,
|
|
hostPressure: providerHostPressureSummary(providerId),
|
|
classification: !providerOnline
|
|
? (nodeStatus === "offline" ? "provider-offline" : "provider-not-online")
|
|
: poolLabelReady
|
|
? "ssh-tcp-pool-ready"
|
|
: pool.transport !== "tcp-pool"
|
|
? "provider-gateway-upgrade-required"
|
|
: desired > 0 && claimed >= desired
|
|
? "provider-data-pool-exhausted"
|
|
: "provider-data-pool-not-ready",
|
|
next: {
|
|
smoke: `trans ${providerId} argv true`,
|
|
fullHealth: "bun scripts/cli.ts debug health",
|
|
gcDegradedSnapshot: `bun scripts/cli.ts gc remote ${providerId} snapshot --no-save`,
|
|
},
|
|
note: "Pool labels are provider-reported channel state; use the smoke command or gc degraded output to verify execution path.",
|
|
};
|
|
}
|
|
|
|
export async function debugEgressProxy(_config: UniDeskConfig, providerId: string): Promise<unknown> {
|
|
const nodesResponse = await coreInternalFetch("/api/nodes");
|
|
const body = recordValue(recordValue(nodesResponse).body);
|
|
const nodes = arrayValue(body.nodes);
|
|
const node = nodes
|
|
.map((item) => recordValue(item))
|
|
.find((item) => item.providerId === providerId) ?? null;
|
|
if (node === null) {
|
|
return {
|
|
ok: false,
|
|
providerId,
|
|
degradedReason: "provider-not-found",
|
|
nodesFetch: nodesResponse,
|
|
next: { fullHealth: "bun scripts/cli.ts debug health" },
|
|
};
|
|
}
|
|
const labels = recordValue(node.labels);
|
|
const active = Number(labels.providerGatewayEgressProxyActiveTunnels ?? 0);
|
|
const pending = Number(labels.providerGatewayEgressProxyPendingTunnels ?? 0);
|
|
const opened = Number(labels.providerGatewayEgressProxyOpenedTunnels ?? 0);
|
|
const stale = Number(labels.providerGatewayEgressProxyStaleTunnels ?? 0);
|
|
const enabled = labels.providerGatewayEgressProxy === true;
|
|
const connected = labels.providerGatewayEgressProxyConnected === true;
|
|
const sshReady = Number(labels.providerGatewaySshDataPoolReady ?? 0);
|
|
const activeTunnelDetails = arrayValue(labels.providerGatewayEgressProxyActiveTunnelDetails);
|
|
const recentClosedTunnels = arrayValue(labels.providerGatewayEgressProxyRecentClosedTunnels);
|
|
const ok = enabled && connected && stale === 0;
|
|
return {
|
|
ok,
|
|
providerId,
|
|
node: {
|
|
providerId: node.providerId,
|
|
name: node.name,
|
|
status: node.status,
|
|
lastHeartbeat: node.lastHeartbeat ?? null,
|
|
updatedAt: node.updatedAt ?? null,
|
|
},
|
|
egressProxy: {
|
|
enabled,
|
|
connected,
|
|
port: labels.providerGatewayEgressProxyPort ?? null,
|
|
activeTunnels: Number.isFinite(active) ? active : 0,
|
|
pendingTunnels: Number.isFinite(pending) ? pending : 0,
|
|
openedTunnels: Number.isFinite(opened) ? opened : 0,
|
|
staleTunnels: Number.isFinite(stale) ? stale : 0,
|
|
oldestTunnelAgeMs: labels.providerGatewayEgressProxyOldestTunnelAgeMs ?? null,
|
|
policy: {
|
|
openTimeoutMs: labels.providerGatewayEgressProxyOpenTimeoutMs ?? null,
|
|
idleTimeoutMs: labels.providerGatewayEgressProxyIdleTimeoutMs ?? null,
|
|
maxTunnelAgeMs: labels.providerGatewayEgressProxyMaxTunnelAgeMs ?? null,
|
|
staleTunnelIdleMs: labels.providerGatewayEgressProxyStaleTunnelIdleMs ?? null,
|
|
maxPendingBytes: labels.providerGatewayEgressProxyMaxPendingBytes ?? null,
|
|
},
|
|
activeTunnelDetails,
|
|
activeTunnelDetailsCount: activeTunnelDetails.length,
|
|
recentClosedTunnels,
|
|
recentClosedTunnelCount: recentClosedTunnels.length,
|
|
},
|
|
controlPlaneSeparation: {
|
|
sshDataTransport: stringValue(labels.providerGatewaySshDataTransport),
|
|
sshDataPoolReady: Number.isFinite(sshReady) ? sshReady : 0,
|
|
sshDataPoolDesired: labels.providerGatewaySshDataPoolDesired ?? null,
|
|
sshDataPoolClaimed: labels.providerGatewaySshDataPoolClaimed ?? null,
|
|
note: "SSH/control readiness is reported separately from egress download tunnel activity.",
|
|
},
|
|
classification: !enabled
|
|
? "egress-proxy-disabled"
|
|
: !connected
|
|
? "egress-proxy-core-channel-disconnected"
|
|
: stale > 0
|
|
? "egress-tunnel-stale"
|
|
: active > 0
|
|
? "egress-tunnel-active"
|
|
: "egress-proxy-ready",
|
|
outputPolicy: {
|
|
redaction: "activeTunnelDetails expose target host/port and path/query-key summary only; URL credentials, headers, tokens and query values are not emitted.",
|
|
boundedRecentClosedTunnels: true,
|
|
},
|
|
next: {
|
|
smoke: `trans ${providerId} argv true`,
|
|
fullHealth: "bun scripts/cli.ts debug health",
|
|
},
|
|
};
|
|
}
|
|
|
|
const providerRootfsMarker = "__UNIDESK_PROVIDER_ROOTFS__";
|
|
|
|
function safeDockerName(value: string): string {
|
|
return value.replace(/[^a-zA-Z0-9_.-]/gu, "-").slice(0, 80);
|
|
}
|
|
|
|
function providerGatewayContainerName(providerId: string): string {
|
|
return `unidesk-provider-gateway-${safeDockerName(providerId)}`.toLowerCase();
|
|
}
|
|
|
|
function providerRootfsDiagnosticScript(providerId: string): string {
|
|
const container = providerGatewayContainerName(providerId);
|
|
return [
|
|
"set +e",
|
|
`MARK=${shellQuote(providerRootfsMarker)}`,
|
|
`container=${shellQuote(container)}`,
|
|
`provider_id=${shellQuote(providerId)}`,
|
|
'emit() { printf "%s%s=%s\\n" "$MARK" "$1" "$2"; }',
|
|
'emit_b64() { key="$1"; value="$2"; encoded=$(printf "%s" "$value" | base64 | tr -d "\\n"); emit "$key" "$encoded"; }',
|
|
'emit provider_id "$provider_id"',
|
|
'emit container_name "$container"',
|
|
"inspect_output=$(docker inspect \"$container\" --format 'id={{.Id}}\nname={{.Name}}\nimageId={{.Image}}\nimageName={{.Config.Image}}\nstatus={{.State.Status}}\nrunning={{.State.Running}}\nrestartPolicy={{.HostConfig.RestartPolicy.Name}}\npidMode={{.HostConfig.PidMode}}' 2>&1)",
|
|
"inspect_code=$?",
|
|
'emit inspect_exit_code "$inspect_code"',
|
|
'emit_b64 inspect_output "$inspect_output"',
|
|
"smoke_output=$(docker exec \"$container\" /bin/sh -lc 'set -eu\n test -x /bin/sh\n test -x /usr/bin/ssh\n /usr/bin/ssh -V >/tmp/unidesk-provider-ssh-version 2>&1 || { cat /tmp/unidesk-provider-ssh-version >&2; exit 41; }\n command -v docker >/dev/null\n env -u DOCKER_HOST docker version --format \"{{.Client.Version}}\" >/dev/null\n test -e /lib/libz.so.1 -o -e /usr/lib/libz.so.1\n command -v bun >/dev/null\n printf \"rootfs-smoke-ok\\n\"' 2>&1)",
|
|
"smoke_code=$?",
|
|
'emit rootfs_smoke_exit_code "$smoke_code"',
|
|
'emit_b64 rootfs_smoke_output "$smoke_output"',
|
|
'system_df_output=$(docker system df 2>&1)',
|
|
"system_df_code=$?",
|
|
'emit docker_system_df_exit_code "$system_df_code"',
|
|
'emit_b64 docker_system_df_output "$system_df_output"',
|
|
"snapshot_summary=$(printf '%s\\n' \"$system_df_output\" | grep -Ei 'snapshot|lowerdir|no such file|not found|missing|lstat|failed' | tail -20)",
|
|
'emit_b64 snapshot_missing_summary "$snapshot_summary"',
|
|
'logs_output=$(docker logs --since 24h --tail 3000 "$container" 2>&1)',
|
|
"logs_code=$?",
|
|
'emit docker_logs_exit_code "$logs_code"',
|
|
"last_started=$(printf '%s\\n' \"$logs_output\" | grep 'host_ssh_session_started' | tail -1)",
|
|
"last_success=$(printf '%s\\n' \"$logs_output\" | grep 'host_ssh_session_completed' | grep '\"exitCode\":0' | tail -1)",
|
|
"last_failure=$(printf '%s\\n' \"$logs_output\" | grep -E 'host_ssh_session_completed|host_ssh_session_failed|host_ssh_session_error' | grep -Ev '\"exitCode\":0' | tail -1)",
|
|
'emit_b64 last_host_ssh_started "$last_started"',
|
|
'emit_b64 last_host_ssh_success "$last_success"',
|
|
'emit_b64 last_host_ssh_failure "$last_failure"',
|
|
"exit 0",
|
|
].join("\n");
|
|
}
|
|
|
|
function parseProviderRootfsFields(stdout: string): Record<string, string> {
|
|
const fields: Record<string, string> = {};
|
|
for (const line of stdout.split(/\r?\n/u)) {
|
|
if (!line.startsWith(providerRootfsMarker)) continue;
|
|
const payload = line.slice(providerRootfsMarker.length);
|
|
const index = payload.indexOf("=");
|
|
if (index <= 0) continue;
|
|
fields[payload.slice(0, index)] = payload.slice(index + 1);
|
|
}
|
|
return fields;
|
|
}
|
|
|
|
function b64Field(fields: Record<string, string>, key: string): string {
|
|
const raw = fields[key] ?? "";
|
|
if (raw.length === 0) return "";
|
|
try {
|
|
return Buffer.from(raw, "base64").toString("utf8");
|
|
} catch {
|
|
return "";
|
|
}
|
|
}
|
|
|
|
function numberField(fields: Record<string, string>, key: string): number | null {
|
|
const value = Number(fields[key]);
|
|
return Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
function parseKeyValueBlock(text: string): Record<string, string> {
|
|
const values: Record<string, string> = {};
|
|
for (const line of text.split(/\r?\n/u)) {
|
|
const index = line.indexOf("=");
|
|
if (index <= 0) continue;
|
|
values[line.slice(0, index)] = line.slice(index + 1);
|
|
}
|
|
return values;
|
|
}
|
|
|
|
function trimForDiagnostic(text: string, maxChars = 1200): string {
|
|
const trimmed = text.trim();
|
|
if (trimmed.length <= maxChars) return trimmed;
|
|
return trimmed.slice(trimmed.length - maxChars);
|
|
}
|
|
|
|
function extractIsoTimeFromLogLine(line: string): string | null {
|
|
const trimmed = line.trim();
|
|
if (trimmed.length === 0) return null;
|
|
try {
|
|
const parsed = JSON.parse(trimmed) as Record<string, unknown>;
|
|
for (const key of ["time", "timestamp", "at"]) {
|
|
const value = parsed[key];
|
|
if (typeof value === "string" && value.length > 0) return value;
|
|
}
|
|
} catch {
|
|
// Fall through to regex extraction for non-JSON log formats.
|
|
}
|
|
return trimmed.match(/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z/u)?.[0] ?? null;
|
|
}
|
|
|
|
export async function debugProviderRootfs(config: UniDeskConfig, providerId: string): Promise<unknown> {
|
|
const script = providerRootfsDiagnosticScript(providerId);
|
|
const capture = await runSshCommandCapture(config, providerId, ["bash"], script);
|
|
const fields = parseProviderRootfsFields(capture.stdout);
|
|
const inspectOutput = b64Field(fields, "inspect_output");
|
|
const inspect = parseKeyValueBlock(inspectOutput);
|
|
const smokeOutput = b64Field(fields, "rootfs_smoke_output");
|
|
const systemDfOutput = b64Field(fields, "docker_system_df_output");
|
|
const snapshotSummary = b64Field(fields, "snapshot_missing_summary");
|
|
const lastStarted = b64Field(fields, "last_host_ssh_started");
|
|
const lastSuccess = b64Field(fields, "last_host_ssh_success");
|
|
const lastFailure = b64Field(fields, "last_host_ssh_failure");
|
|
const inspectExitCode = numberField(fields, "inspect_exit_code");
|
|
const smokeExitCode = numberField(fields, "rootfs_smoke_exit_code");
|
|
const systemDfExitCode = numberField(fields, "docker_system_df_exit_code");
|
|
const dockerLogsExitCode = numberField(fields, "docker_logs_exit_code");
|
|
const transOk = capture.exitCode === 0;
|
|
const ok = transOk && inspectExitCode === 0 && smokeExitCode === 0 && systemDfExitCode === 0;
|
|
return {
|
|
ok,
|
|
providerId,
|
|
containerName: fields.container_name ?? providerGatewayContainerName(providerId),
|
|
trans: {
|
|
ok: transOk,
|
|
exitCode: capture.exitCode,
|
|
stderrTail: trimForDiagnostic(capture.stderr, 1600),
|
|
},
|
|
container: {
|
|
inspectExitCode,
|
|
id: inspect.id ?? null,
|
|
name: inspect.name?.replace(/^\//u, "") ?? null,
|
|
imageId: inspect.imageId ?? null,
|
|
imageName: inspect.imageName ?? null,
|
|
status: inspect.status ?? null,
|
|
running: inspect.running ?? null,
|
|
restartPolicy: inspect.restartPolicy ?? null,
|
|
pidMode: inspect.pidMode ?? null,
|
|
inspectError: inspectExitCode === 0 ? null : trimForDiagnostic(inspectOutput),
|
|
},
|
|
rootfsSmoke: {
|
|
ok: smokeExitCode === 0,
|
|
exitCode: smokeExitCode,
|
|
outputTail: trimForDiagnostic(smokeOutput),
|
|
checks: ["/bin/sh", "/usr/bin/ssh -V", "libz.so.1", "docker client", "bun"],
|
|
},
|
|
dockerSystemDf: {
|
|
ok: systemDfExitCode === 0,
|
|
exitCode: systemDfExitCode,
|
|
outputTail: trimForDiagnostic(systemDfOutput),
|
|
snapshotMissingSummary: trimForDiagnostic(snapshotSummary),
|
|
},
|
|
hostSsh: {
|
|
dockerLogsExitCode,
|
|
lastStarted: {
|
|
at: extractIsoTimeFromLogLine(lastStarted),
|
|
line: trimForDiagnostic(lastStarted),
|
|
},
|
|
lastSuccess: {
|
|
at: extractIsoTimeFromLogLine(lastSuccess),
|
|
line: trimForDiagnostic(lastSuccess),
|
|
},
|
|
lastFailure: {
|
|
at: extractIsoTimeFromLogLine(lastFailure),
|
|
line: trimForDiagnostic(lastFailure),
|
|
},
|
|
note: "Success/failure rows require provider-gateway versions that log host_ssh_session_completed or host_ssh_session_failed; older containers may only expose lastStarted/error.",
|
|
},
|
|
classification: !transOk
|
|
? "trans-channel-unavailable"
|
|
: inspectExitCode !== 0
|
|
? "provider-container-missing"
|
|
: smokeExitCode !== 0
|
|
? "provider-rootfs-smoke-failed"
|
|
: systemDfExitCode !== 0
|
|
? "docker-system-df-failed"
|
|
: "provider-rootfs-healthy",
|
|
next: {
|
|
sshPool: `bun scripts/cli.ts debug ssh-pool ${providerId}`,
|
|
providerUpgrade: `bun scripts/cli.ts debug dispatch ${providerId} provider.upgrade --mode schedule --wait-ms 300000`,
|
|
},
|
|
};
|
|
}
|
|
|
|
async function waitForTask(taskId: string, timeoutMs: number): Promise<unknown> {
|
|
const started = Date.now();
|
|
let latest: unknown = null;
|
|
while (Date.now() - started < timeoutMs) {
|
|
latest = coreInternalFetch("/api/tasks?limit=100");
|
|
const tasks = (latest as { body?: { tasks?: Array<{ id?: string; status?: string; result?: unknown }> } }).body?.tasks ?? [];
|
|
const task = tasks.find((item) => item.id === taskId);
|
|
if (task?.status === "succeeded" || task?.status === "failed") return { ok: true, task };
|
|
await Bun.sleep(500);
|
|
}
|
|
return { ok: false, timeoutMs, latest };
|
|
}
|
|
|
|
export async function debugTask(_config: UniDeskConfig, taskId: string): Promise<unknown> {
|
|
const tasksResponse = coreInternalFetch("/api/tasks?limit=100");
|
|
const tasks = (tasksResponse as { body?: { tasks?: Array<{ id?: string }> } }).body?.tasks ?? [];
|
|
const task = taskId === "latest" ? tasks[0] : tasks.find((item) => item.id === taskId);
|
|
return { tasksResponse, taskId, task: task ?? null };
|
|
}
|
|
|
|
export async function debugDispatch(
|
|
config: UniDeskConfig,
|
|
providerId: string,
|
|
command: DebugDispatchCommand,
|
|
payload?: Record<string, unknown>,
|
|
waitMs = 0,
|
|
): Promise<unknown> {
|
|
const dispatchPayload = payload ?? (command === "provider.upgrade" ? { source: "cli-debug", mode: "plan" } : { source: "cli-debug" });
|
|
const dispatch = coreInternalFetch("/api/dispatch", {
|
|
method: "POST",
|
|
body: { providerId, command, payload: dispatchPayload },
|
|
});
|
|
const taskId = (dispatch as { body?: { taskId?: string } }).body?.taskId ?? "";
|
|
const wait = waitMs > 0 && taskId.length > 0 ? await waitForTask(taskId, waitMs) : null;
|
|
return { dispatch, wait };
|
|
}
|