fix ssh tcp pool transient diagnostics

scripts/src/ssh.ts

@@ -130,6 +130,30 @@ export interface SshStdoutTruncationHint {
   note: string;
 }
 
+export type SshTcpPoolFailureKind =
+  | "provider-data-channel-closed"
+  | "provider-data-channel-missing"
+  | "provider-data-pool-exhausted";
+
+export interface SshTcpPoolHint {
+  code: "ssh-tcp-pool-transient";
+  level: "warning";
+  providerId: string;
+  route: string;
+  transport: "backend-core-broker" | "frontend-websocket";
+  invocationKind: SshInvocationKind;
+  failureKind: SshTcpPoolFailureKind;
+  exitCode: number;
+  message: string;
+  action: string;
+  retry: string;
+  diagnostics: {
+    poolStatus: string;
+    fullHealth: string;
+  };
+  note: string;
+}
+
 const argvQuotedSshSubcommands = new Set(["git", "rg", "grep", "sed", "nl", "stat", "du", "ls", "cat", "head", "tail", "wc", "pwd"]);
 const nativeK3sKubeconfig = "/etc/rancher/k3s/k3s.yaml";
 const windowsBridgeCwd = "/mnt/c/Windows";
@@ -2489,6 +2513,59 @@ export function formatSshStdoutTruncationHint(hint: SshStdoutTruncationHint): st
   return `UNIDESK_SSH_STDOUT_TRUNCATED ${JSON.stringify(hint)}\n`;
 }
 
+export function classifySshTcpPoolFailure(text: string): SshTcpPoolFailureKind | null {
+  const normalized = text.toLowerCase();
+  if (normalized.includes("ssh tcp data channel closed")) return "provider-data-channel-closed";
+  if (
+    normalized.includes("requested ssh tcp data channel is not ready")
+    || normalized.includes("ssh tcp data channel is not available")
+    || normalized.includes('"failurekind":"provider-data-channel-missing"')
+  ) {
+    return "provider-data-channel-missing";
+  }
+  if (
+    normalized.includes("provider ssh tcp data pool has no idle channel")
+    || normalized.includes('"failurekind":"provider-data-pool-exhausted"')
+  ) {
+    return "provider-data-pool-exhausted";
+  }
+  return null;
+}
+
+export function sshTcpPoolHint(options: {
+  invocation: ParsedSshInvocation;
+  transport: SshTcpPoolHint["transport"];
+  exitCode: number;
+  stderrText: string;
+}): SshTcpPoolHint | null {
+  if (options.exitCode === 0) return null;
+  const failureKind = classifySshTcpPoolFailure(options.stderrText);
+  if (failureKind === null) return null;
+  const providerId = safeProviderId(options.invocation.providerId);
+  return {
+    code: "ssh-tcp-pool-transient",
+    level: "warning",
+    providerId,
+    route: options.invocation.route.raw,
+    transport: options.transport,
+    invocationKind: options.invocation.parsed.invocationKind,
+    failureKind,
+    exitCode: options.exitCode,
+    message: "host.ssh.tcp-pool data channel failed during an SSH operation; classify this as transport/data-pool transient until pool labels and a retry prove otherwise.",
+    action: "Inspect providerGatewaySshData* labels, then rerun the same idempotent command through the controlled CLI. Do not treat this alone as HWLAB runtime configuration failure.",
+    retry: `trans ${providerId} argv true`,
+    diagnostics: {
+      poolStatus: `bun scripts/cli.ts debug ssh-pool ${providerId}`,
+      fullHealth: "bun scripts/cli.ts debug health",
+    },
+    note: "Hint is written to stderr and intentionally does not echo the original remote command.",
+  };
+}
+
+export function formatSshTcpPoolHint(hint: SshTcpPoolHint | null): string {
+  return hint === null ? "" : `UNIDESK_SSH_TCP_POOL_HINT ${JSON.stringify(hint)}\n`;
+}
+
 function sshStdoutDumpPath(invocation: ParsedSshInvocation): string {
   mkdirSync(sshStdoutDumpDir, { recursive: true, mode: 0o700 });
   const timestamp = new Date().toISOString().replace(/[:.]/gu, "-");
@@ -2677,6 +2754,16 @@ ws.addEventListener("message", (event) => {
     clearTimeout(openTimer);
     clearRuntimeTimer();
     process.stderr.write(String(message.message || "ssh bridge error") + "\n");
+    if (message.failureKind || message.providerId || message.dataChannelId || message.dataPool) {
+      process.stderr.write("UNIDESK_SSH_ERROR " + JSON.stringify({
+        failureKind: message.failureKind || null,
+        providerId: message.providerId || null,
+        dataChannelId: message.dataChannelId || null,
+        dataPool: message.dataPool || null,
+        transport: message.transport || null,
+        controlFallback: message.controlFallback === true
+      }) + "\n");
+    }
     exitCode = 255;
     ws.close();
     return;
@@ -2915,6 +3002,12 @@ async function runSshCaptureRemoteCommand(config: UniDeskConfig, invocation: Par
         startedAtMs,
       }));
       if (timingHint) stderr += timingHint;
+      stderr += formatSshTcpPoolHint(sshTcpPoolHint({
+        invocation,
+        transport: "backend-core-broker",
+        exitCode,
+        stderrText: stderr,
+      }));
       resolve({ exitCode, stdout, stderr });
     };
     const runtimeTimer = setTimeout(() => {
@@ -3066,6 +3159,12 @@ async function runSshStreamRemoteCommand(
           startedAtMs,
         }));
         if (timingHint) stderr += timingHint;
+        stderr += formatSshTcpPoolHint(sshTcpPoolHint({
+          invocation: streamInvocation,
+          transport: "backend-core-broker",
+          exitCode: finalCode,
+          stderrText: stderr,
+        }));
         resolve({ exitCode: finalCode, stdout, stderr });
       });
     };
@@ -3358,6 +3457,15 @@ export async function runSsh(config: UniDeskConfig, providerId: string, args: st
       restore();
       const hint = timedOut ? null : sshFailureHint(invocation.providerId, parsed, exitCode, stderrTail);
       if (hint !== null) process.stderr.write(formatSshFailureHint(hint));
+      if (!timedOut) {
+        const tcpPoolHint = formatSshTcpPoolHint(sshTcpPoolHint({
+          invocation,
+          transport: "backend-core-broker",
+          exitCode,
+          stderrText: stderrTail,
+        }));
+        if (tcpPoolHint) process.stderr.write(tcpPoolHint);
+      }
       const timingHint = formatSshRuntimeTimingHint(sshRuntimeTimingHint({
         invocation,
         transport: "backend-core-broker",