feat: add provider ssh tcp data pool

scripts/artifact-registry-ssh-timeout-contract-test.ts

@@ -16,7 +16,8 @@ assertCondition(source.includes("downloadRemoteFile(options, remoteArchive, loca
 assertCondition(source.includes("runRemoteScriptBackground(options, remoteScript"), "remote docker save must run as a background job");
 assertCondition(source.includes('runRemoteScriptBackground(options, deployScript, Math.max(options.timeoutMs, 420_000), "d601-k3s-deploy")'), "D601 k3s deploy must use background polling");
 assertCondition(source.includes('"ssh",\n    options.providerId,\n    "download"'), "download helper must route through UniDesk ssh download");
-assertCondition(downloadRemoteFileSource.includes('"--chunk-bytes",\n    "45000"'), "artifact ssh download chunk must keep base64 stdout below the provider bridge truncation boundary");
+assertCondition(downloadRemoteFileSource.includes('"--chunk-bytes",\n    "1048576"'), "artifact ssh download chunk should use MiB-scale blocks after ssh data moved to tcp-pool");
+assertCondition(!downloadRemoteFileSource.includes('"45000"'), "artifact ssh download must not preserve the old provider bridge truncation boundary");
 assertCondition(downloadRemoteFileSource.includes("tee -a") && downloadRemoteFileSource.includes("UNIDESK_JOB_STDERR_FILE"), "artifact ssh download must stream progress stderr into async job logs");
 assertCondition(source.includes("UNIDESK_SSH_CLIENT_TOKEN") && source.includes("UNIDESK_SSH_CLIENT_ROUTE_ALLOWLIST"), "dev frontend artifact deploy must sync scoped ssh runtime keys");
 
@@ -27,7 +28,7 @@ console.log(JSON.stringify({
     "no docker-save stdout stream over ssh",
     "compose artifact uses verified ssh download",
     "remote docker save and k3s deploy use background polling",
-    "artifact downloads keep base64 stdout below the provider bridge truncation boundary",
+    "artifact downloads use MiB-scale tcp-pool chunks instead of the old bridge truncation boundary",
     "artifact download progress is visible in async job stderr",
     "dev frontend artifact deploy syncs scoped ssh runtime keys"
   ]

scripts/code-queue-pr-preflight-contract-test.ts

@@ -298,6 +298,7 @@ async function main(): Promise<void> {
         devFrontend: { port: 18083, containerPort: 8080 },
         database: { port: 15432, containerPort: 5432 },
         providerIngress: { port: 18082, containerPort: 8081 },
+        providerData: { port: 18084, containerPort: 8082 },
       },
       database: { user: "unidesk", password: "<redacted-test-password>", name: "unidesk", volume: "unidesk_pgdata_10gb", volumeSize: "15GB" },
       providerGateway: {

scripts/src/artifact-registry.ts

@@ -1678,7 +1678,7 @@ function downloadRemoteFile(options: ArtifactRegistryOptions, remotePath: string
     options.providerId,
     "download",
     "--chunk-bytes",
-    "45000",
+    "1048576",
     remotePath,
     localPath,
   ];

scripts/src/config.ts

@@ -13,6 +13,7 @@ export interface UniDeskConfig {
     devFrontend: { port: number; containerPort: number };
     database: { port: number; containerPort: number };
     providerIngress: { port: number; containerPort: number };
+    providerData: { port: number; containerPort: number };
     restrictedHostAccess?: { bindHost: string; allowedSourceCidrs: string[] };
   };
   auth: { username: string; password: string; sessionSecret: string; sessionTtlSeconds: number };
@@ -276,6 +277,7 @@ export function readConfig(): UniDeskConfig {
       devFrontend: portPair(network, "devFrontend"),
       database: portPair(network, "database"),
       providerIngress: portPair(network, "providerIngress"),
+      providerData: portPair(network, "providerData"),
       restrictedHostAccess: optionalRestrictedHostAccess(network),
     },
     database: {

scripts/src/docker.ts

@@ -154,6 +154,10 @@ export function writeComposeEnv(config: UniDeskConfig, freshLogPrefix: boolean):
     UNIDESK_DATABASE_PORT: String(config.network.database.port),
     UNIDESK_DATABASE_BIND_HOST: runtimeSecretWithDefault("UNIDESK_DATABASE_BIND_HOST", restrictedHostBind, "127.0.0.1"),
     UNIDESK_PROVIDER_INGRESS_PORT: String(config.network.providerIngress.port),
+    UNIDESK_PROVIDER_DATA_PORT: String(config.network.providerData.port),
+    UNIDESK_PROVIDER_DATA_POOL_SIZE: runtimeSecret("UNIDESK_PROVIDER_DATA_POOL_SIZE") || "10",
+    UNIDESK_PROVIDER_DATA_CONNECT_TIMEOUT_MS: runtimeSecret("UNIDESK_PROVIDER_DATA_CONNECT_TIMEOUT_MS") || "5000",
+    UNIDESK_BACKEND_CORE_CARGO_BUILD_JOBS: runtimeSecret("UNIDESK_BACKEND_CORE_CARGO_BUILD_JOBS") || "1",
     UNIDESK_DATABASE_USER: config.database.user,
     UNIDESK_DATABASE_PASSWORD: config.database.password,
     UNIDESK_DATABASE_NAME: config.database.name,
@@ -397,6 +401,7 @@ function fixedPorts(config: UniDeskConfig): Array<{ name: string; port: number;
     { name: "frontend", port: config.network.frontend.port, listening: isPortListening(config.network.frontend.port) },
     { name: "dev-frontend", port: config.network.devFrontend.port, listening: isPortListening(config.network.devFrontend.port) },
     { name: "provider-ingress", port: config.network.providerIngress.port, listening: isPortListening(config.network.providerIngress.port) },
+    { name: "provider-data", port: config.network.providerData.port, listening: isPortListening(config.network.providerData.port) },
   ];
 }
 
@@ -570,6 +575,7 @@ export async function stackStatus(config: UniDeskConfig): Promise<unknown> {
       frontend: await probe(`http://127.0.0.1:${config.network.frontend.port}/health`),
       devFrontend: await probe(`http://127.0.0.1:${config.network.devFrontend.port}/health`),
       providerIngress: await probe(`http://127.0.0.1:${config.network.providerIngress.port}/health`),
+      providerDataPortListening: isPortListening(config.network.providerData.port),
       database: dockerExec(config, "unidesk-database", ["pg_isready", "-U", config.database.user, "-d", config.database.name]),
       overview,
     },
@@ -577,6 +583,7 @@ export async function stackStatus(config: UniDeskConfig): Promise<unknown> {
       frontend: `http://${config.network.publicHost}:${config.network.frontend.port}`,
       devFrontend: `http://${config.network.publicHost}:${config.network.devFrontend.port}`,
       providerIngress: `ws://${config.network.publicHost}:${config.network.providerIngress.port}/ws/provider`,
+      providerData: `tcp://${config.network.publicHost}:${config.network.providerData.port}`,
       internalCore: `http://backend-core:${config.network.core.containerPort}`,
       internalDatabase: `postgres://${config.database.user}:***@database:${config.network.database.containerPort}/${config.database.name}`,
     },

scripts/src/help.ts

@@ -12,7 +12,7 @@ export function rootHelp(): unknown {
       { command: "help", description: "List supported commands." },
       { command: "--main-server-ip <ip> <command>", description: "Run selected commands through the public frontend API; use --main-server-key only for legacy SSH transport." },
       { command: "config show", description: "Validate and print config.json as the single source of truth." },
-      { command: "check [--full|--files|--scripts-typecheck|--components|--compose|--logs|--recovery-guardrails|--rust] | check recovery-guardrails", description: "Run the lightweight default syntax/config gate or the low-noise read-only D601 recovery guardrails; Rust is opt-in and only allowed from D601 CI/dev execution." },
+      { command: "check [--full|--files|--scripts-typecheck|--components|--compose|--logs|--recovery-guardrails|--rust] | check recovery-guardrails", description: "Run the lightweight default syntax/config gate or the low-noise read-only D601 recovery guardrails; check --rust stays D601-guarded, while backend-core main-server online uses the separate constrained rebuild path." },
       { command: "server start", description: "Fire-and-forget build/start for database, backend-core, frontend, provider gateway, and managed main-server user services." },
       { command: "server stop", description: "Fire-and-forget docker-compose down for the fixed UniDesk stack." },
       { command: "server status", description: "Show fixed ports, containers, service health, and public URLs." },
@@ -22,7 +22,7 @@ export function rootHelp(): unknown {
       { command: "gc plan|run|db-trace|policy|remote [--confirm] [--logs-keep-days N] [--include-browser-cache]", description: "One-time main-server or remote provider disk relief and low-risk anti-bloat policy for logs, journald, allowlisted /tmp artifacts, scoped core dumps and explicit trace telemetry retention; plan is read-only and run requires --confirm." },
       { command: "server rebuild <backend-core|frontend|dev-frontend-proxy|provider-gateway|todo-note|code-queue-mgr|project-manager|baidu-netdisk|oa-event-flow>", description: "Maintenance-only local Compose rebuild for reviewed main-server services; frontend standard release must use CI artifact plus deploy apply dev/prod artifact consumers." },
       { command: "provider attach <providerId> [--master-server URL] [--up] [--force] | provider triage <providerId> [--observed-error text] [--observed-scope scope] [--microservice id ...] [--full|--raw]", description: "Generate the minimal external provider-gateway env/compose bundle or run the low-noise read-only provider health triage contract." },
-      { command: "trans <route> [operation args...] (alias of ssh <route> ...)", description: "Open a Host SSH / WSL SSH maintenance session through the provider-gateway bridge; `trans` is the short alias for this ssh passthrough path." },
+      { command: "trans <route> [operation args...] (alias of ssh <route> ...)", description: "Open a Host SSH / WSL SSH maintenance session; provider WebSocket carries control and host.ssh.tcp-pool carries stdin/stdout/stderr data." },
       { command: "trans gh:/owner/repo[/pr|/issue][/number[/1]] ls|cat|rg|patch-apply", description: "Treat GitHub PRs/issues as virtual text directories; `ls --full` shows state/floors/body length, and `patch-apply` updates first-floor `body.md` through UniDesk gh plus apply-patch v2." },
       { command: "trans <route> apply-patch < patch.diff", description: "Default remote text patch entry: apply a standard patch with the local TypeScript v2 engine while the remote route only reads and writes files." },
       { command: "trans <route> upload <local-file> <remote-file> | trans <route> download <remote-file> <local-file>", description: "Transfer whole files through SSH passthrough with remote temp files, automatic endpoint byte/SHA-256 verification, and client-side chunk fallback." },
@@ -133,7 +133,7 @@ export function serverHelp(action: string | undefined = undefined): unknown {
       providerIngress: "provider-gateway WebSocket ingress",
     },
     rustBoundary: {
-      masterServer: "do not use server rebuild backend-core for Rust iteration; it would build locally",
+      masterServer: "do not use server rebuild backend-core for routine Rust iteration; only reviewed backend-core main-server online may use the constrained rebuild path",
       d601: "use deploy apply --env dev --service backend-core and CI for Rust build/check",
     },
     maintenanceOnly: {

scripts/src/ssh-file-transfer.ts

@@ -50,12 +50,13 @@ class SshFileTransferError extends Error {
   }
 }
 
-const fileTransferReadBlockBytes = 45_000;
+const fileTransferReadBlockBytes = 1_048_576;
 const fileTransferWriteB64ArgvLimit = 48_000;
-const fileTransferWriteB64ChunkChars = 12_000;
+const fileTransferWriteRawChunkBytes = 1_048_576;
+const fileTransferWriteB64ChunkChars = 1_398_104;
 const fileTransferReadBlockMaxAttempts = 12;
 const fileTransferReadEmptyRetryDelayMs = 250;
-const fileTransferProgressEveryChunks = 64;
+const fileTransferProgressEveryChunks = 16;
 
 export function isSshFileTransferOperation(args: string[]): boolean {
   const subcommand = args[0] ?? "";
@@ -163,7 +164,7 @@ function parseSshFileTransferCliOptions(args: string[]): SshFileTransferCliOptio
 function boundedTransferChunkBytes(raw: string, option: string): number {
   const value = Number(raw);
   if (!Number.isInteger(value) || value <= 0) throw new Error(`${option} must be a positive integer`);
-  return Math.min(96_000, Math.max(1024, value));
+  return Math.min(4 * 1024 * 1024, Math.max(1024, value));
 }
 
 async function writeRemoteFileVerified(
@@ -173,26 +174,33 @@ async function writeRemoteFileVerified(
   remotePath: string,
   content: Buffer,
 ): Promise<SshFileTransferWriteResult> {
-  const encoded = content.toString("base64");
   const expectedBytes = String(content.length);
   const expectedSha256 = sha256HexBuffer(content);
+  const encoded = content.length <= fileTransferWriteB64ArgvLimit
+    ? content.toString("base64")
+    : "";
   if (invocation.route.plane !== "win" && encoded.length <= fileTransferWriteB64ArgvLimit) {
     await checkedFileTransfer(invocation, executor, builders, "write-b64-argv", [remotePath, expectedBytes, expectedSha256, ...chunkString(encoded, fileTransferWriteB64ChunkChars)]);
     return { strategy: "argv", chunks: encoded.length === 0 ? 0 : Math.ceil(encoded.length / fileTransferWriteB64ChunkChars) };
   }
-  try {
-    await checkedFileTransfer(invocation, executor, builders, "write-b64-stdin", [remotePath, expectedBytes, expectedSha256], encoded);
-    return { strategy: "stdin", chunks: 1 };
-  } catch {
-    const token = `${process.pid}-${Date.now()}-${randomBytes(4).toString("hex")}-${expectedSha256.slice(0, 12)}`;
-    const chunks = chunkString(encoded, fileTransferWriteB64ChunkChars);
-    await checkedFileTransfer(invocation, executor, builders, "write-b64-begin", [remotePath, token]);
-    for (const chunk of chunks) {
-      await checkedFileTransfer(invocation, executor, builders, "write-b64-append-stdin", [remotePath, token], chunk);
+  if (content.length <= fileTransferWriteRawChunkBytes) {
+    try {
+      await checkedFileTransfer(invocation, executor, builders, "write-b64-stdin", [remotePath, expectedBytes, expectedSha256], content.toString("base64"));
+      return { strategy: "stdin", chunks: 1 };
+    } catch {
+      // Fall through to chunked upload with per-block base64 encoding.
     }
-    await checkedFileTransfer(invocation, executor, builders, "write-b64-commit", [remotePath, token, expectedBytes, expectedSha256]);
-    return { strategy: "chunked-stdin", chunks: encoded.length === 0 ? 0 : chunks.length };
   }
+  const token = `${process.pid}-${Date.now()}-${randomBytes(4).toString("hex")}-${expectedSha256.slice(0, 12)}`;
+  let chunks = 0;
+  await checkedFileTransfer(invocation, executor, builders, "write-b64-begin", [remotePath, token]);
+  for (let offset = 0; offset < content.length; offset += fileTransferWriteRawChunkBytes) {
+    const encodedChunk = content.subarray(offset, Math.min(content.length, offset + fileTransferWriteRawChunkBytes)).toString("base64");
+    await checkedFileTransfer(invocation, executor, builders, "write-b64-append-stdin", [remotePath, token], encodedChunk);
+    chunks += 1;
+  }
+  await checkedFileTransfer(invocation, executor, builders, "write-b64-commit", [remotePath, token, expectedBytes, expectedSha256]);
+  return { strategy: "chunked-stdin", chunks };
 }
 
 async function readRemoteFileVerified(

scripts/ssh-data-tcp-pool-contract-test.ts

@@ -0,0 +1,51 @@
+import { readFileSync } from "node:fs";
+
+function assertCondition(condition: unknown, message: string, detail: unknown = {}): void {
+  if (!condition) throw new Error(`${message}: ${JSON.stringify(detail)}`);
+}
+
+const rustBridge = readFileSync("src/components/backend-core/src/ssh_bridge.rs", "utf8");
+const rustData = readFileSync("src/components/backend-core/src/ssh_data_channel.rs", "utf8");
+const rustMain = readFileSync("src/components/backend-core/src/main.rs", "utf8");
+const rustState = readFileSync("src/components/backend-core/src/state.rs", "utf8");
+const rustProviderRegistry = readFileSync("src/components/backend-core/src/provider_registry.rs", "utf8");
+const provider = readFileSync("src/components/provider-gateway/src/index.ts", "utf8");
+const providerRegistryTs = readFileSync("src/components/backend-core/src/provider-registry.ts", "utf8");
+const providerPackage = JSON.parse(readFileSync("src/components/provider-gateway/package.json", "utf8")) as { version?: unknown };
+const shared = readFileSync("src/components/shared/src/index.ts", "utf8");
+const compose = readFileSync("docker-compose.yml", "utf8");
+const config = readFileSync("config.json", "utf8");
+const backendCoreDockerfile = readFileSync("src/components/backend-core/Dockerfile", "utf8");
+
+assertCondition(rustBridge.includes('"host.ssh.tcp-pool"'), "backend-core ssh bridge must require host.ssh.tcp-pool");
+assertCondition(rustBridge.includes("provider-gateway-upgrade-required"), "old provider must fail with upgrade-required classification");
+assertCondition(rustBridge.includes("provider-data-pool-exhausted"), "tcp pool exhaustion must be visible");
+assertCondition(!rustBridge.includes('"host_ssh_input"'), "ssh input must not fall back to provider control websocket");
+assertCondition(!rustBridge.includes('"host_ssh_eof"'), "ssh eof must not fall back to provider control websocket");
+assertCondition(!rustBridge.includes('"host_ssh_close"'), "ssh close must not fall back to provider control websocket");
+assertCondition(rustBridge.includes('json!({ "type": "input"') && rustBridge.includes('json!({ "type": "eof"'), "ssh bridge must send stdin/eof over data frames");
+assertCondition(!rustProviderRegistry.includes('"host_ssh_opened"') && !rustProviderRegistry.includes('"host_ssh_data"') && !rustProviderRegistry.includes('"host_ssh_exit"'), "rust provider registry must not accept old websocket ssh data messages");
+
+assertCondition(rustData.includes("TcpListener") && rustData.includes("read_data_frame") && rustData.includes("write_data_frame"), "backend-core must expose raw TCP data frame listener");
+assertCondition(rustData.includes("SSH_DATA_PROTOCOL") && rustData.includes("unidesk-host-ssh-tcp-pool-v1"), "tcp data protocol must be explicit");
+assertCondition(rustData.includes("base64::engine::general_purpose::STANDARD.encode(payload)"), "only client-facing websocket payload should remain base64 encoded");
+assertCondition(rustMain.includes("providerDataTcpUrl") && rustMain.includes("spawn_ssh_data_listener"), "backend-core startup must expose provider data listener visibility");
+assertCondition(rustState.includes("active_ssh_data_channels") && rustState.includes("data_channel_id"), "backend state must track data channels separately from provider control socket");
+
+assertCondition(provider.includes("createConnection") && provider.includes("sshDataChannels"), "provider-gateway must use direct TCP sockets for ssh data pool");
+assertCondition(provider.includes("PROVIDER_DATA_POOL_SIZE") && provider.includes("providerGatewaySshDataPoolReady"), "provider-gateway must expose pool config and heartbeat labels");
+assertCondition(provider.includes('capabilities.push("host.ssh", "host.ssh.tcp-pool")'), "provider-gateway must declare tcp-pool capability only with host ssh");
+assertCondition(provider.includes("acquireSshDataChannel") && provider.includes("releaseSshDataChannel"), "provider-gateway must claim/release one data channel per ssh session");
+assertCondition(provider.includes("writeSshDataFrame(dataChannel") && provider.includes("sendSshDataSessionFrame"), "provider-gateway ssh output must use tcp data frames");
+assertCondition(!provider.includes('parsed.type === "host_ssh_input"') && !provider.includes('parsed.type === "host_ssh_close"'), "provider-gateway must not retain old websocket ssh input handlers");
+assertCondition(providerPackage.version === "0.2.28", "provider-gateway behavior change must bump package version", providerPackage);
+
+assertCondition(shared.includes('transport: "tcp-pool"') && shared.includes("dataChannelId: string"), "shared host_ssh_open contract must require tcp-pool fields");
+assertCondition(!shared.includes("CoreHostSshInputMessage") && !shared.includes("ProviderHostSshDataMessage"), "shared protocol must remove old websocket ssh data contracts");
+assertCondition(!providerRegistryTs.includes("host_ssh_data") && !providerRegistryTs.includes("host_ssh_exit"), "typescript backend registry must not forward old websocket ssh data messages");
+assertCondition(compose.includes("UNIDESK_PROVIDER_DATA_PORT") && compose.includes("PROVIDER_DATA_POOL_SIZE"), "compose must wire provider data port and pool size");
+assertCondition(config.includes('"providerData"') && config.includes('"port": 18084') && config.includes('"containerPort": 8082'), "config.json must declare providerData port pair");
+assertCondition(backendCoreDockerfile.includes("ARG CARGO_BUILD_JOBS=1") && backendCoreDockerfile.includes('cargo build --release --jobs "${CARGO_BUILD_JOBS}"'), "backend-core main-server online build must keep cargo concurrency constrained");
+assertCondition(compose.includes("UNIDESK_BACKEND_CORE_CARGO_BUILD_JOBS") && compose.includes("CARGO_BUILD_JOBS"), "compose must pass backend-core cargo build concurrency explicitly");
+
+console.log(JSON.stringify({ ok: true, test: "ssh-data-tcp-pool-contract" }));