Improve ssh tran file transfer reliability

This commit is contained in:
Codex
2026-05-27 04:08:11 +00:00
parent 29ec9254bf
commit 27ed8a261d
8 changed files with 607 additions and 10 deletions
+409
View File
@@ -0,0 +1,409 @@
import { createHash, randomBytes } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import path from "node:path";
import type { ParsedSshInvocation, ParsedSshRoute, SshCaptureResult } from "./ssh";
export interface SshRemoteCommandExecutor {
runRemoteCommand(remoteCommand: string, input?: string): Promise<SshCaptureResult>;
}
export interface SshFileTransferCommandBuilders {
buildRouteCommand(route: ParsedSshRoute, command: string[], options?: { stdin?: boolean }): string;
buildWindowsPowerShellCommand(script: string): string;
}
type SshFileTransferOperation =
| "stat"
| "read-b64-block"
| "write-b64-argv"
| "write-b64-stdin"
| "write-b64-begin"
| "write-b64-append-stdin"
| "write-b64-commit";
interface SshFileTransferCliOptions {
action: "upload" | "download";
localPath: string;
remotePath: string;
readBlockBytes: number;
}
interface SshFileTransferStat {
bytes: number;
sha256: string;
}
interface SshFileTransferWriteResult {
strategy: "argv" | "stdin" | "chunked-stdin";
chunks: number;
}
class SshFileTransferError extends Error {
constructor(message: string, public readonly details: Record<string, unknown> = {}) {
super(message);
this.name = "SshFileTransferError";
}
}
const fileTransferReadBlockBytes = 45_000;
const fileTransferWriteB64ArgvLimit = 48_000;
const fileTransferWriteB64ChunkChars = 12_000;
export function isSshFileTransferOperation(args: string[]): boolean {
const subcommand = args[0] ?? "";
return subcommand === "upload" || subcommand === "download";
}
export async function runSshFileTransferOperation(
invocation: ParsedSshInvocation,
args: string[],
executor: SshRemoteCommandExecutor,
builders: SshFileTransferCommandBuilders,
): Promise<number> {
const options = parseSshFileTransferCliOptions(args);
const localPath = path.resolve(options.localPath);
if (options.action === "upload") {
const content = await readFile(localPath);
const expected = { bytes: content.length, sha256: sha256HexBuffer(content) };
const write = await writeRemoteFileVerified(invocation, executor, builders, options.remotePath, content);
const remote = await statRemoteFile(invocation, executor, builders, options.remotePath);
assertTransferStat("upload final remote verification", options.remotePath, expected, remote);
process.stdout.write(`${JSON.stringify({
ok: true,
command: "ssh upload",
route: invocation.route.raw,
providerId: invocation.providerId,
localPath,
remotePath: options.remotePath,
bytes: expected.bytes,
sha256: expected.sha256,
verified: true,
transfer: write,
}, null, 2)}\n`);
return 0;
}
const read = await readRemoteFileVerified(invocation, executor, builders, options.remotePath, options.readBlockBytes);
await mkdir(path.dirname(localPath), { recursive: true });
await writeFile(localPath, read.content);
const local = await readFile(localPath);
const localStat = { bytes: local.length, sha256: sha256HexBuffer(local) };
assertTransferStat("download final local verification", localPath, read.remote, localStat);
process.stdout.write(`${JSON.stringify({
ok: true,
command: "ssh download",
route: invocation.route.raw,
providerId: invocation.providerId,
remotePath: options.remotePath,
localPath,
bytes: read.remote.bytes,
sha256: read.remote.sha256,
verified: true,
transfer: {
strategy: "chunked-read",
chunks: read.chunks,
chunkBytes: options.readBlockBytes,
},
}, null, 2)}\n`);
return 0;
}
function parseSshFileTransferCliOptions(args: string[]): SshFileTransferCliOptions {
const action = args[0];
if (action !== "upload" && action !== "download") throw new Error("ssh file transfer requires upload or download");
const positionals: string[] = [];
let readBlockBytes = fileTransferReadBlockBytes;
for (let index = 1; index < args.length; index += 1) {
const arg = args[index] ?? "";
if (arg === "--") {
positionals.push(...args.slice(index + 1));
break;
}
if (arg === "--chunk-bytes" || arg === "--block-bytes") {
const value = args[index + 1];
if (value === undefined) throw new Error(`ssh ${action} ${arg} requires a value`);
readBlockBytes = boundedTransferChunkBytes(value, `ssh ${action} ${arg}`);
index += 1;
continue;
}
if (arg === "--help" || arg === "-h" || arg === "help") {
throw new Error(`ssh ${action} usage: ssh <route> ${action} ${action === "upload" ? "<local-file> <remote-file>" : "<remote-file> <local-file>"} [--chunk-bytes N]`);
}
if (arg.startsWith("-")) throw new Error(`unsupported ssh ${action} option: ${arg}`);
positionals.push(arg);
}
if (positionals.length !== 2) {
throw new Error(`ssh ${action} requires exactly two paths: ${action === "upload" ? "<local-file> <remote-file>" : "<remote-file> <local-file>"}`);
}
const [first, second] = positionals;
if (!first || !second) throw new Error(`ssh ${action} paths must be non-empty`);
return action === "upload"
? { action, localPath: first, remotePath: second, readBlockBytes }
: { action, remotePath: first, localPath: second, readBlockBytes };
}
function boundedTransferChunkBytes(raw: string, option: string): number {
const value = Number(raw);
if (!Number.isInteger(value) || value <= 0) throw new Error(`${option} must be a positive integer`);
return Math.min(96_000, Math.max(1024, value));
}
async function writeRemoteFileVerified(
invocation: ParsedSshInvocation,
executor: SshRemoteCommandExecutor,
builders: SshFileTransferCommandBuilders,
remotePath: string,
content: Buffer,
): Promise<SshFileTransferWriteResult> {
const encoded = content.toString("base64");
const expectedBytes = String(content.length);
const expectedSha256 = sha256HexBuffer(content);
if (invocation.route.plane !== "win" && encoded.length <= fileTransferWriteB64ArgvLimit) {
await checkedFileTransfer(invocation, executor, builders, "write-b64-argv", [remotePath, expectedBytes, expectedSha256, ...chunkString(encoded, fileTransferWriteB64ChunkChars)]);
return { strategy: "argv", chunks: encoded.length === 0 ? 0 : Math.ceil(encoded.length / fileTransferWriteB64ChunkChars) };
}
try {
await checkedFileTransfer(invocation, executor, builders, "write-b64-stdin", [remotePath, expectedBytes, expectedSha256], encoded);
return { strategy: "stdin", chunks: 1 };
} catch {
const token = `${process.pid}-${Date.now()}-${randomBytes(4).toString("hex")}-${expectedSha256.slice(0, 12)}`;
const chunks = chunkString(encoded, fileTransferWriteB64ChunkChars);
await checkedFileTransfer(invocation, executor, builders, "write-b64-begin", [remotePath, token]);
for (const chunk of chunks) {
await checkedFileTransfer(invocation, executor, builders, "write-b64-append-stdin", [remotePath, token], chunk);
}
await checkedFileTransfer(invocation, executor, builders, "write-b64-commit", [remotePath, token, expectedBytes, expectedSha256]);
return { strategy: "chunked-stdin", chunks: encoded.length === 0 ? 0 : chunks.length };
}
}
async function readRemoteFileVerified(
invocation: ParsedSshInvocation,
executor: SshRemoteCommandExecutor,
builders: SshFileTransferCommandBuilders,
remotePath: string,
readBlockBytes: number,
): Promise<{ remote: SshFileTransferStat; content: Buffer; chunks: number }> {
const remote = await statRemoteFile(invocation, executor, builders, remotePath);
const chunks: Buffer[] = [];
let actualBytes = 0;
let chunkCount = 0;
for (let blockIndex = 0; actualBytes < remote.bytes; blockIndex += 1) {
const read = await checkedFileTransfer(invocation, executor, builders, "read-b64-block", [remotePath, String(blockIndex), String(readBlockBytes)]);
const encoded = read.stdout.replace(/\s+/gu, "");
const chunk = encoded.length === 0 ? Buffer.alloc(0) : Buffer.from(encoded, "base64");
if (chunk.length === 0) {
throw new SshFileTransferError("remote download returned an empty block before EOF", {
route: invocation.route.raw,
remotePath,
blockIndex,
expectedBytes: remote.bytes,
actualBytes,
});
}
chunks.push(chunk);
actualBytes += chunk.length;
chunkCount += 1;
}
const content = Buffer.concat(chunks);
const actual = { bytes: content.length, sha256: sha256HexBuffer(content) };
assertTransferStat("download remote read verification", remotePath, remote, actual);
return { remote, content, chunks: chunkCount };
}
async function statRemoteFile(
invocation: ParsedSshInvocation,
executor: SshRemoteCommandExecutor,
builders: SshFileTransferCommandBuilders,
remotePath: string,
): Promise<SshFileTransferStat> {
const stat = await checkedFileTransfer(invocation, executor, builders, "stat", [remotePath]);
return parseFileTransferStat(stat.stdout, remotePath);
}
async function checkedFileTransfer(
invocation: ParsedSshInvocation,
executor: SshRemoteCommandExecutor,
builders: SshFileTransferCommandBuilders,
operation: SshFileTransferOperation,
args: string[],
input?: string,
): Promise<SshCaptureResult> {
const remoteCommand = buildFileTransferRemoteCommand(invocation.route, builders, operation, args, input !== undefined);
const result = await executor.runRemoteCommand(remoteCommand, input);
if (result.exitCode === 0) return result;
throw new SshFileTransferError("remote ssh file transfer operation failed", {
route: invocation.route.raw,
operation,
args: args.slice(0, 4),
exitCode: result.exitCode,
stdout: result.stdout.slice(-2000),
stderr: result.stderr.slice(-4000),
});
}
function buildFileTransferRemoteCommand(
route: ParsedSshRoute,
builders: SshFileTransferCommandBuilders,
operation: SshFileTransferOperation,
args: string[],
hasInput: boolean,
): string {
if (route.plane === "win") return buildWindowsFileTransferCommand(route, builders, operation, args);
return builders.buildRouteCommand(route, ["sh", "-c", posixFileTransferScript(), "unidesk-file-transfer", operation, ...args], { stdin: hasInput });
}
function parseFileTransferStat(stdout: string, remotePath: string): SshFileTransferStat {
const [bytesText, sha256] = stdout.trim().split(/\s+/u);
const bytes = Number(bytesText);
if (!Number.isSafeInteger(bytes) || bytes < 0 || !/^[0-9a-f]{64}$/u.test(sha256 ?? "")) {
throw new SshFileTransferError("remote file transfer stat returned invalid metadata", {
remotePath,
stdout: stdout.slice(0, 500),
});
}
return { bytes, sha256: sha256! };
}
function assertTransferStat(label: string, pathName: string, expected: SshFileTransferStat, actual: SshFileTransferStat): void {
if (expected.bytes === actual.bytes && expected.sha256 === actual.sha256) return;
throw new SshFileTransferError(`${label} failed`, {
path: pathName,
expected,
actual,
});
}
function sha256HexBuffer(value: Buffer): string {
return createHash("sha256").update(value).digest("hex");
}
function chunkString(value: string, chunkSize: number): string[] {
const chunks: string[] = [];
for (let index = 0; index < value.length; index += chunkSize) {
chunks.push(value.slice(index, index + chunkSize));
}
return chunks.length > 0 ? chunks : [""];
}
function posixFileTransferScript(): string {
return [
"set -eu",
"sha256_file() {",
" if command -v sha256sum >/dev/null 2>&1; then sha256sum -- \"$1\" | awk '{print $1}'; return; fi",
" if command -v shasum >/dev/null 2>&1; then shasum -a 256 -- \"$1\" | awk '{print $1}'; return; fi",
" if command -v openssl >/dev/null 2>&1; then openssl dgst -sha256 -- \"$1\" | awk '{print $NF}'; return; fi",
" printf 'missing sha256 tool\\n' >&2; return 127",
"}",
"ensure_parent() { case \"$1\" in */*) parent=${1%/*}; mkdir -p -- \"$parent\";; esac; }",
"verify_tmp() {",
" target=$1; tmp=$2; expected_bytes=$3; expected_sha256=$4",
" actual_bytes=$(wc -c < \"$tmp\" | tr -d '[:space:]')",
" if [ \"$actual_bytes\" != \"$expected_bytes\" ]; then rm -f -- \"$tmp\"; printf 'transfer byte count mismatch for %s: expected=%s actual=%s\\n' \"$target\" \"$expected_bytes\" \"$actual_bytes\" >&2; exit 23; fi",
" actual_sha256=$(sha256_file \"$tmp\")",
" if [ \"$actual_sha256\" != \"$expected_sha256\" ]; then rm -f -- \"$tmp\"; printf 'transfer sha256 mismatch for %s: expected=%s actual=%s\\n' \"$target\" \"$expected_sha256\" \"$actual_sha256\" >&2; exit 24; fi",
"}",
"set_tmp_paths() {",
" target=$1; token=$2",
" case \"$token\" in ''|*[!a-zA-Z0-9_.-]*) printf 'invalid transfer temp token\\n' >&2; exit 2;; esac",
" base=${target##*/}; dir=.",
" case \"$target\" in */*) dir=${target%/*};; esac",
" tmp=\"$dir/.${base}.unidesk-transfer-${token}.tmp\"",
" tmp_b64=\"$tmp.b64\"",
"}",
"op=$1; shift",
"case \"$op\" in",
" stat)",
" target=$1; bytes=$(wc -c < \"$target\" | tr -d '[:space:]'); digest=$(sha256_file \"$target\"); printf '%s %s\\n' \"$bytes\" \"$digest\"",
" ;;",
" read-b64-block)",
" target=$1; block_index=$2; block_size=$3",
" case \"$block_index:$block_size\" in *[!0-9:]*|:*) printf 'invalid read block args\\n' >&2; exit 2;; esac",
" dd if=\"$target\" bs=\"$block_size\" skip=\"$block_index\" count=1 2>/dev/null | base64 | tr -d '\\n'",
" ;;",
" write-b64-argv)",
" target=$1; expected_bytes=$2; expected_sha256=$3; shift 3",
" ensure_parent \"$target\"; base=${target##*/}; dir=.; case \"$target\" in */*) dir=${target%/*};; esac",
" tmp=\"$dir/.${base}.unidesk-transfer-$$.tmp\"; tmp_b64=\"$tmp.b64\"; : > \"$tmp_b64\"",
" for chunk in \"$@\"; do printf '%s' \"$chunk\" >> \"$tmp_b64\"; done",
" if ! base64 -d < \"$tmp_b64\" > \"$tmp\"; then rm -f -- \"$tmp\" \"$tmp_b64\"; printf 'transfer base64 decode failed for %s\\n' \"$target\" >&2; exit 22; fi",
" rm -f -- \"$tmp_b64\"; verify_tmp \"$target\" \"$tmp\" \"$expected_bytes\" \"$expected_sha256\"; mv -f -- \"$tmp\" \"$target\"",
" actual_sha256=$(sha256_file \"$target\"); if [ \"$actual_sha256\" != \"$expected_sha256\" ]; then printf 'transfer final sha256 mismatch for %s\\n' \"$target\" >&2; exit 25; fi",
" ;;",
" write-b64-stdin)",
" target=$1; expected_bytes=$2; expected_sha256=$3",
" ensure_parent \"$target\"; base=${target##*/}; dir=.; case \"$target\" in */*) dir=${target%/*};; esac",
" tmp=\"$dir/.${base}.unidesk-transfer-$$.tmp\"",
" if ! base64 -d > \"$tmp\"; then rm -f -- \"$tmp\"; printf 'transfer base64 decode failed for %s\\n' \"$target\" >&2; exit 22; fi",
" verify_tmp \"$target\" \"$tmp\" \"$expected_bytes\" \"$expected_sha256\"; mv -f -- \"$tmp\" \"$target\"",
" actual_sha256=$(sha256_file \"$target\"); if [ \"$actual_sha256\" != \"$expected_sha256\" ]; then printf 'transfer final sha256 mismatch for %s\\n' \"$target\" >&2; exit 25; fi",
" ;;",
" write-b64-begin)",
" target=$1; token=$2; ensure_parent \"$target\"; set_tmp_paths \"$target\" \"$token\"; : > \"$tmp_b64\"",
" ;;",
" write-b64-append-stdin)",
" target=$1; token=$2; set_tmp_paths \"$target\" \"$token\"; cat >> \"$tmp_b64\"",
" ;;",
" write-b64-commit)",
" target=$1; token=$2; expected_bytes=$3; expected_sha256=$4; set_tmp_paths \"$target\" \"$token\"",
" if ! base64 -d < \"$tmp_b64\" > \"$tmp\"; then rm -f -- \"$tmp\" \"$tmp_b64\"; printf 'transfer base64 decode failed for %s\\n' \"$target\" >&2; exit 22; fi",
" rm -f -- \"$tmp_b64\"; verify_tmp \"$target\" \"$tmp\" \"$expected_bytes\" \"$expected_sha256\"; mv -f -- \"$tmp\" \"$target\"",
" actual_sha256=$(sha256_file \"$target\"); if [ \"$actual_sha256\" != \"$expected_sha256\" ]; then printf 'transfer final sha256 mismatch for %s\\n' \"$target\" >&2; exit 25; fi",
" ;;",
" *) printf 'unsupported transfer op: %s\\n' \"$op\" >&2; exit 2;;",
"esac",
].join("\n");
}
function buildWindowsFileTransferCommand(
route: ParsedSshRoute,
builders: SshFileTransferCommandBuilders,
operation: SshFileTransferOperation,
args: string[],
): string {
return builders.buildWindowsPowerShellCommand(windowsFileTransferScript(route.workspace, operation, args));
}
function windowsFileTransferScript(basePath: string | null, operation: SshFileTransferOperation, args: string[]): string {
const target = args[0] ?? "";
const tokenOrBlock = args[1] ?? "";
const third = args[2] ?? "";
const fourth = args[3] ?? "";
const argvChunks = args.slice(3).map(powerShellSingleQuote).join(", ");
return [
"$ErrorActionPreference = 'Stop';",
"$ProgressPreference = 'SilentlyContinue';",
"[Console]::InputEncoding = [System.Text.UTF8Encoding]::new();",
"[Console]::OutputEncoding = [System.Text.UTF8Encoding]::new();",
"$OutputEncoding = [System.Text.UTF8Encoding]::new();",
`$basePath = ${powerShellSingleQuote(basePath ?? "")};`,
`$operation = ${powerShellSingleQuote(operation)};`,
`$targetArg = ${powerShellSingleQuote(target)};`,
`$arg1 = ${powerShellSingleQuote(tokenOrBlock)};`,
`$arg2 = ${powerShellSingleQuote(third)};`,
`$arg3 = ${powerShellSingleQuote(fourth)};`,
`$argvChunks = @(${argvChunks});`,
"function Fail([string]$Message, [int]$Code) { [Console]::Error.WriteLine($Message); exit $Code }",
"function Resolve-UnideskPath([string]$Raw) { if ([string]::IsNullOrWhiteSpace($Raw)) { Fail 'empty transfer path' 2 }; if ([System.IO.Path]::IsPathRooted($Raw)) { return [System.IO.Path]::GetFullPath($Raw) }; if (-not [string]::IsNullOrWhiteSpace($basePath)) { return [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($basePath, $Raw)) }; return [System.IO.Path]::GetFullPath($Raw) }",
"function Ensure-Parent([string]$Target) { $parent = [System.IO.Path]::GetDirectoryName($Target); if (-not [string]::IsNullOrWhiteSpace($parent)) { [System.IO.Directory]::CreateDirectory($parent) | Out-Null } }",
"function Get-Sha256([string]$Path) { return (Get-FileHash -LiteralPath $Path -Algorithm SHA256).Hash.ToLowerInvariant() }",
"function Set-TmpPaths([string]$Target, [string]$Token) { if ($Token -notmatch '^[A-Za-z0-9_.-]+$') { Fail 'invalid transfer temp token' 2 }; $dir = [System.IO.Path]::GetDirectoryName($Target); if ([string]::IsNullOrWhiteSpace($dir)) { $dir = (Get-Location).ProviderPath }; $base = [System.IO.Path]::GetFileName($Target); $script:tmp = [System.IO.Path]::Combine($dir, '.' + $base + '.unidesk-transfer-' + $Token + '.tmp'); $script:tmpB64 = $script:tmp + '.b64' }",
"function Verify-Temp([string]$Target, [string]$Tmp, [Int64]$ExpectedBytes, [string]$ExpectedSha256) { $actualBytes = ([System.IO.FileInfo]$Tmp).Length; if ($actualBytes -ne $ExpectedBytes) { Remove-Item -LiteralPath $Tmp -Force -ErrorAction SilentlyContinue; Fail ('transfer byte count mismatch for ' + $Target + ': expected=' + $ExpectedBytes + ' actual=' + $actualBytes) 23 }; $actualSha256 = Get-Sha256 $Tmp; if ($actualSha256 -ne $ExpectedSha256) { Remove-Item -LiteralPath $Tmp -Force -ErrorAction SilentlyContinue; Fail ('transfer sha256 mismatch for ' + $Target + ': expected=' + $ExpectedSha256 + ' actual=' + $actualSha256) 24 } }",
"function Decode-ToTarget([string]$Target, [string]$Encoded, [Int64]$ExpectedBytes, [string]$ExpectedSha256) { Ensure-Parent $Target; Set-TmpPaths $Target ([guid]::NewGuid().ToString('N')); try { $bytes = [Convert]::FromBase64String(($Encoded -replace '\\s','')) } catch { Fail ('transfer base64 decode failed for ' + $Target + ': ' + $_.Exception.Message) 22 }; [System.IO.File]::WriteAllBytes($script:tmp, $bytes); Verify-Temp $Target $script:tmp $ExpectedBytes $ExpectedSha256; Move-Item -LiteralPath $script:tmp -Destination $Target -Force; $actualSha256 = Get-Sha256 $Target; if ($actualSha256 -ne $ExpectedSha256) { Fail ('transfer final sha256 mismatch for ' + $Target) 25 } }",
"$target = Resolve-UnideskPath $targetArg;",
"switch ($operation) {",
" 'stat' { if (-not (Test-Path -LiteralPath $target -PathType Leaf)) { Fail ('file not found: ' + $target) 1 }; $bytes = ([System.IO.FileInfo]$target).Length; $digest = Get-Sha256 $target; [Console]::Out.WriteLine(([string]$bytes) + ' ' + $digest); break }",
" 'read-b64-block' { $blockIndex = [Int64]$arg1; $blockSize = [Int32]$arg2; if ($blockIndex -lt 0 -or $blockSize -le 0) { Fail 'invalid read block args' 2 }; $fs = [System.IO.File]::Open($target, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, [System.IO.FileShare]::ReadWrite); try { [void]$fs.Seek($blockIndex * $blockSize, [System.IO.SeekOrigin]::Begin); $buffer = New-Object byte[] $blockSize; $read = $fs.Read($buffer, 0, $blockSize); if ($read -gt 0) { [Console]::Out.Write([Convert]::ToBase64String($buffer, 0, $read)) } } finally { $fs.Dispose() }; break }",
" 'write-b64-argv' { Decode-ToTarget $target ([string]::Concat($argvChunks)) ([Int64]$arg1) $arg2; break }",
" 'write-b64-stdin' { Decode-ToTarget $target ([Console]::In.ReadToEnd()) ([Int64]$arg1) $arg2; break }",
" 'write-b64-begin' { Ensure-Parent $target; Set-TmpPaths $target $arg1; [System.IO.File]::WriteAllText($script:tmpB64, '', [System.Text.Encoding]::ASCII); break }",
" 'write-b64-append-stdin' { Set-TmpPaths $target $arg1; $chunk = ([Console]::In.ReadToEnd()) -replace '\\s',''; [System.IO.File]::AppendAllText($script:tmpB64, $chunk, [System.Text.Encoding]::ASCII); break }",
" 'write-b64-commit' { Set-TmpPaths $target $arg1; $encoded = [System.IO.File]::ReadAllText($script:tmpB64, [System.Text.Encoding]::ASCII); Remove-Item -LiteralPath $script:tmpB64 -Force -ErrorAction SilentlyContinue; Decode-ToTarget $target $encoded ([Int64]$arg2) $arg3; break }",
" default { Fail ('unsupported transfer op: ' + $operation) 2 }",
"}",
].join(" ");
}
function powerShellSingleQuote(value: string): string {
return `'${value.replace(/'/g, "''")}'`;
}