fix: retry truncated ssh download blocks

scripts/src/ssh-file-transfer.ts

@@ -206,9 +206,10 @@ async function readRemoteFileVerified(
   const chunks: Buffer[] = [];
   let actualBytes = 0;
   let chunkCount = 0;
-  for (let blockIndex = 0; actualBytes < remote.bytes; blockIndex += 1) {
-    const encoded = await readRemoteBase64BlockWithRetry(invocation, executor, builders, remotePath, readBlockBytes, blockIndex, remote.bytes, actualBytes);
-    const chunk = encoded.length === 0 ? Buffer.alloc(0) : Buffer.from(encoded, "base64");
+  for (let blockIndex = 0; blockIndex * readBlockBytes < remote.bytes; blockIndex += 1) {
+    const offsetBytes = blockIndex * readBlockBytes;
+    const expectedChunkBytes = Math.min(readBlockBytes, remote.bytes - offsetBytes);
+    const chunk = await readRemoteBase64BlockWithRetry(invocation, executor, builders, remotePath, readBlockBytes, blockIndex, expectedChunkBytes, remote.bytes, actualBytes);
     chunks.push(chunk);
     actualBytes += chunk.length;
     chunkCount += 1;
@@ -227,42 +228,62 @@ async function readRemoteBase64BlockWithRetry(
   remotePath: string,
   readBlockBytes: number,
   blockIndex: number,
+  expectedChunkBytes: number,
   expectedBytes: number,
   actualBytes: number,
-): Promise<string> {
+): Promise<Buffer> {
   const attemptErrors: Array<Record<string, unknown>> = [];
   for (let attempt = 1; attempt <= fileTransferReadBlockMaxAttempts; attempt += 1) {
     try {
       const read = await checkedFileTransfer(invocation, executor, builders, "read-b64-block", [remotePath, String(blockIndex), String(readBlockBytes)]);
       const encoded = read.stdout.replace(/\s+/gu, "");
-      if (encoded.length > 0) return encoded;
-      attemptErrors.push({ attempt, exitCode: read.exitCode, stdoutBytes: read.stdout.length, stderrTail: read.stderr.slice(-500) });
+      const chunk = decodeRemoteBase64Block(encoded);
+      if (chunk !== null && chunk.length === expectedChunkBytes) return chunk;
+      const reason = chunk === null ? "invalid-base64" : chunk.length === 0 ? "empty-read" : "short-read";
+      attemptErrors.push({
+        attempt,
+        reason,
+        exitCode: read.exitCode,
+        stdoutBytes: read.stdout.length,
+        encodedBytes: encoded.length,
+        decodedBytes: chunk?.length ?? null,
+        expectedChunkBytes,
+        stderrTail: read.stderr.slice(-500),
+      });
       if (attempt < fileTransferReadBlockMaxAttempts) {
-        emitEmptyReadRetryProgress(invocation, remotePath, blockIndex, attempt, expectedBytes, actualBytes);
+        emitReadRetryProgress(invocation, remotePath, blockIndex, attempt, expectedBytes, actualBytes, expectedChunkBytes, chunk?.length ?? null, reason);
         await delayMs(fileTransferReadEmptyRetryDelayMs);
       }
     } catch (error) {
       attemptErrors.push({
         attempt,
         error: error instanceof Error ? error.message : String(error),
+        expectedChunkBytes,
       });
       if (attempt < fileTransferReadBlockMaxAttempts) {
-        emitEmptyReadRetryProgress(invocation, remotePath, blockIndex, attempt, expectedBytes, actualBytes);
+        emitReadRetryProgress(invocation, remotePath, blockIndex, attempt, expectedBytes, actualBytes, expectedChunkBytes, null, "remote-error");
         await delayMs(fileTransferReadEmptyRetryDelayMs);
       }
     }
   }
-  throw new SshFileTransferError("remote download returned an empty block before EOF after retries", {
+  throw new SshFileTransferError("remote download returned an invalid block before EOF after retries", {
     route: invocation.route.raw,
     remotePath,
     blockIndex,
     attempts: fileTransferReadBlockMaxAttempts,
     expectedBytes,
     actualBytes,
+    expectedChunkBytes,
     attemptErrors,
   });
 }
 
+function decodeRemoteBase64Block(encoded: string): Buffer | null {
+  if (encoded.length === 0) return Buffer.alloc(0);
+  if (!/^[A-Za-z0-9+/]*={0,2}$/u.test(encoded) || encoded.length % 4 !== 0) return null;
+  return Buffer.from(encoded, "base64");
+}
+
 function emitDownloadProgress(
   invocation: ParsedSshInvocation,
   remotePath: string,
@@ -286,24 +307,31 @@ function emitDownloadProgress(
   })}\n`);
 }
 
-function emitEmptyReadRetryProgress(
+function emitReadRetryProgress(
   invocation: ParsedSshInvocation,
   remotePath: string,
   blockIndex: number,
   attempt: number,
   expectedBytes: number,
   actualBytes: number,
+  expectedChunkBytes: number,
+  actualChunkBytes: number | null,
+  reason: string,
 ): void {
+  const event = reason === "empty-read" ? "unidesk.ssh.download.empty-read-retry" : "unidesk.ssh.download.short-read-retry";
   process.stderr.write(`${JSON.stringify({
-    event: "unidesk.ssh.download.empty-read-retry",
+    event,
     route: invocation.route.raw,
     providerId: invocation.providerId,
     remotePath,
     blockIndex,
     attempt,
     maxAttempts: fileTransferReadBlockMaxAttempts,
+    reason,
     actualBytes,
     expectedBytes,
+    actualChunkBytes,
+    expectedChunkBytes,
   })}\n`);
 }