mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: preserve legacy text across trans helpers

Browse Source
This commit is contained in:
Codex
2026-07-10 19:41:44 +02:00
parent d07830dd17
commit 9f8e325ea5
8 changed files with 253 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/apply-patch-v2.ts
+78 -31
View File
@@ -63,10 +63,16 @@ export interface ApplyPatchV2FileSystem {
readBlock(path: string, blockIndex: number, blockBytes: number): Promise<Buffer>;
writeFile(path: string, content: Buffer): Promise<void>;
deleteFile(path: string): Promise<void>;
readFiles?(paths: string[]): Promise<Map<string, string>>;
readFiles?(paths: string[]): Promise<Map<string, ApplyPatchV2ReadFile | string>>;
applyReplacementsBulk?(paths: Iterable<string>, plans: Map<string, ApplyPatchV2BulkReplacementWritePlan>): Promise<void>;
}
export interface ApplyPatchV2ReadFile {
content: string;
bytes: Buffer;
encodingStyle: "utf8" | "utf8-bom" | "legacy-single-byte";
}
export interface ApplyPatchV2FileStat {
bytes: number;
sha256: string;
@@ -114,7 +120,7 @@ export class ApplyPatchV2Error extends Error {
}
}
type PlannedFileState = { exists: true; content: string } | { exists: false; content: "" };
type PlannedFileState = { exists: true; content: string; bytes: Buffer; encodingStyle: ApplyPatchV2ReadFile["encodingStyle"] } | { exists: false; content: "" };
type PlannedOperation = { kind: "write"; path: string; content: string } | { kind: "delete"; path: string };
export type ApplyPatchV2Replacement = [start: number, oldLength: number, newLines: string[]];
type Replacement = ApplyPatchV2Replacement;
@@ -125,7 +131,7 @@ export interface ApplyPatchV2BulkReplacementWritePlan {
finalBytes: number;
finalSha256: string;
newlineStyle: "lf" | "crlf" | "mixed" | "none";
encodingStyle: "utf8" | "utf8-bom";
encodingStyle: ApplyPatchV2ReadFile["encodingStyle"];
replacements: ApplyPatchV2Replacement[];
}
type BulkReplacementWritePlan = ApplyPatchV2BulkReplacementWritePlan;
@@ -653,7 +659,7 @@ async function applyPatchV2UpdateHunksBulk(executor: ApplyPatchV2Executor, hunks
const paths = uniqueStrings(hunks.map((hunk) => hunk.path));
const originalFiles = await readRemoteTextsBulk(executor, paths);
const states = new Map<string, PlannedFileState>();
for (const [filePath, content] of originalFiles) states.set(filePath, { exists: true, content });
for (const [filePath, file] of originalFiles) states.set(filePath, { exists: true, ...file });
const changed: string[] = [];
const outcomes: ApplyPatchV2Outcome[] = [];
@@ -667,12 +673,12 @@ async function applyPatchV2UpdateHunksBulk(executor: ApplyPatchV2Executor, hunks
const state = states.get(hunk.path);
if (state === undefined || !state.exists) throw new ApplyPatchV2Error("cannot update a missing file in bulk patch", { path: hunk.path });
const originalLines = splitContentLines(state.content);
const textFormat = detectApplyPatchTextFormat(state.content);
const textFormat = detectApplyPatchTextFormat(state.content, state.encodingStyle);
const replacements = preserveReplacementNewlines(computeReplacements(hunk.path, originalLines, hunk.chunks), textFormat.newlineStyle);
const newContent = joinLinesWithFinalNewline(applyReplacements(originalLines, replacements));
const originalBuffer = Buffer.from(state.content, "utf8");
const finalBuffer = Buffer.from(newContent, "utf8");
states.set(hunk.path, { exists: true, content: newContent });
const originalBuffer = state.bytes;
const finalBuffer = encodeApplyPatchText(newContent, state.encodingStyle, hunk.path);
states.set(hunk.path, { exists: true, content: newContent, bytes: finalBuffer, encodingStyle: state.encodingStyle });
replacementPlans.set(hunk.path, {
path: hunk.path,
originalBytes: originalBuffer.length,
@@ -1033,7 +1039,7 @@ async function readRemoteText(executor: ApplyPatchV2Executor, target: string): P
return contentBuffer.toString("utf8");
}
async function readRemoteTextsBulk(executor: ApplyPatchV2Executor, targets: string[]): Promise<Map<string, string>> {
async function readRemoteTextsBulk(executor: ApplyPatchV2Executor, targets: string[]): Promise<Map<string, ApplyPatchV2ReadFile>> {
if (targets.length === 0) return new Map();
if (executor.fs?.readFiles !== undefined) {
try {
@@ -1041,14 +1047,14 @@ async function readRemoteTextsBulk(executor: ApplyPatchV2Executor, targets: stri
for (const target of targets) {
if (!files.has(target)) throw new ApplyPatchV2Error("remote apply-patch v2 fs bulk read omitted target", { target });
}
return files;
return new Map(Array.from(files, ([path, file]) => [path, normalizeApplyPatchV2ReadFile(file)]));
} catch (error) {
if (!isBulkReadFailure(error)) throw error;
}
}
if (executor.fs || targets.length === 1) {
const files = new Map<string, string>();
for (const target of targets) files.set(target, await readRemoteText(executor, target));
const files = new Map<string, ApplyPatchV2ReadFile>();
for (const target of targets) files.set(target, decodeApplyPatchV2ReadFile(Buffer.from(await readRemoteText(executor, target), "utf8")));
return files;
}
const result = await checkedRemoteV2(executor, "read-bulk-b64", targets);
@@ -1099,7 +1105,7 @@ function decodeRemoteReadBlock(stdout: string, target: string, blockIndex: numbe
return decoded.length > expectedChunkBytes ? decoded.subarray(0, expectedChunkBytes) : decoded;
}
export function decodeApplyPatchV2BulkRead(stdout: string, targets: string[]): Map<string, string> {
export function decodeApplyPatchV2BulkRead(stdout: string, targets: string[]): Map<string, ApplyPatchV2ReadFile> {
const lines = stdout.split(/\r?\n/u);
const markerIndex = lines.findIndex((line) => line.startsWith(`${remoteBulkReadMarker} `));
if (markerIndex < 0) {
@@ -1113,7 +1119,7 @@ export function decodeApplyPatchV2BulkRead(stdout: string, targets: string[]): M
if (records.length !== declaredCount) {
throw new ApplyPatchV2Error("remote apply-patch v2 bulk read record count mismatch", { expectedTargets: targets.length, records: records.length });
}
const files = new Map<string, string>();
const files = new Map<string, ApplyPatchV2ReadFile>();
for (const record of records) {
const [pathB64, bytesText, sha256, contentB64] = record.split(/\s+/u);
const target = Buffer.from(pathB64, "base64").toString("utf8");
@@ -1132,7 +1138,7 @@ export function decodeApplyPatchV2BulkRead(stdout: string, targets: string[]): M
if (actualSha256 !== sha256) {
throw new ApplyPatchV2Error(`remote apply-patch v2 bulk read sha256 mismatch for ${target}`, { target, expectedSha256: sha256, actualSha256 });
}
files.set(target, contentBuffer.toString("utf8"));
files.set(target, decodeApplyPatchV2ReadFile(contentBuffer));
}
for (const target of targets) {
if (!files.has(target)) throw new ApplyPatchV2Error("remote apply-patch v2 bulk read omitted target", { target });
@@ -1188,7 +1194,7 @@ async function enrichBulkReplacementIntegrityFailure(error: unknown, fs: ApplyPa
const details = error instanceof ApplyPatchV2Error ? error.details : {};
const serialized = `${message}\n${JSON.stringify(details)}`;
if (fs.readFiles === undefined || !serialized.includes("integrity mismatch")) return error instanceof ApplyPatchV2Error ? error : new ApplyPatchV2Error(message);
let currentFiles: Map<string, string>;
let currentFiles: Map<string, ApplyPatchV2ReadFile | string>;
try {
currentFiles = await fs.readFiles(targets);
} catch (readError) {
@@ -1202,10 +1208,11 @@ async function enrichBulkReplacementIntegrityFailure(error: unknown, fs: ApplyPa
const adapter = route.includes(":win") ? "windows" : "posix";
const integrityDiagnostics = targets.flatMap((target) => {
const plan = plans.get(target);
const current = currentFiles.get(target);
if (plan === undefined || current === undefined) return [];
const currentBuffer = Buffer.from(current, "utf8");
const currentFormat = detectApplyPatchTextFormat(current);
const currentValue = currentFiles.get(target);
if (plan === undefined || currentValue === undefined) return [];
const current = normalizeApplyPatchV2ReadFile(currentValue);
const currentBuffer = current.bytes;
const currentFormat = detectApplyPatchTextFormat(current.content, current.encodingStyle);
return [{
adapter,
stage,
@@ -1245,6 +1252,7 @@ export function formatApplyPatchV2BulkReplacementPayload(paths: Iterable<string>
plan.originalSha256,
String(plan.finalBytes),
plan.finalSha256,
plan.encodingStyle,
replacementFields.join(";"),
].join(" "));
}
@@ -1364,11 +1372,14 @@ export function applyPatchV2RemoteCommand(operation: ApplyPatchV2RemoteOperation
" python3 - \"$tmp_payload\" \"$tmp_dir\" \"$expected_count\" <<'PY'",
"import base64, hashlib, pathlib, shutil, sys",
"def text_profile(source):",
" encoding = 'utf8-bom' if source.startswith(b'\\xef\\xbb\\xbf') else 'utf8'",
" text = source.decode('utf-8')",
" crlf = text.count('\\r\\n')",
" bare_lf = text.count('\\n') - crlf",
" lone_cr = text.count('\\r') - crlf",
" try:",
" source.decode('utf-8')",
" encoding = 'utf8-bom' if source.startswith(b'\\xef\\xbb\\xbf') else 'utf8'",
" except UnicodeDecodeError:",
" encoding = 'legacy-single-byte'",
" crlf = source.count(b'\\r\\n')",
" bare_lf = source.count(b'\\n') - crlf",
" lone_cr = source.count(b'\\r') - crlf",
" newline = 'crlf' if crlf > 0 and bare_lf == 0 and lone_cr == 0 else ('lf' if bare_lf > 0 and crlf == 0 and lone_cr == 0 else ('none' if crlf == 0 and bare_lf == 0 and lone_cr == 0 else 'mixed'))",
" return encoding, newline",
"def integrity_failure(adapter, stage, target, expected_bytes, expected_sha256, source):",
@@ -1384,16 +1395,17 @@ export function applyPatchV2RemoteCommand(operation: ApplyPatchV2RemoteOperation
"plans = []",
"for index, record in enumerate(records, start=1):",
" fields = record.split()",
" if len(fields) != 6:",
" if len(fields) != 7:",
" raise SystemExit('bulk replacement malformed record')",
" path_b64, original_bytes_text, original_sha256, final_bytes_text, final_sha256, replacements_text = fields",
" path_b64, original_bytes_text, original_sha256, final_bytes_text, final_sha256, encoding_style, replacements_text = fields",
" target = base64.b64decode(path_b64).decode('utf-8')",
" original_bytes = int(original_bytes_text)",
" final_bytes = int(final_bytes_text)",
" source = pathlib.Path(target).read_bytes()",
" if len(source) != original_bytes or hashlib.sha256(source).hexdigest() != original_sha256:",
" integrity_failure('posix', 'original', target, original_bytes, original_sha256, source)",
" lines = source.decode('utf-8').split('\\n')",
" source_encoding = 'latin-1' if encoding_style == 'legacy-single-byte' else 'utf-8'",
" lines = source.decode(source_encoding).split('\\n')",
" if lines and lines[-1] == '':",
" lines.pop()",
" replacements = []",
@@ -1410,7 +1422,7 @@ export function applyPatchV2RemoteCommand(operation: ApplyPatchV2RemoteOperation
" if start < 0 or old_len < 0 or start + old_len > len(lines):",
" raise SystemExit(f'bulk replacement out of bounds for {target}')",
" lines[start:start + old_len] = new_lines",
" output = (('\\n'.join(lines) + '\\n') if lines else '').encode('utf-8')",
" output = (('\\n'.join(lines) + '\\n') if lines else '').encode(source_encoding)",
" if len(output) != final_bytes or hashlib.sha256(output).hexdigest() != final_sha256:",
" integrity_failure('posix', 'final', target, final_bytes, final_sha256, output)",
" tmp_path = pathlib.Path(tmp_dir) / f'file-{index}.tmp'",
@@ -1636,7 +1648,7 @@ function splitContentLines(content: string): string[] {
return lines;
}
function detectApplyPatchTextFormat(content: string): Pick<ApplyPatchV2BulkReplacementWritePlan, "newlineStyle" | "encodingStyle"> {
function detectApplyPatchTextFormat(content: string, encodingStyle: ApplyPatchV2ReadFile["encodingStyle"] = content.startsWith("\uFEFF") ? "utf8-bom" : "utf8"): Pick<ApplyPatchV2BulkReplacementWritePlan, "newlineStyle" | "encodingStyle"> {
const crlfCount = (content.match(/\r\n/gu) ?? []).length;
const bareLfCount = (content.match(/(?<!\r)\n/gu) ?? []).length;
const loneCrCount = (content.match(/\r(?!\n)/gu) ?? []).length;
@@ -1649,10 +1661,45 @@ function detectApplyPatchTextFormat(content: string): Pick<ApplyPatchV2BulkRepla
: "mixed";
return {
newlineStyle,
encodingStyle: content.startsWith("\uFEFF") ? "utf8-bom" : "utf8",
encodingStyle,
};
}
function decodeApplyPatchV2ReadFile(contentBuffer: Buffer): ApplyPatchV2ReadFile {
try {
const content = new TextDecoder("utf-8", { fatal: true }).decode(contentBuffer);
return {
content,
bytes: contentBuffer,
encodingStyle: content.startsWith("\uFEFF") ? "utf8-bom" : "utf8",
};
} catch {
return {
content: contentBuffer.toString("latin1"),
bytes: contentBuffer,
encodingStyle: "legacy-single-byte",
};
}
}
function normalizeApplyPatchV2ReadFile(file: ApplyPatchV2ReadFile | string): ApplyPatchV2ReadFile {
return typeof file === "string" ? decodeApplyPatchV2ReadFile(Buffer.from(file, "utf8")) : file;
}
function encodeApplyPatchText(content: string, encodingStyle: ApplyPatchV2ReadFile["encodingStyle"], path: string): Buffer {
if (encodingStyle !== "legacy-single-byte") return Buffer.from(content, "utf8");
for (const character of content) {
if ((character.codePointAt(0) ?? 0) > 0xff) {
throw new ApplyPatchV2Error("legacy single-byte localized replacement only supports byte-preserving text", {
path,
encodingStyle,
replacementRequirement: "ASCII or existing single-byte characters",
});
}
}
return Buffer.from(content, "latin1");
}
function preserveReplacementNewlines(replacements: Replacement[], newlineStyle: ApplyPatchV2BulkReplacementWritePlan["newlineStyle"]): Replacement[] {
if (newlineStyle !== "crlf") return replacements;
return replacements.map(([start, oldLength, newLines]) => [