From 3eff8065e757c6e3bab96d164abfceafd340e6d1 Mon Sep 17 00:00:00 2001 From: Afloat Date: Thu, 5 Nov 2026 21:13:14 -0500 Subject: [PATCH] fix(process): preserve UTF-8 at diagnostic truncation boundaries (#8907) * fix(process): preserve UTF-8 at diagnostic truncation boundaries * fix(process): preserve malformed UTF-8 excerpt evidence --- src/process/BoundedOutputExcerpt.test.ts | 96 +++++++++++++++++++++++ src/process/BoundedOutputExcerpt.ts | 47 ++++++++++- src/process/ExecFileBoundedOutput.test.ts | 16 ++++ 3 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 src/process/BoundedOutputExcerpt.test.ts diff --git a/src/process/BoundedOutputExcerpt.test.ts b/src/process/BoundedOutputExcerpt.test.ts new file mode 100644 index 0000000000..d33315e738 --- /dev/null +++ b/src/process/BoundedOutputExcerpt.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; + +import { BoundedOutputExcerpt } from "./BoundedOutputExcerpt.js"; + +describe("bounded output excerpts", () => { + it.each([ + { text: `A🙂${"x".repeat(30)}🙂Z`, limit: 8, head: "A", tail: "Z" }, + { + text: `é中🙂${"x".repeat(30)}🙂中é`, + limit: 16, + head: "é中", + tail: "中é", + }, + { text: `é${"x".repeat(30)}中`, limit: 1, head: "", tail: "" }, + ])( + "omits only cut UTF-8 code points with a $limit byte budget", + ({ text, limit, head, tail }) => { + const bytes = Buffer.from(text); + for (let split = 0; split <= bytes.length; split += 1) { + const excerpt = new BoundedOutputExcerpt(limit); + excerpt.append(bytes.subarray(0, split)); + excerpt.append(bytes.subarray(split)); + expect(excerpt.truncated).toBe(true); + expect(excerpt.text).toBe(`${head} … [output omitted] … ${tail}`); + } + }, + ); + + it("rejoins multibyte characters across the head/tail boundary without truncation", () => { + const bytes = Buffer.from("A🙂"); + for (let split = 0; split <= bytes.length; split += 1) { + const excerpt = new BoundedOutputExcerpt(bytes.length); + excerpt.append(bytes.subarray(0, split)); + excerpt.append(bytes.subarray(split)); + expect(excerpt.truncated).toBe(false); + expect(excerpt.text).toBe("A🙂"); + } + const bytewise = new BoundedOutputExcerpt(bytes.length); + for (const byte of bytes) bytewise.append(Buffer.from([byte])); + expect(bytewise.truncated).toBe(false); + expect(bytewise.text).toBe("A🙂"); + }); + + it.each([ + { suffix: [0xc0], decoded: "�" }, + { suffix: [0xc1], decoded: "�" }, + { suffix: [0xf5], decoded: "�" }, + { suffix: [0xff], decoded: "�" }, + { suffix: [0x80], decoded: "�" }, + { suffix: [0xe0, 0x9f], decoded: "��" }, + { suffix: [0xed, 0xa0], decoded: "��" }, + { suffix: [0xf0, 0x8f], decoded: "��" }, + { suffix: [0xf4, 0x90], decoded: "��" }, + { suffix: [0xf0, 0x8f, 0x80], decoded: "���" }, + { suffix: [0xf4, 0x90, 0x80], decoded: "���" }, + { suffix: [0xc2], decoded: "" }, + { suffix: [0xe0], decoded: "" }, + { suffix: [0xe0, 0xa0], decoded: "" }, + { suffix: [0xed, 0x9f], decoded: "" }, + { suffix: [0xf0], decoded: "" }, + { suffix: [0xf0, 0x90], decoded: "" }, + { suffix: [0xf0, 0x90, 0x80], decoded: "" }, + { suffix: [0xf4, 0x8f], decoded: "" }, + { suffix: [0xf4, 0x8f, 0xbf], decoded: "" }, + ])( + "omits only valid incomplete UTF-8 head suffixes: $suffix", + ({ suffix, decoded }) => { + const head = Buffer.from([0x41, ...suffix]); + const tail = `${"B".repeat(head.length - 1)}Z`; + const bytes = Buffer.concat([ + head, + Buffer.from("x".repeat(30)), + Buffer.from(tail), + ]); + for (let split = 0; split <= bytes.length; split += 1) { + const excerpt = new BoundedOutputExcerpt(head.length * 2); + excerpt.append(bytes.subarray(0, split)); + excerpt.append(bytes.subarray(split)); + expect(excerpt.truncated).toBe(true); + expect(excerpt.text).toBe(`A${decoded} … [output omitted] … ${tail}`); + } + }, + ); + + it("preserves malformed output inside the retained head and tail", () => { + const excerpt = new BoundedOutputExcerpt(4); + excerpt.append( + Buffer.concat([ + Buffer.from([0xff, 0x41]), + Buffer.from("x".repeat(30)), + Buffer.from([0x42, 0xff]), + ]), + ); + expect(excerpt.text).toBe("�A … [output omitted] … B�"); + }); +}); diff --git a/src/process/BoundedOutputExcerpt.ts b/src/process/BoundedOutputExcerpt.ts index 439df43cb2..d65d96ba7b 100644 --- a/src/process/BoundedOutputExcerpt.ts +++ b/src/process/BoundedOutputExcerpt.ts @@ -39,8 +39,49 @@ export class BoundedOutputExcerpt { } get text(): string { - return this.truncated - ? `${this.#head.toString("utf8")} … [output omitted] … ${this.#tail.toString("utf8")}` - : Buffer.concat([this.#head, this.#tail]).toString("utf8"); + if (!this.truncated) + return Buffer.concat([this.#head, this.#tail]).toString("utf8"); + + const head = decodeTruncatedHead(this.#head); + let tailStart = 0; + while ( + tailStart < this.#tail.length && + (this.#tail[tailStart]! & 0xc0) === 0x80 + ) + tailStart += 1; + // A valid cut can leave at most three continuation bytes. Longer runs + // are malformed output, so retain their decoding evidence. + if (tailStart > 3) tailStart = 0; + return `${head} … [output omitted] … ${this.#tail.subarray(tailStart).toString("utf8")}`; } } + +/** Omit only a suffix that could still form a valid UTF-8 code point. */ +const decodeTruncatedHead = (bytes: Buffer): string => { + let start = bytes.length - 1; + while ( + start >= 0 && + bytes.length - start < 4 && + (bytes[start]! & 0xc0) === 0x80 + ) + start -= 1; + const first = bytes[start]; + if (first === undefined) return bytes.toString("utf8"); + const width = + first >= 0xc2 && first <= 0xdf + ? 2 + : first >= 0xe0 && first <= 0xef + ? 3 + : first >= 0xf0 && first <= 0xf4 + ? 4 + : 0; + const second = bytes[start + 1]; + // Exclude overlong encodings, surrogates and values beyond U+10FFFF. + const minimumSecond = first === 0xe0 ? 0xa0 : first === 0xf0 ? 0x90 : 0x80; + const maximumSecond = first === 0xed ? 0x9f : first === 0xf4 ? 0x8f : 0xbf; + const incomplete = + bytes.length - start < width && + (second === undefined || + (second >= minimumSecond && second <= maximumSecond)); + return bytes.toString("utf8", 0, incomplete ? start : bytes.length); +}; diff --git a/src/process/ExecFileBoundedOutput.test.ts b/src/process/ExecFileBoundedOutput.test.ts index 7c603bf241..50a22bb17b 100644 --- a/src/process/ExecFileBoundedOutput.test.ts +++ b/src/process/ExecFileBoundedOutput.test.ts @@ -3,6 +3,22 @@ import { describe, expect, it } from "vitest"; import { execFileBoundedOutput } from "./ExecFileBoundedOutput.js"; describe("execFileBoundedOutput", () => { + it("retains complete UTF-8 characters at both sides of truncated command output", async () => { + const text = `A🙂${"x".repeat(30)}🙂Z`; + const result = await execFileBoundedOutput( + process.execPath, + [ + "-e", + `process.stdout.write(${JSON.stringify(text)}); process.stderr.write(${JSON.stringify(text)});`, + ], + { env: {}, limit: 8 }, + ); + expect(result.exitCode).toBe(0); + expect(result.stdout).toBe("A … [output omitted] … Z"); + expect(result.stderr).toBe("A … [output omitted] … Z"); + expect(result.outputTruncated).toBe(true); + }); + it("lets a command finish after its output exceeds the capture limit", async () => { const result = await execFileBoundedOutput( process.execPath,