diff --git a/src/android/JadxWorkspaceRoot.test.ts b/src/android/JadxWorkspaceRoot.test.ts index e57cf1e405..ce53940983 100644 --- a/src/android/JadxWorkspaceRoot.test.ts +++ b/src/android/JadxWorkspaceRoot.test.ts @@ -30,7 +30,7 @@ afterEach(() => { vi.clearAllMocks(); }); -it("decodes a Windows guardian path across split UTF-8 stdout writes", async () => { +const mockGuardian = () => { const parent = "C:\\Users\\研究者\\AppData\\Local\\Temp"; const path = win32.join(parent, `rea-android-${"a".repeat(32)}`); const child = Object.assign(new EventEmitter(), { @@ -50,14 +50,18 @@ it("decodes a Windows guardian path across split UTF-8 stdout writes", async () }); lstatMock.mockRejectedValue(missingPath); spawnMock.mockReturnValue(child); - const ready = Buffer.from( - `${JSON.stringify({ - path, - volumeSerial: "0123456789abcdef", - fileId: "0123456789abcdef0123456789abcdef", - privateDacl: true, - })}\n`, - ); + const ready = JSON.stringify({ + path, + volumeSerial: "0123456789abcdef", + fileId: "0123456789abcdef0123456789abcdef", + privateDacl: true, + }); + return { child, path, ready }; +}; + +it("decodes a Windows guardian path across split UTF-8 stdout writes", async () => { + const { child, path, ready: line } = mockGuardian(); + const ready = Buffer.from(line + "\n"); const split = ready.indexOf(Buffer.from("研")) + 1; const start = createJadxWorkspaceRoot("win32"); @@ -75,3 +79,62 @@ it("decodes a Windows guardian path across split UTF-8 stdout writes", async () } expect(lstatMock).toHaveBeenCalledWith(path); }); + +it("limits only the first guardian frame and releases its stdout reader after readiness", async () => { + const { child, path, ready } = mockGuardian(); + const startup = createJadxWorkspaceRoot("win32"); + const frame = ready + " ".repeat(4095 - Buffer.byteLength(ready)) + "\n"; + child.stdout.write(frame + "diagnostics after readiness".repeat(200)); + + const root = await startup; + try { + expect(root.path).toBe(path); + expect(child.stdout.destroyed).toBe(false); + expect(child.stdout.listenerCount("data")).toBe(0); + child.stdout.emit("error", new Error("late guardian pipe error")); + expect(root.failureSignal?.aborted).toBe(false); + } finally { + await root.close(); + } + expect(lstatMock).toHaveBeenCalledWith(path); +}); + +it.each([ + { + reason: "a frame above the byte limit", + kind: "limit", + bytes: Buffer.from("测".repeat(1_400) + "\n"), + message: "Android workspace guardian readiness exceeded 4 KiB", + }, + { + reason: "invalid UTF-8", + kind: "malformed", + bytes: Buffer.from([0xff, 0x0a]), + message: "Android workspace guardian readiness is not valid UTF-8", + }, + { + reason: "an unfinished frame", + kind: "truncated", + bytes: Buffer.from("{}"), + message: "Android workspace guardian stdout ended before readiness", + }, +])("cleans up after $reason", async ({ bytes, kind, message }) => { + const { child } = mockGuardian(); + const startup = createJadxWorkspaceRoot("win32"); + child.stdout.end(bytes); + + await expect(startup).rejects.toMatchObject({ kind, message }); + expect(child.stdin.writableEnded).toBe(true); + expect(child.unref).toHaveBeenCalledOnce(); +}); + +it("retains the original pipe error during startup", async () => { + const { child } = mockGuardian(); + const startup = createJadxWorkspaceRoot("win32"); + const failure = new Error("guardian startup pipe error"); + child.stdout.destroy(failure); + + await expect(startup).rejects.toBe(failure); + expect(child.stdin.writableEnded).toBe(true); + expect(child.unref).toHaveBeenCalledOnce(); +}); diff --git a/src/android/JadxWorkspaceRoot.ts b/src/android/JadxWorkspaceRoot.ts index b04ad686d7..4b788f9a91 100644 --- a/src/android/JadxWorkspaceRoot.ts +++ b/src/android/JadxWorkspaceRoot.ts @@ -2,12 +2,20 @@ import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; import { lstat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { win32 } from "node:path"; +import type { Readable } from "node:stream"; import { fileURLToPath } from "node:url"; import { z } from "zod"; +import { invariantViolation } from "../domain/internalInvariant.js"; import { parseJsonWithDuplicatePolicy } from "../domain/jsonDuplicateMember.js"; +import { + decodeLimit, + malformedInput, + truncatedInput, +} from "../domain/parseFailure.js"; import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; +import { Utf8NdjsonLineBuffer } from "../process/Utf8NdjsonLineBuffer.js"; /** Match the native owner's ordinary spelling only for extended local drive paths. */ const localDrivePath = (path: string): string => @@ -23,6 +31,42 @@ const readySchema = z.strictObject({ privateDacl: z.literal(true), }); +const GUARDIAN_READINESS_BYTES = 4096; + +const readGuardianReadiness = async (stdout: Readable): Promise => { + const format = "android-workspace-guardian"; + // The guardian's byte limit includes the LF that completes the record. + const frames = new Utf8NdjsonLineBuffer(() => GUARDIAN_READINESS_BYTES - 1); + for await (const chunk of stdout.iterator({ destroyOnReturn: false })) { + if (!Buffer.isBuffer(chunk)) + throw invariantViolation( + "Android workspace guardian stdout did not provide bytes", + ); + for (const line of frames.pushRaw(chunk)) { + if (line.ok) return line.value; + if (line.error.failure.kind === "limit") + throw decodeLimit( + format, + "Android workspace guardian readiness exceeded 4 KiB", + { + name: "guardian-readiness-bytes", + maximum: GUARDIAN_READINESS_BYTES, + }, + { cause: line.error }, + ); + throw malformedInput( + format, + "Android workspace guardian readiness is not valid UTF-8", + { cause: line.error }, + ); + } + } + throw truncatedInput( + format, + "Android workspace guardian stdout ended before readiness", + ); +}; + /** The Android provider needs a path and a cleanup owner, regardless of host. */ export interface JadxWorkspaceRoot { readonly path: string; @@ -116,22 +160,10 @@ class WindowsJadxWorkspaceRoot implements JadxWorkspaceRoot { ); child.once("close", (code, signal) => resolve({ code, signal })); }); - const ready = new Promise((resolve, reject) => { - let frame = ""; - child.stdout.setEncoding("utf8"); - child.stdout.on("data", (chunk: string) => { - frame += chunk; - if (frame.length > 4096) { - reject( - new Error("Android workspace guardian readiness exceeded 4 KiB"), - ); - return; - } - const end = frame.indexOf("\n"); - if (end !== -1) resolve(frame.slice(0, end)); - }); - child.stdout.once("error", reject); - }); + // The readiness iterator observes errors during startup. Keep the pipe's + // error handler after readiness, while child exit owns lifetime failure. + child.stdout.on("error", () => undefined); + const ready = readGuardianReadiness(child.stdout); try { const line = await boundedWait( Promise.race([ diff --git a/src/browser/WebBundleAnalysisWorker.ts b/src/browser/WebBundleAnalysisWorker.ts index 353c20c893..e676db216a 100644 --- a/src/browser/WebBundleAnalysisWorker.ts +++ b/src/browser/WebBundleAnalysisWorker.ts @@ -20,11 +20,11 @@ import { import type { JavaScriptSemanticPrecisionInput } from "../domain/javascript/javascriptSemanticPrecision.js"; import { analysisErrorWithCancellation } from "../domain/analysisErrorCancellation.js"; import { WorkerFrameSignal } from "../process/WorkerFrameSignal.js"; +import { Utf8NdjsonLineBuffer } from "../process/Utf8NdjsonLineBuffer.js"; import { createHash, randomUUID } from "node:crypto"; import { open, rm } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { StringDecoder } from "node:string_decoder"; import { getHeapStatistics } from "node:v8"; import { ArtifactReaderFailure, @@ -189,8 +189,9 @@ export class WebBundleAnalysisWorker { readonly #runtime: OwnedWorkerRuntime; readonly #files = new ArtifactResourceScope(); readonly #cleanup: AnalysisCleanupLease; - #decoder = new StringDecoder("utf8"); - #pending = ""; + readonly #lines = new Utf8NdjsonLineBuffer( + () => WEB_BUNDLE_WORKER_LIMITS.protocolBytes, + ); #frames: WebBundleWorkerMessage[] = []; #protocolFailure: unknown; #observedFailure: WebBundleObservedFailure | undefined; @@ -206,7 +207,7 @@ export class WebBundleAnalysisWorker { try { // Refuse the chunk before appending it to any retained control text. if ( - Buffer.byteLength(chunk) + Buffer.byteLength(this.#pending) > + Buffer.byteLength(chunk) + this.#lines.pendingBytes > WEB_BUNDLE_WORKER_LIMITS.protocolBytes ) throw new RangeError( @@ -215,24 +216,21 @@ export class WebBundleAnalysisWorker { "control-frame budget", ), ); - this.#pending += - typeof chunk === "string" ? chunk : this.#decoder.write(chunk); - for (;;) { - const end = this.#pending.indexOf("\n"); - if (end < 0) break; + for (const line of this.#lines.pushRaw( + typeof chunk === "string" ? Buffer.from(chunk) : chunk, + )) { + if (!line.ok) throw line.error; if (this.#frames.length >= 16) throw new RangeError("Web analyzer exceeded its control-frame queue"); - this.#frames.push( - webBundleWorkerMessageSchema.parse( - JSON.parse(this.#pending.slice(0, end)), - ), + const frame = webBundleWorkerMessageSchema.parse( + JSON.parse(line.value), ); - this.#observeFrame(this.#frames.at(-1)!); - this.#pending = this.#pending.slice(end + 1); + this.#frames.push(frame); + this.#observeFrame(frame); } this.#frameSignal.wake("frame"); } catch (cause) { - this.#pending = ""; + this.#lines.reset(); this.#protocolFailure = cause; this.#latchFailure({ kind: "protocol", cause }); this.#frameSignal.wake("failure"); @@ -384,7 +382,7 @@ export class WebBundleAnalysisWorker { ownershipInspectionFailure = this.#inspectionFailure(cause, deadline); throw cause; } - if (this.#frames.length > 0 || this.#pending !== "") + if (this.#frames.length > 0 || this.#lines.pendingBytes !== 0) throw new AnalysisOutputError( OPERATION, "Web analyzer emitted unsolicited control data", @@ -995,8 +993,7 @@ export class WebBundleAnalysisWorker { #reset(): void { this.#ready = false; this.#frames = []; - this.#pending = ""; - this.#decoder = new StringDecoder("utf8"); + this.#lines.reset(); this.#protocolFailure = undefined; this.#observedFailure = undefined; this.#activeDeadline = undefined; diff --git a/src/browser/WebBundleAnalysisWorkerServer.test.ts b/src/browser/WebBundleAnalysisWorkerServer.test.ts index b96de558d5..0f633d1ca0 100644 --- a/src/browser/WebBundleAnalysisWorkerServer.test.ts +++ b/src/browser/WebBundleAnalysisWorkerServer.test.ts @@ -279,7 +279,7 @@ it("serves split and coalesced LF/CRLF requests as separate jobs", async () => { it.each([ ["multibyte", Buffer.from("é🙂")], - ["replacement-decoded", Buffer.from([0xff])], + ["encoded replacement character", Buffer.from("�")], ] as const)( "accepts an exact-budget %s JSON request written in split UTF-8 and CRLF writes", async (_name, text) => { @@ -302,14 +302,11 @@ it.each([ text, Buffer.from(`",${request.slice(1)}`), ]); - const decodedBytes = Buffer.byteLength(json.toString("utf8")); const frame = Buffer.concat([ json, - Buffer.alloc(WEB_BUNDLE_WORKER_LIMITS.protocolBytes - decodedBytes, 0x20), + Buffer.alloc(WEB_BUNDLE_WORKER_LIMITS.protocolBytes - json.length, 0x20), ]); - expect(Buffer.byteLength(frame.toString("utf8"))).toBe( - WEB_BUNDLE_WORKER_LIMITS.protocolBytes, - ); + expect(frame.length).toBe(WEB_BUNDLE_WORKER_LIMITS.protocolBytes); await writeInput(server.process, frame.subarray(0, prefix.length + 1)); await writeInput( server.process, @@ -324,7 +321,16 @@ it.each([ }, ); -it("rejects an oversized unterminated frame at EOF without starting analysis", async () => { +it.each([ + [ + "oversized EOF", + "RangeError: Web bundle worker request exceeds its byte budget", + ], + [ + "malformed UTF-8", + "TypeError: Web bundle worker request is not valid UTF-8", + ], +] as const)("rejects %s without starting analysis", async (kind, failure) => { const server = await framingServer(); const input = sourceDescriptor(makeScript("fetch('/not-analyzed');")); const id = randomUUID(); @@ -336,18 +342,21 @@ it("rejects an oversized unterminated frame at EOF without starting analysis", a source: input.descriptor, result_budget_bytes: WEB_BUNDLE_WORKER_LIMITS.resultBytes, }); - const frame = request.padEnd(WEB_BUNDLE_WORKER_LIMITS.protocolBytes + 1, " "); + const frame = + kind === "oversized EOF" + ? request.padEnd(WEB_BUNDLE_WORKER_LIMITS.protocolBytes + 1, " ") + : Buffer.concat([ + Buffer.from('{"kind":"'), + Buffer.of(0xff), + Buffer.from(`",${request.slice(1)}\n`), + ]); const exited = once(server.process, "exit"); server.process.stdin.end(frame); - await expect(server.next()).rejects.toThrow( - "RangeError: Web bundle worker request exceeds its byte budget", - ); + await expect(server.next()).rejects.toThrow(failure); expect(await exited).toEqual([1, null]); await expect( readFile(join(server.root, `${id}.result`)), - ).rejects.toMatchObject({ - code: "ENOENT", - }); + ).rejects.toMatchObject({ code: "ENOENT" }); }); it("bounds and binds per-script worker jobs, then reuses one worker for the next script", async () => { diff --git a/src/browser/WebBundleAnalysisWorkerServer.ts b/src/browser/WebBundleAnalysisWorkerServer.ts index 08c586ccf8..69d0768dad 100644 --- a/src/browser/WebBundleAnalysisWorkerServer.ts +++ b/src/browser/WebBundleAnalysisWorkerServer.ts @@ -85,10 +85,13 @@ export const serveWebBundleAnalysisWorker = async ( for (;;) { const next = await requests.next(); if (next.done) break; - if (!next.value.ok) + if (!next.value.ok) { + if (next.value.error._tag === "ControlLineMalformedUtf8") + throw new TypeError("Web bundle worker request is not valid UTF-8"); throw new RangeError( "Web bundle worker request exceeds its byte budget", ); + } const raw: unknown = JSON.parse(next.value.value); const request = webBundleWorkerRequestSchema.parse(raw); const continueServing = await analyzeJob( diff --git a/src/desktop/AgentEnvComputerHelper.ts b/src/desktop/AgentEnvComputerHelper.ts index 2eca88f157..7778f8b03b 100644 --- a/src/desktop/AgentEnvComputerHelper.ts +++ b/src/desktop/AgentEnvComputerHelper.ts @@ -267,7 +267,12 @@ export class ComputerHelper { if (first !== undefined) continue; first = line.ok ? parseHandshake(line.value) - : { error: "the helper's handshake line is too long" }; + : { + error: + line.error._tag === "ControlLineMalformedUtf8" + ? "the helper's handshake is not valid UTF-8" + : "the helper's handshake line is too long", + }; if ("sandbox_id" in first) { this.#sandboxId = first.sandbox_id; this.#token = first.control_token; diff --git a/src/desktop/AgentEnvSolveHelper.ts b/src/desktop/AgentEnvSolveHelper.ts index 7c9d8f37b0..63c30b8bf3 100644 --- a/src/desktop/AgentEnvSolveHelper.ts +++ b/src/desktop/AgentEnvSolveHelper.ts @@ -236,7 +236,10 @@ export const runSolveHelper = async (options: { report ??= line.ok ? parseReport(line.value) : { - error: "REA's solve helper's report is too long", + error: + line.error._tag === "ControlLineMalformedUtf8" + ? "REA's solve helper's report is not valid UTF-8" + : "REA's solve helper's report is too long", cleanup: { unknown: true, sandboxes: [] }, }; })().catch(() => undefined); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts index 3d668b8a64..54c9eba8f7 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -24,6 +24,10 @@ import { safeParseUtf8Json } from "../../domain/safeJson.js"; import { Budget } from "../../domain/budget.js"; import { analysisSettingLimit } from "../../domain/analysisSettingDefinitions.js"; import { WorkerFrameSignal } from "../../process/WorkerFrameSignal.js"; +import { + Utf8NdjsonLineBuffer, + Utf8NdjsonOversizeError, +} from "../../process/Utf8NdjsonLineBuffer.js"; import type { JavaScriptSemanticProjectionCapacity, JavaScriptSemanticFileProjection, @@ -32,7 +36,6 @@ import { randomUUID } from "node:crypto"; import { readFile, rm, stat } from "node:fs/promises"; import { join } from "node:path"; import { writeJavaScriptAnalysisSourceTransfer } from "./JavaScriptAnalysisSourceTransfer.js"; -import { StringDecoder } from "node:string_decoder"; import { fileURLToPath } from "node:url"; import { getHeapStatistics } from "node:v8"; import { z } from "zod"; @@ -148,8 +151,7 @@ export class JavaScriptAnalysisWorker { #heapLimitBytes = 0; #frames: JavaScriptWorkerMessage[] = []; readonly #frameSignal = new WorkerFrameSignal(); - #pending = ""; - #decoder = new StringDecoder("utf8"); + readonly #lines = new Utf8NdjsonLineBuffer(() => DEFAULTS.protocolBytes); #primaryFailure: WorkerPrimaryFailure | undefined; #activeJobId: string | undefined; #activeJobPhase = "startup"; @@ -181,33 +183,30 @@ export class JavaScriptAnalysisWorker { if (this.#deadline?.interruption !== undefined) return; this.#observeMemoryLimit(); try { - this.#pending += - typeof chunk === "string" ? chunk : this.#decoder.write(chunk); - let newline: number; - while ((newline = this.#pending.indexOf("\n")) >= 0) { - const line = this.#pending.slice(0, newline); - this.#pending = this.#pending.slice(newline + 1); - if ( - Buffer.byteLength(line) > DEFAULTS.protocolBytes || - this.#frames.length >= 16 - ) { + for (const line of this.#lines.pushRaw( + typeof chunk === "string" ? Buffer.from(chunk) : chunk, + )) { + if (!line.ok) throw line.error; + if (this.#frames.length >= 16) { this.#latchOutputFailure( "JavaScript worker exceeded its control-frame budget", ); return; } - const raw: unknown = JSON.parse(line); + const raw: unknown = JSON.parse(line.value); const frame = javaScriptWorkerMessageSchema.parse(raw); this.#frames.push(frame); this.#observeJobFrame(frame); this.#frameSignal.wake("frame"); } - if (Buffer.byteLength(this.#pending) > DEFAULTS.protocolBytes) - this.#latchOutputFailure( - "JavaScript worker control frame is unterminated or oversized", - ); } catch (cause: unknown) { - this.#latchOutputFailure(errorMessage(cause)); + this.#latchOutputFailure( + cause instanceof Utf8NdjsonOversizeError + ? cause.boundary === "tail" + ? "JavaScript worker control frame is unterminated or oversized" + : "JavaScript worker exceeded its control-frame budget" + : errorMessage(cause), + ); } }; @@ -277,7 +276,7 @@ export class JavaScriptAnalysisWorker { this.#primaryFailure === undefined && this.#lastFailure === undefined && this.#frames.length === 0 && - this.#pending === "" && + this.#lines.pendingBytes === 0 && this.#runtime.inputFailure === undefined && !this.#runtime.hasPendingCleanup && !this.#inspectionResources.hasPending && @@ -377,7 +376,7 @@ export class JavaScriptAnalysisWorker { const supervisor = this.#runtime.supervisor; if (rootPath === undefined || supervisor === undefined) throw new TypeError("JavaScript worker resources were not acquired"); - if (this.#frames.length !== 0 || this.#pending !== "") + if (this.#frames.length !== 0 || this.#lines.pendingBytes !== 0) throw new AnalysisOutputError( OPERATION, "JavaScript worker has unsolicited or incomplete control frames", @@ -1546,8 +1545,7 @@ export class JavaScriptAnalysisWorker { this.#ready = false; this.#frames = []; this.#frameSignal.clear(); - this.#pending = ""; - this.#decoder = new StringDecoder("utf8"); + this.#lines.reset(); this.#primaryFailure = undefined; this.#activeJobId = undefined; this.#staticResponseId = undefined; diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts index 8f087b27fc..fbf279b883 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts @@ -140,48 +140,67 @@ it("consumes fragmented CRLF admission and coalesced next-job frames in order", } }); -it("drains oversized checkpoint admission, reports output failure, and serves the next job", async () => { - const server = await startServer(); - try { - await server.next(); - const first = await server.request(); - const second = await server.request(); - server.child.stdin.write(frame(first)); - expect(await server.next()).toMatchObject({ kind: "static", id: first.id }); - server.child.stdin.write( - "x".repeat(JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS.protocolBytes), - ); - server.child.stdin.write( - `x\n${frame(second)}${frame(admission(second.id))}`, - ); - expect(await server.next()).toEqual({ - kind: "failure", - id: first.id, - reason: "output", - resource: null, - limits: null, - message: expect.stringMatching(/admission frame.*missing or oversized/iu), - }); - expect(await server.next()).toMatchObject({ - kind: "static", - id: second.id, - }); - expect(await server.next()).toMatchObject({ - kind: "phase", - id: second.id, - phase: "transfer", - }); - expect(await server.next()).toEqual({ - kind: "complete", - id: second.id, - descriptor: null, - }); - server.child.stdin.end(); - expect((await server.closed)[0]).toBe(0); - } finally { - await server.close(); - } -}); +it.each([ + ["oversized", /admission frame.*missing or oversized/iu], + ["malformed UTF-8", /admission frame.*not valid UTF-8/iu], +] as const)( + "reports %s checkpoint admission and serves the next job", + async (kind, failure) => { + const server = await startServer(); + try { + await server.next(); + const first = await server.request(); + const second = await server.request(); + server.child.stdin.write(frame(first)); + expect(await server.next()).toMatchObject({ + kind: "static", + id: first.id, + }); + if (kind === "oversized") { + server.child.stdin.write( + "x".repeat(JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS.protocolBytes + 1), + ); + } else { + server.child.stdin.write( + Buffer.concat([ + Buffer.from('{"id":"'), + Buffer.of(0xff), + Buffer.from(`",${JSON.stringify(admission(first.id)).slice(1)}`), + ]), + ); + } + server.child.stdin.write( + `\n${frame(second)}${frame(admission(second.id))}`, + ); + expect(await server.next()).toEqual({ + kind: "failure", + id: first.id, + reason: "output", + resource: null, + limits: null, + message: expect.stringMatching(failure), + }); + expect(await server.next()).toMatchObject({ + kind: "static", + id: second.id, + }); + expect(await server.next()).toMatchObject({ + kind: "phase", + id: second.id, + phase: "transfer", + }); + expect(await server.next()).toEqual({ + kind: "complete", + id: second.id, + descriptor: null, + }); + server.child.stdin.end(); + expect((await server.closed)[0]).toBe(0); + } finally { + await server.close(); + } + }, +); it("reports missing checkpoint admission as output failure at stdin EOF", async () => { const server = await startServer(); @@ -227,3 +246,25 @@ it.each(["lf", "eof", "open"] as const)( } }, ); + +it("rejects a valid-shaped initial request containing invalid UTF-8", async () => { + const server = await startServer(); + try { + await server.next(); + const request = await server.request(); + const [before, after] = frame(request).split("app.mjs"); + server.child.stdin.end( + Buffer.concat([ + Buffer.from(before ?? ""), + Buffer.of(0xff), + Buffer.from(after ?? ""), + ]), + ); + expect((await server.closed)[0]).not.toBe(0); + expect(server.stderr()).toContain( + "TypeError: JavaScript worker request is not valid UTF-8", + ); + } finally { + await server.close(); + } +}); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts index 6475f17588..9e9e4e6b70 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts @@ -95,10 +95,13 @@ export const serveJavaScriptAnalysisWorker = async ( for (;;) { const next = await requests.next(); if (next.done) break; - if (!next.value.ok) + if (!next.value.ok) { + if (next.value.error._tag === "ControlLineMalformedUtf8") + throw new TypeError("JavaScript worker request is not valid UTF-8"); throw new RangeError( "JavaScript worker request exceeds its byte budget", ); + } const raw: unknown = JSON.parse(next.value.value); const request = javaScriptWorkerRequestSchema.parse(raw); // Function scope releases source AST/IR/results before accepting another job. @@ -182,6 +185,14 @@ const analyzeJob = async ( sha256: sha256Hex(text), }); const next = await requests.next(); + if ( + !next.done && + !next.value.ok && + next.value.error._tag === "ControlLineMalformedUtf8" + ) + throw new TypeError( + "JavaScript worker admission frame is not valid UTF-8", + ); if (next.done || !next.value.ok) throw new TypeError( "JavaScript worker admission frame is missing or oversized", diff --git a/src/process/BoundedControlLines.test.ts b/src/process/BoundedControlLines.test.ts index 5949562d8a..dcac1cb8e0 100644 --- a/src/process/BoundedControlLines.test.ts +++ b/src/process/BoundedControlLines.test.ts @@ -24,6 +24,7 @@ const collect = async ( }; const exceeded: BoundedControlLine = err({ _tag: "ControlLineLimitExceeded" }); +const malformed: BoundedControlLine = err({ _tag: "ControlLineMalformedUtf8" }); const multibyte = Buffer.from("é😀\n", "utf8"); const exactCrLf = Buffer.from("éa\r\n", "utf8"); const crLfPartitions: { first: number; second: number }[] = []; @@ -61,14 +62,14 @@ describe("bounded control line decoding", () => { ); it.each([ - { bytes: [0xff, 0x0a], maximum: 3, expected: [ok("�")] }, - { bytes: [0xff, 0x0a], maximum: 2, expected: [exceeded] }, - { bytes: [0xe2, 0x82], maximum: 3, expected: [ok("�")] }, - { bytes: [0xe2, 0x82], maximum: 2, expected: [exceeded] }, - { bytes: [0xe2, 0x28, 0xa1, 0x0a], maximum: 7, expected: [ok("�(�")] }, - { bytes: [0xe2, 0x28, 0xa1, 0x0a], maximum: 6, expected: [exceeded] }, + { bytes: [0xff, 0x0a], maximum: 1, expected: [malformed] }, + { bytes: [0xff, 0x0a], maximum: 0, expected: [exceeded] }, + { bytes: [0xe2, 0x82], maximum: 2, expected: [malformed] }, + { bytes: [0xe2, 0x82], maximum: 1, expected: [exceeded] }, + { bytes: [0xe2, 0x28, 0xa1, 0x0a], maximum: 3, expected: [malformed] }, + { bytes: [0xe2, 0x28, 0xa1, 0x0a], maximum: 2, expected: [exceeded] }, ])( - "counts replacement characters against decoded bytes: $bytes / $maximum", + "distinguishes invalid UTF-8 from wire byte overflow: $bytes / $maximum", async ({ bytes, maximum, expected }) => { const encoded = Buffer.from(bytes); expect(await collect([encoded], maximum)).toEqual(expected); @@ -88,6 +89,7 @@ describe("bounded control line boundaries", () => { { text: "\n\r\n", maximum: 0, expected: [ok(""), ok("")] }, { text: " \t \n", maximum: 3, expected: [ok(" \t ")] }, { text: "a\rb\n", maximum: 3, expected: [ok("a\rb")] }, + { text: "\uFEFF�\n", maximum: 6, expected: [ok("\uFEFF�")] }, { text: "abc", maximum: 3, expected: [ok("abc")] }, { text: "ab\r", maximum: 3, expected: [ok("ab\r")] }, { text: "abc\r", maximum: 3, expected: [exceeded] }, @@ -124,10 +126,10 @@ describe("bounded control line boundaries", () => { }, ); - it("recovers from malformed-byte expansion before the following frame", async () => { + it("reports malformed wire bytes before the following valid frame", async () => { expect( await collect([Uint8Array.of(0xff, 0x0a, 0x6f, 0x6b, 0x0a)], 2), - ).toEqual([exceeded, ok("ok")]); + ).toEqual([malformed, ok("ok")]); }); }); diff --git a/src/process/BoundedControlLines.ts b/src/process/BoundedControlLines.ts index 376450dbd4..b52095947f 100644 --- a/src/process/BoundedControlLines.ts +++ b/src/process/BoundedControlLines.ts @@ -1,87 +1,78 @@ -import { StringDecoder } from "node:string_decoder"; import { err, ok, type Result } from "../domain/result.js"; +import { strictUtf8Decoder } from "../domain/strictUtf8.js"; export type BoundedControlLine = Result< string, - { readonly _tag: "ControlLineLimitExceeded" } + { readonly _tag: "ControlLineLimitExceeded" | "ControlLineMalformedUtf8" } >; +const exceeded = err({ _tag: "ControlLineLimitExceeded" } as const); +const malformed = err({ _tag: "ControlLineMalformedUtf8" } as const); +const LF = 0x0a; +const CR = 0x0d; + /** - * Read LF/CRLF control frames without retaining an oversized line. Limits apply - * to decoded UTF-8 content, excluding the delimiter. Oversize is reported once - * as soon as it is known; rejected content is drained before the next frame. - * The caller owns failure classification and whether to continue reading. - * The supplied iterator owns the input stream's early-return cleanup policy. + * Frame wire bytes before decoding. The limit excludes LF or CRLF. Report + * overflow as soon as it is known, then drain to LF before the next frame. + * The caller owns failure policy and the input iterator's early-return cleanup. */ export async function* readBoundedControlLines( chunks: AsyncIterable, maxBytes: number, ): AsyncGenerator { - const decoder = new StringDecoder("utf8"); - let pending = ""; + const decoder = strictUtf8Decoder("preserve"); + let pending: Uint8Array | undefined; let pendingBytes = 0; let oversized = false; - const append = (fragment: string): boolean => { - if (oversized || fragment.length === 0) return false; - const bytes = pendingBytes + Buffer.byteLength(fragment); - // A final CR might be half of a split CRLF. Admit it provisionally, then - // count it as content if another character or EOF follows instead of LF. - if (bytes > maxBytes + (fragment.endsWith("\r") ? 1 : 0)) { - oversized = true; - pending = ""; - pendingBytes = 0; - return true; + const decode = (bytes: Uint8Array): BoundedControlLine => { + if (bytes.length > maxBytes) return exceeded; + try { + return ok(decoder.decode(bytes)); + } catch (cause: unknown) { + if (!(cause instanceof TypeError)) throw cause; + return malformed; } - pending += fragment; - pendingBytes = bytes; - return false; }; - const finish = (delimiter: "lf" | "eof"): BoundedControlLine => { - const line = - delimiter === "lf" && pending.endsWith("\r") - ? pending.slice(0, -1) - : pending; - const result = - oversized || Buffer.byteLength(line) > maxBytes - ? err({ _tag: "ControlLineLimitExceeded" } as const) - : ok(line); - pending = ""; - pendingBytes = 0; - oversized = false; - return result; - }; - - function* consume(decoded: string): Generator { + for await (const chunk of chunks) { let start = 0; - for (;;) { - const newline = decoded.indexOf("\n", start); - if (newline < 0) { - if (append(decoded.slice(start))) - yield err({ _tag: "ControlLineLimitExceeded" }); - return; - } - if (append(decoded.slice(start, newline))) - yield err({ _tag: "ControlLineLimitExceeded" }); - if (!oversized) yield finish("lf"); - else { - pending = ""; - pendingBytes = 0; - oversized = false; + while (start < chunk.length) { + const newline = chunk.indexOf(LF, start); + const end = newline === -1 ? chunk.length : newline; + if (!oversized) { + const fragment = chunk.subarray(start, end); + const count = pendingBytes + fragment.length; + const last = + fragment.length > 0 + ? fragment[fragment.length - 1] + : pending?.[pendingBytes - 1]; + // The byte after the limit can still be CR awaiting LF in the next chunk. + if (count > maxBytes + (last === CR ? 1 : 0)) { + pendingBytes = 0; + oversized = true; + yield exceeded; + } else if (newline !== -1) { + const length = count - (last === CR ? 1 : 0); + if (pending === undefined || pendingBytes === 0) + yield decode(fragment.subarray(0, length)); + else { + pending.set(fragment, pendingBytes); + yield decode(pending.subarray(0, length)); + } + pendingBytes = 0; + } else if (fragment.length > 0) { + pending ??= new Uint8Array(maxBytes + 1); + pending.set(fragment, pendingBytes); + pendingBytes = count; + } } + if (newline === -1) break; + oversized = false; start = newline + 1; } } - for await (const chunk of chunks) { - // Bound decoding expansion as well as retained line content. A malformed - // source byte can expand to a three-byte UTF-8 replacement character. - for (let offset = 0; offset < chunk.byteLength; offset += maxBytes + 1) - yield* consume( - decoder.write(chunk.subarray(offset, offset + maxBytes + 1)), - ); - } - yield* consume(decoder.end()); - if (!oversized && pending.length > 0) yield finish("eof"); + if (pending !== undefined && pendingBytes > 0) + yield decode(pending.subarray(0, pendingBytes)); } diff --git a/src/process/Utf8NdjsonLineBuffer.test.ts b/src/process/Utf8NdjsonLineBuffer.test.ts index 098c25f728..726cc26066 100644 --- a/src/process/Utf8NdjsonLineBuffer.test.ts +++ b/src/process/Utf8NdjsonLineBuffer.test.ts @@ -137,7 +137,7 @@ describe("UTF-8 NDJSON line buffer", () => { const refused = [...lines.push(Buffer.from("y"))]; expect(refused).toHaveLength(1); const refusal = refused[0]?.ok ? undefined : refused[0]?.error; - expect(refusal).toEqual(new Utf8NdjsonOversizeError(16, 17)); + expect(refusal).toEqual(new Utf8NdjsonOversizeError(16, 17, "tail")); // The refusal is the shared limit parse failure, with the observed bytes. expect(refusal).toBeInstanceOf(ParseFailureError); expect(refusal?.failure).toMatchObject({ diff --git a/src/process/Utf8NdjsonLineBuffer.ts b/src/process/Utf8NdjsonLineBuffer.ts index 6953583f0d..f57a024121 100644 --- a/src/process/Utf8NdjsonLineBuffer.ts +++ b/src/process/Utf8NdjsonLineBuffer.ts @@ -11,6 +11,7 @@ export class Utf8NdjsonOversizeError extends ParseFailureError<"limit"> { constructor( readonly maximumBytes: number, readonly observedBytes: number, + readonly boundary: "line" | "tail" = "line", ) { super({ kind: "limit", @@ -55,8 +56,22 @@ export class Utf8NdjsonLineBuffer { * the iteration. A line or the tail still waiting for its LF above the * ceiling is yielded as `Utf8NdjsonOversizeError`, after discarding every * buffered byte, and ends the iteration. + * This bridge projection trims U+FEFF and other ECMAScript whitespace. + * Exact JSON parsers use `pushRaw` to keep the decoded line intact. */ *push(chunk: Buffer): Generator { + for (const line of this.pushRaw(chunk)) { + if (!line.ok) { + yield line; + return; + } + const trimmed = line.value.trim(); + if (trimmed.length > 0) yield ok(trimmed); + } + } + + /** Deliver exact decoded lines, including blanks and leading U+FEFF. */ + *pushRaw(chunk: Buffer): Generator { const maximumBytes = this.#maxLineBytes(); this.#fragments.push(chunk); this.#fragmentBytes += chunk.length; @@ -82,12 +97,8 @@ export class Utf8NdjsonLineBuffer { this.#buffer = rest.length === 0 ? Buffer.alloc(0) : rest; this.#nextNewline = this.#buffer.indexOf(10); const line = this.#decode(encodedLine); - if (!line.ok) { - yield line; - return; - } - const trimmed = line.value.trim(); - if (trimmed.length > 0) yield ok(trimmed); + yield line; + if (!line.ok) return; } const refused = this.#refuseTail(maximumBytes); if (refused !== undefined) yield err(refused); @@ -102,12 +113,12 @@ export class Utf8NdjsonLineBuffer { } #refuseTail(maximumBytes: number): Utf8NdjsonOversizeError | undefined { - const pendingBytes = this.#buffer.length + this.#fragmentBytes; + const pendingBytes = this.pendingBytes; if (this.#nextNewline >= 0 || pendingBytes <= maximumBytes) return undefined; // Drop the refused bytes rather than retaining them until reconnect. this.reset(); - return new Utf8NdjsonOversizeError(maximumBytes, pendingBytes); + return new Utf8NdjsonOversizeError(maximumBytes, pendingBytes, "tail"); } #decode(bytes: Buffer): Result> { @@ -128,12 +139,16 @@ export class Utf8NdjsonLineBuffer { this.#nextNewline = -1; } + get pendingBytes(): number { + return this.#buffer.length + this.#fragmentBytes; + } + /** Bytes still held after complete lines are removed. An empty tail keeps no drained frame. */ retainedTail(): { readonly pendingBytes: number; readonly backingBytes: number; } { - const pendingBytes = this.#buffer.length + this.#fragmentBytes; + const pendingBytes = this.pendingBytes; const backingBytes = this.#buffer.buffer.byteLength + this.#fragments.reduce( diff --git a/tests/boundary/javascript/workerControlUtf8.test.ts b/tests/boundary/javascript/workerControlUtf8.test.ts new file mode 100644 index 0000000000..328a7929a8 --- /dev/null +++ b/tests/boundary/javascript/workerControlUtf8.test.ts @@ -0,0 +1,118 @@ +import { expect, it } from "vitest"; + +import { WebBundleAnalysisWorker } from "../../../src/browser/WebBundleAnalysisWorker.js"; +import { + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../../../src/domain/analysisErrorCore.js"; +import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; +import { createWebTextArtifact } from "../../../src/domain/webContentArtifact.js"; +import type { IncludedScript } from "../../../src/domain/webBundleAnalyzerState.js"; +import { JavaScriptAnalysisWorker } from "../../../src/javascript/analysis/JavaScriptAnalysisWorker.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; + +const source = createWebTextArtifact( + "export const observed = 1;", + "application/javascript", +); +const file: JavaScriptArtifactFile = { + path: "app.mjs", + sha256: source.sha256, + container_sha256: "1".repeat(64), + bytes: source.bytes, + inventory_artifact_id: "fixture-source", + kind: "javascript", + unpacked: false, + text: { included: true, value: source.text }, +}; +const script: IncludedScript = { + script_key: `scr_${source.sha256}`, + frame_id: "fixture", + url: "https://app.example.test/app.js", + origin: "https://app.example.test", + cdp_hash: null, + length: source.bytes, + is_module: true, + language: "JavaScript", + source_map_url: null, + resource_reconciliation: { + status: "unmatched", + reason: "no_exact_sanitized_url", + }, + source: { included: true, artifact: source }, +}; + +type WorkerKind = "web" | "javascript"; +type Encoding = "malformed" | "replacement" | "bom" | "non-json-whitespace"; + +const launchFor = + (kind: WorkerKind, encoding: Encoding): typeof spawnOwnedProviderProcess => + (request) => + spawnOwnedProviderProcess({ + ...request, + arguments: [ + ...request.arguments.slice(0, -1), + "-e", + ` +const { getHeapStatistics } = require("node:v8"); +const { createInterface } = require("node:readline"); +process.stdout.write(JSON.stringify({kind:"ready",heap_limit_bytes:getHeapStatistics().heap_size_limit})+"\\n"); +createInterface({ input: process.stdin }).once("line", (line) => { + const { id } = JSON.parse(line); + const encoding = ${JSON.stringify(encoding)}; + const frame = { + kind: "failure", id, reason: "resource", message: "reported WIRE_VALUE", + ${kind === "javascript" ? 'resource:"memory", limits:null,' : ""} + }; + const [before, after] = JSON.stringify(frame).split("WIRE_VALUE"); + const marker = encoding === "malformed" ? Buffer.from([0xff]) : Buffer.from("\\ufffd"); + const prefix = encoding === "bom" ? "\\ufeff" : encoding === "non-json-whitespace" ? "\\u00a0" : ""; + process.stdout.write(Buffer.concat([Buffer.from(prefix + before), marker, Buffer.from(after + "\\n")])); +}); +`, + ], + }); + +const analyze = async (kind: WorkerKind, encoding: Encoding) => { + const launch = launchFor(kind, encoding); + if (kind === "web") { + const worker = new WebBundleAnalysisWorker(128, launch); + try { + return await worker.analyze(script, 10_000); + } finally { + await worker.close(); + } + } + const worker = new JavaScriptAnalysisWorker( + { heapMb: 128, timeoutMs: 10_000 }, + launch, + ); + try { + return await worker.analyze(file, { nodes: 100, unknowns: 100 }); + } finally { + await worker.close(); + } +}; + +it.each([ + ["web", "malformed"], + ["javascript", "malformed"], + ["web", "replacement"], + ["javascript", "replacement"], + ["web", "bom"], + ["javascript", "bom"], + ["web", "non-json-whitespace"], + ["javascript", "non-json-whitespace"], +] as const)( + "%s distinguishes %s control bytes from a valid resource fact", + async (kind, encoding) => { + const result = await analyze(kind, encoding); + if (result.ok) throw new Error("Expected the fixture's worker failure"); + if (encoding === "replacement") { + expect(result.error.error).toBeInstanceOf( + AnalysisResourceConstraintError, + ); + expect(result.error.error.message).toContain("reported \ufffd"); + } else expect(result.error.error).toBeInstanceOf(AnalysisOutputError); + }, +);