fix(process): decode bounded worker control frames as strict UTF-8 (#12162)

* fix(process): decode analyzer worker control frames as strict UTF-8

Refs #12145.

* fix(process): decode bounded control input without byte replacement

* fix(android): bound guardian readiness by the first UTF-8 frame (#12176)

Follows the readiness framing and lifecycle findings reported by @LcokHcak in #3048.
This commit is contained in:
morluto authored and GitHub committed 2026-11-19 04:00:02 +00:00
1 parent 85ac88fe87
commit 3493d6d252
15 files changed
+496 -208

No files matched your search

@@ -24,6 +24,10 @@ import { safeParseUtf8Json } from "../../domain/safeJson.js";
import { Budget } from "../../domain/budget.js";
import { analysisSettingLimit } from "../../domain/analysisSettingDefinitions.js";
import { WorkerFrameSignal } from "../../process/WorkerFrameSignal.js";
import {
Utf8NdjsonLineBuffer,
Utf8NdjsonOversizeError,
} from "../../process/Utf8NdjsonLineBuffer.js";
import type {
JavaScriptSemanticProjectionCapacity,
JavaScriptSemanticFileProjection,
@@ -32,7 +36,6 @@ import { randomUUID } from "node:crypto";
import { readFile, rm, stat } from "node:fs/promises";
import { join } from "node:path";
import { writeJavaScriptAnalysisSourceTransfer } from "./JavaScriptAnalysisSourceTransfer.js";
import { StringDecoder } from "node:string_decoder";
import { fileURLToPath } from "node:url";
import { getHeapStatistics } from "node:v8";
import { z } from "zod";
@@ -148,8 +151,7 @@ export class JavaScriptAnalysisWorker {
#heapLimitBytes = 0;
#frames: JavaScriptWorkerMessage[] = [];
readonly #frameSignal = new WorkerFrameSignal();
#pending = "";
#decoder = new StringDecoder("utf8");
readonly #lines = new Utf8NdjsonLineBuffer(() => DEFAULTS.protocolBytes);
#primaryFailure: WorkerPrimaryFailure | undefined;
#activeJobId: string | undefined;
#activeJobPhase = "startup";
@@ -181,33 +183,30 @@ export class JavaScriptAnalysisWorker {
if (this.#deadline?.interruption !== undefined) return;
this.#observeMemoryLimit();
try {
this.#pending +=
typeof chunk === "string" ? chunk : this.#decoder.write(chunk);
let newline: number;
while ((newline = this.#pending.indexOf("\n")) >= 0) {
const line = this.#pending.slice(0, newline);
this.#pending = this.#pending.slice(newline + 1);
if (
Buffer.byteLength(line) > DEFAULTS.protocolBytes ||
this.#frames.length >= 16
) {
for (const line of this.#lines.pushRaw(
typeof chunk === "string" ? Buffer.from(chunk) : chunk,
)) {
if (!line.ok) throw line.error;
if (this.#frames.length >= 16) {
this.#latchOutputFailure(
"JavaScript worker exceeded its control-frame budget",
);
return;
}
const raw: unknown = JSON.parse(line);
const raw: unknown = JSON.parse(line.value);
const frame = javaScriptWorkerMessageSchema.parse(raw);
this.#frames.push(frame);
this.#observeJobFrame(frame);
this.#frameSignal.wake("frame");
}
if (Buffer.byteLength(this.#pending) > DEFAULTS.protocolBytes)
this.#latchOutputFailure(
"JavaScript worker control frame is unterminated or oversized",
);
} catch (cause: unknown) {
this.#latchOutputFailure(errorMessage(cause));
this.#latchOutputFailure(
cause instanceof Utf8NdjsonOversizeError
? cause.boundary === "tail"
? "JavaScript worker control frame is unterminated or oversized"
: "JavaScript worker exceeded its control-frame budget"
: errorMessage(cause),
);
}
};
@@ -277,7 +276,7 @@ export class JavaScriptAnalysisWorker {
this.#primaryFailure === undefined &&
this.#lastFailure === undefined &&
this.#frames.length === 0 &&
this.#pending === "" &&
this.#lines.pendingBytes === 0 &&
this.#runtime.inputFailure === undefined &&
!this.#runtime.hasPendingCleanup &&
!this.#inspectionResources.hasPending &&
@@ -377,7 +376,7 @@ export class JavaScriptAnalysisWorker {
const supervisor = this.#runtime.supervisor;
if (rootPath === undefined || supervisor === undefined)
throw new TypeError("JavaScript worker resources were not acquired");
if (this.#frames.length !== 0 || this.#pending !== "")
if (this.#frames.length !== 0 || this.#lines.pendingBytes !== 0)
throw new AnalysisOutputError(
OPERATION,
"JavaScript worker has unsolicited or incomplete control frames",
@@ -1546,8 +1545,7 @@ export class JavaScriptAnalysisWorker {
this.#ready = false;
this.#frames = [];
this.#frameSignal.clear();
this.#pending = "";
this.#decoder = new StringDecoder("utf8");
this.#lines.reset();
this.#primaryFailure = undefined;
this.#activeJobId = undefined;
this.#staticResponseId = undefined;
@@ -140,48 +140,67 @@ it("consumes fragmented CRLF admission and coalesced next-job frames in order",
}
});
it("drains oversized checkpoint admission, reports output failure, and serves the next job", async () => {
const server = await startServer();
try {
await server.next();
const first = await server.request();
const second = await server.request();
server.child.stdin.write(frame(first));
expect(await server.next()).toMatchObject({ kind: "static", id: first.id });
server.child.stdin.write(
"x".repeat(JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS.protocolBytes),
);
server.child.stdin.write(
`x\n${frame(second)}${frame(admission(second.id))}`,
);
expect(await server.next()).toEqual({
kind: "failure",
id: first.id,
reason: "output",
resource: null,
limits: null,
message: expect.stringMatching(/admission frame.*missing or oversized/iu),
});
expect(await server.next()).toMatchObject({
kind: "static",
id: second.id,
});
expect(await server.next()).toMatchObject({
kind: "phase",
id: second.id,
phase: "transfer",
});
expect(await server.next()).toEqual({
kind: "complete",
id: second.id,
descriptor: null,
});
server.child.stdin.end();
expect((await server.closed)[0]).toBe(0);
} finally {
await server.close();
}
});
it.each([
["oversized", /admission frame.*missing or oversized/iu],
["malformed UTF-8", /admission frame.*not valid UTF-8/iu],
] as const)(
"reports %s checkpoint admission and serves the next job",
async (kind, failure) => {
const server = await startServer();
try {
await server.next();
const first = await server.request();
const second = await server.request();
server.child.stdin.write(frame(first));
expect(await server.next()).toMatchObject({
kind: "static",
id: first.id,
});
if (kind === "oversized") {
server.child.stdin.write(
"x".repeat(JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS.protocolBytes + 1),
);
} else {
server.child.stdin.write(
Buffer.concat([
Buffer.from('{"id":"'),
Buffer.of(0xff),
Buffer.from(`",${JSON.stringify(admission(first.id)).slice(1)}`),
]),
);
}
server.child.stdin.write(
`\n${frame(second)}${frame(admission(second.id))}`,
);
expect(await server.next()).toEqual({
kind: "failure",
id: first.id,
reason: "output",
resource: null,
limits: null,
message: expect.stringMatching(failure),
});
expect(await server.next()).toMatchObject({
kind: "static",
id: second.id,
});
expect(await server.next()).toMatchObject({
kind: "phase",
id: second.id,
phase: "transfer",
});
expect(await server.next()).toEqual({
kind: "complete",
id: second.id,
descriptor: null,
});
server.child.stdin.end();
expect((await server.closed)[0]).toBe(0);
} finally {
await server.close();
}
},
);
it("reports missing checkpoint admission as output failure at stdin EOF", async () => {
const server = await startServer();
@@ -227,3 +246,25 @@ it.each(["lf", "eof", "open"] as const)(
}
},
);
it("rejects a valid-shaped initial request containing invalid UTF-8", async () => {
const server = await startServer();
try {
await server.next();
const request = await server.request();
const [before, after] = frame(request).split("app.mjs");
server.child.stdin.end(
Buffer.concat([
Buffer.from(before ?? ""),
Buffer.of(0xff),
Buffer.from(after ?? ""),
]),
);
expect((await server.closed)[0]).not.toBe(0);
expect(server.stderr()).toContain(
"TypeError: JavaScript worker request is not valid UTF-8",
);
} finally {
await server.close();
}
});
@@ -95,10 +95,13 @@ export const serveJavaScriptAnalysisWorker = async (
for (;;) {
const next = await requests.next();
if (next.done) break;
if (!next.value.ok)
if (!next.value.ok) {
if (next.value.error._tag === "ControlLineMalformedUtf8")
throw new TypeError("JavaScript worker request is not valid UTF-8");
throw new RangeError(
"JavaScript worker request exceeds its byte budget",
);
}
const raw: unknown = JSON.parse(next.value.value);
const request = javaScriptWorkerRequestSchema.parse(raw);
// Function scope releases source AST/IR/results before accepting another job.
@@ -182,6 +185,14 @@ const analyzeJob = async (
sha256: sha256Hex(text),
});
const next = await requests.next();
if (
!next.done &&
!next.value.ok &&
next.value.error._tag === "ControlLineMalformedUtf8"
)
throw new TypeError(
"JavaScript worker admission frame is not valid UTF-8",
);
if (next.done || !next.value.ok)
throw new TypeError(
"JavaScript worker admission frame is missing or oversized",