mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 21:23:45 +00:00
fix(process): decode bounded worker control frames as strict UTF-8 (#12162)
* fix(process): decode analyzer worker control frames as strict UTF-8 Refs #12145. * fix(process): decode bounded control input without byte replacement * fix(android): bound guardian readiness by the first UTF-8 frame (#12176) Follows the readiness framing and lifecycle findings reported by @LcokHcak in #3048.
This commit is contained in:
1 parent
85ac88fe87
commit
3493d6d252
15 files changed
+496
-208
No files matched your search
@@ -24,6 +24,10 @@ import { safeParseUtf8Json } from "../../domain/safeJson.js";
|
||||
import { Budget } from "../../domain/budget.js";
|
||||
import { analysisSettingLimit } from "../../domain/analysisSettingDefinitions.js";
|
||||
import { WorkerFrameSignal } from "../../process/WorkerFrameSignal.js";
|
||||
import {
|
||||
Utf8NdjsonLineBuffer,
|
||||
Utf8NdjsonOversizeError,
|
||||
} from "../../process/Utf8NdjsonLineBuffer.js";
|
||||
import type {
|
||||
JavaScriptSemanticProjectionCapacity,
|
||||
JavaScriptSemanticFileProjection,
|
||||
@@ -32,7 +36,6 @@ import { randomUUID } from "node:crypto";
|
||||
import { readFile, rm, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { writeJavaScriptAnalysisSourceTransfer } from "./JavaScriptAnalysisSourceTransfer.js";
|
||||
import { StringDecoder } from "node:string_decoder";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
import { z } from "zod";
|
||||
@@ -148,8 +151,7 @@ export class JavaScriptAnalysisWorker {
|
||||
#heapLimitBytes = 0;
|
||||
#frames: JavaScriptWorkerMessage[] = [];
|
||||
readonly #frameSignal = new WorkerFrameSignal();
|
||||
#pending = "";
|
||||
#decoder = new StringDecoder("utf8");
|
||||
readonly #lines = new Utf8NdjsonLineBuffer(() => DEFAULTS.protocolBytes);
|
||||
#primaryFailure: WorkerPrimaryFailure | undefined;
|
||||
#activeJobId: string | undefined;
|
||||
#activeJobPhase = "startup";
|
||||
@@ -181,33 +183,30 @@ export class JavaScriptAnalysisWorker {
|
||||
if (this.#deadline?.interruption !== undefined) return;
|
||||
this.#observeMemoryLimit();
|
||||
try {
|
||||
this.#pending +=
|
||||
typeof chunk === "string" ? chunk : this.#decoder.write(chunk);
|
||||
let newline: number;
|
||||
while ((newline = this.#pending.indexOf("\n")) >= 0) {
|
||||
const line = this.#pending.slice(0, newline);
|
||||
this.#pending = this.#pending.slice(newline + 1);
|
||||
if (
|
||||
Buffer.byteLength(line) > DEFAULTS.protocolBytes ||
|
||||
this.#frames.length >= 16
|
||||
) {
|
||||
for (const line of this.#lines.pushRaw(
|
||||
typeof chunk === "string" ? Buffer.from(chunk) : chunk,
|
||||
)) {
|
||||
if (!line.ok) throw line.error;
|
||||
if (this.#frames.length >= 16) {
|
||||
this.#latchOutputFailure(
|
||||
"JavaScript worker exceeded its control-frame budget",
|
||||
);
|
||||
return;
|
||||
}
|
||||
const raw: unknown = JSON.parse(line);
|
||||
const raw: unknown = JSON.parse(line.value);
|
||||
const frame = javaScriptWorkerMessageSchema.parse(raw);
|
||||
this.#frames.push(frame);
|
||||
this.#observeJobFrame(frame);
|
||||
this.#frameSignal.wake("frame");
|
||||
}
|
||||
if (Buffer.byteLength(this.#pending) > DEFAULTS.protocolBytes)
|
||||
this.#latchOutputFailure(
|
||||
"JavaScript worker control frame is unterminated or oversized",
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
this.#latchOutputFailure(errorMessage(cause));
|
||||
this.#latchOutputFailure(
|
||||
cause instanceof Utf8NdjsonOversizeError
|
||||
? cause.boundary === "tail"
|
||||
? "JavaScript worker control frame is unterminated or oversized"
|
||||
: "JavaScript worker exceeded its control-frame budget"
|
||||
: errorMessage(cause),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -277,7 +276,7 @@ export class JavaScriptAnalysisWorker {
|
||||
this.#primaryFailure === undefined &&
|
||||
this.#lastFailure === undefined &&
|
||||
this.#frames.length === 0 &&
|
||||
this.#pending === "" &&
|
||||
this.#lines.pendingBytes === 0 &&
|
||||
this.#runtime.inputFailure === undefined &&
|
||||
!this.#runtime.hasPendingCleanup &&
|
||||
!this.#inspectionResources.hasPending &&
|
||||
@@ -377,7 +376,7 @@ export class JavaScriptAnalysisWorker {
|
||||
const supervisor = this.#runtime.supervisor;
|
||||
if (rootPath === undefined || supervisor === undefined)
|
||||
throw new TypeError("JavaScript worker resources were not acquired");
|
||||
if (this.#frames.length !== 0 || this.#pending !== "")
|
||||
if (this.#frames.length !== 0 || this.#lines.pendingBytes !== 0)
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"JavaScript worker has unsolicited or incomplete control frames",
|
||||
@@ -1546,8 +1545,7 @@ export class JavaScriptAnalysisWorker {
|
||||
this.#ready = false;
|
||||
this.#frames = [];
|
||||
this.#frameSignal.clear();
|
||||
this.#pending = "";
|
||||
this.#decoder = new StringDecoder("utf8");
|
||||
this.#lines.reset();
|
||||
this.#primaryFailure = undefined;
|
||||
this.#activeJobId = undefined;
|
||||
this.#staticResponseId = undefined;
|
||||
|
||||
@@ -140,48 +140,67 @@ it("consumes fragmented CRLF admission and coalesced next-job frames in order",
|
||||
}
|
||||
});
|
||||
|
||||
it("drains oversized checkpoint admission, reports output failure, and serves the next job", async () => {
|
||||
const server = await startServer();
|
||||
try {
|
||||
await server.next();
|
||||
const first = await server.request();
|
||||
const second = await server.request();
|
||||
server.child.stdin.write(frame(first));
|
||||
expect(await server.next()).toMatchObject({ kind: "static", id: first.id });
|
||||
server.child.stdin.write(
|
||||
"x".repeat(JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS.protocolBytes),
|
||||
);
|
||||
server.child.stdin.write(
|
||||
`x\n${frame(second)}${frame(admission(second.id))}`,
|
||||
);
|
||||
expect(await server.next()).toEqual({
|
||||
kind: "failure",
|
||||
id: first.id,
|
||||
reason: "output",
|
||||
resource: null,
|
||||
limits: null,
|
||||
message: expect.stringMatching(/admission frame.*missing or oversized/iu),
|
||||
});
|
||||
expect(await server.next()).toMatchObject({
|
||||
kind: "static",
|
||||
id: second.id,
|
||||
});
|
||||
expect(await server.next()).toMatchObject({
|
||||
kind: "phase",
|
||||
id: second.id,
|
||||
phase: "transfer",
|
||||
});
|
||||
expect(await server.next()).toEqual({
|
||||
kind: "complete",
|
||||
id: second.id,
|
||||
descriptor: null,
|
||||
});
|
||||
server.child.stdin.end();
|
||||
expect((await server.closed)[0]).toBe(0);
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
it.each([
|
||||
["oversized", /admission frame.*missing or oversized/iu],
|
||||
["malformed UTF-8", /admission frame.*not valid UTF-8/iu],
|
||||
] as const)(
|
||||
"reports %s checkpoint admission and serves the next job",
|
||||
async (kind, failure) => {
|
||||
const server = await startServer();
|
||||
try {
|
||||
await server.next();
|
||||
const first = await server.request();
|
||||
const second = await server.request();
|
||||
server.child.stdin.write(frame(first));
|
||||
expect(await server.next()).toMatchObject({
|
||||
kind: "static",
|
||||
id: first.id,
|
||||
});
|
||||
if (kind === "oversized") {
|
||||
server.child.stdin.write(
|
||||
"x".repeat(JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS.protocolBytes + 1),
|
||||
);
|
||||
} else {
|
||||
server.child.stdin.write(
|
||||
Buffer.concat([
|
||||
Buffer.from('{"id":"'),
|
||||
Buffer.of(0xff),
|
||||
Buffer.from(`",${JSON.stringify(admission(first.id)).slice(1)}`),
|
||||
]),
|
||||
);
|
||||
}
|
||||
server.child.stdin.write(
|
||||
`\n${frame(second)}${frame(admission(second.id))}`,
|
||||
);
|
||||
expect(await server.next()).toEqual({
|
||||
kind: "failure",
|
||||
id: first.id,
|
||||
reason: "output",
|
||||
resource: null,
|
||||
limits: null,
|
||||
message: expect.stringMatching(failure),
|
||||
});
|
||||
expect(await server.next()).toMatchObject({
|
||||
kind: "static",
|
||||
id: second.id,
|
||||
});
|
||||
expect(await server.next()).toMatchObject({
|
||||
kind: "phase",
|
||||
id: second.id,
|
||||
phase: "transfer",
|
||||
});
|
||||
expect(await server.next()).toEqual({
|
||||
kind: "complete",
|
||||
id: second.id,
|
||||
descriptor: null,
|
||||
});
|
||||
server.child.stdin.end();
|
||||
expect((await server.closed)[0]).toBe(0);
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("reports missing checkpoint admission as output failure at stdin EOF", async () => {
|
||||
const server = await startServer();
|
||||
@@ -227,3 +246,25 @@ it.each(["lf", "eof", "open"] as const)(
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects a valid-shaped initial request containing invalid UTF-8", async () => {
|
||||
const server = await startServer();
|
||||
try {
|
||||
await server.next();
|
||||
const request = await server.request();
|
||||
const [before, after] = frame(request).split("app.mjs");
|
||||
server.child.stdin.end(
|
||||
Buffer.concat([
|
||||
Buffer.from(before ?? ""),
|
||||
Buffer.of(0xff),
|
||||
Buffer.from(after ?? ""),
|
||||
]),
|
||||
);
|
||||
expect((await server.closed)[0]).not.toBe(0);
|
||||
expect(server.stderr()).toContain(
|
||||
"TypeError: JavaScript worker request is not valid UTF-8",
|
||||
);
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
});
|
||||
@@ -95,10 +95,13 @@ export const serveJavaScriptAnalysisWorker = async (
|
||||
for (;;) {
|
||||
const next = await requests.next();
|
||||
if (next.done) break;
|
||||
if (!next.value.ok)
|
||||
if (!next.value.ok) {
|
||||
if (next.value.error._tag === "ControlLineMalformedUtf8")
|
||||
throw new TypeError("JavaScript worker request is not valid UTF-8");
|
||||
throw new RangeError(
|
||||
"JavaScript worker request exceeds its byte budget",
|
||||
);
|
||||
}
|
||||
const raw: unknown = JSON.parse(next.value.value);
|
||||
const request = javaScriptWorkerRequestSchema.parse(raw);
|
||||
// Function scope releases source AST/IR/results before accepting another job.
|
||||
@@ -182,6 +185,14 @@ const analyzeJob = async (
|
||||
sha256: sha256Hex(text),
|
||||
});
|
||||
const next = await requests.next();
|
||||
if (
|
||||
!next.done &&
|
||||
!next.value.ok &&
|
||||
next.value.error._tag === "ControlLineMalformedUtf8"
|
||||
)
|
||||
throw new TypeError(
|
||||
"JavaScript worker admission frame is not valid UTF-8",
|
||||
);
|
||||
if (next.done || !next.value.ok)
|
||||
throw new TypeError(
|
||||
"JavaScript worker admission frame is missing or oversized",
|
||||
|
||||
Reference in new issue
Block a user