mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 19:53:50 +00:00
* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy * fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had * refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803) Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62. * refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871) Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites. * refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration * refactor(application): read MCP registration metadata through the strict UTF-8 owner * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903) * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner * refactor(formats): drop the model readers' unused TextDecoder imports * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935) * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936) * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937) * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938) * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939) * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940) * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner * chore(architecture): match local strict UTF-8 decoders in any option order
213 lines
6.2 KiB
TypeScript
213 lines
6.2 KiB
TypeScript
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
|
import type { GoModule, GoModuleMetadata } from "../domain/go/goBinary.js";
|
|
|
|
const utf8 = strictUtf8Decoder("preserve");
|
|
const escapes = new Map([
|
|
["a", 7],
|
|
["b", 8],
|
|
["f", 12],
|
|
["n", 10],
|
|
["r", 13],
|
|
["t", 9],
|
|
["v", 11],
|
|
["\\", 92],
|
|
['"', 34],
|
|
]);
|
|
|
|
const quoted = (input: string): { value: string; end: number } | null => {
|
|
if (input.startsWith("`")) {
|
|
const end = input.indexOf("`", 1);
|
|
return end < 0
|
|
? null
|
|
: { value: input.slice(1, end).replace(/\r/g, ""), end: end + 1 };
|
|
}
|
|
if (!input.startsWith('"')) return null;
|
|
const output = Buffer.alloc(input.length * 3);
|
|
let length = 0;
|
|
for (let index = 1; index < input.length;) {
|
|
const character = input[index];
|
|
if (character === '"') {
|
|
try {
|
|
return {
|
|
value: utf8.decode(output.subarray(0, length)),
|
|
end: index + 1,
|
|
};
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
if (character === "\n") return null;
|
|
if (character !== "\\") {
|
|
const point = input.codePointAt(index);
|
|
if (point === undefined) return null;
|
|
length += output.write(String.fromCodePoint(point), length, "utf8");
|
|
index += point > 0xffff ? 2 : 1;
|
|
continue;
|
|
}
|
|
const escaped = input[index + 1];
|
|
if (escaped === undefined) return null;
|
|
const simple = escapes.get(escaped);
|
|
if (simple !== undefined) {
|
|
output[length++] = simple;
|
|
index += 2;
|
|
continue;
|
|
}
|
|
const digits =
|
|
escaped === "x" ? 2 : escaped === "u" ? 4 : escaped === "U" ? 8 : 0;
|
|
if (digits > 0) {
|
|
const encoded = input.slice(index + 2, index + 2 + digits);
|
|
if (encoded.length !== digits || !/^[a-f\d]+$/i.test(encoded))
|
|
return null;
|
|
const point = Number.parseInt(encoded, 16);
|
|
if (escaped === "x") output[length++] = point;
|
|
else {
|
|
if (point > 0x10ffff || (point >= 0xd800 && point <= 0xdfff))
|
|
return null;
|
|
length += output.write(String.fromCodePoint(point), length, "utf8");
|
|
}
|
|
index += 2 + digits;
|
|
continue;
|
|
}
|
|
const octal = input.slice(index + 1, index + 4);
|
|
if (!/^[0-7]{3}$/.test(octal)) return null;
|
|
const byte = Number.parseInt(octal, 8);
|
|
if (byte > 255) return null;
|
|
output[length++] = byte;
|
|
index += 4;
|
|
}
|
|
return null;
|
|
};
|
|
|
|
const setting = (input: string): { key: string; value: string } | null => {
|
|
let key: string;
|
|
let rawValue: string;
|
|
if (input.startsWith('"') || input.startsWith("`")) {
|
|
const decoded = quoted(input);
|
|
if (decoded === null || input[decoded.end] !== "=") return null;
|
|
key = decoded.value;
|
|
rawValue = input.slice(decoded.end + 1);
|
|
} else {
|
|
const separator = input.indexOf("=");
|
|
if (separator < 1) return null;
|
|
key = input.slice(0, separator);
|
|
if (/[= \t\r\n"`]/.test(key)) return null;
|
|
rawValue = input.slice(separator + 1);
|
|
}
|
|
if (rawValue.startsWith('"') || rawValue.startsWith("`")) {
|
|
const decoded = quoted(rawValue);
|
|
return decoded !== null && decoded.end === rawValue.length
|
|
? { key, value: decoded.value }
|
|
: null;
|
|
}
|
|
return /[ \t\r\n"`]/.test(rawValue) ? null : { key, value: rawValue };
|
|
};
|
|
|
|
const moduleLine = (fields: string[]): GoModule | null => {
|
|
const [path, version] = fields;
|
|
if (
|
|
(fields.length !== 2 && fields.length !== 3) ||
|
|
path === undefined ||
|
|
version === undefined
|
|
)
|
|
return null;
|
|
return { path, version, sum: fields[2] ?? null, replacement: null };
|
|
};
|
|
|
|
interface ModuleSourceLine {
|
|
readonly line: string | Buffer;
|
|
readonly terminated: boolean;
|
|
}
|
|
|
|
function* moduleSourceLines(source: Buffer): Generator<ModuleSourceLine> {
|
|
let start = 0;
|
|
while (start < source.length) {
|
|
const newline = source.indexOf(10, start);
|
|
const end = newline < 0 ? source.length : newline;
|
|
let line: string | Buffer;
|
|
const bytes = source.subarray(start, end);
|
|
try {
|
|
line = utf8.decode(bytes);
|
|
} catch {
|
|
line = bytes;
|
|
}
|
|
yield { line, terminated: newline >= 0 };
|
|
if (newline < 0) return;
|
|
start = newline + 1;
|
|
}
|
|
}
|
|
|
|
const modulePrefixes = ["mod\t", "dep\t", "=>\t"].map((prefix) =>
|
|
Buffer.from(prefix),
|
|
);
|
|
|
|
/** Decode an unframed Go module body while retaining non-UTF-8 line bytes exactly. */
|
|
export const parseGoModuleBytes = (source: Buffer): GoModuleMetadata => {
|
|
let path: string | null = null;
|
|
let main: GoModule | null = null;
|
|
const dependencies: GoModule[] = [];
|
|
const settings: { key: string; value: string }[] = [];
|
|
const unparsed_lines: string[] = [];
|
|
const unparsed_line_bytes_base64: string[] = [];
|
|
let last: GoModule | null = null;
|
|
for (const { line, terminated } of moduleSourceLines(source)) {
|
|
if (typeof line !== "string") {
|
|
unparsed_line_bytes_base64.push(line.toString("base64"));
|
|
if (
|
|
modulePrefixes.some((prefix) =>
|
|
line.subarray(0, prefix.length).equals(prefix),
|
|
)
|
|
)
|
|
last = null;
|
|
continue;
|
|
}
|
|
if (line === "") continue;
|
|
if (!terminated) {
|
|
unparsed_lines.push(line);
|
|
continue;
|
|
}
|
|
if (line.startsWith("path\t") && path === null) {
|
|
path = line.slice(5);
|
|
continue;
|
|
}
|
|
if (line.startsWith("mod\t") || line.startsWith("dep\t")) {
|
|
const value = moduleLine(line.slice(4).split("\t"));
|
|
last = null;
|
|
if (value !== null && (!line.startsWith("mod\t") || main === null)) {
|
|
if (line.startsWith("mod\t")) main = value;
|
|
else dependencies.push(value);
|
|
last = value;
|
|
continue;
|
|
}
|
|
} else if (line.startsWith("=>\t")) {
|
|
const fields = line.slice(3).split("\t");
|
|
const value = moduleLine(fields);
|
|
if (fields.length === 3 && value !== null && last !== null) {
|
|
last.replacement = {
|
|
path: value.path,
|
|
version: value.version,
|
|
sum: value.sum,
|
|
};
|
|
last = null;
|
|
continue;
|
|
}
|
|
last = null;
|
|
} else if (line.startsWith("build\t")) {
|
|
const value = setting(line.slice(6));
|
|
if (value !== null) {
|
|
settings.push(value);
|
|
continue;
|
|
}
|
|
}
|
|
unparsed_lines.push(line);
|
|
}
|
|
return {
|
|
path,
|
|
main,
|
|
dependencies,
|
|
settings,
|
|
unparsed_lines,
|
|
unparsed_line_bytes_base64,
|
|
complete: unparsed_lines.length + unparsed_line_bytes_base64.length === 0,
|
|
};
|
|
};
|