mirror of
https://github.com/morluto/rea.git
synced 2026-11-20 13:45:30 +00:00
* refactor(domain): give strict UTF-8 decoding one owner with an explicit BOM policy * fix(domain): keep the leading-BOM behaviour every migrated UTF-8 decoder had * refactor(domain): read the domain layer's strict UTF-8 through the BOM-policy owner (#10803) Twelve domain decoders built their own fatal TextDecoder with ignoreBOM: true. They now use strictUtf8Decoder("preserve"), which keeps that behaviour: each reads bytes where a leading U+FEFF is content (binary-format names and fields, JSON and XML read byte-for-byte, recorded bodies, registry seeds whose kept bytes are re-emitted). The counted baseline drops from 74 to 62. * refactor(artifacts): read artifact readers' strict UTF-8 through the BOM-policy owner (#10871) Fourteen artifact reader modules (17 sites) built their own fatal TextDecoder. They now take it from strictUtf8Decoder with the policy each had: every one kept a leading BOM (ignoreBOM: true), so they use "preserve"; JavaScriptArtifactFiles keeps it for JavaScript and HTML entries and strips it otherwise, as before. The strict-utf8-decoder baseline drops those 17 sites. * refactor(domain): drop TextDecoder imports left by the strict UTF-8 migration * refactor(application): read MCP registration metadata through the strict UTF-8 owner * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner (#10903) * refactor(formats): read binary-format readers' strict UTF-8 through the BOM-policy owner * refactor(formats): drop the model readers' unused TextDecoder imports * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner (#10935) * refactor(browser): read browser capture and fetch strict UTF-8 through the BOM-policy owner * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner (#10936) * refactor(native): read LLDB and Mach-O name strict UTF-8 through the BOM-policy owner * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner (#10937) * refactor(process): read MCP and NDJSON transport frames' strict UTF-8 through the BOM-policy owner * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner (#10938) * refactor(application): read JSON and file inputs' strict UTF-8 through the BOM-policy owner * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner (#10939) * refactor(providers): read IDA, Electron, Ghidra and reference strict UTF-8 through the BOM-policy owner * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the owner and refuse new local decoders (#10940) * refactor(evm): read the hex bytecode carrier's strict UTF-8 through the BOM-policy owner * chore(architecture): match local strict UTF-8 decoders in any option order
108 lines
4.0 KiB
TypeScript
108 lines
4.0 KiB
TypeScript
import { strictUtf8Decoder } from "../domain/strictUtf8.js";
|
|
import { constants } from "node:buffer";
|
|
|
|
import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
|
|
import { safeParseJson, type SafeJsonParseResult } from "../domain/safeJson.js";
|
|
|
|
/** Recovery that preserves complete, authenticated Evidence workflow inputs. */
|
|
export const JSON_INPUT_RESOURCE_REMEDIATION =
|
|
"Provide a smaller valid JSON value. For Evidence workflows, re-analyze a smaller selection of the original target and use its complete Evidence; splitting JSON text or trimming Evidence fields does not produce valid workflow input.";
|
|
|
|
/**
|
|
* Strict JSON input keeps rejecting a leading byte-order mark, but names it:
|
|
* the parser's own message quotes the invisible U+FEFF as the bad token.
|
|
*/
|
|
export const JSON_BYTE_ORDER_MARK_MESSAGE =
|
|
"JSON input begins with a UTF-8 byte-order mark (EF BB BF), which strict JSON input does not accept. Save the file as UTF-8 without a byte-order mark.";
|
|
|
|
/** Decode JSON bytes without confusing a runtime string limit with malformed input. */
|
|
export const parseUtf8Json = (
|
|
bytes: Uint8Array,
|
|
operation: string,
|
|
path: string,
|
|
boundary = "json-input",
|
|
): SafeJsonParseResult & { readonly userMessage?: string } => {
|
|
const stringLimitFailure = (
|
|
cause?: unknown,
|
|
decodedUnits?: number,
|
|
): AnalysisResourceConstraintError =>
|
|
new AnalysisResourceConstraintError(
|
|
operation,
|
|
"memory",
|
|
"JSON input exceeds the runtime's maximum decoded string length",
|
|
{
|
|
boundary,
|
|
input_path: path,
|
|
input_bytes: bytes.byteLength,
|
|
max_string_code_units: constants.MAX_STRING_LENGTH,
|
|
...(decodedUnits === undefined
|
|
? {}
|
|
: { decoded_string_code_units_at_least: decodedUnits }),
|
|
},
|
|
{
|
|
...(cause === undefined ? {} : { cause }),
|
|
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
|
|
},
|
|
);
|
|
let text: string;
|
|
try {
|
|
const decoder = strictUtf8Decoder("preserve");
|
|
if (bytes.byteLength <= constants.MAX_STRING_LENGTH) {
|
|
text = decoder.decode(bytes);
|
|
} else {
|
|
// Node 22's UTF-8 fast path bounds encoded bytes before decoding. Decode
|
|
// large inputs in chunks so valid multibyte text is bounded by its actual
|
|
// UTF-16 length. Streaming also preserves characters spanning chunks.
|
|
const parts: string[] = [];
|
|
let decodedUnits = 0;
|
|
const append = (part: string): void => {
|
|
decodedUnits += part.length;
|
|
if (decodedUnits > constants.MAX_STRING_LENGTH)
|
|
throw stringLimitFailure(undefined, decodedUnits);
|
|
parts.push(part);
|
|
};
|
|
const chunkBytes = 1024 * 1024;
|
|
for (let offset = 0; offset < bytes.byteLength; offset += chunkBytes)
|
|
append(
|
|
decoder.decode(bytes.subarray(offset, offset + chunkBytes), {
|
|
stream: true,
|
|
}),
|
|
);
|
|
append(decoder.decode());
|
|
text = parts.join("");
|
|
}
|
|
} catch (cause: unknown) {
|
|
if (cause instanceof AnalysisResourceConstraintError) throw cause;
|
|
const code =
|
|
typeof cause === "object" && cause !== null && "code" in cause
|
|
? cause.code
|
|
: undefined;
|
|
if (code === "ERR_STRING_TOO_LONG") throw stringLimitFailure(cause);
|
|
if (code !== "ERR_ENCODING_INVALID_ENCODED_DATA") throw cause;
|
|
return {
|
|
ok: false,
|
|
error: "JSON input is not valid UTF-8",
|
|
userMessage: "JSON input is not valid UTF-8",
|
|
cause,
|
|
};
|
|
}
|
|
if (text.startsWith("\uFEFF"))
|
|
return {
|
|
ok: false,
|
|
error: JSON_BYTE_ORDER_MARK_MESSAGE,
|
|
cause: undefined,
|
|
userMessage: JSON_BYTE_ORDER_MARK_MESSAGE,
|
|
};
|
|
const parsed = safeParseJson(text);
|
|
// Native syntax errors can quote source snippets containing transport
|
|
// credentials. Keep the original exception internal, never in public text.
|
|
return parsed.ok
|
|
? parsed
|
|
: {
|
|
ok: false,
|
|
error:
|
|
"JSON input is not valid JSON. Check syntax in the selected file.",
|
|
cause: parsed.cause,
|
|
};
|
|
};
|