From 5d2c8c2b78d2552da0e13a11d3e2678efb69810d Mon Sep 17 00:00:00 2001 From: kiran Date: Fri, 23 Oct 2026 23:43:28 +0530 Subject: [PATCH 1/8] feat(cli)!: export row-oriented results as JSON Lines (#5629) * feat(cli): add JSON Lines output for row-oriented results (#5487) - Support opt-in JSON Lines format (--format jsonl) for row-oriented results and collection views with clear record boundaries - Preserve stable source identities and stream records deterministically line-by-line - Emit final coverage and limitation metadata as a distinct line with https://rea.dev/schemas/jsonl-coverage-v1.json to prevent confusion with data rows - Distinguish error outputs with https://rea.dev/schemas/jsonl-error-v1.json - Reject non-row-oriented results that lack record boundaries with an unreadable_output error - Document JSON Lines streaming conventions in docs/cli.md - Add boundary tests covering empty results, multiline strings, partial coverage, large fields, and non-row rejection * fix(cli): preserve diagnostics, provenance, and nested collections in JSONL * fix(cli): frame classified failures and reject unrepresented JSONL facets --------- Co-authored-by: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> --- docs/cli.md | 90 ++++ docs/testing.md | 6 +- .../verification/cli-json-lines-2026-10-23.md | 134 ++++++ scripts/rea.mjs | 45 +- scripts/verify/javascript/output.mjs | 93 +++- src/cli/cliResultOutput.ts | 109 ++++- src/cli/jsonLines.ts | 226 +++++++++ src/cli/jsonLinesOutput.ts | 111 +++++ src/cli/jsonOutput.ts | 13 +- src/cliOutput.ts | 3 +- src/mcpDoctor.ts | 9 +- tests/acceptance/jsonLinesCli.test.ts | 92 ++++ tests/acceptance/managedCliOutput.test.ts | 11 +- tests/boundary/cli/cliOutput.test.ts | 11 +- tests/boundary/cli/dispatcher.test.ts | 41 +- .../boundary/cli/filteredArrayOutput.test.ts | 14 +- .../javascriptApplicationJsonOutput.test.ts | 27 +- tests/boundary/cli/jsonLinesOutput.test.ts | 451 ++++++++++++++++++ tests/boundary/cli/jsonOutput.test.ts | 11 +- tests/boundary/cli/versionReportCli.test.ts | 9 + tests/boundary/mcp/mcpDoctor.test.ts | 44 ++ tests/support/cli/cliCommandSessionWorker.mjs | 5 + tests/support/cli/compiledCliWorker.mjs | 6 + 23 files changed, 1502 insertions(+), 59 deletions(-) create mode 100644 docs/verification/cli-json-lines-2026-10-23.md create mode 100644 src/cli/jsonLines.ts create mode 100644 src/cli/jsonLinesOutput.ts create mode 100644 tests/acceptance/jsonLinesCli.test.ts create mode 100644 tests/boundary/cli/jsonLinesOutput.test.ts diff --git a/docs/cli.md b/docs/cli.md index 84d8c39777..918bce4d33 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -46,6 +46,96 @@ module page. See [JavaScript application workflows](javascript-application-workflows.md) for views, feature traces and comparisons that take the same saved Evidence. +## JSON Lines record output + +Use `--format jsonl` to consume row-oriented results one record per line: + +```bash +cat > observations-view.json <<'JSON' +{"source":{"kind":"evidence-file","path":"app-evidence.json"},"view":{"kind":"javascript-facts","property_keys":[],"offset":0,"limit":32}} +JSON +rea inspect-analysis-view observations-view.json --format jsonl > observations.jsonl +``` + +This request selects up to 32 observation identities and their provenance, +without observation property values. + +This changes JSONL object-collection framing. Scripts that need one complete +report or a scalar field should use `--format json`. For a full application +graph, first project a supported analysis view rather than selecting an +arbitrary nested graph array. + +A top-level array emits its elements in source order. An object must have one +clear `rows`, `items`, `records`, or `occurrences` array, directly or under +`page`. Evidence uses the same boundaries inside `normalized_result`. +An arbitrary report or a scalar selection is refused with an actionable error; +use `--format json` for a complete document. If several collections exist, +select one with `--filter-output`, for example `normalized_result.rows`. + +Object collections finish with a control record: + +```json +{ + "$rea": "jsonl/v1", + "type": "metadata", + "records": { + "path": ["normalized_result", "rows"], + "emitted": 2, + "source": 10, + "filter": "normalized_result.rows[0,2]" + }, + "context": { + "evidence_id": "ev_…", + "normalized_result": { + "coverage": { + "status": "page", + "examined": 10, + "total": 100, + "next_offset": 10, + "exhausted": false + } + } + } +} +``` + +This abbreviated example illustrates framing. The actual `context` preserves +all non-row fields, including Evidence subject, provider, authority, confidence, +locations, links, limitations, parent/view digests, pagination and coverage. +With `--full-output`, it also retains the command envelope and its metadata. +A data filter selects row fields/slices while this context remains available; +excluded peer record collections and unselected Evidence `raw_result` are not +restored. An explicitly selected `raw_result` remains selected. Retain the +original Evidence for complete observations. + +`records.emitted` and `records.source` count the selected output and its source +array; they do not assert analysis completeness. Source coverage and unknowns +retain their original meaning. Existing row identities and indices remain +unchanged unless your field selection excludes them. Empty object collections +still emit metadata; an empty top-level array emits no lines. Bare top-level +arrays have no synthetic coverage record. + +Command and output-validation failures emit `{"$rea":"jsonl/v1","type":"error","ok":false,"result":…}` +and retain the original diagnostic payload. SDK validation/sentinel errors use +the SDK's exit status rather than guessing from data fields. `$rea` is reserved +at the top level of a data record, so control records are unambiguous; collisions +are refused before rows are written. Nested `$rea` values are ordinary data. +A missing final metadata record or nonzero exit can indicate an interrupted +export; do not treat already emitted rows as a complete result. + +`--token-count` estimates this exact representation, including metadata, one +line at a time. Errors remain error records even with token counting enabled. +Token windows cannot preserve JSON Lines and are rejected. JSON encoding and +stdout writes are incremental, including large escaped strings; this is output +streaming over a completed result, not incremental analysis. The analysis still +needs its normal resources. A single record must fit the runtime string limit +for token estimation; remove `--token-count` to stream a larger record. + +Standalone version and MCP doctor diagnostics remain single JSON documents +in JSON Lines mode. Incur's streaming generator events and successful sentinel +responses retain their SDK protocol. MCP application results and Evidence +contracts are unchanged. + ## Shell completion The CLI includes optional shell completion. Run `rea completions --help` to diff --git a/docs/testing.md b/docs/testing.md index e44af7e0d6..92798741ec 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1773,8 +1773,10 @@ the running Node engine's single-string limit. Shared input leaves keep the fixture's graph small; the verifier writes one temporary output file, checks its complete byte count and an independent digest, then removes it. It requires only Node and the built REA runtime, with space for the output plus a 1 GiB reserve. -Run `npm run verify:javascript:output -- jsonl` for compact JSONL coverage. This -opt-in lane is separate from routine tests and the canonical hash check +Run `npm run verify:javascript:output -- jsonl` for compact JSONL coverage. Its +row collection must match independent row and final-metadata byte/digest goldens; +its token count is checked against the actual file one line at a time, even when +the complete export exceeds the engine string limit. This opt-in lane is separate from routine tests and the canonical hash check `npm run verify:javascript:digests`. It verifies serialization rather than an arbitrary third-party application's parsing cost or MCP client capacity. diff --git a/docs/verification/cli-json-lines-2026-10-23.md b/docs/verification/cli-json-lines-2026-10-23.md new file mode 100644 index 0000000000..9e4d358a10 --- /dev/null +++ b/docs/verification/cli-json-lines-2026-10-23.md @@ -0,0 +1,134 @@ +# CLI JSON Lines verification — 2026-10-23 + +The analyst task was to export retained JavaScript observations as independent +records while keeping their Evidence provenance, coverage and unknowns. JSONL +framing stays in the CLI adapter; analysis and view projection continue to use +the same application workflow as MCP. + +## Real artifact and completed workflow + +The installed, locked `semver@7.8.5` npm package was analyzed through the public +`analyze-javascript-application` command, without executing its target code. +It parsed 50 selected files and produced a 12,185,829-byte Evidence document. +Preparation took 12.295 s elapsed, 12.648 s child CPU and 678.5 MiB sampled peak +process-family RSS; preparation is separate from the export measurements. +The artifact digest was +`65c9cce0fbebc34824c1b9657a514e7c5525ab079b232ce9da095697271b7953`. +The saved Evidence file SHA-256 was +`b34627bc12894d4c749e5b24c60731dd5a7eaaf90c98954f0e585eb442d484c3`. + +Preparation ran on main `169512a35`; the exact retained parent was reused by +both refreshed export variants: + +```json +{ + "source": { "kind": "evidence-file", "path": "/absolute/path/parent.json" }, + "view": { + "kind": "javascript-facts", + "property_keys": [], + "include_available_property_keys": true, + "offset": 0, + "limit": 256 + } +} +``` + +Each completed measurement executed one public CLI call: + +```bash +node scripts/rea.mjs inspect-analysis-view request.json --format jsonl +``` + +The view contained 202 observation rows. Every before/after pair reconstructed +exactly equal complete Evidence JSON, including identities, subject/provider, +authority/confidence, locations, parent/view digests, links, original coverage, +17 view limitations and 20 unknowns. These source limitations remain limitations; +exhausting this selected view does not establish complete application analysis. +Before emitted one document; after emitted 202 rows and one metadata record. + +## Performance + +Baseline: refreshed main `2443b949a48c3b2eba687e7381682e49e909d5b9`. +Both variants used Node 24.22.1 and the same locked dependency installation. +Three alternating pairs ran sequentially, with fresh processes and warm +filesystem/build artifacts. Node compile caching was not configured. Cold host +filesystem caches and repeated analysis were not measured. + +| Completed export, median of three | Before | After | +| --------------------------------- | --------: | --------: | +| Elapsed time | 4.510 s | 4.576 s | +| Wait-accounted child CPU | 5.876 s | 5.987 s | +| Sampled peak process-family RSS | 496.3 MiB | 495.1 MiB | +| Emitted UTF-8 bytes | 365,494 | 365,608 | +| `tokenx` token estimate | 99,004 | 99,041 | +| CLI calls | 1 | 1 | + +The small observed overhead does not establish an analysis speedup. +The 114-byte / 37-estimated-token increase is framing overhead, with the same +facts and no additional tool/schema round trips. The identical request file +contributes another 74 estimated tokens to each case. These are heuristic `tokenx` estimates, not model-specific BPE counts; +orchestration prompts, agent reasoning and tool-definition tokens were not +measured. Actual agent context consumption also depends on which rows the +consumer reads. No token reduction is claimed. + +Rows and metadata share the existing bounded JSON encoder and awaited writes. +Token counting reuses that exact record plan, including context, and estimates +one complete line at a time rather than materializing the complete export. +A single line still must fit the estimator's runtime string limit. Token-count +performance, much larger applications and time to first consumed row were not +benchmarked. + +The harness pinned the process family to two CPUs, used a 2 GiB V8 heap limit, +`GOMAXPROCS=2`, `GOMEMLIMIT=2GiB` and two libuv threads. It sampled descendant +RSS every 50 ms and would terminate the task at 6 GiB or 30 minutes. No resource +stop occurred. RSS sums can double-count shared pages and miss short peaks; +the V8 limit is per process and the sampler is not a kernel memory limit. + +## Validation + +`npm run check` passed on the refreshed integration, including type checking, +lint, formatting, dependency/metadata checks and architecture guards. The parsed +output-control contract from current main is retained; handlers, launcher and +reusable test drivers consume those same facts. + +- 241 tests passed across 14 CLI/MCP/acceptance files: actual production MCP + catalogs, authenticated Evidence reconstruction, invalid SDK values, ordinary + and sentinel failures, generator streaming, formatter recovery, empty/partial + views, large output, pipe closure and asynchronous final writes after cleanup + success/failure. The run took 171.097 s elapsed, 203.968 s child CPU and + 1589.3 MiB sampled peak family RSS. +- Another 60 tests passed across four consumer files, including ordinary graph + reports and scalar selections refusing JSONL, malformed JSON input retaining + diagnostics, managed CLI output and the direct SDK update-alias contract. + These took 223.152 s elapsed, 262.538 s child CPU and 1079.2 MiB peak family RSS. +- A new negative control showed that calling a collection encoder for a single + array-valued row undercounted tokens (15 instead of 21, 57 instead of 63). + Emission and counting now share `jsonLineParts`; both cases and all 29 JSONL + tests pass after the fix. The 241-test run includes these regressions. +- A real `rea mcp doctor --format jsonl` run reported healthy in one JSON line. + Startup/validation failures, full-output framing and advertised help are + exercised separately. + +The opt-in verifier used shared 64 KiB leaves and independently checked complete +output bytes and SHA-256, without retaining the complete encoded document: + +| Synthetic serialization fixture | JSON | JSONL | +| ----------------------------------------------- | ---------------: | ----------------: | +| Output bytes (engine string limit: 536,870,888) | 536,961,655 | 536,961,779 | +| Elapsed / child CPU | 9.811 / 10.197 s | 13.940 / 14.451 s | +| Sampled peak family RSS | 171.6 MiB | 186.0 MiB | + +The JSONL run also checked its token count against the actual file line by line: +89,492,357 heuristic tokens matched. JSONL timing includes this extra counting +verification, so the two fixture timings are not equivalent-workflow performance +comparisons. Shared leaves keep input memory small; this verifies serialization +expansion and counting capacity, not a 512 MiB application's analysis footprint. +Both temporary output files were removed by the verifier's cleanup. + +No resource stop occurred. Native provider behavior is outside this CLI change. +`npm run docs:check` passed, including nine README consistency tests. + +The refreshed measurements are in the `refreshed/` subdirectory; earlier +measurements against `169512a35` remain separate. Raw stdout/stderr, commands, +per-run resource samples, equivalence assertions and summaries are retained under +`/tmp/rea-maintenance-20261023/5629/`; they are not committed artifacts. diff --git a/scripts/rea.mjs b/scripts/rea.mjs index 30d9c23e27..c53f64b92c 100755 --- a/scripts/rea.mjs +++ b/scripts/rea.mjs @@ -140,6 +140,7 @@ if (!(await compiledRuntimeExists(runtimeFiles))) { return; } if (output.length > 0) wroteOutput = true; + if (resultOutput?.captureSdkOutput(output)) return; writeStdout(output); }, // A command failure caused by the closed pipe keeps the pipe's status. @@ -147,26 +148,25 @@ if (!(await compiledRuntimeExists(runtimeFiles))) { // shutdown is reported here because this callback is what exits. exit: (code) => { exiting = true; - const status = stdoutClosed ? SIGPIPE_EXIT_STATUS : code; - void Promise.resolve() - .then(() => closeCliJavaScriptResources()) - .then( - () => { - process.exit(status); - }, - (cleanup) => { - process.stderr.write( - `Failed to close the JavaScript analysis worker: ${cleanupFailureText(cleanup)}\n`, - ); - process.exit( - stdoutClosed - ? SIGPIPE_EXIT_STATUS - : status === 0 - ? 1 - : status, - ); - }, - ); + void (async () => { + let status = code; + try { + await closeCliJavaScriptResources(); + } catch (cleanup) { + process.stderr.write( + `Failed to close the JavaScript analysis worker: ${cleanupFailureText(cleanup)}\n`, + ); + if (status === 0) status = 1; + } + try { + await resultOutput?.finishSdkOutput(code); + } catch (cause) { + if (!stdoutClosed) + process.stderr.write(`${cleanupFailureText(cause)}\n`); + if (status === 0) status = 1; + } + process.exit(stdoutClosed ? SIGPIPE_EXIT_STATUS : status); + })(); }, }); } catch (cause) { @@ -184,6 +184,11 @@ if (!(await compiledRuntimeExists(runtimeFiles))) { } if (!wroteOutput && !resultOutput?.handled) writeStdout(renderEmptyFilteredCliOutput(outputOptions) ?? ""); + // An SDK failure exits through the callback, which owns its pending write. + if (!exiting) + await resultOutput?.finishSdkOutput( + typeof process.exitCode === "number" ? process.exitCode : 0, + ); } } catch (cause) { dispatchFailure = cause; diff --git a/scripts/verify/javascript/output.mjs b/scripts/verify/javascript/output.mjs index 1191f9a758..e63856409a 100644 --- a/scripts/verify/javascript/output.mjs +++ b/scripts/verify/javascript/output.mjs @@ -2,6 +2,8 @@ import assert from "node:assert/strict"; import { constants } from "node:buffer"; import { createHash } from "node:crypto"; import { once } from "node:events"; +import { createInterface } from "node:readline"; +import { Writable } from "node:stream"; import { createReadStream, createWriteStream } from "node:fs"; import { mkdtemp, rm, stat, statfs } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -9,6 +11,7 @@ import { join } from "node:path"; import { finished } from "node:stream/promises"; import { Cli } from "incur"; +import { estimateTokenCount } from "tokenx"; import { parseCliOutputArguments } from "../../../dist/cliOutput.js"; import { createCliResultOutput } from "../../../dist/cli/cliResultOutput.js"; @@ -44,26 +47,47 @@ try { { operation: "inspect_fixture", parameters: {}, - result: { payload: Array.from({ length: count }, () => leaf) }, + result: { rows: Array.from({ length: count }, () => leaf) }, }, ); sampleRss(); const metadata = { command: "inspect-output-fixture", duration: "0ms" }; const marker = "__REA_OUTPUT_ENCODING_MARKER__"; - const template = { - ok: true, - data: { ...evidence, normalized_result: { payload: [marker] } }, - meta: metadata, - }; - // The destination is a redirected file, so both formats are compact. - const skeleton = JSON.stringify(template); - const encodedMarker = JSON.stringify(marker); - const markerIndex = skeleton.indexOf(encodedMarker); - assert.ok(markerIndex >= 0); - assert.equal(markerIndex, skeleton.lastIndexOf(encodedMarker)); - const prefix = skeleton.slice(0, markerIndex); - const suffix = `${skeleton.slice(markerIndex + encodedMarker.length)}\n`; - const separator = ","; + let prefix; + let suffix; + if (format === "json") { + const template = { + ok: true, + data: { ...evidence, normalized_result: { rows: [marker] } }, + meta: metadata, + }; + const skeleton = JSON.stringify(template); + const encodedMarker = JSON.stringify(marker); + const markerIndex = skeleton.indexOf(encodedMarker); + assert.ok(markerIndex >= 0); + assert.equal(markerIndex, skeleton.lastIndexOf(encodedMarker)); + prefix = skeleton.slice(0, markerIndex); + suffix = `${skeleton.slice(markerIndex + encodedMarker.length)}\n`; + } else { + // Independent protocol golden: rows are lines; the final record retains + // every non-row fact and the caller's full-output envelope. + prefix = ""; + suffix = `\n${JSON.stringify({ + $rea: "jsonl/v1", + type: "metadata", + records: { + path: ["data", "normalized_result", "rows"], + emitted: count, + source: count, + }, + context: { + ok: true, + data: { ...evidence, normalized_result: {} }, + meta: metadata, + }, + })}\n`; + } + const separator = format === "jsonl" ? "\n" : ","; const expectedHash = createHash("sha256"); expectedHash.update(prefix); for (let index = 0; index < count; index += 1) { @@ -109,6 +133,7 @@ try { assert.equal(exitCode, 0); assert.equal(output.handled, true); assert.equal(output.failed, false); + assert.equal(output.error, undefined); assert.equal(unexpectedOutputBytes, 0); const actualHash = createHash("sha256"); @@ -116,12 +141,50 @@ try { const digest = actualHash.digest("hex"); assert.equal(digest, expectedHash.digest("hex")); assert.equal((await stat(path)).size, expectedBytes); + let verifiedTokenCount; + if (format === "jsonl") { + let expectedTokens = 0; + for await (const line of createInterface({ + input: createReadStream(path), + crlfDelay: Infinity, + })) + expectedTokens += estimateTokenCount(line); + let tokenText = ""; + const counterDestination = new Writable({ + write(chunk, _encoding, callback) { + // The numeric count is tiny; never retain a mistakenly emitted report. + if (tokenText.length + chunk.length > 64) { + callback( + new Error("Token counter emitted more than a numeric count"), + ); + return; + } + tokenText += chunk.toString("utf8"); + callback(); + }, + }); + const counter = createCliResultOutput( + parseCliOutputArguments([...argv, "--token-count"]), + counterDestination, + ); + assert.ok(counter); + assert.equal(await counter.write(evidence, { ...metadata, format }), true); + assert.equal(counter.error, undefined); + assert.equal(counter.failed, false); + counterDestination.end(); + await finished(counterDestination); + assert.equal(tokenText, `${expectedTokens}\n`); + verifiedTokenCount = expectedTokens; + } sampleRss(); finalReport = { json_output_verified: true, format, output_bytes: expectedBytes, engine_string_limit: constants.MAX_STRING_LENGTH, + ...(verifiedTokenCount === undefined + ? {} + : { verified_token_count: verifiedTokenCount }), output_sha256: digest, evidence_id: evidence.evidence_id, rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss }, diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index 9f7d25471c..ff399ed97c 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -16,6 +16,8 @@ import { type AnalysisErrorProjection, } from "../domain/analysisErrorProjection.js"; import { formatCliOutput } from "./cliOutputFormatter.js"; +import { jsonLinesErrorRecord } from "./jsonLines.js"; +import { writeCliJsonLines } from "./jsonLinesOutput.js"; import { jsonOutputParts, writeJsonOutput, @@ -40,6 +42,13 @@ export interface CliResultOutput { readonly handled: boolean; readonly failed: boolean; readonly error: AnalysisErrorProjection | undefined; + /** Start another completed invocation on a reusable command registry. */ + begin(options: ParsedCliOutputArguments): void; + /** SDK generators retain their own streaming event protocol. */ + delegateStreaming(): void; + /** Retain one-shot SDK output until the SDK supplies its authoritative status. */ + captureSdkOutput(text: string): boolean; + finishSdkOutput(exitCode: number): Promise; write(value: unknown, metadata: CliResultOutputMetadata): Promise; /** Recover only before any delegated stdout has been written. */ recoverFormattingFailure(cause: unknown): Promise; @@ -70,8 +79,10 @@ const wrapCommand = ( typeof result === "object" && result !== null && Symbol.asyncIterator in result - ) + ) { + output.delegateStreaming(); return result; + } return Promise.resolve(result).then(async (value: unknown) => { if ( typeof value === "object" && @@ -100,14 +111,22 @@ const wrapCommand = ( /** Preserve Incur output controls and stream JSON while framing failures. */ export const createCliResultOutput = ( - options: ParsedCliOutputArguments, + initialOptions: ParsedCliOutputArguments, destination: Writable, ): CliResultOutput | undefined => { - if (options.parseError || !options.validation.ok) return undefined; - const format = options.format as "json" | "jsonl" | "yaml" | "toon" | "md"; + let options = initialOptions; + if ( + (options.parseError && options.format !== "jsonl") || + !options.validation.ok + ) + return undefined; + let format = options.format as "json" | "jsonl" | "yaml" | "toon" | "md"; let handled = false; let failed = false; let error: AnalysisErrorProjection | undefined; + let streaming = false; + const sdkParts: string[] = []; + let sdkCompletion: Promise | undefined; let completed: | { readonly metadata: CliResultOutputMetadata; @@ -160,7 +179,10 @@ export const createCliResultOutput = ( : error; // Recovery must remain actionable even when a token window/count was // requested. Do not apply those controls to this small typed failure. - const text = formatCliOutput(document, format); + const text = formatCliOutput( + format === "jsonl" ? jsonLinesErrorRecord(document) : document, + format, + ); handled = true; try { await writeOutputText(destination, `${text}\n`); @@ -193,6 +215,55 @@ export const createCliResultOutput = ( get error() { return error; }, + begin(nextOptions) { + options = nextOptions; + format = options.format as typeof format; + handled = false; + failed = false; + error = undefined; + completed = undefined; + streaming = false; + sdkParts.length = 0; + sdkCompletion = undefined; + }, + delegateStreaming() { + streaming = true; + }, + captureSdkOutput(text) { + if (format !== "jsonl" || streaming || handled) return false; + // Incur already materializes one-shot documents. Retain references only; + // generator events bypass this buffer and remain incremental. + if (text.length > 0) sdkParts.push(text); + return true; + }, + finishSdkOutput(exitCode) { + return (sdkCompletion ??= (async () => { + const parts = sdkParts.splice(0); + if (parts.length === 0) return; + try { + if (exitCode === 0) { + for (const part of parts) await writeOutputText(destination, part); + } else { + const text = parts.join(""); + let result: unknown = text; + try { + result = JSON.parse(text); + } catch { + /* Preserve non-JSON SDK diagnostics verbatim. */ + } + await writeJsonOutput( + jsonLinesErrorRecord(result), + destination, + "jsonl", + ); + } + } catch (cause: unknown) { + handled = true; + failed = true; + throw cause; + } + })()); + }, write, recoverFormattingFailure, }; @@ -209,7 +280,7 @@ export const createCliResultOutput = ( ? Filter.parse(options.filterOutput) : undefined; const operationFailure = - options.fullOutput || filterPaths !== undefined + format === "jsonl" || options.fullOutput || filterPaths !== undefined ? cliOperationFailurePayload(value) : undefined; const failure = options.fullOutput ? operationFailure : undefined; @@ -222,6 +293,7 @@ export const createCliResultOutput = ( if ( operationFailure === undefined && metadata.renderOutput === false && + format !== "jsonl" && !options.fullOutput && !options.tokenCount ) @@ -254,6 +326,26 @@ export const createCliResultOutput = ( !ownsSuccessfulTokenWindow ) return false; + if (format === "jsonl") { + handled = true; + try { + error = await writeCliJsonLines( + { + ...options, + source: value, + selected: filtered, + operationFailure, + command: metadata.command, + duration: metadata.duration, + }, + destination, + ); + } catch (cause: unknown) { + failed = true; + throw cause; + } + return true; + } if (options.tokenCount) { // Incur/tokenx materialize JSON. Measure its UTF-16 expansion before // allowing that allocation, independently of UTF-8 bytes. Other formats @@ -266,7 +358,7 @@ export const createCliResultOutput = ( typeof tokenValue !== "function" && typeof tokenValue !== "symbol" ) { - if (format === "json" || format === "jsonl") { + if (format === "json") { for (const part of jsonOutputParts( tokenValue, format, @@ -382,8 +474,7 @@ export const createCliResultOutput = ( const formattedText = format === "md" || ((operationFailure !== undefined || ownsFilteredOutput) && - format !== "json" && - format !== "jsonl") + format !== "json") ? formatCliOutput(document, format) : undefined; if ( diff --git a/src/cli/jsonLines.ts b/src/cli/jsonLines.ts new file mode 100644 index 0000000000..f4ecdab3e1 --- /dev/null +++ b/src/cli/jsonLines.ts @@ -0,0 +1,226 @@ +import { err, ok, type Result } from "../domain/result.js"; + +import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; +import { jsonParts } from "../domain/jsonSerialization.js"; + +/** Reserved control-record marker, not an advertised external JSON Schema. */ +export const JSON_LINES_PROTOCOL = "jsonl/v1"; + +interface JsonLinesErrorRecord { + readonly $rea: typeof JSON_LINES_PROTOCOL; + readonly type: "error"; + readonly ok: false; + readonly result: unknown; +} + +interface JsonLinesMetadataRecord { + readonly $rea: typeof JSON_LINES_PROTOCOL; + readonly type: "metadata"; + readonly records: { + readonly path: readonly string[]; + readonly emitted: number; + readonly source: number | null; + readonly filter?: string; + }; + readonly context: Readonly> | null; +} + +const COLLECTION_KEYS = ["rows", "items", "records", "occurrences"] as const; + +interface Collection { + readonly records: readonly unknown[]; + readonly path: readonly string[]; +} + +/** Parsed once, before output or token counting acquires the stream. */ +export interface JsonLinesPlan extends Collection { + readonly context: Readonly> | undefined; + readonly sourceRecords: number | null; + readonly filter: string | undefined; +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const atPath = (value: unknown, path: readonly string[]): unknown => { + for (const key of path) { + if (!isRecord(value) || !Object.hasOwn(value, key)) return undefined; + value = value[key]; + } + return value; +}; + +const collections = ( + value: unknown, + prefix: readonly string[], +): Collection[] => { + if (Array.isArray(value)) return [{ records: value, path: prefix }]; + if (!isRecord(value)) return []; + const found: Collection[] = []; + const collect = ( + object: Record, + path: readonly string[], + ) => { + for (const key of COLLECTION_KEYS) { + const records = object[key]; + if (Object.hasOwn(object, key) && Array.isArray(records)) + found.push({ records, path: [...path, key] }); + } + }; + collect(value, prefix); + if (Object.hasOwn(value, "page") && isRecord(value.page)) + collect(value.page, [...prefix, "page"]); + return found; +}; + +const documentCollections = ( + value: unknown, + fullOutput: boolean, +): Collection[] => { + const prefix = fullOutput ? ["data"] : []; + const data = atPath(value, prefix); + return [ + ...collections(data, prefix), + ...collections(atPath(data, ["normalized_result"]), [ + ...prefix, + "normalized_result", + ]), + ]; +}; + +/** Shallow copies only ancestors of the emitted array; every other fact survives. */ +const withoutRecords = ( + source: unknown, + selected: unknown, + path: readonly string[], + filtered: boolean, +): Record | undefined => { + if (path.length === 0 || !isRecord(source)) return undefined; + const context = { ...source }; + const selectedObject = isRecord(selected) ? selected : undefined; + // A data filter intentionally excludes raw observation payloads. Keep its + // selected raw value if requested, while always retaining Evidence provenance. + if ( + filtered && + Object.hasOwn(source, "evidence_id") && + typeof source.evidence_id === "string" && + Object.hasOwn(source, "normalized_result") + ) { + if (selectedObject && Object.hasOwn(selectedObject, "raw_result")) + context.raw_result = selectedObject.raw_result; + else delete context.raw_result; + } + const [key, ...rest] = path; + if (key === undefined) return context; + if (rest.length === 0) delete context[key]; + else + context[key] = withoutRecords( + source[key], + selectedObject?.[key], + rest, + filtered, + ); + return context; +}; + +/** Project a clear collection without interpreting or whitelisting its context. */ +export const prepareJsonLines = ( + selected: unknown, + options: { + readonly command: string; + readonly source?: unknown; + readonly filter?: string; + readonly fullOutput?: boolean; + }, +): Result => { + const found = documentCollections(selected, options.fullOutput === true); + const reject = (reason: string) => + err( + new AnalysisOutputError(options.command, reason, { + remediationAction: + "Use --format json for the complete document, or select one record collection with --filter-output.", + }), + ); + if (found.length !== 1) + return reject( + found.length === 0 + ? "JSON Lines requires an array or one explicit rows, items, records, or occurrences collection." + : "JSON Lines cannot choose between multiple record collections. Select one collection explicitly.", + ); + const collection = found[0]; + if (collection === undefined) + return reject("The record collection is unavailable."); + for (const record of collection.records) { + if (isRecord(record) && Object.hasOwn(record, "$rea")) + return reject( + "A data record uses the reserved $rea JSON Lines control marker.", + ); + } + const source = Object.hasOwn(options, "source") ? options.source : selected; + const originalRecords = atPath(source, collection.path); + const sourceMatches = Array.isArray(originalRecords); + let context = withoutRecords( + sourceMatches ? source : selected, + selected, + collection.path, + options.filter !== undefined, + ); + if (options.filter !== undefined && sourceMatches) { + // Excluded peer rows are data, not context. A filter can disambiguate them + // without restoring an unselected collection into the final record. + for (const peer of documentCollections( + source, + options.fullOutput === true, + )) { + if (peer.path.join(".") !== collection.path.join(".")) + context = withoutRecords(context, selected, peer.path, true); + } + } + return ok({ + ...collection, + context, + sourceRecords: sourceMatches ? originalRecords.length : null, + filter: options.filter, + }); +}; + +/** The caller supplies the outcome; error-shaped data is never reclassified here. */ +export const jsonLinesErrorRecord = ( + result: unknown, +): JsonLinesErrorRecord => ({ + $rea: JSON_LINES_PROTOCOL, + type: "error" as const, + ok: false as const, + result, +}); + +export function* jsonLinesParts(plan: JsonLinesPlan): Generator { + for (const record of jsonLinesRecords(plan)) { + yield* jsonLineParts(record); + } +} + +/** One record stays one line even when the record itself is an array. */ +export function* jsonLineParts(record: unknown): Generator { + yield* jsonParts(record); + yield "\n"; +} + +/** Newline-separated records also bound the token estimator's working set. */ +export function* jsonLinesRecords(plan: JsonLinesPlan): Generator { + yield* plan.records; + if (plan.context !== undefined || plan.path.length > 0) { + const metadata: JsonLinesMetadataRecord = { + $rea: JSON_LINES_PROTOCOL, + type: "metadata", + records: { + path: plan.path, + emitted: plan.records.length, + source: plan.sourceRecords, + ...(plan.filter === undefined ? {} : { filter: plan.filter }), + }, + context: plan.context ?? null, + }; + yield metadata; + } +} diff --git a/src/cli/jsonLinesOutput.ts b/src/cli/jsonLinesOutput.ts new file mode 100644 index 0000000000..397b5d4664 --- /dev/null +++ b/src/cli/jsonLinesOutput.ts @@ -0,0 +1,111 @@ +import { constants } from "node:buffer"; +import type { Writable } from "node:stream"; +import { estimateTokenCount } from "tokenx"; + +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { + projectAnalysisError, + type AnalysisErrorProjection, +} from "../domain/analysisErrorProjection.js"; +import { + jsonLineParts, + jsonLinesErrorRecord, + jsonLinesParts, + jsonLinesRecords, + prepareJsonLines, +} from "./jsonLines.js"; +import { + writeJsonOutput, + writeOutputParts, + writeOutputText, +} from "./jsonOutput.js"; + +interface CliJsonLinesInput { + readonly source: unknown; + readonly selected: unknown; + readonly operationFailure: unknown; + readonly command: string; + readonly duration: string; + readonly fullOutput: boolean; + readonly tokenCount: boolean; + readonly filterOutput: string | undefined; +} + +/** Render one already classified result, sharing a plan for output and counting. */ +export const writeCliJsonLines = async ( + options: CliJsonLinesInput, + destination: Writable, +): Promise => { + const { source, selected, operationFailure } = options; + let error: AnalysisErrorProjection | undefined; + const meta = { command: options.command, duration: options.duration }; + const envelope = (data: unknown) => + options.fullOutput ? { ok: true, data, meta } : data; + let failureRecord = + operationFailure === undefined + ? undefined + : jsonLinesErrorRecord( + options.fullOutput + ? { ok: false, error: operationFailure, meta } + : operationFailure, + ); + const plan = + failureRecord === undefined + ? prepareJsonLines(envelope(selected), { + command: options.command, + source: envelope(source), + ...(options.filterOutput === undefined + ? {} + : { filter: options.filterOutput }), + fullOutput: options.fullOutput, + }) + : undefined; + if (plan?.ok === false) { + error = projectAnalysisError(plan.error); + failureRecord = jsonLinesErrorRecord( + options.fullOutput ? { ok: false, error, meta } : error, + ); + } + if (failureRecord !== undefined) { + await writeJsonOutput(failureRecord, destination, "jsonl"); + } else if (plan?.ok === true) { + if (options.tokenCount) { + let count = 0; + for (const record of jsonLinesRecords(plan.value)) { + let characters = 0; + for (const part of jsonLineParts(record)) characters += part.length; + if (characters > constants.MAX_STRING_LENGTH) { + error = projectAnalysisError( + new AnalysisResourceConstraintError( + options.command, + "memory", + "A JSON Lines record exceeds the token estimator's runtime string limit.", + { + formatted_characters: characters, + max_string_characters: constants.MAX_STRING_LENGTH, + }, + { + remediationAction: + "Remove --token-count to stream the records, or select smaller fields with --filter-output.", + }, + ), + ); + break; + } + // Newlines split tokenx segments, so summing complete lines is + // identical to estimating the emitted stream, with one-line memory. + count += estimateTokenCount([...jsonLineParts(record)].join("")); + } + if (error !== undefined) + await writeJsonOutput( + jsonLinesErrorRecord( + options.fullOutput ? { ok: false, error, meta } : error, + ), + destination, + "jsonl", + ); + else await writeOutputText(destination, `${count}\n`); + } else await writeOutputParts(destination, jsonLinesParts(plan.value)); + } + return error; +}; diff --git a/src/cli/jsonOutput.ts b/src/cli/jsonOutput.ts index f3b2bcaa08..5c28b9d6ea 100644 --- a/src/cli/jsonOutput.ts +++ b/src/cli/jsonOutput.ts @@ -16,9 +16,18 @@ export const writeJsonOutput = async ( destination: Writable, format: JsonOutputFormat = "json", ): Promise => { - for (const part of bufferedJsonParts( + await writeOutputParts( + destination, jsonOutputParts(value, format, isTerminal(destination)), - )) + ); +}; + +/** Share bounded writes and destination ownership across structured formats. */ +export const writeOutputParts = async ( + destination: Writable, + parts: Iterable, +): Promise => { + for (const part of bufferedJsonParts(parts)) await writeOutputText(destination, part); }; diff --git a/src/cliOutput.ts b/src/cliOutput.ts index a8f32cbfdc..52405aa590 100644 --- a/src/cliOutput.ts +++ b/src/cliOutput.ts @@ -1,6 +1,7 @@ import { formatCliOutput } from "./cli/cliOutputFormatter.js"; import { jsonOutputParts } from "./cli/jsonOutput.js"; +import { jsonLinesErrorRecord } from "./cli/jsonLines.js"; import { versionReport, type VersionReportHost, @@ -382,7 +383,7 @@ export const renderCliOutputArgumentError = ( }; if (error.format === "yaml") return `ok: false\nerror:\n code: ${error.code}\n message: ${JSON.stringify(error.message)}\n`; - return `${JSON.stringify(value)}\n`; + return `${JSON.stringify(error.format === "jsonl" ? jsonLinesErrorRecord(value) : value)}\n`; }; /** diff --git a/src/mcpDoctor.ts b/src/mcpDoctor.ts index fa33332379..79f6d9a6e6 100644 --- a/src/mcpDoctor.ts +++ b/src/mcpDoctor.ts @@ -6,6 +6,7 @@ import { resolve } from "node:path"; import { StringDecoder } from "node:string_decoder"; import { createCatalogIdentity } from "./catalogIdentity.js"; +import { jsonLinesErrorRecord } from "./cli/jsonLines.js"; import { CLI_OUTPUT_FORMATS, type CliOutputFormat, @@ -343,7 +344,9 @@ export const runProductionMcpDoctorCli = async ( return { output: `${renderMcpDoctorHelp()}\n`, exitCode: 0 }; return { output: renderCliDocument( - { code: parsed.code, message: parsed.message }, + parsed.format === "jsonl" + ? jsonLinesErrorRecord({ code: parsed.code, message: parsed.message }) + : { code: parsed.code, message: parsed.message }, parsed.format, input.stdoutIsTerminal, ), @@ -368,7 +371,9 @@ export const runProductionMcpDoctorCli = async ( : result; return { output: renderCliDocument( - outputValue, + parsed.format === "jsonl" && !result.healthy + ? jsonLinesErrorRecord(outputValue) + : outputValue, parsed.format, input.stdoutIsTerminal, ), diff --git a/tests/acceptance/jsonLinesCli.test.ts b/tests/acceptance/jsonLinesCli.test.ts new file mode 100644 index 0000000000..ba4690ebb4 --- /dev/null +++ b/tests/acceptance/jsonLinesCli.test.ts @@ -0,0 +1,92 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { parseEvidence } from "../../src/domain/evidence.js"; +import { analysisViewJavaScriptEvidence } from "../fixtures/analysisView.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest( + "exports authenticated analysis-view rows, preserving every Evidence fact and empty/partial coverage", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-json-lines-view-"); + const parent = analysisViewJavaScriptEvidence(); + const evidencePath = join(root, "parent.json"); + const requestPath = join(root, "request.json"); + await writeFile(evidencePath, JSON.stringify(parent)); + const request = { + source: { kind: "evidence-file", path: evidencePath }, + view: { kind: "javascript-facts", property_keys: [], limit: 1 }, + }; + await writeFile(requestPath, JSON.stringify(request)); + + const invalid = await cli.run({ + arguments: ["inspect-analysis-view", "--format", "jsonl"], + }); + expect(invalid.exitCode).toBe(1); + expect(JSON.parse(invalid.stdout)).toMatchObject({ + $rea: "jsonl/v1", + type: "error", + result: { code: "VALIDATION_ERROR" }, + }); + + const badOutputValue = await cli.run({ + arguments: ["--format", "jsonl", "--token-limit", "invalid"], + }); + expect(badOutputValue.exitCode).toBe(1); + expect(JSON.parse(badOutputValue.stdout)).toMatchObject({ + $rea: "jsonl/v1", + type: "error", + }); + + for (const where of [undefined, { paths: [] }]) { + await writeFile( + requestPath, + JSON.stringify({ + ...request, + view: { ...request.view, ...(where === undefined ? {} : { where }) }, + }), + ); + const json = await cli.run({ + arguments: ["inspect-analysis-view", requestPath, "--json"], + }); + expect(json.exitCode, json.stderr).toBe(0); + const expected = parseEvidence(json.json); + const result = await cli.run({ + arguments: ["inspect-analysis-view", requestPath, "--format", "jsonl"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const records = result.stdout + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line)); + const final = records.pop(); + expect(final).toMatchObject({ + $rea: "jsonl/v1", + type: "metadata", + records: { + path: ["normalized_result", "rows"], + emitted: records.length, + source: records.length, + }, + context: { + evidence_links: [parent.evidence_id], + normalized_result: { parent_evidence_id: parent.evidence_id }, + }, + }); + const reconstructed = { + ...final.context, + normalized_result: { + ...final.context.normalized_result, + rows: records, + }, + }; + // Authenticate the reconstructed Evidence, rather than checking only a few + // convenient fields from a shortened view-shaped object. + expect(parseEvidence(reconstructed)).toEqual(expected); + expect(records.length).toBe(where === undefined ? 1 : 0); + } + }, +); diff --git a/tests/acceptance/managedCliOutput.test.ts b/tests/acceptance/managedCliOutput.test.ts index b83d237552..44d217bbce 100644 --- a/tests/acceptance/managedCliOutput.test.ts +++ b/tests/acceptance/managedCliOutput.test.ts @@ -315,8 +315,15 @@ cliTest( "operation", ], }); - expect(jsonl.exitCode).toBe(0); - expect(jsonl.json).toBe("inspect_managed_members"); + expect(jsonl.exitCode).toBe(1); + expect(jsonl.json).toMatchObject({ + $rea: "jsonl/v1", + type: "error", + result: { + code: "unreadable_output", + remediation: { action: expect.stringContaining("--format json") }, + }, + }); }, ); diff --git a/tests/boundary/cli/cliOutput.test.ts b/tests/boundary/cli/cliOutput.test.ts index d6473e3b6f..00306b3cc0 100644 --- a/tests/boundary/cli/cliOutput.test.ts +++ b/tests/boundary/cli/cliOutput.test.ts @@ -43,7 +43,16 @@ describe("CLI output argument boundary", () => { if (!validation.ok) { const rendered = renderCliOutputArgumentError(validation); expect(rendered).not.toContain("[truncated:"); - if (format === "json" || format === "jsonl") + if (format === "jsonl") + expect(JSON.parse(rendered)).toMatchObject({ + $rea: "jsonl/v1", + type: "error", + result: { + ok: false, + error: { code: "UNSUPPORTED_OUTPUT_COMBINATION" }, + }, + }); + else if (format === "json") expect(JSON.parse(rendered)).toMatchObject({ ok: false, error: { code: "UNSUPPORTED_OUTPUT_COMBINATION" }, diff --git a/tests/boundary/cli/dispatcher.test.ts b/tests/boundary/cli/dispatcher.test.ts index d1a47feaea..08a5496d5d 100644 --- a/tests/boundary/cli/dispatcher.test.ts +++ b/tests/boundary/cli/dispatcher.test.ts @@ -97,7 +97,7 @@ describe("executable dispatcher", () => { await installCliRuntime( workspace, 'export const closeCliJavaScriptResources = () => {}; export const createCli = () => ({ serve: (_args, options) => { options.stdout("first document\\n"); throw new RangeError("Invalid string length"); } });\n', - 'export const createCliResultOutput = () => ({ handled: false, failed: false, recoverFormattingFailure: () => { process.stdout.write("unexpected second document"); return true; } });\n', + 'export const createCliResultOutput = () => ({ handled: false, failed: false, captureSdkOutput: () => false, finishSdkOutput: async () => {}, recoverFormattingFailure: () => { process.stdout.write("unexpected second document"); return true; } });\n', ); const result = await processes.run( process.execPath, @@ -225,6 +225,45 @@ describe("executable dispatcher", () => { }); describe("executable dispatcher JavaScript shutdown", () => { + workspaceCliTest.for([ + "success", + "cleanup-failure", + "write-failure", + "closed-pipe", + ] as const)( + "awaits SDK output after cleanup and retains finalization status: %s", + async (scenario, { processes, workspace }) => { + const cleanupFails = scenario === "cleanup-failure"; + const finish = + scenario === "closed-pipe" + ? 'process.stdout.emit("error", Object.assign(new Error("broken pipe"), { code: "EPIPE" })); throw new Error("broken pipe");' + : scenario === "write-failure" + ? 'throw new Error("final write failed");' + : 'process.stdout.write("complete document\\n");'; + await installCliRuntime( + workspace, + `export const closeCliJavaScriptResources = () => { globalThis.cleaned = true; ${cleanupFails ? 'return Promise.reject(new Error("worker still owned"));' : ""} }; export const createCli = () => ({ serve: (_args, options) => { options.stdout("pending document"); options.exit(0); } });\n`, + `export const createCliResultOutput = () => ({ handled: false, failed: false, captureSdkOutput: () => true, finishSdkOutput: async () => { if (!globalThis.cleaned) throw new Error("output preceded cleanup"); await new Promise(resolve => setTimeout(resolve, 10)); ${finish} } });\n`, + ); + const result = await processes.run( + process.execPath, + [workspace.path("scripts/rea.mjs"), "fixture"], + { cwd: workspace.root }, + ); + expect(result.exitCode).toBe( + scenario === "closed-pipe" ? 141 : scenario === "success" ? 0 : 1, + ); + expect(result.stdout).toBe( + scenario === "success" || cleanupFails ? "complete document\n" : "", + ); + expect(result.stderr).not.toContain("output preceded cleanup"); + if (cleanupFails) expect(result.stderr).toContain("worker still owned"); + if (scenario === "write-failure") + expect(result.stderr).toContain("final write failed"); + if (scenario === "closed-pipe") expect(result.stderr).toBe(""); + }, + ); + workspaceCliTest( "reports a failed JavaScript worker shutdown instead of exiting successfully", async ({ processes, workspace }) => { diff --git a/tests/boundary/cli/filteredArrayOutput.test.ts b/tests/boundary/cli/filteredArrayOutput.test.ts index 5de47b4142..28900bddf7 100644 --- a/tests/boundary/cli/filteredArrayOutput.test.ts +++ b/tests/boundary/cli/filteredArrayOutput.test.ts @@ -97,7 +97,19 @@ it.each( }, }); const text = Buffer.concat(chunks).toString("utf8"); - if (full) + if (full && format === "jsonl") { + const records = text + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line)); + expect(records.slice(0, -1)).toEqual([1, null, null]); + expect(records.at(-1)).toMatchObject({ + $rea: "jsonl/v1", + type: "metadata", + records: { path: ["data"], emitted: 3, source: 3, filter: "selected" }, + context: { ok: true, meta: { command: "rows" } }, + }); + } else if (full) expect(JSON.parse(text)).toMatchObject({ ok: true, data: [1, null, null], diff --git a/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts b/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts index 3008954f03..5dfd117db2 100644 --- a/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts +++ b/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts @@ -28,7 +28,7 @@ for (const scenario of [ { command: "analyze", format: "jsonl", fullOutput: false, filtered: false }, ] as const) { cliTest( - `${scenario.command} preserves representative ${scenario.format} output`, + `${scenario.command} ${scenario.format === "jsonl" ? "refuses graph documents without a supported record boundary" : "preserves representative JSON output"}`, async ({ cli }) => { const root = await createTestTempDirectory("rea-js-json-output-"); await writeFile(join(root, "main.js"), "fetch('');\n"); @@ -47,6 +47,31 @@ for (const scenario of [ ...(scenario.filtered ? ["--filter-output", filter] : []), ], }); + if (scenario.format === "jsonl") { + expect(output.exitCode).toBe(1); + const record = z + .object({ + $rea: z.literal("jsonl/v1"), + type: z.literal("error"), + result: z.unknown(), + }) + .parse(output.json); + const failure = scenario.fullOutput + ? z + .object({ + ok: z.literal(false), + error: z.unknown(), + meta: z.object({ command: z.literal(scenario.command) }), + }) + .parse(record.result).error + : record.result; + expect(failure).toMatchObject({ + code: "unreadable_output", + remediation: { action: expect.stringContaining("--format json") }, + details: { reason: expect.stringContaining("JSON Lines requires") }, + }); + return; + } expect(output.exitCode).toBe(0); let data: unknown = output.json; if (scenario.fullOutput) { diff --git a/tests/boundary/cli/jsonLinesOutput.test.ts b/tests/boundary/cli/jsonLinesOutput.test.ts new file mode 100644 index 0000000000..aa77b1e6f6 --- /dev/null +++ b/tests/boundary/cli/jsonLinesOutput.test.ts @@ -0,0 +1,451 @@ +import { Writable } from "node:stream"; + +import { Cli, z } from "incur"; +import { estimateTokenCount } from "tokenx"; +import { describe, expect, it } from "vitest"; + +import { + createCliResultOutput, + streamCliCommandResults, +} from "../../../src/cli/cliResultOutput.js"; +import { jsonLinesErrorRecord } from "../../../src/cli/jsonLines.js"; +import { parseCliOutputArguments } from "../../../src/cliOutput.js"; +import { logCliCommand } from "../../../src/cliLogging.js"; +import { silentLogger } from "../../../src/logger.js"; + +const capture = (args: readonly string[]) => { + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + const output = createCliResultOutput( + parseCliOutputArguments(args), + destination, + ); + if (output === undefined) throw new Error("Missing output owner"); + return { + output, + destination, + text: () => Buffer.concat(chunks).toString("utf8"), + }; +}; +const lines = (text: string): unknown[] => + text.trimEnd() === "" + ? [] + : text + .trimEnd() + .split("\n") + .map((line) => JSON.parse(line)); +const metadata = { command: "fixture", duration: "1ms", format: "jsonl" }; + +describe("row-oriented CLI JSON Lines", () => { + it.each(["rows", "items", "records", "occurrences"])( + "preserves all context around %s without a field whitelist", + async (key) => { + const records = [ + { id: "one", index: 8, text: "first\n雪𝟠" }, + { id: "two", index: 9 }, + ]; + const context = { + evidence_id: "ev:parent", + subject: { path: "/local/subject" }, + provider: "actual-provider", + authority: "static", + confidence: "derived", + locations: [{ path: "/source", line: 4 }], + parameters: { field: "caller choice" }, + raw_result: null, + evidence_links: ["ev:source"], + limitations: ["partial source"], + normalized_result: { + kind: "inventory", + parent_digest: "sha256:parent", + view_digest: "sha256:view", + artifact: { id: "art:one" }, + coverage: { complete: false, examined: 2, total: null }, + page: { offset: 8, has_more: true }, + [key]: records, + }, + }; + const stream = capture(["--format", "jsonl"]); + await stream.output.write(context, metadata); + const { [key]: _records, ...viewContext } = context.normalized_result; + expect(lines(stream.text())).toEqual([ + ...records, + { + $rea: "jsonl/v1", + type: "metadata", + records: { path: ["normalized_result", key], emitted: 2, source: 2 }, + context: { ...context, normalized_result: viewContext }, + }, + ]); + expect(context.normalized_result[key]).toBe(records); + }, + ); + + it.each([[], [{ id: "row" }]].map((rows) => ({ rows })))( + "preserves nested pagination without a coverage object (%j)", + async ({ rows }) => { + const stream = capture(["--format", "jsonl", "--full-output"]); + const value = { + page: { rows, offset: 12, total: 31, next_offset: 14 }, + limitations: ["more pages"], + }; + await stream.output.write(value, metadata); + expect(lines(stream.text())).toEqual([ + ...rows, + { + $rea: "jsonl/v1", + type: "metadata", + records: { + path: ["data", "page", "rows"], + emitted: rows.length, + source: rows.length, + }, + context: { + ok: true, + data: { + page: { offset: 12, total: 31, next_offset: 14 }, + limitations: value.limitations, + }, + meta: { command: "fixture", duration: "1ms" }, + }, + }, + ]); + }, + ); +}); + +describe("JSON Lines admission and selection", () => { + it.each([ + null, + {}, + "scalar", + { rows: [], items: [] }, + { rows: [], normalized_result: { rows: [] } }, + Object.create({ rows: [{ inherited: true }] }), + { rows: [{ $rea: "collision" }] }, + ])( + "refuses an unclear or reserved boundary before writing rows: %j", + async (value) => { + const stream = capture(["--format", "jsonl", "--token-count"]); + await stream.output.write(value, metadata); + expect(lines(stream.text())).toEqual([ + jsonLinesErrorRecord(stream.output.error), + ]); + expect(stream.output.error?.code).toBe("unreadable_output"); + }, + ); + + it.each(["missing", "count", "nested.value"])( + "does not admit a scalar/missing selection merely because a filter exists: %s", + async (filter) => { + const stream = capture(["--format", "jsonl", "--filter-output", filter]); + await stream.output.write( + { rows: [{ id: 1 }], count: 0, nested: { value: null } }, + metadata, + ); + expect(lines(stream.text())).toHaveLength(1); + expect(lines(stream.text())[0]).toMatchObject({ + $rea: "jsonl/v1", + type: "error", + result: { code: "unreadable_output" }, + }); + }, + ); + + it("records output selection separately from source coverage and keeps provenance", async () => { + const value = { + evidence_id: "ev:parent", + raw_result: { excluded: "large observation" }, + provider: "source-provider", + normalized_result: { + rows: [ + { id: "a", text: "first" }, + { id: "b", text: "second" }, + { id: "c", text: "third" }, + ], + coverage: { complete: true, examined: 3 }, + }, + }; + const filter = "normalized_result.rows[1,3].id"; + const stream = capture(["--format", "jsonl", "--filter-output", filter]); + await stream.output.write(value, metadata); + expect(lines(stream.text())).toEqual([ + { id: "b" }, + { id: "c" }, + { + $rea: "jsonl/v1", + type: "metadata", + records: { + path: ["normalized_result", "rows"], + emitted: 2, + source: 3, + filter, + }, + context: { + evidence_id: "ev:parent", + provider: "source-provider", + normalized_result: { coverage: { complete: true, examined: 3 } }, + }, + }, + ]); + }); + + it("disambiguates selected collections without restoring excluded peer rows", async () => { + const stream = capture([ + "--format", + "jsonl", + "--filter-output", + "normalized_result.rows", + ]); + await stream.output.write( + { + evidence_id: "ev:parent", + raw_result: null, + normalized_result: { + rows: [{ id: 1 }], + items: [{ excluded: "peer" }], + coverage: { status: "unknown" }, + }, + }, + metadata, + ); + expect(lines(stream.text())).toEqual([ + { id: 1 }, + { + $rea: "jsonl/v1", + type: "metadata", + records: { + path: ["normalized_result", "rows"], + emitted: 1, + source: 1, + filter: "normalized_result.rows", + }, + context: { + evidence_id: "ev:parent", + normalized_result: { coverage: { status: "unknown" } }, + }, + }, + ]); + }); +}); + +describe("JSON Lines counting and transport", () => { + it.each([false, true])( + "frames classified failures without filtering/counting them away (full: %s)", + async (full) => { + const stream = capture([ + "--format", + "jsonl", + "--token-count", + "--filter-output", + "missing", + ...(full ? ["--full-output"] : []), + ]); + const failure = { + healthy: false, + checks: [{ ok: false, name: "source" }], + diagnostics: { path: "/caller/path" }, + }; + await stream.output.write(failure, metadata); + expect(lines(stream.text())).toEqual([ + jsonLinesErrorRecord( + full + ? { + ok: false, + error: failure, + meta: { command: "fixture", duration: "1ms" }, + } + : failure, + ), + ]); + }, + ); + + it.each([false, true])( + "counts array-valued rows as single records (object collection: %s)", + async (wrapped) => { + const rows = [[false, null], [0], [], [{ nested: ["雪", "a\nb"] }]]; + const value = wrapped ? { rows, coverage: { complete: false } } : rows; + const args = ["--format", "jsonl"]; + const stream = capture(args); + await stream.output.write(value, metadata); + const counter = capture([...args, "--token-count"]); + await counter.output.write(value, metadata); + expect(counter.text()).toBe(`${estimateTokenCount(stream.text())}\n`); + }, + ); + + it("counts exactly the emitted representation, including escaped multiline text and final context", async () => { + const value = { + rows: [{ text: "雪𝟠\nline".repeat(4096) }, { text: 'escaped"\\\t' }], + coverage: { complete: false, reasons: ["unknown remainder"] }, + }; + const args = ["--format", "jsonl", "--full-output"]; + const stream = capture(args); + await stream.output.write(value, metadata); + const counter = capture([...args, "--token-count"]); + await counter.output.write(value, metadata); + expect(counter.text()).toBe(`${estimateTokenCount(stream.text())}\n`); + }); + + it("bounds a large escaped field and observes destination backpressure through the final metadata", async () => { + const chunks: Buffer[] = []; + let maxBytes = 0; + const destination = new Writable({ + highWaterMark: 1, + write(chunk: Buffer, _encoding, callback) { + maxBytes = Math.max(maxBytes, chunk.length); + setTimeout(() => { + chunks.push(chunk); + callback(); + }, 1); + }, + }); + const output = createCliResultOutput( + parseCliOutputArguments(["--format", "jsonl"]), + destination, + )!; + const rows = [{ id: "large", text: "\u0000雪𝟠".repeat(32 * 1024) }]; + await output.write({ rows, coverage: { complete: false } }, metadata); + expect(maxBytes).toBeLessThan(128 * 1024); + const records = lines(Buffer.concat(chunks).toString("utf8")); + expect(records[0]).toEqual(rows[0]); + expect(records[1]).toMatchObject({ + type: "metadata", + context: { coverage: { complete: false } }, + }); + expect(destination.writableEnded).toBe(false); + }); + + it("retains an explicitly successful error-shaped row as data", async () => { + const value = [ + { + code: "not-an-error", + message: "ordinary row", + ok: false, + healthy: false, + }, + ]; + const stream = capture(["--format", "jsonl"]); + const cli = Cli.create("explicit-success", { sync: false }).command( + "rows", + { + run: () => + logCliCommand( + silentLogger, + "rows", + async () => value, + () => false, + ), + }, + ); + streamCliCommandResults(cli, stream.output); + await cli.serve(["rows", "--format", "jsonl"], { stdout: () => undefined }); + expect(lines(stream.text())).toEqual(value); + }); +}); + +describe("SDK-owned JSON Lines outcomes", () => { + it("uses SDK status to frame validation errors and resets ownership between invocations", async () => { + const stream = capture(["--format", "jsonl"]); + const cli = Cli.create("sdk-status", { sync: false }).command("rows", { + args: z.object({ required: z.string() }), + run: () => [{ id: "row" }], + }); + streamCliCommandResults(cli, stream.output); + const serve = async (args: string[]) => { + let status = 0; + stream.output.begin(parseCliOutputArguments(args)); + await cli.serve(args, { + stdout: (text) => { + if (!stream.output.handled && !stream.output.captureSdkOutput(text)) + throw new Error("Unexpected delegation"); + }, + exit: (code) => { + status = code; + }, + }); + await stream.output.finishSdkOutput(status); + return status; + }; + expect(await serve(["rows", "--format", "jsonl"])).toBe(1); + expect(lines(stream.text())[0]).toMatchObject({ + $rea: "jsonl/v1", + type: "error", + result: { code: "VALIDATION_ERROR" }, + }); + expect(await serve(["rows", "present", "--format", "jsonl"])).toBe(0); + expect(lines(stream.text())[1]).toEqual({ id: "row" }); + expect(stream.output.error).toBeUndefined(); + }); + + it.each([false, true])( + "preserves sentinel data and frames sentinel failure using SDK outcome (%s)", + async (failure) => { + const stream = capture(["--format", "jsonl"]); + const cli = Cli.create("sdk-sentinel", { sync: false }).command( + "result", + { + run: (context) => + failure + ? context.error({ + code: "EXPLICIT_FAILURE", + message: "Caller-owned error", + }) + : context.ok({ + code: "EXPLICIT_FAILURE", + message: "Successful data", + }), + }, + ); + streamCliCommandResults(cli, stream.output); + let status = 0; + await cli.serve(["result", "--format", "jsonl"], { + stdout: (text) => { + stream.output.captureSdkOutput(text); + }, + exit: (code) => { + status = code; + }, + }); + expect(stream.text()).toBe(""); + await stream.output.finishSdkOutput(status); + expect(lines(stream.text())[0]).toEqual( + failure + ? jsonLinesErrorRecord({ + code: "EXPLICIT_FAILURE", + message: "Caller-owned error", + }) + : { code: "EXPLICIT_FAILURE", message: "Successful data" }, + ); + }, + ); + + it("delegates generator events immediately without retaining the stream", async () => { + const stream = capture(["--format", "jsonl"]); + const delegated: string[] = []; + const cli = Cli.create("sdk-stream", { sync: false }).command("events", { + run: async function* () { + yield { text: "first" }; + expect(delegated.length).toBeGreaterThan(0); + yield { text: "second" }; + }, + }); + streamCliCommandResults(cli, stream.output); + await cli.serve(["events", "--format", "jsonl"], { + stdout: (text) => { + expect(stream.output.captureSdkOutput(text)).toBe(false); + delegated.push(text); + }, + }); + await stream.output.finishSdkOutput(0); + expect(stream.text()).toBe(""); + expect(delegated.join("")).toContain("first"); + expect(delegated.join("")).toContain("second"); + }); +}); diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.ts index 67f46650f4..a013d21235 100644 --- a/tests/boundary/cli/jsonOutput.test.ts +++ b/tests/boundary/cli/jsonOutput.test.ts @@ -634,7 +634,7 @@ describe("CLI result output surface", () => { describe("streamed CLI filtered selections", () => { it.each([ ["raw_result", "json", "null\n"], - ["raw_result", "jsonl", "null\n"], + ["raw_result", "jsonl", "JSONL_BOUNDARY_ERROR"], ["nested.value", "json", '{"nested":{"value":null}}\n'], ["missing", "json", "{}\n"], ["count", "json", "0\n"], @@ -663,7 +663,14 @@ describe("streamed CLI filtered selections", () => { { raw_result: null, nested: { value: null }, count: 0, enabled: false }, { command: "fixture", duration: "1ms", format }, ); - expect(Buffer.concat(chunks).toString("utf8")).toBe(expected); + const text = Buffer.concat(chunks).toString("utf8"); + if (expected === "JSONL_BOUNDARY_ERROR") + expect(JSON.parse(text)).toMatchObject({ + $rea: "jsonl/v1", + type: "error", + result: { code: "unreadable_output" }, + }); + else expect(text).toBe(expected); }, ); }); diff --git a/tests/boundary/cli/versionReportCli.test.ts b/tests/boundary/cli/versionReportCli.test.ts index f7cb5e3729..de4e24638d 100644 --- a/tests/boundary/cli/versionReportCli.test.ts +++ b/tests/boundary/cli/versionReportCli.test.ts @@ -53,6 +53,15 @@ cliTest( }); expect(json.stdout).not.toContain(marker); + const jsonl = await cli.run({ + arguments: ["--version", "--format", "jsonl"], + cwd, + environment, + }); + expect(jsonl.exitCode, jsonl.stderr).toBe(0); + expect(jsonl.stdout.trimEnd().split("\n")).toHaveLength(1); + expect(versionReportSchema.parse(JSON.parse(jsonl.stdout))).toEqual(report); + const yaml = await cli.run({ arguments: ["--version", "--format=yaml"], cwd, diff --git a/tests/boundary/mcp/mcpDoctor.test.ts b/tests/boundary/mcp/mcpDoctor.test.ts index 001ae437b9..9481e4a9f1 100644 --- a/tests/boundary/mcp/mcpDoctor.test.ts +++ b/tests/boundary/mcp/mcpDoctor.test.ts @@ -209,6 +209,50 @@ describe("production MCP doctor", () => { }); describe("production MCP doctor output", () => { + it.each([false, true])( + "frames unhealthy JSONL diagnostics and retains the selected envelope (%s)", + async (full) => { + const result = await runProductionMcpDoctorCli( + ["--format", "jsonl", ...(full ? ["--full-output"] : [])], + { + dispatcherPath: resolve("tests/fixtures/mcpDoctorExit.mjs"), + packageRoot: process.cwd(), + stdoutIsTerminal: true, + }, + ); + expect(result.exitCode).toBe(1); + expect(result.output.trimEnd().split("\n")).toHaveLength(1); + expect(JSON.parse(result.output)).toMatchObject({ + $rea: "jsonl/v1", + type: "error", + result: full + ? { + ok: false, + error: { healthy: false }, + meta: { command: "rea mcp doctor" }, + } + : { healthy: false }, + }); + }, + ); + + it("frames JSONL validation errors before opening a diagnostic session", async () => { + const result = await runProductionMcpDoctorCli( + ["--format", "jsonl", "--unknown"], + { + dispatcherPath: "/must-not-start", + packageRoot: process.cwd(), + stdoutIsTerminal: false, + }, + ); + expect(result.exitCode).toBe(1); + expect(JSON.parse(result.output)).toMatchObject({ + $rea: "jsonl/v1", + type: "error", + result: { code: "VALIDATION_ERROR" }, + }); + }); + it.each([ { name: "report", args: ["--json"] }, { name: "full-output envelope", args: ["--json", "--full-output"] }, diff --git a/tests/support/cli/cliCommandSessionWorker.mjs b/tests/support/cli/cliCommandSessionWorker.mjs index 13da0a2b4d..54d44b60d3 100644 --- a/tests/support/cli/cliCommandSessionWorker.mjs +++ b/tests/support/cli/cliCommandSessionWorker.mjs @@ -182,6 +182,7 @@ async function runOneShot(args) { return; } if (output.length > 0) wroteOutput = true; + if (resultOutput?.captureSdkOutput(output)) return; process.stdout.write(output); }, exit: (code) => { @@ -198,6 +199,10 @@ async function runOneShot(args) { const empty = renderEmptyFilteredCliOutput(outputOptions); if (empty) process.stdout.write(empty); } + await resultOutput?.finishSdkOutput( + explicitExit ?? + (typeof process.exitCode === "number" ? process.exitCode : 0), + ); } } finally { process.stdout.write = originalStdoutWrite; diff --git a/tests/support/cli/compiledCliWorker.mjs b/tests/support/cli/compiledCliWorker.mjs index 9dfdd17de6..fc16840989 100644 --- a/tests/support/cli/compiledCliWorker.mjs +++ b/tests/support/cli/compiledCliWorker.mjs @@ -68,12 +68,14 @@ const runCommand = async (args) => { let wroteOutput = false; let explicitExit; + resultOutput?.begin(outputOptions); try { await cli.serve(cliArgs, { stdout: (output) => { if (resultOutput?.handled) return; if (output.length === 0) return; wroteOutput = true; + if (resultOutput?.captureSdkOutput(output)) return; destination.write(output); }, exit: (code) => { @@ -94,6 +96,10 @@ const runCommand = async (args) => { const empty = renderEmptyFilteredCliOutput(outputOptions); if (empty !== undefined) destination.write(empty); } + await resultOutput?.finishSdkOutput( + explicitExit ?? + (typeof process.exitCode === "number" ? process.exitCode : 0), + ); // logCliCommand reports some failures only through process.exitCode. return { exitCode: From 3787ec5f83eae15f8342c4191bdb88ab690adc13 Mon Sep 17 00:00:00 2001 From: gensokyo-dev <342580529+gensokyo-dev@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:31:32 +0800 Subject: [PATCH 2/8] test(ida): align native search fixtures with Unicode folding (#6898) --- src/ida/IdaNativeProtocol.test.ts | 33 ++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/ida/IdaNativeProtocol.test.ts b/src/ida/IdaNativeProtocol.test.ts index 415a8f0f4d..f99091687c 100644 --- a/src/ida/IdaNativeProtocol.test.ts +++ b/src/ida/IdaNativeProtocol.test.ts @@ -715,6 +715,9 @@ describe("native IDA literal matching", () => { "literal[bracket", "a\\b", "🧪", + "Σ", + "ΟΣ", + "ος", ]; producer.overrideCall = (name, args) => { if (name !== "find_regex") return undefined; @@ -755,12 +758,36 @@ describe("native IDA literal matching", () => { case_sensitive: false, expected: ["İstanbul"], }, - { pattern: "σ", mode: "literal", case_sensitive: false, expected: ["σ"] }, + // Match the shared Unicode contract after the byte-oriented producer has + // returned candidates, including final sigma inside a word. + ...["σ", "ς", "Σ"].map((pattern) => ({ + pattern, + mode: "literal", + case_sensitive: false, + expected: ["σ", "ς", "Σ", "ΟΣ", "ος"], + })), + ...["ΟΣ", "ος"].map((pattern) => ({ + pattern, + mode: "literal", + case_sensitive: false, + expected: ["ΟΣ", "ος"], + })), + ...[ + { pattern: "σ", expected: ["σ"] }, + { pattern: "ς", expected: ["ς", "ος"] }, + { pattern: "Σ", expected: ["Σ", "ΟΣ"] }, + { pattern: "ΟΣ", expected: ["ΟΣ"] }, + { pattern: "ος", expected: ["ος"] }, + ].map((query) => ({ + ...query, + mode: "literal", + case_sensitive: true, + })), { pattern: "σ", mode: "regex", case_sensitive: false, - expected: ["σ", "ς"], + expected: ["σ", "ς", "Σ", "ΟΣ", "ος"], }, { pattern: "[", @@ -792,7 +819,7 @@ describe("native IDA literal matching", () => { const before = producer.calls.length; const result = await client.execute("search_strings", query); if (!result.ok) throw result.error; - expect(result.value.result).toEqual( + expect(result.value.result, JSON.stringify(query)).toEqual( expected.map((value) => ({ address: `0x${(0x3000 + values.indexOf(value)).toString(16)}`, value, From 25aaf8928763c60f84c4d5bb4f369d976d2d6a92 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:32:52 +0800 Subject: [PATCH 3/8] fix(sqlite): budget schema names by returned JSON text Charge the whole-catalog admission guard by escaped decoded names, preserving raw-byte lookup identity and final worker reply limits. Keep UTF-16 inventory and detailed-schema results whose returned facts fit the documented allowance. Fixes #6888 --- docs/sqlite-databases.md | 3 ++ src/sqlite/SqliteDatabaseInspection.ts | 8 +++- tests/acceptance/sqliteDatabaseCli.test.ts | 44 ++++++++++++++++++ .../sqlite/sqliteDatabaseInspection.test.ts | 46 +++++++++++++++++++ tests/fixtures/sqlite/database.ts | 22 +++++++++ 5 files changed, 121 insertions(+), 2 deletions(-) diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md index 3740320b99..f02712240e 100644 --- a/docs/sqlite-databases.md +++ b/docs/sqlite-databases.md @@ -128,6 +128,9 @@ bounds simultaneous input buffering and private snapshot/worker counts. The database and WAL together are limited to 256 MiB. SQLite values/records, SQL statements and the complete worker JSON reply each have a 16 MiB limit. +Schema names count by their returned JSON text, including escaping, for UTF-8, +UTF-16le and UTF-16be databases. Internal raw-byte lookup keys do not consume +the reply allowance; exact name identity still comes from those stored bytes. The owned worker has a 256 MiB JavaScript heap limit and defaults to a 30,000 ms deadline. Select optional MCP `worker_timeout_ms`, or CLI `--worker-timeout-ms `, to change the budget for that call. Values diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index a3d9b2c78c..3ea73c5f3b 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -324,7 +324,7 @@ const readSchema = ( // Names bound eventual output only when the whole catalog is requested. // Selected scope retains one table and at most one possible shadow parent; // unrelated names are examined without retaining them or charging the reply. - const metadataBudget = new OutputBudget(); + const catalogNameBudget = new OutputBudget(); for (const raw of db .prepare( "SELECT schema, name AS display_name, CAST(name AS BLOB) AS name, type, ncol, wr, strict FROM temp.pragma_table_list WHERE schema = 'main'", @@ -333,7 +333,11 @@ const readSchema = ( const value = tableRow.parse(raw); const nameKey = sqliteNameBytesKey(value.name); if (selectedName === undefined) { - metadataBudget.retain(nameKey); + // The private hex key grows with the database encoding, but is never + // returned. Admit catalog names by their decoded JSON representation; + // display text is used only for this lower bound, never for identity. + // Stored schema bytes still undergo strict decoding below. + catalogNameBudget.retain(value.display_name); metadata.set(nameKey, value); } else if (nameKey === selectedKey) { metadata.set(nameKey, value); diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index b58dad8775..83b129c77e 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -21,8 +21,11 @@ import { import { createSqliteApplicationCollationFixture, + createSqliteLongNameFixture, SQLITE_NATIVE_LIMITS_AVAILABLE, } from "../fixtures/sqlite/database.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { sqliteDatabaseSchema } from "../../src/domain/sqlite/sqliteDatabase.js"; import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; @@ -1137,6 +1140,47 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }, ); +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "returns complete UTF-16 schemas through the worker and public CLI when JSON fits", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-large-catalog-"); + for (const [encoding, scope] of [ + ["UTF-16le", "inventory"], + ["UTF-16be", "database"], + ] as const) { + const { path, names } = createSqliteLongNameFixture(root, encoding); + const response = await runFileCli(cli, { + arguments: [ + "inspect-sqlite-database", + path, + "--schema-scope", + scope, + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(response.exitCode, response.stderr).toBe(0); + const evidence = parseEvidence(response.json); + const report = sqliteDatabaseSchema.parse(evidence.normalized_result); + expect(evidence.parameters.schema_scope).toBe(scope); + expect(report.database_encoding).toBe(encoding); + expect(report.schema).toMatchObject({ scope, completeness: "complete" }); + expect(report.schema.tables).toHaveLength(names.length); + expect(report.schema.tables.map((table) => table.name)).toEqual( + expect.arrayContaining(names), + ); + expect( + report.schema.tables.every( + (table) => + table.columns_completeness === + (scope === "inventory" ? "not_requested" : "complete"), + ), + ).toBe(true); + expect(report.rows).toBeNull(); + } + }, +); + cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "keeps selected table metadata tied to exact schema-name bytes through CLI and MCP", async ({ cli }) => { diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts index f30e311ab1..8044a9ada4 100644 --- a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -3,10 +3,12 @@ import { readFile, writeFile } from "node:fs/promises"; import { expect, it as nativeTest, onTestFinished } from "vitest"; import { createSqliteApplicationCollationFixture, + createSqliteLongNameFixture, SQLITE_NATIVE_LIMITS_AVAILABLE, } from "../../fixtures/sqlite/database.js"; import { inspectSqliteDatabaseSnapshot } from "../../../src/sqlite/SqliteDatabaseInspection.js"; import { SqliteInspectionFailure } from "../../../src/sqlite/SqliteDatabaseLimits.js"; +import { jsonByteLength } from "../../../src/domain/jsonSerialization.js"; import { createTestWorkspace, removeTestWorkspace, @@ -238,6 +240,50 @@ it("counts schema names once when enforcing the complete reply budget", async () ); }); +it.each(["UTF-16le", "UTF-16be"] as const)( + "budgets %s catalog names by returned JSON rather than internal hex identity", + async (encoding) => { + const workspace = await createTestWorkspace("rea-sqlite-catalog-names-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const { path, names } = createSqliteLongNameFixture( + workspace.root, + encoding, + ); + for (const schema_scope of ["inventory", "database"] as const) { + const inspection = inspectSqliteDatabaseSnapshot(path, { + path, + schema_scope, + }); + expect(inspection.database_encoding).toBe(encoding); + expect(inspection.schema).toMatchObject({ + scope: schema_scope, + completeness: "complete", + }); + expect(inspection.schema.tables).toHaveLength(names.length); + expect(inspection.schema.tables.map((table) => table.name)).toEqual( + expect.arrayContaining(names), + ); + for (const table of inspection.schema.tables) { + expect(table.columns_completeness).toBe( + schema_scope === "inventory" ? "not_requested" : "complete", + ); + if (schema_scope === "database") { + expect(table.sql).toBe(`CREATE TABLE "${table.name}"(value TEXT)`); + expect(table.columns).toMatchObject([ + { name: "value", declared_type: "TEXT" }, + ]); + } else { + expect(table.sql).toBeUndefined(); + expect(table.columns).toEqual([]); + } + } + expect(jsonByteLength({ ok: true, inspection })).toBeLessThan( + 16 * 1024 * 1024, + ); + } + }, +); + it("keeps unrelated catalog names outside the selected-table reply budget", async () => { const path = await createDatabase( "CREATE TABLE selected(value TEXT); INSERT INTO selected VALUES ('retained');", diff --git a/tests/fixtures/sqlite/database.ts b/tests/fixtures/sqlite/database.ts index 53773c8fe2..445ba6b458 100644 --- a/tests/fixtures/sqlite/database.ts +++ b/tests/fixtures/sqlite/database.ts @@ -100,3 +100,25 @@ export const createSqliteApplicationCollationFixture = ( } return path; }; + +/** UTF-16 hex lookup keys exceed 16 MiB while both public schema scopes fit. */ +export const createSqliteLongNameFixture = ( + root: string, + encoding: "UTF-8" | "UTF-16le" | "UTF-16be", +): { path: string; names: string[] } => { + const path = join(root, `large-schema-${encoding}.db`); + const names = Array.from( + { length: 34 }, + (_, index) => `${index}_${"x".repeat(128 * 1024)}`, + ); + const producer = new DatabaseSync(path); + try { + producer.exec(`PRAGMA encoding='${encoding}'; BEGIN`); + for (const name of names) + producer.exec(`CREATE TABLE "${name}"(value TEXT)`); + producer.exec("COMMIT"); + } finally { + producer.close(); + } + return { path, names }; +}; From 75a757d8ef41772d04bdc7963c95bc2fb08d7025 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:32:53 +0800 Subject: [PATCH 4/8] fix(elf): distinguish compressed note storage from decoded notes Refuse compressed note sections with typed unsupported-target semantics before framing or decoding, preserving stored-range checks and physical descriptor-offset provenance. Flutter retains independent build IDs with partial coverage when compressed notes are skipped. Verify the existing shared alignment fix with prefixed notes across classes and table kinds. Fixes #6869 Closes #6563 --- docs/elf-notes.md | 8 +++ docs/flutter-analysis.md | 3 + src/flutter/ElfBuildId.ts | 15 +++++ src/native/elf/ElfNoteParser.test.ts | 45 +++++++++++++++ src/native/elf/ElfNoteParser.ts | 21 +++++-- src/native/elf/ElfNotes.fixture.ts | 27 +++++++++ tests/acceptance/elfNotesCli.test.ts | 22 +++++++ tests/acceptance/flutterAnalysis.test.ts | 57 ++++++++++++++++++- .../filesystem/flutterElfNotes.test.ts | 57 ++++++++++++++++++- tests/boundary/mcp/elfNotesMcp.test.ts | 28 +++++++++ 10 files changed, 274 insertions(+), 9 deletions(-) diff --git a/docs/elf-notes.md b/docs/elf-notes.md index 6a464fce14..15eb2378ac 100644 --- a/docs/elf-notes.md +++ b/docs/elf-notes.md @@ -58,5 +58,13 @@ numbering (`PN_XNUM`, including a section count stored when `e_shnum` is 0) is unsupported and returns an unsupported-target outcome rather than invalid-input recovery. +`SHF_COMPRESSED` note sections return `unsupported_target` before their payload +is interpreted or decompressed. Their compression header is storage metadata, +not a note header. The current result requires physical file offsets for each +note and descriptor, which decoded compressed notes cannot provide. This refusal +does not label compressed note storage as malformed or allocate its declared +decoded size. Compressed section-name tables remain supported within the limits +above. + This workflow does not execute the image or verify GNU build-id uniqueness against package catalogs. diff --git a/docs/flutter-analysis.md b/docs/flutter-analysis.md index cc4e8a48c1..cab64e9740 100644 --- a/docs/flutter-analysis.md +++ b/docs/flutter-analysis.md @@ -111,6 +111,9 @@ server; older servers keep the complete request after schema discovery. Nonempty GNU build-ID descriptors up to 64 bytes are retained as lowercase hexadecimal, including IDs shorter than four bytes. Matching bytes outside declared note ranges cannot supply a build ID. + Compressed `SHT_NOTE` sections are skipped with an unsupported-compression + limitation and partial coverage; IDs from other note ranges remain candidates. + Encoded note storage is never interpreted as note headers or inflated. - Labeled strings must fit their complete grammar. `dart_version_capped` and `toolchain_capped` independently report labels omitted by grammar or projection bounds; either makes identification coverage partial. Values diff --git a/src/flutter/ElfBuildId.ts b/src/flutter/ElfBuildId.ts index 9d1cccd961..0d6f9a453d 100644 --- a/src/flutter/ElfBuildId.ts +++ b/src/flutter/ElfBuildId.ts @@ -71,6 +71,7 @@ export const readElfBuildIds = async ( string, { offset: number; size: number; alignment: number } >(); + let compressedNotes = false; tables: for (const table of [ { offset: h.e_phoff, @@ -117,6 +118,20 @@ export const readElfBuildIds = async ( size > totalBytes - offset ) return failure("ELF note range is not file-backed"); + if (!table.program) { + const flags = h.is64 + ? bytes.readBigUInt64LE(base + 8) + : BigInt(bytes.readUInt32LE(base + 8)); + if ((flags & 0x800n) !== 0n) { + if (!compressedNotes) { + failures.push({ + reason: "Compressed ELF note sections are unsupported", + }); + compressedNotes = true; + } + continue; + } + } if (size === 0) continue; const alignmentAt = base + (table.program ? (h.is64 ? 48 : 28) : h.is64 ? 48 : 32); diff --git a/src/native/elf/ElfNoteParser.test.ts b/src/native/elf/ElfNoteParser.test.ts index 044a8feafb..eabf204a57 100644 --- a/src/native/elf/ElfNoteParser.test.ts +++ b/src/native/elf/ElfNoteParser.test.ts @@ -6,6 +6,7 @@ import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; import { BinaryFormatFailure } from "../metadata/BinaryContainer.js"; import { compressedElfNoteFixture, + compressedElfNoteSectionFixture, elfNote, elfNoteFixture, elfNoteTablePastEnd, @@ -103,6 +104,50 @@ it("reads the same notes from an SHT_NOTE section including its name", () => { ).toBe(true); }); +it.each([ + [32, true], + [32, false], + [64, true], + [64, false], +] as const)( + "refuses compressed ELF%i %s-endian note storage without parsing its compression header", + (bits, little) => { + expect(() => parse(compressedElfNoteSectionFixture(bits, little))).toThrow( + expect.objectContaining({ + kind: "unsupported", + message: expect.stringContaining( + "Compressed ELF note sections are unsupported", + ), + }), + ); + }, +); + +it("does not allocate the declared decoded note size, but still validates stored ranges", () => { + const bytes = compressedElfNoteSectionFixture(); + const noteSection = 64 + 64; + const storedOffset = Number(bytes.readBigUInt64LE(noteSection + 24)); + bytes.writeBigUInt64LE(0xffffffffffffffffn, storedOffset + 8); + expect(() => parse(bytes)).toThrow( + expect.objectContaining({ kind: "unsupported" }), + ); + bytes.writeBigUInt64LE(0n, noteSection + 32); + expect(() => parse(bytes)).toThrow( + expect.objectContaining({ kind: "unsupported" }), + ); + bytes.writeBigUInt64LE(BigInt(bytes.length), noteSection + 32); + expect(() => parse(bytes)).toThrow(ArtifactReaderFailure); +}); + +it("preserves unknown ELF64 section flag bits without treating them as file sizes", () => { + const bytes = elfNoteFixture({ ownership: "section" }); + bytes.writeBigUInt64LE(0x8000000000000000n, 64 + 64 + 8); + bytes.writeBigUInt64LE(0x8000000000000000n, 64 + 2 * 64 + 8); + expect(parse(bytes).entries[0]?.gnu?.build_id_hex).toBe( + "0123456789abcdef0123456789abcdef01234567", + ); +}); + it.each([ [32, true], [64, false], diff --git a/src/native/elf/ElfNoteParser.ts b/src/native/elf/ElfNoteParser.ts index 8ec5d38f3d..60296e4ebb 100644 --- a/src/native/elf/ElfNoteParser.ts +++ b/src/native/elf/ElfNoteParser.ts @@ -19,6 +19,7 @@ import { decodeElfSectionNameTable } from "../metadata/ElfSectionNameTable.js"; const PT_NOTE = 4; const SHT_NOTE = 7; const SHT_STRTAB = 3; +const SHF_COMPRESSED = 0x800n; const HEX_DESCRIPTOR_LIMIT = 64; interface TextBudget { @@ -111,13 +112,17 @@ class ElfReader { : this.bytes.readUInt32BE(offset); } - word(offset: number): number { + wordBits(offset: number): bigint { const width = this.bits === 64 ? 8 : 4; this.require(offset, width); - if (this.bits === 32) return this.u32(offset); - const value = this.little + if (this.bits === 32) return BigInt(this.u32(offset)); + return this.little ? this.bytes.readBigUInt64LE(offset) : this.bytes.readBigUInt64BE(offset); + } + + word(offset: number): number { + const value = this.wordBits(offset); if (value > BigInt(Number.MAX_SAFE_INTEGER)) fail("ELF offset or size exceeds a safe integer."); return Number(value); @@ -353,10 +358,10 @@ export const parseElfNotes = ( const strOff = reader.word(namesHeader + (wide ? 24 : 16)); const strSize = reader.word(namesHeader + (wide ? 32 : 20)); const rawNames = reader.slice(strOff, strSize); - const flags = reader.word(namesHeader + 8); + const flags = reader.wordBits(namesHeader + 8); const decoded = decodeElfSectionNameTable( rawNames, - BigInt(flags), + flags, header.elfClass, reader.little, input.max_file_bytes - bytes.length, @@ -418,6 +423,12 @@ export const parseElfNotes = ( const nameOffset = reader.u32(at); const fileOffset = reader.word(at + (wide ? 24 : 16)); const fileSize = reader.word(at + (wide ? 32 : 20)); + reader.require(fileOffset, fileSize); + if ((reader.wordBits(at + 8) & SHF_COMPRESSED) !== 0n) + throw new BinaryFormatFailure( + "unsupported", + "Compressed ELF note sections are unsupported; descriptor file offsets require uncompressed note storage.", + ); const alignment = noteAlignment(reader.word(at + (wide ? 48 : 32))); tables.push({ kind: "section", diff --git a/src/native/elf/ElfNotes.fixture.ts b/src/native/elf/ElfNotes.fixture.ts index e404473cb9..7c2a0a653a 100644 --- a/src/native/elf/ElfNotes.fixture.ts +++ b/src/native/elf/ElfNotes.fixture.ts @@ -209,6 +209,33 @@ export const elfNoteFixture = (options: ElfNoteImageOptions = {}): Buffer => { return bytes; }; +/** A non-allocated SHT_NOTE whose stored bytes include an ELF compression header. */ +export const compressedElfNoteSectionFixture = ( + bits: 32 | 64 = 64, + little = true, +): Buffer => { + const notes = concat( + gnuBuildIdNote(undefined, little), + gnuAbiTagNote(little), + ); + const header = Buffer.alloc(bits === 64 ? 24 : 12); + write32(header, 0, 1, little); // ELFCOMPRESS_ZLIB + writeWord(header, bits === 64 ? 8 : 4, notes.length, bits, little); + writeWord(header, bits === 64 ? 16 : 8, 4, bits, little); + const bytes = elfNoteFixture({ + bits, + little, + type: 1, + ownership: "section", + sectionName: ".debug.notes", + alignment: bits === 64 ? 8 : 4, + notes: concat(header, deflateSync(notes)), + }); + const noteSection = (bits === 64 ? 64 : 52) + (bits === 64 ? 64 : 40); + writeWord(bytes, noteSection + 8, 0x800, bits, little); + return bytes; +}; + /** Valid ELF64 section count stored in section header zero per the gABI. */ export const elfExtendedSectionCountFixture = (): Buffer => { const count = 0xff00; diff --git a/tests/acceptance/elfNotesCli.test.ts b/tests/acceptance/elfNotesCli.test.ts index 8d9b734d5e..7905dd039c 100644 --- a/tests/acceptance/elfNotesCli.test.ts +++ b/tests/acceptance/elfNotesCli.test.ts @@ -4,6 +4,7 @@ import { expect } from "vitest"; import { toolContract } from "../../src/contracts/toolContracts.js"; import { compressedElfNoteFixture, + compressedElfNoteSectionFixture, elfExtendedSectionCountFixture, elfNoteFixture, } from "../../src/native/elf/ElfNotes.fixture.js"; @@ -85,6 +86,27 @@ cliTest("reports a truncated ELF as invalid input", async ({ cli }) => { }); }); +cliTest( + "reports compressed note storage as unsupported without mutation", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-elf-notes-storage-cli-"); + const path = join(root, "debug-notes.o"); + const bytes = compressedElfNoteSectionFixture(); + await writeFile(path, bytes); + const result = await cli.run({ + arguments: ["inspect-elf-notes", path, "--json"], + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "unsupported_target", + message: expect.stringContaining( + "Compressed ELF note sections are unsupported", + ), + }); + expect(await readFile(path)).toEqual(bytes); + }, +); + cliTest( "reports valid extended numbering as unsupported without mutation", async ({ cli }) => { diff --git a/tests/acceptance/flutterAnalysis.test.ts b/tests/acceptance/flutterAnalysis.test.ts index 2e364f1c57..7cb29276c4 100644 --- a/tests/acceptance/flutterAnalysis.test.ts +++ b/tests/acceptance/flutterAnalysis.test.ts @@ -1,4 +1,7 @@ -import { gnuBuildIdNote } from "../../src/native/elf/ElfNotes.fixture.js"; +import { + gnuBuildIdNote, + compressedElfNoteSectionFixture, +} from "../../src/native/elf/ElfNotes.fixture.js"; import { createHash } from "node:crypto"; import { expect } from "vitest"; import { toolContract } from "../../src/contracts/toolContracts.js"; @@ -343,3 +346,55 @@ cliTest( } }, ); + +cliTest( + "CLI and stdio MCP retain Flutter build-ID candidates with unsupported compressed notes", + async ({ cli }) => { + const compressed = appendElfRanges( + compressedElfNoteSectionFixture(), + "program", + [{ type: 1, offset: 0, size: 0 }], + ); + const note = gnuBuildIdNote("0123456789abcdef"); + const engine = appendElfRanges( + Buffer.concat([compressed, note]), + "section", + [{ type: 7, offset: compressed.length, size: note.length }], + ); + const path = await createFlutterApk("compressed-notes.apk", [ + { path: "lib/arm64-v8a/libapp.so", bytes: aotFixtureImage() }, + { path: "lib/arm64-v8a/libflutter.so", bytes: engine }, + ]); + const output = await cli.run({ + arguments: ["identify-flutter-build", path, "--json"], + }); + expect(output.exitCode, output.stderr).toBe(0); + const cliEvidence = parseEvidence(output.json); + const { call } = await connectLocalToolsMcp(); + const response = await call("identify_flutter_build", { path }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const mcpEvidence = parseEvidence(response.structuredContent); + expect(mcpEvidence.normalized_result).toEqual( + cliEvidence.normalized_result, + ); + const result = flutterResultSchemas.identify_flutter_build.parse( + cliEvidence.normalized_result, + ); + expect(result).toMatchObject({ + coverage: "partial", + abis: [ + { + libflutter: { + sha256: createHash("sha256").update(engine).digest("hex"), + build_id: null, + build_id_candidates: ["0123456789abcdef"], + }, + }, + ], + }); + for (const evidence of [cliEvidence, mcpEvidence]) + expect(evidence.limitations.join("\n")).toContain( + "Compressed ELF note sections are unsupported", + ); + }, +); diff --git a/tests/boundary/filesystem/flutterElfNotes.test.ts b/tests/boundary/filesystem/flutterElfNotes.test.ts index 80062460bd..2fc7308f6e 100644 --- a/tests/boundary/filesystem/flutterElfNotes.test.ts +++ b/tests/boundary/filesystem/flutterElfNotes.test.ts @@ -5,6 +5,7 @@ import { readElfBuildIds } from "../../../src/flutter/ElfBuildId.js"; import { elfNote, elfNoteFixture, + compressedElfNoteSectionFixture, } from "../../../src/native/elf/ElfNotes.fixture.js"; import { appendElfRanges, @@ -27,11 +28,12 @@ it.each([32, 64] as const)( bits, alignment, ownership: table === "program" ? "program" : "both", - notes: Buffer.concat( - ids.map((id) => + notes: Buffer.concat([ + elfNote("Example", 1, Buffer.from([1]), true, alignment), + ...ids.map((id) => elfNote("GNU", 3, Buffer.from(id, "hex"), true, alignment), ), - ), + ]), }); if (table === "section") image.writeUInt32LE(1, bits === 64 ? 64 : 52); const root = await createTestTempDirectory("rea-flutter-alignment-"); @@ -51,6 +53,55 @@ it.each([32, 64] as const)( }, ); +it.each( + ([32, 64] as const).flatMap((bits) => [ + { bits, empty: false }, + { bits, empty: true }, + ]), +)( + "retains other ELF$bits note IDs with compressed storage (empty: $empty)", + async ({ bits, empty }) => { + const encoded = compressedElfNoteSectionFixture(bits); + const noteSection = (bits === 64 ? 64 : 52) + (bits === 64 ? 64 : 40); + if (empty) { + if (bits === 64) encoded.writeBigUInt64LE(0n, noteSection + 32); + else encoded.writeUInt32LE(0, noteSection + 20); + } + // Engine metadata parsing also requires a program directory. + const compressed = appendElfRanges(encoded, "program", [ + { type: 1, offset: 0, size: 0 }, + ]); + const bytes = appendElfRanges( + Buffer.concat([compressed, note]), + "section", + [{ type: 7, offset: compressed.length, size: note.length }], + ); + const root = await createTestTempDirectory("rea-flutter-compressed-notes-"); + const path = join(root, "engine.so"); + await writeFile(path, bytes); + const file = await open(path, "r"); + const read = vi.spyOn(file, "read"); + try { + expect(await readElfBuildIds(file, bytes.length, undefined)).toEqual({ + candidates: ["0123456789abcdef"], + failures: [{ reason: "Compressed ELF note sections are unsupported" }], + }); + const storedOffset = + bits === 64 + ? Number(compressed.readBigUInt64LE(noteSection + 24)) + : compressed.readUInt32LE(noteSection + 16); + expect(read).not.toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + storedOffset, + ); + } finally { + await file.close(); + } + }, +); + it.each( ([32, 64] as const).flatMap((bits) => [ { bits, program: 8, section: 4, reads: 5 }, diff --git a/tests/boundary/mcp/elfNotesMcp.test.ts b/tests/boundary/mcp/elfNotesMcp.test.ts index 1156308215..62d4ad920c 100644 --- a/tests/boundary/mcp/elfNotesMcp.test.ts +++ b/tests/boundary/mcp/elfNotesMcp.test.ts @@ -7,6 +7,7 @@ import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { elfExtendedSectionCountFixture, + compressedElfNoteSectionFixture, elfNoteFixture, } from "../../../src/native/elf/ElfNotes.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -72,6 +73,33 @@ mcpTest("requires an absolute path for inspect_elf_notes", async ({ mcp }) => { } }); +mcpTest( + "reports compressed note storage as unsupported without mutation", + async ({ mcp }) => { + const { client, session } = await connect(mcp); + try { + const root = await createTestTempDirectory("rea-elf-notes-storage-mcp-"); + const path = join(root, "debug-notes.o"); + const bytes = compressedElfNoteSectionFixture(); + await writeFile(path, bytes); + const response = await client.callTool({ + name: "inspect_elf_notes", + arguments: { path }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "unsupported_target", + message: expect.stringContaining( + "Compressed ELF note sections are unsupported", + ), + }); + expect(await readFile(path)).toEqual(bytes); + } finally { + await session.close(); + } + }, +); + mcpTest( "reports valid extended numbering as unsupported without mutation", async ({ mcp }) => { From e737e6c1c642bc8c12b14b1f27afa18859697ed4 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:32:54 +0800 Subject: [PATCH 5/8] fix(erlang): distinguish empty optional BEAM chunks Select legacy Atom when AtU8 is empty and skip zero-byte optional FunT. Preserve explicitly empty counted arrays, malformed nonempty failures, last-occurrence selection and original chunk provenance. Share structural fixtures across producer and public workflow regressions. Fixes #6816 Fixes #6874 --- docs/erlang-beam.md | 15 ++- src/erlang/Beam.fixture.ts | 22 ++++ src/erlang/BeamParser.test.ts | 153 +++++++++++++++++++++---- src/erlang/BeamParser.ts | 23 ++-- tests/acceptance/erlangBeamCli.test.ts | 86 +++++++++++++- 5 files changed, 264 insertions(+), 35 deletions(-) diff --git a/docs/erlang-beam.md b/docs/erlang-beam.md index 6317dd2f92..b8c6837dd3 100644 --- a/docs/erlang-beam.md +++ b/docs/erlang-beam.md @@ -31,7 +31,7 @@ The BEAM form size bounds the parsed chunks. Bytes after the declared form are ignored by the parser but remain part of the selected artifact's byte length and SHA-256. -When `AtU8` or `Atom` is present, `atoms` is the 1-based name table and +When a selected `AtU8` or `Atom` table is decoded, `atoms` is the 1-based name table and `module` is atom 1. Both the pre-OTP 28 table and the OTP 28+ compact table (negated count, compact-encoded name lengths) are read. `exports`, `imports`, and `lambdas` resolve atom indices to those names when the table is present; unresolved indices keep `function` or @@ -41,6 +41,19 @@ those names when the table is present; unresolved indices keep `function` or function count, and the instruction stream's file range. Missing optional tables are null, not empty-everywhere. +A zero-byte `AtU8` falls back to an available legacy `Atom` table, following +[OTP 20.3's atom selection](https://github.com/erlang/otp/blob/a113f6117fd696ea6f84ed754055b4ec97a7ccb2/erts/emulator/beam/beam_load.c). +A nonempty `AtU8` retains UTF-8 precedence, including an explicit zero atom +count; malformed nonempty data fails instead of falling back. An empty atom +chunk without an alternate table still fails its required count header. +Absent and zero-byte optional `FunT` chunks produce `lambdas: null`, while a +four-byte zero row count produces `lambdas: []`. This follows +[OTP 29's positive-size lambda parsing rule](https://github.com/erlang/otp/blob/550d7b7898706c7822362c42e7b93120c1d1f29a/erts/emulator/beam/beam_file.c). +Empty payloads retain their ranges and SHA-256 in the chunk inventory. These +rules apply to the last occurrence of each chunk name; an empty final occurrence +does not revive an earlier payload of that name. Nonempty truncated tables remain +format failures. + ## Limits and failures The selected file is limited to 64 MiB by default (512 MiB maximum). At most diff --git a/src/erlang/Beam.fixture.ts b/src/erlang/Beam.fixture.ts index da70e33661..dd921624e2 100644 --- a/src/erlang/Beam.fixture.ts +++ b/src/erlang/Beam.fixture.ts @@ -33,6 +33,28 @@ const table = (name: string, rows: readonly (readonly number[])[]): Buffer => { return chunk(name, payload); }; +/** Inert structural container with exact chunk payloads, not executable code. */ +export const erlangBeamChunksFixture = ( + chunks: readonly (readonly [name: string, payload: Buffer])[], +): Buffer => { + const form = concat( + Buffer.from("BEAM", "latin1"), + ...chunks.map(([name, payload]) => chunk(name, payload)), + ); + return concat(Buffer.from("FOR1", "latin1"), u32(form.length), form); +}; + +/** Legacy atom/export container, with an optional exact AtU8 alternate payload. */ +export const erlangBeamLegacyAtomsFixture = (utf8Payload?: Buffer): Buffer => { + const chunks = concat( + atoms(["demo", "ÿ"], false), + utf8Payload === undefined ? Buffer.alloc(0) : chunk("AtU8", utf8Payload), + table("ExpT", [[2, 0, 1]]), + ); + const form = concat(Buffer.from("BEAM", "latin1"), chunks); + return concat(Buffer.from("FOR1", "latin1"), u32(form.length), form); +}; + /** Hand-authored BEAM IFF with AtU8, Code, ExpT, ImpT and FunT chunks. */ export const erlangBeamFixture = (module = "beam_fixture"): Buffer => { const atomNames = [ diff --git a/src/erlang/BeamParser.test.ts b/src/erlang/BeamParser.test.ts index 5051e52590..07775fab3c 100644 --- a/src/erlang/BeamParser.test.ts +++ b/src/erlang/BeamParser.test.ts @@ -7,8 +7,10 @@ import { } from "../domain/erlang/beam.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { + erlangBeamChunksFixture, erlangBeamDeclaredRowsFixture, erlangBeamFixture, + erlangBeamLegacyAtomsFixture, erlangBeamManyAtomsFixture, erlangBeamOtp29Fixture, } from "./Beam.fixture.js"; @@ -119,12 +121,121 @@ it("selects the last table and Code payload for every repeated chunk name", () = it("rejects a malformed selected atom table instead of using an earlier table", () => { const original = erlangBeamFixture(); - const invalid = singleChunkBeam("AtU8", Buffer.alloc(0)); + const invalid = erlangBeamChunksFixture([["AtU8", Buffer.from([0])]]); const repeated = Buffer.concat([original, invalid.subarray(12)]); repeated.writeUInt32BE(repeated.length - 8, 4); expect(() => parse(repeated)).toThrow(/truncated/); }); +it("falls back from an empty AtU8 to legacy Atom, while nonempty alternates keep precedence", () => { + const control = parse(erlangBeamLegacyAtomsFixture()); + const empty = parse(erlangBeamLegacyAtomsFixture(Buffer.alloc(0))); + expect(empty.atoms).toEqual(control.atoms); + expect(empty.atom_encoding).toBe("latin1"); + expect(empty.module).toBe("demo"); + expect(empty.exports?.[0]?.function).toBe("ÿ"); + expect(empty.chunks.map(({ name }) => name)).toEqual([ + "Atom", + "AtU8", + "ExpT", + ]); + expect(empty.chunks[1]?.payload.bytes).toBe(0); + const alternate = parse( + erlangBeamLegacyAtomsFixture( + Buffer.concat([ + Buffer.from([0, 0, 0, 2, 4]), + Buffer.from("demo"), + Buffer.from([3]), + Buffer.from("new"), + ]), + ), + ); + expect(alternate.atom_encoding).toBe("utf8"); + expect(alternate.exports?.[0]?.function).toBe("new"); + const zeroCount = parse(erlangBeamLegacyAtomsFixture(Buffer.alloc(4))); + expect(zeroCount.atom_encoding).toBe("utf8"); + expect(zeroCount.atoms).toEqual([]); + expect(zeroCount.exports?.[0]?.function).toBeNull(); +}); + +it.each([Buffer.from([0]), Buffer.from([0, 0, 0, 1])])( + "rejects a malformed nonempty AtU8 without falling back: %j", + (payload) => { + expect(() => parse(erlangBeamLegacyAtomsFixture(payload))).toThrow( + /truncated/, + ); + }, +); + +it.each(["AtU8", "Atom"])( + "does not turn an empty standalone %s atom table into decoded facts", + (name) => { + expect(() => + parse(erlangBeamChunksFixture([[name, Buffer.alloc(0)]])), + ).toThrow(/atom table is truncated/); + }, +); + +it("uses the last AtU8 occurrence even when empty", () => { + const control = erlangBeamLegacyAtomsFixture(Buffer.from([0, 0, 0, 0])); + const empty = erlangBeamChunksFixture([["AtU8", Buffer.alloc(0)]]); + const bytes = Buffer.concat([control, empty.subarray(12)]); + bytes.writeUInt32BE(bytes.length - 8, 4); + const report = parse(bytes); + expect(report.atom_encoding).toBe("latin1"); + expect(report.exports?.[0]?.function).toBe("ÿ"); + expect( + report.chunks + .filter(({ name }) => name === "AtU8") + .map(({ payload }) => payload.bytes), + ).toEqual([4, 0]); +}); + +it.each([ + { payload: Buffer.alloc(0), expected: null }, + { payload: Buffer.alloc(4), expected: [] }, +])( + "distinguishes empty optional FunT from a zero row count: $payload", + ({ payload, expected }) => { + const original = erlangBeamOtp29Fixture(); + const extension = erlangBeamChunksFixture([["FunT", payload]]); + const bytes = Buffer.concat([original, extension.subarray(12)]); + bytes.writeUInt32BE(bytes.length - 8, 4); + const control = parse(original); + const report = parse(bytes); + expect(control.lambdas).toBeNull(); + expect(report.lambdas).toEqual(expected); + expect(report.atoms).toEqual(control.atoms); + expect(report.exports).toEqual(control.exports); + expect(report.imports).toEqual(control.imports); + expect(report.code).toEqual(control.code); + expect(report.chunks.at(-1)?.payload.bytes).toBe(payload.length); + }, +); + +it("does not reuse an earlier FunT when its last occurrence is empty", () => { + const original = erlangBeamFixture(); + const extension = erlangBeamChunksFixture([["FunT", Buffer.alloc(0)]]); + const bytes = Buffer.concat([original, extension.subarray(12)]); + bytes.writeUInt32BE(bytes.length - 8, 4); + const report = parse(bytes); + expect(report.lambdas).toBeNull(); + expect( + report.chunks + .filter(({ name }) => name === "FunT") + .map(({ payload }) => payload.bytes), + ).toEqual([28, 0]); +}); + +it.each([Buffer.from([0]), Buffer.from([0, 0, 0, 1])])( + "rejects truncated nonempty FunT payloads: %j", + (payload) => { + expect(() => parse(erlangBeamChunksFixture([["FunT", payload]]))).toThrow( + /lambda table is truncated/, + ); + }, +); + // Expected values come from beam_lib:chunks/2 on the same file under OTP 29. it("reads a compiler-produced OTP 29 BEAM with the compact atom table", () => { const report = parse(erlangBeamOtp29Fixture()); @@ -162,10 +273,12 @@ it("reads a compiler-produced OTP 29 BEAM with the compact atom table", () => { it("accepts extended compact atom lengths and rejects truncated or overflowing values", () => { const compactAtom = (length: Buffer, name: Buffer): Buffer => - singleChunkBeam( - "AtU8", - Buffer.concat([Buffer.from([0xff, 0xff, 0xff, 0xff]), length, name]), - ); + erlangBeamChunksFixture([ + [ + "AtU8", + Buffer.concat([Buffer.from([0xff, 0xff, 0xff, 0xff]), length, name]), + ], + ]); const name = Buffer.from("0123456789abcdef", "ascii"); const extended = parse(compactAtom(Buffer.from([0x18, 0x00, 0x10]), name)); expect(extended.module).toBe(name.toString("ascii")); @@ -205,22 +318,6 @@ it("fails a FOR1 form that is not BEAM", () => { expect(() => parse(bytes)).toThrow(ArtifactReaderFailure); }); -const singleChunkBeam = (name: string, payload: Buffer): Buffer => { - const header = Buffer.alloc(8); - header.write(name, 0, "latin1"); - header.writeUInt32BE(payload.length, 4); - const pad = (4 - (payload.length % 4)) % 4; - const form = Buffer.concat([ - Buffer.from("BEAM"), - header, - payload, - Buffer.alloc(pad), - ]); - const size = Buffer.alloc(4); - size.writeUInt32BE(form.length); - return Buffer.concat([Buffer.from("FOR1"), size, form]); -}; - it("rejects a short remainder inside the BEAM form", () => { const bytes = Buffer.alloc(16); bytes.write("FOR1"); @@ -231,13 +328,15 @@ it("rejects a short remainder inside the BEAM form", () => { it("rejects an AtU8 atom name that is not UTF-8", () => { expect(() => - parse(singleChunkBeam("AtU8", Buffer.from([0, 0, 0, 1, 1, 0xff]))), + parse( + erlangBeamChunksFixture([["AtU8", Buffer.from([0, 0, 0, 1, 1, 0xff])]]), + ), ).toThrow(/BEAM atom name is not valid UTF-8/); }); it("keeps a latin1 atom byte unchanged", () => { const report = parse( - singleChunkBeam("Atom", Buffer.from([0, 0, 0, 1, 1, 0xff])), + erlangBeamChunksFixture([["Atom", Buffer.from([0, 0, 0, 1, 1, 0xff])]]), ); expect(report.atom_encoding).toBe("latin1"); expect(report.module).toBe("ÿ"); @@ -263,9 +362,13 @@ it("rejects combined table rows before allocating them", () => { ); const count = Buffer.alloc(4); count.writeUInt32BE(MAX_INSPECT_ERLANG_BEAM_TABLE_ROWS + 1); - expect(() => parse(singleChunkBeam("ExpT", count))).toThrow(/row budget/); + expect(() => parse(erlangBeamChunksFixture([["ExpT", count]]))).toThrow( + /row budget/, + ); count.writeUInt32BE(MAX_INSPECT_ERLANG_BEAM_TABLE_ROWS); - expect(() => parse(singleChunkBeam("ExpT", count))).toThrow(/truncated/); + expect(() => parse(erlangBeamChunksFixture([["ExpT", count]]))).toThrow( + /truncated/, + ); }); it("fails when the atom budget is exhausted", () => { diff --git a/src/erlang/BeamParser.ts b/src/erlang/BeamParser.ts index 47c52d8995..f1c8485e49 100644 --- a/src/erlang/BeamParser.ts +++ b/src/erlang/BeamParser.ts @@ -256,12 +256,16 @@ export const parseErlangBeam = ( payloads.set(name, { payload, offset: at }); at = padded; } - const atomChunk = payloads.get("AtU8") ?? payloads.get("Atom"); - const atomEncoding = payloads.has("AtU8") - ? "utf8" - : payloads.has("Atom") - ? "latin1" - : null; + const utf8AtomChunk = payloads.get("AtU8"); + const legacyAtomChunk = payloads.get("Atom"); + // Historical OTP selects Atom when AtU8 has no payload. Without a legacy + // table, a present empty AtU8 still fails its required count header. + const useUtf8Atoms = + utf8AtomChunk !== undefined && + (utf8AtomChunk.payload.length > 0 || legacyAtomChunk === undefined); + const atomChunk = useUtf8Atoms ? utf8AtomChunk : legacyAtomChunk; + const atomEncoding = + atomChunk === undefined ? null : useUtf8Atoms ? "utf8" : "latin1"; const atoms = atomChunk === undefined || atomEncoding === null ? null @@ -300,10 +304,13 @@ export const parseErlangBeam = ( function: atomAt(atoms, row[1]!), arity: row[2]!, })); + const lambdaChunk = payloads.get("FunT"); + // OTP skips the optional lambda parser for a zero-byte selected chunk. + // A four-byte zero count remains a decoded, explicitly empty table. const lambdas = - payloads.get("FunT") === undefined + lambdaChunk === undefined || lambdaChunk.payload.length === 0 ? null - : parseRows(payloads.get("FunT")!.payload, 6, "lambda").map((row) => ({ + : parseRows(lambdaChunk.payload, 6, "lambda").map((row) => ({ function_index: row[0]!, function: atomAt(atoms, row[0]!), arity: row[1]!, diff --git a/tests/acceptance/erlangBeamCli.test.ts b/tests/acceptance/erlangBeamCli.test.ts index 44209897cd..d7cc3c040c 100644 --- a/tests/acceptance/erlangBeamCli.test.ts +++ b/tests/acceptance/erlangBeamCli.test.ts @@ -1,11 +1,15 @@ -import { writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; import { toolContract } from "../../src/contracts/toolContracts.js"; import { + erlangBeamChunksFixture, erlangBeamFixture, + erlangBeamLegacyAtomsFixture, erlangBeamOtp29Fixture, } from "../../src/erlang/Beam.fixture.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; @@ -72,6 +76,86 @@ cliTest("reports BEAM exports through the public CLI", async ({ cli }) => { ).toBe(true); }); +cliTest( + "inspects empty optional BEAM chunks through CLI and MCP without losing source evidence", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-beam-empty-chunks-"); + // Append an inert chunk to the compiler fixture; no bytecode is executed. + const original = erlangBeamOtp29Fixture(); + const extension = erlangBeamChunksFixture([["FunT", Buffer.alloc(0)]]); + const emptyLambda = Buffer.concat([original, extension.subarray(12)]); + emptyLambda.writeUInt32BE(emptyLambda.length - 8, 4); + const { call } = await connectLocalToolsMcp(); + for (const { + label, + bytes, + chunk, + module, + atomEncoding, + exportedFunction, + } of [ + { + label: "lambda", + bytes: emptyLambda, + chunk: "FunT", + module: "rea_beam_fixture", + atomEncoding: "utf8", + exportedFunction: "hello", + }, + // This reduced legacy container asserts structural facts, not runtime loading. + { + label: "legacy-atoms", + bytes: erlangBeamLegacyAtomsFixture(Buffer.alloc(0)), + chunk: "AtU8", + module: "demo", + atomEncoding: "latin1", + exportedFunction: "ÿ", + }, + ]) { + const path = join(root, `${label}.beam`); + await writeFile(path, bytes); + const output = await cli.run({ + arguments: ["inspect-erlang-beam", path, "--json"], + }); + expect(output.exitCode, output.stderr).toBe(0); + const report = toolContract("inspect_erlang_beam").outputSchema.parse( + output.json, + ).normalized_result; + const reply = await call("inspect_erlang_beam", { path }); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + expect( + toolContract("inspect_erlang_beam").outputSchema.parse( + reply.structuredContent, + ).normalized_result, + ).toEqual(report); + expect(report).toMatchObject({ + module, + atom_encoding: atomEncoding, + lambdas: null, + coverage: { status: "complete" }, + artifact: { + path, + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }, + }); + expect(report.exports).toContainEqual( + expect.objectContaining({ function: exportedFunction }), + ); + expect(report.chunks).toContainEqual( + expect.objectContaining({ + name: chunk, + payload: expect.objectContaining({ + bytes: 0, + sha256: createHash("sha256").update(Buffer.alloc(0)).digest("hex"), + }), + }), + ); + expect(await readFile(path)).toEqual(bytes); + } + }, +); + cliTest( "decodes the last atom table while retaining every chunk occurrence", async ({ cli }) => { From 635f673a8c281cc4f7424ca571094093d8f4a1ab Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:32:54 +0800 Subject: [PATCH 6/8] fix(gpu): own CCOB boundaries and payload representations Use completed codec input to establish legacy CCOB extents, preserving declaration/integrity failures independently. Reuse bounded input windows, charge actual reads and decoding attempts, and qualify recovery after incomplete decoding. Publish only established or qualified terminal extents so interruption retains completed declarations. Recognize bitcode representation in HIP bundles while leaving inner facts unsupported. Fixes #6887 Fixes #6880 --- docs/gpu-code-objects.md | 21 ++- src/gpu/ClangOffloadBundle.ts | 4 +- src/gpu/GpuInventory.test.ts | 109 +++++++++++++++- src/gpu/GpuInventory.ts | 144 ++++++++++++++++++--- src/gpu/GpuPayloadCodec.test.ts | 26 +++- src/gpu/GpuPayloadCodec.ts | 43 ++++-- tests/acceptance/gpuCodeObjectsCli.test.ts | 144 +++++++++++++++++++++ tests/boundary/gpu/codeObjects.test.ts | 116 ++++++++++++++--- tests/fixtures/gpu/codeObjects.ts | 10 ++ 9 files changed, 553 insertions(+), 64 deletions(-) diff --git a/docs/gpu-code-objects.md b/docs/gpu-code-objects.md index 0171b1b0c4..fb02ac7bf7 100644 --- a/docs/gpu-code-objects.md +++ b/docs/gpu-code-objects.md @@ -32,6 +32,12 @@ unexamined payloads remain `unexamined`. An exact AMDGPU target is normalized at its note owner for comparison with a Clang bundle declaration; raw values remain intact. Unresolved feature comparisons stay `unknown`. +A HIP target ID identifies a toolchain target, not its payload representation. +Recognized raw or wrapped LLVM bitcode stays `unsupported` for kernel metadata +inspection, with the bundle ID, range and selected payload digest retained. +Bitcode validity and inner targets remain unexamined; malformed recognized ELF +still reports `malformed`. Standalone bitcode is outside GPU container coverage. + HIP bundle and metadata note recognition accept both the legacy `amdgcn` triple and LLVM's `amdgpu` subarchitecture triples, including declared generic processors such as `gfx11-generic` and `gfx9-4-generic`. Notes with the empty or explicit @@ -40,9 +46,18 @@ retaining their raw triple spelling. CUDA's legacy fatbinary compression flag is retained as an unsupported codec with its encoded and decoded extents; compressed bytes are not treated as PTX text. CCOB is recognized by the exact four signature bytes `CCOB`; high-bit-corrupted bytes are not decoded as ASCII -and accepted as a bundle. CCOB v1 has no stored total size, so a following -legacy bundle is located by the next `CCOB` signature and zero-byte alignment -padding remains part of the first container extent. +and accepted as a bundle. CCOB v1 has no stored total size: the completed zlib +or Zstd stream determines its encoded extent. Embedded `CCOB` text is ordinary +compressed data; zero-byte alignment after the stream belongs to neither +payload. If decoding fails or its codec is unsupported, a later `CCOB` marker +is only an uncertain recovery candidate. The affected row retains that +limitation and discovery remains partial. + +The declared decoded length bounds decompression. An understated length can +stop decoding before codec completion, so any subsequent recovery extent stays +unknown; the decoder does not expand its allowance to prove the bad declaration. +Interrupted observations omit unfinished v1 rows. A terminal capacity row +qualifies its retained range as the containing tail and does not infer siblings. `cuda_target_header` retains the OS ABI, normalized raw `e_flags` and exact header location. CUDA OS ABI `0x33` and the observed `0x41` flags layout use diff --git a/src/gpu/ClangOffloadBundle.ts b/src/gpu/ClangOffloadBundle.ts index 0a42368f5b..fe76dfc9d4 100644 --- a/src/gpu/ClangOffloadBundle.ts +++ b/src/gpu/ClangOffloadBundle.ts @@ -118,8 +118,8 @@ export const readCcobHeader = (bytes: Buffer, remaining: number) => { return { version, method, width, total, decoded, hash }; }; -/** Legacy v1 has no size field; LLVM finds the next CCOB magic as its boundary. */ -export const readLegacyCcobTotal = async ( +/** After failed/unsupported v1 decoding only: a marker is an uncertain recovery candidate. */ +export const findLegacyCcobRecoveryTotal = async ( source: GpuSource, base: number, ): Promise => { diff --git a/src/gpu/GpuInventory.test.ts b/src/gpu/GpuInventory.test.ts index e98be43042..269b14eb22 100644 --- a/src/gpu/GpuInventory.test.ts +++ b/src/gpu/GpuInventory.test.ts @@ -1,24 +1,29 @@ import { expect, it } from "vitest"; import { GpuInventory } from "./GpuInventory.js"; import { GpuBufferSource } from "./GpuSource.js"; +import { assertArtifactReadActive } from "../artifacts/ArtifactFileOperation.js"; import { ccobFixture, cubinFixture, cudaFixture, } from "../../tests/fixtures/gpu/codeObjects.js"; -it.each(["cuda", "ccob"] as const)( +it.each(["cuda", "ccob", "ccob-v1"] as const)( "refuses oversized %s output before reading compressed input", async (kind) => { const bytes = kind === "cuda" ? cudaFixture("zstd", [{ kind: 2, target: 80, bytes: cubinFixture() }]) - : ccobFixture(3); - const payloadOffset = kind === "cuda" ? 80 : 32; - bytes.writeBigUInt64LE( - BigInt(128 * 1024 * 1024 + 1), - kind === "cuda" ? 72 : 16, - ); + : kind === "ccob" + ? ccobFixture(3) + : Buffer.concat([ccobFixture(1), ccobFixture(2)]); + const payloadOffset = kind === "cuda" ? 80 : kind === "ccob" ? 32 : 20; + if (kind === "ccob-v1") bytes.writeUInt32LE(128 * 1024 * 1024 + 1, 8); + else + bytes.writeBigUInt64LE( + BigInt(128 * 1024 * 1024 + 1), + kind === "cuda" ? 72 : 16, + ); const source = new GpuBufferSource(bytes, "fixture"), reads: number[] = [], inventory = new GpuInventory({ path: "/fixture", analysis: "kernels" }); @@ -36,6 +41,65 @@ it.each(["cuda", "ccob"] as const)( sha256: null, }); expect(inventory.kernels).toEqual([]); + if (kind === "ccob-v1") { + expect(inventory.objects).toHaveLength(1); + expect(inventory.objects[0]!.location.size).toBe(bytes.length - 20); + expect(inventory.objects[0]!.limitations.join(" ")).toContain( + "only the containing tail", + ); + expect(inventory.objects[0]!.limitations.join(" ")).toContain( + "Later siblings were not inferred", + ); + } + }, +); +it.each(["zlib", "zstd"] as const)( + "omits an unfinished v1 %s extent when cancellation interrupts its source read", + async (codec) => { + const first = ccobFixture(2, codec), + second = ccobFixture(1, codec), + source = new GpuBufferSource(Buffer.concat([first, second]), "fixture"), + controller = new AbortController(), + reason = { selected: "cancel-v1-payload-read" }, + inventory = new GpuInventory( + { path: "/fixture", analysis: "inventory" }, + controller.signal, + ); + let interrupted = false; + const result = inventory.inspect({ + size: source.size, + containerId: source.containerId, + read: async (offset, size) => { + if (offset === first.length + 20) { + interrupted = true; + controller.abort(reason); + assertArtifactReadActive(controller.signal); + } + return source.read(offset, size); + }, + }); + await expect(result).rejects.toMatchObject({ + reason: "cancelled", + cause: reason, + }); + expect(interrupted).toBe(true); + const wrappers = inventory.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(wrappers).toHaveLength(1); + expect(wrappers[0]).toMatchObject({ + id: "fixture@0", + inspection: "complete", + encoded_size: first.length - 24, + location: { offset: 24, size: first.length - 24 }, + }); + expect( + inventory.objects.every( + ({ id, parent_id }) => + id !== `fixture@${first.length.toString(16)}` && + parent_id !== `fixture@${first.length.toString(16)}`, + ), + ).toBe(true); }, ); it("observes cancellation while walking flat empty bundles in a decoded buffer", async () => { @@ -55,3 +119,34 @@ it("observes cancellation while walking flat empty bundles in a decoded buffer", cause: reason, }); }); +it.each(["zlib", "zstd"] as const)( + "reuses observed %s input across multiple valid legacy CCOB siblings", + async (codec) => { + const count = 16, + bytes = Buffer.concat( + Array.from({ length: count }, () => ccobFixture(1, codec)), + ), + source = new GpuBufferSource(bytes, "fixture"), + inventory = new GpuInventory({ path: "/fixture", analysis: "inventory" }); + let observedReadBytes = 0; + await inventory.inspect({ + size: source.size, + containerId: source.containerId, + read: async (offset, size) => { + observedReadBytes += size; + return source.read(offset, size); + }, + }); + const bundles = inventory.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(bundles).toHaveLength(count); + expect(bundles.every(({ inspection }) => inspection === "complete")).toBe( + true, + ); + expect(inventory.discovery).toBe("complete"); + // Includes prefix/alignment reads. Re-reading every suffix grows with the + // square of the sibling count and exceeds this physical-input allowance. + expect(observedReadBytes).toBeLessThan(2 * bytes.length); + }, +); diff --git a/src/gpu/GpuInventory.ts b/src/gpu/GpuInventory.ts index 8926f0e66e..3b1c310459 100644 --- a/src/gpu/GpuInventory.ts +++ b/src/gpu/GpuInventory.ts @@ -1,14 +1,14 @@ import { CCOB_MAGIC, CLANG_BUNDLE_MAGIC, - readLegacyCcobTotal, + findLegacyCcobRecoveryTotal, readClangBundleDirectory, readCcobHeader, } from "./ClangOffloadBundle.js"; import { gpuObject, gpuLocation, type GpuPayloadTarget } from "./GpuObject.js"; import { readCudaTarget } from "./CudaTarget.js"; import { readCudaFatbinary } from "./CudaFatbinary.js"; -import { decodeGpuPayload } from "./GpuPayloadCodec.js"; +import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; import { createHash } from "node:crypto"; import { scheduler } from "node:timers/promises"; import type { @@ -99,6 +99,11 @@ export class GpuInventory { row.sha256 = sha256; row.decoded_size = size; } + #encodedExtent(row: GpuObject, size: number): void { + this.#retain({ encoded_size: size, size }); + row.encoded_size = size; + row.location.size = size; + } #objectLimitations(row: GpuObject, messages: string[]): void { this.#retain(messages); row.limitations.push(...messages); @@ -394,6 +399,8 @@ export class GpuInventory { this.kernels.push(kernel); } } else if (row.kind === "cubin" || row.kind === "amdgpu") { + if (row.kind === "amdgpu" && isLlvmBitcode(bytes)) + throw new BinaryFormatFailure("unsupported", LLVM_BITCODE_LIMITATION); coverage = await this.#elfFacts(decoded, row, row.kind, preparedAmd); } else { row.inspection = "unsupported"; @@ -464,6 +471,19 @@ export class GpuInventory { }; try { + // An AMD target declaration does not establish an ELF representation. + // Without an outer target, inventory must still avoid an ELF corruption + // claim for recognizable bitcode; its inner target remains unexamined. + if ( + directory === undefined && + isLlvmBitcode(await child.read(0, Math.min(4, child.size))) + ) { + this.discovery = "partial"; + if (this.input.analysis !== "kernels" || !this.#selected(row)) + this.#objectLimitations(row, [LLVM_BITCODE_LIMITATION]); + await this.#payload(source, row); + return; + } const native = directory ?? (await readGpuNativeDirectory( @@ -528,6 +548,7 @@ export class GpuInventory { async #hip(source: GpuSource, depth: number): Promise { let base = 0; let bundles = 0; + let legacyWindow: { offset: number; bytes: Buffer } | undefined; while (base < source.size) { if (++bundles % 128 === 0) { await scheduler.yield(); @@ -552,12 +573,10 @@ export class GpuInventory { } if (!prefix.subarray(0, CCOB_MAGIC.length).equals(CCOB_MAGIC)) rejectGpu("Unexpected bytes between Clang offload bundles"); - const legacyTotal = - prefix.length >= 6 && prefix.readUInt16LE(4) === 1 - ? await readLegacyCcobTotal(source, base) - : source.size - base; - const header = readCcobHeader(prefix, legacyTotal), - { width, total, decoded, hash, version, method } = header; + const header = readCcobHeader(prefix, source.size - base), + { width, decoded, hash, version, method } = header; + let total = header.total; + let extentKnown = version !== 1; const row = gpuObject(source, { offset: base + width, size: total - width, @@ -568,21 +587,81 @@ export class GpuInventory { method === 0 ? "zlib" : method === 1 ? "zstd" : "unsupported"; row.compression_id = `ccob:${version}:${method}`; row.decoded_size = decoded; - this.#add(row); + if (extentKnown) this.#add(row); if (row.compression === "unsupported") { row.inspection = "unsupported"; this.#objectLimitations(row, ["Unsupported CCOB compression method"]); this.discovery = "partial"; } else try { - this.#budgetWork(total - width, decoded); - const data = await decodeGpuPayload( - await source.read(base + width, total - width), - row.compression, - decoded, - this.signal, - version === 1, - ); + // v1 has no encoded size. Supply a bounded suffix, then retain only + // the codec's completed extent, never an embedded CCOB marker. + const inputSize = + version === 1 + ? Math.min(total - width, GPU_OBJECT_BYTES) + : total - width; + let data: Buffer; + if (version === 1) { + const start = base + width; + const cached = + legacyWindow !== undefined && + start >= legacyWindow.offset && + start < legacyWindow.offset + legacyWindow.bytes.length; + const readWindow = async (): Promise => { + this.#budgetWork(inputSize, decoded); + legacyWindow = undefined; + const bytes = await source.read(start, inputSize); + legacyWindow = { offset: start, bytes }; + return bytes; + }; + if (cached) this.#budgetWork(decoded); + let encoded: Buffer | undefined = cached + ? legacyWindow!.bytes.subarray(start - legacyWindow!.offset) + : await readWindow(); + const available = encoded.length; + const compression = row.compression; + const completed = await decodeGpuPayloadStream( + encoded, + compression, + decoded, + this.signal, + ).catch(async (cause: unknown) => { + // A cached window may end inside this sibling. Refill once + // before treating that incomplete view as a malformed stream. + if ( + !(cause instanceof BinaryFormatFailure) || + !cached || + available >= inputSize + ) + throw cause; + // Retire the previous window before allocating its replacement. + encoded = undefined; + return decodeGpuPayloadStream( + await readWindow(), + compression, + decoded, + this.signal, + ); + }); + total = width + completed.consumedBytes; + extentKnown = true; + this.#encodedExtent(row, completed.consumedBytes); + // Publish the completed parent extent before inspecting its children. + this.#add(row); + data = completed.buffer; + if (data.length !== decoded) + rejectGpu( + "GPU decompressed length disagrees with its declaration", + ); + } else { + this.#budgetWork(inputSize, decoded); + data = await decodeGpuPayload( + await source.read(base + width, inputSize), + row.compression, + decoded, + this.signal, + ); + } if ( createHash("md5").update(data).digest().readBigUInt64LE(0) !== hash ) @@ -594,10 +673,41 @@ export class GpuInventory { this.#retainFailure(row, cause); this.discovery = "partial"; } + if (!extentKnown) { + if (row.inspection !== "capacity") + try { + total = await findLegacyCcobRecoveryTotal(source, base); + } catch (cause) { + if (!(cause instanceof BinaryResourceFailure)) throw cause; + this.#retainFailure(row, cause); + } + // A capacity refusal cannot establish a boundary for later siblings. + if (row.inspection === "capacity") { + this.#objectLimitations(row, [ + "CCOB v1 codec extent is unknown; the retained size is only the containing tail, not a completed codec extent. Later siblings were not inferred.", + ]); + this.#add(row); + return; + } + this.#encodedExtent(row, total - width); + this.#objectLimitations(row, [ + total < source.size - base + ? "CCOB v1 codec extent is unknown; the next CCOB marker is only an uncertain recovery candidate." + : "CCOB v1 codec extent is unknown; the retained range extends to the container end.", + ]); + this.#add(row); + } base = await nextGpuContainerOffset(source, base + total); } } } +const LLVM_BITCODE_LIMITATION = + "LLVM bitcode kernel metadata is outside current coverage; bitcode validity and inner target were not inspected."; +/** LLVM raw and wrapper signatures identify representation, not bitcode validity. */ +const isLlvmBitcode = (bytes: Buffer): boolean => + bytes.length >= 4 && + (bytes.readUInt32LE(0) === 0xdec04342 || + bytes.readUInt32LE(0) === 0x0b17c0de); const isGpuSection = (name: string): boolean => [ ".nv_fatbin", diff --git a/src/gpu/GpuPayloadCodec.test.ts b/src/gpu/GpuPayloadCodec.test.ts index 97fc039ee8..465fc725c4 100644 --- a/src/gpu/GpuPayloadCodec.test.ts +++ b/src/gpu/GpuPayloadCodec.test.ts @@ -1,6 +1,6 @@ import { expect, it } from "vitest"; import { deflateSync, zstdCompressSync } from "node:zlib"; -import { decodeGpuPayload } from "./GpuPayloadCodec.js"; +import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; it.each(["zlib", "zstd"] as const)( "requires all declared %s input bytes to belong to the codec stream", async (codec) => { @@ -19,3 +19,27 @@ it.each(["zlib", "zstd"] as const)( ).rejects.toMatchObject({ kind: "malformed" }); }, ); +it.each(["zlib", "zstd"] as const)( + "retains completed %s input accounting before trailing container bytes", + async (codec) => { + const source = Buffer.from("ordinary CCOB marker"), + compressed = + codec === "zstd" + ? zstdCompressSync(source) + : deflateSync(source, { level: 0 }), + input = Buffer.concat([ + compressed, + Buffer.from("CCOB following container"), + ]); + expect(await decodeGpuPayloadStream(input, codec, source.length)).toEqual({ + buffer: source, + consumedBytes: compressed.length, + }); + await expect( + decodeGpuPayloadStream(compressed.subarray(0, -1), codec, source.length), + ).rejects.toMatchObject({ kind: "malformed" }); + await expect( + decodeGpuPayload(compressed, codec, source.length + 1), + ).rejects.toMatchObject({ kind: "malformed" }); + }, +); diff --git a/src/gpu/GpuPayloadCodec.ts b/src/gpu/GpuPayloadCodec.ts index e84a94ad86..29248d5ecc 100644 --- a/src/gpu/GpuPayloadCodec.ts +++ b/src/gpu/GpuPayloadCodec.ts @@ -6,18 +6,17 @@ import { decodeLz4Block } from "./Lz4Block.js"; import { rejectGpu } from "./GpuSource.js"; const unzip = promisify(inflate); const unzstd = promisify(zstdDecompress); -export const decodeGpuPayload = async ( +/** Completed codec bytes and their encoded extent share one completion owner. */ +export const decodeGpuPayloadStream = async ( bytes: Buffer, - compression: "zlib" | "zstd" | "lz4", - expected: number, + compression: "zlib" | "zstd", + maximumDecodedSize: number, signal?: AbortSignal, - allowZeroPadding = false, -): Promise => { +): Promise<{ buffer: Buffer; consumedBytes: number }> => { assertArtifactReadActive(signal); - if (compression === "lz4") return decodeLz4Block(bytes, expected, signal); let result: unknown; try { - const maxOutputLength = Math.max(1, expected); + const maxOutputLength = Math.max(1, maximumDecodedSize); result = await (compression === "zstd" ? unzstd(bytes, { maxOutputLength, @@ -34,16 +33,32 @@ export const decodeGpuPayload = async ( assertArtifactReadActive(signal); const { buffer: decoded, consumedBytes: consumed } = nodeCodecResult(result); if ( - consumed !== bytes.length && - (!allowZeroPadding || - consumed < 0 || - consumed > bytes.length || - !bytes.subarray(consumed).every((byte) => byte === 0)) + !Number.isSafeInteger(consumed) || + consumed <= 0 || + consumed > bytes.length ) + rejectGpu("GPU decoder returned an invalid complete codec extent"); + return { buffer: decoded, consumedBytes: consumed }; +}; +export const decodeGpuPayload = async ( + bytes: Buffer, + compression: "zlib" | "zstd" | "lz4", + expected: number, + signal?: AbortSignal, +): Promise => { + assertArtifactReadActive(signal); + if (compression === "lz4") return decodeLz4Block(bytes, expected, signal); + const decoded = await decodeGpuPayloadStream( + bytes, + compression, + expected, + signal, + ); + if (decoded.consumedBytes !== bytes.length) rejectGpu( "GPU compressed payload contains bytes outside its complete codec stream", ); - if (decoded.length !== expected) + if (decoded.buffer.length !== expected) rejectGpu("GPU decompressed length disagrees with its declaration"); - return decoded; + return decoded.buffer; }; diff --git a/tests/acceptance/gpuCodeObjectsCli.test.ts b/tests/acceptance/gpuCodeObjectsCli.test.ts index b9f5ea8165..7d9c3e73ad 100644 --- a/tests/acceptance/gpuCodeObjectsCli.test.ts +++ b/tests/acceptance/gpuCodeObjectsCli.test.ts @@ -2,6 +2,7 @@ import { expect, it } from "vitest"; import { writeFile, unlink } from "node:fs/promises"; import { join } from "node:path"; import { createHash } from "node:crypto"; +import { deflateSync } from "node:zlib"; import { toolContract } from "../../src/contracts/toolContracts.js"; import { gpuCodeObjectsSchema } from "../../src/domain/gpu/gpuCodeObjects.js"; import { @@ -21,6 +22,7 @@ import { cubinFixture, hipFixture, elfFixture, + llvmBitcodeFixture, } from "../fixtures/gpu/codeObjects.js"; cliTest( @@ -65,6 +67,148 @@ cliTest( }, ); +cliTest( + "inventories a complete CCOB stream containing ordinary CCOB payload text", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-ccob-embedded-marker-"); + const bundle = hipFixture( + Buffer.from("ordinary CCOB marker"), + "host-other", + ), + compressed = deflateSync(bundle, { level: 0 }); + expect(compressed.indexOf("CCOB")).toBeGreaterThan(0); + for (const version of [1, 2] as const) { + const width = version === 1 ? 20 : 24, + header = Buffer.from( + ccobFixture(version, "zlib", bundle).subarray(0, width), + ); + if (version === 2) header.writeUInt32LE(width + compressed.length, 8); + const bytes = Buffer.concat([header, compressed]), + path = join(root, `v${version}`); + await writeFile(path, bytes); + const response = await cli.run({ + arguments: [ + "inspect-gpu-code-objects", + path, + "--analysis", + "inventory", + "--detail", + "complete", + "--json", + ], + }); + expect(response.exitCode, response.stderr).toBe(0); + const evidence = parseEvidence( + toolContract("inspect_gpu_code_objects").outputSchema.parse( + response.json, + ), + ); + const report = gpuCodeObjectsSchema.parse(evidence.normalized_result); + expect(report.coverage).toMatchObject({ + discovery: "complete", + payloads: "not-requested", + }); + expect(report.objects).toHaveLength(3); + expect(report.objects[0]).toMatchObject({ + kind: "hip-bundle", + inspection: "complete", + encoded_size: compressed.length, + decoded_size: bundle.length, + sha256: createHash("sha256").update(bundle).digest("hex"), + location: { offset: width, size: compressed.length }, + }); + expect( + report.objects + .slice(1) + .map(({ bundle_id, inspection }) => [bundle_id, inspection]), + ).toEqual([ + ["host-x86_64-unknown-linux", "complete"], + ["host-other", "complete"], + ]); + } + }, +); + +cliTest( + "distinguishes unsupported HIP bitcode from corrupted ELF through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-hip-bitcode-cli-"), + bitcode = llvmBitcodeFixture(), + corruptedElf = amdFixture(), + bundleId = "hip-amdgcn-amd-amdhsa--gfx900"; + corruptedElf.writeBigUInt64LE(BigInt(corruptedElf.length + 1), 40); + for (const [name, payload, analysis, inspection] of [ + ["inventory", bitcode, "inventory", "inventory"], + ["bitcode", bitcode, "kernels", "unsupported"], + ["corrupt-elf", corruptedElf, "kernels", "malformed"], + ] as const) { + const bytes = hipFixture(payload, bundleId), + path = join(root, name); + await writeFile(path, bytes); + const response = await cli.run({ + arguments: [ + "inspect-gpu-code-objects", + path, + "--analysis", + analysis, + "--targets", + '["gfx900"]', + "--detail", + "complete", + "--json", + ], + }); + expect(response.exitCode, response.stderr).toBe(0); + const evidence = parseEvidence( + toolContract("inspect_gpu_code_objects").outputSchema.parse( + response.json, + ), + ); + const report = gpuCodeObjectsSchema.parse(evidence.normalized_result); + const row = report.objects.find(({ kind }) => kind === "amdgpu")!; + expect(row).toMatchObject({ + bundle_id: bundleId, + target: "gfx900", + payload_target: null, + inspection, + encoded_size: payload.length, + location: { + container_id: null, + offset: bytes.length - payload.length, + size: payload.length, + }, + sha256: + analysis === "inventory" + ? null + : createHash("sha256").update(payload).digest("hex"), + }); + expect(report.coverage.payloads).toBe( + analysis === "inventory" ? "not-requested" : "partial", + ); + expect(report.kernels).toEqual([]); + if (name === "bitcode") { + expect(row.limitations.join(" ")).toContain("LLVM bitcode"); + expect(row.limitations.join(" ")).toContain( + "validity and inner target were not inspected", + ); + } + } + const path = join(root, "raw-bitcode"); + await writeFile(path, bitcode); + const raw = await cli.run({ + arguments: [ + "inspect-gpu-code-objects", + path, + "--analysis", + "kernels", + "--json", + ], + }); + expect(raw.exitCode).toBe(1); + expect(raw.json).toMatchObject({ code: "unsupported_target" }); + }, +); + cliTest( "keeps embedded Clang bundle reads inside their native section", async ({ cli }) => { diff --git a/tests/boundary/gpu/codeObjects.test.ts b/tests/boundary/gpu/codeObjects.test.ts index 029cdb14b8..f172ff2786 100644 --- a/tests/boundary/gpu/codeObjects.test.ts +++ b/tests/boundary/gpu/codeObjects.test.ts @@ -2,6 +2,7 @@ import { expect, it, onTestFinished } from "vitest"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { deflateSync } from "node:zlib"; +import { createHash } from "node:crypto"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { gpuCodeObjectsPartialSchema } from "../../../src/domain/gpu/gpuCodeObjects.js"; import { GpuCodeObjectsProvider } from "../../../src/gpu/GpuCodeObjectsProvider.js"; @@ -15,6 +16,7 @@ import { elfFixture, ptxFixture, amdFixture, + llvmBitcodeFixture, } from "../../fixtures/gpu/codeObjects.js"; const inspect = async ( bytes: Buffer, @@ -278,6 +280,37 @@ it("keeps AMDGPU processor identity unknown when ELF flags are unassigned", asyn inspection: "inventory", }); }); +it.each([false, true])( + "keeps HIP bitcode representation unsupported with wrapped=%s and no declared processor", + async (wrapped) => { + const payload = llvmBitcodeFixture(wrapped), + bytes = hipFixture(payload, "hip-amdgcn-amd-amdhsa--"); + for (const analysis of ["inventory", "kernels"] as const) { + const result = await inspect(bytes, { analysis }); + if (!result.ok) throw result.error; + const row = result.value.objects.find(({ kind }) => kind === "amdgpu")!; + expect(row).toMatchObject({ + bundle_id: "hip-amdgcn-amd-amdhsa--", + target: null, + payload_target: null, + inspection: analysis === "inventory" ? "inventory" : "unsupported", + encoded_size: payload.length, + location: { size: payload.length }, + sha256: + analysis === "inventory" + ? null + : createHash("sha256").update(payload).digest("hex"), + }); + expect(row.limitations.join(" ")).toContain("LLVM bitcode"); + expect(row.limitations.join(" ")).not.toContain("not an ELF image"); + expect(result.value.coverage).toMatchObject({ + discovery: "partial", + payloads: analysis === "inventory" ? "not-requested" : "partial", + }); + expect(result.value.kernels).toEqual([]); + } + }, +); it.each([ ["amdgcn", "gfx900:xnack+"], ["amdgcn", "gfx900+xnack"], @@ -930,26 +963,34 @@ it("does not dispatch a high-bit-corrupted DOS signature to the PE parser", asyn error: { _tag: "AnalysisUnsupportedTargetError" }, }); }); -it("uses each legacy CCOB v1 stream boundary across alignment padding", async () => { - const first = ccobFixture(1, "zlib"), - second = ccobFixture(2, "zlib"), - padding = Buffer.alloc((8 - (first.length % 8)) % 8); - const result = await inspect(Buffer.concat([first, padding, second])); - if (!result.ok) throw result.error; - const bundles = result.value.objects.filter( - ({ kind }) => kind === "hip-bundle", - ); - expect(bundles).toHaveLength(2); - expect( - bundles.map(({ inspection, decoded_size }) => [inspection, decoded_size]), - ).toEqual([ - ["complete", expect.any(Number)], - ["complete", expect.any(Number)], - ]); - expect(bundles[0]!.sha256).toBe(bundles[1]!.sha256); - expect(result.value.kernels).toHaveLength(2); - expect(result.value.coverage.discovery).toBe("complete"); -}); +it.each(["zlib", "zstd"] as const)( + "uses each legacy CCOB v1 %s stream boundary across alignment padding", + async (codec) => { + const first = ccobFixture(1, codec), + second = ccobFixture(2, "zlib"), + padding = Buffer.alloc((8 - (first.length % 8)) % 8); + const result = await inspect(Buffer.concat([first, padding, second])); + if (!result.ok) throw result.error; + const bundles = result.value.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(bundles).toHaveLength(2); + expect( + bundles.map(({ inspection, decoded_size }) => [inspection, decoded_size]), + ).toEqual([ + ["complete", expect.any(Number)], + ["complete", expect.any(Number)], + ]); + expect(bundles[0]!.sha256).toBe(bundles[1]!.sha256); + expect(bundles[0]!.encoded_size).toBe(first.length - 20); + expect(bundles[0]!.location.size).toBe(first.length - 20); + expect(bundles[1]!.metadata_location.offset).toBe( + first.length + padding.length, + ); + expect(result.value.kernels).toHaveLength(2); + expect(result.value.coverage.discovery).toBe("complete"); + }, +); it("keeps a later CCOB bundle after a truncated legacy v1 stream", async () => { const truncated = ccobFixture(1, "zlib").subarray(0, -1), later = ccobFixture(2, "zlib"); @@ -965,7 +1006,42 @@ it("keeps a later CCOB bundle after a truncated legacy v1 stream", async () => { ]); expect(result.value.kernels).toHaveLength(1); expect(result.value.coverage.discovery).toBe("partial"); + expect(bundles[0]!.limitations.join(" ")).toContain( + "uncertain recovery candidate", + ); }); +it.each(["checksum", "decoded-size"] as const)( + "preserves a completed v1 codec extent after a bad %s declaration", + async (declaration) => { + const bundle = hipFixture( + Buffer.from("ordinary CCOB marker"), + "host-other", + ), + compressed = deflateSync(bundle, { level: 0 }), + header = Buffer.from(ccobFixture(1, "zlib", bundle).subarray(0, 20)); + if (declaration === "checksum") header[12] = header[12]! ^ 1; + else header.writeUInt32LE(bundle.length + 1, 8); + const first = Buffer.concat([header, compressed]); + const result = await inspect( + Buffer.concat([first, ccobFixture(2, "zlib")]), + ); + if (!result.ok) throw result.error; + const wrappers = result.value.objects.filter( + ({ kind }) => kind === "hip-bundle", + ); + expect(wrappers.map(({ inspection }) => inspection)).toEqual([ + "malformed", + "complete", + ]); + expect(wrappers[0]).toMatchObject({ + encoded_size: compressed.length, + location: { size: compressed.length }, + }); + expect(wrappers[0]!.limitations.join(" ")).not.toContain("recovery"); + expect(wrappers[1]!.metadata_location.offset).toBe(first.length); + expect(result.value.coverage.discovery).toBe("partial"); + }, +); it("continues at the declared CCOB boundary after an unsupported codec", async () => { const unknown = ccobFixture(); unknown.writeUInt16LE(99, 6); diff --git a/tests/fixtures/gpu/codeObjects.ts b/tests/fixtures/gpu/codeObjects.ts index 7a39a4a562..49a3ca55b7 100644 --- a/tests/fixtures/gpu/codeObjects.ts +++ b/tests/fixtures/gpu/codeObjects.ts @@ -321,6 +321,16 @@ export const hipFixture = ( head.writeBigUInt64LE(2n, 24); return Buffer.concat([head, ...headers, payload]); }; +/** Representation recognition only; these bytes do not establish valid LLVM IR. */ +export const llvmBitcodeFixture = (wrapped = false): Buffer => { + const raw = Buffer.from([0x42, 0x43, 0xc0, 0xde, 0, 0, 0, 0]); + if (!wrapped) return raw; + const header = Buffer.alloc(20); + header.writeUInt32LE(0x0b17c0de, 0); + header.writeUInt32LE(20, 8); + header.writeUInt32LE(raw.length, 12); + return Buffer.concat([header, raw]); +}; export const ccobFixture = ( version: 1 | 2 | 3 = 2, method: "zlib" | "zstd" = "zstd", From f91b7f8009c9af27a5e0148e3b07d20bfe3b7a2a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:04:57 +0800 Subject: [PATCH 7/8] fix(javascript): distinguish resolver discovery from module loading Preserve load and resolve operations through semantic facts, worker transfer and graph Evidence. Only loads propagate executable ownership. Project ambient member writes and escapes through the shared path- and site-aware mutation owner; retain source literals, bases and uncertainty. Cover ownership and mutation boundaries through CLI and MCP. --- docs/javascript-artifact-reconstruction.md | 10 + .../JavaScriptArtifactGraphFindings.ts | 3 + .../javascriptSemanticAmbientWrites.test.ts | 27 ++- .../javascriptSemanticAmbientWrites.ts | 37 +++- .../javascript/javascriptSemanticAnalysis.ts | 15 ++ .../javascriptSemanticMemberMutations.ts | 31 ++- .../javascriptSemanticRequireOrigin.ts | 26 ++- .../javascriptStaticAnalysisCalls.ts | 12 +- ...iptStaticAnalysisLoaderInvocations.test.ts | 8 +- .../javascriptStaticAnalysisState.ts | 1 + .../javascriptStaticAnalysisTypes.ts | 2 + .../JavaScriptAnalysisWorkerSchemas.ts | 1 + .../javascriptIndirectLoaderOwners.test.ts | 180 ++++++++++++++++++ .../javascriptLiteralLoaderOwnership.ts | 15 +- 14 files changed, 351 insertions(+), 17 deletions(-) diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 719e0c93b7..6f0114632e 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -777,6 +777,16 @@ does not change the source file or call-site Evidence identity. If the filename cannot be proven, REA keeps the require target unresolved rather than silently resolving it from the caller's directory. +Module-reference graph edges, including native-addon `loads` edges, retain +`properties.operation`: `load` identifies +an import or loader invocation that may execute the target; `resolve` identifies +a proven `require.resolve()` lookup that discovers a target without executing it. +Both retain the exact specifier, resolution mode, selected resolution base, and +call-site Evidence. Resolve-only edges do not propagate application ownership or +create executable-import gaps when their targets are missing. A modified +`.resolve` member remains an uncertain `load`, because its replacement may execute +code. These static facts do not establish that a call runs at runtime. + ESM relative module paths, legacy package subpaths without exports, and selected package exports targets use URL suffix handling and one percent-decoding pass. Bare specifiers retain their complete query/fragment text for exports key matching. diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.ts index 69adac5a1b..98f9644600 100644 --- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts +++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts @@ -205,6 +205,7 @@ const addReference = ( ? null : resolveReference(context, input.file.path, input.value, input.coverage); if ( + input.value.operation === "load" && (input.value.kind === "static-import" || input.value.kind === "dynamic-import" || input.value.kind === "require") && @@ -222,6 +223,7 @@ const addReference = ( resolved !== null && resolved.path !== null && resolved.file !== null && + input.value.operation === "load" && (input.value.kind === "static-import" || input.value.kind === "dynamic-import" || input.value.kind === "require") @@ -305,6 +307,7 @@ const addReference = ( relation: resolved?.file?.kind === "native-addon" ? "loads" : "imports", properties: { kind: input.value.kind, + operation: input.value.operation, specifier: input.value.specifier, expression: input.value.expression, resolution_base_path: input.value.resolution_base_path ?? null, diff --git a/src/domain/javascript/javascriptSemanticAmbientWrites.test.ts b/src/domain/javascript/javascriptSemanticAmbientWrites.test.ts index 0d785dbef1..f2deb2ba51 100644 --- a/src/domain/javascript/javascriptSemanticAmbientWrites.test.ts +++ b/src/domain/javascript/javascriptSemanticAmbientWrites.test.ts @@ -1,7 +1,10 @@ import { expect, it } from "vitest"; import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; -import { indexJavaScriptBindingsSteps } from "./javascriptSemanticAnalysis.js"; +import { + indexJavaScriptBindingsSteps, + indexJavaScriptBindingsWithReceiversSteps, +} from "./javascriptSemanticAnalysis.js"; import { completeSemanticSteps } from "./javascriptSemanticTraversal.js"; import { parseJavaScriptSource } from "./javascriptSourceParser.js"; @@ -103,3 +106,25 @@ new Worker(new URL("./worker.js", import.meta.url));`; ); expect(worker?.specifier).toBe(specifier); }); + +it("retains getter-derived ambient escapes once per invoked read site", () => { + const source = `const box={loader:require,get trigger(){this.loader.resolve=this.loader;return 0}}; +box.trigger; +box.trigger;`; + const parsed = parseJavaScriptSource(source); + if (parsed === null) throw new Error("Expected getter fixture"); + const index = completeSemanticSteps( + indexJavaScriptBindingsWithReceiversSteps(parsed.program), + ); + completeSemanticSteps(index.indexMemberMutationFactsSteps()); + const effects = index.bindings.ambientEscapeEffects.filter( + ({ path }) => + JSON.stringify(path) === JSON.stringify(["global", "require", null]), + ); + expect( + effects.map(({ node }) => + node === undefined ? null : source.slice(node.start ?? 0, node.end ?? 0), + ), + ).toEqual(["box.trigger", "box.trigger"]); + expect(effects[0]?.node).not.toBe(effects[1]?.node); +}); diff --git a/src/domain/javascript/javascriptSemanticAmbientWrites.ts b/src/domain/javascript/javascriptSemanticAmbientWrites.ts index a3e7c6d8ff..5035c6944b 100644 --- a/src/domain/javascript/javascriptSemanticAmbientWrites.ts +++ b/src/domain/javascript/javascriptSemanticAmbientWrites.ts @@ -49,6 +49,36 @@ const canonicalAmbientPath = ( type AmbientPath = readonly (string | null)[]; +const retainedAmbientEffects = new WeakMap< + JavaScriptSemanticAnalysisState, + Map> +>(); + +/** Retain canonical ambient effects once, whether discovered by syntax or receiver expansion. */ +export const retainSemanticAmbientEffect = ( + state: JavaScriptSemanticAnalysisState, + kind: "write" | "escape", + path: AmbientPath, + node: t.Node | undefined, +): boolean => { + const selected = kind === "escape" ? [...path, null] : path; + const byNode = retainedAmbientEffects.get(state) ?? new Map(); + const keys = byNode.get(node) ?? new Set(); + const key = JSON.stringify([kind, selected]); + if (keys.has(key)) return false; + keys.add(key); + byNode.set(node, keys); + retainedAmbientEffects.set(state, byNode); + if (kind === "write") { + state.ambientWrites.retain(selected); + state.ambientWriteEffects.push({ path: selected, node }); + } else { + state.ambientEscapes.retain(selected); + state.ambientEscapeEffects.push({ path: selected, node }); + } + return true; +}; + /** Undefined means a cyclic or conflicting projection can select any depth. */ type AmbientProjection = AmbientPath | undefined; @@ -641,17 +671,14 @@ export function* indexJavaScriptAmbientWritesSteps( ...path.slice(2), ]); for (const resolved of paths) { - state.ambientWrites.retain(resolved); - state.ambientWriteEffects.push({ path: resolved, node }); + retainSemanticAmbientEffect(state, "write", resolved, node); } } for (const { node, target, browserReceiver, projection } of escapes) for (const path of ambientWrites(target, node, true, projection)) // Only proven browser APIs keep their global receiver from user code. if (!(browserReceiver && path.length === 1 && path[0] === "global")) { - const escaped = [...path, null]; - state.ambientEscapes.retain(escaped); - state.ambientEscapeEffects.push({ path: escaped, node }); + retainSemanticAmbientEffect(state, "escape", path, node); } // A function may call a browser method replaced later in source order. if ( diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index b22047c67f..adfe1b6429 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -68,6 +68,7 @@ import { semanticBundledRequireOrigin } from "./javascriptBundledRequireOrigin.j import { collectSemanticMemberMutationsSteps } from "./javascriptSemanticMemberMutations.js"; import { semanticEffectTiming } from "./javascriptSemanticMutationInitializers.js"; import { + createAmbientPathResolver, indexJavaScriptAmbientWritesSteps, isUnmodifiedGlobalReference, } from "./javascriptSemanticAmbientWrites.js"; @@ -279,6 +280,20 @@ const visitSemanticReferenceEffectsAtSite = ( } else if (lookup.kind === "ambient" || lookup.kind === "uncertain-ambient") { for (const node of state.ambientBindingWrites.get(root.name) ?? []) if (!retain("ambient-write", node)) return metadata; + const ambientRoots = createAmbientPathResolver(state, false)(root); + for (const ambientRoot of ambientRoots) { + const selectedPath = [...ambientRoot, ...path]; + for (const [kind, effects] of [ + ["member-write", state.ambientWriteEffects], + ["escape", state.ambientEscapeEffects], + ] as const) + for (const effect of effects) + if ( + semanticPathsOverlap(selectedPath, effect.path) && + !retain(kind, effect.node, effect.path.slice(ambientRoot.length)) + ) + return metadata; + } } return metadata; }; diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 066fcd64c4..c3b1fe0616 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -1,4 +1,8 @@ import * as t from "@babel/types"; +import { + createAmbientPathResolver, + retainSemanticAmbientEffect, +} from "./javascriptSemanticAmbientWrites.js"; import { isIntrinsicMapLookup } from "./javascriptSemanticMapOperations.js"; import { isIntrinsicTypedArrayConstruction, @@ -522,6 +526,7 @@ export function* collectSemanticMemberMutationsSteps( iterableReferenceKeys.set(reference.node, keys); iterableReferences.push(reference); }; + const ambientPaths = createAmbientPathResolver(state, true); const markValue = ( node: t.Node, path: PropertyPath, @@ -577,6 +582,18 @@ export function* collectSemanticMemberMutationsSteps( if (t.isIdentifier(expression)) { const binding = resolveBinding(expression, expression.name); if (binding === undefined) { + for (const path of ambientPaths( + expression, + current.originAt, + effect === "escape", + current.path.map((key) => + key === null || typeof key === "object" ? null : String(key), + ), + )) + if (retainSemanticAmbientEffect(state, effect, path, mutation)) { + recordedEffectCount += 1; + clearSemanticValueCaches(state); + } const processOrigin = semanticGlobalProcessOrigin(expression, state); if (processOrigin !== undefined) recordRequiredEffect( @@ -1478,6 +1495,7 @@ const observableMutationPaths = ( prefix: PropertyPath, ) => boolean; } => { + const ambientPaths = createAmbientPathResolver(state, false); const bindingNodes = new Map(); const expressions = new WeakMap(); const pendingExpressions: { @@ -1617,6 +1635,8 @@ const observableMutationPaths = ( ["globalThis", "global", "window", "self"].includes(expression.name) ) (observer.origins ??= new Set()).add("ambient"); + else if (ambientPaths(expression).length > 0) + (observer.origins ??= new Set()).add("opaque"); } else if ( t.isMemberExpression(expression) || t.isOptionalMemberExpression(expression) @@ -1686,11 +1706,12 @@ const observableMutationPaths = ( add("opaque"); } else if (origin === "array-prototype" || origin === "opaque") add(origin); - else if ( - origin === "ambient" && - semanticPropertyPathKeyMatches(key, "Array") - ) - add("array"); + else if (origin === "ambient") + add( + semanticPropertyPathKeyMatches(key, "Array") + ? "array" + : "opaque", + ); else if ( origin === "array" && semanticPropertyPathKeyMatches(key, "prototype") diff --git a/src/domain/javascript/javascriptSemanticRequireOrigin.ts b/src/domain/javascript/javascriptSemanticRequireOrigin.ts index 6b6d93824b..afc8bfb41f 100644 --- a/src/domain/javascript/javascriptSemanticRequireOrigin.ts +++ b/src/domain/javascript/javascriptSemanticRequireOrigin.ts @@ -654,6 +654,7 @@ const loaderInvocationInterop = (state: JavaScriptSemanticAnalysisState) => { }; export interface JavaScriptRequireCallFact { + readonly operation: "load" | "resolve"; readonly loader: JavaScriptRequireLoaderFact; readonly arguments: | readonly t.CallExpression["arguments"][number][] @@ -670,7 +671,12 @@ export const semanticRequireCallAt = ( factoryBindingIds: ReadonlySet, ): JavaScriptRequireCallFact | undefined => { if (semanticModuleRequireCallAt(node, state)) - return { loader: "unknown", arguments: undefined, sourceRelative: false }; + return { + operation: "load", + loader: "unknown", + arguments: undefined, + sourceRelative: false, + }; const callee = unwrapJavaScriptExpression(node.callee).node; if ( (t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee)) && @@ -680,7 +686,12 @@ export const semanticRequireCallAt = ( selectedRequireSourcesAt(callee, node, state, factoryBindingIds)?.loader === "local" ) - return { loader: "local", arguments: undefined, sourceRelative: false }; + return { + operation: "load", + loader: "local", + arguments: undefined, + sourceRelative: false, + }; if ( (t.isMemberExpression(callee) || t.isOptionalMemberExpression(callee)) && semanticStaticPropertyKey(callee.property, callee.computed) === "resolve" @@ -691,6 +702,13 @@ export const semanticRequireCallAt = ( state, factoryBindingIds, ); + if (loader === "unknown") + return { + operation: "load", + loader, + arguments: undefined, + sourceRelative: false, + }; // Bundler factory loaders can share the `require` shape, but their // `.resolve` member is not Node's module resolver. if (loader === "commonjs-loader" || loader === "create-require") { @@ -701,6 +719,7 @@ export const semanticRequireCallAt = ( factoryBindingIds, ); return { + operation: "resolve", loader: selected?.loader ?? loader, arguments: node.arguments, sourceRelative: @@ -726,6 +745,7 @@ export const semanticRequireCallAt = ( ); if (selected !== undefined || loader !== "local") return { + operation: "load", loader: selected?.loader ?? loader, arguments: node.arguments, sourceRelative: selected?.sourceRelative ?? loader !== "create-require", @@ -767,6 +787,7 @@ export const semanticRequireCallAt = ( factoryBindingIds, ); return { + operation: "load", loader: invocation.path[1] === "local" ? "local" @@ -790,6 +811,7 @@ export const semanticRequireCallAt = ( return factory === undefined ? undefined : { + operation: "load", loader: factory, arguments: node.arguments.slice(t.isIdentifier(node.callee) ? 0 : 1), sourceRelative: true, diff --git a/src/domain/javascript/javascriptStaticAnalysisCalls.ts b/src/domain/javascript/javascriptStaticAnalysisCalls.ts index 5bfa12e247..1fc7b56304 100644 --- a/src/domain/javascript/javascriptStaticAnalysisCalls.ts +++ b/src/domain/javascript/javascriptStaticAnalysisCalls.ts @@ -91,6 +91,7 @@ export const inspectCall = ( node, kind: "require", specifier: requireCall.specifier, + operation: requireCall.operation, ...(requireCall.resolutionBasePath === undefined ? {} : { resolutionBasePath: requireCall.resolutionBasePath }), @@ -270,6 +271,7 @@ const requireCallSpecifier = ( resolveMemberChanged = false, ): { readonly specifier: string | undefined; + readonly operation: "load" | "resolve"; readonly resolutionBasePath?: string; } | null => { const call = semanticRequireCallAt( @@ -284,9 +286,13 @@ const requireCallSpecifier = ( !call.sourceRelative ) { context.accumulator.unknownFindings += 1; - return { specifier: undefined }; + return { + specifier: undefined, + operation: resolveMemberChanged ? "load" : call.operation, + }; } return { + operation: call.operation, specifier: argumentValue(call.arguments?.[0]), ...(call.sourceRelativePath === undefined ? {} @@ -322,6 +328,7 @@ export const finishModuleRequireCalls = ( node, kind: "require", specifier: call.specifier, + operation: call.operation, ...(call.resolutionBasePath === undefined ? {} : { resolutionBasePath: call.resolutionBasePath }), @@ -947,10 +954,11 @@ export const addReference = ( : { specifier, expression: null }; addLocatedFinding(context, { collection: context.accumulator.references, - key: `reference\0${input.kind}\0${reference.specifier ?? reference.expression}\0${input.resolutionBasePath ?? ""}`, + key: `reference\0${input.kind}\0${input.operation ?? "load"}\0${reference.specifier ?? reference.expression}\0${input.resolutionBasePath ?? ""}`, node: input.node, value: { kind: input.kind, + operation: input.operation ?? "load", ...reference, ...(input.resolutionBasePath === undefined ? {} diff --git a/src/domain/javascript/javascriptStaticAnalysisLoaderInvocations.test.ts b/src/domain/javascript/javascriptStaticAnalysisLoaderInvocations.test.ts index e65a02a6a7..a5d1aafebc 100644 --- a/src/domain/javascript/javascriptStaticAnalysisLoaderInvocations.test.ts +++ b/src/domain/javascript/javascriptStaticAnalysisLoaderInvocations.test.ts @@ -157,7 +157,13 @@ it("retains a mutated require.resolve request as an unknown fact", () => { analyzeJavaScriptStaticSource(source).references.filter( ({ kind }) => kind === "require", ), - ).toEqual([expect.objectContaining({ kind: "require", specifier: null })]); + ).toEqual([ + expect.objectContaining({ + kind: "require", + operation: "load", + specifier: null, + }), + ]); }); it("does not treat bundler factory resolve members as Node resolvers", () => { diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts index dc42744ef0..9a8b0c7c81 100644 --- a/src/domain/javascript/javascriptStaticAnalysisState.ts +++ b/src/domain/javascript/javascriptStaticAnalysisState.ts @@ -110,6 +110,7 @@ export interface JavaScriptFindingContext { /** Candidate import, require, worker, or service-worker reference. */ export interface JavaScriptReferenceInput { + readonly operation?: JavaScriptStaticReference["operation"]; readonly node: t.Node; readonly kind: JavaScriptStaticReference["kind"]; readonly specifier: string | undefined; diff --git a/src/domain/javascript/javascriptStaticAnalysisTypes.ts b/src/domain/javascript/javascriptStaticAnalysisTypes.ts index a2adfb811a..843670a138 100644 --- a/src/domain/javascript/javascriptStaticAnalysisTypes.ts +++ b/src/domain/javascript/javascriptStaticAnalysisTypes.ts @@ -22,6 +22,8 @@ export interface JavaScriptSourceRange { } interface JavaScriptStaticReferenceState { + /** Resolution discovers a target without loading or executing it. */ + readonly operation: "load" | "resolve"; readonly kind: | "static-import" | "dynamic-import" diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts index 2532c3c71d..6d60d651a4 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -296,6 +296,7 @@ const electron = z })); const referenceShape = { ...findingContext, + operation: z.enum(["load", "resolve"]), resolution_base_path: z.string().exactOptional(), kind: z.enum([ "static-import", diff --git a/tests/acceptance/applications/javascriptIndirectLoaderOwners.test.ts b/tests/acceptance/applications/javascriptIndirectLoaderOwners.test.ts index 7725003d7f..886c79d0c7 100644 --- a/tests/acceptance/applications/javascriptIndirectLoaderOwners.test.ts +++ b/tests/acceptance/applications/javascriptIndirectLoaderOwners.test.ts @@ -40,9 +40,112 @@ const cases = [ }, { name: "require.resolve does not execute its resolved module", + resolverSpecifier: "../shared/window.cjs", + resolverTarget: "shared/window.cjs", load: 'require.resolve("../shared/window.cjs");', possible: false, }, + { + name: "resolution discovery does not suppress a subsequent executable load", + resolverSpecifier: "../shared/window.cjs", + resolverTarget: "shared/window.cjs", + load: 'require.resolve("../shared/window.cjs");require("../shared/window.cjs");', + possible: true, + }, + { + name: "sibling wrapper property writes preserve resolver identity", + resolverSpecifier: "../shared/window.cjs", + resolverTarget: "shared/window.cjs", + load: 'require.unrelated=custom;require.resolve("../shared/window.cjs");', + possible: false, + }, + { + name: "later resolver writes do not change an earlier lookup", + resolverSpecifier: "../shared/window.cjs", + resolverTarget: "shared/window.cjs", + load: 'require.resolve("../shared/window.cjs");require.resolve=custom;', + possible: false, + }, + { + name: "conditional wrapper resolver writes retain possible execution", + uncertainResolver: true, + load: 'if(flag)require.resolve=custom;require.resolve("../shared/window.cjs");', + possible: true, + }, + { + name: "escaped wrapper resolver retains possible execution", + uncertainResolver: true, + load: 'consume(require);require.resolve("../shared/window.cjs");', + possible: true, + }, + { + name: "invoked getter retains ambient resolver mutation", + uncertainResolver: true, + load: 'const box={loader:require,get trigger(){this.loader.resolve=this.loader;return 0}};box.trigger;require.resolve("../shared/window.cjs");', + possible: true, + }, + { + name: "invoked getter retains qualified ambient resolver mutation", + uncertainResolver: true, + load: 'const box={loader:globalThis.require,get trigger(){this.loader.resolve=this.loader;return 0}};box.trigger;require.resolve("../shared/window.cjs");', + possible: true, + }, + { + name: "unused getter preserves ambient resolver identity", + resolverSpecifier: "../shared/window.cjs", + resolverTarget: "shared/window.cjs", + load: 'const box={loader:require,get trigger(){this.loader.resolve=this.loader;return 0}};require.resolve("../shared/window.cjs");', + possible: false, + }, + { + name: "later getter read preserves earlier ambient resolution", + resolverSpecifier: "../shared/window.cjs", + resolverTarget: "shared/window.cjs", + load: 'const box={loader:require,get trigger(){this.loader.resolve=this.loader;return 0}};require.resolve("../shared/window.cjs");box.trigger;', + possible: false, + }, + { + name: "invoked getter retains lexical created resolver mutation", + uncertainResolver: true, + mainFile: "main.mjs", + load: 'import {createRequire} from "node:module";const loader=createRequire(import.meta.url);const box={loader,get trigger(){this.loader.resolve=this.loader;return 0}};box.trigger;loader.resolve("../shared/window.cjs");', + possible: true, + }, + { + name: "replaced resolver retains possible executable ownership", + uncertainResolver: true, + load: 'require.resolve=require;require.resolve("../shared/window.cjs");', + possible: true, + }, + { + name: "created resolver retains discovery without module execution", + resolverSpecifier: "../shared/window.cjs", + resolverTarget: "shared/window.cjs", + mainFile: "main.mjs", + load: 'import {createRequire} from "node:module";const load=createRequire(import.meta.url);load.resolve("../shared/window.cjs");', + possible: false, + }, + { + name: "missing resolver target does not create executable ownership gaps", + missingTargetOperation: "resolve", + resolverSpecifier: "../missing/window.cjs", + resolverTarget: null, + load: 'require.resolve("../missing/window.cjs");', + possible: false, + }, + { + name: "dynamic executable target retains application ownership uncertainty", + unresolvedExecutable: true, + load: "require(runtimeTarget);", + possible: true, + }, + { + name: "modified created resolver retains possible executable ownership", + uncertainResolver: true, + mainFile: "main.mjs", + load: 'import {createRequire} from "node:module";const load=createRequire(import.meta.url);load.resolve=load;load.resolve("../shared/window.cjs");', + possible: true, + }, { name: "require.main is not an implicit module loader", load: 'require.main("../shared/window.cjs");', @@ -256,10 +359,87 @@ cliTest.for( root, ); expect(statistics.parse_failures).toBe(0); + if ("resolverSpecifier" in testCase) { + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + kind: "require", + operation: "resolve", + specifier: testCase.resolverSpecifier, + resolution_mode: "require", + resolved_path: testCase.resolverTarget, + }), + }), + ]), + ); + } + + if ("uncertainResolver" in testCase) + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + kind: "require", + operation: "load", + specifier: null, + }), + evidence: expect.objectContaining({ + coverage: expect.objectContaining({ status: "partial" }), + }), + }), + ]), + ); const roles = graph.nodes.filter( ({ kind }) => kind === "electron-renderer", ); expect(roles).toHaveLength(1); + if ( + "missingTargetOperation" in testCase || + "unresolvedExecutable" in testCase + ) { + const baseLimitation = + "Inventoried package metadata does not establish one application root for this source; the loadFile base is unknown."; + const executable = "unresolvedExecutable" in testCase; + expect(graph.edges).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + operation: executable ? "load" : "resolve", + specifier: executable ? null : "../missing/window.cjs", + expression: executable ? "require(runtimeTarget)" : null, + resolved_path: null, + resolution_status: executable ? null : "not-found", + }), + evidence: expect.objectContaining({ + coverage: expect.objectContaining({ status: "complete" }), + limitations: expect.arrayContaining([ + "Static syntax does not prove that this relationship executes at runtime.", + ]), + }), + }), + ]), + ); + expect(roles[0]?.observations).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + limitations: executable ? [baseLimitation] : [], + }), + evidence: expect.objectContaining({ + coverage: expect.objectContaining({ status: "complete" }), + limitations: executable + ? expect.arrayContaining([baseLimitation]) + : expect.not.arrayContaining([baseLimitation]), + }), + }), + ]), + ); + } + expect(roles[0]?.observations).toEqual( expect.arrayContaining([ expect.objectContaining({ diff --git a/tests/acceptance/applications/javascriptLiteralLoaderOwnership.ts b/tests/acceptance/applications/javascriptLiteralLoaderOwnership.ts index a672fe7092..5908002b12 100644 --- a/tests/acceptance/applications/javascriptLiteralLoaderOwnership.ts +++ b/tests/acceptance/applications/javascriptLiteralLoaderOwnership.ts @@ -319,6 +319,7 @@ const expectCompetingLoaderGraph = ( "local module", "projected module", "resolve-only", + "createRequire base", "missing wildcard alias", "earlier reassigned module alias", ].includes(loader); @@ -359,9 +360,21 @@ const expectCompetingLoaderGraph = ( relation: "imports", properties: expect.objectContaining({ kind: "require", - specifier: null, + operation: "load", + specifier: "../shared/window.cjs", + resolution_base_path: "../other/nested/main.cjs", + resolution_status: "not-found", resolved_path: null, }), + evidence: expect.objectContaining({ + location: expect.objectContaining({ + available: true, + value: expect.objectContaining({ + kind: "source-range", + source: "b/main.mjs", + }), + }), + }), }), ]), ); From 34563fcc9bd92af30668820dcac5635e27181094 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:05:22 +0800 Subject: [PATCH 8/8] fix(process): own bounded worker control framing Share incremental replacement UTF-8 decoding across web and JavaScript workers, bound content before complete-line allocation and handle split CRLF consistently. Keep named schemas, request versus checkpoint failures, worker reuse and stream cleanup authority in their servers. Cover real server recovery and deterministic byte boundaries. --- .../WebBundleAnalysisWorkerServer.test.ts | 146 ++++++++++- src/browser/WebBundleAnalysisWorkerServer.ts | 48 +--- .../JavaScriptAnalysisWorkerServer.test.ts | 226 ++++++++++++++++++ .../JavaScriptAnalysisWorkerServer.ts | 27 ++- src/process/BoundedControlLines.test.ts | 197 +++++++++++++++ src/process/BoundedControlLines.ts | 87 +++++++ 6 files changed, 666 insertions(+), 65 deletions(-) create mode 100644 src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts create mode 100644 src/process/BoundedControlLines.test.ts create mode 100644 src/process/BoundedControlLines.ts diff --git a/src/browser/WebBundleAnalysisWorkerServer.test.ts b/src/browser/WebBundleAnalysisWorkerServer.test.ts index c85be1fa3c..8f3db07fd0 100644 --- a/src/browser/WebBundleAnalysisWorkerServer.test.ts +++ b/src/browser/WebBundleAnalysisWorkerServer.test.ts @@ -6,7 +6,7 @@ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createInterface } from "node:readline"; -import { expect, it } from "vitest"; +import { expect, it, onTestFinished } from "vitest"; import { createWebTextArtifact } from "../domain/webContentArtifact.js"; import type { IncludedScript } from "../domain/webBundleAnalyzerState.js"; @@ -110,11 +110,10 @@ const readStream = async (stream: NodeJS.ReadableStream): Promise => { return output; }; -const send = async ( +const writeInput = async ( worker: ChildProcessWithoutNullStreams, - request: unknown, + frame: string | Buffer, ): Promise => { - const frame = `${JSON.stringify(request)}\n`; await new Promise((resolve, reject) => { worker.stdin.write(frame, (error) => error === null || error === undefined ? resolve() : reject(error), @@ -122,6 +121,137 @@ const send = async ( }); }; +const send = async ( + worker: ChildProcessWithoutNullStreams, + request: unknown, +): Promise => writeInput(worker, `${JSON.stringify(request)}\n`); + +const closeServer = async ( + worker: ChildProcessWithoutNullStreams, +): Promise => { + worker.stdin.end(); + if (worker.exitCode === null && worker.signalCode === null) { + const exited = once(worker, "exit"); + const timeout = setTimeout(() => worker.kill("SIGKILL"), 2_000); + await exited; + clearTimeout(timeout); + } +}; + +const framingServer = async () => { + const root = await mkdtemp(join(tmpdir(), "rea-web-worker-framing-")); + let worker: ChildProcessWithoutNullStreams | undefined; + onTestFinished(async () => { + if (worker !== undefined) await closeServer(worker); + await rm(root, { recursive: true, force: true }); + }); + const server = await startServer(root); + worker = server.process; + expect(await server.next()).toMatchObject({ kind: "ready" }); + return { root, ...server }; +}; + +it("serves fragmented and coalesced LF/CRLF requests as separate jobs", async () => { + const server = await framingServer(); + const input = sourceDescriptor(makeScript("fetch('/framed');")); + const ids = [randomUUID(), randomUUID()]; + const requests = ids.map((id) => ({ + kind: "analyze", + id, + source: input.descriptor, + result_budget_bytes: WEB_BUNDLE_WORKER_LIMITS.resultBytes, + })); + for (const id of ids) await writeSource(server.root, id, input.bytes); + const first = JSON.stringify(requests[0]); + await writeInput(server.process, first.slice(0, 17)); + await new Promise((resolve) => setTimeout(resolve, 50)); + await writeInput( + server.process, + `${first.slice(17)}\n${JSON.stringify(requests[1])}\r\n`, + ); + for (const id of ids) { + expect(await server.next()).toMatchObject({ kind: "phase", id }); + expect(await server.next()).toMatchObject({ kind: "complete", id }); + expect( + JSON.parse(await readFile(join(server.root, `${id}.result`), "utf8")), + ).toMatchObject({ status: "analyzed" }); + } +}); + +it.each([ + ["multibyte", Buffer.from("é🙂")], + ["replacement-decoded", Buffer.from([0xff])], +] as const)( + "accepts an exact-budget %s JSON request with fragmented UTF-8 and split CRLF", + async (_name, text) => { + const server = await framingServer(); + const input = sourceDescriptor(makeScript("fetch('/exact-budget');")); + const id = randomUUID(); + await writeSource(server.root, id, input.bytes); + const request = JSON.stringify({ + kind: "analyze", + id, + source: input.descriptor, + result_budget_bytes: WEB_BUNDLE_WORKER_LIMITS.resultBytes, + }); + // JSON.parse retains the final known key, so non-ASCII input exercises the + // decoder while the request admitted by the named schema stays unchanged. + const prefix = Buffer.from('{"kind":"'); + const json = Buffer.concat([ + prefix, + text, + Buffer.from(`",${request.slice(1)}`), + ]); + const decodedBytes = Buffer.byteLength(json.toString("utf8")); + const frame = Buffer.concat([ + json, + Buffer.alloc(WEB_BUNDLE_WORKER_LIMITS.protocolBytes - decodedBytes, 0x20), + ]); + expect(Buffer.byteLength(frame.toString("utf8"))).toBe( + WEB_BUNDLE_WORKER_LIMITS.protocolBytes, + ); + await writeInput(server.process, frame.subarray(0, prefix.length + 1)); + await new Promise((resolve) => setTimeout(resolve, 50)); + await writeInput( + server.process, + Buffer.concat([frame.subarray(prefix.length + 1), Buffer.from("\r")]), + ); + // Let the worker consume the terminal CR before delivering its LF. + await new Promise((resolve) => setTimeout(resolve, 50)); + await writeInput(server.process, "\n"); + expect(await server.next()).toMatchObject({ kind: "phase", id }); + expect(await server.next()).toMatchObject({ kind: "complete", id }); + expect( + JSON.parse(await readFile(join(server.root, `${id}.result`), "utf8")), + ).toMatchObject({ status: "analyzed" }); + }, +); + +it("rejects an oversized unterminated frame at EOF without starting analysis", async () => { + const server = await framingServer(); + const input = sourceDescriptor(makeScript("fetch('/not-analyzed');")); + const id = randomUUID(); + await writeSource(server.root, id, input.bytes); + const request = JSON.stringify({ + kind: "analyze", + id, + source: input.descriptor, + result_budget_bytes: WEB_BUNDLE_WORKER_LIMITS.resultBytes, + }); + const frame = request.padEnd(WEB_BUNDLE_WORKER_LIMITS.protocolBytes + 1, " "); + const exited = once(server.process, "exit"); + server.process.stdin.end(frame); + await expect(server.next()).rejects.toThrow( + "RangeError: Web bundle worker request exceeds its byte budget", + ); + expect(await exited).toEqual([1, null]); + await expect( + readFile(join(server.root, `${id}.result`)), + ).rejects.toMatchObject({ + code: "ENOENT", + }); +}); + it("bounds and binds per-script worker jobs, then reuses one worker for the next script", async () => { const root = await mkdtemp(join(tmpdir(), "rea-web-worker-server-test-")); let worker: ChildProcessWithoutNullStreams | undefined; @@ -256,13 +386,7 @@ it("bounds and binds per-script worker jobs, then reuses one worker for the next expect(selected.route_facts.candidates.length).toBeLessThan(3); } finally { if (worker !== undefined) { - worker.stdin.end(); - if (worker.exitCode === null && worker.signalCode === null) { - const exited = once(worker, "exit"); - const timeout = setTimeout(() => worker?.kill("SIGKILL"), 2_000); - await exited; - clearTimeout(timeout); - } + await closeServer(worker); } await rm(root, { recursive: true, force: true }); } diff --git a/src/browser/WebBundleAnalysisWorkerServer.ts b/src/browser/WebBundleAnalysisWorkerServer.ts index a71560b385..e2259e7ff6 100644 --- a/src/browser/WebBundleAnalysisWorkerServer.ts +++ b/src/browser/WebBundleAnalysisWorkerServer.ts @@ -4,8 +4,8 @@ import type { JsonValue } from "../domain/jsonValue.js"; import { createHash } from "node:crypto"; import { open } from "node:fs/promises"; import { join } from "node:path"; -import { StringDecoder } from "node:string_decoder"; import { getHeapStatistics } from "node:v8"; +import { readBoundedControlLines } from "../process/BoundedControlLines.js"; import { withArtifactFileHandle } from "../artifacts/ArtifactFileOperation.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; @@ -46,50 +46,16 @@ const send = async (input: WebBundleWorkerMessage): Promise => { }); }; -async function* boundedControlLines(): AsyncGenerator { - const decoder = new StringDecoder("utf8"); - let pending = ""; - for await (const chunk of process.stdin) { - const decoded = decoder.write(chunk); - let start = 0; - for (;;) { - const newline = decoded.indexOf("\n", start); - if (newline < 0) { - pending += decoded.slice(start); - if (Buffer.byteLength(pending) > LIMITS.protocolBytes) - throw new RangeError( - "Web bundle worker request exceeds its byte budget", - ); - break; - } - const line = `${pending}${decoded.slice(start, newline)}`.replace( - /\r$/u, - "", - ); - pending = ""; - if (Buffer.byteLength(line) > LIMITS.protocolBytes) - throw new RangeError( - "Web bundle worker request exceeds its byte budget", - ); - yield line; - start = newline + 1; - } - } - pending += decoder.end(); - if (pending !== "") { - if (Buffer.byteLength(pending) > LIMITS.protocolBytes) - throw new RangeError("Web bundle worker request exceeds its byte budget"); - yield pending; - } -} - /** Serve sequential script jobs from the private root owned by the worker runtime. */ export const serveWebBundleAnalysisWorker = async ( analyze: WebBundleScriptAnalyzer, ): Promise => { const root = process.cwd(); const resources = new ArtifactResourceScope(); - const requests = boundedControlLines(); + const requests = readBoundedControlLines( + process.stdin.iterator({ destroyOnReturn: true }), + LIMITS.protocolBytes, + ); await send({ kind: "ready", heap_limit_bytes: @@ -101,11 +67,11 @@ export const serveWebBundleAnalysisWorker = async ( for (;;) { const next = await requests.next(); if (next.done) break; - if (Buffer.byteLength(next.value) > LIMITS.protocolBytes) + if (!next.value.ok) throw new RangeError( "Web bundle worker request exceeds its byte budget", ); - const raw: unknown = JSON.parse(next.value); + const raw: unknown = JSON.parse(next.value.value); const request = webBundleWorkerRequestSchema.parse(raw); const continueServing = await analyzeJob( root, diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts new file mode 100644 index 0000000000..7834a47c2a --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.test.ts @@ -0,0 +1,226 @@ +import { spawn } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import { expect, it } from "vitest"; +import { + JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS, + javaScriptWorkerAdmissionSchema, + javaScriptWorkerMessageSchema, + javaScriptWorkerRequestSchema, +} from "./JavaScriptAnalysisWorkerProtocol.js"; + +const source = "export const observed = 1;"; +const digest = createHash("sha256").update(source).digest("hex"); +const frame = (value: unknown): string => `${JSON.stringify(value)}\n`; +const admission = (id: string) => + javaScriptWorkerAdmissionSchema.parse({ + id, + semantic_result_budget_bytes: 0, + }); + +const startServer = async () => { + const root = await mkdtemp(join(tmpdir(), "rea-js-server-framing-")); + const serverUrl = new URL( + "./JavaScriptAnalysisWorkerServer.ts", + import.meta.url, + ).href; + const analyzerUrl = new URL( + "../../domain/javascript/javascriptStaticAnalysis.ts", + import.meta.url, + ).href; + const child = spawn( + process.execPath, + [ + "--import", + import.meta.resolve("tsx"), + "--input-type=module", + "-e", + `const { serveJavaScriptAnalysisWorker } = await import(${JSON.stringify(serverUrl)}); + const { analyzeJavaScriptStaticSource } = await import(${JSON.stringify(analyzerUrl)}); + await serveJavaScriptAnalysisWorker(async (file, capacity, callbacks) => { + await callbacks.checkpoint(analyzeJavaScriptStaticSource(file.text.value)); + return null; + });`, + ], + { cwd: root, stdio: ["pipe", "pipe", "pipe"] }, + ); + const closed = once(child, "close"); + let stderr = ""; + child.stderr.setEncoding("utf8").on("data", (chunk: string) => { + stderr += chunk; + }); + const lines = createInterface({ input: child.stdout, crlfDelay: Infinity }); + const messages = lines[Symbol.asyncIterator](); + return { + child, + closed, + stderr: () => stderr, + next: async () => { + const next = await messages.next(); + if (next.done) throw new Error(`Worker output ended: ${stderr}`); + return javaScriptWorkerMessageSchema.parse(JSON.parse(next.value)); + }, + request: async () => { + const id = randomUUID(); + await writeFile(join(root, `${id}.source`), source, { flag: "wx" }); + return javaScriptWorkerRequestSchema.parse({ + id, + unknown_budget: 100, + node_budget: 100, + text_sha256: digest, + file: { + path: "app.mjs", + sha256: digest, + container_sha256: digest, + bytes: Buffer.byteLength(source), + inventory_artifact_id: "server-framing-source", + kind: "javascript", + unpacked: false, + }, + }); + }, + close: async () => { + if (child.exitCode === null && child.signalCode === null) child.kill(); + await closed; + lines.close(); + await rm(root, { recursive: true, force: true }); + }, + }; +}; + +it("consumes fragmented CRLF admission and coalesced next-job frames in order", async () => { + const server = await startServer(); + try { + expect(await server.next()).toMatchObject({ kind: "ready" }); + const first = await server.request(); + const second = await server.request(); + server.child.stdin.write(frame(first)); + expect(await server.next()).toMatchObject({ kind: "static", id: first.id }); + const encoded = JSON.stringify(admission(first.id)); + server.child.stdin.write(encoded.slice(0, 12)); + server.child.stdin.write(`${encoded.slice(12)}\r`); + server.child.stdin.write( + `\n${frame(second)}${frame(admission(second.id))}`, + ); + expect(await server.next()).toMatchObject({ + kind: "phase", + id: first.id, + phase: "transfer", + }); + expect(await server.next()).toEqual({ + kind: "complete", + id: first.id, + descriptor: null, + }); + expect(await server.next()).toMatchObject({ + kind: "static", + id: second.id, + }); + expect(await server.next()).toMatchObject({ + kind: "phase", + id: second.id, + phase: "transfer", + }); + expect(await server.next()).toEqual({ + kind: "complete", + id: second.id, + descriptor: null, + }); + server.child.stdin.end(); + expect((await server.closed)[0]).toBe(0); + } finally { + await server.close(); + } +}); + +it("drains oversized checkpoint admission, reports output failure, and serves the next job", async () => { + const server = await startServer(); + try { + await server.next(); + const first = await server.request(); + const second = await server.request(); + server.child.stdin.write(frame(first)); + expect(await server.next()).toMatchObject({ kind: "static", id: first.id }); + server.child.stdin.write( + "x".repeat(JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS.protocolBytes), + ); + server.child.stdin.write( + `x\n${frame(second)}${frame(admission(second.id))}`, + ); + expect(await server.next()).toEqual({ + kind: "failure", + id: first.id, + reason: "output", + resource: null, + limits: null, + message: "JavaScript worker admission frame is missing or oversized", + }); + expect(await server.next()).toMatchObject({ + kind: "static", + id: second.id, + }); + expect(await server.next()).toMatchObject({ + kind: "phase", + id: second.id, + phase: "transfer", + }); + expect(await server.next()).toEqual({ + kind: "complete", + id: second.id, + descriptor: null, + }); + server.child.stdin.end(); + expect((await server.closed)[0]).toBe(0); + } finally { + await server.close(); + } +}); + +it("reports missing checkpoint admission as output failure at stdin EOF", async () => { + const server = await startServer(); + try { + await server.next(); + const request = await server.request(); + server.child.stdin.write(frame(request)); + expect(await server.next()).toMatchObject({ + kind: "static", + id: request.id, + }); + server.child.stdin.end(); + expect(await server.next()).toMatchObject({ + kind: "failure", + id: request.id, + reason: "output", + resource: null, + message: "JavaScript worker admission frame is missing or oversized", + }); + expect((await server.closed)[0]).toBe(0); + } finally { + await server.close(); + } +}); + +it.each(["lf", "eof", "open"] as const)( + "rejects oversized initial requests with %s input", + async (boundary) => { + const server = await startServer(); + try { + await server.next(); + const oversized = "x".repeat( + JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS.protocolBytes + 1, + ); + if (boundary === "open") server.child.stdin.write(oversized); + else server.child.stdin.end(oversized + (boundary === "lf" ? "\n" : "")); + expect((await server.closed)[0]).not.toBe(0); + expect(server.stderr()).toContain( + "RangeError: JavaScript worker request exceeds its byte budget", + ); + } finally { + await server.close(); + } + }, +); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts index bed8508da2..0e4f03f90b 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts @@ -5,8 +5,11 @@ import type { JavaScriptSemanticProjectionCapacity } from "../../domain/javascri import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { createInterface } from "node:readline"; import { getHeapStatistics } from "node:v8"; +import { + readBoundedControlLines, + type BoundedControlLine, +} from "../../process/BoundedControlLines.js"; import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; import { javaScriptRuntime } from "../../domain/runtimeVersion.js"; import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; @@ -63,8 +66,10 @@ export const serveJavaScriptAnalysisWorker = async ( analyze: JavaScriptSourceWorkerAnalyzer, ): Promise => { const root = process.cwd(); - const input = createInterface({ input: process.stdin, crlfDelay: Infinity }); - const requests = input[Symbol.asyncIterator](); + const requests = readBoundedControlLines( + process.stdin.iterator({ destroyOnReturn: false }), + DEFAULTS.protocolBytes, + ); await send({ kind: "ready", heap_limit_bytes: @@ -76,12 +81,11 @@ export const serveJavaScriptAnalysisWorker = async ( for (;;) { const next = await requests.next(); if (next.done) break; - const line = next.value; - if (Buffer.byteLength(line) > DEFAULTS.protocolBytes) + if (!next.value.ok) throw new RangeError( "JavaScript worker request exceeds its byte budget", ); - const raw: unknown = JSON.parse(line); + const raw: unknown = JSON.parse(next.value.value); const request = javaScriptWorkerRequestSchema.parse(raw); // Function scope releases source AST/IR/results before accepting another job. await analyzeJob(root, request, analyze, requests); @@ -92,7 +96,7 @@ export const serveJavaScriptAnalysisWorker = async ( global.gc?.(); } } finally { - input.close(); + await requests.return(undefined); } }; @@ -100,7 +104,7 @@ const analyzeJob = async ( root: string, request: ReturnType, analyze: JavaScriptSourceWorkerAnalyzer, - requests: AsyncIterator, + requests: AsyncIterator, ): Promise => { try { let resultBudgetBytes = 0; @@ -166,14 +170,11 @@ const analyzeJob = async ( sha256: createHash("sha256").update(text).digest("hex"), }); const next = await requests.next(); - if ( - next.done || - Buffer.byteLength(next.value) > DEFAULTS.protocolBytes - ) + if (next.done || !next.value.ok) throw new TypeError( "JavaScript worker admission frame is missing or oversized", ); - const raw: unknown = JSON.parse(next.value); + const raw: unknown = JSON.parse(next.value.value); const admission = javaScriptWorkerAdmissionSchema.parse(raw); if (admission.id !== request.id) throw new TypeError( diff --git a/src/process/BoundedControlLines.test.ts b/src/process/BoundedControlLines.test.ts new file mode 100644 index 0000000000..5949562d8a --- /dev/null +++ b/src/process/BoundedControlLines.test.ts @@ -0,0 +1,197 @@ +import { PassThrough } from "node:stream"; +import { describe, expect, it } from "vitest"; + +import { err, ok } from "../domain/result.js"; +import { + readBoundedControlLines, + type BoundedControlLine, +} from "./BoundedControlLines.js"; + +async function* chunks( + parts: readonly Uint8Array[], +): AsyncGenerator { + yield* parts; +} + +const collect = async ( + parts: readonly Uint8Array[], + maximum: number, +): Promise => { + const lines: BoundedControlLine[] = []; + for await (const line of readBoundedControlLines(chunks(parts), maximum)) + lines.push(line); + return lines; +}; + +const exceeded: BoundedControlLine = err({ _tag: "ControlLineLimitExceeded" }); +const multibyte = Buffer.from("é😀\n", "utf8"); +const exactCrLf = Buffer.from("éa\r\n", "utf8"); +const crLfPartitions: { first: number; second: number }[] = []; +for (let first = 0; first <= exactCrLf.length; first += 1) + for (let second = first; second <= exactCrLf.length; second += 1) + crLfPartitions.push({ first, second }); + +describe("bounded control line decoding", () => { + it.each(Array.from({ length: multibyte.length + 1 }, (_, split) => split))( + "preserves split UTF-8 content at byte %i", + async (split) => { + expect( + await collect( + [multibyte.subarray(0, split), multibyte.subarray(split)], + 6, + ), + ).toEqual([ok("é😀")]); + }, + ); + + it.each(crLfPartitions)( + "excludes the CRLF delimiter at the exact limit across cuts $first/$second", + async ({ first, second }) => { + expect( + await collect( + [ + exactCrLf.subarray(0, first), + exactCrLf.subarray(first, second), + exactCrLf.subarray(second), + ], + 3, + ), + ).toEqual([ok("éa")]); + }, + ); + + it.each([ + { bytes: [0xff, 0x0a], maximum: 3, expected: [ok("�")] }, + { bytes: [0xff, 0x0a], maximum: 2, expected: [exceeded] }, + { bytes: [0xe2, 0x82], maximum: 3, expected: [ok("�")] }, + { bytes: [0xe2, 0x82], maximum: 2, expected: [exceeded] }, + { bytes: [0xe2, 0x28, 0xa1, 0x0a], maximum: 7, expected: [ok("�(�")] }, + { bytes: [0xe2, 0x28, 0xa1, 0x0a], maximum: 6, expected: [exceeded] }, + ])( + "counts replacement characters against decoded bytes: $bytes / $maximum", + async ({ bytes, maximum, expected }) => { + const encoded = Buffer.from(bytes); + expect(await collect([encoded], maximum)).toEqual(expected); + expect( + await collect( + bytes.map((byte) => Uint8Array.of(byte)), + maximum, + ), + ).toEqual(expected); + }, + ); +}); + +describe("bounded control line boundaries", () => { + it.each([ + { text: "", maximum: 3, expected: [] }, + { text: "\n\r\n", maximum: 0, expected: [ok(""), ok("")] }, + { text: " \t \n", maximum: 3, expected: [ok(" \t ")] }, + { text: "a\rb\n", maximum: 3, expected: [ok("a\rb")] }, + { text: "abc", maximum: 3, expected: [ok("abc")] }, + { text: "ab\r", maximum: 3, expected: [ok("ab\r")] }, + { text: "abc\r", maximum: 3, expected: [exceeded] }, + { text: "\r", maximum: 0, expected: [exceeded] }, + { + text: "a\n\nb\r\nc", + maximum: 1, + expected: [ok("a"), ok(""), ok("b"), ok("c")], + }, + ])( + "retains content and EOF semantics: $text", + async ({ text, maximum, expected }) => { + expect(await collect([Buffer.from(text)], maximum)).toEqual(expected); + }, + ); + + it.each(["\nnext\n", "\r\nnext\n", ""])( + "reports oversized drips immediately and drains before the next frame: %j", + async (tail) => { + let consumed = 0; + async function* drips(): AsyncGenerator { + for (const byte of Buffer.from(`overflow${tail}`)) { + consumed += 1; + yield Uint8Array.of(byte); + } + } + const lines = readBoundedControlLines(drips(), 4); + expect(await lines.next()).toEqual({ done: false, value: exceeded }); + expect(consumed).toBe(5); + if (tail !== "") + expect(await lines.next()).toEqual({ done: false, value: ok("next") }); + expect(await lines.next()).toEqual({ done: true, value: undefined }); + expect(consumed).toBe(Buffer.byteLength(`overflow${tail}`)); + }, + ); + + it("recovers from malformed-byte expansion before the following frame", async () => { + expect( + await collect([Uint8Array.of(0xff, 0x0a, 0x6f, 0x6b, 0x0a)], 2), + ).toEqual([exceeded, ok("ok")]); + }); +}); + +describe("bounded control line stream ownership", () => { + it("reports overflow before an open stream supplies a delimiter and then recovers", async () => { + const source = new PassThrough(); + const lines = readBoundedControlLines(source, 4); + try { + source.write("oversized"); + expect(await lines.next()).toEqual({ done: false, value: exceeded }); + expect(source.writableEnded).toBe(false); + expect(source.readableEnded).toBe(false); + expect(source.destroyed).toBe(false); + + source.write(" remaining rejected content\nnext\n"); + expect(await lines.next()).toEqual({ done: false, value: ok("next") }); + source.end(); + expect(await lines.next()).toEqual({ done: true, value: undefined }); + } finally { + source.destroy(); + await lines.return(undefined); + } + }); + + it("propagates the original stream failure after completed lines", async () => { + const source = new PassThrough(); + const lines = readBoundedControlLines(source, 10); + try { + source.write("first\n"); + expect(await lines.next()).toEqual({ done: false, value: ok("first") }); + const failure = new Error("control input failed"); + const next = lines.next(); + source.destroy(failure); + await expect(next).rejects.toBe(failure); + } finally { + source.destroy(); + await lines.return(undefined); + } + }); + + it.each([true, false])( + "leaves early-return destruction to the supplied iterator (%s)", + async (destroyOnReturn) => { + const source = new PassThrough(); + const lines = readBoundedControlLines( + source.iterator({ destroyOnReturn }), + 10, + ); + try { + source.write("first\n"); + expect(await lines.next()).toEqual({ done: false, value: ok("first") }); + await lines.return(undefined); + expect(source.destroyed).toBe(destroyOnReturn); + if (!destroyOnReturn) { + source.end("next\n"); + const remaining: BoundedControlLine[] = []; + for await (const line of readBoundedControlLines(source, 10)) + remaining.push(line); + expect(remaining).toEqual([ok("next")]); + } + } finally { + source.destroy(); + await lines.return(undefined); + } + }, + ); +}); diff --git a/src/process/BoundedControlLines.ts b/src/process/BoundedControlLines.ts new file mode 100644 index 0000000000..376450dbd4 --- /dev/null +++ b/src/process/BoundedControlLines.ts @@ -0,0 +1,87 @@ +import { StringDecoder } from "node:string_decoder"; +import { err, ok, type Result } from "../domain/result.js"; + +export type BoundedControlLine = Result< + string, + { readonly _tag: "ControlLineLimitExceeded" } +>; + +/** + * Read LF/CRLF control frames without retaining an oversized line. Limits apply + * to decoded UTF-8 content, excluding the delimiter. Oversize is reported once + * as soon as it is known; rejected content is drained before the next frame. + * The caller owns failure classification and whether to continue reading. + * The supplied iterator owns the input stream's early-return cleanup policy. + */ +export async function* readBoundedControlLines( + chunks: AsyncIterable, + maxBytes: number, +): AsyncGenerator { + const decoder = new StringDecoder("utf8"); + let pending = ""; + let pendingBytes = 0; + let oversized = false; + + const append = (fragment: string): boolean => { + if (oversized || fragment.length === 0) return false; + const bytes = pendingBytes + Buffer.byteLength(fragment); + // A final CR might be half of a split CRLF. Admit it provisionally, then + // count it as content if another character or EOF follows instead of LF. + if (bytes > maxBytes + (fragment.endsWith("\r") ? 1 : 0)) { + oversized = true; + pending = ""; + pendingBytes = 0; + return true; + } + pending += fragment; + pendingBytes = bytes; + return false; + }; + + const finish = (delimiter: "lf" | "eof"): BoundedControlLine => { + const line = + delimiter === "lf" && pending.endsWith("\r") + ? pending.slice(0, -1) + : pending; + const result = + oversized || Buffer.byteLength(line) > maxBytes + ? err({ _tag: "ControlLineLimitExceeded" } as const) + : ok(line); + pending = ""; + pendingBytes = 0; + oversized = false; + return result; + }; + + function* consume(decoded: string): Generator { + let start = 0; + for (;;) { + const newline = decoded.indexOf("\n", start); + if (newline < 0) { + if (append(decoded.slice(start))) + yield err({ _tag: "ControlLineLimitExceeded" }); + return; + } + if (append(decoded.slice(start, newline))) + yield err({ _tag: "ControlLineLimitExceeded" }); + if (!oversized) yield finish("lf"); + else { + pending = ""; + pendingBytes = 0; + oversized = false; + } + start = newline + 1; + } + } + + for await (const chunk of chunks) { + // Bound decoding expansion as well as retained line content. A malformed + // source byte can expand to a three-byte UTF-8 replacement character. + for (let offset = 0; offset < chunk.byteLength; offset += maxBytes + 1) + yield* consume( + decoder.write(chunk.subarray(offset, offset + maxBytes + 1)), + ); + } + yield* consume(decoder.end()); + if (!oversized && pending.length > 0) yield finish("eof"); +}