diff --git a/docs/design/limits-and-recovery.md b/docs/design/limits-and-recovery.md index 653a4c2571..ee318b1e71 100644 --- a/docs/design/limits-and-recovery.md +++ b/docs/design/limits-and-recovery.md @@ -176,21 +176,22 @@ checks all happen before publishing the replacement snapshot; a rejected batch cannot partially apply. Clients coordinating writes can supply `expected_revision`; without it, valid updates apply to the current state. -| Runtime setting | Application point | -| --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | -| `asar.max_header_bytes` | Next archive operation, including full JavaScript application analysis | -| `asar.max_listing_bytes` | Next expanded ASAR listing, independently of packed header bytes | -| `ghidra.startup_timeout_ms`, `hopper.startup_timeout_ms` | Next provider startup attempt | -| `ghidra.decompile_timeout_seconds` | Next operation on the existing Ghidra Program; `null` means unbounded | -| `ghidra.decompile_max_payload_mib` | Next native decompile; `null` preserves the selected provider's payload default | -| `ghidra.value_flow_max_operations`, `ghidra.value_flow_max_inputs_per_operation`, `ghidra.value_flow_max_edges` | Next function value-flow projection on the existing Program | -| `jadx.operation_timeout_ms` | Next JADX operation | -| `regex.matching_deadline_ms`, `regex.worker_heap_request_mib` | Next search / new regex worker | -| `capture.decoder_old_generation_mib` | Next owned capture decoder | -| `javascript.analysis_timeout_ms`, `javascript.max_heap_mb` | Next source analysis / new JavaScript worker | -| `javascript.member_mutation_alias_states` | Next JavaScript application, web bundle or reference-source analysis | -| `wasm.command_timeout_ms` | Next WABT operation | -| `native.matching_representation_bytes` | Next `match_native_functions` request | +| Runtime setting | Application point | +| --------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | +| `asar.max_header_bytes` | Next archive operation, including full JavaScript application analysis | +| `asar.max_listing_bytes` | Next expanded ASAR listing, independently of packed header bytes | +| `ghidra.startup_timeout_ms`, `hopper.startup_timeout_ms` | Next provider startup attempt | +| `ghidra.decompile_timeout_seconds` | Next operation on the existing Ghidra Program; `null` means unbounded | +| `ghidra.decompile_max_payload_mib` | Next native decompile; `null` preserves the selected provider's payload default | +| `ghidra.value_flow_max_operations`, `ghidra.value_flow_max_inputs_per_operation`, `ghidra.value_flow_max_edges` | Next function value-flow projection on the existing Program | +| `jadx.operation_timeout_ms` | Next JADX operation | +| `regex.matching_deadline_ms`, `regex.worker_heap_request_mib` | Next search / new regex worker | +| `capture.decoder_old_generation_mib` | Next owned capture decoder | +| `javascript.analysis_timeout_ms`, `javascript.max_heap_mb` | Next source analysis / new JavaScript worker | +| `javascript.member_mutation_alias_states` | Next JavaScript application, web bundle or reference-source analysis | +| `wasm.command_timeout_ms` | Next WABT operation | +| `native.matching_representation_bytes` | Next `match_native_functions` request | +| `zstd.max_frames` | Next exact Zstandard decode for an ELF section, OTA `REPLACE_ZSTD` operation, or GPU payload | ### Precedence and lifetime diff --git a/docs/design/rea-limit-inventory.md b/docs/design/rea-limit-inventory.md index 34366447dc..abdd29b080 100644 --- a/docs/design/rea-limit-inventory.md +++ b/docs/design/rea-limit-inventory.md @@ -48,10 +48,10 @@ settings. They are not on the restart-only list. | Category | Count | | --------------------------------------------------------------------- | ---------------------------------------------------------------------------: | -| Runtime analysis settings | 28 (25 have an `REA_` seed; 3 are per-call parameters only) | +| Runtime analysis settings | 29 (26 have an `REA_` seed; 3 are per-call parameters only) | | Per-call numeric `REA_` limits still read from the launch environment | 0 | | Restart-only numeric `REA_` limits | 15 | -| Numeric `REA_` candidates recognized in TypeScript source | 38 | +| Numeric `REA_` candidates recognized in TypeScript source | 39 | | Numeric `REA_` controls with external or generated-reader provenance | 2 | | Non-numeric `REA_` settings kept restart-only | 52 caller settings, plus process-protocol names that are not caller settings | | Hardcoded ceilings left for follow-up | grouped below; none converted in the inventory change | @@ -96,22 +96,23 @@ floor, and maximum. The accepted operation snapshot supplies the value. force. An update outside the range is rejected. A hit names the setting and reports the current value, source, and range. A rejected Windows ChangeTime launch value omits `pending_restart`. -| Environment | Owner | Default | Floor | Maximum | Applies | Fields | -| --------------------------------------------- | ------------------------------------------ | ----------------------: | ------: | -----------: | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | -| `REA_BRIDGE_MAX_REQUEST_BYTES` | `src/config/bridgeMaxRequestBytes.ts` | 16777216 | 7464 | 536870887 | each bridge request, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_BRIDGE_MAX_REPLY_BYTES` | `src/config/bridgeMaxReplyBytes.ts` | 268435456 | 1048576 | 536870888 | each Hopper or Ghidra request, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_CAPTURE_PACKET_JSON_EXPANSION_BYTES` | `src/config/capturePacketJsonExpansion.ts` | 137438953472 | 1048576 | 137438953472 | each packet-capture reply before parsing, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_DEBUGGER_COMMAND_DEADLINE_MS` | `src/domain/analysisResourceLimits.ts` | 120000 | 1 | 2147483647 | each command, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES` | `src/domain/analysisResourceLimits.ts` | 16777216 | 1 | 89303718 | each command, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_HOPPER_CALLER_SCAN_BUDGET_MS` | `src/domain/analysisResourceLimits.ts` | 2000 | 1 | 2147483647 | each caller scan, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_JAVASCRIPT_MEMBER_MUTATION_ALIAS_STATES` | `src/domain/analysisResourceLimits.ts` | 524288 | 1 | 8388608 | each JavaScript source's member-mutation phase, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_JEB_MCP_TIMEOUT_MS` | `src/jeb/JebMcpConnection.ts` | 60000 (MCP SDK default) | 1 | 2147483647 | discovery and each tools/call, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_KUNA_PROJECT_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 1800000 | 1 | 2147483647 | each project export and whole-program scan, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_KUNA_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 120000 | 1 | 2147483647 | each function decompile, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_PWNTOOLS_MAX_INPUT_BYTES` | `src/config/pwntoolsInputLimit.ts` | 33554432 | 1 | 1073741824 | each pwntools decode, unless the call passes `producer_limits.input_bytes`; otherwise the operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_WINDOWS_CHANGE_TIME_DRIFT_MS` | `src/config/windowsChangeTimeSettle.ts` | 2000 | 1 | 60000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | -| `REA_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS` | `src/config/windowsChangeTimeSettle.ts` | 25 | 1 | 1000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | -| `REA_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS` | `src/config/windowsChangeTimeSettle.ts` | 40 | 1 | 1000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | +| Environment | Owner | Default | Floor | Maximum | Applies | Fields | +| --------------------------------------------- | --------------------------------------------------- | ----------------------: | ------: | -----------: | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | +| `REA_BRIDGE_MAX_REQUEST_BYTES` | `src/config/bridgeMaxRequestBytes.ts` | 16777216 | 7464 | 536870887 | each bridge request, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_BRIDGE_MAX_REPLY_BYTES` | `src/config/bridgeMaxReplyBytes.ts` | 268435456 | 1048576 | 536870888 | each Hopper or Ghidra request, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_CAPTURE_PACKET_JSON_EXPANSION_BYTES` | `src/config/capturePacketJsonExpansion.ts` | 137438953472 | 1048576 | 137438953472 | each packet-capture reply before parsing, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_DEBUGGER_COMMAND_DEADLINE_MS` | `src/domain/analysisResourceLimits.ts` | 120000 | 1 | 2147483647 | each command, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES` | `src/domain/analysisResourceLimits.ts` | 16777216 | 1 | 89303718 | each command, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_HOPPER_CALLER_SCAN_BUDGET_MS` | `src/domain/analysisResourceLimits.ts` | 2000 | 1 | 2147483647 | each caller scan, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_JAVASCRIPT_MEMBER_MUTATION_ALIAS_STATES` | `src/domain/analysisResourceLimits.ts` | 524288 | 1 | 8388608 | each JavaScript source's member-mutation phase, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_JEB_MCP_TIMEOUT_MS` | `src/jeb/JebMcpConnection.ts` | 60000 (MCP SDK default) | 1 | 2147483647 | discovery and each tools/call, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_KUNA_PROJECT_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 1800000 | 1 | 2147483647 | each project export and whole-program scan, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_KUNA_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 120000 | 1 | 2147483647 | each function decompile, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_PWNTOOLS_MAX_INPUT_BYTES` | `src/config/pwntoolsInputLimit.ts` | 33554432 | 1 | 1073741824 | each pwntools decode, unless the call passes `producer_limits.input_bytes`; otherwise the operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_WINDOWS_CHANGE_TIME_DRIFT_MS` | `src/config/windowsChangeTimeSettle.ts` | 2000 | 1 | 60000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | +| `REA_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS` | `src/config/windowsChangeTimeSettle.ts` | 25 | 1 | 1000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | +| `REA_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS` | `src/config/windowsChangeTimeSettle.ts` | 40 | 1 | 1000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | +| `REA_ZSTD_MAX_FRAMES` | `src/domain/compression/zstandardFrameAllowance.ts` | 4096 | 1 | 65536 | each exact Zstandard buffer, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | ## Restart-only numeric limits diff --git a/scripts/verify/advertised-schemas/tools/inspect_analysis_settings.json b/scripts/verify/advertised-schemas/tools/inspect_analysis_settings.json index fc17490efc..f79d092fc6 100644 --- a/scripts/verify/advertised-schemas/tools/inspect_analysis_settings.json +++ b/scripts/verify/advertised-schemas/tools/inspect_analysis_settings.json @@ -1,6 +1,6 @@ { "schemas": { - "input compact": "5ea37049dad60cdc", + "input compact": "71360cba238d0c73", "output compact": "b3a429110120aff1" }, "patterns": {} diff --git a/scripts/verify/advertised-schemas/tools/update_analysis_settings.json b/scripts/verify/advertised-schemas/tools/update_analysis_settings.json index d1a23c1a60..637957d379 100644 --- a/scripts/verify/advertised-schemas/tools/update_analysis_settings.json +++ b/scripts/verify/advertised-schemas/tools/update_analysis_settings.json @@ -1,6 +1,6 @@ { "schemas": { - "input compact": "c6bdd8000e43ec47", + "input compact": "4f5355f35275f28b", "output compact": "39446aa1012006cc" }, "patterns": {} diff --git a/src/android/AndroidJniLibraries.ts b/src/android/AndroidJniLibraries.ts index d3de29cf3f..79be283ea1 100644 --- a/src/android/AndroidJniLibraries.ts +++ b/src/android/AndroidJniLibraries.ts @@ -2,6 +2,7 @@ import { ArtifactReaderFailure, type ArtifactResourceScope, streamChunkToBuffer, + yieldArtifactRead, ZipArtifactReader, } from "../resources/index.js"; import type { @@ -11,6 +12,7 @@ import type { AndroidJniLibraryCheckpoint, } from "../domain/android/androidJniBindings.js"; import { compileAndroidJniLibrarySelection } from "../domain/android/androidJniBindings.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { inspectAndroidJniLibrary } from "../domain/android/androidJniLibrary.js"; @@ -18,7 +20,6 @@ import { JadxInterruption, throwIfJadxInterrupted, } from "./JadxInterruption.js"; -import { AnalysisError } from "../domain/analysisErrorBase.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { analysisClientFailureForOperation } from "../providers/AnalysisClientFailure.js"; @@ -43,6 +44,7 @@ export interface AndroidJniLibraries { const jniReadFailure = ( cause: unknown, ): NonNullable => { + if (cause instanceof AnalysisError) throw cause; const message = errorMessage(cause); if (!(cause instanceof ArtifactReaderFailure)) return { kind: "io", message }; // The library report has only a message, so a reader-named ceiling's @@ -209,6 +211,7 @@ export const readAndroidJniLibraries = async (input: { }; const parsed = await inspectAndroidJniLibrary(bytes, { signal: input.signal, + pause: () => yieldArtifactRead(input.signal), }); throwIfJadxInterrupted(input.signal); observation = { diff --git a/src/application/AnalysisSettings.test.ts b/src/application/AnalysisSettings.test.ts index 05889dbdd0..80ec9138bf 100644 --- a/src/application/AnalysisSettings.test.ts +++ b/src/application/AnalysisSettings.test.ts @@ -156,6 +156,53 @@ it("keeps accepted operations and their typed provider views stable across concu }); }); +it("lists zstd.max_frames for every operation that enforces the cap", () => { + const store = new AnalysisSettings({}); + for (const operation of [ + "inspect_artifact", + "extract_artifact", + "inspect_gpu_code_objects", + "inspect_dwarf_units", + "inspect_elf_notes", + "inspect_binary_hardening", + "inspect_elf_unwind_ranges", + "inspect_native_exports", + "inspect_native_initializers", + "inspect_kernel_module", + "inspect_elf_debug_link", + "inspect_go_binary", + "inspect_rust_binary", + "inspect_il2cpp_metadata", + "inspect_android_jni_bindings", + ]) + expect( + store.inspect({ operation }).settings.map(({ setting }) => setting), + ).toContain("zstd.max_frames"); + for (const provider of [ + "gpu-code-objects", + "rea-dwarf-units", + "rea-elf-notes", + "rea-binary-hardening", + "rea-elf-unwind-ranges", + "rea-native-exports", + "rea-native-initializers", + "rea-kernel-module", + "rea-elf-debug-link", + "go-build-info", + "rust-auditable", + "il2cpp-metadata", + "jadx", + ]) + expect( + store.inspect({ provider }).settings.map(({ setting }) => setting), + ).toContain("zstd.max_frames"); + expect( + store + .inspect({ operation: "inspect_binary_layout" }) + .settings.map(({ setting }) => setting), + ).not.toContain("zstd.max_frames"); +}); + it("exposes shared provider controls and preserves the explicit unbounded decompile value", () => { const environment = { REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "7", diff --git a/src/artifacts/AndroidOtaPayloadArtifactReader.boundaries.test.ts b/src/artifacts/AndroidOtaPayloadArtifactReader.boundaries.test.ts index 33b086c594..d4a8ba754e 100644 --- a/src/artifacts/AndroidOtaPayloadArtifactReader.boundaries.test.ts +++ b/src/artifacts/AndroidOtaPayloadArtifactReader.boundaries.test.ts @@ -1,7 +1,9 @@ import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js"; +import { createHash } from "node:crypto"; import { truncate, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { buffer } from "node:stream/consumers"; +import { scheduler } from "node:timers/promises"; import { expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; @@ -164,6 +166,57 @@ it("refuses to open a partition whose distinct operation data exceeds the read b } }); +it("cancels a many-frame REPLACE_ZSTD partition", async () => { + const frame = Buffer.from([ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, 0x61, + ]); + const compressed = Buffer.alloc(frame.length * 32); + for (let index = 0; index < 32; index += 1) + frame.copy(compressed, index * frame.length); + const path = await fixture( + buildAndroidOtaPayload({ + partitions: [ + { + name: "vendor", + operations: [ + { + type: 14, + data: compressed, + dataHash: createHash("sha256").update(compressed).digest(), + dstExtents: [{ startBlock: 0, numBlocks: 1 }], + }, + ], + }, + ], + }), + ); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = new AndroidOtaPayloadArtifactReader(path, admitted); + const reason = new Error("stop ota frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + try { + const [entry] = await entries(reader); + await expect(reader.open(entry!, controller.signal)).rejects.toMatchObject({ + reason: "cancelled", + cause: reason, + }); + } finally { + spy.mockRestore(); + try { + await reader.close(); + } finally { + await admitted.handle.close(); + } + } +}); + it("preserves cancellation while reconstructing a partition", async () => { const path = await fixture( buildAndroidOtaPayload({ diff --git a/src/artifacts/AndroidOtaPayloadArtifactReader.ts b/src/artifacts/AndroidOtaPayloadArtifactReader.ts index 8824a2974d..ad8110b4c2 100644 --- a/src/artifacts/AndroidOtaPayloadArtifactReader.ts +++ b/src/artifacts/AndroidOtaPayloadArtifactReader.ts @@ -9,6 +9,7 @@ import type { import { readArtifactFileBytes } from "./ArtifactFileRange.js"; import { assertArtifactReadActive, + yieldArtifactRead, ArtifactEntryContentUnavailable, ArtifactReaderFailure, copyArtifactEntry, @@ -21,6 +22,8 @@ import { } from "../resources/index.js"; import { ownArtifactStream } from "./OwnedArtifactStream.js"; import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; +import { ZstandardFrameAllowanceError } from "../domain/compression/zstandardFrameAllowance.js"; +import { ConfigurationError } from "../domain/configurationErrors.js"; import { ANDROID_OTA_HEADER_V2_BYTES, androidOtaDistinctOperationDataBytes, @@ -28,7 +31,7 @@ import { androidOtaExtractRefusalLimitation, parseAndroidOtaHeader, parseAndroidOtaPayload, - reconstructAndroidOtaPartition, + reconstructAndroidOtaPartitionCooperative, withAndroidOtaPayloadSignature, type AndroidOtaPartition, type AndroidOtaPayload, @@ -143,11 +146,12 @@ export class AndroidOtaPayloadArtifactReader implements ArtifactReader { } await this.#source.read(signal); this.#assertOpen(signal); - const reconstructed = reconstructAndroidOtaPartition( + const reconstructed = await reconstructAndroidOtaPartitionCooperative( stored.partition, payload.blockSize, (offset, length) => blob.get(`${String(offset)}:${String(length)}`) ?? new Uint8Array(), + () => yieldArtifactRead(signal), // A SHA-256 claim on the entry is checked by the shared scanner under // the caller's integrity policy, so record-and-continue can keep the // observed bytes. Without that claim the reader checks it here. @@ -347,6 +351,12 @@ const reconstructFailure = ( cause: failure, integrityCheck: "declared-digest", }); + if (failure instanceof ConfigurationError) throw failure; + if (failure._tag === "ZstandardFrameWorkRefusal") + throw new ZstandardFrameAllowanceError( + { maxFrames: failure.value, source: failure.source }, + failure.offset, + ); return parseFailure(failure); }; diff --git a/src/artifacts/ZipArtifactReader.test.ts b/src/artifacts/ZipArtifactReader.test.ts index f5c6890c6c..1c06946a8e 100644 --- a/src/artifacts/ZipArtifactReader.test.ts +++ b/src/artifacts/ZipArtifactReader.test.ts @@ -238,31 +238,81 @@ describe("ZIP Zstandard method 93", () => { it.each([ [ - "frameParameter_unsupported", + "reserved_frame_descriptor", "28b52ffddf339d0000686d656d626572206279746573200100e1995f", ], [ - "corruption_detected", + "ZSTD_error_corruption_detected", "28b52ffd20cc9d0000686d656d626572206279746573200100e1995f", ], [ - "dictionary_corrupted", + "ZSTD_error_dictionary_corrupted", "28b52ffd20339d0000976d656d626572206279746573200100e1995f", ], + ] as const)("keeps the %s format fault and its cause", async (code, hex) => { + const content = Buffer.from( + "member bytes member bytes member bytes member bytes", + ); + await expect( + readFirstZstandardMember( + zstandardZipFixture(content, { compressed: Buffer.from(hex, "hex") }), + ), + ).rejects.toMatchObject({ + reason: "format", + message: expect.stringContaining("Invalid compressed data"), + cause: { code }, + }); + }); +}); + +describe("ZIP Zstandard frame completion", () => { + const first = Buffer.from("first text frame\n"); + const second = Buffer.from("second text frame: café\n"); + const firstFrame = zstdCompressSync(first); + const secondFrame = zstdCompressSync(second, { + params: { [constants.ZSTD_c_checksumFlag]: 1 }, + }); + // The skippable payload spans chunks and the next magic straddles a + // zip.js input chunk boundary (64 KiB). + const skipped = Buffer.alloc(1024 * 1024 - firstFrame.length - 2); + skipped.writeUInt32LE(0x184d2a5f, 0); + skipped.writeUInt32LE(skipped.length - 8, 4); + const content = Buffer.concat([first, second]); + + it.each([ + ["two ordinary", [firstFrame, secondFrame]], + ["ordinary and skippable", [firstFrame, skipped, secondFrame]], + ] as const)("streams complete %s frames in order", async (_label, frames) => { + const compressed = Buffer.concat(frames); + const decoded = await readFirstZstandardMember( + zstandardZipFixture(content, { compressed }), + ); + expect(decoded.equals(content)).toBe(true); + }); + + it.each([ + ["a partial frame magic", first, secondFrame.subarray(0, 3)], + ["an incomplete ordinary frame", first, secondFrame.subarray(0, 5)], + [ + "an incomplete skippable frame", + first, + skipped.subarray(0, skipped.length - 1), + ], + [ + "an incomplete checksum", + content, + secondFrame.subarray(0, secondFrame.length - 1), + ], ] as const)( - "keeps the native %s format fault and its cause", - async (code, hex) => { - const content = Buffer.from( - "member bytes member bytes member bytes member bytes", - ); + "refuses %s after a complete frame", + async (_label, output, suffix) => { + const compressed = Buffer.concat([firstFrame, suffix]); await expect( - readFirstZstandardMember( - zstandardZipFixture(content, { compressed: Buffer.from(hex, "hex") }), - ), + readFirstZstandardMember(zstandardZipFixture(output, { compressed })), ).rejects.toMatchObject({ + name: "ArtifactReaderFailure", reason: "format", message: expect.stringContaining("Invalid compressed data"), - cause: { code: `ZSTD_error_${code}` }, }); }, ); diff --git a/src/config/analysisSettings.ts b/src/config/analysisSettings.ts index 49a65f1d94..e9092996b4 100644 --- a/src/config/analysisSettings.ts +++ b/src/config/analysisSettings.ts @@ -34,6 +34,7 @@ import { windowsChangeTimeSettlePollMsSchema, } from "./windowsChangeTimeSettle.js"; import { pwntoolsInputBytesSchema } from "./pwntoolsInputLimit.js"; +import { zstandardMaxFramesSchema } from "../domain/compression/zstandardFrameAllowance.js"; const environmentSchemas: Readonly>> = { REA_ASAR_MAX_HEADER_BYTES: asarMaxHeaderBytesSchema, @@ -67,6 +68,7 @@ const environmentSchemas: Readonly>> = REA_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS: windowsChangeTimeSettlePollMsSchema, REA_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS: windowsChangeTimeSettleMaxPollsSchema, + REA_ZSTD_MAX_FRAMES: zstandardMaxFramesSchema, }; export const launchAnalysisSetting = ( id: AnalysisSettingId, diff --git a/src/domain/analysisResourceLimits.ts b/src/domain/analysisResourceLimits.ts index 5eac06b621..139db98f32 100644 --- a/src/domain/analysisResourceLimits.ts +++ b/src/domain/analysisResourceLimits.ts @@ -276,3 +276,14 @@ export const DEFAULT_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS = 25; export const MAXIMUM_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS = 1_000; export const DEFAULT_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS = 40; export const MAXIMUM_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS = 1_000; + +/** + * Zstandard frames one buffer may contain before decoding is refused. + * Legitimate streams are one frame, sometimes with a few skippable frames. + * 65,536 one-byte frames is the measured Node 22 ceiling: a few hundred + * milliseconds of synchronous decoder work, and far below the millions of + * frames the existing byte allowances would still admit. + */ +export const DEFAULT_ZSTD_MAX_FRAMES = 4_096; +export const MINIMUM_ZSTD_MAX_FRAMES = 1; +export const MAXIMUM_ZSTD_MAX_FRAMES = 65_536; diff --git a/src/domain/analysisSettingDefinitions.ts b/src/domain/analysisSettingDefinitions.ts index 75efc5d202..d4ee650c59 100644 --- a/src/domain/analysisSettingDefinitions.ts +++ b/src/domain/analysisSettingDefinitions.ts @@ -60,6 +60,9 @@ import { MAXIMUM_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS, DEFAULT_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS, MAXIMUM_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS, + DEFAULT_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, + MAXIMUM_ZSTD_MAX_FRAMES, } from "./analysisResourceLimits.js"; import { JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS, @@ -604,6 +607,48 @@ export const ANALYSIS_SETTINGS = { relatedProviders: ["jadx"], operations: windowsChangeTimeOperations, }, + "zstd.max_frames": { + updateMode: "runtime", + environment: "REA_ZSTD_MAX_FRAMES", + defaultValue: DEFAULT_ZSTD_MAX_FRAMES, + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + appliesAt: "next_operation", + provider: "rea-artifact-graph", + relatedProviders: [ + "gpu-code-objects", + "rea-dwarf-units", + "rea-elf-notes", + "rea-binary-hardening", + "rea-elf-unwind-ranges", + "rea-native-exports", + "rea-native-initializers", + "rea-kernel-module", + "rea-elf-debug-link", + "go-build-info", + "rust-auditable", + "il2cpp-metadata", + "jadx", + ], + operations: [ + "inspect_artifact", + "extract_artifact", + "inspect_gpu_code_objects", + "inspect_dwarf_units", + "inspect_elf_notes", + "inspect_binary_hardening", + "inspect_elf_unwind_ranges", + "inspect_native_exports", + "inspect_native_initializers", + "inspect_kernel_module", + "inspect_elf_debug_link", + "inspect_go_binary", + "inspect_rust_binary", + "inspect_il2cpp_metadata", + "inspect_android_jni_bindings", + ], + }, } as const satisfies Record; export const analysisSettingIds = Object.keys( ANALYSIS_SETTINGS, diff --git a/src/domain/analysisSettings.ts b/src/domain/analysisSettings.ts index 59904cc58e..dc47bbf6b0 100644 --- a/src/domain/analysisSettings.ts +++ b/src/domain/analysisSettings.ts @@ -31,6 +31,7 @@ export interface AnalysisSettingValues { readonly "windows.change_time_drift_ms": number; readonly "windows.change_time_settle_poll_ms": number; readonly "windows.change_time_settle_max_polls": number; + readonly "zstd.max_frames": number; } export type AnalysisSettingId = keyof AnalysisSettingValues; export interface AnalysisSettingsSnapshot { diff --git a/src/domain/android/androidJniLibrary.ts b/src/domain/android/androidJniLibrary.ts index 619569c0e3..d067c07eb3 100644 --- a/src/domain/android/androidJniLibrary.ts +++ b/src/domain/android/androidJniLibrary.ts @@ -7,9 +7,13 @@ import type { AndroidJniTableEntry, AndroidJniTableSlot, } from "./androidJniLibraryFacts.js"; -import { parseNativeExportsSteps } from "../native/nativeExportParser.js"; -import { readElfDirectoryFromBuffer } from "../native/nativeSectionDirectory.js"; -import { readElfImage } from "../native/elfImage.js"; +import { parseElfExportsFromDirectorySteps } from "../native/nativeExportParser.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; +import { + readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, +} from "../native/nativeSectionDirectory.js"; +import { readElfImageFromDirectory } from "../native/elfImage.js"; import { BinaryFormatFailure, BinaryMappingFailure, @@ -23,12 +27,14 @@ import { type ElfPointerOverlay, } from "../native/elfPointerRelocations.js"; import { sha256Hex } from "../../kernel/sha256Hex.js"; +import { zstandardFrameAllowanceCause } from "../compression/zstandardFrameAllowance.js"; const MAXIMUM_STRING_BYTES = 65_535; const MAXIMUM_STRING_WORK = 64 * 1024 * 1024; const MAXIMUM_RETAINED_BYTES = 16 * 1024 * 1024; const MAXIMUM_SCAN_BYTES = 64 * 1024 * 1024; const MAXIMUM_TABLE_ENTRIES = 65_536; +const OPERATION = "inspect_android_jni_bindings"; const hex = (value: bigint): string => `0x${value.toString(16)}`; /** Strict Modified UTF-8: no four-byte UTF-8, overlong forms or replacements. */ @@ -90,11 +96,18 @@ const methodDescriptor = (descriptor: string): boolean => { /** Parse borrowed immutable ELF bytes; no filesystem, process or target execution. */ export const inspectAndroidJniLibrary = async ( bytes: Buffer, - options: { readonly signal?: AbortSignal } = {}, + options: { + readonly signal?: AbortSignal; + readonly pause?: ZstandardFramePause; + } = {}, ): Promise> => { const active = (): void => { if (options.signal?.aborted === true) throw options.signal; }; + const pause = async (): Promise => { + await immediate(); + active(); + }; try { active(); if (!bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) @@ -102,8 +115,16 @@ export const inspectAndroidJniLibrary = async ( kind: "unsupported", message: "JNI library inspection requires an ELF image", }); - const directory = readElfDirectoryFromBuffer(bytes); - const image = readElfImage(bytes); + const directory = + options.pause === undefined && options.signal === undefined + ? readElfDirectoryFromBuffer(bytes, OPERATION) + : await readElfDirectoryFromBufferCooperative( + bytes, + options.pause ?? pause, + OPERATION, + ); + active(); + const image = readElfImageFromDirectory(bytes, directory); const mappings = createFileMappingIndex(image.mappings, image.zero_fills); if (image.executable_mappings === undefined) return err({ @@ -126,23 +147,22 @@ export const inspectAndroidJniLibrary = async ( "JNI library facts exceed the 16 MiB retained representation allowance", ); }; - const exportSteps = parseNativeExportsSteps(bytes, { - path: "[admitted immutable ELF bytes]", - sha256: sha256Hex(bytes), - bytes: bytes.length, - }); + const exportSteps = parseElfExportsFromDirectorySteps( + bytes, + { + path: "[admitted immutable ELF bytes]", + sha256: sha256Hex(bytes), + bytes: bytes.length, + }, + directory, + ); + await pause(); let exportStep = exportSteps.next(); while (!exportStep.done) { - await immediate(); - active(); + await pause(); exportStep = exportSteps.next(); } const rawExports = exportStep.value; - if (rawExports.format !== "elf") - return err({ - kind: "unsupported", - message: "JNI library inspection requires ELF exports", - }); const exportRows: AndroidJniLibraryFacts["exports"] = []; let exportsStatus: AndroidJniLibraryFacts["coverage"]["exports_status"] = rawExports.coverage.status === "complete" @@ -499,6 +519,9 @@ export const inspectAndroidJniLibrary = async ( kind: "cancelled", message: "JNI library inspection was cancelled", }); + const frameAllowance = zstandardFrameAllowanceCause(cause); + if (frameAllowance !== undefined) + return err({ kind: "capacity", message: frameAllowance.message }); if ( cause instanceof BinaryResourceFailure || (cause instanceof ElfPointerRelocationFailure && cause.kind === "limit") diff --git a/src/domain/compression/zstandardExactFrame.test.ts b/src/domain/compression/zstandardExactFrame.test.ts new file mode 100644 index 0000000000..0953909ec1 --- /dev/null +++ b/src/domain/compression/zstandardExactFrame.test.ts @@ -0,0 +1,314 @@ +import { constants, zstdCompressSync } from "node:zlib"; +import { describe, expect, it } from "vitest"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, + DEFAULT_ZSTD_MAX_FRAMES, +} from "../analysisResourceLimits.js"; +import { ConfigurationError } from "../configurationErrors.js"; +import { + decodeZstandardFrames, + decodeZstandardFramesCooperative, + ownedZstandardBytes, +} from "./zstandardExactFrame.js"; +import { + resolveZstandardFrameAllowance, + ZSTD_MAX_FRAMES_ENVIRONMENT, + ZSTD_MAX_FRAMES_SETTING, +} from "./zstandardFrameAllowance.js"; + +const rawFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +const repeat = (frame: Buffer, count: number): Buffer => { + const bytes = Buffer.alloc(frame.length * count); + for (let index = 0; index < count; index += 1) + frame.copy(bytes, index * frame.length); + return bytes; +}; + +const skippable = (payload: Buffer, magic = 0x184d2a50): Buffer => { + const bytes = Buffer.alloc(8 + payload.length); + bytes.writeUInt32LE(magic, 0); + bytes.writeUInt32LE(payload.length, 4); + payload.copy(bytes, 8); + return bytes; +}; + +const allowance = ( + maxFrames: number, + source: "default" | "setting" | "runtime", +) => ({ maxFrames, source }); + +it("yields again after the last frame-by-frame decode before success", async () => { + let pauses = 0; + const controller = new AbortController(); + const pending = decodeZstandardFramesCooperative( + rawFrame(0x61), + {}, + async () => { + pauses += 1; + if (pauses === 1) { + controller.abort(); + return; + } + if (controller.signal.aborted) + throw Object.assign(new Error("cancelled"), { reason: "cancelled" }); + }, + ); + await expect(pending).rejects.toMatchObject({ reason: "cancelled" }); + expect(pauses).toBe(2); +}); + +it("observes cancellation while admitting a multi-block frame", async () => { + const first = zstdCompressSync(Buffer.alloc(8 * 1024 * 1024, "read me\n")); + const second = zstdCompressSync(Buffer.from("another frame")); + const bytes = Buffer.concat([first, second]); + const options = { allowance: allowance(1, "runtime") }; + const stopped = new Error("cancelled"); + await expect( + decodeZstandardFramesCooperative(bytes, options, async () => { + throw stopped; + }), + ).rejects.toBe(stopped); + await expect( + decodeZstandardFramesCooperative(bytes, options, async () => {}), + ).resolves.toMatchObject({ + reason: "work_limit", + offset: first.length, + setting: "zstd.max_frames", + value: 1, + source: "runtime", + }); +}); + +describe("exact Zstandard frames", () => { + it("accepts concatenated frames and skippable frames", () => { + const first = rawFrame(0x61); + const second = rawFrame(0x62); + const skip = skippable(Buffer.from("skip"), 0x184d2a5f); + const bytes = Buffer.concat([ + skip, + first, + skippable(Buffer.alloc(0)), + second, + ]); + const decoded = decodeZstandardFrames(bytes, { + allowance: allowance(8, "default"), + }); + expect(decoded).toMatchObject({ + ok: true, + consumedBytes: bytes.length, + frames: 4, + }); + if (!decoded.ok) return; + expect([...decoded.bytes]).toEqual([0x61, 0x62]); + }); + + it.each([ + [ + "frame header", + (frame: Buffer) => frame.subarray(0, 5), + "truncated_frame_header", + ], + ["block", (frame: Buffer) => frame.subarray(0, -1), "truncated_block"], + ] as const)("rejects a truncated %s", (_label, trim, reason) => { + expect(decodeZstandardFrames(trim(rawFrame(0x61)))).toMatchObject({ + ok: false, + reason, + offset: expect.any(Number), + }); + }); + + it("rejects a truncated checksum and a wrong checksum separately", () => { + const frame = zstdCompressSync(Buffer.from("hello zstd checksum"), { + params: { [constants.ZSTD_c_checksumFlag]: 1 }, + }); + expect(decodeZstandardFrames(frame.subarray(0, -1))).toMatchObject({ + ok: false, + reason: "truncated_checksum", + }); + expect(decodeZstandardFrames(frame.subarray(0, -4))).toMatchObject({ + ok: false, + reason: "truncated_checksum", + }); + const wrong = Buffer.from(frame); + const last = wrong.length - 1; + wrong.writeUInt8(wrong.readUInt8(last) ^ 0xff, last); + expect(decodeZstandardFrames(wrong)).toMatchObject({ + ok: false, + reason: "malformed", + code: "ZSTD_error_checksum_wrong", + }); + expect(decodeZstandardFrames(frame)).toMatchObject({ ok: true }); + }); + + it("rejects trailing garbage and a partial next-frame magic", () => { + const frame = rawFrame(0x61); + expect( + decodeZstandardFrames(Buffer.concat([frame, Buffer.of(0)])), + ).toMatchObject({ ok: false, reason: "trailing_bytes" }); + expect( + decodeZstandardFrames( + Buffer.concat([frame, Buffer.of(0x28, 0xb5, 0x2f)]), + ), + ).toMatchObject({ ok: false, reason: "trailing_bytes" }); + expect( + decodeZstandardFrames( + Buffer.concat([frame, Buffer.of(0x28, 0xb5, 0x2f, 0xfd)]), + ), + ).toMatchObject({ ok: false, reason: "truncated_frame_header" }); + }); + + it("stops a prefix at the first non-frame byte", async () => { + const frame = rawFrame(0x61); + const suffix = Buffer.from("CCOB"); + const input = Buffer.concat([frame, suffix]); + const options = { mode: "prefix" } as const; + const expected = { + ok: true, + bytes: Buffer.from("a"), + consumedBytes: frame.length, + frames: 1, + }; + expect(decodeZstandardFrames(input, options)).toMatchObject(expected); + await expect( + decodeZstandardFramesCooperative(input, options, async () => {}), + ).resolves.toMatchObject(expected); + }); + + it("refuses 65536 productive frames on the work allowance", () => { + const count = 65_536; + const frame = rawFrame(0x61); + const bytes = repeat(frame, count); + expect( + decodeZstandardFrames(bytes, { + maxOutputLength: count, + allowance: allowance(count - 1, "setting"), + }), + ).toMatchObject({ + ok: false, + reason: "work_limit", + setting: ZSTD_MAX_FRAMES_SETTING, + value: count - 1, + source: "setting", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + }); + const decoded = decodeZstandardFrames(bytes, { + maxOutputLength: count, + allowance: allowance(count, "runtime"), + }); + expect(decoded).toMatchObject({ + ok: true, + frames: count, + consumedBytes: bytes.length, + }); + if (!decoded.ok) return; + expect(decoded.bytes.length).toBe(count); + expect(decoded.bytes[0]).toBe(0x61); + expect(decoded.bytes[count - 1]).toBe(0x61); + }); +}); + +it("does not retain a decoder chunk behind a short frame", () => { + const backing = Buffer.alloc(16 * 1024, 0x61); + const slice = backing.subarray(0, 1); + const owned = ownedZstandardBytes(slice); + expect(owned.length).toBe(1); + expect(owned[0]).toBe(0x61); + expect(owned.buffer).not.toBe(backing.buffer); + const exact = Buffer.allocUnsafeSlow(1); + exact[0] = 0x62; + expect(ownedZstandardBytes(exact)).toBe(exact); +}); + +describe("Zstandard frame header widths", () => { + const magic = (bytes: number[]) => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, ...bytes]); + const decode = (bytes: Buffer) => + decodeZstandardFrames(bytes, { + allowance: allowance(8, "default"), + }); + + it("accepts content-size, window, dictionary-id, and RLE frames", () => { + const fcs4 = magic([0xa0, 1, 0, 0, 0, 0x09, 0, 0, 0x41]); + const fcs8 = magic([0xe0, 1, 0, 0, 0, 0, 0, 0, 0, 0x09, 0, 0, 0x41]); + const fcs2 = Buffer.alloc(10 + 256); + fcs2.writeUInt32LE(0xfd2fb528, 0); + fcs2[4] = 0x60; + fcs2.writeUInt16LE(0, 5); + fcs2.writeUIntLE(1 | (256 << 3), 7, 3); + fcs2.fill(0x42, 10); + const windowed = magic([0x00, 0x50, 0x01, 0, 0]); + const dictionary1 = magic([0x21, 0x00, 0x00, 0x01, 0, 0]); + const dictionary2 = magic([0x22, 0, 0, 0x00, 0x01, 0, 0]); + const dictionary4 = magic([0x23, 0, 0, 0, 0, 0x00, 0x01, 0, 0]); + const rle = magic([0x20, 0x04, 0x23, 0, 0, 0x41]); + expect(decode(fcs4)).toMatchObject({ ok: true, frames: 1 }); + expect(decode(fcs8)).toMatchObject({ ok: true, frames: 1 }); + expect(decode(fcs2)).toMatchObject({ ok: true, frames: 1 }); + const empty = decode(windowed); + const repeated = decode(rle); + const wide = decode(fcs2); + const sized = decode(fcs4); + expect(empty.ok && empty.bytes.length).toBe(0); + expect(decode(dictionary1).ok).toBe(true); + expect(decode(dictionary2).ok).toBe(true); + expect(decode(dictionary4).ok).toBe(true); + expect(sized.ok && sized.bytes[0]).toBe(0x41); + expect( + repeated.ok && Buffer.compare(repeated.bytes, Buffer.from("AAAA")) === 0, + ).toBe(true); + expect(wide.ok && wide.bytes.length).toBe(256); + }); + + it.each([ + ["content size", () => magic([0xa0, 1, 0]), "truncated_frame_header"], + [ + "eight-byte content size", + () => magic([0xe0, 1, 0, 0, 0]), + "truncated_frame_header", + ], + ["window descriptor", () => magic([0x00]), "truncated_frame_header"], + ["one-byte dictionary id", () => magic([0x21]), "truncated_frame_header"], + [ + "two-byte dictionary id", + () => magic([0x22, 0]), + "truncated_frame_header", + ], + [ + "four-byte dictionary id", + () => magic([0x23, 0, 0]), + "truncated_frame_header", + ], + ["RLE block", () => magic([0x20, 0x04, 0x23, 0, 0]), "truncated_block"], + ] as const)("rejects a truncated %s", (_label, frame, reason) => { + expect(decode(frame())).toMatchObject({ ok: false, reason }); + }); +}); + +describe("zstd.max_frames allowance", () => { + it("resolves default, launch, and runtime sources", () => { + expect(resolveZstandardFrameAllowance({})).toEqual({ + ok: true, + value: { maxFrames: DEFAULT_ZSTD_MAX_FRAMES, source: "default" }, + }); + expect( + resolveZstandardFrameAllowance({ [ZSTD_MAX_FRAMES_ENVIRONMENT]: "8" }), + ).toEqual({ ok: true, value: { maxFrames: 8, source: "setting" } }); + expect(resolveZstandardFrameAllowance({}, 3)).toEqual({ + ok: true, + value: { maxFrames: 3, source: "runtime" }, + }); + const invalid = resolveZstandardFrameAllowance({ + [ZSTD_MAX_FRAMES_ENVIRONMENT]: "0", + }); + expect(invalid.ok).toBe(false); + if (!invalid.ok) expect(invalid.error).toBeInstanceOf(ConfigurationError); + const above = resolveZstandardFrameAllowance({ + [ZSTD_MAX_FRAMES_ENVIRONMENT]: String(MAXIMUM_ZSTD_MAX_FRAMES + 1), + }); + expect(above.ok).toBe(false); + }); +}); diff --git a/src/domain/compression/zstandardExactFrame.ts b/src/domain/compression/zstandardExactFrame.ts new file mode 100644 index 0000000000..6a37899dfe --- /dev/null +++ b/src/domain/compression/zstandardExactFrame.ts @@ -0,0 +1,522 @@ +import { constants as bufferConstants } from "node:buffer"; +import { promisify } from "node:util"; +import { + constants, + zstdCompressSync, + zstdDecompress, + zstdDecompressSync, +} from "node:zlib"; +import { nodeCodecResult } from "../../kernel/nodeCodecResult.js"; +import { hostRangeErrorKind } from "../parseFailure.js"; +import type { ConfigurationError } from "../configurationErrors.js"; +import { + currentZstandardFrameAllowance, + type ZstandardFrameAllowance, + type ZstandardFrameSource, + zstandardWorkLimitFacts, +} from "./zstandardFrameAllowance.js"; +import { + ZSTANDARD_MAGIC_BYTES, + zstandardFrameKind, + zstandardFrameReads, + type ZstandardFrameKind, +} from "./zstandardFrameWire.js"; + +/** Zstandard's default maximum decoder window, 2^27 bytes (128 MiB). */ +export const ZSTANDARD_WINDOW_LOG_MAX = 27; + +/** + * Structural framing failures. `work_limit` is a decoder-work refusal, not + * evidence that the bytes are malformed. + */ +export const ZSTANDARD_FRAME_REASONS = [ + "truncated_frame_header", + "truncated_block", + "truncated_checksum", + "trailing_bytes", + "work_limit", + "malformed", + "output_limit", + "invalid_allowance", +] as const; + +export type ZstandardFrameReason = (typeof ZSTANDARD_FRAME_REASONS)[number]; + +export type ZstandardFrameMode = "exact" | "prefix"; + +export interface ZstandardDecodeOptions { + /** Aggregate decoded-output ceiling. Node applies it to one decoder call. */ + readonly maxOutputLength?: number; + /** `ZSTD_d_windowLogMax`. The default is Zstandard's 128 MiB window. */ + readonly windowLogMax?: number; + /** Optional dictionary. Frames that name a dictionary fail without one. */ + readonly dictionary?: Uint8Array; + /** + * `exact` rejects every byte that is not part of a complete frame. + * `prefix` returns the decoded frames and the byte count they occupied, + * leaving a non-frame suffix unconsumed. + */ + readonly mode?: ZstandardFrameMode; + /** When omitted, the accepted operation's `zstd.max_frames` allowance. */ + readonly allowance?: ZstandardFrameAllowance; +} + +export interface ZstandardFrameFailure { + readonly ok: false; + readonly reason: ZstandardFrameReason; + readonly offset: number; + /** Node or structural code for a malformed frame. Absent for truncation. */ + readonly code?: string; + /** Decoder text for a malformed frame. Callers may keep it; tests use `code`. */ + readonly detail?: string; + readonly setting?: "zstd.max_frames"; + readonly value?: number; + readonly source?: ZstandardFrameSource; + readonly minimum?: number; + readonly maximum?: number; + /** Present when the launch seed for `zstd.max_frames` cannot be parsed. */ + readonly configuration?: ConfigurationError; +} + +export interface ZstandardFrameSuccess { + readonly ok: true; + readonly bytes: Buffer; + readonly consumedBytes: number; + readonly frames: number; +} + +export type ZstandardFrameResult = + | ZstandardFrameSuccess + | ZstandardFrameFailure; + +interface FrameSpan { + readonly offset: number; + readonly length: number; + readonly kind: ZstandardFrameKind; +} + +interface WalkSuccess { + readonly ok: true; + readonly frames: readonly FrameSpan[]; + readonly consumed: number; +} + +// Bound structural work between cooperative pauses, including across frames. +const READS_PER_PAUSE = 64; +interface FrameWalkProgress { + reads: number; +} +const needsFrameWalkPause = (progress: FrameWalkProgress): boolean => + ++progress.reads % READS_PER_PAUSE === 0; + +const failure = ( + reason: ZstandardFrameReason, + offset: number, + code?: string, + detail?: string, +): ZstandardFrameFailure => ({ + ok: false, + reason, + offset, + ...(code === undefined ? {} : { code }), + ...(detail === undefined ? {} : { detail }), +}); + +const workLimit = ( + allowance: ZstandardFrameAllowance, + offset: number, +): ZstandardFrameFailure => ({ + ok: false, + offset, + ...zstandardWorkLimitFacts(allowance), +}); + +const unreadBytes = new Uint8Array(0); + +function* frameAt( + bytes: Uint8Array, + offset: number, + kind: ZstandardFrameKind, + progress: FrameWalkProgress, +): Generator { + let cursor = offset + ZSTANDARD_MAGIC_BYTES; + const reads = zstandardFrameReads(kind, offset); + let step = reads.next(); + while (!step.done) { + if (needsFrameWalkPause(progress)) yield; + const read = step.value; + const end = cursor + read.length; + if (end > bytes.length) return failure(read.eof, read.failureOffset); + step = reads.next(read.inspect ? bytes.subarray(cursor, end) : unreadBytes); + cursor = end; + } + if (step.value !== undefined) + return failure("malformed", step.value.offset, step.value.code); + return { offset, length: cursor - offset, kind }; +} + +/** + * Exact frame boundaries for one buffer. The walk does not ask the decoder + * how many bytes it consumed. + */ +function* walkFrames( + bytes: Uint8Array, + allowance: ZstandardFrameAllowance, + mode: ZstandardFrameMode, +): Generator { + const frames: FrameSpan[] = []; + const progress = { reads: 0 }; + let offset = 0; + while (offset < bytes.length) { + const kind = zstandardFrameKind(bytes, offset); + if (frames.length === allowance.maxFrames) { + if (kind !== undefined) return workLimit(allowance, offset); + if (mode === "exact") return failure("trailing_bytes", offset); + break; + } + if (offset + ZSTANDARD_MAGIC_BYTES > bytes.length) { + if (frames.length === 0) return failure("truncated_frame_header", offset); + if (mode === "exact") return failure("trailing_bytes", offset); + break; + } + if (kind === undefined) { + if (frames.length === 0) return failure("malformed", offset, "magic"); + if (mode === "exact") return failure("trailing_bytes", offset); + break; + } + const next = yield* frameAt(bytes, offset, kind, progress); + if ("ok" in next) return next; + frames.push(next); + offset += next.length; + } + if (frames.length === 0) return failure("truncated_frame_header", 0); + return { ok: true, frames, consumed: offset }; +} + +const decoderCode = (cause: unknown): string | undefined => { + if (typeof cause !== "object" || cause === null || !("code" in cause)) + return undefined; + const code = cause.code; + return typeof code === "string" ? code : undefined; +}; + +const decoderFailure = ( + cause: unknown, + offset: number, +): ZstandardFrameFailure => { + if (hostRangeErrorKind(cause) === "host-capacity") + return failure("output_limit", offset); + const code = decoderCode(cause); + const detail = cause instanceof Error ? cause.message : undefined; + if (code === "ZSTD_error_dstSize_tooSmall" || code === "ERR_BUFFER_TOO_LARGE") + return failure("output_limit", offset, code, detail); + return failure("malformed", offset, code, detail); +}; + +const isFrameFailure = ( + value: { readonly bytes: Buffer } | ZstandardFrameFailure, +): value is ZstandardFrameFailure => "reason" in value; + +interface DecompressOptions { + readonly maxOutputLength: number; + readonly windowLogMax: number; + readonly dictionary?: Uint8Array; + readonly rejectTrailing: boolean; +} + +/** + * Node 22 stops after the first frame and ignores `rejectGarbageAfterEnd`. + * Node 24.19+ and Node 26 decode a whole sequence and reject a suffix with + * that option. Probe once; do not trust a major-version check alone. + */ +let strictSequence: boolean | undefined; +const nodeDecodesStrictSequence = (): boolean => { + if (strictSequence !== undefined) return strictSequence; + const frame = zstdCompressSync(Buffer.from([0x61])); + const pair = Buffer.concat([frame, frame]); + let concatenated = false; + try { + concatenated = zstdDecompressSync(pair).length === 2; + } catch { + concatenated = false; + } + let strict = false; + if (concatenated) { + try { + zstdDecompressSync(Buffer.concat([frame, Buffer.of(0)]), { + rejectGarbageAfterEnd: true, + }); + } catch { + strict = true; + } + } + strictSequence = strict; + return strictSequence; +}; + +const unzstd = promisify(zstdDecompress); + +const decompress = ( + input: Uint8Array, + offset: number, + options: DecompressOptions, +): { readonly bytes: Buffer } | ZstandardFrameFailure => { + try { + const result = nodeCodecResult( + zstdDecompressSync(input, { + info: true, + maxOutputLength: Math.max(1, options.maxOutputLength), + ...(options.rejectTrailing ? { rejectGarbageAfterEnd: true } : {}), + ...(options.dictionary === undefined + ? {} + : { dictionary: options.dictionary }), + params: { [constants.ZSTD_d_windowLogMax]: options.windowLogMax }, + }), + ); + // The walk already fixed the input extent. A short read means the + // decoder did not accept that extent, so the bytes are not returned. + if (result.consumedBytes !== input.length) + return failure("malformed", offset, "unconsumed_input"); + return { bytes: result.buffer }; + } catch (cause: unknown) { + return decoderFailure(cause, offset); + } +}; + +/** Async strict decode so a large sequence does not block the event loop. */ +const decompressSequence = async ( + input: Uint8Array, + offset: number, + options: DecompressOptions, +): Promise<{ readonly bytes: Buffer } | ZstandardFrameFailure> => { + try { + const result = nodeCodecResult( + await unzstd(input, { + info: true, + rejectGarbageAfterEnd: true, + maxOutputLength: Math.max(1, options.maxOutputLength), + ...(options.dictionary === undefined + ? {} + : { dictionary: options.dictionary }), + params: { [constants.ZSTD_d_windowLogMax]: options.windowLogMax }, + }), + ); + if (result.consumedBytes !== input.length) + return failure("malformed", offset, "unconsumed_input"); + return { bytes: ownedZstandardBytes(result.buffer) }; + } catch (cause: unknown) { + return decoderFailure(cause, offset); + } +}; + +/** + * Node's synchronous decoder returns a view onto a larger chunk allocation. + * Retaining that view keeps the whole chunk. One byte from a 16 KiB chunk, + * repeated for every frame, retains mebibytes for a few kibibytes of output. + * Copy only when the backing store is larger than the bytes in use. + */ +export const ownedZstandardBytes = (bytes: Buffer): Buffer => { + if (bytes.byteOffset === 0 && bytes.buffer.byteLength === bytes.byteLength) + return bytes; + const owned = Buffer.allocUnsafe(bytes.byteLength); + bytes.copy(owned); + return owned; +}; + +const withinOutput = ( + bytes: Buffer, + remaining: number, + offset: number, +): { readonly bytes: Buffer } | ZstandardFrameFailure => + bytes.length > remaining ? failure("output_limit", offset) : { bytes }; + +const decodeDataFrame = ( + bytes: Uint8Array, + frame: FrameSpan, + total: number, + options: DecompressOptions, +): { readonly bytes: Buffer } | ZstandardFrameFailure => { + const remaining = options.maxOutputLength - total; + const decoded = decompress( + bytes.subarray(frame.offset, frame.offset + frame.length), + frame.offset, + { ...options, maxOutputLength: Math.max(1, remaining) }, + ); + if (isFrameFailure(decoded)) return decoded; + const bounded = withinOutput(decoded.bytes, remaining, frame.offset); + if (isFrameFailure(bounded)) return bounded; + return { bytes: ownedZstandardBytes(bounded.bytes) }; +}; + +const finishFrames = ( + chunks: readonly Buffer[], + consumed: number, + frames: number, +): ZstandardFrameSuccess => { + const only = chunks.length === 1 ? chunks[0] : undefined; + return { + ok: true, + bytes: + only !== undefined + ? only + : chunks.length === 0 + ? Buffer.alloc(0) + : Buffer.concat(chunks), + consumedBytes: consumed, + frames, + }; +}; + +const decodeFrames = ( + bytes: Uint8Array, + frames: readonly FrameSpan[], + consumed: number, + options: DecompressOptions, +): ZstandardFrameSuccess | ZstandardFrameFailure => { + if (options.rejectTrailing) { + const decoded = decompress(bytes.subarray(0, consumed), 0, options); + if (isFrameFailure(decoded)) return decoded; + const bounded = withinOutput(decoded.bytes, options.maxOutputLength, 0); + if (isFrameFailure(bounded)) return bounded; + return { + ...finishFrames( + [ownedZstandardBytes(bounded.bytes)], + consumed, + frames.length, + ), + }; + } + const chunks: Buffer[] = []; + let total = 0; + for (const frame of frames) { + if (frame.kind === "skippable") continue; + const step = decodeDataFrame(bytes, frame, total, options); + if (isFrameFailure(step)) return step; + total += step.bytes.length; + chunks.push(step.bytes); + } + return finishFrames(chunks, consumed, frames.length); +}; + +/** + * Decode one Zstandard buffer. Frame boundaries come from the structural + * walk. Where Node's public strict decoder accepts a whole sequence, one + * call decodes that sequence and checks its checksums. Otherwise each + * walked frame is decoded on its own, which is what Node 22 requires. + */ +interface PreparedDecode { + readonly frames: readonly FrameSpan[]; + readonly consumed: number; + readonly options: DecompressOptions; +} + +function* prepareDecode( + bytes: Uint8Array, + options: ZstandardDecodeOptions, +): Generator { + const resolved = + options.allowance === undefined + ? currentZstandardFrameAllowance() + : { ok: true as const, value: options.allowance }; + if (!resolved.ok) + return { + ok: false, + reason: "invalid_allowance", + offset: 0, + configuration: resolved.error, + }; + const mode = options.mode ?? "exact"; + const walked = yield* walkFrames(bytes, resolved.value, mode); + if ("reason" in walked) return walked; + return { + frames: walked.frames, + consumed: walked.consumed, + options: { + maxOutputLength: options.maxOutputLength ?? bufferConstants.MAX_LENGTH, + windowLogMax: options.windowLogMax ?? ZSTANDARD_WINDOW_LOG_MAX, + ...(options.dictionary === undefined + ? {} + : { dictionary: options.dictionary }), + rejectTrailing: nodeDecodesStrictSequence(), + }, + }; +} + +/** + * Decode one Zstandard buffer. Frame boundaries come from the structural + * walk. Where Node's public strict decoder accepts a whole sequence, one + * call decodes that sequence and checks its checksums. Otherwise each + * walked frame is decoded on its own, which is what Node 22 requires. + */ +export const decodeZstandardFrames = ( + bytes: Uint8Array, + options: ZstandardDecodeOptions = {}, +): ZstandardFrameResult => { + const steps = prepareDecode(bytes, options); + let next = steps.next(); + while (!next.done) next = steps.next(); + const prepared = next.value; + if ("reason" in prepared) return prepared; + return decodeFrames( + bytes, + prepared.frames, + prepared.consumed, + prepared.options, + ); +}; + +/** Yield point the caller uses to observe its existing abort signal. */ +export type ZstandardFramePause = () => Promise; + +/** + * Pause during structural validation, before decoding, and before returning + * successful output. The strict sequence uses the async decoder. The fallback + * pauses before each synchronous data-frame decode. + */ +export const decodeZstandardFramesCooperative = async ( + bytes: Uint8Array, + options: ZstandardDecodeOptions, + pause: ZstandardFramePause, +): Promise => { + const steps = prepareDecode(bytes, options); + let next = steps.next(); + while (!next.done) { + await pause(); + next = steps.next(); + } + const prepared = next.value; + if ("reason" in prepared) return prepared; + if (prepared.options.rejectTrailing) { + await pause(); + const decoded = await decompressSequence( + bytes.subarray(0, prepared.consumed), + 0, + prepared.options, + ); + if (isFrameFailure(decoded)) return decoded; + const bounded = withinOutput( + decoded.bytes, + prepared.options.maxOutputLength, + 0, + ); + if (isFrameFailure(bounded)) return bounded; + await pause(); + return finishFrames( + [bounded.bytes], + prepared.consumed, + prepared.frames.length, + ); + } + const chunks: Buffer[] = []; + let total = 0; + for (const frame of prepared.frames) { + if (frame.kind === "skippable") continue; + await pause(); + const step = decodeDataFrame(bytes, frame, total, prepared.options); + if (isFrameFailure(step)) return step; + total += step.bytes.length; + chunks.push(step.bytes); + } + await pause(); + return finishFrames(chunks, prepared.consumed, prepared.frames.length); +}; diff --git a/src/domain/compression/zstandardFrameAllowance.ts b/src/domain/compression/zstandardFrameAllowance.ts new file mode 100644 index 0000000000..3294db2a92 --- /dev/null +++ b/src/domain/compression/zstandardFrameAllowance.ts @@ -0,0 +1,176 @@ +import { z } from "zod"; +import { + DEFAULT_ZSTD_MAX_FRAMES, + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../analysisResourceLimits.js"; +import { analysisSettingLimit } from "../analysisSettingDefinitions.js"; +import { AnalysisResourceConstraintError } from "../analysisErrorCore.js"; +import { currentAnalysisSettingOverride } from "../analysisOperationSnapshot.js"; +import { ConfigurationError } from "../configurationErrors.js"; +import { parseLimitConstraint } from "../parseFailureAnalysisError.js"; +import { ParseFailureError } from "../parseFailure.js"; +import type { Result } from "../result.js"; +import { err, ok } from "../result.js"; + +/** Launch seed for the shared Zstandard frame allowance. */ +export const ZSTD_MAX_FRAMES_ENVIRONMENT = "REA_ZSTD_MAX_FRAMES"; + +/** Registered analysis setting that owns this allowance. */ +export const ZSTD_MAX_FRAMES_SETTING = "zstd.max_frames" as const; + +export type ZstandardFrameSource = "default" | "setting" | "runtime"; + +/** The frame allowance in force for one decode, and where it came from. */ +export interface ZstandardFrameAllowance { + readonly maxFrames: number; + readonly source: ZstandardFrameSource; +} + +const constraint = `${ZSTD_MAX_FRAMES_ENVIRONMENT} must be a decimal integer from ${String(MINIMUM_ZSTD_MAX_FRAMES)} to ${String(MAXIMUM_ZSTD_MAX_FRAMES)} frames; the default is ${String(DEFAULT_ZSTD_MAX_FRAMES)}.`; + +const decimalInteger = /^[1-9]\d*$/u; + +/** Environment seed for `zstd.max_frames`, shared with launch admission. */ +export const zstandardMaxFramesSchema = z + .string() + .regex(decimalInteger, { error: constraint }) + .refine( + (value) => { + if (!decimalInteger.test(value)) return true; + const parsed = BigInt(value); + return ( + parsed >= BigInt(MINIMUM_ZSTD_MAX_FRAMES) && + parsed <= BigInt(MAXIMUM_ZSTD_MAX_FRAMES) + ); + }, + { error: constraint }, + ) + .transform(Number); + +const invalidSeed = (): ConfigurationError => + new ConfigurationError(constraint, { + remediationAction: `On a running MCP session, call update_analysis_settings for ${ZSTD_MAX_FRAMES_SETTING}, then retry. For CLI or future REA processes, correct ${ZSTD_MAX_FRAMES_ENVIRONMENT} in the launch environment.`, + settings: [ + { + setting: ZSTD_MAX_FRAMES_ENVIRONMENT, + constraint, + update_mode: "runtime", + pending_restart: false, + }, + ], + }); + +/** + * Resolve the allowance for one decode. A runtime override wins, then the + * launch seed, then the default. An invalid seed is refused; it is not + * replaced by the default. + */ +export const resolveZstandardFrameAllowance = ( + environment: Readonly>, + override: number | undefined = currentAnalysisSettingOverride( + ZSTD_MAX_FRAMES_SETTING, + ), +): Result => { + if (override !== undefined) + return ok({ maxFrames: override, source: "runtime" }); + const raw = environment[ZSTD_MAX_FRAMES_ENVIRONMENT]; + if (raw === undefined) + return ok({ maxFrames: DEFAULT_ZSTD_MAX_FRAMES, source: "default" }); + const parsed = zstandardMaxFramesSchema.safeParse(raw); + return parsed.success + ? ok({ maxFrames: parsed.data, source: "setting" }) + : err(invalidSeed()); +}; + +/** Allowance for the operation on this stack, or the process launch environment. */ +export const currentZstandardFrameAllowance = (): Result< + ZstandardFrameAllowance, + ConfigurationError +> => resolveZstandardFrameAllowance(process.env); + +/** Facts a work refusal carries so callers do not restate the registry. */ +export const zstandardWorkLimitFacts = ( + allowance: ZstandardFrameAllowance, +): { + readonly reason: "work_limit"; + readonly setting: typeof ZSTD_MAX_FRAMES_SETTING; + readonly value: number; + readonly source: ZstandardFrameSource; + readonly minimum: number; + readonly maximum: number; +} => ({ + reason: "work_limit", + setting: ZSTD_MAX_FRAMES_SETTING, + value: allowance.maxFrames, + source: allowance.source, + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, +}); + +/** + * Decoder-work refusal. Distinct from malformed framing: the bytes were not + * shown to be invalid, and the named setting can admit a larger frame count. + */ +export class ZstandardFrameAllowanceError extends ParseFailureError<"limit"> { + readonly reason = "work_limit" as const; + readonly setting = ZSTD_MAX_FRAMES_SETTING; + readonly value: number; + readonly source: ZstandardFrameSource; + readonly minimum: number; + readonly maximum: number; + readonly offset: number; + + constructor(allowance: ZstandardFrameAllowance, offset: number) { + const facts = zstandardWorkLimitFacts(allowance); + const detail = `${facts.setting} is ${String(facts.value)} (source ${facts.source}, range ${String(facts.minimum)}..${String(facts.maximum)})`; + super({ + kind: "limit", + format: "zstd", + detail, + offset, + field: facts.reason, + limit: { + name: facts.setting, + maximum: facts.value, + required: facts.value + 1, + }, + }); + this.name = "ZstandardFrameAllowanceError"; + this.value = facts.value; + this.source = facts.source; + this.minimum = facts.minimum; + this.maximum = facts.maximum; + this.offset = offset; + } +} + +/** Read the frame refusal carried by a raw parser or its public projection. */ +export const zstandardFrameAllowanceCause = (cause: unknown) => + cause instanceof ZstandardFrameAllowanceError + ? cause + : cause instanceof AnalysisResourceConstraintError && + cause.cause instanceof ZstandardFrameAllowanceError + ? cause.cause + : undefined; + +/** + * The public resource refusal for one frame-cap. Callers keep the setting, + * effective value, source, and range instead of rewriting the error as I/O + * or malformed content. + */ +export const projectZstandardFrameAllowance = ( + operation: string, + error: ZstandardFrameAllowanceError, +) => + parseLimitConstraint(operation, "cpu", error, { + retryable: false, + limit: analysisSettingLimit(error.setting, error.value, error.source), + reportedLimitContext: { + setting: error.setting, + value: error.value, + source: error.source, + minimum: error.minimum, + maximum: error.maximum, + }, + }); diff --git a/src/domain/compression/zstandardFrameChunks.ts b/src/domain/compression/zstandardFrameChunks.ts new file mode 100644 index 0000000000..c10933fe2f --- /dev/null +++ b/src/domain/compression/zstandardFrameChunks.ts @@ -0,0 +1,89 @@ +import { + ZSTANDARD_MAGIC_BYTES, + zstandardFrameKind, + zstandardFrameReads, + type ZstandardFrameRead, +} from "./zstandardFrameWire.js"; + +export class ZstandardFramingError extends Error { + constructor( + readonly code: string, + readonly offset: number, + ) { + super(`Zstandard framing failed at byte ${offset}: ${code}`); + } +} + +const magicRead = (offset: number): ZstandardFrameRead => ({ + length: ZSTANDARD_MAGIC_BYTES, + inspect: true, + eof: "truncated_frame_header", + failureOffset: offset, +}); + +/** + * Yield compressed slices whose boundaries never cross two frames. Node 22 + * stops a decode call at its first frame end, but its stream can accept later + * frames in separate writes. Chunk boundaries may also split a frame. + */ +export async function* zstandardFrameChunks( + input: AsyncIterable, +): AsyncGenerator { + // Only inspected fields that span chunks are copied. Magic and skippable + // size are the widest inspected fields (four bytes). + const header = new Uint8Array(ZSTANDARD_MAGIC_BYTES); + const unread = new Uint8Array(0); + let offset = 0; + let frameOffset = 0; + let read = magicRead(frameOffset); + let supplied = 0; + let reads: ReturnType | undefined; + for await (const bytes of input) { + let start = 0; + let cursor = 0; + while (cursor < bytes.length) { + const count = Math.min(read.length - supplied, bytes.length - cursor); + const end = cursor + count; + const complete = supplied + count === read.length; + let field: Uint8Array = unread; + if (read.inspect) { + if (supplied === 0 && complete) { + field = bytes.subarray(cursor, end); + } else { + header.set(bytes.subarray(cursor, end), supplied); + field = header; + } + } + cursor = end; + offset += count; + if (!complete) { + supplied += count; + break; + } + supplied = 0; + if (reads === undefined) { + const kind = zstandardFrameKind(field, 0); + if (kind === undefined) + throw new ZstandardFramingError("magic", frameOffset); + reads = zstandardFrameReads(kind, frameOffset); + } + let step = reads.next(field); + // A reserved/empty length never needs another input chunk. + while (!step.done && step.value.length === 0) step = reads.next(unread); + if (!step.done) { + read = step.value; + continue; + } + if (step.value !== undefined) + throw new ZstandardFramingError(step.value.code, step.value.offset); + yield bytes.subarray(start, cursor); + start = cursor; + frameOffset = offset; + reads = undefined; + read = magicRead(frameOffset); + } + if (start < bytes.length) yield bytes.subarray(start); + } + if (reads !== undefined || supplied !== 0 || frameOffset === 0) + throw new ZstandardFramingError(read.eof, read.failureOffset); +} diff --git a/src/domain/compression/zstandardFrameWire.ts b/src/domain/compression/zstandardFrameWire.ts new file mode 100644 index 0000000000..bc13387131 --- /dev/null +++ b/src/domain/compression/zstandardFrameWire.ts @@ -0,0 +1,117 @@ +const ZSTD_MAGIC = 0xfd2fb528; +const SKIPPABLE_MAGIC_MIN = 0x184d2a50; +const SKIPPABLE_MAGIC_MAX = 0x184d2a5f; +export const ZSTANDARD_MAGIC_BYTES = 4; + +export type ZstandardFrameKind = "zstd" | "skippable"; + +type Truncation = + | "truncated_frame_header" + | "truncated_block" + | "truncated_checksum"; + +export interface ZstandardFrameRead { + readonly length: number; + readonly inspect: boolean; + readonly eof: Truncation; + readonly failureOffset: number; +} + +type FrameWireFailureCode = "reserved_frame_descriptor" | "reserved_block"; + +const u24 = (bytes: Uint8Array, offset: number): number => + (bytes[offset] ?? 0) | + ((bytes[offset + 1] ?? 0) << 8) | + ((bytes[offset + 2] ?? 0) << 16); + +const u32 = (bytes: Uint8Array, offset: number): number => + (u24(bytes, offset) | ((bytes[offset + 3] ?? 0) << 24)) >>> 0; + +export const zstandardFrameKind = ( + bytes: Uint8Array, + offset: number, +): ZstandardFrameKind | undefined => { + if (bytes.length - offset < ZSTANDARD_MAGIC_BYTES) return undefined; + const magic = u32(bytes, offset); + if (magic === ZSTD_MAGIC) return "zstd"; + if (magic >= SKIPPABLE_MAGIC_MIN && magic <= SKIPPABLE_MAGIC_MAX) + return "skippable"; + return undefined; +}; + +/** + * RFC 8878 §3.1 frame extent after its magic. The caller supplies inspected + * fields and skips opaque bytes, which remain the native decoder's concern. + */ +export function* zstandardFrameReads( + kind: ZstandardFrameKind, + offset: number, +): Generator< + ZstandardFrameRead, + { readonly code: FrameWireFailureCode; readonly offset: number } | undefined, + Uint8Array +> { + let cursor = offset + ZSTANDARD_MAGIC_BYTES; + const header = (length: number, inspect: boolean): ZstandardFrameRead => ({ + length, + inspect, + eof: "truncated_frame_header", + failureOffset: offset, + }); + if (kind === "skippable") { + const userSize = yield header(4, true); + yield { + length: u32(userSize, 0), + inspect: false, + eof: "truncated_block", + failureOffset: cursor + 4, + }; + return undefined; + } + + const descriptor = (yield header(1, true))[0] ?? 0; + if (descriptor & 0x08) return { code: "reserved_frame_descriptor", offset }; + const singleSegment = (descriptor & 0x20) !== 0; + const dictionaryFlag = descriptor & 0x03; + const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag; + const contentSizeFlag = descriptor >>> 6; + const contentSizeBytes = + contentSizeFlag === 0 ? Number(singleSegment) : 2 ** contentSizeFlag; + const headerRest = + Number(!singleSegment) + dictionaryBytes + contentSizeBytes; + yield header(headerRest, false); + cursor += 1 + headerRest; + + for (;;) { + const block = u24( + yield { + length: 3, + inspect: true, + eof: "truncated_block", + failureOffset: cursor, + }, + 0, + ); + const type = (block >>> 1) & 3; + if (type === 3) return { code: "reserved_block", offset: cursor }; + cursor += 3; + // An RLE block stores one byte and repeats it the declared number of times. + const length = type === 1 ? 1 : block >>> 3; + yield { + length, + inspect: false, + eof: "truncated_block", + failureOffset: cursor, + }; + cursor += length; + if (block & 1) break; + } + if (descriptor & 0x04) + yield { + length: 4, + inspect: false, + eof: "truncated_checksum", + failureOffset: cursor, + }; + return undefined; +} diff --git a/src/domain/firmware/androidOtaPayload.test.ts b/src/domain/firmware/androidOtaPayload.test.ts index 3712c8f8ce..f4db766d7c 100644 --- a/src/domain/firmware/androidOtaPayload.test.ts +++ b/src/domain/firmware/androidOtaPayload.test.ts @@ -1,5 +1,12 @@ import { createHash } from "node:crypto"; import { zstdCompressSync } from "node:zlib"; +import { runWithAnalysisOperationSnapshot } from "../analysisOperationSnapshot.js"; +import { ConfigurationError } from "../configurationErrors.js"; +import { ZSTD_MAX_FRAMES_ENVIRONMENT } from "../compression/zstandardFrameAllowance.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../analysisResourceLimits.js"; import { BinaryWriter, WireType } from "@bufbuild/protobuf/wire"; import { expect, it } from "vitest"; @@ -240,6 +247,140 @@ it("reconstructs REPLACE_ZSTD payloads", () => { ).toBe(true); }); +const zstdByteFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +it("rejects unread REPLACE_ZSTD bytes and accepts a second frame", () => { + const plain = Buffer.from("zstd partition"); + const trailing = buildAndroidOtaPayload({ + partitions: [ + { + name: "vendor", + operations: [ + { + type: 14, + data: Buffer.concat([zstdCompressSync(plain), Buffer.of(0)]), + }, + ], + }, + ], + }); + const trailingParsed = parse(trailing); + if (!trailingParsed.ok) throw trailingParsed.error; + const trailingBlob = trailing.subarray( + trailingParsed.value.header.blobOffset, + ); + expect( + reconstructAndroidOtaPartition( + trailingParsed.value.partitions[0]!, + trailingParsed.value.blockSize, + (offset, length) => trailingBlob.subarray(offset, offset + length), + ), + ).toMatchObject({ + ok: false, + error: { + kind: "malformed", + failure: { field: "trailing_bytes" }, + }, + }); + + const frames = Buffer.concat([zstdByteFrame(0x61), zstdByteFrame(0x62)]); + const concatenated = buildAndroidOtaPayload({ + partitions: [ + { + name: "vendor", + newSize: 2, + omitNewHash: true, + operations: [ + { + type: 14, + data: frames, + dstExtents: [{ startBlock: 0, numBlocks: 1 }], + }, + ], + }, + ], + }); + const parsed = parse(concatenated); + if (!parsed.ok) throw parsed.error; + const blob = concatenated.subarray(parsed.value.header.blobOffset); + const read = (offset: number, length: number) => + blob.subarray(offset, offset + length); + const reconstructed = reconstructAndroidOtaPartition( + parsed.value.partitions[0]!, + parsed.value.blockSize, + read, + { verifyDeclaredHash: false }, + ); + if (!reconstructed.ok) throw reconstructed.error; + expect([...reconstructed.value.subarray(0, 2)]).toEqual([0x61, 0x62]); + expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => + reconstructAndroidOtaPartition( + parsed.value.partitions[0]!, + parsed.value.blockSize, + read, + { verifyDeclaredHash: false }, + ), + ), + ).toMatchObject({ + ok: false, + error: { + _tag: "ZstandardFrameWorkRefusal", + reason: "work_limit", + setting: "zstd.max_frames", + value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + }, + }); +}); + +it("keeps an invalid REPLACE_ZSTD allowance out of the malformed path", () => { + const plain = Buffer.from("zstd-partition-bytes"); + const bytes = buildAndroidOtaPayload({ + partitions: [ + { + name: "vendor", + operations: [ + { + type: 14, + data: zstdCompressSync(plain), + dstExtents: [{ startBlock: 0, numBlocks: 1 }], + }, + ], + }, + ], + }); + const parsed = parse(bytes); + if (!parsed.ok) throw parsed.error; + const blob = bytes.subarray(parsed.value.header.blobOffset); + const reconstruct = () => + reconstructAndroidOtaPartition( + parsed.value.partitions[0]!, + parsed.value.blockSize, + (offset, length) => blob.subarray(offset, offset + length), + ); + const previous = process.env[ZSTD_MAX_FRAMES_ENVIRONMENT]; + process.env[ZSTD_MAX_FRAMES_ENVIRONMENT] = "0"; + try { + const refused = reconstruct(); + expect(refused.ok).toBe(false); + if (!refused.ok) expect(refused.error).toBeInstanceOf(ConfigurationError); + const recovered = runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 8 } }, + reconstruct, + ); + expect(recovered.ok).toBe(true); + } finally { + if (previous === undefined) delete process.env[ZSTD_MAX_FRAMES_ENVIRONMENT]; + else process.env[ZSTD_MAX_FRAMES_ENVIRONMENT] = previous; + } +}); + it("refuses REPLACE_ZSTD output past destination extents as a capacity limit", () => { const plain = Buffer.alloc(8192, 0x61); const bytes = buildAndroidOtaPayload({ diff --git a/src/domain/firmware/androidOtaPayload.ts b/src/domain/firmware/androidOtaPayload.ts index 6bcb44a97d..bb7d5118c3 100644 --- a/src/domain/firmware/androidOtaPayload.ts +++ b/src/domain/firmware/androidOtaPayload.ts @@ -1,11 +1,21 @@ -import { zstdDecompressSync } from "node:zlib"; +import { + decodeZstandardFrames, + decodeZstandardFramesCooperative, + type ZstandardFrameFailure, + type ZstandardFramePause, + type ZstandardFrameResult, +} from "../compression/zstandardExactFrame.js"; +import { + zstandardWorkLimitFacts, + type ZstandardFrameSource, +} from "../compression/zstandardFrameAllowance.js"; +import { ConfigurationError } from "../configurationErrors.js"; import { BinaryReader, WireType } from "@bufbuild/protobuf/wire"; import { sha256Hex } from "../../kernel/sha256Hex.js"; import { decodeLimit, - hostRangeErrorKind, malformedInput, ParseFailureError, truncatedInput, @@ -122,6 +132,7 @@ export type AndroidOtaPayloadParseError = export type AndroidOtaPayloadReconstructError = | AndroidOtaPayloadParseError + | ConfigurationError | { readonly _tag: "AndroidOtaPayloadUnsupportedOperation"; readonly message: string; @@ -136,6 +147,17 @@ export type AndroidOtaPayloadReconstructError = | { readonly _tag: "AndroidOtaPayloadBlobTruncated"; readonly message: string; + } + | { + readonly _tag: "ZstandardFrameWorkRefusal"; + readonly reason: "work_limit"; + readonly setting: "zstd.max_frames"; + readonly value: number; + readonly source: ZstandardFrameSource; + readonly minimum: number; + readonly maximum: number; + readonly offset: number; + readonly message: string; }; const ANDROID_OTA_FORMAT = "android-ota-payload"; @@ -166,6 +188,21 @@ const capacity = ( { name: `Android OTA payload ${name}`, maximum }, ); +const zstdWorkRefusal = ( + decoded: ZstandardFrameFailure, +): AndroidOtaPayloadReconstructError => { + const facts = zstandardWorkLimitFacts({ + maxFrames: decoded.value ?? 0, + source: decoded.source ?? "default", + }); + return { + _tag: "ZstandardFrameWorkRefusal", + ...facts, + offset: decoded.offset, + message: `${facts.setting} is ${String(facts.value)} (source ${facts.source}, range ${String(facts.minimum)}..${String(facts.maximum)})`, + }; +}; + const androidOtaUnsupportedVersion = (version: bigint): string => `Android OTA payload major version ${String(version)} is unsupported; this reader supports version 2 only`; @@ -1391,17 +1428,16 @@ const hasCompleteExtentCoverage = ( return false; }; -/** - * Rebuild a full-payload partition image after checking declared blob hashes. - * A partition image hash mismatch fails unless `verifyDeclaredHash` is false, - * which leaves that decision to a caller applying its own integrity policy. - */ -export const reconstructAndroidOtaPartition = ( +function* reconstructAndroidOtaPartitionSteps( partition: AndroidOtaPartition, blockSize: number, readBlob: (offset: number, length: number) => Uint8Array, options: { readonly verifyDeclaredHash?: boolean } = {}, -): Result => { +): Generator< + { readonly raw: Uint8Array; readonly maxOutputLength: number }, + Result, + ZstandardFrameResult +> { const refused = androidOtaExtractRefusalError(partition); if (refused !== null) return err(refused); const size = partition.newPartitionInfo.size!; @@ -1455,17 +1491,28 @@ export const reconstructAndroidOtaPartition = ( 1, Math.min(dstBytes.value, ANDROID_OTA_MAX_EXTRACT_BYTES), ); - try { - payload = zstdDecompressSync(raw, { maxOutputLength }); - } catch (cause: unknown) { + const decoded = yield { raw, maxOutputLength }; + if (!decoded.ok) { + if ( + decoded.reason === "invalid_allowance" && + decoded.configuration !== undefined + ) + return err(decoded.configuration); + if (decoded.reason === "work_limit") + return err(zstdWorkRefusal(decoded)); + if (decoded.reason === "output_limit") + return err( + capacity("REPLACE_ZSTD decoded output", maxOutputLength, "byte"), + ); return err( - hostRangeErrorKind(cause) === "host-capacity" - ? capacity("REPLACE_ZSTD decoded output", maxOutputLength, "byte") - : malformed( - "REPLACE_ZSTD operation payload is not a valid Zstandard frame", - ), + malformedInput( + ANDROID_OTA_FORMAT, + "REPLACE_ZSTD operation payload is not a valid Zstandard frame", + { offset: decoded.offset, field: decoded.reason }, + ), ); } + payload = decoded.bytes; } const written = writeExtents( image, @@ -1502,4 +1549,77 @@ export const reconstructAndroidOtaPartition = ( observedHashHex: observed, }); return ok(image); +} + +const driveAndroidOtaReconstruction = ( + steps: Generator< + { readonly raw: Uint8Array; readonly maxOutputLength: number }, + Result, + ZstandardFrameResult + >, + decode: ( + raw: Uint8Array, + maxOutputLength: number, + ) => ZstandardFrameResult | Promise, +): Promise> => { + const run = async (): Promise< + Result + > => { + let next = steps.next(); + while (!next.done) + next = steps.next( + await decode(next.value.raw, next.value.maxOutputLength), + ); + return next.value; + }; + return run(); }; + +/** + * Rebuild a full-payload partition image after checking declared blob hashes. + * A partition image hash mismatch fails unless `verifyDeclaredHash` is false, + * which leaves that decision to a caller applying its own integrity policy. + */ +export const reconstructAndroidOtaPartition = ( + partition: AndroidOtaPartition, + blockSize: number, + readBlob: (offset: number, length: number) => Uint8Array, + options: { readonly verifyDeclaredHash?: boolean } = {}, +): Result => { + const steps = reconstructAndroidOtaPartitionSteps( + partition, + blockSize, + readBlob, + options, + ); + let next = steps.next(); + while (!next.done) + next = steps.next( + decodeZstandardFrames(next.value.raw, { + maxOutputLength: next.value.maxOutputLength, + }), + ); + return next.value; +}; + +/** + * Same reconstruction as {@link reconstructAndroidOtaPartition}. REPLACE_ZSTD + * frames yield through `pause` so the caller's abort signal can land. + */ +export const reconstructAndroidOtaPartitionCooperative = ( + partition: AndroidOtaPartition, + blockSize: number, + readBlob: (offset: number, length: number) => Uint8Array, + pause: ZstandardFramePause, + options: { readonly verifyDeclaredHash?: boolean } = {}, +): Promise> => + driveAndroidOtaReconstruction( + reconstructAndroidOtaPartitionSteps( + partition, + blockSize, + readBlob, + options, + ), + (raw, maxOutputLength) => + decodeZstandardFramesCooperative(raw, { maxOutputLength }, pause), + ); diff --git a/src/domain/native/elfCompressedSection.test.ts b/src/domain/native/elfCompressedSection.test.ts index b8924411e3..78006e5e3d 100644 --- a/src/domain/native/elfCompressedSection.test.ts +++ b/src/domain/native/elfCompressedSection.test.ts @@ -1,5 +1,12 @@ import { deflateSync, zstdCompressSync } from "node:zlib"; +import { ConfigurationError } from "../configurationErrors.js"; +import { ZSTD_MAX_FRAMES_ENVIRONMENT } from "../compression/zstandardFrameAllowance.js"; import { describe, expect, it } from "vitest"; +import { runWithAnalysisOperationSnapshot } from "../analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../analysisResourceLimits.js"; import { decodeElfCompressedSection, decodeElfGnuZlibSection, @@ -190,6 +197,93 @@ describe("ELF compressed section payloads", () => { }); }); +const zstdFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +it("accepts a Zstandard skippable suffix and rejects a trailing byte", () => { + const payload = zstdCompressSync(plain); + const skip = Buffer.alloc(8); + skip.writeUInt32LE(0x184d2a50, 0); + const decode = (body: Buffer) => + decodeElfCompressedSection( + Buffer.concat([chdr(64, true, 2, plain.length), body]), + 0x800n, + 64, + true, + 1024, + "ELF sample section", + ); + expect(decode(payload)).toMatchObject({ state: "available" }); + expect(decode(Buffer.concat([payload, skip]))).toMatchObject({ + state: "available", + compression: "zstd", + }); + expect(decode(Buffer.concat([payload, Buffer.of(0)]))).toMatchObject({ + state: "malformed", + reason: "trailing_bytes", + }); +}); + +it("keeps an invalid frame allowance out of the malformed path", () => { + const payload = zstdCompressSync(plain); + const bytes = Buffer.concat([chdr(64, true, 2, plain.length), payload]); + const decode = () => + decodeElfCompressedSection( + bytes, + 0x800n, + 64, + true, + 1024, + "ELF sample section", + ); + const previous = process.env[ZSTD_MAX_FRAMES_ENVIRONMENT]; + process.env[ZSTD_MAX_FRAMES_ENVIRONMENT] = "0"; + try { + expect(decode()).toMatchObject({ + state: "configuration", + error: expect.any(ConfigurationError), + }); + expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 8 } }, + decode, + ), + ).toMatchObject({ state: "available" }); + } finally { + if (previous === undefined) delete process.env[ZSTD_MAX_FRAMES_ENVIRONMENT]; + else process.env[ZSTD_MAX_FRAMES_ENVIRONMENT] = previous; + } +}); + +it("refuses a Zstandard section that exceeds zstd.max_frames", () => { + const body = Buffer.concat([zstdFrame(0x61), zstdFrame(0x62)]); + const bytes = Buffer.concat([chdr(64, true, 2, 2), body]); + const decode = () => + decodeElfCompressedSection( + bytes, + 0x800n, + 64, + true, + 1024, + "ELF sample section", + ); + expect(decode()).toMatchObject({ state: "available" }); + expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + decode, + ), + ).toMatchObject({ + state: "work_limit", + reason: "work_limit", + setting: "zstd.max_frames", + value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + }); +}); + describe("ELF compressed section declaration admission", () => { it.each([ [32, false, 0x8000_0000n], diff --git a/src/domain/native/elfCompressedSection.ts b/src/domain/native/elfCompressedSection.ts index 0966903e3f..7dac37cf8a 100644 --- a/src/domain/native/elfCompressedSection.ts +++ b/src/domain/native/elfCompressedSection.ts @@ -1,6 +1,18 @@ import { BinaryReader } from "../binaryContainer.js"; -import { constants, inflateSync, zstdDecompressSync } from "node:zlib"; +import { inflateSync } from "node:zlib"; import { nodeCodecResult } from "../../kernel/nodeCodecResult.js"; +import { + decodeZstandardFrames, + decodeZstandardFramesCooperative, + ZSTANDARD_WINDOW_LOG_MAX, + type ZstandardFrameFailure, + type ZstandardFramePause, +} from "../compression/zstandardExactFrame.js"; +import { + zstandardWorkLimitFacts, + type ZstandardFrameSource, +} from "../compression/zstandardFrameAllowance.js"; +import type { ConfigurationError } from "../configurationErrors.js"; const SHF_COMPRESSED = 0x800n; const ELFCOMPRESS_ZLIB = 1; @@ -21,6 +33,27 @@ export type ElfCompressedSectionResult = readonly message: string; /** Larger of declared output and compressed input read; 0 when no attempt started. */ readonly inflatedBytes: number; + /** Present when the shared Zstandard framer classified the payload. */ + readonly reason?: ZstandardFrameFailure["reason"]; + readonly offset?: number; + readonly code?: string; + } + | { + readonly state: "configuration"; + readonly error: ConfigurationError; + readonly inflatedBytes: number; + } + | { + readonly state: "work_limit"; + readonly reason: "work_limit"; + readonly setting: "zstd.max_frames"; + readonly value: number; + readonly source: ZstandardFrameSource; + readonly minimum: number; + readonly maximum: number; + readonly offset: number; + /** Larger of declared output and compressed input read. */ + readonly inflatedBytes: number; } | { readonly state: "limit"; @@ -70,18 +103,23 @@ const readElfCompressedSectionHeader = ( return { state: "compressed", compressionType, declaredSize, headerBytes }; }; -/** - * Decode one ELF section payload. `subject` is the noun phrase used in - * failure text so existing section-name messages stay stable. - */ -export const decodeElfCompressedSection = ( +type PreparedElfSection = + | ElfCompressedSectionResult + | { + readonly state: "payload"; + readonly payload: Buffer; + readonly declaredSize: bigint; + readonly compression: "zlib" | "zstd"; + }; + +const prepareElfCompressedSection = ( bytes: Buffer, flags: bigint, bits: 32 | 64, little: boolean, maximumDecodedBytes: number, subject: string, -): ElfCompressedSectionResult => { +): PreparedElfSection => { const header = readElfCompressedSectionHeader( bytes, flags, @@ -106,12 +144,109 @@ export const decodeElfCompressedSection = ( requiredBytes: declaredSize, maximumBytes: maximumDecodedBytes, }; - return inflateDeclared( - bytes.subarray(headerBytes), + return { + state: "payload", + payload: bytes.subarray(headerBytes), declaredSize, - compressionType === ELFCOMPRESS_ZSTD ? "zstd" : "zlib", + compression: compressionType === ELFCOMPRESS_ZSTD ? "zstd" : "zlib", + }; +}; + +const zstdDecodeOptions = (expected: number) => ({ + maxOutputLength: Math.max(1, expected), + // Retain Zstandard's default maximum decoder window of 128 MiB. + windowLogMax: ZSTANDARD_WINDOW_LOG_MAX, +}); + +const finishInflated = ( + decoded: Buffer, + expected: number, + compression: "zlib" | "zstd", + subject: string, + inflatedBytes: number, +): ElfCompressedSectionResult => { + if (decoded.length !== expected) + return { + state: "malformed", + message: `${subject} size is inconsistent`, + inflatedBytes, + }; + return { + state: "available", + bytes: decoded, + compressed: true, + compression, + inflatedBytes, + }; +}; + +/** + * Decode one ELF section payload. `subject` is the noun phrase used in + * failure text so existing section-name messages stay stable. + */ +export const decodeElfCompressedSection = ( + bytes: Buffer, + flags: bigint, + bits: 32 | 64, + little: boolean, + maximumDecodedBytes: number, + subject: string, +): ElfCompressedSectionResult => { + const prepared = prepareElfCompressedSection( + bytes, + flags, + bits, + little, + maximumDecodedBytes, subject, ); + if (prepared.state !== "payload") return prepared; + return inflateDeclared( + prepared.payload, + prepared.declaredSize, + prepared.compression, + subject, + ); +}; + +/** + * Same admission as {@link decodeElfCompressedSection}. Zstandard frames use + * the cooperative decoder so `pause` can observe the caller's abort signal. + */ +export const decodeElfCompressedSectionCooperative = async ( + bytes: Buffer, + flags: bigint, + bits: 32 | 64, + little: boolean, + maximumDecodedBytes: number, + subject: string, + pause: ZstandardFramePause, +): Promise => { + const prepared = prepareElfCompressedSection( + bytes, + flags, + bits, + little, + maximumDecodedBytes, + subject, + ); + if (prepared.state !== "payload") return prepared; + if (prepared.compression !== "zstd") + return inflateDeclared( + prepared.payload, + prepared.declaredSize, + prepared.compression, + subject, + ); + const expected = Number(prepared.declaredSize); + const inflatedBytes = Math.max(expected, prepared.payload.length); + const result = await decodeZstandardFramesCooperative( + prepared.payload, + zstdDecodeOptions(expected), + pause, + ); + if (!result.ok) return zstdSectionFailure(result, subject, inflatedBytes); + return finishInflated(result.bytes, expected, "zstd", subject, inflatedBytes); }; /** @@ -140,6 +275,45 @@ export const decodeElfGnuZlibSection = ( return inflateDeclared(bytes.subarray(12), declaredSize, "zlib", subject); }; +const zstdSectionFailure = ( + result: ZstandardFrameFailure, + subject: string, + inflatedBytes: number, +): ElfCompressedSectionResult => { + if (result.reason === "invalid_allowance" && result.configuration) + return { + state: "configuration", + error: result.configuration, + inflatedBytes, + }; + if (result.reason === "work_limit") + return { + state: "work_limit", + ...zstandardWorkLimitFacts({ + maxFrames: result.value ?? 0, + source: result.source ?? "default", + }), + offset: result.offset, + inflatedBytes, + }; + if (result.reason === "output_limit") + return { + state: "malformed", + reason: "output_limit", + offset: result.offset, + message: `${subject} size is inconsistent`, + inflatedBytes, + }; + return { + state: "malformed", + reason: result.reason, + offset: result.offset, + ...(result.code === undefined ? {} : { code: result.code }), + message: `${subject} could not be decompressed: ${result.detail ?? result.code ?? result.reason}`, + inflatedBytes, + }; +}; + const inflateDeclared = ( payload: Buffer, declaredSize: bigint, @@ -152,11 +326,12 @@ const inflateDeclared = ( try { const maxOutputLength = Math.max(1, expected); if (compression === "zstd") { - decoded = zstdDecompressSync(payload, { - maxOutputLength, - // Retain Zstandard's default maximum decoder window of 128 MiB. - params: { [constants.ZSTD_d_windowLogMax]: 27 }, - }); + const result = decodeZstandardFrames( + payload, + zstdDecodeOptions(expected), + ); + if (!result.ok) return zstdSectionFailure(result, subject, inflatedBytes); + decoded = result.bytes; } else { const result = nodeCodecResult( inflateSync(payload, { info: true, maxOutputLength }), @@ -176,17 +351,5 @@ const inflateDeclared = ( message: `${subject} could not be decompressed: ${cause instanceof Error ? cause.message : "invalid compressed data"}`, }; } - if (decoded.length !== expected) - return { - state: "malformed", - message: `${subject} size is inconsistent`, - inflatedBytes, - }; - return { - state: "available", - bytes: decoded, - compressed: true, - compression, - inflatedBytes, - }; + return finishInflated(decoded, expected, compression, subject, inflatedBytes); }; diff --git a/src/domain/native/elfImage.ts b/src/domain/native/elfImage.ts index 5044c38523..850b847aff 100644 --- a/src/domain/native/elfImage.ts +++ b/src/domain/native/elfImage.ts @@ -10,17 +10,18 @@ import { type LoadRegion, } from "../binaryContainer.js"; import { elfArchitecture } from "../elfFixedHeader.js"; -import { readElfDirectoryFromBuffer } from "./nativeSectionDirectory.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; +import type { CooperativeElfSectionNameDecoder } from "./elfSectionNameTable.js"; +import { + readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, + type ElfSectionDirectory, +} from "./nativeSectionDirectory.js"; -/** Read ELF sections and file-backed program mappings without interpreting language metadata. */ -export const readElfImage = ( +export const readElfImageFromDirectory = ( bytes: Buffer, - maximumSourceAndDecodedNamesBytes?: number, + directory: ElfSectionDirectory, ): Extract => { - const directory = readElfDirectoryFromBuffer( - bytes, - maximumSourceAndDecodedNamesBytes, - ); const { bits } = directory; const reader = new BinaryReader(bytes, directory.little, "elf"); const mappings: FileMapping[] = []; @@ -138,3 +139,37 @@ export const architectureForElfMachine = (machine: number): string => { return `elf-machine-${machine}`; } }; + +/** Read ELF sections and file-backed program mappings without interpreting language metadata. */ +export const readElfImage = ( + bytes: Buffer, + operation: string, + maximumSourceAndDecodedNamesBytes?: number, +): Extract => + readElfImageFromDirectory( + bytes, + readElfDirectoryFromBuffer( + bytes, + operation, + maximumSourceAndDecodedNamesBytes, + ), + ); + +/** Same image as {@link readElfImage}. Section names yield through `pause`. */ +export const readElfImageCooperative = async ( + bytes: Buffer, + operation: string, + pause: ZstandardFramePause, + maximumSourceAndDecodedNamesBytes?: number, + decode?: CooperativeElfSectionNameDecoder, +): Promise> => + readElfImageFromDirectory( + bytes, + await readElfDirectoryFromBufferCooperative( + bytes, + pause, + operation, + maximumSourceAndDecodedNamesBytes, + decode, + ), + ); diff --git a/src/domain/native/elfSectionNameTable.ts b/src/domain/native/elfSectionNameTable.ts index e93807d0e1..497804db9a 100644 --- a/src/domain/native/elfSectionNameTable.ts +++ b/src/domain/native/elfSectionNameTable.ts @@ -1,8 +1,10 @@ import { decodeLimit, ParseFailureError } from "../parseFailure.js"; import { decodeElfCompressedSection, + decodeElfCompressedSectionCooperative, type ElfCompressedSectionResult, } from "./elfCompressedSection.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; const SUBJECT = "ELF compressed section-name table"; @@ -50,6 +52,19 @@ export const elfSectionNameBudget = ( }; }; +/** One section-name table a directory or note driver asks the owner to decode. */ +export interface ElfSectionNameDecodeRequest { + readonly raw: Buffer; + readonly flags: bigint; + readonly bits: 32 | 64; + readonly little: boolean; + readonly maximumDecodedBytes: number; +} + +export type CooperativeElfSectionNameDecoder = ( + request: ElfSectionNameDecodeRequest, +) => Promise; + /** Decode section names through the shared ELF compressed-section owner. */ export const decodeElfSectionNameTable = ( bytes: Buffer, @@ -66,3 +81,85 @@ export const decodeElfSectionNameTable = ( maximumDecodedBytes, SUBJECT, ); + +/** Cooperative section-name decode. `pause` observes the caller's abort signal. */ +export const decodeElfSectionNameTableCooperative = ( + bytes: Buffer, + flags: bigint, + bits: 32 | 64, + little: boolean, + maximumDecodedBytes: number, + pause: ZstandardFramePause, +): Promise => + decodeElfCompressedSectionCooperative( + bytes, + flags, + bits, + little, + maximumDecodedBytes, + SUBJECT, + pause, + ); + +export const decodeElfSectionNameRequest = ( + request: ElfSectionNameDecodeRequest, +): ElfCompressedSectionResult => + decodeElfSectionNameTable( + request.raw, + request.flags, + request.bits, + request.little, + request.maximumDecodedBytes, + ); + +export const decodeElfSectionNameRequestCooperative = ( + request: ElfSectionNameDecodeRequest, + pause: ZstandardFramePause, +): Promise => + decodeElfSectionNameTableCooperative( + request.raw, + request.flags, + request.bits, + request.little, + request.maximumDecodedBytes, + pause, + ); + +/** Reuse a decoded table across readers of one borrowed immutable artifact. */ +export const sharedElfSectionNameDecoder = ( + pause: ZstandardFramePause, +): CooperativeElfSectionNameDecoder => { + let admitted: + | { + request: ElfSectionNameDecodeRequest; + decoded: Extract< + ElfCompressedSectionResult, + { state: "available" | "unsupported" } + >; + } + | undefined; + return async (request) => { + if (admitted !== undefined) { + const { request: previous, decoded } = admitted; + if ( + request.raw.buffer === previous.raw.buffer && + request.raw.byteOffset === previous.raw.byteOffset && + request.raw.byteLength === previous.raw.byteLength && + request.bits === previous.bits && + request.little === previous.little && + request.flags === previous.flags && + (decoded.state === "unsupported" || + !decoded.compressed || + decoded.bytes.length <= request.maximumDecodedBytes) + ) + return decoded; + } + const decoded = await decodeElfSectionNameRequestCooperative( + request, + pause, + ); + if (decoded.state === "available" || decoded.state === "unsupported") + admitted = { request, decoded }; + return decoded; + }; +}; diff --git a/src/domain/native/nativeExportParser.ts b/src/domain/native/nativeExportParser.ts index 306f4190b5..63c4412f3b 100644 --- a/src/domain/native/nativeExportParser.ts +++ b/src/domain/native/nativeExportParser.ts @@ -12,8 +12,10 @@ import { readElfStringBytes } from "./elfStringBytes.js"; import { elfSymbolSize, readElfSymbol } from "../elfSymbol.js"; import { readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, type NativeSection, } from "./nativeSectionDirectory.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; import { peDeclaredFileMappings, readPeImage } from "./peImage.js"; import { allKeyed, @@ -32,6 +34,10 @@ import { type Artifact = NativeExports["artifact"]; +const ELF_MAGIC = Buffer.from([0x7f, 0x45, 0x4c, 0x46]); +const hasElfMagic = (bytes: Buffer): boolean => + bytes.subarray(0, ELF_MAGIC.length).equals(ELF_MAGIC); +const OPERATION = "inspect_native_exports"; const SHT_STRTAB = 3; const SHT_DYNSYM = 11; const SHT_GNU_VERDEF = 0x6ffffffd; @@ -668,12 +674,12 @@ function* readVersionNames( return namesByIndex; } -/** Read defined exported entries of `.dynsym` with their symbol versions. */ -function* parseElfExports( +/** Read ELF exports and versions from an already-admitted directory for these bytes. */ +export function* parseElfExportsFromDirectorySteps( bytes: Buffer, artifact: Artifact, -): Generator { - const directory = readElfDirectoryFromBuffer(bytes); + directory: ReturnType, +): Generator> { const reader = new BinaryReader(bytes, directory.little, "elf"); const result = { artifact, @@ -930,8 +936,12 @@ export function* parseNativeExportsSteps( artifact: Artifact, ): Generator { yield; - if (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) - return yield* parseElfExports(bytes, artifact); + if (hasElfMagic(bytes)) + return yield* parseElfExportsFromDirectorySteps( + bytes, + artifact, + readElfDirectoryFromBuffer(bytes, OPERATION), + ); if (bytes.subarray(0, 2).equals(Buffer.from("MZ"))) return parsePeExports(bytes, artifact); throw new BinaryFormatFailure( @@ -944,8 +954,32 @@ export function* parseNativeExportsSteps( export const parseNativeExports = ( bytes: Buffer, artifact: Artifact, +): NativeExports => finishExports(parseNativeExportsSteps(bytes, artifact)); + +/** Yield during ELF section-name decoding and bounded export inventory work. */ +export const parseNativeExportsCooperative = async ( + bytes: Buffer, + artifact: Artifact, + pause: ZstandardFramePause, +): Promise => { + if (!hasElfMagic(bytes)) return parseNativeExports(bytes, artifact); + const directory = await readElfDirectoryFromBufferCooperative( + bytes, + pause, + OPERATION, + ); + const steps = parseElfExportsFromDirectorySteps(bytes, artifact, directory); + let step = steps.next(); + while (!step.done) { + await pause(); + step = steps.next(); + } + return step.value; +}; + +const finishExports = ( + steps: Generator, ): NativeExports => { - const steps = parseNativeExportsSteps(bytes, artifact); let step = steps.next(); while (!step.done) step = steps.next(); return step.value; diff --git a/src/domain/native/nativeSectionDirectory.ts b/src/domain/native/nativeSectionDirectory.ts index f4e3a0c2b8..15f1b0233f 100644 --- a/src/domain/native/nativeSectionDirectory.ts +++ b/src/domain/native/nativeSectionDirectory.ts @@ -28,14 +28,29 @@ import { type ElfDynamicSegment, } from "../binaryContainer.js"; import { - decodeElfSectionNameTable, + decodeElfSectionNameRequest, + decodeElfSectionNameRequestCooperative, elfSectionNameBudgetFailure, + type CooperativeElfSectionNameDecoder, + type ElfSectionNameDecodeRequest, } from "./elfSectionNameTable.js"; +import type { ElfCompressedSectionResult } from "./elfCompressedSection.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../compression/zstandardFrameAllowance.js"; /** A borrowed positional source. Drivers own admission, cancellation and file identity. */ export interface NativeReadRequest { readonly offset: number; readonly size: number; + /** + * When set, the driver decodes this section-name table and calls + * `fulfillSectionNames` before resuming. A byte read does not satisfy it. + */ + readonly sectionNames?: ElfSectionNameDecodeRequest; + readonly fulfillSectionNames?: (result: ElfCompressedSectionResult) => void; } export interface NativeSection { readonly index: number; @@ -102,7 +117,7 @@ export interface NativeSectionDirectory { readonly programs: readonly ElfProgramDeclaration[]; readonly sections: readonly NativeSection[]; } -type ElfSectionDirectory = NativeSectionDirectory & { +export type ElfSectionDirectory = NativeSectionDirectory & { readonly format: "elf"; readonly elf_type: number; }; @@ -111,6 +126,13 @@ const STRUCTURE_BYTES = 16 * 1024 * 1024; const reject = (message: string): never => { throw new BinaryFormatFailure("malformed", message); }; +const requireSectionNames = ( + result: ElfCompressedSectionResult | undefined, +): ElfCompressedSectionResult => { + if (result === undefined) + return reject("ELF section-name decode was not satisfied"); + return result; +}; const exact = (value: bigint): number => { if (value < 0n || value > BigInt(Number.MAX_SAFE_INTEGER)) reject("Native offset exceeds exact host precision"); @@ -247,34 +269,50 @@ export function* readElfSectionDirectory( maximumSourceAndDecodedNamesBytes === undefined ? structuralRemaining : Math.max(0, maximumSourceAndDecodedNamesBytes - fileSize); - const decoded = decodeElfSectionNameTable( - rawNames, - namesSection.flags, + let decoded: ElfCompressedSectionResult | undefined; + const sectionNames: ElfSectionNameDecodeRequest = { + raw: rawNames, + flags: namesSection.flags, bits, little, - Math.min(structuralRemaining, callerRemaining), - ); - if (decoded.state === "malformed") return reject(decoded.message); + maximumDecodedBytes: Math.min(structuralRemaining, callerRemaining), + }; + yield { + offset: 0, + size: 0, + sectionNames, + fulfillSectionNames: (result) => { + decoded = result; + }, + }; + const admitted = requireSectionNames(decoded); + if (admitted.state === "malformed") return reject(admitted.message); + if (admitted.state === "configuration") throw admitted.error; + if (admitted.state === "work_limit") + throw new ZstandardFrameAllowanceError( + { maxFrames: admitted.value, source: admitted.source }, + admitted.offset, + ); if ( - decoded.state === "limit" && + admitted.state === "limit" && maximumSourceAndDecodedNamesBytes !== undefined && - decoded.requiredBytes > BigInt(callerRemaining) + admitted.requiredBytes > BigInt(callerRemaining) ) throw elfSectionNameBudgetFailure( - BigInt(fileSize) + decoded.requiredBytes, + BigInt(fileSize) + admitted.requiredBytes, maximumSourceAndDecodedNamesBytes, ); - if (decoded.state === "limit") + if (admitted.state === "limit") throw new BinaryResourceFailure( "structure", STRUCTURE_BYTES, - `Decoded ELF section names require ${String(decoded.requiredBytes)} bytes, above the ${String(structuralRemaining)}-byte remaining structural budget`, + `Decoded ELF section names require ${String(admitted.requiredBytes)} bytes, above the ${String(structuralRemaining)}-byte remaining structural budget`, "structure", "bytes", ); - if (decoded.state === "unsupported") - section_names_limitation = `ELF section names are unavailable because compression type ${String(decoded.compressionType)} is unsupported`; - else names = decoded.bytes; + if (admitted.state === "unsupported") + section_names_limitation = `ELF section names are unavailable because compression type ${String(admitted.compressionType)} is unsupported`; + else names = admitted.bytes; } else if (count > 1) { section_names_limitation = "ELF section names are unavailable because no section-name string table is declared"; @@ -476,27 +514,110 @@ export function* readPeSectionDirectory( }; } +const resumeDirectoryStep = ( + bytes: Buffer, + step: NativeReadRequest, + names: (request: ElfSectionNameDecodeRequest) => ElfCompressedSectionResult, +): Buffer => { + if ( + step.sectionNames !== undefined && + step.fulfillSectionNames !== undefined + ) { + step.fulfillSectionNames(names(step.sectionNames)); + return Buffer.alloc(0); + } + return bytes.subarray(step.offset, step.offset + step.size); +}; + const driveFromBuffer = ( bytes: Buffer, steps: Generator, ): T => { let next = steps.next(); while (!next.done) { - const { offset, size } = next.value; - next = steps.next(bytes.subarray(offset, offset + size)); + next = steps.next( + resumeDirectoryStep(bytes, next.value, decodeElfSectionNameRequest), + ); } return next.value; }; -/** Borrow ELF bytes; file drivers satisfy the identical requests without loading payloads. */ +const driveElfDirectoryCooperative = async ( + bytes: Buffer, + decode: CooperativeElfSectionNameDecoder, + maximumSourceAndDecodedNamesBytes?: number, +): Promise => { + const steps = readElfSectionDirectory( + bytes.length, + maximumSourceAndDecodedNamesBytes, + ); + let next = steps.next(); + while (!next.done) { + const step = next.value; + if ( + step.sectionNames !== undefined && + step.fulfillSectionNames !== undefined + ) { + step.fulfillSectionNames(await decode(step.sectionNames)); + next = steps.next(Buffer.alloc(0)); + continue; + } + next = steps.next(bytes.subarray(step.offset, step.offset + step.size)); + } + return next.value; +}; + +const projectDirectoryFrameAllowance = ( + operation: string, + cause: unknown, +): never => { + if (cause instanceof ZstandardFrameAllowanceError) + throw projectZstandardFrameAllowance(operation, cause); + throw cause; +}; + +/** + * Borrow ELF bytes. `operation` names the public refusal when a compressed + * section-name table exceeds `zstd.max_frames`, so every caller projects that + * adjustable limit instead of an I/O or adapter failure. + */ export const readElfDirectoryFromBuffer = ( bytes: Buffer, + operation: string, maximumSourceAndDecodedNamesBytes?: number, -): ElfSectionDirectory => - driveFromBuffer( - bytes, - readElfSectionDirectory(bytes.length, maximumSourceAndDecodedNamesBytes), - ); +): ElfSectionDirectory => { + try { + return driveFromBuffer( + bytes, + readElfSectionDirectory(bytes.length, maximumSourceAndDecodedNamesBytes), + ); + } catch (cause) { + return projectDirectoryFrameAllowance(operation, cause); + } +}; + +/** + * Same directory as {@link readElfDirectoryFromBuffer}. Compressed section + * names yield through `pause` so the caller's abort signal can land. + */ +export const readElfDirectoryFromBufferCooperative = async ( + bytes: Buffer, + pause: ZstandardFramePause, + operation: string, + maximumSourceAndDecodedNamesBytes?: number, + decode: CooperativeElfSectionNameDecoder = (request) => + decodeElfSectionNameRequestCooperative(request, pause), +): Promise => { + try { + return await driveElfDirectoryCooperative( + bytes, + decode, + maximumSourceAndDecodedNamesBytes, + ); + } catch (cause) { + return projectDirectoryFrameAllowance(operation, cause); + } +}; /** Reject a program or section payload the declared tables place outside these bytes. */ export const assertElfDeclaredRangesFromBuffer = (bytes: Buffer): void => { diff --git a/src/domain/native/readBinaryContainer.ts b/src/domain/native/readBinaryContainer.ts index b9b0fb670f..246dfc1925 100644 --- a/src/domain/native/readBinaryContainer.ts +++ b/src/domain/native/readBinaryContainer.ts @@ -8,14 +8,35 @@ import { readFatHeaderPrefix, readMachoHeaderPrefixFacts, } from "../apple/machoContainer.js"; -import { readElfImage } from "./elfImage.js"; +import { readElfImage, readElfImageCooperative } from "./elfImage.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; import { readPeImage } from "./peImage.js"; import { readMachoImage } from "../apple/machoImage.js"; +const ELF_MAGIC = Buffer.from([0x7f, 0x45, 0x4c, 0x46]); +const hasElfMagic = (bytes: Buffer): boolean => + bytes.subarray(0, ELF_MAGIC.length).equals(ELF_MAGIC); + /** Identify an ELF, PE, or thin Mach-O image and its file sections. */ -export const readBinaryContainer = (bytes: Buffer): BinaryContainer => { - if (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) - return readElfImage(bytes); +export const readBinaryContainer = ( + bytes: Buffer, + operation: string, +): BinaryContainer => + hasElfMagic(bytes) + ? readElfImage(bytes, operation) + : readNonElfContainer(bytes); + +/** Read the same facts, allowing the operation to pause during ELF name decoding. */ +export const readBinaryContainerCooperative = async ( + bytes: Buffer, + operation: string, + pause: ZstandardFramePause, +): Promise => + hasElfMagic(bytes) + ? readElfImageCooperative(bytes, operation, pause) + : readNonElfContainer(bytes); + +const readNonElfContainer = (bytes: Buffer): BinaryContainer => { if (bytes.subarray(0, 2).equals(Buffer.from("MZ"))) return readPeImage(bytes); if (bytes.length >= 4) { const fatPrefix = readFatHeaderPrefix(bytes); diff --git a/src/go/GoBinaryImage.ts b/src/go/GoBinaryImage.ts index 996d5f0da6..361a856ce3 100644 --- a/src/go/GoBinaryImage.ts +++ b/src/go/GoBinaryImage.ts @@ -13,7 +13,11 @@ import { isZeroFilledSpan, type BinaryContainer, } from "../domain/binaryContainer.js"; -import { readBinaryContainer } from "../domain/native/readBinaryContainer.js"; +import type { ZstandardFramePause } from "../domain/compression/zstandardExactFrame.js"; +import { + readBinaryContainer, + readBinaryContainerCooperative, +} from "../domain/native/readBinaryContainer.js"; import { goAnalysisBudgets, type GoAnalysisBudgets, @@ -58,9 +62,31 @@ export interface ReadGoBinaryImageOptions { export const readGoBinaryImage = ( bytes: Buffer, options?: ReadGoBinaryImageOptions, +): GoBinaryImage => + finishGoBinaryImage( + bytes, + options, + readBinaryContainer(bytes, "inspect_go_binary"), + ); + +/** Same image as {@link readGoBinaryImage}. ELF section names yield through `pause`. */ +export const readGoBinaryImageCooperative = async ( + bytes: Buffer, + options: ReadGoBinaryImageOptions | undefined, + pause: ZstandardFramePause, +): Promise => + finishGoBinaryImage( + bytes, + options, + await readBinaryContainerCooperative(bytes, "inspect_go_binary", pause), + ); + +const finishGoBinaryImage = ( + bytes: Buffer, + options: ReadGoBinaryImageOptions | undefined, + container: ReturnType, ): GoBinaryImage => { const budgets = goAnalysisBudgets(options?.budgets); - const container = readBinaryContainer(bytes); const { format, architecture, bits, byte_order } = container; const stringLiterals = options?.includeStringLiterals === true diff --git a/src/go/GoBinaryProvider.ts b/src/go/GoBinaryProvider.ts index 665ac56d0f..ed1f96a26c 100644 --- a/src/go/GoBinaryProvider.ts +++ b/src/go/GoBinaryProvider.ts @@ -17,6 +17,7 @@ import { artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, assertArtifactReadActive, + yieldArtifactRead, ArtifactResourceScope, ArtifactReaderFailure, } from "../resources/index.js"; @@ -40,7 +41,11 @@ import { import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { err, ok, type Result } from "../domain/result.js"; -import { readGoBinaryImage, type GoBinaryImage } from "./GoBinaryImage.js"; +import { + readGoBinaryImage, + readGoBinaryImageCooperative, + type GoBinaryImage, +} from "./GoBinaryImage.js"; import { requireSelectedNativeImage } from "../domain/apple/machoSliceSelection.js"; import { BinaryResourceFailure, @@ -333,11 +338,19 @@ export class GoBinaryProvider implements GoBinaryPort { snapshot.bytes, input.architecture, ); - const image = readGoBinaryImage(selected.bytes, { + const imageOptions = { includeStringLiterals: input.include_string_literals === true, budgets, checkActive: () => assertArtifactReadActive(options?.signal), - }); + }; + const image = + options?.signal === undefined + ? readGoBinaryImage(selected.bytes, imageOptions) + : await readGoBinaryImageCooperative( + selected.bytes, + imageOptions, + () => yieldArtifactRead(options.signal), + ); const { section_names_limitation: _sectionNamesLimitation, macho_chained_fixups: _machoChainedFixups, diff --git a/src/gpu/GpuCodeObjectsProvider.ts b/src/gpu/GpuCodeObjectsProvider.ts index 38db322a97..f4f3ee7ec7 100644 --- a/src/gpu/GpuCodeObjectsProvider.ts +++ b/src/gpu/GpuCodeObjectsProvider.ts @@ -34,6 +34,10 @@ import { carriedReason, } from "../domain/analysisErrorCore.js"; import { parseFailureAnalysisError } from "../domain/parseFailureAnalysisError.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../domain/compression/zstandardFrameAllowance.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; @@ -114,8 +118,19 @@ export class GpuCodeObjectsProvider implements GpuCodeObjectsPort { input.path, primary, ); + const frameAllowance = + primary instanceof ZstandardFrameAllowanceError + ? primary + : cause instanceof ZstandardFrameAllowanceError + ? cause + : undefined; let error: AnalysisError; - if (origin === "owned-or-classified" && primary instanceof AnalysisError) + if (frameAllowance !== undefined) + error = projectZstandardFrameAllowance(OPERATION, frameAllowance); + else if ( + origin === "owned-or-classified" && + primary instanceof AnalysisError + ) error = primary; else if (primary instanceof BinaryResourceFailure) error = new AnalysisResourceConstraintError( diff --git a/src/gpu/GpuElfKernels.ts b/src/gpu/GpuElfKernels.ts index dbca939186..37df71fae4 100644 --- a/src/gpu/GpuElfKernels.ts +++ b/src/gpu/GpuElfKernels.ts @@ -17,6 +17,7 @@ import { readCudaParameters, } from "./CudaInfo.js"; import type { GpuKernel, GpuLocation } from "../domain/gpu/gpuCodeObjects.js"; +import type { ZstandardFramePause } from "../domain/compression/zstandardExactFrame.js"; import { BinaryReader, BinaryFormatFailure, @@ -100,6 +101,7 @@ export const readGpuElfKernels = async ( source: GpuSource, kind: "cubin" | "amdgpu", onKernel: (kernel: GpuKernel) => void, + pause: ZstandardFramePause, preparedAmd?: AmdElfMetadata, onTarget?: (target: GpuPayloadTarget) => void, ): Promise<{ @@ -111,6 +113,7 @@ export const readGpuElfKernels = async ( const directory = await readGpuNativeDirectory( source, readElfSectionDirectory(source.size), + pause, ); if (directory.machine !== (kind === "cubin" ? 190 : 224)) rejectGpu("GPU entry kind disagrees with ELF machine"); diff --git a/src/gpu/GpuInventory.ts b/src/gpu/GpuInventory.ts index 544620a06c..8d5ac0c3a5 100644 --- a/src/gpu/GpuInventory.ts +++ b/src/gpu/GpuInventory.ts @@ -22,6 +22,11 @@ import { import { readCudaTarget } from "./CudaTarget.js"; import { readCudaFatbinary } from "./CudaFatbinary.js"; import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../domain/compression/zstandardFrameAllowance.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; import { createHash } from "node:crypto"; import { scheduler } from "node:timers/promises"; import { @@ -35,7 +40,10 @@ import { readPeSectionDirectory, type NativeSectionDirectory, } from "../domain/native/nativeSectionDirectory.js"; -import { assertArtifactReadActive } from "../resources/index.js"; +import { + assertArtifactReadActive, + yieldArtifactRead, +} from "../resources/index.js"; import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { @@ -179,6 +187,7 @@ export class GpuInventory { elf ? readElfSectionDirectory(source.size) : readPeSectionDirectory(source.size), + () => yieldArtifactRead(this.signal), ); if (elf && (directory.machine === 190 || directory.machine === 224)) { const row = gpuObject(source, { @@ -295,6 +304,9 @@ export class GpuInventory { } #retainFailure(row: GpuObject, cause: unknown): void { if (cause instanceof GpuReportCapacityFailure) throw cause; + if (cause instanceof ZstandardFrameAllowanceError) + throw projectZstandardFrameAllowance("inspect_gpu_code_objects", cause); + if (cause instanceof AnalysisError) throw cause; const capacity = cause instanceof BinaryResourceFailure || isBinaryStringReadLimit(cause); if (!capacity && !(cause instanceof BinaryFormatFailure)) throw cause; @@ -358,7 +370,16 @@ export class GpuInventory { this.signal, ); } catch (cause) { - if (cause instanceof BinaryResourceFailure) throw cause; + if (cause instanceof ZstandardFrameAllowanceError) + throw projectZstandardFrameAllowance( + "inspect_gpu_code_objects", + cause, + ); + if ( + cause instanceof BinaryResourceFailure || + cause instanceof AnalysisError + ) + throw cause; assertArtifactReadActive(this.signal); throw new BinaryFormatFailure( "malformed", @@ -515,6 +536,7 @@ export class GpuInventory { (await readGpuNativeDirectory( child, readElfSectionDirectory(child.size), + () => yieldArtifactRead(this.signal), )); const inspected = await readAmdElfMetadata(child, native); const prepared = inspected.ok ? inspected.value : inspected.error.partial; @@ -558,6 +580,7 @@ export class GpuInventory { this.#retain(kernel); this.kernels.push(kernel); }, + () => yieldArtifactRead(this.signal), preparedAmd, (target) => { conflict = this.#target(row, target); diff --git a/src/gpu/GpuPayloadCodec.test.ts b/src/gpu/GpuPayloadCodec.test.ts index 6b1f11f9aa..e399e07599 100644 --- a/src/gpu/GpuPayloadCodec.test.ts +++ b/src/gpu/GpuPayloadCodec.test.ts @@ -1,5 +1,10 @@ import { expect, it } from "vitest"; import { deflateSync, zstdCompressSync } from "node:zlib"; +import { runWithAnalysisOperationSnapshot } from "../domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../domain/analysisResourceLimits.js"; import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; it.each(["zlib", "zstd"] as const)( "requires all declared %s input bytes to belong to the codec stream", @@ -22,6 +27,49 @@ it.each(["zlib", "zstd"] as const)( ).rejects.toMatchObject({ kind: "malformed" }); }, ); +const zstdByteFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +it("stops a many-frame Zstandard payload when the read is cancelled", async () => { + const frame = zstdByteFrame(0x61); + const count = 32; + const bytes = Buffer.alloc(frame.length * count); + for (let index = 0; index < count; index += 1) + frame.copy(bytes, index * frame.length); + const controller = new AbortController(); + const pending = decodeGpuPayload(bytes, "zstd", count, controller.signal); + controller.abort(); + await expect(pending).rejects.toMatchObject({ reason: "cancelled" }); +}); + +it("decodes every Zstandard frame before a container suffix", async () => { + const frames = Buffer.concat([zstdByteFrame(0x61), zstdByteFrame(0x62)]); + const plain = Buffer.from([0x61, 0x62]); + expect(Buffer.compare(await decodeGpuPayload(frames, "zstd", 2), plain)).toBe( + 0, + ); + const streamed = await decodeGpuPayloadStream( + Buffer.concat([frames, Buffer.from("CCOB")]), + "zstd", + 2, + ); + expect(Buffer.compare(streamed.buffer, plain)).toBe(0); + expect(streamed.consumedBytes).toBe(frames.length); + await expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => decodeGpuPayload(frames, "zstd", 2), + ), + ).rejects.toMatchObject({ + reason: "work_limit", + setting: "zstd.max_frames", + value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + }); +}); + it.each(["zlib", "zstd"] as const)( "retains completed %s input accounting before trailing container bytes", async (codec) => { diff --git a/src/gpu/GpuPayloadCodec.ts b/src/gpu/GpuPayloadCodec.ts index fc7b9417eb..4e2f74951c 100644 --- a/src/gpu/GpuPayloadCodec.ts +++ b/src/gpu/GpuPayloadCodec.ts @@ -1,11 +1,17 @@ import { nodeCodecResult } from "../kernel/nodeCodecResult.js"; -import { constants, inflate, zstdDecompress } from "node:zlib"; +import { inflate } from "node:zlib"; +import { scheduler } from "node:timers/promises"; import { promisify } from "node:util"; import { assertArtifactReadActive } from "../resources/index.js"; import { decodeLz4Block } from "../compression/Lz4Block.js"; +import { + decodeZstandardFramesCooperative, + ZSTANDARD_WINDOW_LOG_MAX, +} from "../domain/compression/zstandardExactFrame.js"; +import { ZstandardFrameAllowanceError } from "../domain/compression/zstandardFrameAllowance.js"; +import { BinaryFormatFailure } from "../domain/binaryContainer.js"; import { GPU_OBJECT_BYTES, rejectGpu } from "./GpuSource.js"; const unzip = promisify(inflate); -const unzstd = promisify(zstdDecompress); /** Completed codec bytes and their encoded extent share one completion owner. */ export const decodeGpuPayloadStream = async ( bytes: Buffer, @@ -14,16 +20,50 @@ export const decodeGpuPayloadStream = async ( signal?: AbortSignal, ): Promise<{ buffer: Buffer; consumedBytes: number }> => { assertArtifactReadActive(signal); + const maxOutputLength = Math.max(1, maximumDecodedSize); + if (compression === "zstd") { + const decoded = await decodeZstandardFramesCooperative( + bytes, + { + maxOutputLength, + windowLogMax: ZSTANDARD_WINDOW_LOG_MAX, + mode: "prefix", + }, + async () => { + await scheduler.yield(); + assertArtifactReadActive(signal); + }, + ); + assertArtifactReadActive(signal); + if (!decoded.ok) { + if ( + decoded.reason === "invalid_allowance" && + decoded.configuration !== undefined + ) + throw decoded.configuration; + if (decoded.reason === "work_limit") + throw new ZstandardFrameAllowanceError( + { + maxFrames: decoded.value ?? 0, + source: decoded.source ?? "default", + }, + decoded.offset, + ); + throw new BinaryFormatFailure( + "malformed", + "GPU payload decompression failed or exceeded its declared length", + { + format: "zstd", + field: decoded.reason, + offset: decoded.offset, + }, + ); + } + return { buffer: decoded.bytes, consumedBytes: decoded.consumedBytes }; + } let result: unknown; try { - const maxOutputLength = Math.max(1, maximumDecodedSize); - result = await (compression === "zstd" - ? unzstd(bytes, { - maxOutputLength, - info: true, - params: { [constants.ZSTD_d_windowLogMax]: 27 }, - }) - : unzip(bytes, { maxOutputLength, info: true })); + result = await unzip(bytes, { maxOutputLength, info: true }); } catch { assertArtifactReadActive(signal); return rejectGpu( diff --git a/src/gpu/GpuSource.ts b/src/gpu/GpuSource.ts index d843500742..63205dade0 100644 --- a/src/gpu/GpuSource.ts +++ b/src/gpu/GpuSource.ts @@ -8,6 +8,8 @@ import { BinaryFormatFailure, BinaryResourceFailure, } from "../domain/binaryContainer.js"; +import { decodeElfSectionNameRequestCooperative } from "../domain/native/elfSectionNameTable.js"; +import type { ZstandardFramePause } from "../domain/compression/zstandardExactFrame.js"; import type { NativeReadRequest, NativeSectionDirectory, @@ -118,11 +120,22 @@ export class GpuBufferSource implements GpuSource { export const readGpuNativeDirectory = async ( source: GpuSource, steps: Generator, + pause: ZstandardFramePause, ): Promise => { let next = steps.next(); while (!next.done) { - const { offset, size } = next.value; - next = steps.next(await source.read(offset, size)); + const step = next.value; + if ( + step.sectionNames !== undefined && + step.fulfillSectionNames !== undefined + ) { + step.fulfillSectionNames( + await decodeElfSectionNameRequestCooperative(step.sectionNames, pause), + ); + next = steps.next(Buffer.alloc(0)); + continue; + } + next = steps.next(await source.read(step.offset, step.size)); } return next.value; }; diff --git a/src/il2cpp/Il2cppBinaryFile.ts b/src/il2cpp/Il2cppBinaryFile.ts index 16102a197e..965eb94df1 100644 --- a/src/il2cpp/Il2cppBinaryFile.ts +++ b/src/il2cpp/Il2cppBinaryFile.ts @@ -1,6 +1,12 @@ import type { Il2cppMethodAddress } from "../domain/il2cpp/il2cppMetadata.js"; import { Il2cppBinaryFormatFailure } from "./Il2cppBinaryFormat.js"; -import { openImageSteps, readSlot, isExecutable } from "./Il2cppBinaryImage.js"; +import { + openImageSteps, + readSlot, + isExecutable, + readIl2cppBinaryContainerCooperative, + type NativeImage, +} from "./Il2cppBinaryImage.js"; import { findModules, findCodeRegistration, @@ -74,6 +80,15 @@ export type Il2cppBinaryReadOptions = Pick< readonly genericInstances?: boolean; }; +const assertGenericLinkLayout = ( + metadata: Il2cppMetadataDecoded, + options: Il2cppBinaryReadOptions, +): void => { + // Refuse an undecoded version before any binary tables are read. + if (options.genericInstances === true) + genericMethodLayouts(metadata.metadata_version); +}; + /** * Link metadata methods to native code addresses without executing the image. * A method token's row id selects its module's methodPointers slot. @@ -84,11 +99,43 @@ export function* linkIl2cppBinarySteps( facts: Il2cppMetadataLinkFacts, options: Il2cppBinaryReadOptions = {}, ): Generator { - const version = metadata.metadata_version; - // Refuse an undecoded version before any table is read. - if (options.genericInstances === true) genericMethodLayouts(version); + assertGenericLinkLayout(metadata, options); yield; - const image = yield* openImageSteps(bytes); + return yield* linkImageSteps(openImageSteps(bytes), metadata, facts, options); +} + +export const linkIl2cppBinaryCooperative = async ( + bytes: Buffer, + metadata: Il2cppMetadataDecoded, + facts: Il2cppMetadataLinkFacts, + options: Il2cppBinaryReadOptions, + pause: () => Promise, +): Promise => { + assertGenericLinkLayout(metadata, options); + await pause(); + const container = await readIl2cppBinaryContainerCooperative(bytes, pause); + const steps = linkImageSteps( + openImageSteps(bytes, container), + metadata, + facts, + options, + ); + let step = steps.next(); + while (!step.done) { + await pause(); + step = steps.next(); + } + return step.value; +}; + +function* linkImageSteps( + opening: Generator, + metadata: Il2cppMetadataDecoded, + facts: Il2cppMetadataLinkFacts, + options: Il2cppBinaryReadOptions, +): Generator { + const image = yield* opening; + const version = metadata.metadata_version; const modules = yield* findModules(image, metadata); const registration_ = yield* findCodeRegistration(image, modules); const methodImage = new Int32Array(metadata.methods.length).fill(-1); diff --git a/src/il2cpp/Il2cppBinaryImage.ts b/src/il2cpp/Il2cppBinaryImage.ts index d8e97fe54d..13b621bf50 100644 --- a/src/il2cpp/Il2cppBinaryImage.ts +++ b/src/il2cpp/Il2cppBinaryImage.ts @@ -14,7 +14,10 @@ import { type BinaryContainer, type FileMappingIndex, } from "../domain/binaryContainer.js"; -import { readBinaryContainer } from "../domain/native/readBinaryContainer.js"; +import { + readBinaryContainer, + readBinaryContainerCooperative, +} from "../domain/native/readBinaryContainer.js"; import { requireSelectedNativeImage, type SelectedNativeImage, @@ -81,28 +84,51 @@ export interface NativeImage { readonly machoPointerFixups?: PointerFixups; } -/** Open one thin image; a universal Mach-O slice is selected before this. */ -export function* openImageSteps(image: Buffer): Generator { - let container: BinaryContainer; +const OPERATION = "inspect_il2cpp_metadata"; + +const rejectContainer = (cause: unknown): never => { + if (cause instanceof BinaryResourceFailure) + throw new Il2cppBinaryFormatFailure( + "native-index-capacity", + cause.message, + cause.failure.limit.maximum, + "bytes", + ); + if (cause instanceof BinaryFormatFailure) + throw new Il2cppBinaryFormatFailure( + cause.kind === "unsupported" + ? "unsupported-container" + : "malformed-container", + cause.message, + ); + throw cause; +}; + +const readContainer = (image: Buffer): BinaryContainer => { try { - container = readBinaryContainer(image); + return readBinaryContainer(image, OPERATION); } catch (cause) { - if (cause instanceof BinaryResourceFailure) - throw new Il2cppBinaryFormatFailure( - "native-index-capacity", - cause.message, - cause.failure.limit.maximum, - "bytes", - ); - if (cause instanceof BinaryFormatFailure) - throw new Il2cppBinaryFormatFailure( - cause.kind === "unsupported" - ? "unsupported-container" - : "malformed-container", - cause.message, - ); - throw cause; + return rejectContainer(cause); } +}; + +export const readIl2cppBinaryContainerCooperative = async ( + image: Buffer, + pause: () => Promise, +): Promise => { + try { + return await readBinaryContainerCooperative(image, OPERATION, pause); + } catch (cause) { + return rejectContainer(cause); + } +}; + +/** Open one thin image; a universal Mach-O slice is selected before this. */ +export function* openImageSteps( + image: Buffer, + preparedContainer?: BinaryContainer, +): Generator { + const container = preparedContainer ?? readContainer(image); if (container.bits !== 64 || container.byte_order !== "little") throw new Il2cppBinaryFormatFailure( "unsupported-container", diff --git a/src/il2cpp/Il2cppMetadataProvider.ts b/src/il2cpp/Il2cppMetadataProvider.ts index e078fc0d0b..27e602f16c 100644 --- a/src/il2cpp/Il2cppMetadataProvider.ts +++ b/src/il2cpp/Il2cppMetadataProvider.ts @@ -61,7 +61,7 @@ import { type Il2cppMetadataRowCounts, } from "./Il2cppMetadataFile.js"; import { - linkIl2cppBinarySteps, + linkIl2cppBinaryCooperative, type Il2cppBinaryLink, } from "./Il2cppBinaryFile.js"; import { errorMessage } from "../kernel/errorMessage.js"; @@ -563,7 +563,7 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { input.binary_architecture, ); sliceLimitations = selected.limitations; - const linking = linkIl2cppBinarySteps( + const linking = await linkIl2cppBinaryCooperative( selected.bytes, decoded, link, @@ -571,14 +571,12 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { ...admission, genericInstances: input.generic_instances === true, }, + async () => { + await this.yieldControl(); + assertArtifactReadActive(options?.signal); + }, ); - let linkStep = linking.next(); - while (!linkStep.done) { - await this.yieldControl(); - assertArtifactReadActive(options?.signal); - linkStep = linking.next(); - } - linked = binaryReport(linkStep.value, { + linked = binaryReport(linking, { path: binaryPath, sha256: binary.sha256, bytes: binary.bytes.length, diff --git a/src/native/dwarf/DwarfSections.ts b/src/native/dwarf/DwarfSections.ts index abd4109042..ece6332ee2 100644 --- a/src/native/dwarf/DwarfSections.ts +++ b/src/native/dwarf/DwarfSections.ts @@ -16,12 +16,16 @@ import { } from "../../domain/binaryContainer.js"; import { decodeElfCompressedSection, + decodeElfCompressedSectionCooperative, decodeElfGnuZlibSection, type ElfCompressedSectionResult, } from "../../domain/native/elfCompressedSection.js"; +import type { ZstandardFramePause } from "../../domain/compression/zstandardExactFrame.js"; +import { ZstandardFrameAllowanceError } from "../../domain/compression/zstandardFrameAllowance.js"; import { readMachoImage } from "../../domain/apple/machoImage.js"; import { readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, readPeSectionDirectory, type NativeSection, type NativeSectionDirectory, @@ -128,14 +132,16 @@ interface Candidate { } /** Select DWARF sections from one thin ELF, PE, or Mach-O image. */ -export const locateDwarfSections = ( +export const locateDwarfSections = async ( bytes: Buffer, maximumDecodedBytes = MAX_DWARF_DECOMPRESSED_SECTION_BYTES, -): LocatedDwarfImage => { + pause?: ZstandardFramePause, +): Promise => { const budget: InflateBudget = { allowance: maximumDecodedBytes, used: 0 }; const magic = bytes.length >= 4 ? new BinaryReader(bytes, true, "dwarf").u32(0) : null; - if (magic === 0x464c457f) return finish(locateElf(bytes, budget)); + if (magic === 0x464c457f) + return finish(await locateElf(bytes, budget, pause)); if (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) return finish(locatePe(bytes, budget)); if ( @@ -259,6 +265,12 @@ const admitDecoded = ( : decoded.requiredBytes + BigInt(budget.used); throw new DwarfSectionAllowanceFailure(name, demanded, budget.allowance); } + if (decoded.state === "configuration") throw decoded.error; + if (decoded.state === "work_limit") + throw new ZstandardFrameAllowanceError( + { maxFrames: decoded.value, source: decoded.source }, + decoded.offset, + ); if (decoded.state === "malformed") { // The owner charges the larger of declared output and compressed input // read on failure. Rejected headers never inflate and charge 0. @@ -413,8 +425,19 @@ const finish = (image: ImageIdentity): LocatedDwarfImage => { }; }; -const locateElf = (bytes: Buffer, budget: InflateBudget): ImageIdentity => { - const directory = readElfDirectoryFromBuffer(bytes); +const locateElf = async ( + bytes: Buffer, + budget: InflateBudget, + pause?: ZstandardFramePause, +): Promise => { + const directory = + pause === undefined + ? readElfDirectoryFromBuffer(bytes, "inspect_dwarf_units") + : await readElfDirectoryFromBufferCooperative( + bytes, + pause, + "inspect_dwarf_units", + ); if (directory.section_names_limitation !== undefined) throw new BinaryFormatFailure( "unsupported", @@ -480,14 +503,24 @@ const locateElf = (bytes: Buffer, budget: InflateBudget): ImageIdentity => { continue; } const decoded = compressed - ? decodeElfCompressedSection( - stored, - section.flags, - directory.bits, - directory.little, - decodeBudget(budget, name), - `ELF ${name} section`, - ) + ? pause === undefined + ? decodeElfCompressedSection( + stored, + section.flags, + directory.bits, + directory.little, + decodeBudget(budget, name), + `ELF ${name} section`, + ) + : await decodeElfCompressedSectionCooperative( + stored, + section.flags, + directory.bits, + directory.little, + decodeBudget(budget, name), + `ELF ${name} section`, + pause, + ) : decodeElfGnuZlibSection( stored, decodeBudget(budget, name), diff --git a/src/native/dwarf/DwarfUnitsProvider.test.ts b/src/native/dwarf/DwarfUnitsProvider.test.ts index 31d2efd4d9..d2964538a9 100644 --- a/src/native/dwarf/DwarfUnitsProvider.test.ts +++ b/src/native/dwarf/DwarfUnitsProvider.test.ts @@ -1,7 +1,9 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { scheduler } from "node:timers/promises"; import { expect, it, vi } from "vitest"; import { + AnalysisCancelledError, AnalysisInputError, AnalysisResourceConstraintError, } from "../../domain/analysisErrorCore.js"; @@ -55,6 +57,55 @@ const inspect = async ( } }; +const manyProductiveFrames = (count: number): Buffer => { + const frame = Buffer.from([ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, 0x61, + ]); + const bytes = Buffer.alloc(frame.length * count); + for (let index = 0; index < count; index += 1) + frame.copy(bytes, index * frame.length); + return bytes; +}; + +it("cancels a many-frame compressed DWARF section", async () => { + const plain = Buffer.alloc(32, 0x61); + const header = elfCompressedBody(plain, "zstd").subarray(0, 24); + const image = elfDebugImage( + { info: Buffer.concat([header, manyProductiveFrames(32)]) }, + { infoFlags: 0x800n }, + ); + const root = await createTestTempDirectory("rea-dwarf-zstd-cancel-"); + const path = join(root, "image"); + await writeFile(path, image); + const reason = new Error("stop dwarf frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + const provider = new DwarfUnitsProvider(); + try { + const result = await provider.inspect( + inspectDwarfUnitsInputSchema.parse({ path }), + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_dwarf_units", + cause: reason, + }); + } finally { + spy.mockRestore(); + await provider.close(); + } +}); + it("reads a little-endian ELF unit and refuses architecture on a thin file", async () => { const named = namedUnit({ name: "main.c", dwoName: "main.dwo" }); const image = elfDebugImage(named); @@ -379,7 +430,7 @@ it("does not inflate a duplicate section and caps decompressed bytes across role expect(aggregate.error).toBeInstanceOf(AnalysisResourceConstraintError); }); -it("charges failed decompression attempts against the shared allowance", () => { +it("charges failed decompression attempts against the shared allowance", async () => { const allowance = 1024 * 1024; const bytes = Buffer.alloc(allowance); for (const gnu of [false, true]) { @@ -393,13 +444,13 @@ it("charges failed decompression attempts against the shared allowance", () => { flags: gnu ? 0n : 0x800n, })), }); - expect(() => locateDwarfSections(image, allowance)).toThrow( + await expect(locateDwarfSections(image, allowance)).rejects.toThrow( DwarfSectionAllowanceFailure, ); } }); -it("charges valid empty streams against the shared allowance", () => { +it("charges valid empty streams against the shared allowance", async () => { const empty = elfCompressedBody(Buffer.alloc(0), "zlib"); const blocks = Buffer.alloc(5 * 2048); for (let at = 0; at < blocks.length; at += 5) @@ -417,12 +468,12 @@ it("charges valid empty streams against the shared allowance", () => { flags: 0x800n, })), }); - expect(() => locateDwarfSections(image, payloadLength * 2)).toThrow( + await expect(locateDwarfSections(image, payloadLength * 2)).rejects.toThrow( DwarfSectionAllowanceFailure, ); }); -it("charges zero-output streams with bad checksums against the shared allowance", () => { +it("charges zero-output streams with bad checksums against the shared allowance", async () => { const empty = elfCompressedBody(Buffer.alloc(0), "zlib"); // Insert non-final empty stored blocks after the ELF and zlib headers. const blocks = Buffer.alloc(5 * 2048); @@ -442,7 +493,7 @@ it("charges zero-output streams with bad checksums against the shared allowance" flags: 0x800n, })), }); - expect(() => locateDwarfSections(image, payloadLength * 2)).toThrow( + await expect(locateDwarfSections(image, payloadLength * 2)).rejects.toThrow( DwarfSectionAllowanceFailure, ); }); diff --git a/src/native/dwarf/DwarfUnitsProvider.ts b/src/native/dwarf/DwarfUnitsProvider.ts index bd52ad38ad..66cbe6060f 100644 --- a/src/native/dwarf/DwarfUnitsProvider.ts +++ b/src/native/dwarf/DwarfUnitsProvider.ts @@ -7,6 +7,7 @@ import { type StableArtifactSnapshot, ArtifactReaderFailure, readStableArtifact, + yieldArtifactRead, artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, ArtifactResourceScope, @@ -17,6 +18,10 @@ import { } from "../../domain/parseFailureAnalysisError.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../../domain/compression/zstandardFrameAllowance.js"; import { decodeLimit } from "../../domain/parseFailure.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { DWARF_UNITS_PROVIDER } from "../../domain/investigationProviders.js"; @@ -93,7 +98,13 @@ export class DwarfUnitsProvider { snapshot.bytes, input.architecture, ); - const located = locateDwarfSections(selected.bytes); + const located = await locateDwarfSections( + selected.bytes, + undefined, + async () => { + await yieldArtifactRead(options?.signal); + }, + ); const report = dwarfUnitsSchema.parse( buildReport( input, @@ -250,6 +261,8 @@ const dwarfUnitsFailurePrimary = ( input: InspectDwarfUnitsInput, ) => { const { cause, origin } = facts; + if (cause instanceof ZstandardFrameAllowanceError) + return projectZstandardFrameAllowance(OPERATION, cause); if (origin === "owned-or-classified" && cause instanceof AnalysisError) return cause; if (cause instanceof DwarfSectionAllowanceFailure) diff --git a/src/native/elf/ElfDebugLink.test.ts b/src/native/elf/ElfDebugLink.test.ts index c14d7e0048..8a8875a1d5 100644 --- a/src/native/elf/ElfDebugLink.test.ts +++ b/src/native/elf/ElfDebugLink.test.ts @@ -376,7 +376,7 @@ const verify = (image: Buffer, candidate: Buffer, candidatePath: string) => { it("counts legacy .zdebug_info as present candidate debug info", () => { const candidate = elfDebugCandidateFixture({ zdebugInfo: true }); - const sections = readElfImage(candidate).sections; + const sections = readElfImage(candidate, "elf-directory").sections; expect( sections.some( (section) => section.format === "elf" && section.name === ".zdebug_info", @@ -397,7 +397,7 @@ it("counts legacy .zdebug_info as present candidate debug info", () => { /** Point one named section of a little-endian ELF64 candidate past its end. */ const moveSectionPastEnd = (candidate: Buffer, name: string): Buffer => { const damaged = Buffer.from(candidate); - const declared = readElfImage(candidate).sections.find( + const declared = readElfImage(candidate, "elf-directory").sections.find( (section) => section.format === "elf" && section.name === name, ); if (declared?.format !== "elf") throw new Error(`missing ${name}`); diff --git a/src/native/elf/ElfDebugLink.ts b/src/native/elf/ElfDebugLink.ts index a7a26f5cc4..1d586928c0 100644 --- a/src/native/elf/ElfDebugLink.ts +++ b/src/native/elf/ElfDebugLink.ts @@ -1,9 +1,18 @@ import { crc32 } from "node:zlib"; import { isParseLimit } from "../../domain/parseFailureAnalysisError.js"; -import { ArtifactReaderFailure } from "../../resources/index.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + ArtifactReaderFailure, + assertArtifactReadActive, + yieldArtifactRead, +} from "../../resources/index.js"; import { readElfFixedHeader } from "../../domain/elfFixedHeader.js"; import { elfObjectTypeName } from "../../domain/native/elfNotes.js"; +import { + zstandardFrameAllowanceCause, + type ZstandardFrameAllowanceError, +} from "../../domain/compression/zstandardFrameAllowance.js"; import { debugFileNameIsSingleSegment, deriveDebugFileLookupPaths, @@ -26,8 +35,21 @@ import { BinaryFormatFailure, BinaryResourceFailure, } from "../../domain/binaryContainer.js"; -import { readElfImage } from "../../domain/native/elfImage.js"; -import { parseElfNotes } from "./ElfNoteParser.js"; +import { + readElfImage, + readElfImageCooperative, +} from "../../domain/native/elfImage.js"; +import { parseElfNotes, parseElfNotesCooperative } from "./ElfNoteParser.js"; +import type { ZstandardFramePause } from "../../domain/compression/zstandardExactFrame.js"; +import { + sharedElfSectionNameDecoder, + type CooperativeElfSectionNameDecoder, +} from "../../domain/native/elfSectionNameTable.js"; + +import { err, ok, type Result } from "../../domain/result.js"; + +type PreparedCandidateImage = Result, unknown>; +type PreparedNotes = Result, unknown>; type ElfSection = Extract; type Report = ElfDebugLink; @@ -371,21 +393,26 @@ const observeBuildId = ( bytes: Buffer, artifact: Report["artifact"], input: InspectElfDebugLinkInput, + preparedNotes?: PreparedNotes, ): ObservedBuildId => { let notes: ReturnType; try { - notes = parseElfNotes( - bytes, - artifact, - { - path: artifact.path, - max_file_bytes: input.max_file_bytes, - max_entries: input.max_entries, - }, - { maxGnuBuildIdBytes: MAX_ELF_DEBUG_ALT_BUILD_ID_BYTES }, - ); + if (preparedNotes !== undefined && !preparedNotes.ok) + throw preparedNotes.error; + notes = + preparedNotes?.value ?? + parseElfNotes( + bytes, + artifact, + buildIdNoteInput(artifact, input), + BUILD_ID_NOTE_OPTIONS, + ); } catch (cause) { - if (isRethrownBudget(cause)) throw cause; + if ( + isRethrownBudget(cause) || + zstandardFrameAllowanceCause(cause) !== undefined + ) + throw cause; if (cause instanceof ArtifactReaderFailure && cause.reason === "format") return unknownBuildId( "note_tables_malformed", @@ -469,13 +496,32 @@ const sectionsNamed = ( name: string, ): readonly ElfSection[] => sections.filter((section) => section.name === name); +const BUILD_ID_NOTE_OPTIONS = { + maxGnuBuildIdBytes: MAX_ELF_DEBUG_ALT_BUILD_ID_BYTES, +}; +const buildIdNoteInput = ( + artifact: Report["artifact"], + input: InspectElfDebugLinkInput, +) => ({ + path: artifact.path, + max_file_bytes: input.max_file_bytes, + max_entries: input.max_entries, +}); + /** Decode separate-debug linkage from one in-memory ELF image. */ +const ELF_DEBUG_LINK_OPERATION = "inspect_elf_debug_link"; + export const parseElfDebugLink = ( bytes: Buffer, artifact: Report["artifact"], input: InspectElfDebugLinkInput, + image?: ReturnType, + preparedNotes?: PreparedNotes, ): Report => { - const image = readElfImage(bytes, input.max_file_bytes); + // The directory admits section names before either decoder inflates them. + image = + image ?? + readElfImage(bytes, ELF_DEBUG_LINK_OPERATION, input.max_file_bytes); const sections = image.sections.filter( (section): section is ElfSection => section.format === "elf", ); @@ -499,7 +545,7 @@ export const parseElfDebugLink = ( sectionsNamed(sections, ".gnu_debugdata"), namesComplete, ); - const observed = observeBuildId(bytes, artifact, input); + const observed = observeBuildId(bytes, artifact, input, preparedNotes); const limitations: Limitations = [...ELF_DEBUG_LINK_LIMITATIONS]; if (image.section_names_limitation !== undefined) limitations.push(image.section_names_limitation); @@ -579,23 +625,6 @@ const candidateSections = ( }; }; -/** - * The candidate is decoded with the image's budgets. A candidate that exceeds - * them leaves its identity unknown instead of failing the image report. - */ -const observeCandidateBuildId = ( - content: Buffer, - artifact: Report["artifact"], - input: InspectElfDebugLinkInput, -): ObservedBuildId | null => { - try { - return observeBuildId(content, artifact, input); - } catch (cause) { - if (isRethrownBudget(cause)) return null; - throw cause; - } -}; - const hasElfMagic = (bytes: Buffer): boolean => bytes.length >= 4 && bytes.toString("hex", 0, 4) === "7f454c46"; @@ -604,6 +633,8 @@ const readCandidateIdentity = ( totalBytes: number, artifact: Report["artifact"], input: InspectElfDebugLinkInput, + preparedImage?: PreparedCandidateImage, + preparedNotes?: PreparedNotes, ): { readonly format: NonNullable["format"]; readonly byte_order: NonNullable["byte_order"]; @@ -613,6 +644,7 @@ const readCandidateIdentity = ( >; readonly identity: CandidateIdentity; readonly budgetExceeded: boolean; + readonly frameAllowance?: ZstandardFrameAllowanceError; } => { const complete = content.length === totalBytes; const header = readElfFixedHeader(content, totalBytes); @@ -678,13 +710,33 @@ const readCandidateIdentity = ( let image: ReturnType | null = null; let imageMalformed = false; let imageBudgetExceeded = false; + let frameAllowance: ZstandardFrameAllowanceError | undefined; try { - image = readElfImage(content, input.max_file_bytes); + if (preparedImage !== undefined && !preparedImage.ok) + throw preparedImage.error; + image = + preparedImage?.value ?? + readElfImage(content, ELF_DEBUG_LINK_OPERATION, input.max_file_bytes); } catch (cause) { - if (isRethrownBudget(cause)) imageBudgetExceeded = true; - else imageMalformed = true; + frameAllowance = zstandardFrameAllowanceCause(cause); + if (frameAllowance === undefined) { + if (cause instanceof AnalysisError) throw cause; + if (isRethrownBudget(cause)) imageBudgetExceeded = true; + else imageMalformed = true; + } + } + // The candidate may lose named evidence without losing its file identity + // or CRC. If its section names already exceeded the frame allowance, the + // note reader cannot decode that same table either. + let observed: ObservedBuildId | null = null; + if (frameAllowance === undefined) { + try { + observed = observeBuildId(content, artifact, input, preparedNotes); + } catch (cause) { + frameAllowance = zstandardFrameAllowanceCause(cause); + if (frameAllowance === undefined && !isRethrownBudget(cause)) throw cause; + } } - const observed = observeCandidateBuildId(content, artifact, input); const buildId: BuildId = observed?.buildId ?? { status: "unknown", hex: null, @@ -703,7 +755,10 @@ const readCandidateIdentity = ( buildId, candidateHex: buildId.status === "observed" ? buildId.hex : null, }, - budgetExceeded: imageBudgetExceeded || observed === null, + budgetExceeded: + imageBudgetExceeded || + (frameAllowance === undefined && observed === null), + ...(frameAllowance === undefined ? {} : { frameAllowance }), }; }; @@ -838,17 +893,16 @@ export const verifyElfDebugCandidate = ( report: Report, candidate: ElfDebugCandidateFile, input: InspectElfDebugLinkInput, + preparedImage?: PreparedCandidateImage, + preparedNotes?: PreparedNotes, ): Report => { - const identityArtifact = { - path: candidate.path, - sha256: candidate.sha256, - bytes: candidate.bytes, - }; const read = readCandidateIdentity( candidate.prefix, candidate.bytes, - identityArtifact, + candidateArtifact(candidate), input, + preparedImage, + preparedNotes, ); const crc = crcCheck(report.debuglink.crc32_hex, candidate.crc32); const build = buildIdCheck(report.build_id, read.identity); @@ -858,6 +912,10 @@ export const verifyElfDebugCandidate = ( limitations.push(ELF_DEBUG_LINK_CANDIDATE_PREFIX_LIMITATION); if (read.budgetExceeded) limitations.push(ELF_DEBUG_LINK_CANDIDATE_BUDGET_LIMITATION); + if (read.frameAllowance !== undefined) + limitations.push( + `Candidate ELF section names could not be decoded because ${read.frameAllowance.failure.detail}.`, + ); if ( verification.status === "matches" && (crc.status === "unavailable" || build.status === "unavailable") @@ -880,6 +938,124 @@ export const verifyElfDebugCandidate = ( }; }; +const debugLinkPause = (signal?: AbortSignal) => () => + yieldArtifactRead(signal); + +const candidateArtifact = (candidate: ElfDebugCandidateFile) => ({ + path: candidate.path, + sha256: candidate.sha256, + bytes: candidate.bytes, +}); + +const prepareBuildIdNotes = async ( + bytes: Buffer, + artifact: Report["artifact"], + input: InspectElfDebugLinkInput, + pause: ZstandardFramePause, + decode: CooperativeElfSectionNameDecoder, + signal: AbortSignal | undefined, +): Promise => { + assertArtifactReadActive(signal); + try { + return ok( + await parseElfNotesCooperative( + bytes, + artifact, + buildIdNoteInput(artifact, input), + pause, + BUILD_ID_NOTE_OPTIONS, + decode, + ), + ); + } catch (cause) { + assertArtifactReadActive(signal); + return err(cause); + } +}; + +/** Same linkage as {@link parseElfDebugLink}. Section names yield for `signal`. */ +export const parseElfDebugLinkCooperative = async ( + bytes: Buffer, + artifact: Report["artifact"], + input: InspectElfDebugLinkInput, + signal?: AbortSignal, +): Promise => { + const pause = debugLinkPause(signal); + const decode = sharedElfSectionNameDecoder(pause); + const image = await readElfImageCooperative( + bytes, + ELF_DEBUG_LINK_OPERATION, + pause, + input.max_file_bytes, + decode, + ); + const notes = await prepareBuildIdNotes( + bytes, + artifact, + input, + pause, + decode, + signal, + ); + assertArtifactReadActive(signal); + return parseElfDebugLink(bytes, artifact, input, image, notes); +}; + +/** Same comparison as {@link verifyElfDebugCandidate}. Section names yield for `signal`. */ +export const verifyElfDebugCandidateCooperative = async ( + report: Report, + candidate: ElfDebugCandidateFile, + input: InspectElfDebugLinkInput, + signal?: AbortSignal, +): Promise => { + assertArtifactReadActive(signal); + if ( + candidate.prefix.length !== candidate.bytes || + !hasElfMagic(candidate.prefix) + ) + return verifyElfDebugCandidate(report, candidate, input); + const pause = debugLinkPause(signal); + const decode = sharedElfSectionNameDecoder(pause); + let preparedImage: PreparedCandidateImage; + try { + preparedImage = ok( + await readElfImageCooperative( + candidate.prefix, + ELF_DEBUG_LINK_OPERATION, + pause, + input.max_file_bytes, + decode, + ), + ); + } catch (cause) { + assertArtifactReadActive(signal); + preparedImage = err(cause); + } + // A typed image error already determines the candidate outcome. + if ( + !preparedImage.ok && + (preparedImage.error instanceof AnalysisError || + zstandardFrameAllowanceCause(preparedImage.error) !== undefined) + ) + return verifyElfDebugCandidate(report, candidate, input, preparedImage); + const notes = await prepareBuildIdNotes( + candidate.prefix, + candidateArtifact(candidate), + input, + pause, + decode, + signal, + ); + assertArtifactReadActive(signal); + return verifyElfDebugCandidate( + report, + candidate, + input, + preparedImage, + notes, + ); +}; + /** Keep the decoded image linkage when the selected candidate cannot be read. */ export const unreadableElfDebugCandidate = (report: Report): Report => ({ ...report, diff --git a/src/native/elf/ElfDebugLinkProvider.test.ts b/src/native/elf/ElfDebugLinkProvider.test.ts index eb63748990..8c9220695d 100644 --- a/src/native/elf/ElfDebugLinkProvider.test.ts +++ b/src/native/elf/ElfDebugLinkProvider.test.ts @@ -1,3 +1,4 @@ +import { scheduler } from "node:timers/promises"; import { primaryAnalysisError } from "../../domain/analysisErrorBase.js"; import { expect, it, vi } from "vitest"; @@ -11,6 +12,7 @@ import { inspectElfDebugLinkInputSchema, elfDebugLinkSchema, } from "../../domain/native/elfDebugLink.js"; +import { elfFrameAllowanceFixture } from "../../../tests/fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { elfDebugLinkFixture } from "./ElfDebugLink.fixture.js"; const image = elfDebugLinkFixture(); @@ -144,6 +146,52 @@ it("keeps a completed candidate stream's identity when cancellation arrives duri ); }); +it("preserves the completed candidate when its ELF comparison is cancelled", async () => { + const { streamStableArtifact } = await import("../../resources/index.js"); + const prefix = elfFrameAllowanceFixture(); + const sha256 = "cd".repeat(32); + vi.mocked(streamStableArtifact).mockResolvedValueOnce({ + prefix, + sha256, + size: prefix.length, + }); + const provider = new ElfDebugLinkProvider(); + const controller = new AbortController(); + const reason = new Error("candidate comparison cancelled"); + const originalYield = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await originalYield(); + controller.abort(reason); + }); + try { + const result = await provider.inspect( + inspectElfDebugLinkInputSchema.parse({ + path: "/artifacts/prog", + candidate_path: "/artifacts/prog.debug", + }), + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected interrupted comparison"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + }); + expect(retainedReport(result.error)).toMatchObject({ + candidate: { + path: "/artifacts/prog.debug", + sha256, + bytes: prefix.length, + }, + verification: null, + }); + } finally { + pause.mockRestore(); + await provider.close(); + } +}); + it("recovers a completed stream's whole-file size, not its prefix length", () => { expect( completedStableArtifactObservation( diff --git a/src/native/elf/ElfDebugLinkProvider.ts b/src/native/elf/ElfDebugLinkProvider.ts index 8c9850a6e7..b600fc9fde 100644 --- a/src/native/elf/ElfDebugLinkProvider.ts +++ b/src/native/elf/ElfDebugLinkProvider.ts @@ -4,6 +4,7 @@ import { parseLimitConstraint, } from "../../domain/parseFailureAnalysisError.js"; import { decodeLimit } from "../../domain/parseFailure.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { projectArtifactCloseFailure, projectOwnedArtifactIdentityFailure, @@ -41,9 +42,11 @@ import { } from "../../domain/binaryContainer.js"; import { continueGnuDebuglinkCrc32, - parseElfDebugLink, + parseElfDebugLinkCooperative, unreadableElfDebugCandidate, verifyElfDebugCandidate, + verifyElfDebugCandidateCooperative, + type ElfDebugCandidateFile, } from "./ElfDebugLink.js"; import { ELF_NOTE_TEXT_ALLOWANCE } from "./ElfNoteParser.js"; import { elfSectionNameBudget } from "../../domain/native/elfSectionNameTable.js"; @@ -72,6 +75,7 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { async inspect(input: InspectElfDebugLinkInput, options?: ExecutionOptions) { let field: SelectedField = "path"; let candidatePhase: CandidatePhase | undefined; + let candidate: StableArtifactStream | undefined; let crc32 = 0; let snapshot: StableArtifactSnapshot | undefined; try { @@ -84,7 +88,7 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { undefined, this.#resources, ); - const image = parseElfDebugLink( + const image = await parseElfDebugLinkCooperative( snapshot.bytes, { path: input.path, @@ -92,11 +96,11 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { bytes: snapshot.bytes.length, }, input, + options?.signal, ); if (input.candidate_path === undefined) return ok(image); field = "candidate_path"; candidatePhase = { report: image, path: input.candidate_path }; - let candidate; try { // Stream the whole candidate for the debuglink CRC32 and SHA-256. // Only a file within max_file_bytes is retained for ELF decoding; @@ -118,7 +122,15 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { if (!candidateUnreadable(cause)) throw cause; return ok(unreadableElfDebugCandidate(image)); } - return ok(verifiedCandidate(candidatePhase, candidate, crc32, input)); + return ok( + await verifiedCandidate( + candidatePhase, + candidate, + crc32, + input, + options?.signal, + ), + ); }); } catch (cause) { return err( @@ -134,6 +146,7 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { cause, crc32, input, + candidate, ), }, ), @@ -144,45 +157,81 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { type CandidatePhase = { readonly report: ElfDebugLink; readonly path: string }; +const candidateFile = ( + phase: CandidatePhase, + candidate: StableArtifactStream, + crc32: number, +): ElfDebugCandidateFile => ({ + path: phase.path, + sha256: candidate.sha256, + bytes: candidate.size, + crc32, + prefix: candidate.prefix, +}); + const verifiedCandidate = ( phase: CandidatePhase, candidate: StableArtifactStream, crc32: number, input: InspectElfDebugLinkInput, -): ElfDebugLink => - verifyElfDebugCandidate( + signal?: AbortSignal, +): Promise => + verifyElfDebugCandidateCooperative( phase.report, - { - path: phase.path, - sha256: candidate.sha256, - bytes: candidate.size, - crc32, - prefix: candidate.prefix, - }, + candidateFile(phase, candidate, crc32), input, + signal, ); -/** - * The decoded image survives a failed candidate phase. A candidate stream that - * completed before cancellation or a failed close keeps its full comparison; - * otherwise only the image linkage is retained, without any comparison. - */ +const unverifiedCandidateObservation = ( + report: ElfDebugLink, + candidate: ElfDebugCandidateFile, +): ElfDebugLink => ({ + ...report, + candidate: { + path: candidate.path, + sha256: candidate.sha256, + bytes: candidate.bytes, + format: null, + byte_order: null, + debug_info: "unknown", + symtab: "unknown", + }, + limitations: [ + ...report.limitations, + "The candidate read completed. Its ELF comparison did not complete.", + ], +}); + +/** Retain the completed stream without restarting an interrupted comparison. */ const candidatePhaseObservation = ( phase: CandidatePhase, cause: unknown, crc32: number, input: InspectElfDebugLinkInput, + candidate: StableArtifactStream | undefined, ): AnalysisPartialObservation => { + if (candidate !== undefined) + return unverifiedCandidateObservation( + phase.report, + candidateFile(phase, candidate, crc32), + ); const completed = completedStableArtifactStream(cause); - return completed === undefined - ? { - ...phase.report, - limitations: [ - ...phase.report.limitations, - ELF_DEBUG_LINK_CANDIDATE_INCOMPLETE_LIMITATION, - ], - } - : verifiedCandidate(phase, completed, crc32, input); + if (completed === undefined) + return { + ...phase.report, + limitations: [ + ...phase.report.limitations, + ELF_DEBUG_LINK_CANDIDATE_INCOMPLETE_LIMITATION, + ], + }; + const file = candidateFile(phase, completed, crc32); + // A stream whose close failed can still supply a complete comparison. + try { + return verifyElfDebugCandidate(phase.report, file, input); + } catch { + return unverifiedCandidateObservation(phase.report, file); + } }; /** Enough retained bytes for an ELF32 or ELF64 fixed header. */ @@ -254,6 +303,8 @@ const elfDebugLinkFailurePrimary = ( field: SelectedField, ) => { const { cause, origin } = facts; + if (origin === "owned-or-classified" && cause instanceof AnalysisError) + return cause; const path = selectedPath(input, field); const sectionNameBudget = elfSectionNameBudget(cause); if (isParseLimit(cause, "elf") && sectionNameBudget !== undefined) diff --git a/src/native/elf/ElfNoteParser.test.ts b/src/native/elf/ElfNoteParser.test.ts index 2fdac63559..c77bf08fc8 100644 --- a/src/native/elf/ElfNoteParser.test.ts +++ b/src/native/elf/ElfNoteParser.test.ts @@ -653,7 +653,7 @@ it.each([ bytes.set([0x41, 0x42, 0xe9, 0], bytes.length - 4); bytes.writeUInt32LE(offset, 64 + 2 * 64); const notes = parse(bytes); - const directory = readElfDirectoryFromBuffer(bytes); + const directory = readElfDirectoryFromBuffer(bytes, "elf-directory"); expect(notes.tables[0]?.name).toBe(name); expect(directory.sections[2]).toMatchObject({ name, name_state: state }); if (name === null) { @@ -675,7 +675,7 @@ it.each([65535, 65536])( bytes[bytes.length - 1] = 0; const name = "A".repeat(terminatorOffset); const notes = parse(bytes); - const directory = readElfDirectoryFromBuffer(bytes); + const directory = readElfDirectoryFromBuffer(bytes, "elf-directory"); expect(notes.tables[0]?.name).toBe(name); expect(notes.coverage.status).toBe("complete"); expect(directory.sections[2]).toMatchObject( @@ -757,14 +757,16 @@ it("checks direct section count encoding in notes and the native directory", () bytes.writeBigUInt64LE(0n, sectionZero + 32); bytes.writeUInt16LE(0xfeff, 60); expect(parse(bytes).tables[0]?.name).toBe(".note.gnu.build-id"); - expect(readElfDirectoryFromBuffer(bytes).sections[1]?.name).toBe( - ".note.gnu.build-id", - ); + expect( + readElfDirectoryFromBuffer(bytes, "elf-directory").sections[1]?.name, + ).toBe(".note.gnu.build-id"); bytes.writeUInt16LE(0xff00, 60); const reason = /e_shnum uses a reserved direct section count/u; expect(() => parse(bytes)).toThrow(reason); - expect(() => readElfDirectoryFromBuffer(bytes)).toThrow(reason); + expect(() => readElfDirectoryFromBuffer(bytes, "elf-directory")).toThrow( + reason, + ); }); const repeatedNotes = (count: number, notes: Buffer): Buffer => { diff --git a/src/native/elf/ElfNoteParser.ts b/src/native/elf/ElfNoteParser.ts index 1297d4f8e1..eceddb837d 100644 --- a/src/native/elf/ElfNoteParser.ts +++ b/src/native/elf/ElfNoteParser.ts @@ -33,9 +33,15 @@ import { import { BinaryFormatFailure } from "../../domain/binaryContainer.js"; import { decodeLimit, ParseFailureError } from "../../domain/parseFailure.js"; import { - decodeElfSectionNameTable, + decodeElfSectionNameRequest, + decodeElfSectionNameRequestCooperative, elfSectionNameBudgetFailure, + type CooperativeElfSectionNameDecoder, + type ElfSectionNameDecodeRequest, } from "../../domain/native/elfSectionNameTable.js"; +import type { ElfCompressedSectionResult } from "../../domain/native/elfCompressedSection.js"; +import type { ZstandardFramePause } from "../../domain/compression/zstandardExactFrame.js"; +import { ZstandardFrameAllowanceError } from "../../domain/compression/zstandardFrameAllowance.js"; import { sha256Hex } from "../../kernel/sha256Hex.js"; import { decodeFdoNote, type FdoDecodeBudget } from "./ElfFdoNote.js"; @@ -697,12 +703,16 @@ const decodeAndroid = ( }; /** Decode PT_NOTE and SHT_NOTE tables from one in-memory ELF image. */ -export const parseElfNotes = ( +const parseElfNoteSteps = function* ( bytes: Buffer, artifact: ElfNotes["artifact"], input: InspectElfNotesInput, options: ElfNoteParseOptions = {}, -): ElfNotes => { +): Generator< + ElfSectionNameDecodeRequest, + ElfNotes, + ElfCompressedSectionResult +> { const fixed = readElfFixedHeader(bytes); if (!fixed.ok) return fail(fixed.error); const header = fixed.value; @@ -780,14 +790,20 @@ export const parseElfNotes = ( Number(namesSection.size), ); const flags = namesSection.flags; - const decoded = decodeElfSectionNameTable( - rawNames, + const decoded = yield { + raw: rawNames, flags, - header.elfClass, - reader.little, - input.max_file_bytes - bytes.length, - ); + bits: header.elfClass, + little: reader.little, + maximumDecodedBytes: input.max_file_bytes - bytes.length, + }; if (decoded.state === "malformed") return fail(decoded.message); + if (decoded.state === "configuration") throw decoded.error; + if (decoded.state === "work_limit") + throw new ZstandardFrameAllowanceError( + { maxFrames: decoded.value, source: decoded.source }, + decoded.offset, + ); if (decoded.state === "limit") throw elfSectionNameBudgetFailure( BigInt(bytes.length) + decoded.requiredBytes, @@ -974,6 +990,52 @@ export const parseElfNotes = ( }; }; +const driveElfNotes = ( + steps: Generator< + ElfSectionNameDecodeRequest, + ElfNotes, + ElfCompressedSectionResult + >, + decode: ( + request: ElfSectionNameDecodeRequest, + ) => ElfCompressedSectionResult | Promise, +): Promise => { + const run = async (): Promise => { + let next = steps.next(); + while (!next.done) next = steps.next(await decode(next.value)); + return next.value; + }; + return run(); +}; + +/** Decode PT_NOTE and SHT_NOTE tables from one in-memory ELF image. */ +export const parseElfNotes = ( + bytes: Buffer, + artifact: ElfNotes["artifact"], + input: InspectElfNotesInput, + options: ElfNoteParseOptions = {}, +): ElfNotes => { + const steps = parseElfNoteSteps(bytes, artifact, input, options); + let next = steps.next(); + while (!next.done) next = steps.next(decodeElfSectionNameRequest(next.value)); + return next.value; +}; + +/** + * Same note decode as {@link parseElfNotes}. Section-name Zstandard frames + * yield through `pause` so an existing caller abort can land between frames. + */ +export const parseElfNotesCooperative = ( + bytes: Buffer, + artifact: ElfNotes["artifact"], + input: InspectElfNotesInput, + pause: ZstandardFramePause, + options: ElfNoteParseOptions = {}, + decode: CooperativeElfSectionNameDecoder = (request) => + decodeElfSectionNameRequestCooperative(request, pause), +): Promise => + driveElfNotes(parseElfNoteSteps(bytes, artifact, input, options), decode); + const toEntry = ( note: ParsedNote, tableKind: "program" | "section", diff --git a/src/native/elf/ElfNotes.fixture.ts b/src/native/elf/ElfNotes.fixture.ts index a8d0628cd0..fe6567fa85 100644 --- a/src/native/elf/ElfNotes.fixture.ts +++ b/src/native/elf/ElfNotes.fixture.ts @@ -321,6 +321,7 @@ export const compressedElfNoteFixture = ( little = true, compressionType = 1, sectionName = ".note.gnu.build-id", + compressedPayload?: Buffer, ): Buffer => { const image = elfNoteFixture({ bits, @@ -349,7 +350,8 @@ export const compressedElfNoteFixture = ( const namesSize = word(namesHeader + (wide ? 32 : 20)); const names = image.subarray(namesOffset, namesOffset + namesSize); const compressed = - compressionType === 2 ? zstdCompressSync(names) : deflateSync(names); + compressedPayload ?? + (compressionType === 2 ? zstdCompressSync(names) : deflateSync(names)); const compressionHeaderBytes = wide ? 24 : 12; const replacement = Buffer.alloc(compressionHeaderBytes); if (little) replacement.writeUInt32LE(compressionType, 0); diff --git a/src/native/elf/ElfNotesProvider.test.ts b/src/native/elf/ElfNotesProvider.test.ts index 52692ccbdb..45669a6a09 100644 --- a/src/native/elf/ElfNotesProvider.test.ts +++ b/src/native/elf/ElfNotesProvider.test.ts @@ -1,15 +1,25 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import { runWithAnalysisOperationSnapshot } from "../../domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../../domain/analysisResourceLimits.js"; import { ElfNotesService } from "../../application/binaryDiagnostics/ElfNotesService.js"; import { + AnalysisCancelledError, AnalysisInputError, AnalysisResourceConstraintError, AnalysisUnsupportedTargetError, } from "../../domain/analysisErrorCore.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { analysisRefusalRemediation } from "../../domain/analysisRefusal.js"; -import { elfNotesSchema } from "../../domain/native/elfNotes.js"; +import { + elfNotesSchema, + inspectElfNotesInputSchema, +} from "../../domain/native/elfNotes.js"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { compressedElfNoteFixture, @@ -20,6 +30,26 @@ import { } from "./ElfNotes.fixture.js"; import { ElfNotesProvider } from "./ElfNotesProvider.js"; +const manyProductiveFrames = (count: number): Buffer => { + const frame = zstdByteFrame(0x61); + const bytes = Buffer.alloc(frame.length * count); + for (let index = 0; index < count; index += 1) + frame.copy(bytes, index * frame.length); + return bytes; +}; + +const abortOnFirstYield = (reason: Error): AbortSignal => { + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + return controller.signal; +}; + const inspect = async ( path: string, input: { readonly max_file_bytes?: number } = {}, @@ -32,6 +62,77 @@ const inspect = async ( } }; +const zstdByteFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +it("projects a Zstandard frame-cap on section names as an adjustable limit", async () => { + const root = await createTestTempDirectory("rea-elf-note-zstd-frames-"); + const path = join(root, "notes.elf"); + await writeFile( + path, + compressedElfNoteFixture( + 64, + true, + 2, + ".note.gnu.build-id", + Buffer.concat([zstdByteFrame(0x61), zstdByteFrame(0x62)]), + ), + ); + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => inspect(path), + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); +}); + +it("cancels a many-frame compressed section-name table", async () => { + const root = await createTestTempDirectory("rea-elf-note-zstd-cancel-"); + const path = join(root, "notes.elf"); + await writeFile( + path, + compressedElfNoteFixture( + 64, + true, + 2, + ".note.gnu.build-id", + manyProductiveFrames(32), + ), + ); + const reason = new Error("stop note frames"); + const provider = new ElfNotesProvider(); + const signal = abortOnFirstYield(reason); + try { + const result = await provider.inspect( + inspectElfNotesInputSchema.parse({ path }), + { signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_elf_notes", + cause: reason, + }); + } finally { + vi.restoreAllMocks(); + await provider.close(); + } +}); + describe("ELF note provider failures", () => { it("projects compressed note sections as unsupported rather than malformed input", async () => { const root = await createTestTempDirectory("rea-compressed-notes-"); diff --git a/src/native/elf/ElfNotesProvider.ts b/src/native/elf/ElfNotesProvider.ts index da82f6619b..9b41f455d0 100644 --- a/src/native/elf/ElfNotesProvider.ts +++ b/src/native/elf/ElfNotesProvider.ts @@ -12,6 +12,7 @@ import { ArtifactReaderFailure, type StableArtifactSnapshot, readStableArtifact, + yieldArtifactRead, artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, ArtifactResourceScope, @@ -20,6 +21,11 @@ import type { ElfNotesPort } from "../../ports/binaryDiagnostics/ElfNotesPort.js import type { ExecutionOptions } from "../../ports/AnalysisProvider.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; import { decodeLimit } from "../../domain/parseFailure.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../../domain/compression/zstandardFrameAllowance.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { MAX_INSPECT_ELF_NOTE_ENTRIES, @@ -28,8 +34,11 @@ import { } from "../../domain/native/elfNotes.js"; import { err, ok } from "../../domain/result.js"; import { ELF_NOTES_PROVIDER } from "../../domain/investigationProviders.js"; -import { ELF_NOTE_TEXT_ALLOWANCE, parseElfNotes } from "./ElfNoteParser.js"; import { elfSectionNameBudget } from "../../domain/native/elfSectionNameTable.js"; +import { + ELF_NOTE_TEXT_ALLOWANCE, + parseElfNotesCooperative, +} from "./ElfNoteParser.js"; const OPERATION = "inspect_elf_notes"; @@ -62,7 +71,7 @@ export class ElfNotesProvider implements ElfNotesPort { this.#resources, ); return ok( - parseElfNotes( + await parseElfNotesCooperative( snapshot.bytes, { path: input.path, @@ -70,6 +79,7 @@ export class ElfNotesProvider implements ElfNotesPort { bytes: snapshot.bytes.length, }, input, + () => yieldArtifactRead(options?.signal), ), ); }); @@ -101,6 +111,9 @@ const elfNotesFailurePrimary = ( input: InspectElfNotesInput, ) => { const { cause, origin } = facts; + if (cause instanceof AnalysisError) return cause; + if (cause instanceof ZstandardFrameAllowanceError) + return projectZstandardFrameAllowance(OPERATION, cause); const sectionNameBudget = elfSectionNameBudget(cause); if (isParseLimit(cause, "elf") && sectionNameBudget !== undefined) return parseLimitConstraint(OPERATION, "memory", cause, { diff --git a/src/native/elf/ElfUnwindRangesProvider.ts b/src/native/elf/ElfUnwindRangesProvider.ts index eba7ece81d..aa935cf442 100644 --- a/src/native/elf/ElfUnwindRangesProvider.ts +++ b/src/native/elf/ElfUnwindRangesProvider.ts @@ -11,6 +11,7 @@ import { projectArtifactCloseFailure, ArtifactReaderFailure, readStableArtifact, + type StableArtifactSnapshot, artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, ArtifactResourceScope, @@ -18,6 +19,7 @@ import { import type { ElfUnwindRangesPort } from "../../ports/binaryDiagnostics/ElfUnwindRangesPort.js"; import type { ExecutionOptions } from "../../ports/AnalysisProvider.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { @@ -34,7 +36,7 @@ import { import { ELF_UNWIND_FORMAT, ELF_UNWIND_RECORD_LIMIT, - parseElfUnwindRanges, + parseElfUnwindRangesCooperative, } from "./ElfUnwindTables.js"; const OPERATION = "inspect_elf_unwind_ranges"; @@ -60,30 +62,25 @@ export class ElfUnwindRangesProvider implements ElfUnwindRangesPort { input: InspectElfUnwindRangesInput, options?: ExecutionOptions, ) { + const signal = options?.signal; + let snapshot: StableArtifactSnapshot | undefined; try { return await this.#resources.run(async () => { - const snapshot = await readStableArtifact( + snapshot = await readStableArtifact( input.path, input.max_file_bytes, - options?.signal, + signal, undefined, this.#resources, ); + const { bytes, sha256 } = snapshot; + const artifact = { path: input.path, sha256, bytes: bytes.length }; return ok( - parseElfUnwindRanges( - snapshot.bytes, - { - path: input.path, - sha256: snapshot.sha256, - bytes: snapshot.bytes.length, - }, - input, - options?.signal, - ), + await parseElfUnwindRangesCooperative(bytes, artifact, input, signal), ); }); } catch (cause) { - return err(elfUnwindFailure(cause, input)); + return err(elfUnwindFailure(cause, input, snapshot)); } } } @@ -103,6 +100,8 @@ const elfUnwindFailurePrimary = ( input: InspectElfUnwindRangesInput, ) => { const { cause, origin } = facts; + if (origin === "owned-or-classified" && cause instanceof AnalysisError) + return cause; if (isParseLimit(cause, ELF_UNWIND_FORMAT)) { const { detail, limit } = cause.failure; const records = limit.name === ELF_UNWIND_RECORD_LIMIT; @@ -184,6 +183,7 @@ const elfUnwindFailurePrimary = ( const elfUnwindFailure = ( cause: unknown, input: InspectElfUnwindRangesInput, + snapshot?: StableArtifactSnapshot, ) => { const primary = elfUnwindFailurePrimary( artifactFilePrimaryFacts(cause), @@ -195,5 +195,6 @@ const elfUnwindFailure = ( input.path, ELF_UNWIND_RANGES_PROVIDER.id, OPERATION, + snapshot, ); }; diff --git a/src/native/elf/ElfUnwindTables.ts b/src/native/elf/ElfUnwindTables.ts index 619c541595..ceb7358184 100644 --- a/src/native/elf/ElfUnwindTables.ts +++ b/src/native/elf/ElfUnwindTables.ts @@ -5,7 +5,10 @@ import { keyedBy, uniqueKeyed, } from "../../domain/keyedCollection.js"; -import { assertArtifactReadActive } from "../../resources/index.js"; +import { + assertArtifactReadActive, + yieldArtifactRead, +} from "../../resources/index.js"; import { decodeLimit } from "../../domain/parseFailure.js"; import { readElfDynamicEntry } from "../../domain/elfDynamicEntry.js"; import { elfSymbolSize, readElfSymbol } from "../../domain/elfSymbol.js"; @@ -27,6 +30,7 @@ import { buildIntervalIndex } from "./ElfIntervalIndex.js"; import { ELF_PT_GNU_EH_FRAME, readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, type ElfProgramDeclaration, type NativeSection, } from "../../domain/native/nativeSectionDirectory.js"; @@ -242,20 +246,56 @@ const placeSection = ( export const ELF_UNWIND_FORMAT = "elf-unwind-ranges"; export const ELF_UNWIND_RECORD_LIMIT = "max_ranges"; +const OPERATION = "inspect_elf_unwind_ranges"; /** Read .eh_frame/.eh_frame_hdr function ranges. Truncation stays incomplete. */ export const parseElfUnwindRanges = ( bytes: Buffer, - artifact: { - readonly path: string; - readonly sha256: string; - readonly bytes: number; - }, + artifact: ElfUnwindRanges["artifact"], input: InspectElfUnwindRangesInput, signal?: AbortSignal, ): ElfUnwindRanges => { assertArtifactReadActive(signal); - const directory = readElfDirectoryFromBuffer(bytes); + return elfUnwindRangesFromDirectory( + bytes, + artifact, + input, + readElfDirectoryFromBuffer(bytes, OPERATION), + signal, + ); +}; + +/** Read the same unwind facts while allowing compressed section names to yield. */ +export const parseElfUnwindRangesCooperative = async ( + bytes: Buffer, + artifact: ElfUnwindRanges["artifact"], + input: InspectElfUnwindRangesInput, + signal?: AbortSignal, +): Promise => { + if (signal === undefined) return parseElfUnwindRanges(bytes, artifact, input); + assertArtifactReadActive(signal); + const directory = await readElfDirectoryFromBufferCooperative( + bytes, + () => yieldArtifactRead(signal), + OPERATION, + ); + return elfUnwindRangesFromDirectory( + bytes, + artifact, + input, + directory, + signal, + ); +}; + +const elfUnwindRangesFromDirectory = ( + bytes: Buffer, + artifact: ElfUnwindRanges["artifact"], + input: InspectElfUnwindRangesInput, + directory: ReturnType, + signal?: AbortSignal, +): ElfUnwindRanges => { + assertArtifactReadActive(signal); const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const little = directory.little; const bits = directory.bits; diff --git a/src/native/elf/KernelModuleParser.ts b/src/native/elf/KernelModuleParser.ts index 672b39017b..7000bfeee2 100644 --- a/src/native/elf/KernelModuleParser.ts +++ b/src/native/elf/KernelModuleParser.ts @@ -1,4 +1,8 @@ -import { ArtifactReaderFailure } from "../../resources/index.js"; +import { + ArtifactReaderFailure, + assertArtifactReadActive, + yieldArtifactRead, +} from "../../resources/index.js"; import { allKeyed, firstKeyed, keyedBy } from "../../domain/keyedCollection.js"; import { elfObjectTypeName } from "../../domain/native/elfNotes.js"; import { @@ -18,7 +22,10 @@ import { type ImageSection, type BinaryContainer, } from "../../domain/binaryContainer.js"; -import { readElfImage } from "../../domain/native/elfImage.js"; +import { + readElfImage, + readElfImageCooperative, +} from "../../domain/native/elfImage.js"; import { kernelCompressionCodecAt } from "./KernelCompressionCodec.js"; import { sha256Hex } from "../../kernel/sha256Hex.js"; @@ -67,11 +74,6 @@ const inFile = (section: ElfSection, fileLength: number): boolean => const rawHex = (bytes: Buffer): string | null => bytes.length > MAX_KERNEL_MODULE_RAW_BYTES ? null : bytes.toString("hex"); -const compressionCodec = (bytes: Buffer): Report["compression"] => { - const codec = kernelCompressionCodecAt(bytes, 0, MODULE_CODECS); - return codec === null ? null : { codec, status: "unsupported" }; -}; - const baseLimitations = (): Limitations => [ "This workflow does not execute the module, resolve imported symbols, or validate an appended signature.", "struct module in .gnu.linkonce.this_module is kernel-version specific and is not decoded.", @@ -124,6 +126,17 @@ const unsupportedCompression = ( ], }); +/** A recognized outer container never enters the ELF image parser. */ +const moduleCompressionReport = ( + bytes: Buffer, + artifact: Report["artifact"], +): Report | null => { + const codec = kernelCompressionCodecAt(bytes, 0, MODULE_CODECS); + return codec === null + ? null + : unsupportedCompression(artifact, { codec, status: "unsupported" }); +}; + const notApplicable = ( artifact: Report["artifact"], facts: ReturnType, @@ -506,16 +519,13 @@ const signatureLimitation = (signature: Report["signature"]): string => { return "The appended signature trailer was not accepted as an unvalidated PKCS#7 range."; }; -/** Decode kernel-module metadata from one in-memory ET_REL image. */ -export const parseKernelModule = ( +/** Decode kernel-module metadata from an admitted in-memory ELF image. */ +const parseKernelModuleImage = ( bytes: Buffer, artifact: Report["artifact"], input: InspectKernelModuleInput, + image: ReturnType, ): Report => { - const compression = compressionCodec(bytes); - if (compression !== null) - return unsupportedCompression(artifact, compression); - const image = readElfImage(bytes); const facts = headerFacts(image); if (facts.type !== 1) return notApplicable(artifact, facts); const sections = image.sections.filter( @@ -663,3 +673,37 @@ export const parseKernelModule = ( limitations, }; }; + +export const parseKernelModule = ( + bytes: Buffer, + artifact: Report["artifact"], + input: InspectKernelModuleInput, +): Report => + moduleCompressionReport(bytes, artifact) ?? + parseKernelModuleImage( + bytes, + artifact, + input, + readElfImage(bytes, "inspect_kernel_module"), + ); + +/** Same report as {@link parseKernelModule}. Section names yield for `signal`. */ +export const parseKernelModuleCooperative = async ( + bytes: Buffer, + artifact: Report["artifact"], + input: InspectKernelModuleInput, + signal?: AbortSignal, +): Promise => { + assertArtifactReadActive(signal); + return ( + moduleCompressionReport(bytes, artifact) ?? + parseKernelModuleImage( + bytes, + artifact, + input, + await readElfImageCooperative(bytes, "inspect_kernel_module", () => + yieldArtifactRead(signal), + ), + ) + ); +}; diff --git a/src/native/elf/KernelModuleProvider.ts b/src/native/elf/KernelModuleProvider.ts index 7b4c51c316..3284616342 100644 --- a/src/native/elf/KernelModuleProvider.ts +++ b/src/native/elf/KernelModuleProvider.ts @@ -18,6 +18,7 @@ import { import type { KernelModulePort } from "../../ports/binaryDiagnostics/KernelModulePort.js"; import type { ExecutionOptions } from "../../ports/AnalysisProvider.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { @@ -31,7 +32,7 @@ import { BinaryResourceFailure, binaryResourceCeiling, } from "../../domain/binaryContainer.js"; -import { parseKernelModule } from "./KernelModuleParser.js"; +import { parseKernelModuleCooperative } from "./KernelModuleParser.js"; const OPERATION = "inspect_kernel_module"; @@ -64,7 +65,7 @@ export class KernelModuleProvider implements KernelModulePort { this.#resources, ); return ok( - parseKernelModule( + await parseKernelModuleCooperative( snapshot.bytes, { path: input.path, @@ -72,6 +73,7 @@ export class KernelModuleProvider implements KernelModulePort { bytes: snapshot.bytes.length, }, input, + options?.signal, ), ); }); @@ -98,6 +100,8 @@ const kernelModuleFailurePrimary = ( input: InspectKernelModuleInput, ) => { const { cause, origin } = facts; + if (origin === "owned-or-classified" && cause instanceof AnalysisError) + return cause; if (cause instanceof BinaryResourceFailure) return parseLimitConstraint( OPERATION, diff --git a/src/native/exports/NativeExportsProvider.ts b/src/native/exports/NativeExportsProvider.ts index 666d34c5eb..f56ff00f96 100644 --- a/src/native/exports/NativeExportsProvider.ts +++ b/src/native/exports/NativeExportsProvider.ts @@ -15,6 +15,7 @@ import { artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, assertArtifactReadActive, + yieldArtifactRead, ArtifactResourceScope, } from "../../resources/index.js"; import type { NativeExportsPort } from "../../ports/binaryDiagnostics/NativeExportsPort.js"; @@ -39,6 +40,7 @@ import { NATIVE_EXPORT_LIMITS, type NativeExportLimit, parseNativeExports, + parseNativeExportsCooperative, } from "../../domain/native/nativeExportParser.js"; import { errorMessage } from "../../kernel/errorMessage.js"; @@ -62,23 +64,32 @@ export class NativeExportsProvider implements NativeExportsPort { } async inspect(input: InspectNativeExportsInput, options?: ExecutionOptions) { + const signal = options?.signal; let snapshot: StableArtifactSnapshot | undefined; try { return await this.#resources.run(async () => { snapshot = await readStableArtifact( input.path, input.max_file_bytes, - options?.signal, + signal, undefined, this.#resources, ); - assertArtifactReadActive(options?.signal); - const report = parseNativeExports(snapshot.bytes, { + assertArtifactReadActive(signal); + const artifact = { path: input.path, sha256: snapshot.sha256, bytes: snapshot.bytes.length, - }); - assertArtifactReadActive(options?.signal); + }; + const report = + signal === undefined + ? parseNativeExports(snapshot.bytes, artifact) + : await parseNativeExportsCooperative( + snapshot.bytes, + artifact, + () => yieldArtifactRead(signal), + ); + assertArtifactReadActive(signal); return ok(report); }); } catch (cause) { diff --git a/src/native/hardening/BinaryHardeningParsers.test.ts b/src/native/hardening/BinaryHardeningParsers.test.ts index cb42d69894..9f6fdc1d16 100644 --- a/src/native/hardening/BinaryHardeningParsers.test.ts +++ b/src/native/hardening/BinaryHardeningParsers.test.ts @@ -1112,7 +1112,7 @@ it("skips an empty SHT_NOTE section instead of rejecting the image", () => { } expect(sectionAt).toBeGreaterThan(0); bytes.writeBigUInt64LE(0n, sectionAt + 32); - expect(() => readElfImage(bytes)).not.toThrow(); + expect(() => readElfImage(bytes, "elf-directory")).not.toThrow(); const image = readElfHardening(bytes); expect(image.status).toBe("parsed"); expect(findRow(image.rows, "nx").state).toBe("enabled"); @@ -1715,7 +1715,7 @@ it("rejects p_filesz one above p_memsz like the ELF image reader", () => { const load = elfProgramAt(bytes, PT_LOAD); bytes.writeBigUInt64LE(bytes.readBigUInt64LE(load + 32) - 1n, load + 40); const message = "ELF load segment sizes or addresses are invalid"; - expect(() => readElfImage(bytes)).toThrow(message); + expect(() => readElfImage(bytes, "elf-directory")).toThrow(message); expect(() => readElfHardening(bytes)).toThrow( expect.objectContaining({ kind: "malformed", message }), ); diff --git a/src/native/hardening/BinaryHardeningProvider.ts b/src/native/hardening/BinaryHardeningProvider.ts index ac1bdb3a36..df354be7f8 100644 --- a/src/native/hardening/BinaryHardeningProvider.ts +++ b/src/native/hardening/BinaryHardeningProvider.ts @@ -35,7 +35,7 @@ import { BinaryResourceFailure, binaryResourceCeiling, } from "../../domain/binaryContainer.js"; -import { readElfHardening } from "./ElfHardening.js"; +import { readElfHardeningCooperative } from "./ElfHardening.js"; import { readMachoHardening } from "./MachoHardening.js"; import { readPeHardening } from "./PeHardening.js"; import { errorMessage } from "../../kernel/errorMessage.js"; @@ -109,7 +109,7 @@ export const readBinaryHardening = async ( return { artifact, format: "elf", - images: [readElfHardening(bytes, signal)], + images: [await readElfHardeningCooperative(bytes, signal)], limitations: [...BINARY_HARDENING_LIMITATIONS], }; if (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) diff --git a/src/native/hardening/ElfHardening.ts b/src/native/hardening/ElfHardening.ts index 49870b9887..e9ce543272 100644 --- a/src/native/hardening/ElfHardening.ts +++ b/src/native/hardening/ElfHardening.ts @@ -2,6 +2,7 @@ import { isParseLimit } from "../../domain/parseFailureAnalysisError.js"; import { assertArtifactReadActive, ArtifactReaderFailure, + yieldArtifactRead, } from "../../resources/index.js"; import { firstKeyed, @@ -68,12 +69,14 @@ import { ELF_PT_NOTE, assertElfDeclaredRangesFromBuffer, readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, type ElfProgramDeclaration, type NativeSection, } from "../../domain/native/nativeSectionDirectory.js"; const ET_EXEC = 2; const ET_DYN = 3; +const OPERATION = "inspect_binary_hardening"; const SHT_SYMTAB = 2; const SHT_STRTAB = 3; const SHT_DYNSYM = 11; @@ -184,7 +187,34 @@ export const readElfHardening = ( signal?: AbortSignal, ): BinaryHardeningImage => { assertArtifactReadActive(signal); - const directory = readElfDirectoryFromBuffer(bytes); + return hardeningFromDirectory( + bytes, + readElfDirectoryFromBuffer(bytes, OPERATION), + signal, + ); +}; + +/** A signal-owning inspection can pause while decoding ELF section names. */ +export const readElfHardeningCooperative = async ( + bytes: Buffer, + signal?: AbortSignal, +): Promise => { + if (signal === undefined) return readElfHardening(bytes); + assertArtifactReadActive(signal); + const directory = await readElfDirectoryFromBufferCooperative( + bytes, + () => yieldArtifactRead(signal), + OPERATION, + ); + assertArtifactReadActive(signal); + return hardeningFromDirectory(bytes, directory, signal); +}; + +const hardeningFromDirectory = ( + bytes: Buffer, + directory: ReturnType, + signal?: AbortSignal, +): BinaryHardeningImage => { // A report needs every declared payload, not only the tables that name them. assertElfDeclaredRangesFromBuffer(bytes); const { bits, little, machine, programs, sections } = directory; diff --git a/src/native/initializers/ElfInitializers.ts b/src/native/initializers/ElfInitializers.ts index 91bf66abfe..4273c34a04 100644 --- a/src/native/initializers/ElfInitializers.ts +++ b/src/native/initializers/ElfInitializers.ts @@ -1,5 +1,8 @@ import { setImmediate as immediate } from "node:timers/promises"; -import { assertArtifactReadActive } from "../../resources/index.js"; +import { + assertArtifactReadActive, + yieldArtifactRead, +} from "../../resources/index.js"; import { joinCoverageStatus, type CoverageStatus, @@ -21,7 +24,10 @@ import { elfRelativeRelocationType, readElfRelocationWritesSteps, } from "../../domain/native/elfPointerRelocations.js"; -import { readElfImage } from "../../domain/native/elfImage.js"; +import { + readElfImage, + readElfImageFromDirectory, +} from "../../domain/native/elfImage.js"; import { BinaryFormatFailure, BinaryMappingFailure, @@ -30,7 +36,10 @@ import { mappedFileOffset, type FileMappingIndex, } from "../../domain/binaryContainer.js"; -import { readElfDirectoryFromBuffer } from "../../domain/native/nativeSectionDirectory.js"; +import { + readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, +} from "../../domain/native/nativeSectionDirectory.js"; import type { NativeSection } from "../../domain/native/nativeSectionDirectory.js"; import { loadElfSymbolIndex, @@ -972,8 +981,17 @@ export const readElfInitializers = async ( signal?: AbortSignal, ): Promise => { assertArtifactReadActive(signal); - const image = readElfImage(bytes); - const directory = readElfDirectoryFromBuffer(bytes); + const operation = "inspect_native_initializers"; + const directory = + signal === undefined + ? readElfDirectoryFromBuffer(bytes, operation) + : await readElfDirectoryFromBufferCooperative( + bytes, + () => yieldArtifactRead(signal), + operation, + ); + assertArtifactReadActive(signal); + const image = readElfImageFromDirectory(bytes, directory); const scan: ElfScan = { bytes, bits: image.bits, diff --git a/src/native/initializers/NativeInitializersParsers.test.ts b/src/native/initializers/NativeInitializersParsers.test.ts index f90ce1e15e..d08acc725e 100644 --- a/src/native/initializers/NativeInitializersParsers.test.ts +++ b/src/native/initializers/NativeInitializersParsers.test.ts @@ -36,7 +36,11 @@ vi.mock("../../domain/binaryContainer.js", async (importOriginal) => { }; }); /** Element reads of every parsed ELF section table, so tests can count section walks. */ -const sectionReads = vi.hoisted(() => ({ count: 0, budget: Infinity })); +const sectionReads = vi.hoisted(() => ({ + count: 0, + budget: Infinity, + directories: 0, +})); vi.mock( "../../domain/native/nativeSectionDirectory.js", async (importOriginal) => { @@ -49,6 +53,7 @@ vi.mock( readElfDirectoryFromBuffer: ( ...args: Parameters ) => { + sectionReads.directories += 1; const directory = actual.readElfDirectoryFromBuffer(...args); return { ...directory, @@ -138,6 +143,13 @@ it.each([ }, ); +it("uses one ELF directory for initializer mappings and section facts", async () => { + sectionReads.directories = 0; + const report = await read(elf64PieInitializers()); + expect(report.entries.length).toBeGreaterThan(0); + expect(sectionReads.directories).toBe(1); +}); + it("preserves a real mapping to file offset zero instead of treating zero as missing", async () => { const bytes = elfUnmappedInitializerTargets({ type: 3 }); bytes.writeBigUInt64LE(0n, 64 + 16); diff --git a/src/native/initializers/NativeInitializersProvider.ts b/src/native/initializers/NativeInitializersProvider.ts index af9227cfa7..04f9680d62 100644 --- a/src/native/initializers/NativeInitializersProvider.ts +++ b/src/native/initializers/NativeInitializersProvider.ts @@ -15,6 +15,7 @@ import { artifactFilePrimaryFacts, assertArtifactReadActive, type ArtifactFilePrimaryFacts, + type StableArtifactSnapshot, ArtifactResourceScope, } from "../../resources/index.js"; import { @@ -77,9 +78,10 @@ export class NativeInitializersProvider implements NativeInitializersPort { input: InspectNativeInitializersInput, options?: ExecutionOptions, ) { + let snapshot: StableArtifactSnapshot | undefined; try { return await this.#resources.run(async () => { - const snapshot = await readStableArtifact( + snapshot = await readStableArtifact( input.path, input.max_file_bytes, options?.signal, @@ -101,7 +103,7 @@ export class NativeInitializersProvider implements NativeInitializersPort { return ok(report); }); } catch (cause) { - return err(nativeInitializersFailure(cause, input)); + return err(nativeInitializersFailure(cause, input, snapshot)); } } } @@ -264,6 +266,7 @@ const failurePrimary = ( const nativeInitializersFailure = ( cause: unknown, input: InspectNativeInitializersInput, + snapshot?: StableArtifactSnapshot, ): AnalysisError => projectStableArtifactFailure( cause, @@ -271,4 +274,5 @@ const nativeInitializersFailure = ( input.path, NATIVE_INITIALIZERS_PROVIDER.id, OPERATION, + snapshot, ); diff --git a/src/native/metadata/BinaryContainer.test.ts b/src/native/metadata/BinaryContainer.test.ts index 31c78ad036..e720bcfcb6 100644 --- a/src/native/metadata/BinaryContainer.test.ts +++ b/src/native/metadata/BinaryContainer.test.ts @@ -325,7 +325,7 @@ it.each([ ({ magic, bits, headerSize }) => { const bytes = Buffer.alloc(4); bytes.writeUInt32BE(magic); - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( new BinaryFormatFailure( "malformed", `truncated Mach-O header: a ${bits}-bit header needs ${headerSize} bytes; the file has 4`, @@ -338,14 +338,14 @@ it("keeps a JVM-sized FAT count unconfirmed while rejecting malformed zero-count const bytes = Buffer.alloc(8); bytes.writeUInt32BE(0xcafebabe); bytes.writeUInt32BE(0x0000003d, 4); // JVM major version 61; no FAT table follows. - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( new BinaryFormatFailure( "unsupported", "Only ELF, PE, and thin Mach-O native images are supported", ), ); bytes.writeUInt32BE(0, 4); - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( new BinaryFormatFailure( "malformed", "FAT header declares no architectures", @@ -376,10 +376,12 @@ const elfWithLoadSegment = (fileSize: bigint): Buffer => { }; it("names a load segment that runs past the end of the file as truncated", () => { - expect(readBinaryContainer(elfWithLoadSegment(120n)).mappings).toHaveLength( - 1, - ); - expect(() => readBinaryContainer(elfWithLoadSegment(0x1000n))).toThrow( + expect( + readBinaryContainer(elfWithLoadSegment(120n), "elf-directory").mappings, + ).toHaveLength(1); + expect(() => + readBinaryContainer(elfWithLoadSegment(0x1000n), "elf-directory"), + ).toThrow( expect.objectContaining({ failure: { kind: "truncated", diff --git a/src/native/metadata/Il2cppContainer.test.ts b/src/native/metadata/Il2cppContainer.test.ts index 25f3045eaa..dc5f1eb835 100644 --- a/src/native/metadata/Il2cppContainer.test.ts +++ b/src/native/metadata/Il2cppContainer.test.ts @@ -15,8 +15,8 @@ it.each(["unmapped", "zero-fill", "duplicate", "out-of-file"] as const)( expect(() => readGoBinaryImage(goNamed)).toThrow(/build-info|mapped/i); expect(readGoBinaryImage(control).build_info).toBeNull(); - expect(readBinaryContainer(goNamed).load_regions).toEqual( - readBinaryContainer(control).load_regions, + expect(readBinaryContainer(goNamed, "elf-directory").load_regions).toEqual( + readBinaryContainer(control, "elf-directory").load_regions, ); const image = openImageSteps(goNamed).next(); diff --git a/src/native/metadata/NativeSectionDirectory.test.ts b/src/native/metadata/NativeSectionDirectory.test.ts index 91c2d48f18..002f02228d 100644 --- a/src/native/metadata/NativeSectionDirectory.test.ts +++ b/src/native/metadata/NativeSectionDirectory.test.ts @@ -59,6 +59,7 @@ it("does not classify an unconfirmed pointer overlapping DOS data as a PE layout it("admits PT_INTERP, PT_GNU_STACK and PT_GNU_RELRO program headers", () => { const directory = readElfDirectoryFromBuffer( elfHardeningFixture({ interp: true, gnuRelro: true }), + "elf-directory", ); expect(directory.elf_type).toBe(3); expect(directory.programs.map((program) => program.type)).toEqual( @@ -74,7 +75,7 @@ it("admits PT_INTERP, PT_GNU_STACK and PT_GNU_RELRO program headers", () => { it("retains PT_GNU_EH_FRAME with its file and address span", () => { const bytes = elfUnwindSectionlessFixture(); - const directory = readElfDirectoryFromBuffer(bytes); + const directory = readElfDirectoryFromBuffer(bytes, "elf-directory"); expect(directory.programs.map((program) => program.type)).toEqual([ 1, ELF_PT_GNU_EH_FRAME, @@ -110,9 +111,12 @@ it.each([ "retains ELF%i little=%s compression independently of stored section coverage", (bits, little) => { const bytes = compressedElfNoteSectionFixture(bits, little); - const initial = readElfDirectoryFromBuffer(bytes).sections[1]!; + const initial = readElfDirectoryFromBuffer(bytes, "elf-directory") + .sections[1]!; expect(initial).toMatchObject({ compressed: true, file_backed: true }); - expect(readElfDirectoryFromBuffer(bytes).sections[2]).toMatchObject({ + expect( + readElfDirectoryFromBuffer(bytes, "elf-directory").sections[2], + ).toMatchObject({ compressed: false, file_backed: true, }); @@ -124,7 +128,9 @@ it.each([ else bytes.writeBigUInt64BE(BigInt(bytes.length), sizeAt); } else if (little) bytes.writeUInt32LE(bytes.length, sizeAt); else bytes.writeUInt32BE(bytes.length, sizeAt); - expect(readElfDirectoryFromBuffer(bytes).sections[1]).toMatchObject({ + expect( + readElfDirectoryFromBuffer(bytes, "elf-directory").sections[1], + ).toMatchObject({ compressed: true, file_backed: false, offset: initial.offset, diff --git a/src/native/metadata/PeImage.test.ts b/src/native/metadata/PeImage.test.ts index 19a1049337..7a789dc8d2 100644 --- a/src/native/metadata/PeImage.test.ts +++ b/src/native/metadata/PeImage.test.ts @@ -68,7 +68,7 @@ it.each(["NE", "LE", "LX"])( (signature) => { const bytes = pe(0x014c).subarray(0, 66); bytes.write(signature, 64, "ascii"); - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( expect.objectContaining({ kind: "unsupported", message: expect.stringContaining(signature), diff --git a/src/resources/ArtifactFileOperation.ts b/src/resources/ArtifactFileOperation.ts index 7463441995..dce0fb61e2 100644 --- a/src/resources/ArtifactFileOperation.ts +++ b/src/resources/ArtifactFileOperation.ts @@ -1,4 +1,5 @@ import type { FileHandle } from "node:fs/promises"; +import { scheduler } from "node:timers/promises"; import { RegularFileOperationCancelled, type StableRegularFileDescriptor, @@ -245,6 +246,17 @@ export const assertArtifactReadActive = ( }); }; +/** + * Let a queued abort land, then report it. Synchronous multi-frame decoding + * cannot observe the signal until the caller yields. + */ +export const yieldArtifactRead = async ( + signal: AbortSignal | undefined, +): Promise => { + await scheduler.yield(); + assertArtifactReadActive(signal); +}; + /** * Project a failure raised by a domain byte parser over artifact bytes onto the * reader's reasons: the caller's cancellation (`signal.reason`) becomes diff --git a/src/resources/ArtifactOperationFailure.test.ts b/src/resources/ArtifactOperationFailure.test.ts index b973a765bf..7fe6ec4590 100644 --- a/src/resources/ArtifactOperationFailure.test.ts +++ b/src/resources/ArtifactOperationFailure.test.ts @@ -1,4 +1,9 @@ import { expect, it } from "vitest"; +import { ZstandardFrameAllowanceError } from "../domain/compression/zstandardFrameAllowance.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../domain/analysisResourceLimits.js"; import { createHash } from "node:crypto"; import { @@ -29,6 +34,26 @@ import { unsupportedInput, } from "../domain/parseFailure.js"; +it("projects a Zstandard frame-cap as an adjustable artifact limit", () => { + expect( + translateArtifactFailure( + "extract_artifact", + new ZstandardFrameAllowanceError({ maxFrames: 4, source: "runtime" }, 10), + ), + ).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 4, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); +}); + it.each([ malformedInput("pe", "bad field"), truncatedInput("pe", "short field"), diff --git a/src/resources/ArtifactOperationFailure.ts b/src/resources/ArtifactOperationFailure.ts index b4991900dc..9a60bc216b 100644 --- a/src/resources/ArtifactOperationFailure.ts +++ b/src/resources/ArtifactOperationFailure.ts @@ -1,4 +1,8 @@ import { analysisSettingLimit } from "../domain/analysisSettingDefinitions.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../domain/compression/zstandardFrameAllowance.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { ArtifactOperationError } from "../domain/artifactOperationError.js"; import { InternalInvariantError } from "../domain/internalInvariant.js"; @@ -18,6 +22,8 @@ export const translateArtifactFailure = ( cause: unknown, ): AnalysisError => { if (cause instanceof InternalInvariantError) throw cause; + if (cause instanceof ZstandardFrameAllowanceError) + return projectZstandardFrameAllowance(operation, cause); if (cause instanceof ArtifactReaderFailure) { const error = new ArtifactOperationError( operation, diff --git a/src/resources/ZipArtifactReader.ts b/src/resources/ZipArtifactReader.ts index 37cfe1fab7..c5f048ea5b 100644 --- a/src/resources/ZipArtifactReader.ts +++ b/src/resources/ZipArtifactReader.ts @@ -13,7 +13,7 @@ import { import { ArtifactFileSource } from "./ArtifactFileSource.js"; import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js"; import type { FileHandle } from "node:fs/promises"; -import { Duplex, PassThrough, type Readable } from "node:stream"; +import { compose, Duplex, PassThrough, type Readable } from "node:stream"; import { once } from "node:events"; import { constants, createZstdDecompress } from "node:zlib"; @@ -56,6 +56,10 @@ import { readFileHandleRange } from "../filesystem/readFileHandleRange.js"; import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { decodeZipCp437 } from "../domain/zip/zipEntryText.js"; import { invariantViolation } from "../domain/internalInvariant.js"; +import { + ZstandardFramingError, + zstandardFrameChunks, +} from "../domain/compression/zstandardFrameChunks.js"; const ZIP_UTF8_DECODER = strictUtf8Decoder("preserve"); @@ -83,13 +87,16 @@ class ZipZstandardDecompressionStream { constructor(_format: string, options?: DecompressionStreamOptions) { const stream = Duplex.toWeb( - createZstdDecompress({ - chunkSize: options?.chunkSize, - // Match the existing ELF/GPU Zstandard decoder's 128 MiB window cap. - params: { - [constants.ZSTD_d_windowLogMax]: ZIP_ZSTANDARD_WINDOW_LOG_MAX, - }, - }), + compose( + new PassThrough(), + zstandardFrameChunks, + createZstdDecompress({ + chunkSize: options?.chunkSize, + params: { + [constants.ZSTD_d_windowLogMax]: ZIP_ZSTANDARD_WINDOW_LOG_MAX, + }, + }), + ), ); this.readable = stream.readable; this.writable = stream.writable; @@ -648,7 +655,8 @@ export const typedZipFailure = ( }); const invalidZstandard = compressionMethod === 93 && - (code === "ZSTD_error_prefix_unknown" || + (cause instanceof ZstandardFramingError || + code === "ZSTD_error_prefix_unknown" || code === "ZSTD_error_checksum_wrong" || code === "ZSTD_error_corruption_detected" || code === "ZSTD_error_dictionary_corrupted" || diff --git a/src/resources/index.ts b/src/resources/index.ts index c8963a4af6..e2c2cbce60 100644 --- a/src/resources/index.ts +++ b/src/resources/index.ts @@ -37,6 +37,7 @@ export { artifactReadCancelledBy, artifactSelectedPathErrno, assertArtifactReadActive, + yieldArtifactRead, bindArtifactFileCleanup, mapArtifactFileFailure, withArtifactFileHandle, diff --git a/src/rust/RustBinaryImage.ts b/src/rust/RustBinaryImage.ts index 73d93729f8..23a83f005a 100644 --- a/src/rust/RustBinaryImage.ts +++ b/src/rust/RustBinaryImage.ts @@ -1,7 +1,11 @@ import type { ImageSection } from "../domain/binaryContainer.js"; import { malformedInput, unsupportedInput } from "../domain/parseFailure.js"; import { keyedBy, uniqueKeyed } from "../domain/keyedCollection.js"; -import { readBinaryContainer } from "../domain/native/readBinaryContainer.js"; +import type { ZstandardFramePause } from "../domain/compression/zstandardExactFrame.js"; +import { + readBinaryContainer, + readBinaryContainerCooperative, +} from "../domain/native/readBinaryContainer.js"; import type { RustAuditableMetadata, RustBinary, @@ -59,9 +63,44 @@ export const readRustBinaryImage = ( bytes: Buffer, maxRustcHashes = DEFAULT_RUSTC_HASH_LIMIT, maxOutputBytes: number = RUST_PRODUCER_DEFAULTS.output_bytes, +): RustBinaryImage => + finishRustBinaryImage( + bytes, + maxRustcHashes, + maxOutputBytes, + hasWasmPreamble(bytes) ? wasmLayout(bytes) : nativeLayout(bytes), + ); + +/** Same image as {@link readRustBinaryImage}. ELF section names yield through `pause`. */ +export const readRustBinaryImageCooperative = async ( + bytes: Buffer, + maxRustcHashes: number | undefined, + maxOutputBytes: number | undefined, + pause: ZstandardFramePause, +): Promise => + finishRustBinaryImage( + bytes, + maxRustcHashes ?? DEFAULT_RUSTC_HASH_LIMIT, + maxOutputBytes ?? RUST_PRODUCER_DEFAULTS.output_bytes, + hasWasmPreamble(bytes) + ? wasmLayout(bytes) + : nativeLayout( + bytes, + await readBinaryContainerCooperative( + bytes, + "inspect_rust_binary", + pause, + ), + ), + ); + +const finishRustBinaryImage = ( + bytes: Buffer, + maxRustcHashes: number, + maxOutputBytes: number, + layout: RustImageLayout, ): RustBinaryImage => { - const { section_name, dependency, compiler_strings, ...identity } = - hasWasmPreamble(bytes) ? wasmLayout(bytes) : nativeLayout(bytes); + const { section_name, dependency, compiler_strings, ...identity } = layout; return { ...identity, auditable_metadata: @@ -85,8 +124,10 @@ export const readRustBinaryImage = ( }; }; -const nativeLayout = (bytes: Buffer): RustImageLayout => { - const container = readBinaryContainer(bytes); +const nativeLayout = ( + bytes: Buffer, + container = readBinaryContainer(bytes, "inspect_rust_binary"), +): RustImageLayout => { const section_name = rustDependencySectionName(container.format); const dependency = dependencySection(section_name, container.sections); return { diff --git a/src/rust/RustBinaryProvider.ts b/src/rust/RustBinaryProvider.ts index 64a487e450..11df5765b2 100644 --- a/src/rust/RustBinaryProvider.ts +++ b/src/rust/RustBinaryProvider.ts @@ -13,6 +13,7 @@ import { type ArtifactFilePrimaryFacts, artifactFileCancellationCause, assertArtifactReadActive, + yieldArtifactRead, ArtifactResourceScope, ArtifactReaderFailure, } from "../resources/index.js"; @@ -37,6 +38,7 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; import { readRustBinaryImage, + readRustBinaryImageCooperative, type RustBinaryImage, } from "./RustBinaryImage.js"; import { requireSelectedNativeImage } from "../domain/apple/machoSliceSelection.js"; @@ -121,11 +123,18 @@ export class RustBinaryProvider implements RustBinaryPort { input.architecture, ); const image = inFile( - readRustBinaryImage( - selected.bytes, - input.max_rustc_hashes, - limits.output_bytes, - ), + options?.signal === undefined + ? readRustBinaryImage( + selected.bytes, + input.max_rustc_hashes, + limits.output_bytes, + ) + : await readRustBinaryImageCooperative( + selected.bytes, + input.max_rustc_hashes, + limits.output_bytes, + () => yieldArtifactRead(options.signal), + ), selected.fileOffset, ); const report: RustBinary = { diff --git a/tests/acceptance/nativeElfFrameAllowanceMcp.test.ts b/tests/acceptance/nativeElfFrameAllowanceMcp.test.ts new file mode 100644 index 0000000000..997332309a --- /dev/null +++ b/tests/acceptance/nativeElfFrameAllowanceMcp.test.ts @@ -0,0 +1,103 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { elfNoteFixture } from "../../src/native/elf/ElfNotes.fixture.js"; +import { INLINE_EVIDENCE_MCP_ENVIRONMENT } from "../../scripts/lib/mcp-verification-environment.mjs"; +import { elfFrameAllowanceFixture } from "../fixtures/binaryDiagnostics/elfFrameAllowance.js"; +import { createIl2cppMetadataFixture } from "../fixtures/il2cpp/metadata.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +it("reports an ELF section-name frame refusal through each native diagnostic", async () => { + const root = await createTestTempDirectory("rea-native-elf-frame-cap-"); + const path = join(root, "image.elf"); + const primary = join(root, "primary.elf"); + const metadata = join(root, "global-metadata.dat"); + const bytes = elfFrameAllowanceFixture(); + await writeFile(path, bytes); + await writeFile(primary, elfNoteFixture()); + await writeFile(metadata, createIl2cppMetadataFixture().bytes); + const { call } = await connectLocalToolsMcp(INLINE_EVIDENCE_MCP_ENVIRONMENT); + const operations = [ + "inspect_binary_hardening", + "inspect_elf_debug_link", + "inspect_elf_unwind_ranges", + "inspect_kernel_module", + "inspect_native_exports", + "inspect_native_initializers", + "inspect_go_binary", + "inspect_rust_binary", + ] as const; + const setAllowance = async (value: number) => { + const result = await call("update_analysis_settings", { + changes: [{ setting: "zstd.max_frames", action: "set", value }], + }); + expect(result.isError).not.toBe(true); + }; + + await setAllowance(1); + for (const operation of [...operations, "inspect_il2cpp_metadata"]) { + const settings = await call("inspect_analysis_settings", { operation }); + expect(settings.structuredContent, operation).toMatchObject({ + settings: expect.arrayContaining([ + expect.objectContaining({ setting: "zstd.max_frames" }), + ]), + }); + const result = await call( + operation, + operation === "inspect_il2cpp_metadata" + ? { path: metadata, binary_path: path } + : { path }, + ); + expect(parseMcpToolError(result).error, operation).toMatchObject({ + code: "resource_constraint", + details: { + operation, + resource: "cpu", + reported_limits: { + boundary: "zstd.max_frames", + maximum: 1, + required: 2, + }, + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + unit: "count", + }, + }, + }); + } + + const candidate = await call("inspect_elf_debug_link", { + path: primary, + candidate_path: path, + }); + expect(candidate.isError).not.toBe(true); + expect(candidate.structuredContent).toMatchObject({ + normalized_result: { + candidate: { + path, + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + format: "elf64", + debug_info: "unknown", + symtab: "unknown", + }, + limitations: expect.arrayContaining([ + expect.stringContaining("zstd.max_frames"), + ]), + }, + }); + + await setAllowance(2); + for (const operation of operations) { + const result = await call(operation, { path }); + expect( + result.isError, + `${operation}: ${JSON.stringify(result.content)}`, + ).not.toBe(true); + } +}); diff --git a/tests/boundary/android/androidJniLibraries.test.ts b/tests/boundary/android/androidJniLibraries.test.ts index 3f58fb60ea..4e8ee18fa5 100644 --- a/tests/boundary/android/androidJniLibraries.test.ts +++ b/tests/boundary/android/androidJniLibraries.test.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { scheduler } from "node:timers/promises"; import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { expect, it, onTestFinished, vi } from "vitest"; import { Ajv2020 } from "ajv/dist/2020.js"; @@ -31,6 +32,8 @@ import { zstandardZipFixture, } from "../../fixtures/zipZstandard.js"; import { jniElfFixture } from "../../fixtures/android/jniElf.js"; +import { runWithAnalysisOperationSnapshot } from "../../../src/domain/analysisOperationSnapshot.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { ArtifactReaderFailure, ZipArtifactReader, @@ -240,6 +243,58 @@ it("keeps the ZIP decoder window's recompression advice in the library failure", }); }); +it("retains a backed library identity when the ELF section-name frame budget is refused", async () => { + const bytes = elfFrameAllowanceFixture(); + const library = "lib/x86_64/libframe.so"; + const source = await fixture([ + [library, bytes], + [LIBRARY, jniElfFixture().bytes], + ]); + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => read(source), + ); + expect(result.libraries).toEqual([ + expect.objectContaining({ + path: library, + size: bytes.length, + sha256: digest(bytes), + facts: null, + failure: { + kind: "capacity", + message: expect.stringContaining("zstd.max_frames"), + }, + }), + expect.objectContaining({ + path: LIBRARY, + facts: expect.any(Object), + failure: null, + }), + ]); + expect(source.resources.hasPending).toBe(false); + + const checkpoints: AndroidJniLibraryCheckpoint[] = []; + const close = vi + .spyOn(ZipArtifactReader.prototype, "close") + .mockRejectedValueOnce(new Error("controlled ZIP close failure")); + try { + await expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => + read(source, {}, undefined, undefined, (checkpoint) => + checkpoints.push(checkpoint), + ), + ), + ).rejects.toBeInstanceOf(ProviderCleanupError); + expect(checkpoints[0]?.libraries).toEqual(result.libraries); + expect(source.resources.hasPending).toBe(true); + } finally { + close.mockRestore(); + await source.resources.close(); + } +}); + it("preserves declared capacity metadata without presenting unread bytes as verified", async () => { const source = await fixture(); const originalEntries = ZipArtifactReader.prototype.entries; @@ -560,82 +615,109 @@ for (const first of ["caller", "deadline"] as const) } }); -it("keeps completed facts and the interrupted carrier when a later library is cancelled", async () => { - const second = "lib/arm64-v8a/libsecond.so", - third = "lib/arm64-v8a/libunexamined.so"; - const bytes = jniElfFixture().bytes; - const source = await fixture([ - [LIBRARY, bytes], - [second, bytes], - [third, bytes], - ]); - const controller = new AbortController(), - reason = { caller: "stop before second member bytes" }; - const originalOpen = ZipArtifactReader.prototype.open; - const open = vi - .spyOn(ZipArtifactReader.prototype, "open") - .mockImplementation(async function ( - this: ZipArtifactReader, - entry, - signal, - ) { - if (entry.path === second) { - controller.abort(reason); - throw new JadxInterruption(reason); - } - return originalOpen.call(this, entry, signal); - }); - let checkpoint: AndroidJniLibraryCheckpoint | undefined; - try { - const failure = await read( - source, - {}, - controller.signal, - undefined, - (value) => { - checkpoint = value; - }, - ).catch((cause: unknown) => cause); - expect(failure).toBeInstanceOf(AnalysisCancelledError); - if (!(failure instanceof AnalysisCancelledError)) throw failure; - expect(failure.cause).toBe(reason); - expect(checkpoint).toMatchObject({ - scan_state: "interrupted", - inventory: { - status: "partial", - examined_entries: 2, - selected_libraries: 2, - }, - }); - expect(checkpoint?.libraries.map((library) => library.path)).toEqual([ - LIBRARY, - second, +it.each(["member open", "section names"] as const)( + "keeps completed facts and the interrupted carrier when a later library is cancelled during %s", + async (stage) => { + const parsingNames = stage === "section names"; + const second = "lib/arm64-v8a/libsecond.so", + third = "lib/arm64-v8a/libunexamined.so"; + const bytes = jniElfFixture().bytes; + const secondBytes = parsingNames ? elfFrameAllowanceFixture() : bytes; + const source = await fixture([ + [LIBRARY, bytes], + [second, secondBytes], + [third, bytes], ]); - expect(checkpoint?.libraries[0]).toMatchObject({ - sha256: digest(bytes), - size: bytes.length, - facts: { tables: expect.any(Array) }, - }); - expect(checkpoint?.libraries[1]).toMatchObject({ - sha256: null, - size: null, - declared_size: bytes.length, - compressed_size: bytes.length, - facts: null, - failure: { kind: "incomplete" }, - }); - expect(checkpoint?.inventory.unknowns.join("\n")).toContain( - "unexamined selected occurrences remain unknown", - ); - if (checkpoint === undefined) - throw new Error("Missing interrupted scan checkpoint"); - expect( - androidPartialObservationSchema.safeParse(partial(source, checkpoint)) - .success, - ).toBe(true); - expect(JSON.parse(JSON.stringify(checkpoint))).toEqual(checkpoint); - expect(source.resources.hasPending).toBe(false); - } finally { - open.mockRestore(); - } -}); + const controller = new AbortController(), + reason = { caller: "stop during " + stage }; + const originalOpen = ZipArtifactReader.prototype.open; + const open = vi + .spyOn(ZipArtifactReader.prototype, "open") + .mockImplementation(async function ( + this: ZipArtifactReader, + entry, + signal, + ) { + if (entry.path === second && !parsingNames) { + controller.abort(reason); + throw new JadxInterruption(reason); + } + return originalOpen.call(this, entry, signal); + }); + const yieldToEventLoop = scheduler.yield.bind(scheduler); + const pause = parsingNames + ? vi.spyOn(scheduler, "yield").mockImplementationOnce(async () => { + await yieldToEventLoop(); + controller.abort(reason); + }) + : undefined; + let checkpoint: AndroidJniLibraryCheckpoint | undefined; + try { + const failure = await read( + source, + {}, + controller.signal, + undefined, + (value) => { + checkpoint = value; + }, + ).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(AnalysisCancelledError); + if (!(failure instanceof AnalysisCancelledError)) throw failure; + expect(failure.cause).toBe(reason); + expect(checkpoint).toMatchObject({ + scan_state: "interrupted", + inventory: { + status: "partial", + examined_entries: 2, + selected_libraries: 2, + }, + }); + expect(checkpoint?.libraries.map((library) => library.path)).toEqual([ + LIBRARY, + second, + ]); + expect(checkpoint?.libraries[0]).toMatchObject({ + sha256: digest(bytes), + size: bytes.length, + facts: { tables: expect.any(Array) }, + }); + expect(checkpoint?.libraries[1]).toMatchObject({ + sha256: parsingNames ? digest(secondBytes) : null, + size: parsingNames ? secondBytes.length : null, + declared_size: secondBytes.length, + compressed_size: secondBytes.length, + facts: null, + failure: { kind: "incomplete" }, + }); + expect(checkpoint?.inventory.unknowns.join("\n")).toContain( + "unexamined selected occurrences remain unknown", + ); + if (checkpoint === undefined) + throw new Error("Missing interrupted scan checkpoint"); + expect( + androidPartialObservationSchema.safeParse(partial(source, checkpoint)) + .success, + ).toBe(true); + expect(JSON.parse(JSON.stringify(checkpoint))).toEqual(checkpoint); + expect(source.resources.hasPending).toBe(false); + if (parsingNames) { + const retry = await read(source); + expect(retry.inventory).toMatchObject({ + status: "complete", + selected_libraries: 3, + }); + expect(retry.libraries[1]).toMatchObject({ + path: second, + sha256: digest(secondBytes), + size: secondBytes.length, + facts: expect.any(Object), + failure: null, + }); + } + } finally { + pause?.mockRestore(); + open.mockRestore(); + } + }, +); diff --git a/tests/boundary/filesystem/artifactZipIntegrity.test.ts b/tests/boundary/filesystem/artifactZipIntegrity.test.ts index c501298943..a18c4fad7c 100644 --- a/tests/boundary/filesystem/artifactZipIntegrity.test.ts +++ b/tests/boundary/filesystem/artifactZipIntegrity.test.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { zstdCompressSync } from "node:zlib"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { expect, it, onTestFinished } from "vitest"; @@ -84,6 +85,36 @@ cliTest( }, ); +it("does not publish a method-93 member with an incomplete final frame magic", async () => { + const root = await createTestTempDirectory("rea-zip-zstd-incomplete-"); + const path = join(root, "method93.zip"); + const outputRoot = join(root, "extracted"); + const content = Buffer.from("first complete text frame\n"); + const second = zstdCompressSync(Buffer.from("second text frame\n")); + const compressed = Buffer.concat([ + zstdCompressSync(content), + second.subarray(0, 3), + ]); + const archive = zstandardZipFixture(content, { compressed }); + await writeFile(path, archive); + + const target = await parseBinaryTarget(path); + if (!target.ok) throw target.error; + const result = await withArtifactClient(target.value, (client) => + client.execute("extract_artifact", { output_root: outputRoot }), + ); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "ArtifactOperationError", + reason: "format", + detail: "ZIP archive is malformed: Invalid compressed data", + }, + }); + expect(await readdir(root)).toEqual(["method93.zip"]); + expect((await readFile(path)).equals(archive)).toBe(true); +}); + for (const zip64 of [false, true]) { cliTest( `inspects and extracts an empty ${zip64 ? "Zip64" : "classic ZIP"} without changing its identity`, diff --git a/tests/boundary/filesystem/binaryHardening.test.ts b/tests/boundary/filesystem/binaryHardening.test.ts index ea1ce36ff8..87eee28373 100644 --- a/tests/boundary/filesystem/binaryHardening.test.ts +++ b/tests/boundary/filesystem/binaryHardening.test.ts @@ -1,8 +1,15 @@ import { readFile, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import { runWithAnalysisOperationSnapshot } from "../../../src/domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../../../src/domain/analysisResourceLimits.js"; import { createBinaryHardeningService } from "../../../src/composition/binaryDiagnostics.js"; import { elfHardeningFixture } from "../../../src/native/hardening/BinaryHardening.fixture.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("binary hardening stable filesystem boundary", () => { @@ -98,3 +105,68 @@ describe("binary hardening stable filesystem boundary", () => { } }); }); + +describe("binary hardening compressed ELF names", () => { + it("projects a section-name frame cap as an adjustable zstd.max_frames limit", async () => { + const root = await createTestTempDirectory("rea-binary-hardening-zstd-"); + const path = join(root, "app"); + await writeFile(path, elfFrameAllowanceFixture()); + const service = createBinaryHardeningService(); + try { + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => + service.inspect({ path }, { signal: new AbortController().signal }), + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + operation: "inspect_binary_hardening", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); + } finally { + await service.close(); + } + }); + + it("cancels a many-frame compressed section-name table", async () => { + const root = await createTestTempDirectory("rea-binary-hardening-cancel-"); + const path = join(root, "app"); + await writeFile(path, elfFrameAllowanceFixture()); + const reason = new Error("stop hardening frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + const service = createBinaryHardeningService(); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_binary_hardening", + cause: reason, + }); + } finally { + spy.mockRestore(); + await service.close(); + } + }); +}); diff --git a/tests/boundary/filesystem/elfDebugLink.test.ts b/tests/boundary/filesystem/elfDebugLink.test.ts index c28cb83ffe..e1ec8f85c5 100644 --- a/tests/boundary/filesystem/elfDebugLink.test.ts +++ b/tests/boundary/filesystem/elfDebugLink.test.ts @@ -1,7 +1,9 @@ -import { crc32 } from "node:zlib"; +import * as zlib from "node:zlib"; +import { createHash } from "node:crypto"; import { mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; import { AnalysisResourceConstraintError } from "../../../src/domain/analysisErrorCore.js"; import { createElfDebugLinkService } from "../../../src/composition/binaryDiagnostics.js"; import { @@ -10,7 +12,10 @@ import { ELF_DEBUG_LINK_UNAVAILABLE_CHECK_LIMITATION, elfDebugLinkSchema, } from "../../../src/domain/native/elfDebugLink.js"; -import { gnuBuildIdNote } from "../../../src/native/elf/ElfNotes.fixture.js"; +import { + compressedElfNoteFixture, + gnuBuildIdNote, +} from "../../../src/native/elf/ElfNotes.fixture.js"; import { debuglinkSection, elfDebugCandidateFixture, @@ -19,6 +24,37 @@ import { FIXTURE_DEBUG_NAME, } from "../../../src/native/elf/ElfDebugLink.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; + +vi.mock(import("node:zlib"), { spy: true }); + +it("inflates section names once per artifact while preserving build-ID comparison", async () => { + const root = await createTestTempDirectory("rea-elf-debug-link-name-reuse-"); + const path = join(root, "prog"); + const candidatePath = join(root, "prog.debug"); + const bytes = compressedElfNoteFixture(); + await writeFile(path, bytes); + await writeFile(candidatePath, bytes); + const service = createElfDebugLinkService(); + const inflate = vi.mocked(zlib.inflateSync).mockClear(); + try { + const result = await service.inspect( + { path, candidate_path: candidatePath }, + { signal: new AbortController().signal }, + ); + if (!result.ok) throw result.error; + const report = elfDebugLinkSchema.parse(result.value.normalized_result); + expect(report.build_id).toMatchObject({ + status: "observed", + sources: [{ table_kind: "section", table_index: 1 }], + }); + expect(report.verification?.build_id.status).toBe("match"); + expect(inflate).toHaveBeenCalledTimes(2); + } finally { + inflate.mockClear(); + await service.close(); + } +}); describe("ELF debug-link stable filesystem boundary", () => { it("reports non-file and malformed ELF selections as caller input errors", async () => { @@ -153,6 +189,52 @@ describe("ELF debug-link stable filesystem boundary", () => { }); describe("ELF debug-link streamed candidate boundary", () => { + it("retains streamed candidate identity when cancellation stops ELF comparison", async () => { + const root = await createTestTempDirectory( + "rea-elf-debug-link-cancel-candidate-", + ); + const path = join(root, "prog"); + const candidatePath = join(root, "prog.debug"); + const candidate = elfFrameAllowanceFixture(); + await writeFile(path, elfDebugLinkFixture()); + await writeFile(candidatePath, candidate); + const service = createElfDebugLinkService(); + const controller = new AbortController(); + const reason = new Error("stop while decoding candidate names"); + const yieldToEventLoop = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await yieldToEventLoop(); + controller.abort(reason); + }); + try { + const result = await service.inspect( + { path, candidate_path: candidatePath }, + { signal: controller.signal }, + ); + if (result.ok) + throw new Error("Expected candidate comparison cancellation"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + partialObservation: { + build_id: { status: "observed", hex: FIXTURE_BUILD_ID_HEX }, + candidate: { + path: candidatePath, + sha256: createHash("sha256").update(candidate).digest("hex"), + bytes: candidate.length, + debug_info: "unknown", + symtab: "unknown", + }, + verification: null, + }, + }); + } finally { + pause.mockRestore(); + await service.close(); + } + }); it("matches the known CRC32 of a non-ELF candidate", async () => { const root = await createTestTempDirectory("rea-elf-debug-link-crc32-"); const path = join(root, "prog"); @@ -261,7 +343,11 @@ describe("ELF debug-link candidate degradation", () => { candidate.set([0x7f, 0x45, 0x4c, 0x46, 2, 1, 9]); const image = elfDebugLinkFixture({ ...(presence === "with" ? {} : { notes: null }), - debuglink: debuglinkSection(FIXTURE_DEBUG_NAME, crc32(candidate), true), + debuglink: debuglinkSection( + FIXTURE_DEBUG_NAME, + zlib.crc32(candidate), + true, + ), }); const path = join(root, "prog"); const candidatePath = join(root, FIXTURE_DEBUG_NAME); @@ -303,7 +389,11 @@ describe("ELF debug-link candidate degradation", () => { ]), }); const image = elfDebugLinkFixture({ - debuglink: debuglinkSection(FIXTURE_DEBUG_NAME, crc32(candidate), true), + debuglink: debuglinkSection( + FIXTURE_DEBUG_NAME, + zlib.crc32(candidate), + true, + ), }); const path = join(root, "prog"); const candidatePath = join(root, FIXTURE_DEBUG_NAME); diff --git a/tests/boundary/filesystem/elfSectionNameCancellation.test.ts b/tests/boundary/filesystem/elfSectionNameCancellation.test.ts new file mode 100644 index 0000000000..51177fa472 --- /dev/null +++ b/tests/boundary/filesystem/elfSectionNameCancellation.test.ts @@ -0,0 +1,73 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { scheduler } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; +import { + createBinaryHardeningService, + createElfUnwindRangesService, + createNativeExportsService, +} from "../../../src/composition/binaryDiagnostics.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it.each([ + [ + "inspect_binary_hardening", + createBinaryHardeningService, + { format: "elf", images: [{ status: "parsed" }] }, + ], + ["inspect_elf_unwind_ranges", createElfUnwindRangesService, {}], + ["inspect_native_exports", createNativeExportsService, { format: "elf" }], +] as const)( + "%s observes cancellation while decoding ELF names after the file snapshot closes", + async (operation, createService, expectedReport) => { + const root = await createTestTempDirectory("rea-elf-names-cancel-"); + const path = join(root, "app"); + const bytes = elfFrameAllowanceFixture(); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + await writeFile(path, bytes); + const service = createService(); + const controller = new AbortController(); + const reason = new Error("stop section-name decoding"); + const yieldToEventLoop = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await yieldToEventLoop(); + controller.abort(reason); + }); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation, + cause: reason, + partialObservation: { + kind: "file-snapshot-hash", + path, + sha256, + bytes: bytes.length, + }, + }); + const retry = await service.inspect( + { path }, + { signal: new AbortController().signal }, + ); + if (!retry.ok) throw retry.error; + expect(retry.value.normalized_result).toMatchObject({ + artifact: { path, sha256, bytes: bytes.length }, + ...expectedReport, + }); + expect((await readFile(path)).equals(bytes)).toBe(true); + } finally { + pause.mockRestore(); + await service.close(); + } + }, +); diff --git a/tests/boundary/filesystem/elfUnwindRanges.test.ts b/tests/boundary/filesystem/elfUnwindRanges.test.ts index 195cc20df4..b434d59e5b 100644 --- a/tests/boundary/filesystem/elfUnwindRanges.test.ts +++ b/tests/boundary/filesystem/elfUnwindRanges.test.ts @@ -1,9 +1,16 @@ import { symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import { runWithAnalysisOperationSnapshot } from "../../../src/domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../../../src/domain/analysisResourceLimits.js"; import { createElfUnwindRangesService } from "../../../src/composition/binaryDiagnostics.js"; import { MAX_ELF_UNWIND_SYMBOL_TEXT_BYTES } from "../../../src/domain/native/elfUnwindRanges.js"; import { elfUnwindFixture } from "../../../src/native/elf/ElfUnwindTables.fixture.js"; +import { compressedElfNoteFixture } from "../../../src/native/elf/ElfNotes.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("ELF unwind range stable filesystem boundary", () => { @@ -176,3 +183,86 @@ describe("ELF unwind range budgets", () => { } }); }); + +it("projects a section-name frame cap as an adjustable zstd.max_frames limit", async () => { + const root = await createTestTempDirectory("rea-elf-unwind-zstd-"); + const path = join(root, "program"); + const frame = Buffer.from([ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, 0x61, + ]); + await writeFile( + path, + compressedElfNoteFixture( + 64, + true, + 2, + ".note.gnu.build-id", + Buffer.concat([frame, frame]), + ), + ); + const service = createElfUnwindRangesService(); + try { + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => service.inspect({ path }), + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + operation: "inspect_elf_unwind_ranges", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); + } finally { + await service.close(); + } +}); + +it("cancels a many-frame compressed section-name table", async () => { + const root = await createTestTempDirectory("rea-elf-unwind-cancel-"); + const path = join(root, "program"); + const frame = Buffer.from([ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, 0x61, + ]); + const frames = Buffer.alloc(frame.length * 32); + for (let index = 0; index < 32; index += 1) + frame.copy(frames, index * frame.length); + await writeFile( + path, + compressedElfNoteFixture(64, true, 2, ".note.gnu.build-id", frames), + ); + const reason = new Error("stop unwind frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + const service = createElfUnwindRangesService(); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_elf_unwind_ranges", + cause: reason, + }); + } finally { + spy.mockRestore(); + await service.close(); + } +}); diff --git a/tests/boundary/filesystem/kernelModule.test.ts b/tests/boundary/filesystem/kernelModule.test.ts index 86ec5e3089..0c5deb8d6d 100644 --- a/tests/boundary/filesystem/kernelModule.test.ts +++ b/tests/boundary/filesystem/kernelModule.test.ts @@ -103,23 +103,26 @@ describe("kernel module stable filesystem boundary", () => { } }); - it("reports a gzip container as unsupported without treating it as a module", async () => { - const root = await createTestTempDirectory("rea-kernel-module-gzip-"); - const path = join(root, "demo.ko.gz"); - await writeFile(path, compressedKernelModuleContainer("gzip")); - const service = createKernelModuleService(); - try { - const result = await service.inspect({ path }); - if (!result.ok) throw result.error; - expect(fileIdentity(result.value.subject).format).toBe("file"); - expect(result.value.normalized_result).toMatchObject({ - applicability: "unsupported", - compression: { codec: "gzip", status: "unsupported" }, - next_probe: { operation: "external_decompression" }, - modinfo: null, - }); - } finally { - await service.close(); - } - }); + it.each(["gzip", "xz", "zstd"] as const)( + "reports a %s container as unsupported without treating it as a module", + async (codec) => { + const root = await createTestTempDirectory(`rea-kernel-module-${codec}-`); + const path = join(root, `demo.ko.${codec === "gzip" ? "gz" : codec}`); + await writeFile(path, compressedKernelModuleContainer(codec)); + const service = createKernelModuleService(); + try { + const result = await service.inspect({ path }); + if (!result.ok) throw result.error; + expect(fileIdentity(result.value.subject).format).toBe("file"); + expect(result.value.normalized_result).toMatchObject({ + applicability: "unsupported", + compression: { codec, status: "unsupported" }, + next_probe: { operation: "external_decompression" }, + modinfo: null, + }); + } finally { + await service.close(); + } + }, + ); }); diff --git a/tests/boundary/filesystem/machoCommandAdmission.test.ts b/tests/boundary/filesystem/machoCommandAdmission.test.ts index 6e73b193bf..be98845ccb 100644 --- a/tests/boundary/filesystem/machoCommandAdmission.test.ts +++ b/tests/boundary/filesystem/machoCommandAdmission.test.ts @@ -117,7 +117,9 @@ it.each([ ); await writeFile(path, bytes); expect(parseExecutableHeader(bytes, "arm64").ok).toBe(false); - expect(() => readBinaryContainer(bytes)).toThrow(/load command/iu); + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( + /load command/iu, + ); const resolved = await parseBinaryTarget(path); expect(resolved.ok).toBe(false); if (resolved.ok || !(resolved.error instanceof BinaryTargetError)) @@ -144,7 +146,7 @@ it.each([ "image", ); await writeFile(path, bytes); - expect(readBinaryContainer(bytes)).toMatchObject({ + expect(readBinaryContainer(bytes, "elf-directory")).toMatchObject({ format: "macho", bits: wide ? 64 : 32, byte_order: littleEndian ? "little" : "big", @@ -219,8 +221,10 @@ const OVERSIZED_COMMAND_BYTES = 16 * 1024 * 1024 + 8; it("reports truncation when an oversized command table cannot fit the file", () => { const bytes = commandImage(true, true, [16]); bytes.writeUInt32LE(OVERSIZED_COMMAND_BYTES, 20); - expect(() => readBinaryContainer(bytes)).toThrow(BinaryFormatFailure); - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( + BinaryFormatFailure, + ); + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( expect.objectContaining({ kind: "truncated", failure: expect.objectContaining({ field: "Mach-O load commands" }), @@ -238,5 +242,7 @@ it("retains the native capacity result for a backed oversized command table", () ok: true, value: { commandBytes: OVERSIZED_COMMAND_BYTES }, }); - expect(() => readBinaryContainer(bytes)).toThrow(BinaryResourceFailure); + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( + BinaryResourceFailure, + ); }); diff --git a/tests/boundary/filesystem/nativeExports.test.ts b/tests/boundary/filesystem/nativeExports.test.ts index 5fe0d18101..0b44414143 100644 --- a/tests/boundary/filesystem/nativeExports.test.ts +++ b/tests/boundary/filesystem/nativeExports.test.ts @@ -1,7 +1,8 @@ import { createHash } from "node:crypto"; import { symlink, truncate, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; import { createNativeExportsService } from "../../../src/composition/binaryDiagnostics.js"; import { analysisErrorRemediationAction } from "../../../src/domain/analysisErrorPresentation.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; @@ -149,6 +150,54 @@ describe("native export stable filesystem boundary", () => { }); }); +it("observes cancellation during a bounded ELF export inventory", async () => { + const root = await createTestTempDirectory("rea-native-export-cancel-"); + const path = join(root, "libfixture.so"); + const bytes = elfExportImage({ + symbols: Array.from({ length: 1025 }, () => ({ name: "exported" })), + }); + await writeFile(path, bytes); + const service = createNativeExportsService(); + const controller = new AbortController(); + const reason = new Error("stop during export rows"); + const yieldToEventLoop = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await yieldToEventLoop(); + controller.abort(reason); + }); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation during export rows"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_native_exports", + cause: reason, + partialObservation: { + kind: "file-snapshot-hash", + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + }); + const retry = await service.inspect( + { path }, + { signal: new AbortController().signal }, + ); + if (!retry.ok) throw retry.error; + expect(retry.value.normalized_result).toMatchObject({ + coverage: { status: "complete", examined_entries: 1025, exports: 1025 }, + }); + } finally { + pause.mockRestore(); + await service.close(); + } +}); + describe("native export text capacity", () => { it("retains valid exports and partial Evidence when a symbol name exceeds the read allowance", async () => { const root = await createTestTempDirectory("rea-native-export-name-limit-"); diff --git a/tests/boundary/filesystem/nativeInitializers.test.ts b/tests/boundary/filesystem/nativeInitializers.test.ts index b3bbfaf36c..0dd3ae881b 100644 --- a/tests/boundary/filesystem/nativeInitializers.test.ts +++ b/tests/boundary/filesystem/nativeInitializers.test.ts @@ -1,11 +1,14 @@ -import { symlink, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { readFile, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; import { createNativeInitializersService } from "../../../src/composition/binaryDiagnostics.js"; import { elf64PieInitializers, machoInitializerSymbolKinds, } from "../../../src/native/initializers/NativeInitializers.fixture.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("native initializer stable filesystem boundary", () => { @@ -90,3 +93,47 @@ describe("native initializer stable filesystem boundary", () => { expect(cancelled.error._tag).toBe("AnalysisCancelledError"); }); }); + +it("cancels ELF section-name decoding while retaining the initializer snapshot", async () => { + const root = await createTestTempDirectory("rea-native-init-zstd-cancel-"); + const path = join(root, "program"); + const bytes = elfFrameAllowanceFixture(); + await writeFile(path, bytes); + const service = createNativeInitializersService(); + const controller = new AbortController(); + const reason = new Error("section-name reading cancelled"); + const nativeYield = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await nativeYield(); + controller.abort(reason); + }); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation during ELF names"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + partialObservation: { + kind: "file-snapshot-hash", + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + }); + const retry = await service.inspect({ path }); + if (!retry.ok) throw retry.error; + expect(retry.value.normalized_result).toMatchObject({ + format: "elf", + status: "parsed", + }); + expect((await readFile(path)).equals(bytes)).toBe(true); + } finally { + pause.mockRestore(); + await service.close(); + } +}); diff --git a/tests/boundary/go/goBinaryImage.test.ts b/tests/boundary/go/goBinaryImage.test.ts index cf258de989..b645c82c22 100644 --- a/tests/boundary/go/goBinaryImage.test.ts +++ b/tests/boundary/go/goBinaryImage.test.ts @@ -50,7 +50,7 @@ it("preserves exact Mach-O protection masks for Go fallback selection", () => { bytes.writeUInt32LE(3, secondSegment + 56); bytes.writeUInt32LE(3, secondSegment + 60); - const container = readBinaryContainer(bytes); + const container = readBinaryContainer(bytes, "elf-directory"); expect( container.load_regions?.map((region) => region.protection_masks), ).toEqual([ diff --git a/tests/boundary/go/goBinaryProvider.test.ts b/tests/boundary/go/goBinaryProvider.test.ts index fbd1c0500b..48c0637635 100644 --- a/tests/boundary/go/goBinaryProvider.test.ts +++ b/tests/boundary/go/goBinaryProvider.test.ts @@ -1,6 +1,7 @@ import { truncate, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; import { GoBinaryService } from "../../../src/application/go/GoBinaryService.js"; import { GoBinaryProvider } from "../../../src/go/GoBinaryProvider.js"; import { @@ -16,6 +17,7 @@ import { } from "../../fixtures/go/image.js"; import { createGoRodataStringFixture } from "../../fixtures/go/rodataStringImage.js"; import { CPU_ARM64, universalMacho } from "../../fixtures/universalMacho.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; it("rejects advertised PE32 data directories outside the optional header", async () => { @@ -356,3 +358,33 @@ it.each([ } }, ); + +it("cancels a many-frame compressed ELF section-name table", async () => { + const root = await createTestTempDirectory("rea-go-zstd-cancel-"); + const path = join(root, "app"); + await writeFile(path, elfFrameAllowanceFixture()); + const reason = new Error("stop go frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + try { + const result = await new GoBinaryProvider().inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_go_binary", + cause: reason, + }); + } finally { + spy.mockRestore(); + } +}); diff --git a/tests/boundary/gpu/codeObjects.test.ts b/tests/boundary/gpu/codeObjects.test.ts index bee7fb48bc..59bfa02907 100644 --- a/tests/boundary/gpu/codeObjects.test.ts +++ b/tests/boundary/gpu/codeObjects.test.ts @@ -1,12 +1,18 @@ import { expect, it, onTestFinished } from "vitest"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { deflateSync } from "node:zlib"; +import { deflateSync, zstdCompressSync } from "node:zlib"; import { createHash } from "node:crypto"; +import { runWithAnalysisOperationSnapshot } from "../../../src/domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../../../src/domain/analysisResourceLimits.js"; import { pe } from "../../../src/domain/binaryTarget.fixture.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { gpuCodeObjectsPartialSchema } from "../../../src/domain/gpu/gpuCodeObjects.js"; import { GpuCodeObjectsProvider } from "../../../src/gpu/GpuCodeObjectsProvider.js"; +import { GpuInventory } from "../../../src/gpu/GpuInventory.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cudaFixture, @@ -36,6 +42,45 @@ const inspect = async ( onTestFinished(() => provider.close()); return provider.inspect({ path, analysis: "kernels", ...options }); }; +it("reports a Zstandard frame-count refusal as an adjustable GPU limit", async () => { + const frame = (byte: number) => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + const compressed = Buffer.concat([frame(0x61), frame(0x62)]); + const entry = Buffer.alloc(64 + Math.ceil(compressed.length / 8) * 8); + entry.writeUInt16LE(1, 0); + entry.writeUInt16LE(0x101, 2); + entry.writeUInt32LE(64, 4); + entry.writeBigUInt64LE(BigInt(entry.length - 64), 8); + entry.writeUInt32LE(compressed.length, 16); + entry.writeUInt32LE(80, 28); + entry.writeBigUInt64LE(0x11n | 0x8000n, 40); + entry.writeBigUInt64LE(2n, 56); + compressed.copy(entry, 64); + const header = Buffer.alloc(16); + header.writeUInt32LE(0xba55ed50); + header.writeUInt16LE(1, 4); + header.writeUInt16LE(16, 6); + header.writeBigUInt64LE(BigInt(entry.length), 8); + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => inspect(Buffer.concat([header, entry])), + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); +}); + it.each(["none", "zstd", "lz4"] as const)( "reads selected CUDA metadata and PTX with %s compression", async (compression) => { @@ -70,6 +115,52 @@ it.each(["none", "zstd", "lz4"] as const)( ]); }, ); + +it("observes cancellation during a cubin kernel directory read", async () => { + const cubin = cubinFixture(); + const section = Number(cubin.readBigUInt64LE(40)) + 64; + const offset = Number(cubin.readBigUInt64LE(section + 24)); + const size = Number(cubin.readBigUInt64LE(section + 32)); + const names = cubin.subarray(offset, offset + size); + const compressionHeader = Buffer.alloc(24); + compressionHeader.writeUInt32LE(2); + compressionHeader.writeBigUInt64LE(BigInt(names.length), 8); + compressionHeader.writeBigUInt64LE(1n, 16); + const encodedNames = Buffer.concat([ + compressionHeader, + zstdCompressSync(names), + ]); + const bytes = Buffer.concat([cubin, encodedNames]); + bytes.writeBigUInt64LE(0x800n, section + 8); + bytes.writeBigUInt64LE(BigInt(cubin.length), section + 24); + bytes.writeBigUInt64LE(BigInt(encodedNames.length), section + 32); + + const controller = new AbortController(); + const inventory = new GpuInventory( + { path: "/artifacts/cubin", analysis: "kernels" }, + controller.signal, + ); + const cancelled = new Error("kernel directory read cancelled"); + let payloadReads = 0; + let pendingAbort: NodeJS.Immediate | undefined; + try { + await expect( + inventory.inspect({ + size: bytes.length, + containerId: null, + read: async (at, count) => { + // The selected payload is re-read into the kernel parser's buffer. + if (at === 0 && count === bytes.length && ++payloadReads === 1) + pendingAbort = setImmediate(() => controller.abort(cancelled)); + return bytes.subarray(at, at + count); + }, + }), + ).rejects.toMatchObject({ reason: "cancelled", cause: cancelled }); + expect(payloadReads).toBe(1); + } finally { + if (pendingAbort !== undefined) clearImmediate(pendingAbort); + } +}); it("decodes a CUDA 0x2000 LZ4 entry to the exact declared payload", async () => { const ptx = Buffer.from(ptxFixture), bytes = cudaFixture("lz4", [{ kind: 1, target: 80, bytes: ptx }]), diff --git a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts index 1916c635a4..fae8059031 100644 --- a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts +++ b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts @@ -1112,7 +1112,7 @@ it.each(["outside", "misaligned", "duplicate"] as const)( bytes.copy(bytes, 176, 120, 176); bytes.writeUInt16LE(3, 56); } - expect(() => readBinaryContainer(bytes)).not.toThrow(); + expect(() => readBinaryContainer(bytes, "elf-directory")).not.toThrow(); expect(() => link(bytes)).toThrow( shape === "duplicate" ? "multiple PT_DYNAMIC segments" diff --git a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts index 1a36c77c53..e1a0b82750 100644 --- a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts +++ b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts @@ -2,6 +2,7 @@ import { setImmediate } from "node:timers/promises"; import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { zstdCompressSync } from "node:zlib"; import { afterEach, expect, it, vi } from "vitest"; import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import { @@ -17,10 +18,12 @@ import { inspectIl2cppMetadataV108 } from "../../../src/il2cpp/Il2cppMetadataV10 import { Il2cppGhidraSeedWriter } from "../../../src/il2cpp/Il2cppGhidraSeedWriter.js"; import { Il2cppMetadataService } from "../../../src/application/il2cpp/Il2cppMetadataService.js"; import { il2cppMetadataSchema } from "../../../src/domain/il2cpp/il2cppMetadata.js"; +import * as zstandard from "../../../src/domain/compression/zstandardExactFrame.js"; import { createElfFixture, createIl2cppBinaryFixture, FIXTURE_GENERIC_INSTANCES, + FIXTURE_METHOD_ADDRESSES, } from "../../fixtures/il2cpp/binary.js"; import { createIl2cppMetadataFixture, @@ -755,6 +758,137 @@ it("accepts cancellation between binary linking batches and keeps the metadata s } }); +/** Add a compressed ELF64 .shstrtab outside the fixture's PT_LOAD. */ +const withCompressedElfNames = (image: Buffer): Buffer => { + const names = Buffer.from("\0.shstrtab\0"); + const compressionHeader = Buffer.alloc(24); + compressionHeader.writeUInt32LE(2, 0); // ELFCOMPRESS_ZSTD + compressionHeader.writeBigUInt64LE(BigInt(names.length), 8); + compressionHeader.writeBigUInt64LE(1n, 16); + const payload = Buffer.concat([compressionHeader, zstdCompressSync(names)]); + const padding = Buffer.alloc((8 - (image.length % 8)) % 8); + const tableOffset = image.length + padding.length; + const table = Buffer.alloc(128); + const namesHeader = table.subarray(64); + namesHeader.writeUInt32LE(1, 0); // ".shstrtab" offset in names + namesHeader.writeUInt32LE(3, 4); // SHT_STRTAB + namesHeader.writeBigUInt64LE(0x800n, 8); // SHF_COMPRESSED + namesHeader.writeBigUInt64LE(BigInt(tableOffset + table.length), 24); + namesHeader.writeBigUInt64LE(BigInt(payload.length), 32); + namesHeader.writeBigUInt64LE(1n, 48); + const bytes = Buffer.concat([image, padding, table, payload]); + bytes.writeBigUInt64LE(BigInt(tableOffset), 40); + bytes.writeUInt16LE(64, 58); + bytes.writeUInt16LE(2, 60); + bytes.writeUInt16LE(1, 62); + return bytes; +}; + +it("retains selected source evidence when cancelled decoding compressed ELF names", async () => { + const root = await createTestTempDirectory("rea-il2cpp-elf-names-cancel-"); + const path = join(root, "global-metadata.dat"); + const binary_path = join(root, "libil2cpp.so"); + const metadata = createLinkedIl2cppMetadataFixture().bytes; + const binary = withCompressedElfNames( + createElfFixture(createIl2cppBinaryFixture()), + ); + await Promise.all([ + writeFile(path, metadata), + writeFile(binary_path, binary), + ]); + + const controller = new AbortController(); + const reason = new Error("cancel while decoding ELF names"); + const decode = zstandard.decodeZstandardFramesCooperative; + const checkpoint = vi + .spyOn(zstandard, "decodeZstandardFramesCooperative") + .mockImplementation((bytes, options, pause) => + decode(bytes, options, async () => { + controller.abort(reason); + await pause(); + }), + ); + const provider = new Il2cppMetadataProvider(); + try { + const result = await provider.inspect( + { path, binary_path }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation during ELF decoding"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(result.error.partialObservation).toMatchObject({ + operation: "observe_il2cpp_metadata_source", + normalized_result: { + artifact: { path }, + selected_binary: { + path: binary_path, + identity_status: "observed", + bytes: binary.length, + sha256: createHash("sha256").update(binary).digest("hex"), + }, + inventory_status: "not-returned-as-complete", + }, + }); + checkpoint.mockRestore(); + const retry = await provider.inspect({ path, binary_path }); + if (!retry.ok) throw retry.error; + expect(retry.value.binary?.format).toBe("elf"); + expect(retry.value.method_addresses[0]?.address).toBe( + `0x${FIXTURE_METHOD_ADDRESSES[0].toString(16)}`, + ); + } finally { + await provider.close(); + } +}); + +it.each([false, true])( + "refuses unsupported generic metadata before reading an invalid ELF directory (signal=%s)", + async (withSignal) => { + const root = await createTestTempDirectory("rea-il2cpp-generic-layout-"); + const path = join(root, "global-metadata.dat"); + const binary_path = join(root, "libil2cpp.so"); + const metadata = createLinkedIl2cppMetadataFixture({ version: 39 }).bytes; + const binary = withCompressedElfNames( + createElfFixture(createIl2cppBinaryFixture()), + ); + binary.writeBigUInt64LE(BigInt(binary.length + 8), 40); + await Promise.all([ + writeFile(path, metadata), + writeFile(binary_path, binary), + ]); + const options = withSignal + ? { signal: new AbortController().signal } + : undefined; + const provider = new Il2cppMetadataProvider(); + try { + const result = await provider.inspect( + { path, binary_path, generic_instances: true }, + options, + ); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisUnsupportedTargetError", + reason: expect.stringContaining( + "generic method instances are not decoded for metadata version 39", + ), + }, + }); + const ordinary = await provider.inspect({ path, binary_path }, options); + expect(ordinary).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisInputError", + issues: [{ path: ["binary_path"], reason: "invalid_format" }], + }, + }); + } finally { + await provider.close(); + } + }, +); + it("refuses duplicate MethodDef tokens before publishing linked Evidence or Ghidra seeds", async () => { const root = await createTestTempDirectory( "rea-il2cpp-duplicate-method-token-", diff --git a/tests/boundary/rust/rustBinaryBudget.test.ts b/tests/boundary/rust/rustBinaryBudget.test.ts index 4e459d4b05..c6ed61bc3b 100644 --- a/tests/boundary/rust/rustBinaryBudget.test.ts +++ b/tests/boundary/rust/rustBinaryBudget.test.ts @@ -1,12 +1,14 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { RustBinaryProvider } from "../../../src/rust/RustBinaryProvider.js"; import { createRustImage, createRustHashOverflowImage, } from "../../fixtures/rust/image.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; it("reports a recoverable compiler prefix without refusing the dependency report", async () => { @@ -204,3 +206,33 @@ it("reports byte trimming only when it removes a compiler path", async () => { await provider.close(); } }); + +it("cancels a many-frame compressed ELF section-name table", async () => { + const root = await createTestTempDirectory("rea-rust-zstd-cancel-"); + const path = join(root, "app"); + await writeFile(path, elfFrameAllowanceFixture()); + const reason = new Error("stop rust frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + try { + const result = await new RustBinaryProvider().inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_rust_binary", + cause: reason, + }); + } finally { + spy.mockRestore(); + } +}); diff --git a/tests/boundary/rust/rustBinaryImage.test.ts b/tests/boundary/rust/rustBinaryImage.test.ts index a5cc1ec126..209d1e8a49 100644 --- a/tests/boundary/rust/rustBinaryImage.test.ts +++ b/tests/boundary/rust/rustBinaryImage.test.ts @@ -48,7 +48,7 @@ it.each([ "retains read-only ELF section bytes with a %s name", (name, expectedName) => { const bytes = rustElfSectionNameFixture(name); - const container = readBinaryContainer(bytes); + const container = readBinaryContainer(bytes, "elf-directory"); expect(container.load_regions?.every(({ write }) => write)).toBe(true); expect(container.sections).toContainEqual( expect.objectContaining({ @@ -77,7 +77,9 @@ it.each([ const bytes = rustElfSectionNameFixture("empty"); bytes.writeUInt32LE(type, 384 + 4); bytes.writeBigUInt64LE(flags, 384 + 8); - expect(readBinaryContainer(bytes).sections).not.toContainEqual( + expect( + readBinaryContainer(bytes, "elf-directory").sections, + ).not.toContainEqual( expect.objectContaining({ offset: 2048, read_only: true }), ); expect(readRustBinaryImage(bytes).rustc_commit).toBeNull(); @@ -88,9 +90,9 @@ it("does not scan an unnamed ELF section whose declared range leaves the file", const bytes = rustElfSectionNameFixture("empty"); // The compiler path still exists at 2048, but its declared section no longer fits. bytes.writeBigUInt64LE(BigInt(bytes.length), 384 + 32); - expect(readBinaryContainer(bytes).sections).not.toContainEqual( - expect.objectContaining({ read_only: true }), - ); + expect( + readBinaryContainer(bytes, "elf-directory").sections, + ).not.toContainEqual(expect.objectContaining({ read_only: true })); expect(readRustBinaryImage(bytes).rustc_commit).toBeNull(); }); @@ -520,7 +522,7 @@ it("keeps the container's truncated failure and its facts unchanged", () => { } throw new Error("expected a container failure"); }; - const container = thrown(() => readBinaryContainer(bytes)); + const container = thrown(() => readBinaryContainer(bytes, "elf-directory")); const rust = thrown(() => readRustBinaryImage(bytes)); expect(rust).toBeInstanceOf(ParseFailureError); expect((rust as ParseFailureError).failure).toEqual( @@ -623,8 +625,8 @@ it.each([ } else { sectionName.copy(control, sectionOffset); } - expect(readBinaryContainer(bytes).load_regions).toEqual( - readBinaryContainer(control).load_regions, + expect(readBinaryContainer(bytes, "elf-directory").load_regions).toEqual( + readBinaryContainer(control, "elf-directory").load_regions, ); expect(() => readGoBinaryImage(bytes)).toThrow(/build-info|mapped/i); expect(readRustBinaryImage(bytes)).toMatchObject({ diff --git a/tests/fixtures/binaryDiagnostics/elfFrameAllowance.ts b/tests/fixtures/binaryDiagnostics/elfFrameAllowance.ts new file mode 100644 index 0000000000..7cbdfb6228 --- /dev/null +++ b/tests/fixtures/binaryDiagnostics/elfFrameAllowance.ts @@ -0,0 +1,17 @@ +import { zstdCompressSync } from "node:zlib"; +import { compressedElfNoteFixture } from "../../../src/native/elf/ElfNotes.fixture.js"; + +/** Two ordinary frames encode the original ELF note fixture's names. */ +export const elfFrameAllowanceFixture = () => { + const section = ".note.gnu.build-id"; + return compressedElfNoteFixture( + 64, + true, + 2, + section, + Buffer.concat([ + zstdCompressSync(Buffer.from(`\0${section}\0`)), + zstdCompressSync(Buffer.from(".shstrtab\0")), + ]), + ); +};