From 7fd50eb3614aec28849d3f68ac65be9c3c5f8ca2 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:45:26 -0500 Subject: [PATCH] fix(zstd): share exact-frame Zstandard decoding across ELF and OTA (#12416) * fix(zstd): share exact-frame Zstandard decoding across ELF and OTA Walk RFC 8878 frames once so ELF sections, Android OTA REPLACE_ZSTD, and GPU payloads reject truncated headers, blocks, checksums, and trailing bytes on every supported Node runtime. Bound that walk with the runtime zstd.max_frames setting. * fix(zstd): keep frame output tight and report GPU frame-cap refusals Copy each Node 22 frame off its decoder chunk before the next frame is decoded. Map a GPU zstd.max_frames refusal through the resource-limit contract instead of a provider failure. Co-authored-by: morluto * fix(zstd): keep frame-cap refusals and bad seeds out of malformed results Project zstd.max_frames through one resource-limit helper for ELF notes, DWARF, GPU, and artifact reads. An invalid REA_ZSTD_MAX_FRAMES seed stays a configuration failure, and frame-header tests cover content size, window, dictionary id, and RLE blocks. Co-authored-by: morluto * fix(zstd): yield GPU frame decoding so cancellation can land The frame-by-frame fallback awaits a caller pause before each data frame. GPU yields and checks the read signal there, so a many-frame payload does not finish before a disconnect is observed. Co-authored-by: morluto * fix(zstd): advertise the frame cap and keep strict GPU decode cancellable List every operation that enforces zstd.max_frames, including GPU, DWARF, and ELF notes. The strict sequence uses the async decoder and checks the caller signal again after it returns, and a single output buffer is not copied a second time. Co-authored-by: morluto * fix(zstd): refresh analysis-setting schemas and GPU frame-cap projection Regenerate the advertised input summaries for inspect_analysis_settings and update_analysis_settings. GPU inventory now throws the projected zstd.max_frames resource limit from payload decode and #retainFailure, so the adjustable setting survives the public failure. Co-authored-by: morluto * style(gpu): format the frame-cap projection throw oxfmt keeps the zstd.max_frames throw in #retainFailure on one line so CI format:check matches the local formatter. Co-authored-by: morluto * fix(zstd): yield after the last frame-by-frame GPU decode The fallback returned success in the same turn as the final synchronous frame, so a queued abort was still invisible to the caller's signal check. Pause once more before that success, matching the strict sequence. Co-authored-by: morluto * fix(zstd): yield ELF, OTA, and DWARF frame decoding on the caller signal The synchronous frame loop ignored abort signals owned by inspect_elf_notes, inspect_dwarf_units, and Android OTA partition reads. Those callers now use the cooperative decoder and yield before Zstandard work, and 32-frame cancellations cover each path. Co-authored-by: morluto * fix(zstd): observe cancellation during structural parsing * fix(zstd): project section-name frame caps at the ELF directory boundary readElfDirectoryFromBuffer now requires the caller's operation and throws the adjustable zstd.max_frames resource limit. Hardening, unwind, and the other directory consumers keep that typed refusal instead of an adapter or I/O error. Co-authored-by: morluto * fix(zstd): preserve optional ELF evidence when frames are capped * fix(zip): validate Zstandard members through frame completion (#12461) fix(zip): require complete Zstandard member frames * fix(elf): yield hardening section name decoding * fix(elf): yield section names for unwind and export reads * fix(zstd): yield ELF directory reads that already own a cancellation signal Hardening, unwind, initializers, exports, kernel modules, debug link, and Android JNI now read section names through the cooperative directory when their caller has a signal. Readers with no signal stay synchronous. Co-authored-by: morluto * fix(elf): share one initializer directory and retain its snapshot * fix(android): reuse the admitted JNI ELF directory * fix(elf): retain optional debug candidate evidence through cooperative decoding * fix(zstd): yield ELF section names for Go, Rust, and IL2CPP These inspections already own an abort signal. Their ELF directory read now uses the cooperative container decoder so a queued cancellation can land during a many-frame section-name table. Co-authored-by: morluto * fix(elf): recognize compressed kernel modules before ELF decoding The cooperative module reader decoded the bytes as ELF before the container check, so gzip, xz, and zstd modules became malformed images. The compression check now returns the unsupported-container report first. Co-authored-by: morluto * refactor(elf): classify each container once before reading it * fix(il2cpp): preserve admission order during cooperative linking * fix(elf): reuse section-name decoding in debug-link comparison (#12472) * style(elf): format updated section-reader test calls --------- Co-authored-by: Cursor Agent Co-authored-by: morluto --- docs/design/limits-and-recovery.md | 31 +- docs/design/rea-limit-inventory.md | 37 +- .../tools/inspect_analysis_settings.json | 2 +- .../tools/update_analysis_settings.json | 2 +- src/android/AndroidJniLibraries.ts | 5 +- src/application/AnalysisSettings.test.ts | 47 ++ ...taPayloadArtifactReader.boundaries.test.ts | 53 ++ .../AndroidOtaPayloadArtifactReader.ts | 14 +- src/artifacts/ZipArtifactReader.test.ts | 74 ++- src/config/analysisSettings.ts | 2 + src/domain/analysisResourceLimits.ts | 11 + src/domain/analysisSettingDefinitions.ts | 45 ++ src/domain/analysisSettings.ts | 1 + src/domain/android/androidJniLibrary.ts | 59 +- .../compression/zstandardExactFrame.test.ts | 314 +++++++++++ src/domain/compression/zstandardExactFrame.ts | 522 ++++++++++++++++++ .../compression/zstandardFrameAllowance.ts | 176 ++++++ .../compression/zstandardFrameChunks.ts | 89 +++ src/domain/compression/zstandardFrameWire.ts | 117 ++++ src/domain/firmware/androidOtaPayload.test.ts | 141 +++++ src/domain/firmware/androidOtaPayload.ts | 154 +++++- .../native/elfCompressedSection.test.ts | 94 ++++ src/domain/native/elfCompressedSection.ts | 219 +++++++- src/domain/native/elfImage.ts | 51 +- src/domain/native/elfSectionNameTable.ts | 97 ++++ src/domain/native/nativeExportParser.ts | 48 +- src/domain/native/nativeSectionDirectory.ts | 169 +++++- src/domain/native/readBinaryContainer.ts | 29 +- src/go/GoBinaryImage.ts | 30 +- src/go/GoBinaryProvider.ts | 19 +- src/gpu/GpuCodeObjectsProvider.ts | 17 +- src/gpu/GpuElfKernels.ts | 3 + src/gpu/GpuInventory.ts | 27 +- src/gpu/GpuPayloadCodec.test.ts | 48 ++ src/gpu/GpuPayloadCodec.ts | 60 +- src/gpu/GpuSource.ts | 17 +- src/il2cpp/Il2cppBinaryFile.ts | 57 +- src/il2cpp/Il2cppBinaryImage.ts | 66 ++- src/il2cpp/Il2cppMetadataProvider.ts | 16 +- src/native/dwarf/DwarfSections.ts | 59 +- src/native/dwarf/DwarfUnitsProvider.test.ts | 63 ++- src/native/dwarf/DwarfUnitsProvider.ts | 15 +- src/native/elf/ElfDebugLink.test.ts | 4 +- src/native/elf/ElfDebugLink.ts | 264 +++++++-- src/native/elf/ElfDebugLinkProvider.test.ts | 48 ++ src/native/elf/ElfDebugLinkProvider.ts | 105 +++- src/native/elf/ElfNoteParser.test.ts | 14 +- src/native/elf/ElfNoteParser.ts | 80 ++- src/native/elf/ElfNotes.fixture.ts | 4 +- src/native/elf/ElfNotesProvider.test.ts | 105 +++- src/native/elf/ElfNotesProvider.ts | 17 +- src/native/elf/ElfUnwindRangesProvider.ts | 29 +- src/native/elf/ElfUnwindTables.ts | 54 +- src/native/elf/KernelModuleParser.ts | 70 ++- src/native/elf/KernelModuleProvider.ts | 8 +- src/native/exports/NativeExportsProvider.ts | 21 +- .../hardening/BinaryHardeningParsers.test.ts | 4 +- .../hardening/BinaryHardeningProvider.ts | 4 +- src/native/hardening/ElfHardening.ts | 32 +- src/native/initializers/ElfInitializers.ts | 28 +- .../NativeInitializersParsers.test.ts | 14 +- .../NativeInitializersProvider.ts | 8 +- src/native/metadata/BinaryContainer.test.ts | 16 +- src/native/metadata/Il2cppContainer.test.ts | 4 +- .../metadata/NativeSectionDirectory.test.ts | 14 +- src/native/metadata/PeImage.test.ts | 2 +- src/resources/ArtifactFileOperation.ts | 12 + .../ArtifactOperationFailure.test.ts | 25 + src/resources/ArtifactOperationFailure.ts | 6 + src/resources/ZipArtifactReader.ts | 26 +- src/resources/index.ts | 1 + src/rust/RustBinaryImage.ts | 51 +- src/rust/RustBinaryProvider.ts | 19 +- .../nativeElfFrameAllowanceMcp.test.ts | 103 ++++ .../android/androidJniLibraries.test.ts | 238 +++++--- .../filesystem/artifactZipIntegrity.test.ts | 31 ++ .../filesystem/binaryHardening.test.ts | 74 ++- .../boundary/filesystem/elfDebugLink.test.ts | 100 +++- .../elfSectionNameCancellation.test.ts | 73 +++ .../filesystem/elfUnwindRanges.test.ts | 92 ++- .../boundary/filesystem/kernelModule.test.ts | 41 +- .../filesystem/machoCommandAdmission.test.ts | 16 +- .../boundary/filesystem/nativeExports.test.ts | 51 +- .../filesystem/nativeInitializers.test.ts | 51 +- tests/boundary/go/goBinaryImage.test.ts | 2 +- tests/boundary/go/goBinaryProvider.test.ts | 34 +- tests/boundary/gpu/codeObjects.test.ts | 93 +++- .../boundary/il2cpp/il2cppBinaryFile.test.ts | 2 +- .../il2cpp/il2cppMetadataProvider.test.ts | 134 +++++ tests/boundary/rust/rustBinaryBudget.test.ts | 34 +- tests/boundary/rust/rustBinaryImage.test.ts | 18 +- .../binaryDiagnostics/elfFrameAllowance.ts | 17 + 92 files changed, 4858 insertions(+), 535 deletions(-) create mode 100644 src/domain/compression/zstandardExactFrame.test.ts create mode 100644 src/domain/compression/zstandardExactFrame.ts create mode 100644 src/domain/compression/zstandardFrameAllowance.ts create mode 100644 src/domain/compression/zstandardFrameChunks.ts create mode 100644 src/domain/compression/zstandardFrameWire.ts create mode 100644 tests/acceptance/nativeElfFrameAllowanceMcp.test.ts create mode 100644 tests/boundary/filesystem/elfSectionNameCancellation.test.ts create mode 100644 tests/fixtures/binaryDiagnostics/elfFrameAllowance.ts diff --git a/docs/design/limits-and-recovery.md b/docs/design/limits-and-recovery.md index 653a4c2571..ee318b1e71 100644 --- a/docs/design/limits-and-recovery.md +++ b/docs/design/limits-and-recovery.md @@ -176,21 +176,22 @@ checks all happen before publishing the replacement snapshot; a rejected batch cannot partially apply. Clients coordinating writes can supply `expected_revision`; without it, valid updates apply to the current state. -| Runtime setting | Application point | -| --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | -| `asar.max_header_bytes` | Next archive operation, including full JavaScript application analysis | -| `asar.max_listing_bytes` | Next expanded ASAR listing, independently of packed header bytes | -| `ghidra.startup_timeout_ms`, `hopper.startup_timeout_ms` | Next provider startup attempt | -| `ghidra.decompile_timeout_seconds` | Next operation on the existing Ghidra Program; `null` means unbounded | -| `ghidra.decompile_max_payload_mib` | Next native decompile; `null` preserves the selected provider's payload default | -| `ghidra.value_flow_max_operations`, `ghidra.value_flow_max_inputs_per_operation`, `ghidra.value_flow_max_edges` | Next function value-flow projection on the existing Program | -| `jadx.operation_timeout_ms` | Next JADX operation | -| `regex.matching_deadline_ms`, `regex.worker_heap_request_mib` | Next search / new regex worker | -| `capture.decoder_old_generation_mib` | Next owned capture decoder | -| `javascript.analysis_timeout_ms`, `javascript.max_heap_mb` | Next source analysis / new JavaScript worker | -| `javascript.member_mutation_alias_states` | Next JavaScript application, web bundle or reference-source analysis | -| `wasm.command_timeout_ms` | Next WABT operation | -| `native.matching_representation_bytes` | Next `match_native_functions` request | +| Runtime setting | Application point | +| --------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | +| `asar.max_header_bytes` | Next archive operation, including full JavaScript application analysis | +| `asar.max_listing_bytes` | Next expanded ASAR listing, independently of packed header bytes | +| `ghidra.startup_timeout_ms`, `hopper.startup_timeout_ms` | Next provider startup attempt | +| `ghidra.decompile_timeout_seconds` | Next operation on the existing Ghidra Program; `null` means unbounded | +| `ghidra.decompile_max_payload_mib` | Next native decompile; `null` preserves the selected provider's payload default | +| `ghidra.value_flow_max_operations`, `ghidra.value_flow_max_inputs_per_operation`, `ghidra.value_flow_max_edges` | Next function value-flow projection on the existing Program | +| `jadx.operation_timeout_ms` | Next JADX operation | +| `regex.matching_deadline_ms`, `regex.worker_heap_request_mib` | Next search / new regex worker | +| `capture.decoder_old_generation_mib` | Next owned capture decoder | +| `javascript.analysis_timeout_ms`, `javascript.max_heap_mb` | Next source analysis / new JavaScript worker | +| `javascript.member_mutation_alias_states` | Next JavaScript application, web bundle or reference-source analysis | +| `wasm.command_timeout_ms` | Next WABT operation | +| `native.matching_representation_bytes` | Next `match_native_functions` request | +| `zstd.max_frames` | Next exact Zstandard decode for an ELF section, OTA `REPLACE_ZSTD` operation, or GPU payload | ### Precedence and lifetime diff --git a/docs/design/rea-limit-inventory.md b/docs/design/rea-limit-inventory.md index 34366447dc..abdd29b080 100644 --- a/docs/design/rea-limit-inventory.md +++ b/docs/design/rea-limit-inventory.md @@ -48,10 +48,10 @@ settings. They are not on the restart-only list. | Category | Count | | --------------------------------------------------------------------- | ---------------------------------------------------------------------------: | -| Runtime analysis settings | 28 (25 have an `REA_` seed; 3 are per-call parameters only) | +| Runtime analysis settings | 29 (26 have an `REA_` seed; 3 are per-call parameters only) | | Per-call numeric `REA_` limits still read from the launch environment | 0 | | Restart-only numeric `REA_` limits | 15 | -| Numeric `REA_` candidates recognized in TypeScript source | 38 | +| Numeric `REA_` candidates recognized in TypeScript source | 39 | | Numeric `REA_` controls with external or generated-reader provenance | 2 | | Non-numeric `REA_` settings kept restart-only | 52 caller settings, plus process-protocol names that are not caller settings | | Hardcoded ceilings left for follow-up | grouped below; none converted in the inventory change | @@ -96,22 +96,23 @@ floor, and maximum. The accepted operation snapshot supplies the value. force. An update outside the range is rejected. A hit names the setting and reports the current value, source, and range. A rejected Windows ChangeTime launch value omits `pending_restart`. -| Environment | Owner | Default | Floor | Maximum | Applies | Fields | -| --------------------------------------------- | ------------------------------------------ | ----------------------: | ------: | -----------: | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | -| `REA_BRIDGE_MAX_REQUEST_BYTES` | `src/config/bridgeMaxRequestBytes.ts` | 16777216 | 7464 | 536870887 | each bridge request, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_BRIDGE_MAX_REPLY_BYTES` | `src/config/bridgeMaxReplyBytes.ts` | 268435456 | 1048576 | 536870888 | each Hopper or Ghidra request, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_CAPTURE_PACKET_JSON_EXPANSION_BYTES` | `src/config/capturePacketJsonExpansion.ts` | 137438953472 | 1048576 | 137438953472 | each packet-capture reply before parsing, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_DEBUGGER_COMMAND_DEADLINE_MS` | `src/domain/analysisResourceLimits.ts` | 120000 | 1 | 2147483647 | each command, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES` | `src/domain/analysisResourceLimits.ts` | 16777216 | 1 | 89303718 | each command, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_HOPPER_CALLER_SCAN_BUDGET_MS` | `src/domain/analysisResourceLimits.ts` | 2000 | 1 | 2147483647 | each caller scan, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_JAVASCRIPT_MEMBER_MUTATION_ALIAS_STATES` | `src/domain/analysisResourceLimits.ts` | 524288 | 1 | 8388608 | each JavaScript source's member-mutation phase, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_JEB_MCP_TIMEOUT_MS` | `src/jeb/JebMcpConnection.ts` | 60000 (MCP SDK default) | 1 | 2147483647 | discovery and each tools/call, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_KUNA_PROJECT_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 1800000 | 1 | 2147483647 | each project export and whole-program scan, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_KUNA_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 120000 | 1 | 2147483647 | each function decompile, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_PWNTOOLS_MAX_INPUT_BYTES` | `src/config/pwntoolsInputLimit.ts` | 33554432 | 1 | 1073741824 | each pwntools decode, unless the call passes `producer_limits.input_bytes`; otherwise the operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | -| `REA_WINDOWS_CHANGE_TIME_DRIFT_MS` | `src/config/windowsChangeTimeSettle.ts` | 2000 | 1 | 60000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | -| `REA_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS` | `src/config/windowsChangeTimeSettle.ts` | 25 | 1 | 1000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | -| `REA_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS` | `src/config/windowsChangeTimeSettle.ts` | 40 | 1 | 1000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | +| Environment | Owner | Default | Floor | Maximum | Applies | Fields | +| --------------------------------------------- | --------------------------------------------------- | ----------------------: | ------: | -----------: | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------- | +| `REA_BRIDGE_MAX_REQUEST_BYTES` | `src/config/bridgeMaxRequestBytes.ts` | 16777216 | 7464 | 536870887 | each bridge request, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_BRIDGE_MAX_REPLY_BYTES` | `src/config/bridgeMaxReplyBytes.ts` | 268435456 | 1048576 | 536870888 | each Hopper or Ghidra request, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_CAPTURE_PACKET_JSON_EXPANSION_BYTES` | `src/config/capturePacketJsonExpansion.ts` | 137438953472 | 1048576 | 137438953472 | each packet-capture reply before parsing, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_DEBUGGER_COMMAND_DEADLINE_MS` | `src/domain/analysisResourceLimits.ts` | 120000 | 1 | 2147483647 | each command, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES` | `src/domain/analysisResourceLimits.ts` | 16777216 | 1 | 89303718 | each command, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_HOPPER_CALLER_SCAN_BUDGET_MS` | `src/domain/analysisResourceLimits.ts` | 2000 | 1 | 2147483647 | each caller scan, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_JAVASCRIPT_MEMBER_MUTATION_ALIAS_STATES` | `src/domain/analysisResourceLimits.ts` | 524288 | 1 | 8388608 | each JavaScript source's member-mutation phase, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_JEB_MCP_TIMEOUT_MS` | `src/jeb/JebMcpConnection.ts` | 60000 (MCP SDK default) | 1 | 2147483647 | discovery and each tools/call, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_KUNA_PROJECT_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 1800000 | 1 | 2147483647 | each project export and whole-program scan, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_KUNA_TIMEOUT_MS` | `src/kuna/KunaConfiguration.ts` | 120000 | 1 | 2147483647 | each function decompile, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_PWNTOOLS_MAX_INPUT_BYTES` | `src/config/pwntoolsInputLimit.ts` | 33554432 | 1 | 1073741824 | each pwntools decode, unless the call passes `producer_limits.input_bytes`; otherwise the operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | +| `REA_WINDOWS_CHANGE_TIME_DRIFT_MS` | `src/config/windowsChangeTimeSettle.ts` | 2000 | 1 | 60000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | +| `REA_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS` | `src/config/windowsChangeTimeSettle.ts` | 25 | 1 | 1000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | +| `REA_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS` | `src/config/windowsChangeTimeSettle.ts` | 40 | 1 | 1000 | each ChangeTime observation, from the accepted operation snapshot | `update_mode: "runtime"`; invalid launch omits `pending_restart` | +| `REA_ZSTD_MAX_FRAMES` | `src/domain/compression/zstandardFrameAllowance.ts` | 4096 | 1 | 65536 | each exact Zstandard buffer, from the accepted operation snapshot | `update_mode: "runtime"`, `pending_restart: false` | ## Restart-only numeric limits diff --git a/scripts/verify/advertised-schemas/tools/inspect_analysis_settings.json b/scripts/verify/advertised-schemas/tools/inspect_analysis_settings.json index fc17490efc..f79d092fc6 100644 --- a/scripts/verify/advertised-schemas/tools/inspect_analysis_settings.json +++ b/scripts/verify/advertised-schemas/tools/inspect_analysis_settings.json @@ -1,6 +1,6 @@ { "schemas": { - "input compact": "5ea37049dad60cdc", + "input compact": "71360cba238d0c73", "output compact": "b3a429110120aff1" }, "patterns": {} diff --git a/scripts/verify/advertised-schemas/tools/update_analysis_settings.json b/scripts/verify/advertised-schemas/tools/update_analysis_settings.json index d1a23c1a60..637957d379 100644 --- a/scripts/verify/advertised-schemas/tools/update_analysis_settings.json +++ b/scripts/verify/advertised-schemas/tools/update_analysis_settings.json @@ -1,6 +1,6 @@ { "schemas": { - "input compact": "c6bdd8000e43ec47", + "input compact": "4f5355f35275f28b", "output compact": "39446aa1012006cc" }, "patterns": {} diff --git a/src/android/AndroidJniLibraries.ts b/src/android/AndroidJniLibraries.ts index d3de29cf3f..79be283ea1 100644 --- a/src/android/AndroidJniLibraries.ts +++ b/src/android/AndroidJniLibraries.ts @@ -2,6 +2,7 @@ import { ArtifactReaderFailure, type ArtifactResourceScope, streamChunkToBuffer, + yieldArtifactRead, ZipArtifactReader, } from "../resources/index.js"; import type { @@ -11,6 +12,7 @@ import type { AndroidJniLibraryCheckpoint, } from "../domain/android/androidJniBindings.js"; import { compileAndroidJniLibrarySelection } from "../domain/android/androidJniBindings.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { inspectAndroidJniLibrary } from "../domain/android/androidJniLibrary.js"; @@ -18,7 +20,6 @@ import { JadxInterruption, throwIfJadxInterrupted, } from "./JadxInterruption.js"; -import { AnalysisError } from "../domain/analysisErrorBase.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; import { analysisClientFailureForOperation } from "../providers/AnalysisClientFailure.js"; @@ -43,6 +44,7 @@ export interface AndroidJniLibraries { const jniReadFailure = ( cause: unknown, ): NonNullable => { + if (cause instanceof AnalysisError) throw cause; const message = errorMessage(cause); if (!(cause instanceof ArtifactReaderFailure)) return { kind: "io", message }; // The library report has only a message, so a reader-named ceiling's @@ -209,6 +211,7 @@ export const readAndroidJniLibraries = async (input: { }; const parsed = await inspectAndroidJniLibrary(bytes, { signal: input.signal, + pause: () => yieldArtifactRead(input.signal), }); throwIfJadxInterrupted(input.signal); observation = { diff --git a/src/application/AnalysisSettings.test.ts b/src/application/AnalysisSettings.test.ts index 05889dbdd0..80ec9138bf 100644 --- a/src/application/AnalysisSettings.test.ts +++ b/src/application/AnalysisSettings.test.ts @@ -156,6 +156,53 @@ it("keeps accepted operations and their typed provider views stable across concu }); }); +it("lists zstd.max_frames for every operation that enforces the cap", () => { + const store = new AnalysisSettings({}); + for (const operation of [ + "inspect_artifact", + "extract_artifact", + "inspect_gpu_code_objects", + "inspect_dwarf_units", + "inspect_elf_notes", + "inspect_binary_hardening", + "inspect_elf_unwind_ranges", + "inspect_native_exports", + "inspect_native_initializers", + "inspect_kernel_module", + "inspect_elf_debug_link", + "inspect_go_binary", + "inspect_rust_binary", + "inspect_il2cpp_metadata", + "inspect_android_jni_bindings", + ]) + expect( + store.inspect({ operation }).settings.map(({ setting }) => setting), + ).toContain("zstd.max_frames"); + for (const provider of [ + "gpu-code-objects", + "rea-dwarf-units", + "rea-elf-notes", + "rea-binary-hardening", + "rea-elf-unwind-ranges", + "rea-native-exports", + "rea-native-initializers", + "rea-kernel-module", + "rea-elf-debug-link", + "go-build-info", + "rust-auditable", + "il2cpp-metadata", + "jadx", + ]) + expect( + store.inspect({ provider }).settings.map(({ setting }) => setting), + ).toContain("zstd.max_frames"); + expect( + store + .inspect({ operation: "inspect_binary_layout" }) + .settings.map(({ setting }) => setting), + ).not.toContain("zstd.max_frames"); +}); + it("exposes shared provider controls and preserves the explicit unbounded decompile value", () => { const environment = { REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "7", diff --git a/src/artifacts/AndroidOtaPayloadArtifactReader.boundaries.test.ts b/src/artifacts/AndroidOtaPayloadArtifactReader.boundaries.test.ts index 33b086c594..d4a8ba754e 100644 --- a/src/artifacts/AndroidOtaPayloadArtifactReader.boundaries.test.ts +++ b/src/artifacts/AndroidOtaPayloadArtifactReader.boundaries.test.ts @@ -1,7 +1,9 @@ import { artifactPreparationRecoveryCases } from "../../tests/fixtures/artifactReaderLifecycle.js"; +import { createHash } from "node:crypto"; import { truncate, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { buffer } from "node:stream/consumers"; +import { scheduler } from "node:timers/promises"; import { expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; @@ -164,6 +166,57 @@ it("refuses to open a partition whose distinct operation data exceeds the read b } }); +it("cancels a many-frame REPLACE_ZSTD partition", async () => { + const frame = Buffer.from([ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, 0x61, + ]); + const compressed = Buffer.alloc(frame.length * 32); + for (let index = 0; index < 32; index += 1) + frame.copy(compressed, index * frame.length); + const path = await fixture( + buildAndroidOtaPayload({ + partitions: [ + { + name: "vendor", + operations: [ + { + type: 14, + data: compressed, + dataHash: createHash("sha256").update(compressed).digest(), + dstExtents: [{ startBlock: 0, numBlocks: 1 }], + }, + ], + }, + ], + }), + ); + const admitted = await openRegularFile(path, { symlinks: "reject" }); + const reader = new AndroidOtaPayloadArtifactReader(path, admitted); + const reason = new Error("stop ota frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + try { + const [entry] = await entries(reader); + await expect(reader.open(entry!, controller.signal)).rejects.toMatchObject({ + reason: "cancelled", + cause: reason, + }); + } finally { + spy.mockRestore(); + try { + await reader.close(); + } finally { + await admitted.handle.close(); + } + } +}); + it("preserves cancellation while reconstructing a partition", async () => { const path = await fixture( buildAndroidOtaPayload({ diff --git a/src/artifacts/AndroidOtaPayloadArtifactReader.ts b/src/artifacts/AndroidOtaPayloadArtifactReader.ts index 8824a2974d..ad8110b4c2 100644 --- a/src/artifacts/AndroidOtaPayloadArtifactReader.ts +++ b/src/artifacts/AndroidOtaPayloadArtifactReader.ts @@ -9,6 +9,7 @@ import type { import { readArtifactFileBytes } from "./ArtifactFileRange.js"; import { assertArtifactReadActive, + yieldArtifactRead, ArtifactEntryContentUnavailable, ArtifactReaderFailure, copyArtifactEntry, @@ -21,6 +22,8 @@ import { } from "../resources/index.js"; import { ownArtifactStream } from "./OwnedArtifactStream.js"; import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; +import { ZstandardFrameAllowanceError } from "../domain/compression/zstandardFrameAllowance.js"; +import { ConfigurationError } from "../domain/configurationErrors.js"; import { ANDROID_OTA_HEADER_V2_BYTES, androidOtaDistinctOperationDataBytes, @@ -28,7 +31,7 @@ import { androidOtaExtractRefusalLimitation, parseAndroidOtaHeader, parseAndroidOtaPayload, - reconstructAndroidOtaPartition, + reconstructAndroidOtaPartitionCooperative, withAndroidOtaPayloadSignature, type AndroidOtaPartition, type AndroidOtaPayload, @@ -143,11 +146,12 @@ export class AndroidOtaPayloadArtifactReader implements ArtifactReader { } await this.#source.read(signal); this.#assertOpen(signal); - const reconstructed = reconstructAndroidOtaPartition( + const reconstructed = await reconstructAndroidOtaPartitionCooperative( stored.partition, payload.blockSize, (offset, length) => blob.get(`${String(offset)}:${String(length)}`) ?? new Uint8Array(), + () => yieldArtifactRead(signal), // A SHA-256 claim on the entry is checked by the shared scanner under // the caller's integrity policy, so record-and-continue can keep the // observed bytes. Without that claim the reader checks it here. @@ -347,6 +351,12 @@ const reconstructFailure = ( cause: failure, integrityCheck: "declared-digest", }); + if (failure instanceof ConfigurationError) throw failure; + if (failure._tag === "ZstandardFrameWorkRefusal") + throw new ZstandardFrameAllowanceError( + { maxFrames: failure.value, source: failure.source }, + failure.offset, + ); return parseFailure(failure); }; diff --git a/src/artifacts/ZipArtifactReader.test.ts b/src/artifacts/ZipArtifactReader.test.ts index f5c6890c6c..1c06946a8e 100644 --- a/src/artifacts/ZipArtifactReader.test.ts +++ b/src/artifacts/ZipArtifactReader.test.ts @@ -238,31 +238,81 @@ describe("ZIP Zstandard method 93", () => { it.each([ [ - "frameParameter_unsupported", + "reserved_frame_descriptor", "28b52ffddf339d0000686d656d626572206279746573200100e1995f", ], [ - "corruption_detected", + "ZSTD_error_corruption_detected", "28b52ffd20cc9d0000686d656d626572206279746573200100e1995f", ], [ - "dictionary_corrupted", + "ZSTD_error_dictionary_corrupted", "28b52ffd20339d0000976d656d626572206279746573200100e1995f", ], + ] as const)("keeps the %s format fault and its cause", async (code, hex) => { + const content = Buffer.from( + "member bytes member bytes member bytes member bytes", + ); + await expect( + readFirstZstandardMember( + zstandardZipFixture(content, { compressed: Buffer.from(hex, "hex") }), + ), + ).rejects.toMatchObject({ + reason: "format", + message: expect.stringContaining("Invalid compressed data"), + cause: { code }, + }); + }); +}); + +describe("ZIP Zstandard frame completion", () => { + const first = Buffer.from("first text frame\n"); + const second = Buffer.from("second text frame: café\n"); + const firstFrame = zstdCompressSync(first); + const secondFrame = zstdCompressSync(second, { + params: { [constants.ZSTD_c_checksumFlag]: 1 }, + }); + // The skippable payload spans chunks and the next magic straddles a + // zip.js input chunk boundary (64 KiB). + const skipped = Buffer.alloc(1024 * 1024 - firstFrame.length - 2); + skipped.writeUInt32LE(0x184d2a5f, 0); + skipped.writeUInt32LE(skipped.length - 8, 4); + const content = Buffer.concat([first, second]); + + it.each([ + ["two ordinary", [firstFrame, secondFrame]], + ["ordinary and skippable", [firstFrame, skipped, secondFrame]], + ] as const)("streams complete %s frames in order", async (_label, frames) => { + const compressed = Buffer.concat(frames); + const decoded = await readFirstZstandardMember( + zstandardZipFixture(content, { compressed }), + ); + expect(decoded.equals(content)).toBe(true); + }); + + it.each([ + ["a partial frame magic", first, secondFrame.subarray(0, 3)], + ["an incomplete ordinary frame", first, secondFrame.subarray(0, 5)], + [ + "an incomplete skippable frame", + first, + skipped.subarray(0, skipped.length - 1), + ], + [ + "an incomplete checksum", + content, + secondFrame.subarray(0, secondFrame.length - 1), + ], ] as const)( - "keeps the native %s format fault and its cause", - async (code, hex) => { - const content = Buffer.from( - "member bytes member bytes member bytes member bytes", - ); + "refuses %s after a complete frame", + async (_label, output, suffix) => { + const compressed = Buffer.concat([firstFrame, suffix]); await expect( - readFirstZstandardMember( - zstandardZipFixture(content, { compressed: Buffer.from(hex, "hex") }), - ), + readFirstZstandardMember(zstandardZipFixture(output, { compressed })), ).rejects.toMatchObject({ + name: "ArtifactReaderFailure", reason: "format", message: expect.stringContaining("Invalid compressed data"), - cause: { code: `ZSTD_error_${code}` }, }); }, ); diff --git a/src/config/analysisSettings.ts b/src/config/analysisSettings.ts index 49a65f1d94..e9092996b4 100644 --- a/src/config/analysisSettings.ts +++ b/src/config/analysisSettings.ts @@ -34,6 +34,7 @@ import { windowsChangeTimeSettlePollMsSchema, } from "./windowsChangeTimeSettle.js"; import { pwntoolsInputBytesSchema } from "./pwntoolsInputLimit.js"; +import { zstandardMaxFramesSchema } from "../domain/compression/zstandardFrameAllowance.js"; const environmentSchemas: Readonly>> = { REA_ASAR_MAX_HEADER_BYTES: asarMaxHeaderBytesSchema, @@ -67,6 +68,7 @@ const environmentSchemas: Readonly>> = REA_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS: windowsChangeTimeSettlePollMsSchema, REA_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS: windowsChangeTimeSettleMaxPollsSchema, + REA_ZSTD_MAX_FRAMES: zstandardMaxFramesSchema, }; export const launchAnalysisSetting = ( id: AnalysisSettingId, diff --git a/src/domain/analysisResourceLimits.ts b/src/domain/analysisResourceLimits.ts index 5eac06b621..139db98f32 100644 --- a/src/domain/analysisResourceLimits.ts +++ b/src/domain/analysisResourceLimits.ts @@ -276,3 +276,14 @@ export const DEFAULT_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS = 25; export const MAXIMUM_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS = 1_000; export const DEFAULT_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS = 40; export const MAXIMUM_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS = 1_000; + +/** + * Zstandard frames one buffer may contain before decoding is refused. + * Legitimate streams are one frame, sometimes with a few skippable frames. + * 65,536 one-byte frames is the measured Node 22 ceiling: a few hundred + * milliseconds of synchronous decoder work, and far below the millions of + * frames the existing byte allowances would still admit. + */ +export const DEFAULT_ZSTD_MAX_FRAMES = 4_096; +export const MINIMUM_ZSTD_MAX_FRAMES = 1; +export const MAXIMUM_ZSTD_MAX_FRAMES = 65_536; diff --git a/src/domain/analysisSettingDefinitions.ts b/src/domain/analysisSettingDefinitions.ts index 75efc5d202..d4ee650c59 100644 --- a/src/domain/analysisSettingDefinitions.ts +++ b/src/domain/analysisSettingDefinitions.ts @@ -60,6 +60,9 @@ import { MAXIMUM_WINDOWS_CHANGE_TIME_SETTLE_POLL_MS, DEFAULT_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS, MAXIMUM_WINDOWS_CHANGE_TIME_SETTLE_MAX_POLLS, + DEFAULT_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, + MAXIMUM_ZSTD_MAX_FRAMES, } from "./analysisResourceLimits.js"; import { JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS, @@ -604,6 +607,48 @@ export const ANALYSIS_SETTINGS = { relatedProviders: ["jadx"], operations: windowsChangeTimeOperations, }, + "zstd.max_frames": { + updateMode: "runtime", + environment: "REA_ZSTD_MAX_FRAMES", + defaultValue: DEFAULT_ZSTD_MAX_FRAMES, + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + appliesAt: "next_operation", + provider: "rea-artifact-graph", + relatedProviders: [ + "gpu-code-objects", + "rea-dwarf-units", + "rea-elf-notes", + "rea-binary-hardening", + "rea-elf-unwind-ranges", + "rea-native-exports", + "rea-native-initializers", + "rea-kernel-module", + "rea-elf-debug-link", + "go-build-info", + "rust-auditable", + "il2cpp-metadata", + "jadx", + ], + operations: [ + "inspect_artifact", + "extract_artifact", + "inspect_gpu_code_objects", + "inspect_dwarf_units", + "inspect_elf_notes", + "inspect_binary_hardening", + "inspect_elf_unwind_ranges", + "inspect_native_exports", + "inspect_native_initializers", + "inspect_kernel_module", + "inspect_elf_debug_link", + "inspect_go_binary", + "inspect_rust_binary", + "inspect_il2cpp_metadata", + "inspect_android_jni_bindings", + ], + }, } as const satisfies Record; export const analysisSettingIds = Object.keys( ANALYSIS_SETTINGS, diff --git a/src/domain/analysisSettings.ts b/src/domain/analysisSettings.ts index 59904cc58e..dc47bbf6b0 100644 --- a/src/domain/analysisSettings.ts +++ b/src/domain/analysisSettings.ts @@ -31,6 +31,7 @@ export interface AnalysisSettingValues { readonly "windows.change_time_drift_ms": number; readonly "windows.change_time_settle_poll_ms": number; readonly "windows.change_time_settle_max_polls": number; + readonly "zstd.max_frames": number; } export type AnalysisSettingId = keyof AnalysisSettingValues; export interface AnalysisSettingsSnapshot { diff --git a/src/domain/android/androidJniLibrary.ts b/src/domain/android/androidJniLibrary.ts index 619569c0e3..d067c07eb3 100644 --- a/src/domain/android/androidJniLibrary.ts +++ b/src/domain/android/androidJniLibrary.ts @@ -7,9 +7,13 @@ import type { AndroidJniTableEntry, AndroidJniTableSlot, } from "./androidJniLibraryFacts.js"; -import { parseNativeExportsSteps } from "../native/nativeExportParser.js"; -import { readElfDirectoryFromBuffer } from "../native/nativeSectionDirectory.js"; -import { readElfImage } from "../native/elfImage.js"; +import { parseElfExportsFromDirectorySteps } from "../native/nativeExportParser.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; +import { + readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, +} from "../native/nativeSectionDirectory.js"; +import { readElfImageFromDirectory } from "../native/elfImage.js"; import { BinaryFormatFailure, BinaryMappingFailure, @@ -23,12 +27,14 @@ import { type ElfPointerOverlay, } from "../native/elfPointerRelocations.js"; import { sha256Hex } from "../../kernel/sha256Hex.js"; +import { zstandardFrameAllowanceCause } from "../compression/zstandardFrameAllowance.js"; const MAXIMUM_STRING_BYTES = 65_535; const MAXIMUM_STRING_WORK = 64 * 1024 * 1024; const MAXIMUM_RETAINED_BYTES = 16 * 1024 * 1024; const MAXIMUM_SCAN_BYTES = 64 * 1024 * 1024; const MAXIMUM_TABLE_ENTRIES = 65_536; +const OPERATION = "inspect_android_jni_bindings"; const hex = (value: bigint): string => `0x${value.toString(16)}`; /** Strict Modified UTF-8: no four-byte UTF-8, overlong forms or replacements. */ @@ -90,11 +96,18 @@ const methodDescriptor = (descriptor: string): boolean => { /** Parse borrowed immutable ELF bytes; no filesystem, process or target execution. */ export const inspectAndroidJniLibrary = async ( bytes: Buffer, - options: { readonly signal?: AbortSignal } = {}, + options: { + readonly signal?: AbortSignal; + readonly pause?: ZstandardFramePause; + } = {}, ): Promise> => { const active = (): void => { if (options.signal?.aborted === true) throw options.signal; }; + const pause = async (): Promise => { + await immediate(); + active(); + }; try { active(); if (!bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) @@ -102,8 +115,16 @@ export const inspectAndroidJniLibrary = async ( kind: "unsupported", message: "JNI library inspection requires an ELF image", }); - const directory = readElfDirectoryFromBuffer(bytes); - const image = readElfImage(bytes); + const directory = + options.pause === undefined && options.signal === undefined + ? readElfDirectoryFromBuffer(bytes, OPERATION) + : await readElfDirectoryFromBufferCooperative( + bytes, + options.pause ?? pause, + OPERATION, + ); + active(); + const image = readElfImageFromDirectory(bytes, directory); const mappings = createFileMappingIndex(image.mappings, image.zero_fills); if (image.executable_mappings === undefined) return err({ @@ -126,23 +147,22 @@ export const inspectAndroidJniLibrary = async ( "JNI library facts exceed the 16 MiB retained representation allowance", ); }; - const exportSteps = parseNativeExportsSteps(bytes, { - path: "[admitted immutable ELF bytes]", - sha256: sha256Hex(bytes), - bytes: bytes.length, - }); + const exportSteps = parseElfExportsFromDirectorySteps( + bytes, + { + path: "[admitted immutable ELF bytes]", + sha256: sha256Hex(bytes), + bytes: bytes.length, + }, + directory, + ); + await pause(); let exportStep = exportSteps.next(); while (!exportStep.done) { - await immediate(); - active(); + await pause(); exportStep = exportSteps.next(); } const rawExports = exportStep.value; - if (rawExports.format !== "elf") - return err({ - kind: "unsupported", - message: "JNI library inspection requires ELF exports", - }); const exportRows: AndroidJniLibraryFacts["exports"] = []; let exportsStatus: AndroidJniLibraryFacts["coverage"]["exports_status"] = rawExports.coverage.status === "complete" @@ -499,6 +519,9 @@ export const inspectAndroidJniLibrary = async ( kind: "cancelled", message: "JNI library inspection was cancelled", }); + const frameAllowance = zstandardFrameAllowanceCause(cause); + if (frameAllowance !== undefined) + return err({ kind: "capacity", message: frameAllowance.message }); if ( cause instanceof BinaryResourceFailure || (cause instanceof ElfPointerRelocationFailure && cause.kind === "limit") diff --git a/src/domain/compression/zstandardExactFrame.test.ts b/src/domain/compression/zstandardExactFrame.test.ts new file mode 100644 index 0000000000..0953909ec1 --- /dev/null +++ b/src/domain/compression/zstandardExactFrame.test.ts @@ -0,0 +1,314 @@ +import { constants, zstdCompressSync } from "node:zlib"; +import { describe, expect, it } from "vitest"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, + DEFAULT_ZSTD_MAX_FRAMES, +} from "../analysisResourceLimits.js"; +import { ConfigurationError } from "../configurationErrors.js"; +import { + decodeZstandardFrames, + decodeZstandardFramesCooperative, + ownedZstandardBytes, +} from "./zstandardExactFrame.js"; +import { + resolveZstandardFrameAllowance, + ZSTD_MAX_FRAMES_ENVIRONMENT, + ZSTD_MAX_FRAMES_SETTING, +} from "./zstandardFrameAllowance.js"; + +const rawFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +const repeat = (frame: Buffer, count: number): Buffer => { + const bytes = Buffer.alloc(frame.length * count); + for (let index = 0; index < count; index += 1) + frame.copy(bytes, index * frame.length); + return bytes; +}; + +const skippable = (payload: Buffer, magic = 0x184d2a50): Buffer => { + const bytes = Buffer.alloc(8 + payload.length); + bytes.writeUInt32LE(magic, 0); + bytes.writeUInt32LE(payload.length, 4); + payload.copy(bytes, 8); + return bytes; +}; + +const allowance = ( + maxFrames: number, + source: "default" | "setting" | "runtime", +) => ({ maxFrames, source }); + +it("yields again after the last frame-by-frame decode before success", async () => { + let pauses = 0; + const controller = new AbortController(); + const pending = decodeZstandardFramesCooperative( + rawFrame(0x61), + {}, + async () => { + pauses += 1; + if (pauses === 1) { + controller.abort(); + return; + } + if (controller.signal.aborted) + throw Object.assign(new Error("cancelled"), { reason: "cancelled" }); + }, + ); + await expect(pending).rejects.toMatchObject({ reason: "cancelled" }); + expect(pauses).toBe(2); +}); + +it("observes cancellation while admitting a multi-block frame", async () => { + const first = zstdCompressSync(Buffer.alloc(8 * 1024 * 1024, "read me\n")); + const second = zstdCompressSync(Buffer.from("another frame")); + const bytes = Buffer.concat([first, second]); + const options = { allowance: allowance(1, "runtime") }; + const stopped = new Error("cancelled"); + await expect( + decodeZstandardFramesCooperative(bytes, options, async () => { + throw stopped; + }), + ).rejects.toBe(stopped); + await expect( + decodeZstandardFramesCooperative(bytes, options, async () => {}), + ).resolves.toMatchObject({ + reason: "work_limit", + offset: first.length, + setting: "zstd.max_frames", + value: 1, + source: "runtime", + }); +}); + +describe("exact Zstandard frames", () => { + it("accepts concatenated frames and skippable frames", () => { + const first = rawFrame(0x61); + const second = rawFrame(0x62); + const skip = skippable(Buffer.from("skip"), 0x184d2a5f); + const bytes = Buffer.concat([ + skip, + first, + skippable(Buffer.alloc(0)), + second, + ]); + const decoded = decodeZstandardFrames(bytes, { + allowance: allowance(8, "default"), + }); + expect(decoded).toMatchObject({ + ok: true, + consumedBytes: bytes.length, + frames: 4, + }); + if (!decoded.ok) return; + expect([...decoded.bytes]).toEqual([0x61, 0x62]); + }); + + it.each([ + [ + "frame header", + (frame: Buffer) => frame.subarray(0, 5), + "truncated_frame_header", + ], + ["block", (frame: Buffer) => frame.subarray(0, -1), "truncated_block"], + ] as const)("rejects a truncated %s", (_label, trim, reason) => { + expect(decodeZstandardFrames(trim(rawFrame(0x61)))).toMatchObject({ + ok: false, + reason, + offset: expect.any(Number), + }); + }); + + it("rejects a truncated checksum and a wrong checksum separately", () => { + const frame = zstdCompressSync(Buffer.from("hello zstd checksum"), { + params: { [constants.ZSTD_c_checksumFlag]: 1 }, + }); + expect(decodeZstandardFrames(frame.subarray(0, -1))).toMatchObject({ + ok: false, + reason: "truncated_checksum", + }); + expect(decodeZstandardFrames(frame.subarray(0, -4))).toMatchObject({ + ok: false, + reason: "truncated_checksum", + }); + const wrong = Buffer.from(frame); + const last = wrong.length - 1; + wrong.writeUInt8(wrong.readUInt8(last) ^ 0xff, last); + expect(decodeZstandardFrames(wrong)).toMatchObject({ + ok: false, + reason: "malformed", + code: "ZSTD_error_checksum_wrong", + }); + expect(decodeZstandardFrames(frame)).toMatchObject({ ok: true }); + }); + + it("rejects trailing garbage and a partial next-frame magic", () => { + const frame = rawFrame(0x61); + expect( + decodeZstandardFrames(Buffer.concat([frame, Buffer.of(0)])), + ).toMatchObject({ ok: false, reason: "trailing_bytes" }); + expect( + decodeZstandardFrames( + Buffer.concat([frame, Buffer.of(0x28, 0xb5, 0x2f)]), + ), + ).toMatchObject({ ok: false, reason: "trailing_bytes" }); + expect( + decodeZstandardFrames( + Buffer.concat([frame, Buffer.of(0x28, 0xb5, 0x2f, 0xfd)]), + ), + ).toMatchObject({ ok: false, reason: "truncated_frame_header" }); + }); + + it("stops a prefix at the first non-frame byte", async () => { + const frame = rawFrame(0x61); + const suffix = Buffer.from("CCOB"); + const input = Buffer.concat([frame, suffix]); + const options = { mode: "prefix" } as const; + const expected = { + ok: true, + bytes: Buffer.from("a"), + consumedBytes: frame.length, + frames: 1, + }; + expect(decodeZstandardFrames(input, options)).toMatchObject(expected); + await expect( + decodeZstandardFramesCooperative(input, options, async () => {}), + ).resolves.toMatchObject(expected); + }); + + it("refuses 65536 productive frames on the work allowance", () => { + const count = 65_536; + const frame = rawFrame(0x61); + const bytes = repeat(frame, count); + expect( + decodeZstandardFrames(bytes, { + maxOutputLength: count, + allowance: allowance(count - 1, "setting"), + }), + ).toMatchObject({ + ok: false, + reason: "work_limit", + setting: ZSTD_MAX_FRAMES_SETTING, + value: count - 1, + source: "setting", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + }); + const decoded = decodeZstandardFrames(bytes, { + maxOutputLength: count, + allowance: allowance(count, "runtime"), + }); + expect(decoded).toMatchObject({ + ok: true, + frames: count, + consumedBytes: bytes.length, + }); + if (!decoded.ok) return; + expect(decoded.bytes.length).toBe(count); + expect(decoded.bytes[0]).toBe(0x61); + expect(decoded.bytes[count - 1]).toBe(0x61); + }); +}); + +it("does not retain a decoder chunk behind a short frame", () => { + const backing = Buffer.alloc(16 * 1024, 0x61); + const slice = backing.subarray(0, 1); + const owned = ownedZstandardBytes(slice); + expect(owned.length).toBe(1); + expect(owned[0]).toBe(0x61); + expect(owned.buffer).not.toBe(backing.buffer); + const exact = Buffer.allocUnsafeSlow(1); + exact[0] = 0x62; + expect(ownedZstandardBytes(exact)).toBe(exact); +}); + +describe("Zstandard frame header widths", () => { + const magic = (bytes: number[]) => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, ...bytes]); + const decode = (bytes: Buffer) => + decodeZstandardFrames(bytes, { + allowance: allowance(8, "default"), + }); + + it("accepts content-size, window, dictionary-id, and RLE frames", () => { + const fcs4 = magic([0xa0, 1, 0, 0, 0, 0x09, 0, 0, 0x41]); + const fcs8 = magic([0xe0, 1, 0, 0, 0, 0, 0, 0, 0, 0x09, 0, 0, 0x41]); + const fcs2 = Buffer.alloc(10 + 256); + fcs2.writeUInt32LE(0xfd2fb528, 0); + fcs2[4] = 0x60; + fcs2.writeUInt16LE(0, 5); + fcs2.writeUIntLE(1 | (256 << 3), 7, 3); + fcs2.fill(0x42, 10); + const windowed = magic([0x00, 0x50, 0x01, 0, 0]); + const dictionary1 = magic([0x21, 0x00, 0x00, 0x01, 0, 0]); + const dictionary2 = magic([0x22, 0, 0, 0x00, 0x01, 0, 0]); + const dictionary4 = magic([0x23, 0, 0, 0, 0, 0x00, 0x01, 0, 0]); + const rle = magic([0x20, 0x04, 0x23, 0, 0, 0x41]); + expect(decode(fcs4)).toMatchObject({ ok: true, frames: 1 }); + expect(decode(fcs8)).toMatchObject({ ok: true, frames: 1 }); + expect(decode(fcs2)).toMatchObject({ ok: true, frames: 1 }); + const empty = decode(windowed); + const repeated = decode(rle); + const wide = decode(fcs2); + const sized = decode(fcs4); + expect(empty.ok && empty.bytes.length).toBe(0); + expect(decode(dictionary1).ok).toBe(true); + expect(decode(dictionary2).ok).toBe(true); + expect(decode(dictionary4).ok).toBe(true); + expect(sized.ok && sized.bytes[0]).toBe(0x41); + expect( + repeated.ok && Buffer.compare(repeated.bytes, Buffer.from("AAAA")) === 0, + ).toBe(true); + expect(wide.ok && wide.bytes.length).toBe(256); + }); + + it.each([ + ["content size", () => magic([0xa0, 1, 0]), "truncated_frame_header"], + [ + "eight-byte content size", + () => magic([0xe0, 1, 0, 0, 0]), + "truncated_frame_header", + ], + ["window descriptor", () => magic([0x00]), "truncated_frame_header"], + ["one-byte dictionary id", () => magic([0x21]), "truncated_frame_header"], + [ + "two-byte dictionary id", + () => magic([0x22, 0]), + "truncated_frame_header", + ], + [ + "four-byte dictionary id", + () => magic([0x23, 0, 0]), + "truncated_frame_header", + ], + ["RLE block", () => magic([0x20, 0x04, 0x23, 0, 0]), "truncated_block"], + ] as const)("rejects a truncated %s", (_label, frame, reason) => { + expect(decode(frame())).toMatchObject({ ok: false, reason }); + }); +}); + +describe("zstd.max_frames allowance", () => { + it("resolves default, launch, and runtime sources", () => { + expect(resolveZstandardFrameAllowance({})).toEqual({ + ok: true, + value: { maxFrames: DEFAULT_ZSTD_MAX_FRAMES, source: "default" }, + }); + expect( + resolveZstandardFrameAllowance({ [ZSTD_MAX_FRAMES_ENVIRONMENT]: "8" }), + ).toEqual({ ok: true, value: { maxFrames: 8, source: "setting" } }); + expect(resolveZstandardFrameAllowance({}, 3)).toEqual({ + ok: true, + value: { maxFrames: 3, source: "runtime" }, + }); + const invalid = resolveZstandardFrameAllowance({ + [ZSTD_MAX_FRAMES_ENVIRONMENT]: "0", + }); + expect(invalid.ok).toBe(false); + if (!invalid.ok) expect(invalid.error).toBeInstanceOf(ConfigurationError); + const above = resolveZstandardFrameAllowance({ + [ZSTD_MAX_FRAMES_ENVIRONMENT]: String(MAXIMUM_ZSTD_MAX_FRAMES + 1), + }); + expect(above.ok).toBe(false); + }); +}); diff --git a/src/domain/compression/zstandardExactFrame.ts b/src/domain/compression/zstandardExactFrame.ts new file mode 100644 index 0000000000..6a37899dfe --- /dev/null +++ b/src/domain/compression/zstandardExactFrame.ts @@ -0,0 +1,522 @@ +import { constants as bufferConstants } from "node:buffer"; +import { promisify } from "node:util"; +import { + constants, + zstdCompressSync, + zstdDecompress, + zstdDecompressSync, +} from "node:zlib"; +import { nodeCodecResult } from "../../kernel/nodeCodecResult.js"; +import { hostRangeErrorKind } from "../parseFailure.js"; +import type { ConfigurationError } from "../configurationErrors.js"; +import { + currentZstandardFrameAllowance, + type ZstandardFrameAllowance, + type ZstandardFrameSource, + zstandardWorkLimitFacts, +} from "./zstandardFrameAllowance.js"; +import { + ZSTANDARD_MAGIC_BYTES, + zstandardFrameKind, + zstandardFrameReads, + type ZstandardFrameKind, +} from "./zstandardFrameWire.js"; + +/** Zstandard's default maximum decoder window, 2^27 bytes (128 MiB). */ +export const ZSTANDARD_WINDOW_LOG_MAX = 27; + +/** + * Structural framing failures. `work_limit` is a decoder-work refusal, not + * evidence that the bytes are malformed. + */ +export const ZSTANDARD_FRAME_REASONS = [ + "truncated_frame_header", + "truncated_block", + "truncated_checksum", + "trailing_bytes", + "work_limit", + "malformed", + "output_limit", + "invalid_allowance", +] as const; + +export type ZstandardFrameReason = (typeof ZSTANDARD_FRAME_REASONS)[number]; + +export type ZstandardFrameMode = "exact" | "prefix"; + +export interface ZstandardDecodeOptions { + /** Aggregate decoded-output ceiling. Node applies it to one decoder call. */ + readonly maxOutputLength?: number; + /** `ZSTD_d_windowLogMax`. The default is Zstandard's 128 MiB window. */ + readonly windowLogMax?: number; + /** Optional dictionary. Frames that name a dictionary fail without one. */ + readonly dictionary?: Uint8Array; + /** + * `exact` rejects every byte that is not part of a complete frame. + * `prefix` returns the decoded frames and the byte count they occupied, + * leaving a non-frame suffix unconsumed. + */ + readonly mode?: ZstandardFrameMode; + /** When omitted, the accepted operation's `zstd.max_frames` allowance. */ + readonly allowance?: ZstandardFrameAllowance; +} + +export interface ZstandardFrameFailure { + readonly ok: false; + readonly reason: ZstandardFrameReason; + readonly offset: number; + /** Node or structural code for a malformed frame. Absent for truncation. */ + readonly code?: string; + /** Decoder text for a malformed frame. Callers may keep it; tests use `code`. */ + readonly detail?: string; + readonly setting?: "zstd.max_frames"; + readonly value?: number; + readonly source?: ZstandardFrameSource; + readonly minimum?: number; + readonly maximum?: number; + /** Present when the launch seed for `zstd.max_frames` cannot be parsed. */ + readonly configuration?: ConfigurationError; +} + +export interface ZstandardFrameSuccess { + readonly ok: true; + readonly bytes: Buffer; + readonly consumedBytes: number; + readonly frames: number; +} + +export type ZstandardFrameResult = + | ZstandardFrameSuccess + | ZstandardFrameFailure; + +interface FrameSpan { + readonly offset: number; + readonly length: number; + readonly kind: ZstandardFrameKind; +} + +interface WalkSuccess { + readonly ok: true; + readonly frames: readonly FrameSpan[]; + readonly consumed: number; +} + +// Bound structural work between cooperative pauses, including across frames. +const READS_PER_PAUSE = 64; +interface FrameWalkProgress { + reads: number; +} +const needsFrameWalkPause = (progress: FrameWalkProgress): boolean => + ++progress.reads % READS_PER_PAUSE === 0; + +const failure = ( + reason: ZstandardFrameReason, + offset: number, + code?: string, + detail?: string, +): ZstandardFrameFailure => ({ + ok: false, + reason, + offset, + ...(code === undefined ? {} : { code }), + ...(detail === undefined ? {} : { detail }), +}); + +const workLimit = ( + allowance: ZstandardFrameAllowance, + offset: number, +): ZstandardFrameFailure => ({ + ok: false, + offset, + ...zstandardWorkLimitFacts(allowance), +}); + +const unreadBytes = new Uint8Array(0); + +function* frameAt( + bytes: Uint8Array, + offset: number, + kind: ZstandardFrameKind, + progress: FrameWalkProgress, +): Generator { + let cursor = offset + ZSTANDARD_MAGIC_BYTES; + const reads = zstandardFrameReads(kind, offset); + let step = reads.next(); + while (!step.done) { + if (needsFrameWalkPause(progress)) yield; + const read = step.value; + const end = cursor + read.length; + if (end > bytes.length) return failure(read.eof, read.failureOffset); + step = reads.next(read.inspect ? bytes.subarray(cursor, end) : unreadBytes); + cursor = end; + } + if (step.value !== undefined) + return failure("malformed", step.value.offset, step.value.code); + return { offset, length: cursor - offset, kind }; +} + +/** + * Exact frame boundaries for one buffer. The walk does not ask the decoder + * how many bytes it consumed. + */ +function* walkFrames( + bytes: Uint8Array, + allowance: ZstandardFrameAllowance, + mode: ZstandardFrameMode, +): Generator { + const frames: FrameSpan[] = []; + const progress = { reads: 0 }; + let offset = 0; + while (offset < bytes.length) { + const kind = zstandardFrameKind(bytes, offset); + if (frames.length === allowance.maxFrames) { + if (kind !== undefined) return workLimit(allowance, offset); + if (mode === "exact") return failure("trailing_bytes", offset); + break; + } + if (offset + ZSTANDARD_MAGIC_BYTES > bytes.length) { + if (frames.length === 0) return failure("truncated_frame_header", offset); + if (mode === "exact") return failure("trailing_bytes", offset); + break; + } + if (kind === undefined) { + if (frames.length === 0) return failure("malformed", offset, "magic"); + if (mode === "exact") return failure("trailing_bytes", offset); + break; + } + const next = yield* frameAt(bytes, offset, kind, progress); + if ("ok" in next) return next; + frames.push(next); + offset += next.length; + } + if (frames.length === 0) return failure("truncated_frame_header", 0); + return { ok: true, frames, consumed: offset }; +} + +const decoderCode = (cause: unknown): string | undefined => { + if (typeof cause !== "object" || cause === null || !("code" in cause)) + return undefined; + const code = cause.code; + return typeof code === "string" ? code : undefined; +}; + +const decoderFailure = ( + cause: unknown, + offset: number, +): ZstandardFrameFailure => { + if (hostRangeErrorKind(cause) === "host-capacity") + return failure("output_limit", offset); + const code = decoderCode(cause); + const detail = cause instanceof Error ? cause.message : undefined; + if (code === "ZSTD_error_dstSize_tooSmall" || code === "ERR_BUFFER_TOO_LARGE") + return failure("output_limit", offset, code, detail); + return failure("malformed", offset, code, detail); +}; + +const isFrameFailure = ( + value: { readonly bytes: Buffer } | ZstandardFrameFailure, +): value is ZstandardFrameFailure => "reason" in value; + +interface DecompressOptions { + readonly maxOutputLength: number; + readonly windowLogMax: number; + readonly dictionary?: Uint8Array; + readonly rejectTrailing: boolean; +} + +/** + * Node 22 stops after the first frame and ignores `rejectGarbageAfterEnd`. + * Node 24.19+ and Node 26 decode a whole sequence and reject a suffix with + * that option. Probe once; do not trust a major-version check alone. + */ +let strictSequence: boolean | undefined; +const nodeDecodesStrictSequence = (): boolean => { + if (strictSequence !== undefined) return strictSequence; + const frame = zstdCompressSync(Buffer.from([0x61])); + const pair = Buffer.concat([frame, frame]); + let concatenated = false; + try { + concatenated = zstdDecompressSync(pair).length === 2; + } catch { + concatenated = false; + } + let strict = false; + if (concatenated) { + try { + zstdDecompressSync(Buffer.concat([frame, Buffer.of(0)]), { + rejectGarbageAfterEnd: true, + }); + } catch { + strict = true; + } + } + strictSequence = strict; + return strictSequence; +}; + +const unzstd = promisify(zstdDecompress); + +const decompress = ( + input: Uint8Array, + offset: number, + options: DecompressOptions, +): { readonly bytes: Buffer } | ZstandardFrameFailure => { + try { + const result = nodeCodecResult( + zstdDecompressSync(input, { + info: true, + maxOutputLength: Math.max(1, options.maxOutputLength), + ...(options.rejectTrailing ? { rejectGarbageAfterEnd: true } : {}), + ...(options.dictionary === undefined + ? {} + : { dictionary: options.dictionary }), + params: { [constants.ZSTD_d_windowLogMax]: options.windowLogMax }, + }), + ); + // The walk already fixed the input extent. A short read means the + // decoder did not accept that extent, so the bytes are not returned. + if (result.consumedBytes !== input.length) + return failure("malformed", offset, "unconsumed_input"); + return { bytes: result.buffer }; + } catch (cause: unknown) { + return decoderFailure(cause, offset); + } +}; + +/** Async strict decode so a large sequence does not block the event loop. */ +const decompressSequence = async ( + input: Uint8Array, + offset: number, + options: DecompressOptions, +): Promise<{ readonly bytes: Buffer } | ZstandardFrameFailure> => { + try { + const result = nodeCodecResult( + await unzstd(input, { + info: true, + rejectGarbageAfterEnd: true, + maxOutputLength: Math.max(1, options.maxOutputLength), + ...(options.dictionary === undefined + ? {} + : { dictionary: options.dictionary }), + params: { [constants.ZSTD_d_windowLogMax]: options.windowLogMax }, + }), + ); + if (result.consumedBytes !== input.length) + return failure("malformed", offset, "unconsumed_input"); + return { bytes: ownedZstandardBytes(result.buffer) }; + } catch (cause: unknown) { + return decoderFailure(cause, offset); + } +}; + +/** + * Node's synchronous decoder returns a view onto a larger chunk allocation. + * Retaining that view keeps the whole chunk. One byte from a 16 KiB chunk, + * repeated for every frame, retains mebibytes for a few kibibytes of output. + * Copy only when the backing store is larger than the bytes in use. + */ +export const ownedZstandardBytes = (bytes: Buffer): Buffer => { + if (bytes.byteOffset === 0 && bytes.buffer.byteLength === bytes.byteLength) + return bytes; + const owned = Buffer.allocUnsafe(bytes.byteLength); + bytes.copy(owned); + return owned; +}; + +const withinOutput = ( + bytes: Buffer, + remaining: number, + offset: number, +): { readonly bytes: Buffer } | ZstandardFrameFailure => + bytes.length > remaining ? failure("output_limit", offset) : { bytes }; + +const decodeDataFrame = ( + bytes: Uint8Array, + frame: FrameSpan, + total: number, + options: DecompressOptions, +): { readonly bytes: Buffer } | ZstandardFrameFailure => { + const remaining = options.maxOutputLength - total; + const decoded = decompress( + bytes.subarray(frame.offset, frame.offset + frame.length), + frame.offset, + { ...options, maxOutputLength: Math.max(1, remaining) }, + ); + if (isFrameFailure(decoded)) return decoded; + const bounded = withinOutput(decoded.bytes, remaining, frame.offset); + if (isFrameFailure(bounded)) return bounded; + return { bytes: ownedZstandardBytes(bounded.bytes) }; +}; + +const finishFrames = ( + chunks: readonly Buffer[], + consumed: number, + frames: number, +): ZstandardFrameSuccess => { + const only = chunks.length === 1 ? chunks[0] : undefined; + return { + ok: true, + bytes: + only !== undefined + ? only + : chunks.length === 0 + ? Buffer.alloc(0) + : Buffer.concat(chunks), + consumedBytes: consumed, + frames, + }; +}; + +const decodeFrames = ( + bytes: Uint8Array, + frames: readonly FrameSpan[], + consumed: number, + options: DecompressOptions, +): ZstandardFrameSuccess | ZstandardFrameFailure => { + if (options.rejectTrailing) { + const decoded = decompress(bytes.subarray(0, consumed), 0, options); + if (isFrameFailure(decoded)) return decoded; + const bounded = withinOutput(decoded.bytes, options.maxOutputLength, 0); + if (isFrameFailure(bounded)) return bounded; + return { + ...finishFrames( + [ownedZstandardBytes(bounded.bytes)], + consumed, + frames.length, + ), + }; + } + const chunks: Buffer[] = []; + let total = 0; + for (const frame of frames) { + if (frame.kind === "skippable") continue; + const step = decodeDataFrame(bytes, frame, total, options); + if (isFrameFailure(step)) return step; + total += step.bytes.length; + chunks.push(step.bytes); + } + return finishFrames(chunks, consumed, frames.length); +}; + +/** + * Decode one Zstandard buffer. Frame boundaries come from the structural + * walk. Where Node's public strict decoder accepts a whole sequence, one + * call decodes that sequence and checks its checksums. Otherwise each + * walked frame is decoded on its own, which is what Node 22 requires. + */ +interface PreparedDecode { + readonly frames: readonly FrameSpan[]; + readonly consumed: number; + readonly options: DecompressOptions; +} + +function* prepareDecode( + bytes: Uint8Array, + options: ZstandardDecodeOptions, +): Generator { + const resolved = + options.allowance === undefined + ? currentZstandardFrameAllowance() + : { ok: true as const, value: options.allowance }; + if (!resolved.ok) + return { + ok: false, + reason: "invalid_allowance", + offset: 0, + configuration: resolved.error, + }; + const mode = options.mode ?? "exact"; + const walked = yield* walkFrames(bytes, resolved.value, mode); + if ("reason" in walked) return walked; + return { + frames: walked.frames, + consumed: walked.consumed, + options: { + maxOutputLength: options.maxOutputLength ?? bufferConstants.MAX_LENGTH, + windowLogMax: options.windowLogMax ?? ZSTANDARD_WINDOW_LOG_MAX, + ...(options.dictionary === undefined + ? {} + : { dictionary: options.dictionary }), + rejectTrailing: nodeDecodesStrictSequence(), + }, + }; +} + +/** + * Decode one Zstandard buffer. Frame boundaries come from the structural + * walk. Where Node's public strict decoder accepts a whole sequence, one + * call decodes that sequence and checks its checksums. Otherwise each + * walked frame is decoded on its own, which is what Node 22 requires. + */ +export const decodeZstandardFrames = ( + bytes: Uint8Array, + options: ZstandardDecodeOptions = {}, +): ZstandardFrameResult => { + const steps = prepareDecode(bytes, options); + let next = steps.next(); + while (!next.done) next = steps.next(); + const prepared = next.value; + if ("reason" in prepared) return prepared; + return decodeFrames( + bytes, + prepared.frames, + prepared.consumed, + prepared.options, + ); +}; + +/** Yield point the caller uses to observe its existing abort signal. */ +export type ZstandardFramePause = () => Promise; + +/** + * Pause during structural validation, before decoding, and before returning + * successful output. The strict sequence uses the async decoder. The fallback + * pauses before each synchronous data-frame decode. + */ +export const decodeZstandardFramesCooperative = async ( + bytes: Uint8Array, + options: ZstandardDecodeOptions, + pause: ZstandardFramePause, +): Promise => { + const steps = prepareDecode(bytes, options); + let next = steps.next(); + while (!next.done) { + await pause(); + next = steps.next(); + } + const prepared = next.value; + if ("reason" in prepared) return prepared; + if (prepared.options.rejectTrailing) { + await pause(); + const decoded = await decompressSequence( + bytes.subarray(0, prepared.consumed), + 0, + prepared.options, + ); + if (isFrameFailure(decoded)) return decoded; + const bounded = withinOutput( + decoded.bytes, + prepared.options.maxOutputLength, + 0, + ); + if (isFrameFailure(bounded)) return bounded; + await pause(); + return finishFrames( + [bounded.bytes], + prepared.consumed, + prepared.frames.length, + ); + } + const chunks: Buffer[] = []; + let total = 0; + for (const frame of prepared.frames) { + if (frame.kind === "skippable") continue; + await pause(); + const step = decodeDataFrame(bytes, frame, total, prepared.options); + if (isFrameFailure(step)) return step; + total += step.bytes.length; + chunks.push(step.bytes); + } + await pause(); + return finishFrames(chunks, prepared.consumed, prepared.frames.length); +}; diff --git a/src/domain/compression/zstandardFrameAllowance.ts b/src/domain/compression/zstandardFrameAllowance.ts new file mode 100644 index 0000000000..3294db2a92 --- /dev/null +++ b/src/domain/compression/zstandardFrameAllowance.ts @@ -0,0 +1,176 @@ +import { z } from "zod"; +import { + DEFAULT_ZSTD_MAX_FRAMES, + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../analysisResourceLimits.js"; +import { analysisSettingLimit } from "../analysisSettingDefinitions.js"; +import { AnalysisResourceConstraintError } from "../analysisErrorCore.js"; +import { currentAnalysisSettingOverride } from "../analysisOperationSnapshot.js"; +import { ConfigurationError } from "../configurationErrors.js"; +import { parseLimitConstraint } from "../parseFailureAnalysisError.js"; +import { ParseFailureError } from "../parseFailure.js"; +import type { Result } from "../result.js"; +import { err, ok } from "../result.js"; + +/** Launch seed for the shared Zstandard frame allowance. */ +export const ZSTD_MAX_FRAMES_ENVIRONMENT = "REA_ZSTD_MAX_FRAMES"; + +/** Registered analysis setting that owns this allowance. */ +export const ZSTD_MAX_FRAMES_SETTING = "zstd.max_frames" as const; + +export type ZstandardFrameSource = "default" | "setting" | "runtime"; + +/** The frame allowance in force for one decode, and where it came from. */ +export interface ZstandardFrameAllowance { + readonly maxFrames: number; + readonly source: ZstandardFrameSource; +} + +const constraint = `${ZSTD_MAX_FRAMES_ENVIRONMENT} must be a decimal integer from ${String(MINIMUM_ZSTD_MAX_FRAMES)} to ${String(MAXIMUM_ZSTD_MAX_FRAMES)} frames; the default is ${String(DEFAULT_ZSTD_MAX_FRAMES)}.`; + +const decimalInteger = /^[1-9]\d*$/u; + +/** Environment seed for `zstd.max_frames`, shared with launch admission. */ +export const zstandardMaxFramesSchema = z + .string() + .regex(decimalInteger, { error: constraint }) + .refine( + (value) => { + if (!decimalInteger.test(value)) return true; + const parsed = BigInt(value); + return ( + parsed >= BigInt(MINIMUM_ZSTD_MAX_FRAMES) && + parsed <= BigInt(MAXIMUM_ZSTD_MAX_FRAMES) + ); + }, + { error: constraint }, + ) + .transform(Number); + +const invalidSeed = (): ConfigurationError => + new ConfigurationError(constraint, { + remediationAction: `On a running MCP session, call update_analysis_settings for ${ZSTD_MAX_FRAMES_SETTING}, then retry. For CLI or future REA processes, correct ${ZSTD_MAX_FRAMES_ENVIRONMENT} in the launch environment.`, + settings: [ + { + setting: ZSTD_MAX_FRAMES_ENVIRONMENT, + constraint, + update_mode: "runtime", + pending_restart: false, + }, + ], + }); + +/** + * Resolve the allowance for one decode. A runtime override wins, then the + * launch seed, then the default. An invalid seed is refused; it is not + * replaced by the default. + */ +export const resolveZstandardFrameAllowance = ( + environment: Readonly>, + override: number | undefined = currentAnalysisSettingOverride( + ZSTD_MAX_FRAMES_SETTING, + ), +): Result => { + if (override !== undefined) + return ok({ maxFrames: override, source: "runtime" }); + const raw = environment[ZSTD_MAX_FRAMES_ENVIRONMENT]; + if (raw === undefined) + return ok({ maxFrames: DEFAULT_ZSTD_MAX_FRAMES, source: "default" }); + const parsed = zstandardMaxFramesSchema.safeParse(raw); + return parsed.success + ? ok({ maxFrames: parsed.data, source: "setting" }) + : err(invalidSeed()); +}; + +/** Allowance for the operation on this stack, or the process launch environment. */ +export const currentZstandardFrameAllowance = (): Result< + ZstandardFrameAllowance, + ConfigurationError +> => resolveZstandardFrameAllowance(process.env); + +/** Facts a work refusal carries so callers do not restate the registry. */ +export const zstandardWorkLimitFacts = ( + allowance: ZstandardFrameAllowance, +): { + readonly reason: "work_limit"; + readonly setting: typeof ZSTD_MAX_FRAMES_SETTING; + readonly value: number; + readonly source: ZstandardFrameSource; + readonly minimum: number; + readonly maximum: number; +} => ({ + reason: "work_limit", + setting: ZSTD_MAX_FRAMES_SETTING, + value: allowance.maxFrames, + source: allowance.source, + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, +}); + +/** + * Decoder-work refusal. Distinct from malformed framing: the bytes were not + * shown to be invalid, and the named setting can admit a larger frame count. + */ +export class ZstandardFrameAllowanceError extends ParseFailureError<"limit"> { + readonly reason = "work_limit" as const; + readonly setting = ZSTD_MAX_FRAMES_SETTING; + readonly value: number; + readonly source: ZstandardFrameSource; + readonly minimum: number; + readonly maximum: number; + readonly offset: number; + + constructor(allowance: ZstandardFrameAllowance, offset: number) { + const facts = zstandardWorkLimitFacts(allowance); + const detail = `${facts.setting} is ${String(facts.value)} (source ${facts.source}, range ${String(facts.minimum)}..${String(facts.maximum)})`; + super({ + kind: "limit", + format: "zstd", + detail, + offset, + field: facts.reason, + limit: { + name: facts.setting, + maximum: facts.value, + required: facts.value + 1, + }, + }); + this.name = "ZstandardFrameAllowanceError"; + this.value = facts.value; + this.source = facts.source; + this.minimum = facts.minimum; + this.maximum = facts.maximum; + this.offset = offset; + } +} + +/** Read the frame refusal carried by a raw parser or its public projection. */ +export const zstandardFrameAllowanceCause = (cause: unknown) => + cause instanceof ZstandardFrameAllowanceError + ? cause + : cause instanceof AnalysisResourceConstraintError && + cause.cause instanceof ZstandardFrameAllowanceError + ? cause.cause + : undefined; + +/** + * The public resource refusal for one frame-cap. Callers keep the setting, + * effective value, source, and range instead of rewriting the error as I/O + * or malformed content. + */ +export const projectZstandardFrameAllowance = ( + operation: string, + error: ZstandardFrameAllowanceError, +) => + parseLimitConstraint(operation, "cpu", error, { + retryable: false, + limit: analysisSettingLimit(error.setting, error.value, error.source), + reportedLimitContext: { + setting: error.setting, + value: error.value, + source: error.source, + minimum: error.minimum, + maximum: error.maximum, + }, + }); diff --git a/src/domain/compression/zstandardFrameChunks.ts b/src/domain/compression/zstandardFrameChunks.ts new file mode 100644 index 0000000000..c10933fe2f --- /dev/null +++ b/src/domain/compression/zstandardFrameChunks.ts @@ -0,0 +1,89 @@ +import { + ZSTANDARD_MAGIC_BYTES, + zstandardFrameKind, + zstandardFrameReads, + type ZstandardFrameRead, +} from "./zstandardFrameWire.js"; + +export class ZstandardFramingError extends Error { + constructor( + readonly code: string, + readonly offset: number, + ) { + super(`Zstandard framing failed at byte ${offset}: ${code}`); + } +} + +const magicRead = (offset: number): ZstandardFrameRead => ({ + length: ZSTANDARD_MAGIC_BYTES, + inspect: true, + eof: "truncated_frame_header", + failureOffset: offset, +}); + +/** + * Yield compressed slices whose boundaries never cross two frames. Node 22 + * stops a decode call at its first frame end, but its stream can accept later + * frames in separate writes. Chunk boundaries may also split a frame. + */ +export async function* zstandardFrameChunks( + input: AsyncIterable, +): AsyncGenerator { + // Only inspected fields that span chunks are copied. Magic and skippable + // size are the widest inspected fields (four bytes). + const header = new Uint8Array(ZSTANDARD_MAGIC_BYTES); + const unread = new Uint8Array(0); + let offset = 0; + let frameOffset = 0; + let read = magicRead(frameOffset); + let supplied = 0; + let reads: ReturnType | undefined; + for await (const bytes of input) { + let start = 0; + let cursor = 0; + while (cursor < bytes.length) { + const count = Math.min(read.length - supplied, bytes.length - cursor); + const end = cursor + count; + const complete = supplied + count === read.length; + let field: Uint8Array = unread; + if (read.inspect) { + if (supplied === 0 && complete) { + field = bytes.subarray(cursor, end); + } else { + header.set(bytes.subarray(cursor, end), supplied); + field = header; + } + } + cursor = end; + offset += count; + if (!complete) { + supplied += count; + break; + } + supplied = 0; + if (reads === undefined) { + const kind = zstandardFrameKind(field, 0); + if (kind === undefined) + throw new ZstandardFramingError("magic", frameOffset); + reads = zstandardFrameReads(kind, frameOffset); + } + let step = reads.next(field); + // A reserved/empty length never needs another input chunk. + while (!step.done && step.value.length === 0) step = reads.next(unread); + if (!step.done) { + read = step.value; + continue; + } + if (step.value !== undefined) + throw new ZstandardFramingError(step.value.code, step.value.offset); + yield bytes.subarray(start, cursor); + start = cursor; + frameOffset = offset; + reads = undefined; + read = magicRead(frameOffset); + } + if (start < bytes.length) yield bytes.subarray(start); + } + if (reads !== undefined || supplied !== 0 || frameOffset === 0) + throw new ZstandardFramingError(read.eof, read.failureOffset); +} diff --git a/src/domain/compression/zstandardFrameWire.ts b/src/domain/compression/zstandardFrameWire.ts new file mode 100644 index 0000000000..bc13387131 --- /dev/null +++ b/src/domain/compression/zstandardFrameWire.ts @@ -0,0 +1,117 @@ +const ZSTD_MAGIC = 0xfd2fb528; +const SKIPPABLE_MAGIC_MIN = 0x184d2a50; +const SKIPPABLE_MAGIC_MAX = 0x184d2a5f; +export const ZSTANDARD_MAGIC_BYTES = 4; + +export type ZstandardFrameKind = "zstd" | "skippable"; + +type Truncation = + | "truncated_frame_header" + | "truncated_block" + | "truncated_checksum"; + +export interface ZstandardFrameRead { + readonly length: number; + readonly inspect: boolean; + readonly eof: Truncation; + readonly failureOffset: number; +} + +type FrameWireFailureCode = "reserved_frame_descriptor" | "reserved_block"; + +const u24 = (bytes: Uint8Array, offset: number): number => + (bytes[offset] ?? 0) | + ((bytes[offset + 1] ?? 0) << 8) | + ((bytes[offset + 2] ?? 0) << 16); + +const u32 = (bytes: Uint8Array, offset: number): number => + (u24(bytes, offset) | ((bytes[offset + 3] ?? 0) << 24)) >>> 0; + +export const zstandardFrameKind = ( + bytes: Uint8Array, + offset: number, +): ZstandardFrameKind | undefined => { + if (bytes.length - offset < ZSTANDARD_MAGIC_BYTES) return undefined; + const magic = u32(bytes, offset); + if (magic === ZSTD_MAGIC) return "zstd"; + if (magic >= SKIPPABLE_MAGIC_MIN && magic <= SKIPPABLE_MAGIC_MAX) + return "skippable"; + return undefined; +}; + +/** + * RFC 8878 §3.1 frame extent after its magic. The caller supplies inspected + * fields and skips opaque bytes, which remain the native decoder's concern. + */ +export function* zstandardFrameReads( + kind: ZstandardFrameKind, + offset: number, +): Generator< + ZstandardFrameRead, + { readonly code: FrameWireFailureCode; readonly offset: number } | undefined, + Uint8Array +> { + let cursor = offset + ZSTANDARD_MAGIC_BYTES; + const header = (length: number, inspect: boolean): ZstandardFrameRead => ({ + length, + inspect, + eof: "truncated_frame_header", + failureOffset: offset, + }); + if (kind === "skippable") { + const userSize = yield header(4, true); + yield { + length: u32(userSize, 0), + inspect: false, + eof: "truncated_block", + failureOffset: cursor + 4, + }; + return undefined; + } + + const descriptor = (yield header(1, true))[0] ?? 0; + if (descriptor & 0x08) return { code: "reserved_frame_descriptor", offset }; + const singleSegment = (descriptor & 0x20) !== 0; + const dictionaryFlag = descriptor & 0x03; + const dictionaryBytes = dictionaryFlag === 3 ? 4 : dictionaryFlag; + const contentSizeFlag = descriptor >>> 6; + const contentSizeBytes = + contentSizeFlag === 0 ? Number(singleSegment) : 2 ** contentSizeFlag; + const headerRest = + Number(!singleSegment) + dictionaryBytes + contentSizeBytes; + yield header(headerRest, false); + cursor += 1 + headerRest; + + for (;;) { + const block = u24( + yield { + length: 3, + inspect: true, + eof: "truncated_block", + failureOffset: cursor, + }, + 0, + ); + const type = (block >>> 1) & 3; + if (type === 3) return { code: "reserved_block", offset: cursor }; + cursor += 3; + // An RLE block stores one byte and repeats it the declared number of times. + const length = type === 1 ? 1 : block >>> 3; + yield { + length, + inspect: false, + eof: "truncated_block", + failureOffset: cursor, + }; + cursor += length; + if (block & 1) break; + } + if (descriptor & 0x04) + yield { + length: 4, + inspect: false, + eof: "truncated_checksum", + failureOffset: cursor, + }; + return undefined; +} diff --git a/src/domain/firmware/androidOtaPayload.test.ts b/src/domain/firmware/androidOtaPayload.test.ts index 3712c8f8ce..f4db766d7c 100644 --- a/src/domain/firmware/androidOtaPayload.test.ts +++ b/src/domain/firmware/androidOtaPayload.test.ts @@ -1,5 +1,12 @@ import { createHash } from "node:crypto"; import { zstdCompressSync } from "node:zlib"; +import { runWithAnalysisOperationSnapshot } from "../analysisOperationSnapshot.js"; +import { ConfigurationError } from "../configurationErrors.js"; +import { ZSTD_MAX_FRAMES_ENVIRONMENT } from "../compression/zstandardFrameAllowance.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../analysisResourceLimits.js"; import { BinaryWriter, WireType } from "@bufbuild/protobuf/wire"; import { expect, it } from "vitest"; @@ -240,6 +247,140 @@ it("reconstructs REPLACE_ZSTD payloads", () => { ).toBe(true); }); +const zstdByteFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +it("rejects unread REPLACE_ZSTD bytes and accepts a second frame", () => { + const plain = Buffer.from("zstd partition"); + const trailing = buildAndroidOtaPayload({ + partitions: [ + { + name: "vendor", + operations: [ + { + type: 14, + data: Buffer.concat([zstdCompressSync(plain), Buffer.of(0)]), + }, + ], + }, + ], + }); + const trailingParsed = parse(trailing); + if (!trailingParsed.ok) throw trailingParsed.error; + const trailingBlob = trailing.subarray( + trailingParsed.value.header.blobOffset, + ); + expect( + reconstructAndroidOtaPartition( + trailingParsed.value.partitions[0]!, + trailingParsed.value.blockSize, + (offset, length) => trailingBlob.subarray(offset, offset + length), + ), + ).toMatchObject({ + ok: false, + error: { + kind: "malformed", + failure: { field: "trailing_bytes" }, + }, + }); + + const frames = Buffer.concat([zstdByteFrame(0x61), zstdByteFrame(0x62)]); + const concatenated = buildAndroidOtaPayload({ + partitions: [ + { + name: "vendor", + newSize: 2, + omitNewHash: true, + operations: [ + { + type: 14, + data: frames, + dstExtents: [{ startBlock: 0, numBlocks: 1 }], + }, + ], + }, + ], + }); + const parsed = parse(concatenated); + if (!parsed.ok) throw parsed.error; + const blob = concatenated.subarray(parsed.value.header.blobOffset); + const read = (offset: number, length: number) => + blob.subarray(offset, offset + length); + const reconstructed = reconstructAndroidOtaPartition( + parsed.value.partitions[0]!, + parsed.value.blockSize, + read, + { verifyDeclaredHash: false }, + ); + if (!reconstructed.ok) throw reconstructed.error; + expect([...reconstructed.value.subarray(0, 2)]).toEqual([0x61, 0x62]); + expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => + reconstructAndroidOtaPartition( + parsed.value.partitions[0]!, + parsed.value.blockSize, + read, + { verifyDeclaredHash: false }, + ), + ), + ).toMatchObject({ + ok: false, + error: { + _tag: "ZstandardFrameWorkRefusal", + reason: "work_limit", + setting: "zstd.max_frames", + value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + }, + }); +}); + +it("keeps an invalid REPLACE_ZSTD allowance out of the malformed path", () => { + const plain = Buffer.from("zstd-partition-bytes"); + const bytes = buildAndroidOtaPayload({ + partitions: [ + { + name: "vendor", + operations: [ + { + type: 14, + data: zstdCompressSync(plain), + dstExtents: [{ startBlock: 0, numBlocks: 1 }], + }, + ], + }, + ], + }); + const parsed = parse(bytes); + if (!parsed.ok) throw parsed.error; + const blob = bytes.subarray(parsed.value.header.blobOffset); + const reconstruct = () => + reconstructAndroidOtaPartition( + parsed.value.partitions[0]!, + parsed.value.blockSize, + (offset, length) => blob.subarray(offset, offset + length), + ); + const previous = process.env[ZSTD_MAX_FRAMES_ENVIRONMENT]; + process.env[ZSTD_MAX_FRAMES_ENVIRONMENT] = "0"; + try { + const refused = reconstruct(); + expect(refused.ok).toBe(false); + if (!refused.ok) expect(refused.error).toBeInstanceOf(ConfigurationError); + const recovered = runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 8 } }, + reconstruct, + ); + expect(recovered.ok).toBe(true); + } finally { + if (previous === undefined) delete process.env[ZSTD_MAX_FRAMES_ENVIRONMENT]; + else process.env[ZSTD_MAX_FRAMES_ENVIRONMENT] = previous; + } +}); + it("refuses REPLACE_ZSTD output past destination extents as a capacity limit", () => { const plain = Buffer.alloc(8192, 0x61); const bytes = buildAndroidOtaPayload({ diff --git a/src/domain/firmware/androidOtaPayload.ts b/src/domain/firmware/androidOtaPayload.ts index 6bcb44a97d..bb7d5118c3 100644 --- a/src/domain/firmware/androidOtaPayload.ts +++ b/src/domain/firmware/androidOtaPayload.ts @@ -1,11 +1,21 @@ -import { zstdDecompressSync } from "node:zlib"; +import { + decodeZstandardFrames, + decodeZstandardFramesCooperative, + type ZstandardFrameFailure, + type ZstandardFramePause, + type ZstandardFrameResult, +} from "../compression/zstandardExactFrame.js"; +import { + zstandardWorkLimitFacts, + type ZstandardFrameSource, +} from "../compression/zstandardFrameAllowance.js"; +import { ConfigurationError } from "../configurationErrors.js"; import { BinaryReader, WireType } from "@bufbuild/protobuf/wire"; import { sha256Hex } from "../../kernel/sha256Hex.js"; import { decodeLimit, - hostRangeErrorKind, malformedInput, ParseFailureError, truncatedInput, @@ -122,6 +132,7 @@ export type AndroidOtaPayloadParseError = export type AndroidOtaPayloadReconstructError = | AndroidOtaPayloadParseError + | ConfigurationError | { readonly _tag: "AndroidOtaPayloadUnsupportedOperation"; readonly message: string; @@ -136,6 +147,17 @@ export type AndroidOtaPayloadReconstructError = | { readonly _tag: "AndroidOtaPayloadBlobTruncated"; readonly message: string; + } + | { + readonly _tag: "ZstandardFrameWorkRefusal"; + readonly reason: "work_limit"; + readonly setting: "zstd.max_frames"; + readonly value: number; + readonly source: ZstandardFrameSource; + readonly minimum: number; + readonly maximum: number; + readonly offset: number; + readonly message: string; }; const ANDROID_OTA_FORMAT = "android-ota-payload"; @@ -166,6 +188,21 @@ const capacity = ( { name: `Android OTA payload ${name}`, maximum }, ); +const zstdWorkRefusal = ( + decoded: ZstandardFrameFailure, +): AndroidOtaPayloadReconstructError => { + const facts = zstandardWorkLimitFacts({ + maxFrames: decoded.value ?? 0, + source: decoded.source ?? "default", + }); + return { + _tag: "ZstandardFrameWorkRefusal", + ...facts, + offset: decoded.offset, + message: `${facts.setting} is ${String(facts.value)} (source ${facts.source}, range ${String(facts.minimum)}..${String(facts.maximum)})`, + }; +}; + const androidOtaUnsupportedVersion = (version: bigint): string => `Android OTA payload major version ${String(version)} is unsupported; this reader supports version 2 only`; @@ -1391,17 +1428,16 @@ const hasCompleteExtentCoverage = ( return false; }; -/** - * Rebuild a full-payload partition image after checking declared blob hashes. - * A partition image hash mismatch fails unless `verifyDeclaredHash` is false, - * which leaves that decision to a caller applying its own integrity policy. - */ -export const reconstructAndroidOtaPartition = ( +function* reconstructAndroidOtaPartitionSteps( partition: AndroidOtaPartition, blockSize: number, readBlob: (offset: number, length: number) => Uint8Array, options: { readonly verifyDeclaredHash?: boolean } = {}, -): Result => { +): Generator< + { readonly raw: Uint8Array; readonly maxOutputLength: number }, + Result, + ZstandardFrameResult +> { const refused = androidOtaExtractRefusalError(partition); if (refused !== null) return err(refused); const size = partition.newPartitionInfo.size!; @@ -1455,17 +1491,28 @@ export const reconstructAndroidOtaPartition = ( 1, Math.min(dstBytes.value, ANDROID_OTA_MAX_EXTRACT_BYTES), ); - try { - payload = zstdDecompressSync(raw, { maxOutputLength }); - } catch (cause: unknown) { + const decoded = yield { raw, maxOutputLength }; + if (!decoded.ok) { + if ( + decoded.reason === "invalid_allowance" && + decoded.configuration !== undefined + ) + return err(decoded.configuration); + if (decoded.reason === "work_limit") + return err(zstdWorkRefusal(decoded)); + if (decoded.reason === "output_limit") + return err( + capacity("REPLACE_ZSTD decoded output", maxOutputLength, "byte"), + ); return err( - hostRangeErrorKind(cause) === "host-capacity" - ? capacity("REPLACE_ZSTD decoded output", maxOutputLength, "byte") - : malformed( - "REPLACE_ZSTD operation payload is not a valid Zstandard frame", - ), + malformedInput( + ANDROID_OTA_FORMAT, + "REPLACE_ZSTD operation payload is not a valid Zstandard frame", + { offset: decoded.offset, field: decoded.reason }, + ), ); } + payload = decoded.bytes; } const written = writeExtents( image, @@ -1502,4 +1549,77 @@ export const reconstructAndroidOtaPartition = ( observedHashHex: observed, }); return ok(image); +} + +const driveAndroidOtaReconstruction = ( + steps: Generator< + { readonly raw: Uint8Array; readonly maxOutputLength: number }, + Result, + ZstandardFrameResult + >, + decode: ( + raw: Uint8Array, + maxOutputLength: number, + ) => ZstandardFrameResult | Promise, +): Promise> => { + const run = async (): Promise< + Result + > => { + let next = steps.next(); + while (!next.done) + next = steps.next( + await decode(next.value.raw, next.value.maxOutputLength), + ); + return next.value; + }; + return run(); }; + +/** + * Rebuild a full-payload partition image after checking declared blob hashes. + * A partition image hash mismatch fails unless `verifyDeclaredHash` is false, + * which leaves that decision to a caller applying its own integrity policy. + */ +export const reconstructAndroidOtaPartition = ( + partition: AndroidOtaPartition, + blockSize: number, + readBlob: (offset: number, length: number) => Uint8Array, + options: { readonly verifyDeclaredHash?: boolean } = {}, +): Result => { + const steps = reconstructAndroidOtaPartitionSteps( + partition, + blockSize, + readBlob, + options, + ); + let next = steps.next(); + while (!next.done) + next = steps.next( + decodeZstandardFrames(next.value.raw, { + maxOutputLength: next.value.maxOutputLength, + }), + ); + return next.value; +}; + +/** + * Same reconstruction as {@link reconstructAndroidOtaPartition}. REPLACE_ZSTD + * frames yield through `pause` so the caller's abort signal can land. + */ +export const reconstructAndroidOtaPartitionCooperative = ( + partition: AndroidOtaPartition, + blockSize: number, + readBlob: (offset: number, length: number) => Uint8Array, + pause: ZstandardFramePause, + options: { readonly verifyDeclaredHash?: boolean } = {}, +): Promise> => + driveAndroidOtaReconstruction( + reconstructAndroidOtaPartitionSteps( + partition, + blockSize, + readBlob, + options, + ), + (raw, maxOutputLength) => + decodeZstandardFramesCooperative(raw, { maxOutputLength }, pause), + ); diff --git a/src/domain/native/elfCompressedSection.test.ts b/src/domain/native/elfCompressedSection.test.ts index b8924411e3..78006e5e3d 100644 --- a/src/domain/native/elfCompressedSection.test.ts +++ b/src/domain/native/elfCompressedSection.test.ts @@ -1,5 +1,12 @@ import { deflateSync, zstdCompressSync } from "node:zlib"; +import { ConfigurationError } from "../configurationErrors.js"; +import { ZSTD_MAX_FRAMES_ENVIRONMENT } from "../compression/zstandardFrameAllowance.js"; import { describe, expect, it } from "vitest"; +import { runWithAnalysisOperationSnapshot } from "../analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../analysisResourceLimits.js"; import { decodeElfCompressedSection, decodeElfGnuZlibSection, @@ -190,6 +197,93 @@ describe("ELF compressed section payloads", () => { }); }); +const zstdFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +it("accepts a Zstandard skippable suffix and rejects a trailing byte", () => { + const payload = zstdCompressSync(plain); + const skip = Buffer.alloc(8); + skip.writeUInt32LE(0x184d2a50, 0); + const decode = (body: Buffer) => + decodeElfCompressedSection( + Buffer.concat([chdr(64, true, 2, plain.length), body]), + 0x800n, + 64, + true, + 1024, + "ELF sample section", + ); + expect(decode(payload)).toMatchObject({ state: "available" }); + expect(decode(Buffer.concat([payload, skip]))).toMatchObject({ + state: "available", + compression: "zstd", + }); + expect(decode(Buffer.concat([payload, Buffer.of(0)]))).toMatchObject({ + state: "malformed", + reason: "trailing_bytes", + }); +}); + +it("keeps an invalid frame allowance out of the malformed path", () => { + const payload = zstdCompressSync(plain); + const bytes = Buffer.concat([chdr(64, true, 2, plain.length), payload]); + const decode = () => + decodeElfCompressedSection( + bytes, + 0x800n, + 64, + true, + 1024, + "ELF sample section", + ); + const previous = process.env[ZSTD_MAX_FRAMES_ENVIRONMENT]; + process.env[ZSTD_MAX_FRAMES_ENVIRONMENT] = "0"; + try { + expect(decode()).toMatchObject({ + state: "configuration", + error: expect.any(ConfigurationError), + }); + expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 8 } }, + decode, + ), + ).toMatchObject({ state: "available" }); + } finally { + if (previous === undefined) delete process.env[ZSTD_MAX_FRAMES_ENVIRONMENT]; + else process.env[ZSTD_MAX_FRAMES_ENVIRONMENT] = previous; + } +}); + +it("refuses a Zstandard section that exceeds zstd.max_frames", () => { + const body = Buffer.concat([zstdFrame(0x61), zstdFrame(0x62)]); + const bytes = Buffer.concat([chdr(64, true, 2, 2), body]); + const decode = () => + decodeElfCompressedSection( + bytes, + 0x800n, + 64, + true, + 1024, + "ELF sample section", + ); + expect(decode()).toMatchObject({ state: "available" }); + expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + decode, + ), + ).toMatchObject({ + state: "work_limit", + reason: "work_limit", + setting: "zstd.max_frames", + value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + }); +}); + describe("ELF compressed section declaration admission", () => { it.each([ [32, false, 0x8000_0000n], diff --git a/src/domain/native/elfCompressedSection.ts b/src/domain/native/elfCompressedSection.ts index 0966903e3f..7dac37cf8a 100644 --- a/src/domain/native/elfCompressedSection.ts +++ b/src/domain/native/elfCompressedSection.ts @@ -1,6 +1,18 @@ import { BinaryReader } from "../binaryContainer.js"; -import { constants, inflateSync, zstdDecompressSync } from "node:zlib"; +import { inflateSync } from "node:zlib"; import { nodeCodecResult } from "../../kernel/nodeCodecResult.js"; +import { + decodeZstandardFrames, + decodeZstandardFramesCooperative, + ZSTANDARD_WINDOW_LOG_MAX, + type ZstandardFrameFailure, + type ZstandardFramePause, +} from "../compression/zstandardExactFrame.js"; +import { + zstandardWorkLimitFacts, + type ZstandardFrameSource, +} from "../compression/zstandardFrameAllowance.js"; +import type { ConfigurationError } from "../configurationErrors.js"; const SHF_COMPRESSED = 0x800n; const ELFCOMPRESS_ZLIB = 1; @@ -21,6 +33,27 @@ export type ElfCompressedSectionResult = readonly message: string; /** Larger of declared output and compressed input read; 0 when no attempt started. */ readonly inflatedBytes: number; + /** Present when the shared Zstandard framer classified the payload. */ + readonly reason?: ZstandardFrameFailure["reason"]; + readonly offset?: number; + readonly code?: string; + } + | { + readonly state: "configuration"; + readonly error: ConfigurationError; + readonly inflatedBytes: number; + } + | { + readonly state: "work_limit"; + readonly reason: "work_limit"; + readonly setting: "zstd.max_frames"; + readonly value: number; + readonly source: ZstandardFrameSource; + readonly minimum: number; + readonly maximum: number; + readonly offset: number; + /** Larger of declared output and compressed input read. */ + readonly inflatedBytes: number; } | { readonly state: "limit"; @@ -70,18 +103,23 @@ const readElfCompressedSectionHeader = ( return { state: "compressed", compressionType, declaredSize, headerBytes }; }; -/** - * Decode one ELF section payload. `subject` is the noun phrase used in - * failure text so existing section-name messages stay stable. - */ -export const decodeElfCompressedSection = ( +type PreparedElfSection = + | ElfCompressedSectionResult + | { + readonly state: "payload"; + readonly payload: Buffer; + readonly declaredSize: bigint; + readonly compression: "zlib" | "zstd"; + }; + +const prepareElfCompressedSection = ( bytes: Buffer, flags: bigint, bits: 32 | 64, little: boolean, maximumDecodedBytes: number, subject: string, -): ElfCompressedSectionResult => { +): PreparedElfSection => { const header = readElfCompressedSectionHeader( bytes, flags, @@ -106,12 +144,109 @@ export const decodeElfCompressedSection = ( requiredBytes: declaredSize, maximumBytes: maximumDecodedBytes, }; - return inflateDeclared( - bytes.subarray(headerBytes), + return { + state: "payload", + payload: bytes.subarray(headerBytes), declaredSize, - compressionType === ELFCOMPRESS_ZSTD ? "zstd" : "zlib", + compression: compressionType === ELFCOMPRESS_ZSTD ? "zstd" : "zlib", + }; +}; + +const zstdDecodeOptions = (expected: number) => ({ + maxOutputLength: Math.max(1, expected), + // Retain Zstandard's default maximum decoder window of 128 MiB. + windowLogMax: ZSTANDARD_WINDOW_LOG_MAX, +}); + +const finishInflated = ( + decoded: Buffer, + expected: number, + compression: "zlib" | "zstd", + subject: string, + inflatedBytes: number, +): ElfCompressedSectionResult => { + if (decoded.length !== expected) + return { + state: "malformed", + message: `${subject} size is inconsistent`, + inflatedBytes, + }; + return { + state: "available", + bytes: decoded, + compressed: true, + compression, + inflatedBytes, + }; +}; + +/** + * Decode one ELF section payload. `subject` is the noun phrase used in + * failure text so existing section-name messages stay stable. + */ +export const decodeElfCompressedSection = ( + bytes: Buffer, + flags: bigint, + bits: 32 | 64, + little: boolean, + maximumDecodedBytes: number, + subject: string, +): ElfCompressedSectionResult => { + const prepared = prepareElfCompressedSection( + bytes, + flags, + bits, + little, + maximumDecodedBytes, subject, ); + if (prepared.state !== "payload") return prepared; + return inflateDeclared( + prepared.payload, + prepared.declaredSize, + prepared.compression, + subject, + ); +}; + +/** + * Same admission as {@link decodeElfCompressedSection}. Zstandard frames use + * the cooperative decoder so `pause` can observe the caller's abort signal. + */ +export const decodeElfCompressedSectionCooperative = async ( + bytes: Buffer, + flags: bigint, + bits: 32 | 64, + little: boolean, + maximumDecodedBytes: number, + subject: string, + pause: ZstandardFramePause, +): Promise => { + const prepared = prepareElfCompressedSection( + bytes, + flags, + bits, + little, + maximumDecodedBytes, + subject, + ); + if (prepared.state !== "payload") return prepared; + if (prepared.compression !== "zstd") + return inflateDeclared( + prepared.payload, + prepared.declaredSize, + prepared.compression, + subject, + ); + const expected = Number(prepared.declaredSize); + const inflatedBytes = Math.max(expected, prepared.payload.length); + const result = await decodeZstandardFramesCooperative( + prepared.payload, + zstdDecodeOptions(expected), + pause, + ); + if (!result.ok) return zstdSectionFailure(result, subject, inflatedBytes); + return finishInflated(result.bytes, expected, "zstd", subject, inflatedBytes); }; /** @@ -140,6 +275,45 @@ export const decodeElfGnuZlibSection = ( return inflateDeclared(bytes.subarray(12), declaredSize, "zlib", subject); }; +const zstdSectionFailure = ( + result: ZstandardFrameFailure, + subject: string, + inflatedBytes: number, +): ElfCompressedSectionResult => { + if (result.reason === "invalid_allowance" && result.configuration) + return { + state: "configuration", + error: result.configuration, + inflatedBytes, + }; + if (result.reason === "work_limit") + return { + state: "work_limit", + ...zstandardWorkLimitFacts({ + maxFrames: result.value ?? 0, + source: result.source ?? "default", + }), + offset: result.offset, + inflatedBytes, + }; + if (result.reason === "output_limit") + return { + state: "malformed", + reason: "output_limit", + offset: result.offset, + message: `${subject} size is inconsistent`, + inflatedBytes, + }; + return { + state: "malformed", + reason: result.reason, + offset: result.offset, + ...(result.code === undefined ? {} : { code: result.code }), + message: `${subject} could not be decompressed: ${result.detail ?? result.code ?? result.reason}`, + inflatedBytes, + }; +}; + const inflateDeclared = ( payload: Buffer, declaredSize: bigint, @@ -152,11 +326,12 @@ const inflateDeclared = ( try { const maxOutputLength = Math.max(1, expected); if (compression === "zstd") { - decoded = zstdDecompressSync(payload, { - maxOutputLength, - // Retain Zstandard's default maximum decoder window of 128 MiB. - params: { [constants.ZSTD_d_windowLogMax]: 27 }, - }); + const result = decodeZstandardFrames( + payload, + zstdDecodeOptions(expected), + ); + if (!result.ok) return zstdSectionFailure(result, subject, inflatedBytes); + decoded = result.bytes; } else { const result = nodeCodecResult( inflateSync(payload, { info: true, maxOutputLength }), @@ -176,17 +351,5 @@ const inflateDeclared = ( message: `${subject} could not be decompressed: ${cause instanceof Error ? cause.message : "invalid compressed data"}`, }; } - if (decoded.length !== expected) - return { - state: "malformed", - message: `${subject} size is inconsistent`, - inflatedBytes, - }; - return { - state: "available", - bytes: decoded, - compressed: true, - compression, - inflatedBytes, - }; + return finishInflated(decoded, expected, compression, subject, inflatedBytes); }; diff --git a/src/domain/native/elfImage.ts b/src/domain/native/elfImage.ts index 5044c38523..850b847aff 100644 --- a/src/domain/native/elfImage.ts +++ b/src/domain/native/elfImage.ts @@ -10,17 +10,18 @@ import { type LoadRegion, } from "../binaryContainer.js"; import { elfArchitecture } from "../elfFixedHeader.js"; -import { readElfDirectoryFromBuffer } from "./nativeSectionDirectory.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; +import type { CooperativeElfSectionNameDecoder } from "./elfSectionNameTable.js"; +import { + readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, + type ElfSectionDirectory, +} from "./nativeSectionDirectory.js"; -/** Read ELF sections and file-backed program mappings without interpreting language metadata. */ -export const readElfImage = ( +export const readElfImageFromDirectory = ( bytes: Buffer, - maximumSourceAndDecodedNamesBytes?: number, + directory: ElfSectionDirectory, ): Extract => { - const directory = readElfDirectoryFromBuffer( - bytes, - maximumSourceAndDecodedNamesBytes, - ); const { bits } = directory; const reader = new BinaryReader(bytes, directory.little, "elf"); const mappings: FileMapping[] = []; @@ -138,3 +139,37 @@ export const architectureForElfMachine = (machine: number): string => { return `elf-machine-${machine}`; } }; + +/** Read ELF sections and file-backed program mappings without interpreting language metadata. */ +export const readElfImage = ( + bytes: Buffer, + operation: string, + maximumSourceAndDecodedNamesBytes?: number, +): Extract => + readElfImageFromDirectory( + bytes, + readElfDirectoryFromBuffer( + bytes, + operation, + maximumSourceAndDecodedNamesBytes, + ), + ); + +/** Same image as {@link readElfImage}. Section names yield through `pause`. */ +export const readElfImageCooperative = async ( + bytes: Buffer, + operation: string, + pause: ZstandardFramePause, + maximumSourceAndDecodedNamesBytes?: number, + decode?: CooperativeElfSectionNameDecoder, +): Promise> => + readElfImageFromDirectory( + bytes, + await readElfDirectoryFromBufferCooperative( + bytes, + pause, + operation, + maximumSourceAndDecodedNamesBytes, + decode, + ), + ); diff --git a/src/domain/native/elfSectionNameTable.ts b/src/domain/native/elfSectionNameTable.ts index e93807d0e1..497804db9a 100644 --- a/src/domain/native/elfSectionNameTable.ts +++ b/src/domain/native/elfSectionNameTable.ts @@ -1,8 +1,10 @@ import { decodeLimit, ParseFailureError } from "../parseFailure.js"; import { decodeElfCompressedSection, + decodeElfCompressedSectionCooperative, type ElfCompressedSectionResult, } from "./elfCompressedSection.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; const SUBJECT = "ELF compressed section-name table"; @@ -50,6 +52,19 @@ export const elfSectionNameBudget = ( }; }; +/** One section-name table a directory or note driver asks the owner to decode. */ +export interface ElfSectionNameDecodeRequest { + readonly raw: Buffer; + readonly flags: bigint; + readonly bits: 32 | 64; + readonly little: boolean; + readonly maximumDecodedBytes: number; +} + +export type CooperativeElfSectionNameDecoder = ( + request: ElfSectionNameDecodeRequest, +) => Promise; + /** Decode section names through the shared ELF compressed-section owner. */ export const decodeElfSectionNameTable = ( bytes: Buffer, @@ -66,3 +81,85 @@ export const decodeElfSectionNameTable = ( maximumDecodedBytes, SUBJECT, ); + +/** Cooperative section-name decode. `pause` observes the caller's abort signal. */ +export const decodeElfSectionNameTableCooperative = ( + bytes: Buffer, + flags: bigint, + bits: 32 | 64, + little: boolean, + maximumDecodedBytes: number, + pause: ZstandardFramePause, +): Promise => + decodeElfCompressedSectionCooperative( + bytes, + flags, + bits, + little, + maximumDecodedBytes, + SUBJECT, + pause, + ); + +export const decodeElfSectionNameRequest = ( + request: ElfSectionNameDecodeRequest, +): ElfCompressedSectionResult => + decodeElfSectionNameTable( + request.raw, + request.flags, + request.bits, + request.little, + request.maximumDecodedBytes, + ); + +export const decodeElfSectionNameRequestCooperative = ( + request: ElfSectionNameDecodeRequest, + pause: ZstandardFramePause, +): Promise => + decodeElfSectionNameTableCooperative( + request.raw, + request.flags, + request.bits, + request.little, + request.maximumDecodedBytes, + pause, + ); + +/** Reuse a decoded table across readers of one borrowed immutable artifact. */ +export const sharedElfSectionNameDecoder = ( + pause: ZstandardFramePause, +): CooperativeElfSectionNameDecoder => { + let admitted: + | { + request: ElfSectionNameDecodeRequest; + decoded: Extract< + ElfCompressedSectionResult, + { state: "available" | "unsupported" } + >; + } + | undefined; + return async (request) => { + if (admitted !== undefined) { + const { request: previous, decoded } = admitted; + if ( + request.raw.buffer === previous.raw.buffer && + request.raw.byteOffset === previous.raw.byteOffset && + request.raw.byteLength === previous.raw.byteLength && + request.bits === previous.bits && + request.little === previous.little && + request.flags === previous.flags && + (decoded.state === "unsupported" || + !decoded.compressed || + decoded.bytes.length <= request.maximumDecodedBytes) + ) + return decoded; + } + const decoded = await decodeElfSectionNameRequestCooperative( + request, + pause, + ); + if (decoded.state === "available" || decoded.state === "unsupported") + admitted = { request, decoded }; + return decoded; + }; +}; diff --git a/src/domain/native/nativeExportParser.ts b/src/domain/native/nativeExportParser.ts index 306f4190b5..63c4412f3b 100644 --- a/src/domain/native/nativeExportParser.ts +++ b/src/domain/native/nativeExportParser.ts @@ -12,8 +12,10 @@ import { readElfStringBytes } from "./elfStringBytes.js"; import { elfSymbolSize, readElfSymbol } from "../elfSymbol.js"; import { readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, type NativeSection, } from "./nativeSectionDirectory.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; import { peDeclaredFileMappings, readPeImage } from "./peImage.js"; import { allKeyed, @@ -32,6 +34,10 @@ import { type Artifact = NativeExports["artifact"]; +const ELF_MAGIC = Buffer.from([0x7f, 0x45, 0x4c, 0x46]); +const hasElfMagic = (bytes: Buffer): boolean => + bytes.subarray(0, ELF_MAGIC.length).equals(ELF_MAGIC); +const OPERATION = "inspect_native_exports"; const SHT_STRTAB = 3; const SHT_DYNSYM = 11; const SHT_GNU_VERDEF = 0x6ffffffd; @@ -668,12 +674,12 @@ function* readVersionNames( return namesByIndex; } -/** Read defined exported entries of `.dynsym` with their symbol versions. */ -function* parseElfExports( +/** Read ELF exports and versions from an already-admitted directory for these bytes. */ +export function* parseElfExportsFromDirectorySteps( bytes: Buffer, artifact: Artifact, -): Generator { - const directory = readElfDirectoryFromBuffer(bytes); + directory: ReturnType, +): Generator> { const reader = new BinaryReader(bytes, directory.little, "elf"); const result = { artifact, @@ -930,8 +936,12 @@ export function* parseNativeExportsSteps( artifact: Artifact, ): Generator { yield; - if (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) - return yield* parseElfExports(bytes, artifact); + if (hasElfMagic(bytes)) + return yield* parseElfExportsFromDirectorySteps( + bytes, + artifact, + readElfDirectoryFromBuffer(bytes, OPERATION), + ); if (bytes.subarray(0, 2).equals(Buffer.from("MZ"))) return parsePeExports(bytes, artifact); throw new BinaryFormatFailure( @@ -944,8 +954,32 @@ export function* parseNativeExportsSteps( export const parseNativeExports = ( bytes: Buffer, artifact: Artifact, +): NativeExports => finishExports(parseNativeExportsSteps(bytes, artifact)); + +/** Yield during ELF section-name decoding and bounded export inventory work. */ +export const parseNativeExportsCooperative = async ( + bytes: Buffer, + artifact: Artifact, + pause: ZstandardFramePause, +): Promise => { + if (!hasElfMagic(bytes)) return parseNativeExports(bytes, artifact); + const directory = await readElfDirectoryFromBufferCooperative( + bytes, + pause, + OPERATION, + ); + const steps = parseElfExportsFromDirectorySteps(bytes, artifact, directory); + let step = steps.next(); + while (!step.done) { + await pause(); + step = steps.next(); + } + return step.value; +}; + +const finishExports = ( + steps: Generator, ): NativeExports => { - const steps = parseNativeExportsSteps(bytes, artifact); let step = steps.next(); while (!step.done) step = steps.next(); return step.value; diff --git a/src/domain/native/nativeSectionDirectory.ts b/src/domain/native/nativeSectionDirectory.ts index f4e3a0c2b8..15f1b0233f 100644 --- a/src/domain/native/nativeSectionDirectory.ts +++ b/src/domain/native/nativeSectionDirectory.ts @@ -28,14 +28,29 @@ import { type ElfDynamicSegment, } from "../binaryContainer.js"; import { - decodeElfSectionNameTable, + decodeElfSectionNameRequest, + decodeElfSectionNameRequestCooperative, elfSectionNameBudgetFailure, + type CooperativeElfSectionNameDecoder, + type ElfSectionNameDecodeRequest, } from "./elfSectionNameTable.js"; +import type { ElfCompressedSectionResult } from "./elfCompressedSection.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../compression/zstandardFrameAllowance.js"; /** A borrowed positional source. Drivers own admission, cancellation and file identity. */ export interface NativeReadRequest { readonly offset: number; readonly size: number; + /** + * When set, the driver decodes this section-name table and calls + * `fulfillSectionNames` before resuming. A byte read does not satisfy it. + */ + readonly sectionNames?: ElfSectionNameDecodeRequest; + readonly fulfillSectionNames?: (result: ElfCompressedSectionResult) => void; } export interface NativeSection { readonly index: number; @@ -102,7 +117,7 @@ export interface NativeSectionDirectory { readonly programs: readonly ElfProgramDeclaration[]; readonly sections: readonly NativeSection[]; } -type ElfSectionDirectory = NativeSectionDirectory & { +export type ElfSectionDirectory = NativeSectionDirectory & { readonly format: "elf"; readonly elf_type: number; }; @@ -111,6 +126,13 @@ const STRUCTURE_BYTES = 16 * 1024 * 1024; const reject = (message: string): never => { throw new BinaryFormatFailure("malformed", message); }; +const requireSectionNames = ( + result: ElfCompressedSectionResult | undefined, +): ElfCompressedSectionResult => { + if (result === undefined) + return reject("ELF section-name decode was not satisfied"); + return result; +}; const exact = (value: bigint): number => { if (value < 0n || value > BigInt(Number.MAX_SAFE_INTEGER)) reject("Native offset exceeds exact host precision"); @@ -247,34 +269,50 @@ export function* readElfSectionDirectory( maximumSourceAndDecodedNamesBytes === undefined ? structuralRemaining : Math.max(0, maximumSourceAndDecodedNamesBytes - fileSize); - const decoded = decodeElfSectionNameTable( - rawNames, - namesSection.flags, + let decoded: ElfCompressedSectionResult | undefined; + const sectionNames: ElfSectionNameDecodeRequest = { + raw: rawNames, + flags: namesSection.flags, bits, little, - Math.min(structuralRemaining, callerRemaining), - ); - if (decoded.state === "malformed") return reject(decoded.message); + maximumDecodedBytes: Math.min(structuralRemaining, callerRemaining), + }; + yield { + offset: 0, + size: 0, + sectionNames, + fulfillSectionNames: (result) => { + decoded = result; + }, + }; + const admitted = requireSectionNames(decoded); + if (admitted.state === "malformed") return reject(admitted.message); + if (admitted.state === "configuration") throw admitted.error; + if (admitted.state === "work_limit") + throw new ZstandardFrameAllowanceError( + { maxFrames: admitted.value, source: admitted.source }, + admitted.offset, + ); if ( - decoded.state === "limit" && + admitted.state === "limit" && maximumSourceAndDecodedNamesBytes !== undefined && - decoded.requiredBytes > BigInt(callerRemaining) + admitted.requiredBytes > BigInt(callerRemaining) ) throw elfSectionNameBudgetFailure( - BigInt(fileSize) + decoded.requiredBytes, + BigInt(fileSize) + admitted.requiredBytes, maximumSourceAndDecodedNamesBytes, ); - if (decoded.state === "limit") + if (admitted.state === "limit") throw new BinaryResourceFailure( "structure", STRUCTURE_BYTES, - `Decoded ELF section names require ${String(decoded.requiredBytes)} bytes, above the ${String(structuralRemaining)}-byte remaining structural budget`, + `Decoded ELF section names require ${String(admitted.requiredBytes)} bytes, above the ${String(structuralRemaining)}-byte remaining structural budget`, "structure", "bytes", ); - if (decoded.state === "unsupported") - section_names_limitation = `ELF section names are unavailable because compression type ${String(decoded.compressionType)} is unsupported`; - else names = decoded.bytes; + if (admitted.state === "unsupported") + section_names_limitation = `ELF section names are unavailable because compression type ${String(admitted.compressionType)} is unsupported`; + else names = admitted.bytes; } else if (count > 1) { section_names_limitation = "ELF section names are unavailable because no section-name string table is declared"; @@ -476,27 +514,110 @@ export function* readPeSectionDirectory( }; } +const resumeDirectoryStep = ( + bytes: Buffer, + step: NativeReadRequest, + names: (request: ElfSectionNameDecodeRequest) => ElfCompressedSectionResult, +): Buffer => { + if ( + step.sectionNames !== undefined && + step.fulfillSectionNames !== undefined + ) { + step.fulfillSectionNames(names(step.sectionNames)); + return Buffer.alloc(0); + } + return bytes.subarray(step.offset, step.offset + step.size); +}; + const driveFromBuffer = ( bytes: Buffer, steps: Generator, ): T => { let next = steps.next(); while (!next.done) { - const { offset, size } = next.value; - next = steps.next(bytes.subarray(offset, offset + size)); + next = steps.next( + resumeDirectoryStep(bytes, next.value, decodeElfSectionNameRequest), + ); } return next.value; }; -/** Borrow ELF bytes; file drivers satisfy the identical requests without loading payloads. */ +const driveElfDirectoryCooperative = async ( + bytes: Buffer, + decode: CooperativeElfSectionNameDecoder, + maximumSourceAndDecodedNamesBytes?: number, +): Promise => { + const steps = readElfSectionDirectory( + bytes.length, + maximumSourceAndDecodedNamesBytes, + ); + let next = steps.next(); + while (!next.done) { + const step = next.value; + if ( + step.sectionNames !== undefined && + step.fulfillSectionNames !== undefined + ) { + step.fulfillSectionNames(await decode(step.sectionNames)); + next = steps.next(Buffer.alloc(0)); + continue; + } + next = steps.next(bytes.subarray(step.offset, step.offset + step.size)); + } + return next.value; +}; + +const projectDirectoryFrameAllowance = ( + operation: string, + cause: unknown, +): never => { + if (cause instanceof ZstandardFrameAllowanceError) + throw projectZstandardFrameAllowance(operation, cause); + throw cause; +}; + +/** + * Borrow ELF bytes. `operation` names the public refusal when a compressed + * section-name table exceeds `zstd.max_frames`, so every caller projects that + * adjustable limit instead of an I/O or adapter failure. + */ export const readElfDirectoryFromBuffer = ( bytes: Buffer, + operation: string, maximumSourceAndDecodedNamesBytes?: number, -): ElfSectionDirectory => - driveFromBuffer( - bytes, - readElfSectionDirectory(bytes.length, maximumSourceAndDecodedNamesBytes), - ); +): ElfSectionDirectory => { + try { + return driveFromBuffer( + bytes, + readElfSectionDirectory(bytes.length, maximumSourceAndDecodedNamesBytes), + ); + } catch (cause) { + return projectDirectoryFrameAllowance(operation, cause); + } +}; + +/** + * Same directory as {@link readElfDirectoryFromBuffer}. Compressed section + * names yield through `pause` so the caller's abort signal can land. + */ +export const readElfDirectoryFromBufferCooperative = async ( + bytes: Buffer, + pause: ZstandardFramePause, + operation: string, + maximumSourceAndDecodedNamesBytes?: number, + decode: CooperativeElfSectionNameDecoder = (request) => + decodeElfSectionNameRequestCooperative(request, pause), +): Promise => { + try { + return await driveElfDirectoryCooperative( + bytes, + decode, + maximumSourceAndDecodedNamesBytes, + ); + } catch (cause) { + return projectDirectoryFrameAllowance(operation, cause); + } +}; /** Reject a program or section payload the declared tables place outside these bytes. */ export const assertElfDeclaredRangesFromBuffer = (bytes: Buffer): void => { diff --git a/src/domain/native/readBinaryContainer.ts b/src/domain/native/readBinaryContainer.ts index b9b0fb670f..246dfc1925 100644 --- a/src/domain/native/readBinaryContainer.ts +++ b/src/domain/native/readBinaryContainer.ts @@ -8,14 +8,35 @@ import { readFatHeaderPrefix, readMachoHeaderPrefixFacts, } from "../apple/machoContainer.js"; -import { readElfImage } from "./elfImage.js"; +import { readElfImage, readElfImageCooperative } from "./elfImage.js"; +import type { ZstandardFramePause } from "../compression/zstandardExactFrame.js"; import { readPeImage } from "./peImage.js"; import { readMachoImage } from "../apple/machoImage.js"; +const ELF_MAGIC = Buffer.from([0x7f, 0x45, 0x4c, 0x46]); +const hasElfMagic = (bytes: Buffer): boolean => + bytes.subarray(0, ELF_MAGIC.length).equals(ELF_MAGIC); + /** Identify an ELF, PE, or thin Mach-O image and its file sections. */ -export const readBinaryContainer = (bytes: Buffer): BinaryContainer => { - if (bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) - return readElfImage(bytes); +export const readBinaryContainer = ( + bytes: Buffer, + operation: string, +): BinaryContainer => + hasElfMagic(bytes) + ? readElfImage(bytes, operation) + : readNonElfContainer(bytes); + +/** Read the same facts, allowing the operation to pause during ELF name decoding. */ +export const readBinaryContainerCooperative = async ( + bytes: Buffer, + operation: string, + pause: ZstandardFramePause, +): Promise => + hasElfMagic(bytes) + ? readElfImageCooperative(bytes, operation, pause) + : readNonElfContainer(bytes); + +const readNonElfContainer = (bytes: Buffer): BinaryContainer => { if (bytes.subarray(0, 2).equals(Buffer.from("MZ"))) return readPeImage(bytes); if (bytes.length >= 4) { const fatPrefix = readFatHeaderPrefix(bytes); diff --git a/src/go/GoBinaryImage.ts b/src/go/GoBinaryImage.ts index 996d5f0da6..361a856ce3 100644 --- a/src/go/GoBinaryImage.ts +++ b/src/go/GoBinaryImage.ts @@ -13,7 +13,11 @@ import { isZeroFilledSpan, type BinaryContainer, } from "../domain/binaryContainer.js"; -import { readBinaryContainer } from "../domain/native/readBinaryContainer.js"; +import type { ZstandardFramePause } from "../domain/compression/zstandardExactFrame.js"; +import { + readBinaryContainer, + readBinaryContainerCooperative, +} from "../domain/native/readBinaryContainer.js"; import { goAnalysisBudgets, type GoAnalysisBudgets, @@ -58,9 +62,31 @@ export interface ReadGoBinaryImageOptions { export const readGoBinaryImage = ( bytes: Buffer, options?: ReadGoBinaryImageOptions, +): GoBinaryImage => + finishGoBinaryImage( + bytes, + options, + readBinaryContainer(bytes, "inspect_go_binary"), + ); + +/** Same image as {@link readGoBinaryImage}. ELF section names yield through `pause`. */ +export const readGoBinaryImageCooperative = async ( + bytes: Buffer, + options: ReadGoBinaryImageOptions | undefined, + pause: ZstandardFramePause, +): Promise => + finishGoBinaryImage( + bytes, + options, + await readBinaryContainerCooperative(bytes, "inspect_go_binary", pause), + ); + +const finishGoBinaryImage = ( + bytes: Buffer, + options: ReadGoBinaryImageOptions | undefined, + container: ReturnType, ): GoBinaryImage => { const budgets = goAnalysisBudgets(options?.budgets); - const container = readBinaryContainer(bytes); const { format, architecture, bits, byte_order } = container; const stringLiterals = options?.includeStringLiterals === true diff --git a/src/go/GoBinaryProvider.ts b/src/go/GoBinaryProvider.ts index 665ac56d0f..ed1f96a26c 100644 --- a/src/go/GoBinaryProvider.ts +++ b/src/go/GoBinaryProvider.ts @@ -17,6 +17,7 @@ import { artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, assertArtifactReadActive, + yieldArtifactRead, ArtifactResourceScope, ArtifactReaderFailure, } from "../resources/index.js"; @@ -40,7 +41,11 @@ import { import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; import { err, ok, type Result } from "../domain/result.js"; -import { readGoBinaryImage, type GoBinaryImage } from "./GoBinaryImage.js"; +import { + readGoBinaryImage, + readGoBinaryImageCooperative, + type GoBinaryImage, +} from "./GoBinaryImage.js"; import { requireSelectedNativeImage } from "../domain/apple/machoSliceSelection.js"; import { BinaryResourceFailure, @@ -333,11 +338,19 @@ export class GoBinaryProvider implements GoBinaryPort { snapshot.bytes, input.architecture, ); - const image = readGoBinaryImage(selected.bytes, { + const imageOptions = { includeStringLiterals: input.include_string_literals === true, budgets, checkActive: () => assertArtifactReadActive(options?.signal), - }); + }; + const image = + options?.signal === undefined + ? readGoBinaryImage(selected.bytes, imageOptions) + : await readGoBinaryImageCooperative( + selected.bytes, + imageOptions, + () => yieldArtifactRead(options.signal), + ); const { section_names_limitation: _sectionNamesLimitation, macho_chained_fixups: _machoChainedFixups, diff --git a/src/gpu/GpuCodeObjectsProvider.ts b/src/gpu/GpuCodeObjectsProvider.ts index 38db322a97..f4f3ee7ec7 100644 --- a/src/gpu/GpuCodeObjectsProvider.ts +++ b/src/gpu/GpuCodeObjectsProvider.ts @@ -34,6 +34,10 @@ import { carriedReason, } from "../domain/analysisErrorCore.js"; import { parseFailureAnalysisError } from "../domain/parseFailureAnalysisError.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../domain/compression/zstandardFrameAllowance.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; @@ -114,8 +118,19 @@ export class GpuCodeObjectsProvider implements GpuCodeObjectsPort { input.path, primary, ); + const frameAllowance = + primary instanceof ZstandardFrameAllowanceError + ? primary + : cause instanceof ZstandardFrameAllowanceError + ? cause + : undefined; let error: AnalysisError; - if (origin === "owned-or-classified" && primary instanceof AnalysisError) + if (frameAllowance !== undefined) + error = projectZstandardFrameAllowance(OPERATION, frameAllowance); + else if ( + origin === "owned-or-classified" && + primary instanceof AnalysisError + ) error = primary; else if (primary instanceof BinaryResourceFailure) error = new AnalysisResourceConstraintError( diff --git a/src/gpu/GpuElfKernels.ts b/src/gpu/GpuElfKernels.ts index dbca939186..37df71fae4 100644 --- a/src/gpu/GpuElfKernels.ts +++ b/src/gpu/GpuElfKernels.ts @@ -17,6 +17,7 @@ import { readCudaParameters, } from "./CudaInfo.js"; import type { GpuKernel, GpuLocation } from "../domain/gpu/gpuCodeObjects.js"; +import type { ZstandardFramePause } from "../domain/compression/zstandardExactFrame.js"; import { BinaryReader, BinaryFormatFailure, @@ -100,6 +101,7 @@ export const readGpuElfKernels = async ( source: GpuSource, kind: "cubin" | "amdgpu", onKernel: (kernel: GpuKernel) => void, + pause: ZstandardFramePause, preparedAmd?: AmdElfMetadata, onTarget?: (target: GpuPayloadTarget) => void, ): Promise<{ @@ -111,6 +113,7 @@ export const readGpuElfKernels = async ( const directory = await readGpuNativeDirectory( source, readElfSectionDirectory(source.size), + pause, ); if (directory.machine !== (kind === "cubin" ? 190 : 224)) rejectGpu("GPU entry kind disagrees with ELF machine"); diff --git a/src/gpu/GpuInventory.ts b/src/gpu/GpuInventory.ts index 544620a06c..8d5ac0c3a5 100644 --- a/src/gpu/GpuInventory.ts +++ b/src/gpu/GpuInventory.ts @@ -22,6 +22,11 @@ import { import { readCudaTarget } from "./CudaTarget.js"; import { readCudaFatbinary } from "./CudaFatbinary.js"; import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../domain/compression/zstandardFrameAllowance.js"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; import { createHash } from "node:crypto"; import { scheduler } from "node:timers/promises"; import { @@ -35,7 +40,10 @@ import { readPeSectionDirectory, type NativeSectionDirectory, } from "../domain/native/nativeSectionDirectory.js"; -import { assertArtifactReadActive } from "../resources/index.js"; +import { + assertArtifactReadActive, + yieldArtifactRead, +} from "../resources/index.js"; import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { @@ -179,6 +187,7 @@ export class GpuInventory { elf ? readElfSectionDirectory(source.size) : readPeSectionDirectory(source.size), + () => yieldArtifactRead(this.signal), ); if (elf && (directory.machine === 190 || directory.machine === 224)) { const row = gpuObject(source, { @@ -295,6 +304,9 @@ export class GpuInventory { } #retainFailure(row: GpuObject, cause: unknown): void { if (cause instanceof GpuReportCapacityFailure) throw cause; + if (cause instanceof ZstandardFrameAllowanceError) + throw projectZstandardFrameAllowance("inspect_gpu_code_objects", cause); + if (cause instanceof AnalysisError) throw cause; const capacity = cause instanceof BinaryResourceFailure || isBinaryStringReadLimit(cause); if (!capacity && !(cause instanceof BinaryFormatFailure)) throw cause; @@ -358,7 +370,16 @@ export class GpuInventory { this.signal, ); } catch (cause) { - if (cause instanceof BinaryResourceFailure) throw cause; + if (cause instanceof ZstandardFrameAllowanceError) + throw projectZstandardFrameAllowance( + "inspect_gpu_code_objects", + cause, + ); + if ( + cause instanceof BinaryResourceFailure || + cause instanceof AnalysisError + ) + throw cause; assertArtifactReadActive(this.signal); throw new BinaryFormatFailure( "malformed", @@ -515,6 +536,7 @@ export class GpuInventory { (await readGpuNativeDirectory( child, readElfSectionDirectory(child.size), + () => yieldArtifactRead(this.signal), )); const inspected = await readAmdElfMetadata(child, native); const prepared = inspected.ok ? inspected.value : inspected.error.partial; @@ -558,6 +580,7 @@ export class GpuInventory { this.#retain(kernel); this.kernels.push(kernel); }, + () => yieldArtifactRead(this.signal), preparedAmd, (target) => { conflict = this.#target(row, target); diff --git a/src/gpu/GpuPayloadCodec.test.ts b/src/gpu/GpuPayloadCodec.test.ts index 6b1f11f9aa..e399e07599 100644 --- a/src/gpu/GpuPayloadCodec.test.ts +++ b/src/gpu/GpuPayloadCodec.test.ts @@ -1,5 +1,10 @@ import { expect, it } from "vitest"; import { deflateSync, zstdCompressSync } from "node:zlib"; +import { runWithAnalysisOperationSnapshot } from "../domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../domain/analysisResourceLimits.js"; import { decodeGpuPayload, decodeGpuPayloadStream } from "./GpuPayloadCodec.js"; it.each(["zlib", "zstd"] as const)( "requires all declared %s input bytes to belong to the codec stream", @@ -22,6 +27,49 @@ it.each(["zlib", "zstd"] as const)( ).rejects.toMatchObject({ kind: "malformed" }); }, ); +const zstdByteFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +it("stops a many-frame Zstandard payload when the read is cancelled", async () => { + const frame = zstdByteFrame(0x61); + const count = 32; + const bytes = Buffer.alloc(frame.length * count); + for (let index = 0; index < count; index += 1) + frame.copy(bytes, index * frame.length); + const controller = new AbortController(); + const pending = decodeGpuPayload(bytes, "zstd", count, controller.signal); + controller.abort(); + await expect(pending).rejects.toMatchObject({ reason: "cancelled" }); +}); + +it("decodes every Zstandard frame before a container suffix", async () => { + const frames = Buffer.concat([zstdByteFrame(0x61), zstdByteFrame(0x62)]); + const plain = Buffer.from([0x61, 0x62]); + expect(Buffer.compare(await decodeGpuPayload(frames, "zstd", 2), plain)).toBe( + 0, + ); + const streamed = await decodeGpuPayloadStream( + Buffer.concat([frames, Buffer.from("CCOB")]), + "zstd", + 2, + ); + expect(Buffer.compare(streamed.buffer, plain)).toBe(0); + expect(streamed.consumedBytes).toBe(frames.length); + await expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => decodeGpuPayload(frames, "zstd", 2), + ), + ).rejects.toMatchObject({ + reason: "work_limit", + setting: "zstd.max_frames", + value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + }); +}); + it.each(["zlib", "zstd"] as const)( "retains completed %s input accounting before trailing container bytes", async (codec) => { diff --git a/src/gpu/GpuPayloadCodec.ts b/src/gpu/GpuPayloadCodec.ts index fc7b9417eb..4e2f74951c 100644 --- a/src/gpu/GpuPayloadCodec.ts +++ b/src/gpu/GpuPayloadCodec.ts @@ -1,11 +1,17 @@ import { nodeCodecResult } from "../kernel/nodeCodecResult.js"; -import { constants, inflate, zstdDecompress } from "node:zlib"; +import { inflate } from "node:zlib"; +import { scheduler } from "node:timers/promises"; import { promisify } from "node:util"; import { assertArtifactReadActive } from "../resources/index.js"; import { decodeLz4Block } from "../compression/Lz4Block.js"; +import { + decodeZstandardFramesCooperative, + ZSTANDARD_WINDOW_LOG_MAX, +} from "../domain/compression/zstandardExactFrame.js"; +import { ZstandardFrameAllowanceError } from "../domain/compression/zstandardFrameAllowance.js"; +import { BinaryFormatFailure } from "../domain/binaryContainer.js"; import { GPU_OBJECT_BYTES, rejectGpu } from "./GpuSource.js"; const unzip = promisify(inflate); -const unzstd = promisify(zstdDecompress); /** Completed codec bytes and their encoded extent share one completion owner. */ export const decodeGpuPayloadStream = async ( bytes: Buffer, @@ -14,16 +20,50 @@ export const decodeGpuPayloadStream = async ( signal?: AbortSignal, ): Promise<{ buffer: Buffer; consumedBytes: number }> => { assertArtifactReadActive(signal); + const maxOutputLength = Math.max(1, maximumDecodedSize); + if (compression === "zstd") { + const decoded = await decodeZstandardFramesCooperative( + bytes, + { + maxOutputLength, + windowLogMax: ZSTANDARD_WINDOW_LOG_MAX, + mode: "prefix", + }, + async () => { + await scheduler.yield(); + assertArtifactReadActive(signal); + }, + ); + assertArtifactReadActive(signal); + if (!decoded.ok) { + if ( + decoded.reason === "invalid_allowance" && + decoded.configuration !== undefined + ) + throw decoded.configuration; + if (decoded.reason === "work_limit") + throw new ZstandardFrameAllowanceError( + { + maxFrames: decoded.value ?? 0, + source: decoded.source ?? "default", + }, + decoded.offset, + ); + throw new BinaryFormatFailure( + "malformed", + "GPU payload decompression failed or exceeded its declared length", + { + format: "zstd", + field: decoded.reason, + offset: decoded.offset, + }, + ); + } + return { buffer: decoded.bytes, consumedBytes: decoded.consumedBytes }; + } let result: unknown; try { - const maxOutputLength = Math.max(1, maximumDecodedSize); - result = await (compression === "zstd" - ? unzstd(bytes, { - maxOutputLength, - info: true, - params: { [constants.ZSTD_d_windowLogMax]: 27 }, - }) - : unzip(bytes, { maxOutputLength, info: true })); + result = await unzip(bytes, { maxOutputLength, info: true }); } catch { assertArtifactReadActive(signal); return rejectGpu( diff --git a/src/gpu/GpuSource.ts b/src/gpu/GpuSource.ts index d843500742..63205dade0 100644 --- a/src/gpu/GpuSource.ts +++ b/src/gpu/GpuSource.ts @@ -8,6 +8,8 @@ import { BinaryFormatFailure, BinaryResourceFailure, } from "../domain/binaryContainer.js"; +import { decodeElfSectionNameRequestCooperative } from "../domain/native/elfSectionNameTable.js"; +import type { ZstandardFramePause } from "../domain/compression/zstandardExactFrame.js"; import type { NativeReadRequest, NativeSectionDirectory, @@ -118,11 +120,22 @@ export class GpuBufferSource implements GpuSource { export const readGpuNativeDirectory = async ( source: GpuSource, steps: Generator, + pause: ZstandardFramePause, ): Promise => { let next = steps.next(); while (!next.done) { - const { offset, size } = next.value; - next = steps.next(await source.read(offset, size)); + const step = next.value; + if ( + step.sectionNames !== undefined && + step.fulfillSectionNames !== undefined + ) { + step.fulfillSectionNames( + await decodeElfSectionNameRequestCooperative(step.sectionNames, pause), + ); + next = steps.next(Buffer.alloc(0)); + continue; + } + next = steps.next(await source.read(step.offset, step.size)); } return next.value; }; diff --git a/src/il2cpp/Il2cppBinaryFile.ts b/src/il2cpp/Il2cppBinaryFile.ts index 16102a197e..965eb94df1 100644 --- a/src/il2cpp/Il2cppBinaryFile.ts +++ b/src/il2cpp/Il2cppBinaryFile.ts @@ -1,6 +1,12 @@ import type { Il2cppMethodAddress } from "../domain/il2cpp/il2cppMetadata.js"; import { Il2cppBinaryFormatFailure } from "./Il2cppBinaryFormat.js"; -import { openImageSteps, readSlot, isExecutable } from "./Il2cppBinaryImage.js"; +import { + openImageSteps, + readSlot, + isExecutable, + readIl2cppBinaryContainerCooperative, + type NativeImage, +} from "./Il2cppBinaryImage.js"; import { findModules, findCodeRegistration, @@ -74,6 +80,15 @@ export type Il2cppBinaryReadOptions = Pick< readonly genericInstances?: boolean; }; +const assertGenericLinkLayout = ( + metadata: Il2cppMetadataDecoded, + options: Il2cppBinaryReadOptions, +): void => { + // Refuse an undecoded version before any binary tables are read. + if (options.genericInstances === true) + genericMethodLayouts(metadata.metadata_version); +}; + /** * Link metadata methods to native code addresses without executing the image. * A method token's row id selects its module's methodPointers slot. @@ -84,11 +99,43 @@ export function* linkIl2cppBinarySteps( facts: Il2cppMetadataLinkFacts, options: Il2cppBinaryReadOptions = {}, ): Generator { - const version = metadata.metadata_version; - // Refuse an undecoded version before any table is read. - if (options.genericInstances === true) genericMethodLayouts(version); + assertGenericLinkLayout(metadata, options); yield; - const image = yield* openImageSteps(bytes); + return yield* linkImageSteps(openImageSteps(bytes), metadata, facts, options); +} + +export const linkIl2cppBinaryCooperative = async ( + bytes: Buffer, + metadata: Il2cppMetadataDecoded, + facts: Il2cppMetadataLinkFacts, + options: Il2cppBinaryReadOptions, + pause: () => Promise, +): Promise => { + assertGenericLinkLayout(metadata, options); + await pause(); + const container = await readIl2cppBinaryContainerCooperative(bytes, pause); + const steps = linkImageSteps( + openImageSteps(bytes, container), + metadata, + facts, + options, + ); + let step = steps.next(); + while (!step.done) { + await pause(); + step = steps.next(); + } + return step.value; +}; + +function* linkImageSteps( + opening: Generator, + metadata: Il2cppMetadataDecoded, + facts: Il2cppMetadataLinkFacts, + options: Il2cppBinaryReadOptions, +): Generator { + const image = yield* opening; + const version = metadata.metadata_version; const modules = yield* findModules(image, metadata); const registration_ = yield* findCodeRegistration(image, modules); const methodImage = new Int32Array(metadata.methods.length).fill(-1); diff --git a/src/il2cpp/Il2cppBinaryImage.ts b/src/il2cpp/Il2cppBinaryImage.ts index d8e97fe54d..13b621bf50 100644 --- a/src/il2cpp/Il2cppBinaryImage.ts +++ b/src/il2cpp/Il2cppBinaryImage.ts @@ -14,7 +14,10 @@ import { type BinaryContainer, type FileMappingIndex, } from "../domain/binaryContainer.js"; -import { readBinaryContainer } from "../domain/native/readBinaryContainer.js"; +import { + readBinaryContainer, + readBinaryContainerCooperative, +} from "../domain/native/readBinaryContainer.js"; import { requireSelectedNativeImage, type SelectedNativeImage, @@ -81,28 +84,51 @@ export interface NativeImage { readonly machoPointerFixups?: PointerFixups; } -/** Open one thin image; a universal Mach-O slice is selected before this. */ -export function* openImageSteps(image: Buffer): Generator { - let container: BinaryContainer; +const OPERATION = "inspect_il2cpp_metadata"; + +const rejectContainer = (cause: unknown): never => { + if (cause instanceof BinaryResourceFailure) + throw new Il2cppBinaryFormatFailure( + "native-index-capacity", + cause.message, + cause.failure.limit.maximum, + "bytes", + ); + if (cause instanceof BinaryFormatFailure) + throw new Il2cppBinaryFormatFailure( + cause.kind === "unsupported" + ? "unsupported-container" + : "malformed-container", + cause.message, + ); + throw cause; +}; + +const readContainer = (image: Buffer): BinaryContainer => { try { - container = readBinaryContainer(image); + return readBinaryContainer(image, OPERATION); } catch (cause) { - if (cause instanceof BinaryResourceFailure) - throw new Il2cppBinaryFormatFailure( - "native-index-capacity", - cause.message, - cause.failure.limit.maximum, - "bytes", - ); - if (cause instanceof BinaryFormatFailure) - throw new Il2cppBinaryFormatFailure( - cause.kind === "unsupported" - ? "unsupported-container" - : "malformed-container", - cause.message, - ); - throw cause; + return rejectContainer(cause); } +}; + +export const readIl2cppBinaryContainerCooperative = async ( + image: Buffer, + pause: () => Promise, +): Promise => { + try { + return await readBinaryContainerCooperative(image, OPERATION, pause); + } catch (cause) { + return rejectContainer(cause); + } +}; + +/** Open one thin image; a universal Mach-O slice is selected before this. */ +export function* openImageSteps( + image: Buffer, + preparedContainer?: BinaryContainer, +): Generator { + const container = preparedContainer ?? readContainer(image); if (container.bits !== 64 || container.byte_order !== "little") throw new Il2cppBinaryFormatFailure( "unsupported-container", diff --git a/src/il2cpp/Il2cppMetadataProvider.ts b/src/il2cpp/Il2cppMetadataProvider.ts index e078fc0d0b..27e602f16c 100644 --- a/src/il2cpp/Il2cppMetadataProvider.ts +++ b/src/il2cpp/Il2cppMetadataProvider.ts @@ -61,7 +61,7 @@ import { type Il2cppMetadataRowCounts, } from "./Il2cppMetadataFile.js"; import { - linkIl2cppBinarySteps, + linkIl2cppBinaryCooperative, type Il2cppBinaryLink, } from "./Il2cppBinaryFile.js"; import { errorMessage } from "../kernel/errorMessage.js"; @@ -563,7 +563,7 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { input.binary_architecture, ); sliceLimitations = selected.limitations; - const linking = linkIl2cppBinarySteps( + const linking = await linkIl2cppBinaryCooperative( selected.bytes, decoded, link, @@ -571,14 +571,12 @@ export class Il2cppMetadataProvider implements Il2cppMetadataPort { ...admission, genericInstances: input.generic_instances === true, }, + async () => { + await this.yieldControl(); + assertArtifactReadActive(options?.signal); + }, ); - let linkStep = linking.next(); - while (!linkStep.done) { - await this.yieldControl(); - assertArtifactReadActive(options?.signal); - linkStep = linking.next(); - } - linked = binaryReport(linkStep.value, { + linked = binaryReport(linking, { path: binaryPath, sha256: binary.sha256, bytes: binary.bytes.length, diff --git a/src/native/dwarf/DwarfSections.ts b/src/native/dwarf/DwarfSections.ts index abd4109042..ece6332ee2 100644 --- a/src/native/dwarf/DwarfSections.ts +++ b/src/native/dwarf/DwarfSections.ts @@ -16,12 +16,16 @@ import { } from "../../domain/binaryContainer.js"; import { decodeElfCompressedSection, + decodeElfCompressedSectionCooperative, decodeElfGnuZlibSection, type ElfCompressedSectionResult, } from "../../domain/native/elfCompressedSection.js"; +import type { ZstandardFramePause } from "../../domain/compression/zstandardExactFrame.js"; +import { ZstandardFrameAllowanceError } from "../../domain/compression/zstandardFrameAllowance.js"; import { readMachoImage } from "../../domain/apple/machoImage.js"; import { readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, readPeSectionDirectory, type NativeSection, type NativeSectionDirectory, @@ -128,14 +132,16 @@ interface Candidate { } /** Select DWARF sections from one thin ELF, PE, or Mach-O image. */ -export const locateDwarfSections = ( +export const locateDwarfSections = async ( bytes: Buffer, maximumDecodedBytes = MAX_DWARF_DECOMPRESSED_SECTION_BYTES, -): LocatedDwarfImage => { + pause?: ZstandardFramePause, +): Promise => { const budget: InflateBudget = { allowance: maximumDecodedBytes, used: 0 }; const magic = bytes.length >= 4 ? new BinaryReader(bytes, true, "dwarf").u32(0) : null; - if (magic === 0x464c457f) return finish(locateElf(bytes, budget)); + if (magic === 0x464c457f) + return finish(await locateElf(bytes, budget, pause)); if (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) return finish(locatePe(bytes, budget)); if ( @@ -259,6 +265,12 @@ const admitDecoded = ( : decoded.requiredBytes + BigInt(budget.used); throw new DwarfSectionAllowanceFailure(name, demanded, budget.allowance); } + if (decoded.state === "configuration") throw decoded.error; + if (decoded.state === "work_limit") + throw new ZstandardFrameAllowanceError( + { maxFrames: decoded.value, source: decoded.source }, + decoded.offset, + ); if (decoded.state === "malformed") { // The owner charges the larger of declared output and compressed input // read on failure. Rejected headers never inflate and charge 0. @@ -413,8 +425,19 @@ const finish = (image: ImageIdentity): LocatedDwarfImage => { }; }; -const locateElf = (bytes: Buffer, budget: InflateBudget): ImageIdentity => { - const directory = readElfDirectoryFromBuffer(bytes); +const locateElf = async ( + bytes: Buffer, + budget: InflateBudget, + pause?: ZstandardFramePause, +): Promise => { + const directory = + pause === undefined + ? readElfDirectoryFromBuffer(bytes, "inspect_dwarf_units") + : await readElfDirectoryFromBufferCooperative( + bytes, + pause, + "inspect_dwarf_units", + ); if (directory.section_names_limitation !== undefined) throw new BinaryFormatFailure( "unsupported", @@ -480,14 +503,24 @@ const locateElf = (bytes: Buffer, budget: InflateBudget): ImageIdentity => { continue; } const decoded = compressed - ? decodeElfCompressedSection( - stored, - section.flags, - directory.bits, - directory.little, - decodeBudget(budget, name), - `ELF ${name} section`, - ) + ? pause === undefined + ? decodeElfCompressedSection( + stored, + section.flags, + directory.bits, + directory.little, + decodeBudget(budget, name), + `ELF ${name} section`, + ) + : await decodeElfCompressedSectionCooperative( + stored, + section.flags, + directory.bits, + directory.little, + decodeBudget(budget, name), + `ELF ${name} section`, + pause, + ) : decodeElfGnuZlibSection( stored, decodeBudget(budget, name), diff --git a/src/native/dwarf/DwarfUnitsProvider.test.ts b/src/native/dwarf/DwarfUnitsProvider.test.ts index 31d2efd4d9..d2964538a9 100644 --- a/src/native/dwarf/DwarfUnitsProvider.test.ts +++ b/src/native/dwarf/DwarfUnitsProvider.test.ts @@ -1,7 +1,9 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { scheduler } from "node:timers/promises"; import { expect, it, vi } from "vitest"; import { + AnalysisCancelledError, AnalysisInputError, AnalysisResourceConstraintError, } from "../../domain/analysisErrorCore.js"; @@ -55,6 +57,55 @@ const inspect = async ( } }; +const manyProductiveFrames = (count: number): Buffer => { + const frame = Buffer.from([ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, 0x61, + ]); + const bytes = Buffer.alloc(frame.length * count); + for (let index = 0; index < count; index += 1) + frame.copy(bytes, index * frame.length); + return bytes; +}; + +it("cancels a many-frame compressed DWARF section", async () => { + const plain = Buffer.alloc(32, 0x61); + const header = elfCompressedBody(plain, "zstd").subarray(0, 24); + const image = elfDebugImage( + { info: Buffer.concat([header, manyProductiveFrames(32)]) }, + { infoFlags: 0x800n }, + ); + const root = await createTestTempDirectory("rea-dwarf-zstd-cancel-"); + const path = join(root, "image"); + await writeFile(path, image); + const reason = new Error("stop dwarf frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + const provider = new DwarfUnitsProvider(); + try { + const result = await provider.inspect( + inspectDwarfUnitsInputSchema.parse({ path }), + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_dwarf_units", + cause: reason, + }); + } finally { + spy.mockRestore(); + await provider.close(); + } +}); + it("reads a little-endian ELF unit and refuses architecture on a thin file", async () => { const named = namedUnit({ name: "main.c", dwoName: "main.dwo" }); const image = elfDebugImage(named); @@ -379,7 +430,7 @@ it("does not inflate a duplicate section and caps decompressed bytes across role expect(aggregate.error).toBeInstanceOf(AnalysisResourceConstraintError); }); -it("charges failed decompression attempts against the shared allowance", () => { +it("charges failed decompression attempts against the shared allowance", async () => { const allowance = 1024 * 1024; const bytes = Buffer.alloc(allowance); for (const gnu of [false, true]) { @@ -393,13 +444,13 @@ it("charges failed decompression attempts against the shared allowance", () => { flags: gnu ? 0n : 0x800n, })), }); - expect(() => locateDwarfSections(image, allowance)).toThrow( + await expect(locateDwarfSections(image, allowance)).rejects.toThrow( DwarfSectionAllowanceFailure, ); } }); -it("charges valid empty streams against the shared allowance", () => { +it("charges valid empty streams against the shared allowance", async () => { const empty = elfCompressedBody(Buffer.alloc(0), "zlib"); const blocks = Buffer.alloc(5 * 2048); for (let at = 0; at < blocks.length; at += 5) @@ -417,12 +468,12 @@ it("charges valid empty streams against the shared allowance", () => { flags: 0x800n, })), }); - expect(() => locateDwarfSections(image, payloadLength * 2)).toThrow( + await expect(locateDwarfSections(image, payloadLength * 2)).rejects.toThrow( DwarfSectionAllowanceFailure, ); }); -it("charges zero-output streams with bad checksums against the shared allowance", () => { +it("charges zero-output streams with bad checksums against the shared allowance", async () => { const empty = elfCompressedBody(Buffer.alloc(0), "zlib"); // Insert non-final empty stored blocks after the ELF and zlib headers. const blocks = Buffer.alloc(5 * 2048); @@ -442,7 +493,7 @@ it("charges zero-output streams with bad checksums against the shared allowance" flags: 0x800n, })), }); - expect(() => locateDwarfSections(image, payloadLength * 2)).toThrow( + await expect(locateDwarfSections(image, payloadLength * 2)).rejects.toThrow( DwarfSectionAllowanceFailure, ); }); diff --git a/src/native/dwarf/DwarfUnitsProvider.ts b/src/native/dwarf/DwarfUnitsProvider.ts index bd52ad38ad..66cbe6060f 100644 --- a/src/native/dwarf/DwarfUnitsProvider.ts +++ b/src/native/dwarf/DwarfUnitsProvider.ts @@ -7,6 +7,7 @@ import { type StableArtifactSnapshot, ArtifactReaderFailure, readStableArtifact, + yieldArtifactRead, artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, ArtifactResourceScope, @@ -17,6 +18,10 @@ import { } from "../../domain/parseFailureAnalysisError.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../../domain/compression/zstandardFrameAllowance.js"; import { decodeLimit } from "../../domain/parseFailure.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { DWARF_UNITS_PROVIDER } from "../../domain/investigationProviders.js"; @@ -93,7 +98,13 @@ export class DwarfUnitsProvider { snapshot.bytes, input.architecture, ); - const located = locateDwarfSections(selected.bytes); + const located = await locateDwarfSections( + selected.bytes, + undefined, + async () => { + await yieldArtifactRead(options?.signal); + }, + ); const report = dwarfUnitsSchema.parse( buildReport( input, @@ -250,6 +261,8 @@ const dwarfUnitsFailurePrimary = ( input: InspectDwarfUnitsInput, ) => { const { cause, origin } = facts; + if (cause instanceof ZstandardFrameAllowanceError) + return projectZstandardFrameAllowance(OPERATION, cause); if (origin === "owned-or-classified" && cause instanceof AnalysisError) return cause; if (cause instanceof DwarfSectionAllowanceFailure) diff --git a/src/native/elf/ElfDebugLink.test.ts b/src/native/elf/ElfDebugLink.test.ts index c14d7e0048..8a8875a1d5 100644 --- a/src/native/elf/ElfDebugLink.test.ts +++ b/src/native/elf/ElfDebugLink.test.ts @@ -376,7 +376,7 @@ const verify = (image: Buffer, candidate: Buffer, candidatePath: string) => { it("counts legacy .zdebug_info as present candidate debug info", () => { const candidate = elfDebugCandidateFixture({ zdebugInfo: true }); - const sections = readElfImage(candidate).sections; + const sections = readElfImage(candidate, "elf-directory").sections; expect( sections.some( (section) => section.format === "elf" && section.name === ".zdebug_info", @@ -397,7 +397,7 @@ it("counts legacy .zdebug_info as present candidate debug info", () => { /** Point one named section of a little-endian ELF64 candidate past its end. */ const moveSectionPastEnd = (candidate: Buffer, name: string): Buffer => { const damaged = Buffer.from(candidate); - const declared = readElfImage(candidate).sections.find( + const declared = readElfImage(candidate, "elf-directory").sections.find( (section) => section.format === "elf" && section.name === name, ); if (declared?.format !== "elf") throw new Error(`missing ${name}`); diff --git a/src/native/elf/ElfDebugLink.ts b/src/native/elf/ElfDebugLink.ts index a7a26f5cc4..1d586928c0 100644 --- a/src/native/elf/ElfDebugLink.ts +++ b/src/native/elf/ElfDebugLink.ts @@ -1,9 +1,18 @@ import { crc32 } from "node:zlib"; import { isParseLimit } from "../../domain/parseFailureAnalysisError.js"; -import { ArtifactReaderFailure } from "../../resources/index.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + ArtifactReaderFailure, + assertArtifactReadActive, + yieldArtifactRead, +} from "../../resources/index.js"; import { readElfFixedHeader } from "../../domain/elfFixedHeader.js"; import { elfObjectTypeName } from "../../domain/native/elfNotes.js"; +import { + zstandardFrameAllowanceCause, + type ZstandardFrameAllowanceError, +} from "../../domain/compression/zstandardFrameAllowance.js"; import { debugFileNameIsSingleSegment, deriveDebugFileLookupPaths, @@ -26,8 +35,21 @@ import { BinaryFormatFailure, BinaryResourceFailure, } from "../../domain/binaryContainer.js"; -import { readElfImage } from "../../domain/native/elfImage.js"; -import { parseElfNotes } from "./ElfNoteParser.js"; +import { + readElfImage, + readElfImageCooperative, +} from "../../domain/native/elfImage.js"; +import { parseElfNotes, parseElfNotesCooperative } from "./ElfNoteParser.js"; +import type { ZstandardFramePause } from "../../domain/compression/zstandardExactFrame.js"; +import { + sharedElfSectionNameDecoder, + type CooperativeElfSectionNameDecoder, +} from "../../domain/native/elfSectionNameTable.js"; + +import { err, ok, type Result } from "../../domain/result.js"; + +type PreparedCandidateImage = Result, unknown>; +type PreparedNotes = Result, unknown>; type ElfSection = Extract; type Report = ElfDebugLink; @@ -371,21 +393,26 @@ const observeBuildId = ( bytes: Buffer, artifact: Report["artifact"], input: InspectElfDebugLinkInput, + preparedNotes?: PreparedNotes, ): ObservedBuildId => { let notes: ReturnType; try { - notes = parseElfNotes( - bytes, - artifact, - { - path: artifact.path, - max_file_bytes: input.max_file_bytes, - max_entries: input.max_entries, - }, - { maxGnuBuildIdBytes: MAX_ELF_DEBUG_ALT_BUILD_ID_BYTES }, - ); + if (preparedNotes !== undefined && !preparedNotes.ok) + throw preparedNotes.error; + notes = + preparedNotes?.value ?? + parseElfNotes( + bytes, + artifact, + buildIdNoteInput(artifact, input), + BUILD_ID_NOTE_OPTIONS, + ); } catch (cause) { - if (isRethrownBudget(cause)) throw cause; + if ( + isRethrownBudget(cause) || + zstandardFrameAllowanceCause(cause) !== undefined + ) + throw cause; if (cause instanceof ArtifactReaderFailure && cause.reason === "format") return unknownBuildId( "note_tables_malformed", @@ -469,13 +496,32 @@ const sectionsNamed = ( name: string, ): readonly ElfSection[] => sections.filter((section) => section.name === name); +const BUILD_ID_NOTE_OPTIONS = { + maxGnuBuildIdBytes: MAX_ELF_DEBUG_ALT_BUILD_ID_BYTES, +}; +const buildIdNoteInput = ( + artifact: Report["artifact"], + input: InspectElfDebugLinkInput, +) => ({ + path: artifact.path, + max_file_bytes: input.max_file_bytes, + max_entries: input.max_entries, +}); + /** Decode separate-debug linkage from one in-memory ELF image. */ +const ELF_DEBUG_LINK_OPERATION = "inspect_elf_debug_link"; + export const parseElfDebugLink = ( bytes: Buffer, artifact: Report["artifact"], input: InspectElfDebugLinkInput, + image?: ReturnType, + preparedNotes?: PreparedNotes, ): Report => { - const image = readElfImage(bytes, input.max_file_bytes); + // The directory admits section names before either decoder inflates them. + image = + image ?? + readElfImage(bytes, ELF_DEBUG_LINK_OPERATION, input.max_file_bytes); const sections = image.sections.filter( (section): section is ElfSection => section.format === "elf", ); @@ -499,7 +545,7 @@ export const parseElfDebugLink = ( sectionsNamed(sections, ".gnu_debugdata"), namesComplete, ); - const observed = observeBuildId(bytes, artifact, input); + const observed = observeBuildId(bytes, artifact, input, preparedNotes); const limitations: Limitations = [...ELF_DEBUG_LINK_LIMITATIONS]; if (image.section_names_limitation !== undefined) limitations.push(image.section_names_limitation); @@ -579,23 +625,6 @@ const candidateSections = ( }; }; -/** - * The candidate is decoded with the image's budgets. A candidate that exceeds - * them leaves its identity unknown instead of failing the image report. - */ -const observeCandidateBuildId = ( - content: Buffer, - artifact: Report["artifact"], - input: InspectElfDebugLinkInput, -): ObservedBuildId | null => { - try { - return observeBuildId(content, artifact, input); - } catch (cause) { - if (isRethrownBudget(cause)) return null; - throw cause; - } -}; - const hasElfMagic = (bytes: Buffer): boolean => bytes.length >= 4 && bytes.toString("hex", 0, 4) === "7f454c46"; @@ -604,6 +633,8 @@ const readCandidateIdentity = ( totalBytes: number, artifact: Report["artifact"], input: InspectElfDebugLinkInput, + preparedImage?: PreparedCandidateImage, + preparedNotes?: PreparedNotes, ): { readonly format: NonNullable["format"]; readonly byte_order: NonNullable["byte_order"]; @@ -613,6 +644,7 @@ const readCandidateIdentity = ( >; readonly identity: CandidateIdentity; readonly budgetExceeded: boolean; + readonly frameAllowance?: ZstandardFrameAllowanceError; } => { const complete = content.length === totalBytes; const header = readElfFixedHeader(content, totalBytes); @@ -678,13 +710,33 @@ const readCandidateIdentity = ( let image: ReturnType | null = null; let imageMalformed = false; let imageBudgetExceeded = false; + let frameAllowance: ZstandardFrameAllowanceError | undefined; try { - image = readElfImage(content, input.max_file_bytes); + if (preparedImage !== undefined && !preparedImage.ok) + throw preparedImage.error; + image = + preparedImage?.value ?? + readElfImage(content, ELF_DEBUG_LINK_OPERATION, input.max_file_bytes); } catch (cause) { - if (isRethrownBudget(cause)) imageBudgetExceeded = true; - else imageMalformed = true; + frameAllowance = zstandardFrameAllowanceCause(cause); + if (frameAllowance === undefined) { + if (cause instanceof AnalysisError) throw cause; + if (isRethrownBudget(cause)) imageBudgetExceeded = true; + else imageMalformed = true; + } + } + // The candidate may lose named evidence without losing its file identity + // or CRC. If its section names already exceeded the frame allowance, the + // note reader cannot decode that same table either. + let observed: ObservedBuildId | null = null; + if (frameAllowance === undefined) { + try { + observed = observeBuildId(content, artifact, input, preparedNotes); + } catch (cause) { + frameAllowance = zstandardFrameAllowanceCause(cause); + if (frameAllowance === undefined && !isRethrownBudget(cause)) throw cause; + } } - const observed = observeCandidateBuildId(content, artifact, input); const buildId: BuildId = observed?.buildId ?? { status: "unknown", hex: null, @@ -703,7 +755,10 @@ const readCandidateIdentity = ( buildId, candidateHex: buildId.status === "observed" ? buildId.hex : null, }, - budgetExceeded: imageBudgetExceeded || observed === null, + budgetExceeded: + imageBudgetExceeded || + (frameAllowance === undefined && observed === null), + ...(frameAllowance === undefined ? {} : { frameAllowance }), }; }; @@ -838,17 +893,16 @@ export const verifyElfDebugCandidate = ( report: Report, candidate: ElfDebugCandidateFile, input: InspectElfDebugLinkInput, + preparedImage?: PreparedCandidateImage, + preparedNotes?: PreparedNotes, ): Report => { - const identityArtifact = { - path: candidate.path, - sha256: candidate.sha256, - bytes: candidate.bytes, - }; const read = readCandidateIdentity( candidate.prefix, candidate.bytes, - identityArtifact, + candidateArtifact(candidate), input, + preparedImage, + preparedNotes, ); const crc = crcCheck(report.debuglink.crc32_hex, candidate.crc32); const build = buildIdCheck(report.build_id, read.identity); @@ -858,6 +912,10 @@ export const verifyElfDebugCandidate = ( limitations.push(ELF_DEBUG_LINK_CANDIDATE_PREFIX_LIMITATION); if (read.budgetExceeded) limitations.push(ELF_DEBUG_LINK_CANDIDATE_BUDGET_LIMITATION); + if (read.frameAllowance !== undefined) + limitations.push( + `Candidate ELF section names could not be decoded because ${read.frameAllowance.failure.detail}.`, + ); if ( verification.status === "matches" && (crc.status === "unavailable" || build.status === "unavailable") @@ -880,6 +938,124 @@ export const verifyElfDebugCandidate = ( }; }; +const debugLinkPause = (signal?: AbortSignal) => () => + yieldArtifactRead(signal); + +const candidateArtifact = (candidate: ElfDebugCandidateFile) => ({ + path: candidate.path, + sha256: candidate.sha256, + bytes: candidate.bytes, +}); + +const prepareBuildIdNotes = async ( + bytes: Buffer, + artifact: Report["artifact"], + input: InspectElfDebugLinkInput, + pause: ZstandardFramePause, + decode: CooperativeElfSectionNameDecoder, + signal: AbortSignal | undefined, +): Promise => { + assertArtifactReadActive(signal); + try { + return ok( + await parseElfNotesCooperative( + bytes, + artifact, + buildIdNoteInput(artifact, input), + pause, + BUILD_ID_NOTE_OPTIONS, + decode, + ), + ); + } catch (cause) { + assertArtifactReadActive(signal); + return err(cause); + } +}; + +/** Same linkage as {@link parseElfDebugLink}. Section names yield for `signal`. */ +export const parseElfDebugLinkCooperative = async ( + bytes: Buffer, + artifact: Report["artifact"], + input: InspectElfDebugLinkInput, + signal?: AbortSignal, +): Promise => { + const pause = debugLinkPause(signal); + const decode = sharedElfSectionNameDecoder(pause); + const image = await readElfImageCooperative( + bytes, + ELF_DEBUG_LINK_OPERATION, + pause, + input.max_file_bytes, + decode, + ); + const notes = await prepareBuildIdNotes( + bytes, + artifact, + input, + pause, + decode, + signal, + ); + assertArtifactReadActive(signal); + return parseElfDebugLink(bytes, artifact, input, image, notes); +}; + +/** Same comparison as {@link verifyElfDebugCandidate}. Section names yield for `signal`. */ +export const verifyElfDebugCandidateCooperative = async ( + report: Report, + candidate: ElfDebugCandidateFile, + input: InspectElfDebugLinkInput, + signal?: AbortSignal, +): Promise => { + assertArtifactReadActive(signal); + if ( + candidate.prefix.length !== candidate.bytes || + !hasElfMagic(candidate.prefix) + ) + return verifyElfDebugCandidate(report, candidate, input); + const pause = debugLinkPause(signal); + const decode = sharedElfSectionNameDecoder(pause); + let preparedImage: PreparedCandidateImage; + try { + preparedImage = ok( + await readElfImageCooperative( + candidate.prefix, + ELF_DEBUG_LINK_OPERATION, + pause, + input.max_file_bytes, + decode, + ), + ); + } catch (cause) { + assertArtifactReadActive(signal); + preparedImage = err(cause); + } + // A typed image error already determines the candidate outcome. + if ( + !preparedImage.ok && + (preparedImage.error instanceof AnalysisError || + zstandardFrameAllowanceCause(preparedImage.error) !== undefined) + ) + return verifyElfDebugCandidate(report, candidate, input, preparedImage); + const notes = await prepareBuildIdNotes( + candidate.prefix, + candidateArtifact(candidate), + input, + pause, + decode, + signal, + ); + assertArtifactReadActive(signal); + return verifyElfDebugCandidate( + report, + candidate, + input, + preparedImage, + notes, + ); +}; + /** Keep the decoded image linkage when the selected candidate cannot be read. */ export const unreadableElfDebugCandidate = (report: Report): Report => ({ ...report, diff --git a/src/native/elf/ElfDebugLinkProvider.test.ts b/src/native/elf/ElfDebugLinkProvider.test.ts index eb63748990..8c9220695d 100644 --- a/src/native/elf/ElfDebugLinkProvider.test.ts +++ b/src/native/elf/ElfDebugLinkProvider.test.ts @@ -1,3 +1,4 @@ +import { scheduler } from "node:timers/promises"; import { primaryAnalysisError } from "../../domain/analysisErrorBase.js"; import { expect, it, vi } from "vitest"; @@ -11,6 +12,7 @@ import { inspectElfDebugLinkInputSchema, elfDebugLinkSchema, } from "../../domain/native/elfDebugLink.js"; +import { elfFrameAllowanceFixture } from "../../../tests/fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { elfDebugLinkFixture } from "./ElfDebugLink.fixture.js"; const image = elfDebugLinkFixture(); @@ -144,6 +146,52 @@ it("keeps a completed candidate stream's identity when cancellation arrives duri ); }); +it("preserves the completed candidate when its ELF comparison is cancelled", async () => { + const { streamStableArtifact } = await import("../../resources/index.js"); + const prefix = elfFrameAllowanceFixture(); + const sha256 = "cd".repeat(32); + vi.mocked(streamStableArtifact).mockResolvedValueOnce({ + prefix, + sha256, + size: prefix.length, + }); + const provider = new ElfDebugLinkProvider(); + const controller = new AbortController(); + const reason = new Error("candidate comparison cancelled"); + const originalYield = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await originalYield(); + controller.abort(reason); + }); + try { + const result = await provider.inspect( + inspectElfDebugLinkInputSchema.parse({ + path: "/artifacts/prog", + candidate_path: "/artifacts/prog.debug", + }), + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected interrupted comparison"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + }); + expect(retainedReport(result.error)).toMatchObject({ + candidate: { + path: "/artifacts/prog.debug", + sha256, + bytes: prefix.length, + }, + verification: null, + }); + } finally { + pause.mockRestore(); + await provider.close(); + } +}); + it("recovers a completed stream's whole-file size, not its prefix length", () => { expect( completedStableArtifactObservation( diff --git a/src/native/elf/ElfDebugLinkProvider.ts b/src/native/elf/ElfDebugLinkProvider.ts index 8c9850a6e7..b600fc9fde 100644 --- a/src/native/elf/ElfDebugLinkProvider.ts +++ b/src/native/elf/ElfDebugLinkProvider.ts @@ -4,6 +4,7 @@ import { parseLimitConstraint, } from "../../domain/parseFailureAnalysisError.js"; import { decodeLimit } from "../../domain/parseFailure.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { projectArtifactCloseFailure, projectOwnedArtifactIdentityFailure, @@ -41,9 +42,11 @@ import { } from "../../domain/binaryContainer.js"; import { continueGnuDebuglinkCrc32, - parseElfDebugLink, + parseElfDebugLinkCooperative, unreadableElfDebugCandidate, verifyElfDebugCandidate, + verifyElfDebugCandidateCooperative, + type ElfDebugCandidateFile, } from "./ElfDebugLink.js"; import { ELF_NOTE_TEXT_ALLOWANCE } from "./ElfNoteParser.js"; import { elfSectionNameBudget } from "../../domain/native/elfSectionNameTable.js"; @@ -72,6 +75,7 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { async inspect(input: InspectElfDebugLinkInput, options?: ExecutionOptions) { let field: SelectedField = "path"; let candidatePhase: CandidatePhase | undefined; + let candidate: StableArtifactStream | undefined; let crc32 = 0; let snapshot: StableArtifactSnapshot | undefined; try { @@ -84,7 +88,7 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { undefined, this.#resources, ); - const image = parseElfDebugLink( + const image = await parseElfDebugLinkCooperative( snapshot.bytes, { path: input.path, @@ -92,11 +96,11 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { bytes: snapshot.bytes.length, }, input, + options?.signal, ); if (input.candidate_path === undefined) return ok(image); field = "candidate_path"; candidatePhase = { report: image, path: input.candidate_path }; - let candidate; try { // Stream the whole candidate for the debuglink CRC32 and SHA-256. // Only a file within max_file_bytes is retained for ELF decoding; @@ -118,7 +122,15 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { if (!candidateUnreadable(cause)) throw cause; return ok(unreadableElfDebugCandidate(image)); } - return ok(verifiedCandidate(candidatePhase, candidate, crc32, input)); + return ok( + await verifiedCandidate( + candidatePhase, + candidate, + crc32, + input, + options?.signal, + ), + ); }); } catch (cause) { return err( @@ -134,6 +146,7 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { cause, crc32, input, + candidate, ), }, ), @@ -144,45 +157,81 @@ export class ElfDebugLinkProvider implements ElfDebugLinkPort { type CandidatePhase = { readonly report: ElfDebugLink; readonly path: string }; +const candidateFile = ( + phase: CandidatePhase, + candidate: StableArtifactStream, + crc32: number, +): ElfDebugCandidateFile => ({ + path: phase.path, + sha256: candidate.sha256, + bytes: candidate.size, + crc32, + prefix: candidate.prefix, +}); + const verifiedCandidate = ( phase: CandidatePhase, candidate: StableArtifactStream, crc32: number, input: InspectElfDebugLinkInput, -): ElfDebugLink => - verifyElfDebugCandidate( + signal?: AbortSignal, +): Promise => + verifyElfDebugCandidateCooperative( phase.report, - { - path: phase.path, - sha256: candidate.sha256, - bytes: candidate.size, - crc32, - prefix: candidate.prefix, - }, + candidateFile(phase, candidate, crc32), input, + signal, ); -/** - * The decoded image survives a failed candidate phase. A candidate stream that - * completed before cancellation or a failed close keeps its full comparison; - * otherwise only the image linkage is retained, without any comparison. - */ +const unverifiedCandidateObservation = ( + report: ElfDebugLink, + candidate: ElfDebugCandidateFile, +): ElfDebugLink => ({ + ...report, + candidate: { + path: candidate.path, + sha256: candidate.sha256, + bytes: candidate.bytes, + format: null, + byte_order: null, + debug_info: "unknown", + symtab: "unknown", + }, + limitations: [ + ...report.limitations, + "The candidate read completed. Its ELF comparison did not complete.", + ], +}); + +/** Retain the completed stream without restarting an interrupted comparison. */ const candidatePhaseObservation = ( phase: CandidatePhase, cause: unknown, crc32: number, input: InspectElfDebugLinkInput, + candidate: StableArtifactStream | undefined, ): AnalysisPartialObservation => { + if (candidate !== undefined) + return unverifiedCandidateObservation( + phase.report, + candidateFile(phase, candidate, crc32), + ); const completed = completedStableArtifactStream(cause); - return completed === undefined - ? { - ...phase.report, - limitations: [ - ...phase.report.limitations, - ELF_DEBUG_LINK_CANDIDATE_INCOMPLETE_LIMITATION, - ], - } - : verifiedCandidate(phase, completed, crc32, input); + if (completed === undefined) + return { + ...phase.report, + limitations: [ + ...phase.report.limitations, + ELF_DEBUG_LINK_CANDIDATE_INCOMPLETE_LIMITATION, + ], + }; + const file = candidateFile(phase, completed, crc32); + // A stream whose close failed can still supply a complete comparison. + try { + return verifyElfDebugCandidate(phase.report, file, input); + } catch { + return unverifiedCandidateObservation(phase.report, file); + } }; /** Enough retained bytes for an ELF32 or ELF64 fixed header. */ @@ -254,6 +303,8 @@ const elfDebugLinkFailurePrimary = ( field: SelectedField, ) => { const { cause, origin } = facts; + if (origin === "owned-or-classified" && cause instanceof AnalysisError) + return cause; const path = selectedPath(input, field); const sectionNameBudget = elfSectionNameBudget(cause); if (isParseLimit(cause, "elf") && sectionNameBudget !== undefined) diff --git a/src/native/elf/ElfNoteParser.test.ts b/src/native/elf/ElfNoteParser.test.ts index 2fdac63559..c77bf08fc8 100644 --- a/src/native/elf/ElfNoteParser.test.ts +++ b/src/native/elf/ElfNoteParser.test.ts @@ -653,7 +653,7 @@ it.each([ bytes.set([0x41, 0x42, 0xe9, 0], bytes.length - 4); bytes.writeUInt32LE(offset, 64 + 2 * 64); const notes = parse(bytes); - const directory = readElfDirectoryFromBuffer(bytes); + const directory = readElfDirectoryFromBuffer(bytes, "elf-directory"); expect(notes.tables[0]?.name).toBe(name); expect(directory.sections[2]).toMatchObject({ name, name_state: state }); if (name === null) { @@ -675,7 +675,7 @@ it.each([65535, 65536])( bytes[bytes.length - 1] = 0; const name = "A".repeat(terminatorOffset); const notes = parse(bytes); - const directory = readElfDirectoryFromBuffer(bytes); + const directory = readElfDirectoryFromBuffer(bytes, "elf-directory"); expect(notes.tables[0]?.name).toBe(name); expect(notes.coverage.status).toBe("complete"); expect(directory.sections[2]).toMatchObject( @@ -757,14 +757,16 @@ it("checks direct section count encoding in notes and the native directory", () bytes.writeBigUInt64LE(0n, sectionZero + 32); bytes.writeUInt16LE(0xfeff, 60); expect(parse(bytes).tables[0]?.name).toBe(".note.gnu.build-id"); - expect(readElfDirectoryFromBuffer(bytes).sections[1]?.name).toBe( - ".note.gnu.build-id", - ); + expect( + readElfDirectoryFromBuffer(bytes, "elf-directory").sections[1]?.name, + ).toBe(".note.gnu.build-id"); bytes.writeUInt16LE(0xff00, 60); const reason = /e_shnum uses a reserved direct section count/u; expect(() => parse(bytes)).toThrow(reason); - expect(() => readElfDirectoryFromBuffer(bytes)).toThrow(reason); + expect(() => readElfDirectoryFromBuffer(bytes, "elf-directory")).toThrow( + reason, + ); }); const repeatedNotes = (count: number, notes: Buffer): Buffer => { diff --git a/src/native/elf/ElfNoteParser.ts b/src/native/elf/ElfNoteParser.ts index 1297d4f8e1..eceddb837d 100644 --- a/src/native/elf/ElfNoteParser.ts +++ b/src/native/elf/ElfNoteParser.ts @@ -33,9 +33,15 @@ import { import { BinaryFormatFailure } from "../../domain/binaryContainer.js"; import { decodeLimit, ParseFailureError } from "../../domain/parseFailure.js"; import { - decodeElfSectionNameTable, + decodeElfSectionNameRequest, + decodeElfSectionNameRequestCooperative, elfSectionNameBudgetFailure, + type CooperativeElfSectionNameDecoder, + type ElfSectionNameDecodeRequest, } from "../../domain/native/elfSectionNameTable.js"; +import type { ElfCompressedSectionResult } from "../../domain/native/elfCompressedSection.js"; +import type { ZstandardFramePause } from "../../domain/compression/zstandardExactFrame.js"; +import { ZstandardFrameAllowanceError } from "../../domain/compression/zstandardFrameAllowance.js"; import { sha256Hex } from "../../kernel/sha256Hex.js"; import { decodeFdoNote, type FdoDecodeBudget } from "./ElfFdoNote.js"; @@ -697,12 +703,16 @@ const decodeAndroid = ( }; /** Decode PT_NOTE and SHT_NOTE tables from one in-memory ELF image. */ -export const parseElfNotes = ( +const parseElfNoteSteps = function* ( bytes: Buffer, artifact: ElfNotes["artifact"], input: InspectElfNotesInput, options: ElfNoteParseOptions = {}, -): ElfNotes => { +): Generator< + ElfSectionNameDecodeRequest, + ElfNotes, + ElfCompressedSectionResult +> { const fixed = readElfFixedHeader(bytes); if (!fixed.ok) return fail(fixed.error); const header = fixed.value; @@ -780,14 +790,20 @@ export const parseElfNotes = ( Number(namesSection.size), ); const flags = namesSection.flags; - const decoded = decodeElfSectionNameTable( - rawNames, + const decoded = yield { + raw: rawNames, flags, - header.elfClass, - reader.little, - input.max_file_bytes - bytes.length, - ); + bits: header.elfClass, + little: reader.little, + maximumDecodedBytes: input.max_file_bytes - bytes.length, + }; if (decoded.state === "malformed") return fail(decoded.message); + if (decoded.state === "configuration") throw decoded.error; + if (decoded.state === "work_limit") + throw new ZstandardFrameAllowanceError( + { maxFrames: decoded.value, source: decoded.source }, + decoded.offset, + ); if (decoded.state === "limit") throw elfSectionNameBudgetFailure( BigInt(bytes.length) + decoded.requiredBytes, @@ -974,6 +990,52 @@ export const parseElfNotes = ( }; }; +const driveElfNotes = ( + steps: Generator< + ElfSectionNameDecodeRequest, + ElfNotes, + ElfCompressedSectionResult + >, + decode: ( + request: ElfSectionNameDecodeRequest, + ) => ElfCompressedSectionResult | Promise, +): Promise => { + const run = async (): Promise => { + let next = steps.next(); + while (!next.done) next = steps.next(await decode(next.value)); + return next.value; + }; + return run(); +}; + +/** Decode PT_NOTE and SHT_NOTE tables from one in-memory ELF image. */ +export const parseElfNotes = ( + bytes: Buffer, + artifact: ElfNotes["artifact"], + input: InspectElfNotesInput, + options: ElfNoteParseOptions = {}, +): ElfNotes => { + const steps = parseElfNoteSteps(bytes, artifact, input, options); + let next = steps.next(); + while (!next.done) next = steps.next(decodeElfSectionNameRequest(next.value)); + return next.value; +}; + +/** + * Same note decode as {@link parseElfNotes}. Section-name Zstandard frames + * yield through `pause` so an existing caller abort can land between frames. + */ +export const parseElfNotesCooperative = ( + bytes: Buffer, + artifact: ElfNotes["artifact"], + input: InspectElfNotesInput, + pause: ZstandardFramePause, + options: ElfNoteParseOptions = {}, + decode: CooperativeElfSectionNameDecoder = (request) => + decodeElfSectionNameRequestCooperative(request, pause), +): Promise => + driveElfNotes(parseElfNoteSteps(bytes, artifact, input, options), decode); + const toEntry = ( note: ParsedNote, tableKind: "program" | "section", diff --git a/src/native/elf/ElfNotes.fixture.ts b/src/native/elf/ElfNotes.fixture.ts index a8d0628cd0..fe6567fa85 100644 --- a/src/native/elf/ElfNotes.fixture.ts +++ b/src/native/elf/ElfNotes.fixture.ts @@ -321,6 +321,7 @@ export const compressedElfNoteFixture = ( little = true, compressionType = 1, sectionName = ".note.gnu.build-id", + compressedPayload?: Buffer, ): Buffer => { const image = elfNoteFixture({ bits, @@ -349,7 +350,8 @@ export const compressedElfNoteFixture = ( const namesSize = word(namesHeader + (wide ? 32 : 20)); const names = image.subarray(namesOffset, namesOffset + namesSize); const compressed = - compressionType === 2 ? zstdCompressSync(names) : deflateSync(names); + compressedPayload ?? + (compressionType === 2 ? zstdCompressSync(names) : deflateSync(names)); const compressionHeaderBytes = wide ? 24 : 12; const replacement = Buffer.alloc(compressionHeaderBytes); if (little) replacement.writeUInt32LE(compressionType, 0); diff --git a/src/native/elf/ElfNotesProvider.test.ts b/src/native/elf/ElfNotesProvider.test.ts index 52692ccbdb..45669a6a09 100644 --- a/src/native/elf/ElfNotesProvider.test.ts +++ b/src/native/elf/ElfNotesProvider.test.ts @@ -1,15 +1,25 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import { runWithAnalysisOperationSnapshot } from "../../domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../../domain/analysisResourceLimits.js"; import { ElfNotesService } from "../../application/binaryDiagnostics/ElfNotesService.js"; import { + AnalysisCancelledError, AnalysisInputError, AnalysisResourceConstraintError, AnalysisUnsupportedTargetError, } from "../../domain/analysisErrorCore.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { analysisRefusalRemediation } from "../../domain/analysisRefusal.js"; -import { elfNotesSchema } from "../../domain/native/elfNotes.js"; +import { + elfNotesSchema, + inspectElfNotesInputSchema, +} from "../../domain/native/elfNotes.js"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { compressedElfNoteFixture, @@ -20,6 +30,26 @@ import { } from "./ElfNotes.fixture.js"; import { ElfNotesProvider } from "./ElfNotesProvider.js"; +const manyProductiveFrames = (count: number): Buffer => { + const frame = zstdByteFrame(0x61); + const bytes = Buffer.alloc(frame.length * count); + for (let index = 0; index < count; index += 1) + frame.copy(bytes, index * frame.length); + return bytes; +}; + +const abortOnFirstYield = (reason: Error): AbortSignal => { + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + return controller.signal; +}; + const inspect = async ( path: string, input: { readonly max_file_bytes?: number } = {}, @@ -32,6 +62,77 @@ const inspect = async ( } }; +const zstdByteFrame = (byte: number): Buffer => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + +it("projects a Zstandard frame-cap on section names as an adjustable limit", async () => { + const root = await createTestTempDirectory("rea-elf-note-zstd-frames-"); + const path = join(root, "notes.elf"); + await writeFile( + path, + compressedElfNoteFixture( + 64, + true, + 2, + ".note.gnu.build-id", + Buffer.concat([zstdByteFrame(0x61), zstdByteFrame(0x62)]), + ), + ); + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => inspect(path), + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); +}); + +it("cancels a many-frame compressed section-name table", async () => { + const root = await createTestTempDirectory("rea-elf-note-zstd-cancel-"); + const path = join(root, "notes.elf"); + await writeFile( + path, + compressedElfNoteFixture( + 64, + true, + 2, + ".note.gnu.build-id", + manyProductiveFrames(32), + ), + ); + const reason = new Error("stop note frames"); + const provider = new ElfNotesProvider(); + const signal = abortOnFirstYield(reason); + try { + const result = await provider.inspect( + inspectElfNotesInputSchema.parse({ path }), + { signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_elf_notes", + cause: reason, + }); + } finally { + vi.restoreAllMocks(); + await provider.close(); + } +}); + describe("ELF note provider failures", () => { it("projects compressed note sections as unsupported rather than malformed input", async () => { const root = await createTestTempDirectory("rea-compressed-notes-"); diff --git a/src/native/elf/ElfNotesProvider.ts b/src/native/elf/ElfNotesProvider.ts index da82f6619b..9b41f455d0 100644 --- a/src/native/elf/ElfNotesProvider.ts +++ b/src/native/elf/ElfNotesProvider.ts @@ -12,6 +12,7 @@ import { ArtifactReaderFailure, type StableArtifactSnapshot, readStableArtifact, + yieldArtifactRead, artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, ArtifactResourceScope, @@ -20,6 +21,11 @@ import type { ElfNotesPort } from "../../ports/binaryDiagnostics/ElfNotesPort.js import type { ExecutionOptions } from "../../ports/AnalysisProvider.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; import { decodeLimit } from "../../domain/parseFailure.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../../domain/compression/zstandardFrameAllowance.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { MAX_INSPECT_ELF_NOTE_ENTRIES, @@ -28,8 +34,11 @@ import { } from "../../domain/native/elfNotes.js"; import { err, ok } from "../../domain/result.js"; import { ELF_NOTES_PROVIDER } from "../../domain/investigationProviders.js"; -import { ELF_NOTE_TEXT_ALLOWANCE, parseElfNotes } from "./ElfNoteParser.js"; import { elfSectionNameBudget } from "../../domain/native/elfSectionNameTable.js"; +import { + ELF_NOTE_TEXT_ALLOWANCE, + parseElfNotesCooperative, +} from "./ElfNoteParser.js"; const OPERATION = "inspect_elf_notes"; @@ -62,7 +71,7 @@ export class ElfNotesProvider implements ElfNotesPort { this.#resources, ); return ok( - parseElfNotes( + await parseElfNotesCooperative( snapshot.bytes, { path: input.path, @@ -70,6 +79,7 @@ export class ElfNotesProvider implements ElfNotesPort { bytes: snapshot.bytes.length, }, input, + () => yieldArtifactRead(options?.signal), ), ); }); @@ -101,6 +111,9 @@ const elfNotesFailurePrimary = ( input: InspectElfNotesInput, ) => { const { cause, origin } = facts; + if (cause instanceof AnalysisError) return cause; + if (cause instanceof ZstandardFrameAllowanceError) + return projectZstandardFrameAllowance(OPERATION, cause); const sectionNameBudget = elfSectionNameBudget(cause); if (isParseLimit(cause, "elf") && sectionNameBudget !== undefined) return parseLimitConstraint(OPERATION, "memory", cause, { diff --git a/src/native/elf/ElfUnwindRangesProvider.ts b/src/native/elf/ElfUnwindRangesProvider.ts index eba7ece81d..aa935cf442 100644 --- a/src/native/elf/ElfUnwindRangesProvider.ts +++ b/src/native/elf/ElfUnwindRangesProvider.ts @@ -11,6 +11,7 @@ import { projectArtifactCloseFailure, ArtifactReaderFailure, readStableArtifact, + type StableArtifactSnapshot, artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, ArtifactResourceScope, @@ -18,6 +19,7 @@ import { import type { ElfUnwindRangesPort } from "../../ports/binaryDiagnostics/ElfUnwindRangesPort.js"; import type { ExecutionOptions } from "../../ports/AnalysisProvider.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { @@ -34,7 +36,7 @@ import { import { ELF_UNWIND_FORMAT, ELF_UNWIND_RECORD_LIMIT, - parseElfUnwindRanges, + parseElfUnwindRangesCooperative, } from "./ElfUnwindTables.js"; const OPERATION = "inspect_elf_unwind_ranges"; @@ -60,30 +62,25 @@ export class ElfUnwindRangesProvider implements ElfUnwindRangesPort { input: InspectElfUnwindRangesInput, options?: ExecutionOptions, ) { + const signal = options?.signal; + let snapshot: StableArtifactSnapshot | undefined; try { return await this.#resources.run(async () => { - const snapshot = await readStableArtifact( + snapshot = await readStableArtifact( input.path, input.max_file_bytes, - options?.signal, + signal, undefined, this.#resources, ); + const { bytes, sha256 } = snapshot; + const artifact = { path: input.path, sha256, bytes: bytes.length }; return ok( - parseElfUnwindRanges( - snapshot.bytes, - { - path: input.path, - sha256: snapshot.sha256, - bytes: snapshot.bytes.length, - }, - input, - options?.signal, - ), + await parseElfUnwindRangesCooperative(bytes, artifact, input, signal), ); }); } catch (cause) { - return err(elfUnwindFailure(cause, input)); + return err(elfUnwindFailure(cause, input, snapshot)); } } } @@ -103,6 +100,8 @@ const elfUnwindFailurePrimary = ( input: InspectElfUnwindRangesInput, ) => { const { cause, origin } = facts; + if (origin === "owned-or-classified" && cause instanceof AnalysisError) + return cause; if (isParseLimit(cause, ELF_UNWIND_FORMAT)) { const { detail, limit } = cause.failure; const records = limit.name === ELF_UNWIND_RECORD_LIMIT; @@ -184,6 +183,7 @@ const elfUnwindFailurePrimary = ( const elfUnwindFailure = ( cause: unknown, input: InspectElfUnwindRangesInput, + snapshot?: StableArtifactSnapshot, ) => { const primary = elfUnwindFailurePrimary( artifactFilePrimaryFacts(cause), @@ -195,5 +195,6 @@ const elfUnwindFailure = ( input.path, ELF_UNWIND_RANGES_PROVIDER.id, OPERATION, + snapshot, ); }; diff --git a/src/native/elf/ElfUnwindTables.ts b/src/native/elf/ElfUnwindTables.ts index 619c541595..ceb7358184 100644 --- a/src/native/elf/ElfUnwindTables.ts +++ b/src/native/elf/ElfUnwindTables.ts @@ -5,7 +5,10 @@ import { keyedBy, uniqueKeyed, } from "../../domain/keyedCollection.js"; -import { assertArtifactReadActive } from "../../resources/index.js"; +import { + assertArtifactReadActive, + yieldArtifactRead, +} from "../../resources/index.js"; import { decodeLimit } from "../../domain/parseFailure.js"; import { readElfDynamicEntry } from "../../domain/elfDynamicEntry.js"; import { elfSymbolSize, readElfSymbol } from "../../domain/elfSymbol.js"; @@ -27,6 +30,7 @@ import { buildIntervalIndex } from "./ElfIntervalIndex.js"; import { ELF_PT_GNU_EH_FRAME, readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, type ElfProgramDeclaration, type NativeSection, } from "../../domain/native/nativeSectionDirectory.js"; @@ -242,20 +246,56 @@ const placeSection = ( export const ELF_UNWIND_FORMAT = "elf-unwind-ranges"; export const ELF_UNWIND_RECORD_LIMIT = "max_ranges"; +const OPERATION = "inspect_elf_unwind_ranges"; /** Read .eh_frame/.eh_frame_hdr function ranges. Truncation stays incomplete. */ export const parseElfUnwindRanges = ( bytes: Buffer, - artifact: { - readonly path: string; - readonly sha256: string; - readonly bytes: number; - }, + artifact: ElfUnwindRanges["artifact"], input: InspectElfUnwindRangesInput, signal?: AbortSignal, ): ElfUnwindRanges => { assertArtifactReadActive(signal); - const directory = readElfDirectoryFromBuffer(bytes); + return elfUnwindRangesFromDirectory( + bytes, + artifact, + input, + readElfDirectoryFromBuffer(bytes, OPERATION), + signal, + ); +}; + +/** Read the same unwind facts while allowing compressed section names to yield. */ +export const parseElfUnwindRangesCooperative = async ( + bytes: Buffer, + artifact: ElfUnwindRanges["artifact"], + input: InspectElfUnwindRangesInput, + signal?: AbortSignal, +): Promise => { + if (signal === undefined) return parseElfUnwindRanges(bytes, artifact, input); + assertArtifactReadActive(signal); + const directory = await readElfDirectoryFromBufferCooperative( + bytes, + () => yieldArtifactRead(signal), + OPERATION, + ); + return elfUnwindRangesFromDirectory( + bytes, + artifact, + input, + directory, + signal, + ); +}; + +const elfUnwindRangesFromDirectory = ( + bytes: Buffer, + artifact: ElfUnwindRanges["artifact"], + input: InspectElfUnwindRangesInput, + directory: ReturnType, + signal?: AbortSignal, +): ElfUnwindRanges => { + assertArtifactReadActive(signal); const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const little = directory.little; const bits = directory.bits; diff --git a/src/native/elf/KernelModuleParser.ts b/src/native/elf/KernelModuleParser.ts index 672b39017b..7000bfeee2 100644 --- a/src/native/elf/KernelModuleParser.ts +++ b/src/native/elf/KernelModuleParser.ts @@ -1,4 +1,8 @@ -import { ArtifactReaderFailure } from "../../resources/index.js"; +import { + ArtifactReaderFailure, + assertArtifactReadActive, + yieldArtifactRead, +} from "../../resources/index.js"; import { allKeyed, firstKeyed, keyedBy } from "../../domain/keyedCollection.js"; import { elfObjectTypeName } from "../../domain/native/elfNotes.js"; import { @@ -18,7 +22,10 @@ import { type ImageSection, type BinaryContainer, } from "../../domain/binaryContainer.js"; -import { readElfImage } from "../../domain/native/elfImage.js"; +import { + readElfImage, + readElfImageCooperative, +} from "../../domain/native/elfImage.js"; import { kernelCompressionCodecAt } from "./KernelCompressionCodec.js"; import { sha256Hex } from "../../kernel/sha256Hex.js"; @@ -67,11 +74,6 @@ const inFile = (section: ElfSection, fileLength: number): boolean => const rawHex = (bytes: Buffer): string | null => bytes.length > MAX_KERNEL_MODULE_RAW_BYTES ? null : bytes.toString("hex"); -const compressionCodec = (bytes: Buffer): Report["compression"] => { - const codec = kernelCompressionCodecAt(bytes, 0, MODULE_CODECS); - return codec === null ? null : { codec, status: "unsupported" }; -}; - const baseLimitations = (): Limitations => [ "This workflow does not execute the module, resolve imported symbols, or validate an appended signature.", "struct module in .gnu.linkonce.this_module is kernel-version specific and is not decoded.", @@ -124,6 +126,17 @@ const unsupportedCompression = ( ], }); +/** A recognized outer container never enters the ELF image parser. */ +const moduleCompressionReport = ( + bytes: Buffer, + artifact: Report["artifact"], +): Report | null => { + const codec = kernelCompressionCodecAt(bytes, 0, MODULE_CODECS); + return codec === null + ? null + : unsupportedCompression(artifact, { codec, status: "unsupported" }); +}; + const notApplicable = ( artifact: Report["artifact"], facts: ReturnType, @@ -506,16 +519,13 @@ const signatureLimitation = (signature: Report["signature"]): string => { return "The appended signature trailer was not accepted as an unvalidated PKCS#7 range."; }; -/** Decode kernel-module metadata from one in-memory ET_REL image. */ -export const parseKernelModule = ( +/** Decode kernel-module metadata from an admitted in-memory ELF image. */ +const parseKernelModuleImage = ( bytes: Buffer, artifact: Report["artifact"], input: InspectKernelModuleInput, + image: ReturnType, ): Report => { - const compression = compressionCodec(bytes); - if (compression !== null) - return unsupportedCompression(artifact, compression); - const image = readElfImage(bytes); const facts = headerFacts(image); if (facts.type !== 1) return notApplicable(artifact, facts); const sections = image.sections.filter( @@ -663,3 +673,37 @@ export const parseKernelModule = ( limitations, }; }; + +export const parseKernelModule = ( + bytes: Buffer, + artifact: Report["artifact"], + input: InspectKernelModuleInput, +): Report => + moduleCompressionReport(bytes, artifact) ?? + parseKernelModuleImage( + bytes, + artifact, + input, + readElfImage(bytes, "inspect_kernel_module"), + ); + +/** Same report as {@link parseKernelModule}. Section names yield for `signal`. */ +export const parseKernelModuleCooperative = async ( + bytes: Buffer, + artifact: Report["artifact"], + input: InspectKernelModuleInput, + signal?: AbortSignal, +): Promise => { + assertArtifactReadActive(signal); + return ( + moduleCompressionReport(bytes, artifact) ?? + parseKernelModuleImage( + bytes, + artifact, + input, + await readElfImageCooperative(bytes, "inspect_kernel_module", () => + yieldArtifactRead(signal), + ), + ) + ); +}; diff --git a/src/native/elf/KernelModuleProvider.ts b/src/native/elf/KernelModuleProvider.ts index 7b4c51c316..3284616342 100644 --- a/src/native/elf/KernelModuleProvider.ts +++ b/src/native/elf/KernelModuleProvider.ts @@ -18,6 +18,7 @@ import { import type { KernelModulePort } from "../../ports/binaryDiagnostics/KernelModulePort.js"; import type { ExecutionOptions } from "../../ports/AnalysisProvider.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { @@ -31,7 +32,7 @@ import { BinaryResourceFailure, binaryResourceCeiling, } from "../../domain/binaryContainer.js"; -import { parseKernelModule } from "./KernelModuleParser.js"; +import { parseKernelModuleCooperative } from "./KernelModuleParser.js"; const OPERATION = "inspect_kernel_module"; @@ -64,7 +65,7 @@ export class KernelModuleProvider implements KernelModulePort { this.#resources, ); return ok( - parseKernelModule( + await parseKernelModuleCooperative( snapshot.bytes, { path: input.path, @@ -72,6 +73,7 @@ export class KernelModuleProvider implements KernelModulePort { bytes: snapshot.bytes.length, }, input, + options?.signal, ), ); }); @@ -98,6 +100,8 @@ const kernelModuleFailurePrimary = ( input: InspectKernelModuleInput, ) => { const { cause, origin } = facts; + if (origin === "owned-or-classified" && cause instanceof AnalysisError) + return cause; if (cause instanceof BinaryResourceFailure) return parseLimitConstraint( OPERATION, diff --git a/src/native/exports/NativeExportsProvider.ts b/src/native/exports/NativeExportsProvider.ts index 666d34c5eb..f56ff00f96 100644 --- a/src/native/exports/NativeExportsProvider.ts +++ b/src/native/exports/NativeExportsProvider.ts @@ -15,6 +15,7 @@ import { artifactFilePrimaryFacts, type ArtifactFilePrimaryFacts, assertArtifactReadActive, + yieldArtifactRead, ArtifactResourceScope, } from "../../resources/index.js"; import type { NativeExportsPort } from "../../ports/binaryDiagnostics/NativeExportsPort.js"; @@ -39,6 +40,7 @@ import { NATIVE_EXPORT_LIMITS, type NativeExportLimit, parseNativeExports, + parseNativeExportsCooperative, } from "../../domain/native/nativeExportParser.js"; import { errorMessage } from "../../kernel/errorMessage.js"; @@ -62,23 +64,32 @@ export class NativeExportsProvider implements NativeExportsPort { } async inspect(input: InspectNativeExportsInput, options?: ExecutionOptions) { + const signal = options?.signal; let snapshot: StableArtifactSnapshot | undefined; try { return await this.#resources.run(async () => { snapshot = await readStableArtifact( input.path, input.max_file_bytes, - options?.signal, + signal, undefined, this.#resources, ); - assertArtifactReadActive(options?.signal); - const report = parseNativeExports(snapshot.bytes, { + assertArtifactReadActive(signal); + const artifact = { path: input.path, sha256: snapshot.sha256, bytes: snapshot.bytes.length, - }); - assertArtifactReadActive(options?.signal); + }; + const report = + signal === undefined + ? parseNativeExports(snapshot.bytes, artifact) + : await parseNativeExportsCooperative( + snapshot.bytes, + artifact, + () => yieldArtifactRead(signal), + ); + assertArtifactReadActive(signal); return ok(report); }); } catch (cause) { diff --git a/src/native/hardening/BinaryHardeningParsers.test.ts b/src/native/hardening/BinaryHardeningParsers.test.ts index cb42d69894..9f6fdc1d16 100644 --- a/src/native/hardening/BinaryHardeningParsers.test.ts +++ b/src/native/hardening/BinaryHardeningParsers.test.ts @@ -1112,7 +1112,7 @@ it("skips an empty SHT_NOTE section instead of rejecting the image", () => { } expect(sectionAt).toBeGreaterThan(0); bytes.writeBigUInt64LE(0n, sectionAt + 32); - expect(() => readElfImage(bytes)).not.toThrow(); + expect(() => readElfImage(bytes, "elf-directory")).not.toThrow(); const image = readElfHardening(bytes); expect(image.status).toBe("parsed"); expect(findRow(image.rows, "nx").state).toBe("enabled"); @@ -1715,7 +1715,7 @@ it("rejects p_filesz one above p_memsz like the ELF image reader", () => { const load = elfProgramAt(bytes, PT_LOAD); bytes.writeBigUInt64LE(bytes.readBigUInt64LE(load + 32) - 1n, load + 40); const message = "ELF load segment sizes or addresses are invalid"; - expect(() => readElfImage(bytes)).toThrow(message); + expect(() => readElfImage(bytes, "elf-directory")).toThrow(message); expect(() => readElfHardening(bytes)).toThrow( expect.objectContaining({ kind: "malformed", message }), ); diff --git a/src/native/hardening/BinaryHardeningProvider.ts b/src/native/hardening/BinaryHardeningProvider.ts index ac1bdb3a36..df354be7f8 100644 --- a/src/native/hardening/BinaryHardeningProvider.ts +++ b/src/native/hardening/BinaryHardeningProvider.ts @@ -35,7 +35,7 @@ import { BinaryResourceFailure, binaryResourceCeiling, } from "../../domain/binaryContainer.js"; -import { readElfHardening } from "./ElfHardening.js"; +import { readElfHardeningCooperative } from "./ElfHardening.js"; import { readMachoHardening } from "./MachoHardening.js"; import { readPeHardening } from "./PeHardening.js"; import { errorMessage } from "../../kernel/errorMessage.js"; @@ -109,7 +109,7 @@ export const readBinaryHardening = async ( return { artifact, format: "elf", - images: [readElfHardening(bytes, signal)], + images: [await readElfHardeningCooperative(bytes, signal)], limitations: [...BINARY_HARDENING_LIMITATIONS], }; if (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) diff --git a/src/native/hardening/ElfHardening.ts b/src/native/hardening/ElfHardening.ts index 49870b9887..e9ce543272 100644 --- a/src/native/hardening/ElfHardening.ts +++ b/src/native/hardening/ElfHardening.ts @@ -2,6 +2,7 @@ import { isParseLimit } from "../../domain/parseFailureAnalysisError.js"; import { assertArtifactReadActive, ArtifactReaderFailure, + yieldArtifactRead, } from "../../resources/index.js"; import { firstKeyed, @@ -68,12 +69,14 @@ import { ELF_PT_NOTE, assertElfDeclaredRangesFromBuffer, readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, type ElfProgramDeclaration, type NativeSection, } from "../../domain/native/nativeSectionDirectory.js"; const ET_EXEC = 2; const ET_DYN = 3; +const OPERATION = "inspect_binary_hardening"; const SHT_SYMTAB = 2; const SHT_STRTAB = 3; const SHT_DYNSYM = 11; @@ -184,7 +187,34 @@ export const readElfHardening = ( signal?: AbortSignal, ): BinaryHardeningImage => { assertArtifactReadActive(signal); - const directory = readElfDirectoryFromBuffer(bytes); + return hardeningFromDirectory( + bytes, + readElfDirectoryFromBuffer(bytes, OPERATION), + signal, + ); +}; + +/** A signal-owning inspection can pause while decoding ELF section names. */ +export const readElfHardeningCooperative = async ( + bytes: Buffer, + signal?: AbortSignal, +): Promise => { + if (signal === undefined) return readElfHardening(bytes); + assertArtifactReadActive(signal); + const directory = await readElfDirectoryFromBufferCooperative( + bytes, + () => yieldArtifactRead(signal), + OPERATION, + ); + assertArtifactReadActive(signal); + return hardeningFromDirectory(bytes, directory, signal); +}; + +const hardeningFromDirectory = ( + bytes: Buffer, + directory: ReturnType, + signal?: AbortSignal, +): BinaryHardeningImage => { // A report needs every declared payload, not only the tables that name them. assertElfDeclaredRangesFromBuffer(bytes); const { bits, little, machine, programs, sections } = directory; diff --git a/src/native/initializers/ElfInitializers.ts b/src/native/initializers/ElfInitializers.ts index 91bf66abfe..4273c34a04 100644 --- a/src/native/initializers/ElfInitializers.ts +++ b/src/native/initializers/ElfInitializers.ts @@ -1,5 +1,8 @@ import { setImmediate as immediate } from "node:timers/promises"; -import { assertArtifactReadActive } from "../../resources/index.js"; +import { + assertArtifactReadActive, + yieldArtifactRead, +} from "../../resources/index.js"; import { joinCoverageStatus, type CoverageStatus, @@ -21,7 +24,10 @@ import { elfRelativeRelocationType, readElfRelocationWritesSteps, } from "../../domain/native/elfPointerRelocations.js"; -import { readElfImage } from "../../domain/native/elfImage.js"; +import { + readElfImage, + readElfImageFromDirectory, +} from "../../domain/native/elfImage.js"; import { BinaryFormatFailure, BinaryMappingFailure, @@ -30,7 +36,10 @@ import { mappedFileOffset, type FileMappingIndex, } from "../../domain/binaryContainer.js"; -import { readElfDirectoryFromBuffer } from "../../domain/native/nativeSectionDirectory.js"; +import { + readElfDirectoryFromBuffer, + readElfDirectoryFromBufferCooperative, +} from "../../domain/native/nativeSectionDirectory.js"; import type { NativeSection } from "../../domain/native/nativeSectionDirectory.js"; import { loadElfSymbolIndex, @@ -972,8 +981,17 @@ export const readElfInitializers = async ( signal?: AbortSignal, ): Promise => { assertArtifactReadActive(signal); - const image = readElfImage(bytes); - const directory = readElfDirectoryFromBuffer(bytes); + const operation = "inspect_native_initializers"; + const directory = + signal === undefined + ? readElfDirectoryFromBuffer(bytes, operation) + : await readElfDirectoryFromBufferCooperative( + bytes, + () => yieldArtifactRead(signal), + operation, + ); + assertArtifactReadActive(signal); + const image = readElfImageFromDirectory(bytes, directory); const scan: ElfScan = { bytes, bits: image.bits, diff --git a/src/native/initializers/NativeInitializersParsers.test.ts b/src/native/initializers/NativeInitializersParsers.test.ts index f90ce1e15e..d08acc725e 100644 --- a/src/native/initializers/NativeInitializersParsers.test.ts +++ b/src/native/initializers/NativeInitializersParsers.test.ts @@ -36,7 +36,11 @@ vi.mock("../../domain/binaryContainer.js", async (importOriginal) => { }; }); /** Element reads of every parsed ELF section table, so tests can count section walks. */ -const sectionReads = vi.hoisted(() => ({ count: 0, budget: Infinity })); +const sectionReads = vi.hoisted(() => ({ + count: 0, + budget: Infinity, + directories: 0, +})); vi.mock( "../../domain/native/nativeSectionDirectory.js", async (importOriginal) => { @@ -49,6 +53,7 @@ vi.mock( readElfDirectoryFromBuffer: ( ...args: Parameters ) => { + sectionReads.directories += 1; const directory = actual.readElfDirectoryFromBuffer(...args); return { ...directory, @@ -138,6 +143,13 @@ it.each([ }, ); +it("uses one ELF directory for initializer mappings and section facts", async () => { + sectionReads.directories = 0; + const report = await read(elf64PieInitializers()); + expect(report.entries.length).toBeGreaterThan(0); + expect(sectionReads.directories).toBe(1); +}); + it("preserves a real mapping to file offset zero instead of treating zero as missing", async () => { const bytes = elfUnmappedInitializerTargets({ type: 3 }); bytes.writeBigUInt64LE(0n, 64 + 16); diff --git a/src/native/initializers/NativeInitializersProvider.ts b/src/native/initializers/NativeInitializersProvider.ts index af9227cfa7..04f9680d62 100644 --- a/src/native/initializers/NativeInitializersProvider.ts +++ b/src/native/initializers/NativeInitializersProvider.ts @@ -15,6 +15,7 @@ import { artifactFilePrimaryFacts, assertArtifactReadActive, type ArtifactFilePrimaryFacts, + type StableArtifactSnapshot, ArtifactResourceScope, } from "../../resources/index.js"; import { @@ -77,9 +78,10 @@ export class NativeInitializersProvider implements NativeInitializersPort { input: InspectNativeInitializersInput, options?: ExecutionOptions, ) { + let snapshot: StableArtifactSnapshot | undefined; try { return await this.#resources.run(async () => { - const snapshot = await readStableArtifact( + snapshot = await readStableArtifact( input.path, input.max_file_bytes, options?.signal, @@ -101,7 +103,7 @@ export class NativeInitializersProvider implements NativeInitializersPort { return ok(report); }); } catch (cause) { - return err(nativeInitializersFailure(cause, input)); + return err(nativeInitializersFailure(cause, input, snapshot)); } } } @@ -264,6 +266,7 @@ const failurePrimary = ( const nativeInitializersFailure = ( cause: unknown, input: InspectNativeInitializersInput, + snapshot?: StableArtifactSnapshot, ): AnalysisError => projectStableArtifactFailure( cause, @@ -271,4 +274,5 @@ const nativeInitializersFailure = ( input.path, NATIVE_INITIALIZERS_PROVIDER.id, OPERATION, + snapshot, ); diff --git a/src/native/metadata/BinaryContainer.test.ts b/src/native/metadata/BinaryContainer.test.ts index 31c78ad036..e720bcfcb6 100644 --- a/src/native/metadata/BinaryContainer.test.ts +++ b/src/native/metadata/BinaryContainer.test.ts @@ -325,7 +325,7 @@ it.each([ ({ magic, bits, headerSize }) => { const bytes = Buffer.alloc(4); bytes.writeUInt32BE(magic); - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( new BinaryFormatFailure( "malformed", `truncated Mach-O header: a ${bits}-bit header needs ${headerSize} bytes; the file has 4`, @@ -338,14 +338,14 @@ it("keeps a JVM-sized FAT count unconfirmed while rejecting malformed zero-count const bytes = Buffer.alloc(8); bytes.writeUInt32BE(0xcafebabe); bytes.writeUInt32BE(0x0000003d, 4); // JVM major version 61; no FAT table follows. - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( new BinaryFormatFailure( "unsupported", "Only ELF, PE, and thin Mach-O native images are supported", ), ); bytes.writeUInt32BE(0, 4); - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( new BinaryFormatFailure( "malformed", "FAT header declares no architectures", @@ -376,10 +376,12 @@ const elfWithLoadSegment = (fileSize: bigint): Buffer => { }; it("names a load segment that runs past the end of the file as truncated", () => { - expect(readBinaryContainer(elfWithLoadSegment(120n)).mappings).toHaveLength( - 1, - ); - expect(() => readBinaryContainer(elfWithLoadSegment(0x1000n))).toThrow( + expect( + readBinaryContainer(elfWithLoadSegment(120n), "elf-directory").mappings, + ).toHaveLength(1); + expect(() => + readBinaryContainer(elfWithLoadSegment(0x1000n), "elf-directory"), + ).toThrow( expect.objectContaining({ failure: { kind: "truncated", diff --git a/src/native/metadata/Il2cppContainer.test.ts b/src/native/metadata/Il2cppContainer.test.ts index 25f3045eaa..dc5f1eb835 100644 --- a/src/native/metadata/Il2cppContainer.test.ts +++ b/src/native/metadata/Il2cppContainer.test.ts @@ -15,8 +15,8 @@ it.each(["unmapped", "zero-fill", "duplicate", "out-of-file"] as const)( expect(() => readGoBinaryImage(goNamed)).toThrow(/build-info|mapped/i); expect(readGoBinaryImage(control).build_info).toBeNull(); - expect(readBinaryContainer(goNamed).load_regions).toEqual( - readBinaryContainer(control).load_regions, + expect(readBinaryContainer(goNamed, "elf-directory").load_regions).toEqual( + readBinaryContainer(control, "elf-directory").load_regions, ); const image = openImageSteps(goNamed).next(); diff --git a/src/native/metadata/NativeSectionDirectory.test.ts b/src/native/metadata/NativeSectionDirectory.test.ts index 91c2d48f18..002f02228d 100644 --- a/src/native/metadata/NativeSectionDirectory.test.ts +++ b/src/native/metadata/NativeSectionDirectory.test.ts @@ -59,6 +59,7 @@ it("does not classify an unconfirmed pointer overlapping DOS data as a PE layout it("admits PT_INTERP, PT_GNU_STACK and PT_GNU_RELRO program headers", () => { const directory = readElfDirectoryFromBuffer( elfHardeningFixture({ interp: true, gnuRelro: true }), + "elf-directory", ); expect(directory.elf_type).toBe(3); expect(directory.programs.map((program) => program.type)).toEqual( @@ -74,7 +75,7 @@ it("admits PT_INTERP, PT_GNU_STACK and PT_GNU_RELRO program headers", () => { it("retains PT_GNU_EH_FRAME with its file and address span", () => { const bytes = elfUnwindSectionlessFixture(); - const directory = readElfDirectoryFromBuffer(bytes); + const directory = readElfDirectoryFromBuffer(bytes, "elf-directory"); expect(directory.programs.map((program) => program.type)).toEqual([ 1, ELF_PT_GNU_EH_FRAME, @@ -110,9 +111,12 @@ it.each([ "retains ELF%i little=%s compression independently of stored section coverage", (bits, little) => { const bytes = compressedElfNoteSectionFixture(bits, little); - const initial = readElfDirectoryFromBuffer(bytes).sections[1]!; + const initial = readElfDirectoryFromBuffer(bytes, "elf-directory") + .sections[1]!; expect(initial).toMatchObject({ compressed: true, file_backed: true }); - expect(readElfDirectoryFromBuffer(bytes).sections[2]).toMatchObject({ + expect( + readElfDirectoryFromBuffer(bytes, "elf-directory").sections[2], + ).toMatchObject({ compressed: false, file_backed: true, }); @@ -124,7 +128,9 @@ it.each([ else bytes.writeBigUInt64BE(BigInt(bytes.length), sizeAt); } else if (little) bytes.writeUInt32LE(bytes.length, sizeAt); else bytes.writeUInt32BE(bytes.length, sizeAt); - expect(readElfDirectoryFromBuffer(bytes).sections[1]).toMatchObject({ + expect( + readElfDirectoryFromBuffer(bytes, "elf-directory").sections[1], + ).toMatchObject({ compressed: true, file_backed: false, offset: initial.offset, diff --git a/src/native/metadata/PeImage.test.ts b/src/native/metadata/PeImage.test.ts index 19a1049337..7a789dc8d2 100644 --- a/src/native/metadata/PeImage.test.ts +++ b/src/native/metadata/PeImage.test.ts @@ -68,7 +68,7 @@ it.each(["NE", "LE", "LX"])( (signature) => { const bytes = pe(0x014c).subarray(0, 66); bytes.write(signature, 64, "ascii"); - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( expect.objectContaining({ kind: "unsupported", message: expect.stringContaining(signature), diff --git a/src/resources/ArtifactFileOperation.ts b/src/resources/ArtifactFileOperation.ts index 7463441995..dce0fb61e2 100644 --- a/src/resources/ArtifactFileOperation.ts +++ b/src/resources/ArtifactFileOperation.ts @@ -1,4 +1,5 @@ import type { FileHandle } from "node:fs/promises"; +import { scheduler } from "node:timers/promises"; import { RegularFileOperationCancelled, type StableRegularFileDescriptor, @@ -245,6 +246,17 @@ export const assertArtifactReadActive = ( }); }; +/** + * Let a queued abort land, then report it. Synchronous multi-frame decoding + * cannot observe the signal until the caller yields. + */ +export const yieldArtifactRead = async ( + signal: AbortSignal | undefined, +): Promise => { + await scheduler.yield(); + assertArtifactReadActive(signal); +}; + /** * Project a failure raised by a domain byte parser over artifact bytes onto the * reader's reasons: the caller's cancellation (`signal.reason`) becomes diff --git a/src/resources/ArtifactOperationFailure.test.ts b/src/resources/ArtifactOperationFailure.test.ts index b973a765bf..7fe6ec4590 100644 --- a/src/resources/ArtifactOperationFailure.test.ts +++ b/src/resources/ArtifactOperationFailure.test.ts @@ -1,4 +1,9 @@ import { expect, it } from "vitest"; +import { ZstandardFrameAllowanceError } from "../domain/compression/zstandardFrameAllowance.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../domain/analysisResourceLimits.js"; import { createHash } from "node:crypto"; import { @@ -29,6 +34,26 @@ import { unsupportedInput, } from "../domain/parseFailure.js"; +it("projects a Zstandard frame-cap as an adjustable artifact limit", () => { + expect( + translateArtifactFailure( + "extract_artifact", + new ZstandardFrameAllowanceError({ maxFrames: 4, source: "runtime" }, 10), + ), + ).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 4, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); +}); + it.each([ malformedInput("pe", "bad field"), truncatedInput("pe", "short field"), diff --git a/src/resources/ArtifactOperationFailure.ts b/src/resources/ArtifactOperationFailure.ts index b4991900dc..9a60bc216b 100644 --- a/src/resources/ArtifactOperationFailure.ts +++ b/src/resources/ArtifactOperationFailure.ts @@ -1,4 +1,8 @@ import { analysisSettingLimit } from "../domain/analysisSettingDefinitions.js"; +import { + projectZstandardFrameAllowance, + ZstandardFrameAllowanceError, +} from "../domain/compression/zstandardFrameAllowance.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { ArtifactOperationError } from "../domain/artifactOperationError.js"; import { InternalInvariantError } from "../domain/internalInvariant.js"; @@ -18,6 +22,8 @@ export const translateArtifactFailure = ( cause: unknown, ): AnalysisError => { if (cause instanceof InternalInvariantError) throw cause; + if (cause instanceof ZstandardFrameAllowanceError) + return projectZstandardFrameAllowance(operation, cause); if (cause instanceof ArtifactReaderFailure) { const error = new ArtifactOperationError( operation, diff --git a/src/resources/ZipArtifactReader.ts b/src/resources/ZipArtifactReader.ts index 37cfe1fab7..c5f048ea5b 100644 --- a/src/resources/ZipArtifactReader.ts +++ b/src/resources/ZipArtifactReader.ts @@ -13,7 +13,7 @@ import { import { ArtifactFileSource } from "./ArtifactFileSource.js"; import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js"; import type { FileHandle } from "node:fs/promises"; -import { Duplex, PassThrough, type Readable } from "node:stream"; +import { compose, Duplex, PassThrough, type Readable } from "node:stream"; import { once } from "node:events"; import { constants, createZstdDecompress } from "node:zlib"; @@ -56,6 +56,10 @@ import { readFileHandleRange } from "../filesystem/readFileHandleRange.js"; import { strictUtf8Decoder } from "../domain/strictUtf8.js"; import { decodeZipCp437 } from "../domain/zip/zipEntryText.js"; import { invariantViolation } from "../domain/internalInvariant.js"; +import { + ZstandardFramingError, + zstandardFrameChunks, +} from "../domain/compression/zstandardFrameChunks.js"; const ZIP_UTF8_DECODER = strictUtf8Decoder("preserve"); @@ -83,13 +87,16 @@ class ZipZstandardDecompressionStream { constructor(_format: string, options?: DecompressionStreamOptions) { const stream = Duplex.toWeb( - createZstdDecompress({ - chunkSize: options?.chunkSize, - // Match the existing ELF/GPU Zstandard decoder's 128 MiB window cap. - params: { - [constants.ZSTD_d_windowLogMax]: ZIP_ZSTANDARD_WINDOW_LOG_MAX, - }, - }), + compose( + new PassThrough(), + zstandardFrameChunks, + createZstdDecompress({ + chunkSize: options?.chunkSize, + params: { + [constants.ZSTD_d_windowLogMax]: ZIP_ZSTANDARD_WINDOW_LOG_MAX, + }, + }), + ), ); this.readable = stream.readable; this.writable = stream.writable; @@ -648,7 +655,8 @@ export const typedZipFailure = ( }); const invalidZstandard = compressionMethod === 93 && - (code === "ZSTD_error_prefix_unknown" || + (cause instanceof ZstandardFramingError || + code === "ZSTD_error_prefix_unknown" || code === "ZSTD_error_checksum_wrong" || code === "ZSTD_error_corruption_detected" || code === "ZSTD_error_dictionary_corrupted" || diff --git a/src/resources/index.ts b/src/resources/index.ts index c8963a4af6..e2c2cbce60 100644 --- a/src/resources/index.ts +++ b/src/resources/index.ts @@ -37,6 +37,7 @@ export { artifactReadCancelledBy, artifactSelectedPathErrno, assertArtifactReadActive, + yieldArtifactRead, bindArtifactFileCleanup, mapArtifactFileFailure, withArtifactFileHandle, diff --git a/src/rust/RustBinaryImage.ts b/src/rust/RustBinaryImage.ts index 73d93729f8..23a83f005a 100644 --- a/src/rust/RustBinaryImage.ts +++ b/src/rust/RustBinaryImage.ts @@ -1,7 +1,11 @@ import type { ImageSection } from "../domain/binaryContainer.js"; import { malformedInput, unsupportedInput } from "../domain/parseFailure.js"; import { keyedBy, uniqueKeyed } from "../domain/keyedCollection.js"; -import { readBinaryContainer } from "../domain/native/readBinaryContainer.js"; +import type { ZstandardFramePause } from "../domain/compression/zstandardExactFrame.js"; +import { + readBinaryContainer, + readBinaryContainerCooperative, +} from "../domain/native/readBinaryContainer.js"; import type { RustAuditableMetadata, RustBinary, @@ -59,9 +63,44 @@ export const readRustBinaryImage = ( bytes: Buffer, maxRustcHashes = DEFAULT_RUSTC_HASH_LIMIT, maxOutputBytes: number = RUST_PRODUCER_DEFAULTS.output_bytes, +): RustBinaryImage => + finishRustBinaryImage( + bytes, + maxRustcHashes, + maxOutputBytes, + hasWasmPreamble(bytes) ? wasmLayout(bytes) : nativeLayout(bytes), + ); + +/** Same image as {@link readRustBinaryImage}. ELF section names yield through `pause`. */ +export const readRustBinaryImageCooperative = async ( + bytes: Buffer, + maxRustcHashes: number | undefined, + maxOutputBytes: number | undefined, + pause: ZstandardFramePause, +): Promise => + finishRustBinaryImage( + bytes, + maxRustcHashes ?? DEFAULT_RUSTC_HASH_LIMIT, + maxOutputBytes ?? RUST_PRODUCER_DEFAULTS.output_bytes, + hasWasmPreamble(bytes) + ? wasmLayout(bytes) + : nativeLayout( + bytes, + await readBinaryContainerCooperative( + bytes, + "inspect_rust_binary", + pause, + ), + ), + ); + +const finishRustBinaryImage = ( + bytes: Buffer, + maxRustcHashes: number, + maxOutputBytes: number, + layout: RustImageLayout, ): RustBinaryImage => { - const { section_name, dependency, compiler_strings, ...identity } = - hasWasmPreamble(bytes) ? wasmLayout(bytes) : nativeLayout(bytes); + const { section_name, dependency, compiler_strings, ...identity } = layout; return { ...identity, auditable_metadata: @@ -85,8 +124,10 @@ export const readRustBinaryImage = ( }; }; -const nativeLayout = (bytes: Buffer): RustImageLayout => { - const container = readBinaryContainer(bytes); +const nativeLayout = ( + bytes: Buffer, + container = readBinaryContainer(bytes, "inspect_rust_binary"), +): RustImageLayout => { const section_name = rustDependencySectionName(container.format); const dependency = dependencySection(section_name, container.sections); return { diff --git a/src/rust/RustBinaryProvider.ts b/src/rust/RustBinaryProvider.ts index 64a487e450..11df5765b2 100644 --- a/src/rust/RustBinaryProvider.ts +++ b/src/rust/RustBinaryProvider.ts @@ -13,6 +13,7 @@ import { type ArtifactFilePrimaryFacts, artifactFileCancellationCause, assertArtifactReadActive, + yieldArtifactRead, ArtifactResourceScope, ArtifactReaderFailure, } from "../resources/index.js"; @@ -37,6 +38,7 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; import { readRustBinaryImage, + readRustBinaryImageCooperative, type RustBinaryImage, } from "./RustBinaryImage.js"; import { requireSelectedNativeImage } from "../domain/apple/machoSliceSelection.js"; @@ -121,11 +123,18 @@ export class RustBinaryProvider implements RustBinaryPort { input.architecture, ); const image = inFile( - readRustBinaryImage( - selected.bytes, - input.max_rustc_hashes, - limits.output_bytes, - ), + options?.signal === undefined + ? readRustBinaryImage( + selected.bytes, + input.max_rustc_hashes, + limits.output_bytes, + ) + : await readRustBinaryImageCooperative( + selected.bytes, + input.max_rustc_hashes, + limits.output_bytes, + () => yieldArtifactRead(options.signal), + ), selected.fileOffset, ); const report: RustBinary = { diff --git a/tests/acceptance/nativeElfFrameAllowanceMcp.test.ts b/tests/acceptance/nativeElfFrameAllowanceMcp.test.ts new file mode 100644 index 0000000000..997332309a --- /dev/null +++ b/tests/acceptance/nativeElfFrameAllowanceMcp.test.ts @@ -0,0 +1,103 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { elfNoteFixture } from "../../src/native/elf/ElfNotes.fixture.js"; +import { INLINE_EVIDENCE_MCP_ENVIRONMENT } from "../../scripts/lib/mcp-verification-environment.mjs"; +import { elfFrameAllowanceFixture } from "../fixtures/binaryDiagnostics/elfFrameAllowance.js"; +import { createIl2cppMetadataFixture } from "../fixtures/il2cpp/metadata.js"; +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +it("reports an ELF section-name frame refusal through each native diagnostic", async () => { + const root = await createTestTempDirectory("rea-native-elf-frame-cap-"); + const path = join(root, "image.elf"); + const primary = join(root, "primary.elf"); + const metadata = join(root, "global-metadata.dat"); + const bytes = elfFrameAllowanceFixture(); + await writeFile(path, bytes); + await writeFile(primary, elfNoteFixture()); + await writeFile(metadata, createIl2cppMetadataFixture().bytes); + const { call } = await connectLocalToolsMcp(INLINE_EVIDENCE_MCP_ENVIRONMENT); + const operations = [ + "inspect_binary_hardening", + "inspect_elf_debug_link", + "inspect_elf_unwind_ranges", + "inspect_kernel_module", + "inspect_native_exports", + "inspect_native_initializers", + "inspect_go_binary", + "inspect_rust_binary", + ] as const; + const setAllowance = async (value: number) => { + const result = await call("update_analysis_settings", { + changes: [{ setting: "zstd.max_frames", action: "set", value }], + }); + expect(result.isError).not.toBe(true); + }; + + await setAllowance(1); + for (const operation of [...operations, "inspect_il2cpp_metadata"]) { + const settings = await call("inspect_analysis_settings", { operation }); + expect(settings.structuredContent, operation).toMatchObject({ + settings: expect.arrayContaining([ + expect.objectContaining({ setting: "zstd.max_frames" }), + ]), + }); + const result = await call( + operation, + operation === "inspect_il2cpp_metadata" + ? { path: metadata, binary_path: path } + : { path }, + ); + expect(parseMcpToolError(result).error, operation).toMatchObject({ + code: "resource_constraint", + details: { + operation, + resource: "cpu", + reported_limits: { + boundary: "zstd.max_frames", + maximum: 1, + required: 2, + }, + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + unit: "count", + }, + }, + }); + } + + const candidate = await call("inspect_elf_debug_link", { + path: primary, + candidate_path: path, + }); + expect(candidate.isError).not.toBe(true); + expect(candidate.structuredContent).toMatchObject({ + normalized_result: { + candidate: { + path, + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + format: "elf64", + debug_info: "unknown", + symtab: "unknown", + }, + limitations: expect.arrayContaining([ + expect.stringContaining("zstd.max_frames"), + ]), + }, + }); + + await setAllowance(2); + for (const operation of operations) { + const result = await call(operation, { path }); + expect( + result.isError, + `${operation}: ${JSON.stringify(result.content)}`, + ).not.toBe(true); + } +}); diff --git a/tests/boundary/android/androidJniLibraries.test.ts b/tests/boundary/android/androidJniLibraries.test.ts index 3f58fb60ea..4e8ee18fa5 100644 --- a/tests/boundary/android/androidJniLibraries.test.ts +++ b/tests/boundary/android/androidJniLibraries.test.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { scheduler } from "node:timers/promises"; import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { expect, it, onTestFinished, vi } from "vitest"; import { Ajv2020 } from "ajv/dist/2020.js"; @@ -31,6 +32,8 @@ import { zstandardZipFixture, } from "../../fixtures/zipZstandard.js"; import { jniElfFixture } from "../../fixtures/android/jniElf.js"; +import { runWithAnalysisOperationSnapshot } from "../../../src/domain/analysisOperationSnapshot.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { ArtifactReaderFailure, ZipArtifactReader, @@ -240,6 +243,58 @@ it("keeps the ZIP decoder window's recompression advice in the library failure", }); }); +it("retains a backed library identity when the ELF section-name frame budget is refused", async () => { + const bytes = elfFrameAllowanceFixture(); + const library = "lib/x86_64/libframe.so"; + const source = await fixture([ + [library, bytes], + [LIBRARY, jniElfFixture().bytes], + ]); + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => read(source), + ); + expect(result.libraries).toEqual([ + expect.objectContaining({ + path: library, + size: bytes.length, + sha256: digest(bytes), + facts: null, + failure: { + kind: "capacity", + message: expect.stringContaining("zstd.max_frames"), + }, + }), + expect.objectContaining({ + path: LIBRARY, + facts: expect.any(Object), + failure: null, + }), + ]); + expect(source.resources.hasPending).toBe(false); + + const checkpoints: AndroidJniLibraryCheckpoint[] = []; + const close = vi + .spyOn(ZipArtifactReader.prototype, "close") + .mockRejectedValueOnce(new Error("controlled ZIP close failure")); + try { + await expect( + runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => + read(source, {}, undefined, undefined, (checkpoint) => + checkpoints.push(checkpoint), + ), + ), + ).rejects.toBeInstanceOf(ProviderCleanupError); + expect(checkpoints[0]?.libraries).toEqual(result.libraries); + expect(source.resources.hasPending).toBe(true); + } finally { + close.mockRestore(); + await source.resources.close(); + } +}); + it("preserves declared capacity metadata without presenting unread bytes as verified", async () => { const source = await fixture(); const originalEntries = ZipArtifactReader.prototype.entries; @@ -560,82 +615,109 @@ for (const first of ["caller", "deadline"] as const) } }); -it("keeps completed facts and the interrupted carrier when a later library is cancelled", async () => { - const second = "lib/arm64-v8a/libsecond.so", - third = "lib/arm64-v8a/libunexamined.so"; - const bytes = jniElfFixture().bytes; - const source = await fixture([ - [LIBRARY, bytes], - [second, bytes], - [third, bytes], - ]); - const controller = new AbortController(), - reason = { caller: "stop before second member bytes" }; - const originalOpen = ZipArtifactReader.prototype.open; - const open = vi - .spyOn(ZipArtifactReader.prototype, "open") - .mockImplementation(async function ( - this: ZipArtifactReader, - entry, - signal, - ) { - if (entry.path === second) { - controller.abort(reason); - throw new JadxInterruption(reason); - } - return originalOpen.call(this, entry, signal); - }); - let checkpoint: AndroidJniLibraryCheckpoint | undefined; - try { - const failure = await read( - source, - {}, - controller.signal, - undefined, - (value) => { - checkpoint = value; - }, - ).catch((cause: unknown) => cause); - expect(failure).toBeInstanceOf(AnalysisCancelledError); - if (!(failure instanceof AnalysisCancelledError)) throw failure; - expect(failure.cause).toBe(reason); - expect(checkpoint).toMatchObject({ - scan_state: "interrupted", - inventory: { - status: "partial", - examined_entries: 2, - selected_libraries: 2, - }, - }); - expect(checkpoint?.libraries.map((library) => library.path)).toEqual([ - LIBRARY, - second, +it.each(["member open", "section names"] as const)( + "keeps completed facts and the interrupted carrier when a later library is cancelled during %s", + async (stage) => { + const parsingNames = stage === "section names"; + const second = "lib/arm64-v8a/libsecond.so", + third = "lib/arm64-v8a/libunexamined.so"; + const bytes = jniElfFixture().bytes; + const secondBytes = parsingNames ? elfFrameAllowanceFixture() : bytes; + const source = await fixture([ + [LIBRARY, bytes], + [second, secondBytes], + [third, bytes], ]); - expect(checkpoint?.libraries[0]).toMatchObject({ - sha256: digest(bytes), - size: bytes.length, - facts: { tables: expect.any(Array) }, - }); - expect(checkpoint?.libraries[1]).toMatchObject({ - sha256: null, - size: null, - declared_size: bytes.length, - compressed_size: bytes.length, - facts: null, - failure: { kind: "incomplete" }, - }); - expect(checkpoint?.inventory.unknowns.join("\n")).toContain( - "unexamined selected occurrences remain unknown", - ); - if (checkpoint === undefined) - throw new Error("Missing interrupted scan checkpoint"); - expect( - androidPartialObservationSchema.safeParse(partial(source, checkpoint)) - .success, - ).toBe(true); - expect(JSON.parse(JSON.stringify(checkpoint))).toEqual(checkpoint); - expect(source.resources.hasPending).toBe(false); - } finally { - open.mockRestore(); - } -}); + const controller = new AbortController(), + reason = { caller: "stop during " + stage }; + const originalOpen = ZipArtifactReader.prototype.open; + const open = vi + .spyOn(ZipArtifactReader.prototype, "open") + .mockImplementation(async function ( + this: ZipArtifactReader, + entry, + signal, + ) { + if (entry.path === second && !parsingNames) { + controller.abort(reason); + throw new JadxInterruption(reason); + } + return originalOpen.call(this, entry, signal); + }); + const yieldToEventLoop = scheduler.yield.bind(scheduler); + const pause = parsingNames + ? vi.spyOn(scheduler, "yield").mockImplementationOnce(async () => { + await yieldToEventLoop(); + controller.abort(reason); + }) + : undefined; + let checkpoint: AndroidJniLibraryCheckpoint | undefined; + try { + const failure = await read( + source, + {}, + controller.signal, + undefined, + (value) => { + checkpoint = value; + }, + ).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(AnalysisCancelledError); + if (!(failure instanceof AnalysisCancelledError)) throw failure; + expect(failure.cause).toBe(reason); + expect(checkpoint).toMatchObject({ + scan_state: "interrupted", + inventory: { + status: "partial", + examined_entries: 2, + selected_libraries: 2, + }, + }); + expect(checkpoint?.libraries.map((library) => library.path)).toEqual([ + LIBRARY, + second, + ]); + expect(checkpoint?.libraries[0]).toMatchObject({ + sha256: digest(bytes), + size: bytes.length, + facts: { tables: expect.any(Array) }, + }); + expect(checkpoint?.libraries[1]).toMatchObject({ + sha256: parsingNames ? digest(secondBytes) : null, + size: parsingNames ? secondBytes.length : null, + declared_size: secondBytes.length, + compressed_size: secondBytes.length, + facts: null, + failure: { kind: "incomplete" }, + }); + expect(checkpoint?.inventory.unknowns.join("\n")).toContain( + "unexamined selected occurrences remain unknown", + ); + if (checkpoint === undefined) + throw new Error("Missing interrupted scan checkpoint"); + expect( + androidPartialObservationSchema.safeParse(partial(source, checkpoint)) + .success, + ).toBe(true); + expect(JSON.parse(JSON.stringify(checkpoint))).toEqual(checkpoint); + expect(source.resources.hasPending).toBe(false); + if (parsingNames) { + const retry = await read(source); + expect(retry.inventory).toMatchObject({ + status: "complete", + selected_libraries: 3, + }); + expect(retry.libraries[1]).toMatchObject({ + path: second, + sha256: digest(secondBytes), + size: secondBytes.length, + facts: expect.any(Object), + failure: null, + }); + } + } finally { + pause?.mockRestore(); + open.mockRestore(); + } + }, +); diff --git a/tests/boundary/filesystem/artifactZipIntegrity.test.ts b/tests/boundary/filesystem/artifactZipIntegrity.test.ts index c501298943..a18c4fad7c 100644 --- a/tests/boundary/filesystem/artifactZipIntegrity.test.ts +++ b/tests/boundary/filesystem/artifactZipIntegrity.test.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { zstdCompressSync } from "node:zlib"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { expect, it, onTestFinished } from "vitest"; @@ -84,6 +85,36 @@ cliTest( }, ); +it("does not publish a method-93 member with an incomplete final frame magic", async () => { + const root = await createTestTempDirectory("rea-zip-zstd-incomplete-"); + const path = join(root, "method93.zip"); + const outputRoot = join(root, "extracted"); + const content = Buffer.from("first complete text frame\n"); + const second = zstdCompressSync(Buffer.from("second text frame\n")); + const compressed = Buffer.concat([ + zstdCompressSync(content), + second.subarray(0, 3), + ]); + const archive = zstandardZipFixture(content, { compressed }); + await writeFile(path, archive); + + const target = await parseBinaryTarget(path); + if (!target.ok) throw target.error; + const result = await withArtifactClient(target.value, (client) => + client.execute("extract_artifact", { output_root: outputRoot }), + ); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "ArtifactOperationError", + reason: "format", + detail: "ZIP archive is malformed: Invalid compressed data", + }, + }); + expect(await readdir(root)).toEqual(["method93.zip"]); + expect((await readFile(path)).equals(archive)).toBe(true); +}); + for (const zip64 of [false, true]) { cliTest( `inspects and extracts an empty ${zip64 ? "Zip64" : "classic ZIP"} without changing its identity`, diff --git a/tests/boundary/filesystem/binaryHardening.test.ts b/tests/boundary/filesystem/binaryHardening.test.ts index ea1ce36ff8..87eee28373 100644 --- a/tests/boundary/filesystem/binaryHardening.test.ts +++ b/tests/boundary/filesystem/binaryHardening.test.ts @@ -1,8 +1,15 @@ import { readFile, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import { runWithAnalysisOperationSnapshot } from "../../../src/domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../../../src/domain/analysisResourceLimits.js"; import { createBinaryHardeningService } from "../../../src/composition/binaryDiagnostics.js"; import { elfHardeningFixture } from "../../../src/native/hardening/BinaryHardening.fixture.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("binary hardening stable filesystem boundary", () => { @@ -98,3 +105,68 @@ describe("binary hardening stable filesystem boundary", () => { } }); }); + +describe("binary hardening compressed ELF names", () => { + it("projects a section-name frame cap as an adjustable zstd.max_frames limit", async () => { + const root = await createTestTempDirectory("rea-binary-hardening-zstd-"); + const path = join(root, "app"); + await writeFile(path, elfFrameAllowanceFixture()); + const service = createBinaryHardeningService(); + try { + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => + service.inspect({ path }, { signal: new AbortController().signal }), + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + operation: "inspect_binary_hardening", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); + } finally { + await service.close(); + } + }); + + it("cancels a many-frame compressed section-name table", async () => { + const root = await createTestTempDirectory("rea-binary-hardening-cancel-"); + const path = join(root, "app"); + await writeFile(path, elfFrameAllowanceFixture()); + const reason = new Error("stop hardening frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + const service = createBinaryHardeningService(); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_binary_hardening", + cause: reason, + }); + } finally { + spy.mockRestore(); + await service.close(); + } + }); +}); diff --git a/tests/boundary/filesystem/elfDebugLink.test.ts b/tests/boundary/filesystem/elfDebugLink.test.ts index c28cb83ffe..e1ec8f85c5 100644 --- a/tests/boundary/filesystem/elfDebugLink.test.ts +++ b/tests/boundary/filesystem/elfDebugLink.test.ts @@ -1,7 +1,9 @@ -import { crc32 } from "node:zlib"; +import * as zlib from "node:zlib"; +import { createHash } from "node:crypto"; import { mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; import { AnalysisResourceConstraintError } from "../../../src/domain/analysisErrorCore.js"; import { createElfDebugLinkService } from "../../../src/composition/binaryDiagnostics.js"; import { @@ -10,7 +12,10 @@ import { ELF_DEBUG_LINK_UNAVAILABLE_CHECK_LIMITATION, elfDebugLinkSchema, } from "../../../src/domain/native/elfDebugLink.js"; -import { gnuBuildIdNote } from "../../../src/native/elf/ElfNotes.fixture.js"; +import { + compressedElfNoteFixture, + gnuBuildIdNote, +} from "../../../src/native/elf/ElfNotes.fixture.js"; import { debuglinkSection, elfDebugCandidateFixture, @@ -19,6 +24,37 @@ import { FIXTURE_DEBUG_NAME, } from "../../../src/native/elf/ElfDebugLink.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; + +vi.mock(import("node:zlib"), { spy: true }); + +it("inflates section names once per artifact while preserving build-ID comparison", async () => { + const root = await createTestTempDirectory("rea-elf-debug-link-name-reuse-"); + const path = join(root, "prog"); + const candidatePath = join(root, "prog.debug"); + const bytes = compressedElfNoteFixture(); + await writeFile(path, bytes); + await writeFile(candidatePath, bytes); + const service = createElfDebugLinkService(); + const inflate = vi.mocked(zlib.inflateSync).mockClear(); + try { + const result = await service.inspect( + { path, candidate_path: candidatePath }, + { signal: new AbortController().signal }, + ); + if (!result.ok) throw result.error; + const report = elfDebugLinkSchema.parse(result.value.normalized_result); + expect(report.build_id).toMatchObject({ + status: "observed", + sources: [{ table_kind: "section", table_index: 1 }], + }); + expect(report.verification?.build_id.status).toBe("match"); + expect(inflate).toHaveBeenCalledTimes(2); + } finally { + inflate.mockClear(); + await service.close(); + } +}); describe("ELF debug-link stable filesystem boundary", () => { it("reports non-file and malformed ELF selections as caller input errors", async () => { @@ -153,6 +189,52 @@ describe("ELF debug-link stable filesystem boundary", () => { }); describe("ELF debug-link streamed candidate boundary", () => { + it("retains streamed candidate identity when cancellation stops ELF comparison", async () => { + const root = await createTestTempDirectory( + "rea-elf-debug-link-cancel-candidate-", + ); + const path = join(root, "prog"); + const candidatePath = join(root, "prog.debug"); + const candidate = elfFrameAllowanceFixture(); + await writeFile(path, elfDebugLinkFixture()); + await writeFile(candidatePath, candidate); + const service = createElfDebugLinkService(); + const controller = new AbortController(); + const reason = new Error("stop while decoding candidate names"); + const yieldToEventLoop = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await yieldToEventLoop(); + controller.abort(reason); + }); + try { + const result = await service.inspect( + { path, candidate_path: candidatePath }, + { signal: controller.signal }, + ); + if (result.ok) + throw new Error("Expected candidate comparison cancellation"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + partialObservation: { + build_id: { status: "observed", hex: FIXTURE_BUILD_ID_HEX }, + candidate: { + path: candidatePath, + sha256: createHash("sha256").update(candidate).digest("hex"), + bytes: candidate.length, + debug_info: "unknown", + symtab: "unknown", + }, + verification: null, + }, + }); + } finally { + pause.mockRestore(); + await service.close(); + } + }); it("matches the known CRC32 of a non-ELF candidate", async () => { const root = await createTestTempDirectory("rea-elf-debug-link-crc32-"); const path = join(root, "prog"); @@ -261,7 +343,11 @@ describe("ELF debug-link candidate degradation", () => { candidate.set([0x7f, 0x45, 0x4c, 0x46, 2, 1, 9]); const image = elfDebugLinkFixture({ ...(presence === "with" ? {} : { notes: null }), - debuglink: debuglinkSection(FIXTURE_DEBUG_NAME, crc32(candidate), true), + debuglink: debuglinkSection( + FIXTURE_DEBUG_NAME, + zlib.crc32(candidate), + true, + ), }); const path = join(root, "prog"); const candidatePath = join(root, FIXTURE_DEBUG_NAME); @@ -303,7 +389,11 @@ describe("ELF debug-link candidate degradation", () => { ]), }); const image = elfDebugLinkFixture({ - debuglink: debuglinkSection(FIXTURE_DEBUG_NAME, crc32(candidate), true), + debuglink: debuglinkSection( + FIXTURE_DEBUG_NAME, + zlib.crc32(candidate), + true, + ), }); const path = join(root, "prog"); const candidatePath = join(root, FIXTURE_DEBUG_NAME); diff --git a/tests/boundary/filesystem/elfSectionNameCancellation.test.ts b/tests/boundary/filesystem/elfSectionNameCancellation.test.ts new file mode 100644 index 0000000000..51177fa472 --- /dev/null +++ b/tests/boundary/filesystem/elfSectionNameCancellation.test.ts @@ -0,0 +1,73 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { scheduler } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; +import { + createBinaryHardeningService, + createElfUnwindRangesService, + createNativeExportsService, +} from "../../../src/composition/binaryDiagnostics.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it.each([ + [ + "inspect_binary_hardening", + createBinaryHardeningService, + { format: "elf", images: [{ status: "parsed" }] }, + ], + ["inspect_elf_unwind_ranges", createElfUnwindRangesService, {}], + ["inspect_native_exports", createNativeExportsService, { format: "elf" }], +] as const)( + "%s observes cancellation while decoding ELF names after the file snapshot closes", + async (operation, createService, expectedReport) => { + const root = await createTestTempDirectory("rea-elf-names-cancel-"); + const path = join(root, "app"); + const bytes = elfFrameAllowanceFixture(); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + await writeFile(path, bytes); + const service = createService(); + const controller = new AbortController(); + const reason = new Error("stop section-name decoding"); + const yieldToEventLoop = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await yieldToEventLoop(); + controller.abort(reason); + }); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation, + cause: reason, + partialObservation: { + kind: "file-snapshot-hash", + path, + sha256, + bytes: bytes.length, + }, + }); + const retry = await service.inspect( + { path }, + { signal: new AbortController().signal }, + ); + if (!retry.ok) throw retry.error; + expect(retry.value.normalized_result).toMatchObject({ + artifact: { path, sha256, bytes: bytes.length }, + ...expectedReport, + }); + expect((await readFile(path)).equals(bytes)).toBe(true); + } finally { + pause.mockRestore(); + await service.close(); + } + }, +); diff --git a/tests/boundary/filesystem/elfUnwindRanges.test.ts b/tests/boundary/filesystem/elfUnwindRanges.test.ts index 195cc20df4..b434d59e5b 100644 --- a/tests/boundary/filesystem/elfUnwindRanges.test.ts +++ b/tests/boundary/filesystem/elfUnwindRanges.test.ts @@ -1,9 +1,16 @@ import { symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; +import { runWithAnalysisOperationSnapshot } from "../../../src/domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../../../src/domain/analysisResourceLimits.js"; import { createElfUnwindRangesService } from "../../../src/composition/binaryDiagnostics.js"; import { MAX_ELF_UNWIND_SYMBOL_TEXT_BYTES } from "../../../src/domain/native/elfUnwindRanges.js"; import { elfUnwindFixture } from "../../../src/native/elf/ElfUnwindTables.fixture.js"; +import { compressedElfNoteFixture } from "../../../src/native/elf/ElfNotes.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("ELF unwind range stable filesystem boundary", () => { @@ -176,3 +183,86 @@ describe("ELF unwind range budgets", () => { } }); }); + +it("projects a section-name frame cap as an adjustable zstd.max_frames limit", async () => { + const root = await createTestTempDirectory("rea-elf-unwind-zstd-"); + const path = join(root, "program"); + const frame = Buffer.from([ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, 0x61, + ]); + await writeFile( + path, + compressedElfNoteFixture( + 64, + true, + 2, + ".note.gnu.build-id", + Buffer.concat([frame, frame]), + ), + ); + const service = createElfUnwindRangesService(); + try { + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => service.inspect({ path }), + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + operation: "inspect_elf_unwind_ranges", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); + } finally { + await service.close(); + } +}); + +it("cancels a many-frame compressed section-name table", async () => { + const root = await createTestTempDirectory("rea-elf-unwind-cancel-"); + const path = join(root, "program"); + const frame = Buffer.from([ + 0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, 0x61, + ]); + const frames = Buffer.alloc(frame.length * 32); + for (let index = 0; index < 32; index += 1) + frame.copy(frames, index * frame.length); + await writeFile( + path, + compressedElfNoteFixture(64, true, 2, ".note.gnu.build-id", frames), + ); + const reason = new Error("stop unwind frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + const service = createElfUnwindRangesService(); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_elf_unwind_ranges", + cause: reason, + }); + } finally { + spy.mockRestore(); + await service.close(); + } +}); diff --git a/tests/boundary/filesystem/kernelModule.test.ts b/tests/boundary/filesystem/kernelModule.test.ts index 86ec5e3089..0c5deb8d6d 100644 --- a/tests/boundary/filesystem/kernelModule.test.ts +++ b/tests/boundary/filesystem/kernelModule.test.ts @@ -103,23 +103,26 @@ describe("kernel module stable filesystem boundary", () => { } }); - it("reports a gzip container as unsupported without treating it as a module", async () => { - const root = await createTestTempDirectory("rea-kernel-module-gzip-"); - const path = join(root, "demo.ko.gz"); - await writeFile(path, compressedKernelModuleContainer("gzip")); - const service = createKernelModuleService(); - try { - const result = await service.inspect({ path }); - if (!result.ok) throw result.error; - expect(fileIdentity(result.value.subject).format).toBe("file"); - expect(result.value.normalized_result).toMatchObject({ - applicability: "unsupported", - compression: { codec: "gzip", status: "unsupported" }, - next_probe: { operation: "external_decompression" }, - modinfo: null, - }); - } finally { - await service.close(); - } - }); + it.each(["gzip", "xz", "zstd"] as const)( + "reports a %s container as unsupported without treating it as a module", + async (codec) => { + const root = await createTestTempDirectory(`rea-kernel-module-${codec}-`); + const path = join(root, `demo.ko.${codec === "gzip" ? "gz" : codec}`); + await writeFile(path, compressedKernelModuleContainer(codec)); + const service = createKernelModuleService(); + try { + const result = await service.inspect({ path }); + if (!result.ok) throw result.error; + expect(fileIdentity(result.value.subject).format).toBe("file"); + expect(result.value.normalized_result).toMatchObject({ + applicability: "unsupported", + compression: { codec, status: "unsupported" }, + next_probe: { operation: "external_decompression" }, + modinfo: null, + }); + } finally { + await service.close(); + } + }, + ); }); diff --git a/tests/boundary/filesystem/machoCommandAdmission.test.ts b/tests/boundary/filesystem/machoCommandAdmission.test.ts index 6e73b193bf..be98845ccb 100644 --- a/tests/boundary/filesystem/machoCommandAdmission.test.ts +++ b/tests/boundary/filesystem/machoCommandAdmission.test.ts @@ -117,7 +117,9 @@ it.each([ ); await writeFile(path, bytes); expect(parseExecutableHeader(bytes, "arm64").ok).toBe(false); - expect(() => readBinaryContainer(bytes)).toThrow(/load command/iu); + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( + /load command/iu, + ); const resolved = await parseBinaryTarget(path); expect(resolved.ok).toBe(false); if (resolved.ok || !(resolved.error instanceof BinaryTargetError)) @@ -144,7 +146,7 @@ it.each([ "image", ); await writeFile(path, bytes); - expect(readBinaryContainer(bytes)).toMatchObject({ + expect(readBinaryContainer(bytes, "elf-directory")).toMatchObject({ format: "macho", bits: wide ? 64 : 32, byte_order: littleEndian ? "little" : "big", @@ -219,8 +221,10 @@ const OVERSIZED_COMMAND_BYTES = 16 * 1024 * 1024 + 8; it("reports truncation when an oversized command table cannot fit the file", () => { const bytes = commandImage(true, true, [16]); bytes.writeUInt32LE(OVERSIZED_COMMAND_BYTES, 20); - expect(() => readBinaryContainer(bytes)).toThrow(BinaryFormatFailure); - expect(() => readBinaryContainer(bytes)).toThrow( + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( + BinaryFormatFailure, + ); + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( expect.objectContaining({ kind: "truncated", failure: expect.objectContaining({ field: "Mach-O load commands" }), @@ -238,5 +242,7 @@ it("retains the native capacity result for a backed oversized command table", () ok: true, value: { commandBytes: OVERSIZED_COMMAND_BYTES }, }); - expect(() => readBinaryContainer(bytes)).toThrow(BinaryResourceFailure); + expect(() => readBinaryContainer(bytes, "elf-directory")).toThrow( + BinaryResourceFailure, + ); }); diff --git a/tests/boundary/filesystem/nativeExports.test.ts b/tests/boundary/filesystem/nativeExports.test.ts index 5fe0d18101..0b44414143 100644 --- a/tests/boundary/filesystem/nativeExports.test.ts +++ b/tests/boundary/filesystem/nativeExports.test.ts @@ -1,7 +1,8 @@ import { createHash } from "node:crypto"; import { symlink, truncate, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; import { createNativeExportsService } from "../../../src/composition/binaryDiagnostics.js"; import { analysisErrorRemediationAction } from "../../../src/domain/analysisErrorPresentation.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; @@ -149,6 +150,54 @@ describe("native export stable filesystem boundary", () => { }); }); +it("observes cancellation during a bounded ELF export inventory", async () => { + const root = await createTestTempDirectory("rea-native-export-cancel-"); + const path = join(root, "libfixture.so"); + const bytes = elfExportImage({ + symbols: Array.from({ length: 1025 }, () => ({ name: "exported" })), + }); + await writeFile(path, bytes); + const service = createNativeExportsService(); + const controller = new AbortController(); + const reason = new Error("stop during export rows"); + const yieldToEventLoop = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await yieldToEventLoop(); + controller.abort(reason); + }); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation during export rows"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_native_exports", + cause: reason, + partialObservation: { + kind: "file-snapshot-hash", + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + }); + const retry = await service.inspect( + { path }, + { signal: new AbortController().signal }, + ); + if (!retry.ok) throw retry.error; + expect(retry.value.normalized_result).toMatchObject({ + coverage: { status: "complete", examined_entries: 1025, exports: 1025 }, + }); + } finally { + pause.mockRestore(); + await service.close(); + } +}); + describe("native export text capacity", () => { it("retains valid exports and partial Evidence when a symbol name exceeds the read allowance", async () => { const root = await createTestTempDirectory("rea-native-export-name-limit-"); diff --git a/tests/boundary/filesystem/nativeInitializers.test.ts b/tests/boundary/filesystem/nativeInitializers.test.ts index b3bbfaf36c..0dd3ae881b 100644 --- a/tests/boundary/filesystem/nativeInitializers.test.ts +++ b/tests/boundary/filesystem/nativeInitializers.test.ts @@ -1,11 +1,14 @@ -import { symlink, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { readFile, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { describe, expect, it, vi } from "vitest"; import { createNativeInitializersService } from "../../../src/composition/binaryDiagnostics.js"; import { elf64PieInitializers, machoInitializerSymbolKinds, } from "../../../src/native/initializers/NativeInitializers.fixture.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("native initializer stable filesystem boundary", () => { @@ -90,3 +93,47 @@ describe("native initializer stable filesystem boundary", () => { expect(cancelled.error._tag).toBe("AnalysisCancelledError"); }); }); + +it("cancels ELF section-name decoding while retaining the initializer snapshot", async () => { + const root = await createTestTempDirectory("rea-native-init-zstd-cancel-"); + const path = join(root, "program"); + const bytes = elfFrameAllowanceFixture(); + await writeFile(path, bytes); + const service = createNativeInitializersService(); + const controller = new AbortController(); + const reason = new Error("section-name reading cancelled"); + const nativeYield = scheduler.yield.bind(scheduler); + const pause = vi + .spyOn(scheduler, "yield") + .mockImplementationOnce(async () => { + await nativeYield(); + controller.abort(reason); + }); + try { + const result = await service.inspect( + { path }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation during ELF names"); + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + partialObservation: { + kind: "file-snapshot-hash", + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + }); + const retry = await service.inspect({ path }); + if (!retry.ok) throw retry.error; + expect(retry.value.normalized_result).toMatchObject({ + format: "elf", + status: "parsed", + }); + expect((await readFile(path)).equals(bytes)).toBe(true); + } finally { + pause.mockRestore(); + await service.close(); + } +}); diff --git a/tests/boundary/go/goBinaryImage.test.ts b/tests/boundary/go/goBinaryImage.test.ts index cf258de989..b645c82c22 100644 --- a/tests/boundary/go/goBinaryImage.test.ts +++ b/tests/boundary/go/goBinaryImage.test.ts @@ -50,7 +50,7 @@ it("preserves exact Mach-O protection masks for Go fallback selection", () => { bytes.writeUInt32LE(3, secondSegment + 56); bytes.writeUInt32LE(3, secondSegment + 60); - const container = readBinaryContainer(bytes); + const container = readBinaryContainer(bytes, "elf-directory"); expect( container.load_regions?.map((region) => region.protection_masks), ).toEqual([ diff --git a/tests/boundary/go/goBinaryProvider.test.ts b/tests/boundary/go/goBinaryProvider.test.ts index fbd1c0500b..48c0637635 100644 --- a/tests/boundary/go/goBinaryProvider.test.ts +++ b/tests/boundary/go/goBinaryProvider.test.ts @@ -1,6 +1,7 @@ import { truncate, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; import { GoBinaryService } from "../../../src/application/go/GoBinaryService.js"; import { GoBinaryProvider } from "../../../src/go/GoBinaryProvider.js"; import { @@ -16,6 +17,7 @@ import { } from "../../fixtures/go/image.js"; import { createGoRodataStringFixture } from "../../fixtures/go/rodataStringImage.js"; import { CPU_ARM64, universalMacho } from "../../fixtures/universalMacho.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; it("rejects advertised PE32 data directories outside the optional header", async () => { @@ -356,3 +358,33 @@ it.each([ } }, ); + +it("cancels a many-frame compressed ELF section-name table", async () => { + const root = await createTestTempDirectory("rea-go-zstd-cancel-"); + const path = join(root, "app"); + await writeFile(path, elfFrameAllowanceFixture()); + const reason = new Error("stop go frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + try { + const result = await new GoBinaryProvider().inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_go_binary", + cause: reason, + }); + } finally { + spy.mockRestore(); + } +}); diff --git a/tests/boundary/gpu/codeObjects.test.ts b/tests/boundary/gpu/codeObjects.test.ts index bee7fb48bc..59bfa02907 100644 --- a/tests/boundary/gpu/codeObjects.test.ts +++ b/tests/boundary/gpu/codeObjects.test.ts @@ -1,12 +1,18 @@ import { expect, it, onTestFinished } from "vitest"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { deflateSync } from "node:zlib"; +import { deflateSync, zstdCompressSync } from "node:zlib"; import { createHash } from "node:crypto"; +import { runWithAnalysisOperationSnapshot } from "../../../src/domain/analysisOperationSnapshot.js"; +import { + MAXIMUM_ZSTD_MAX_FRAMES, + MINIMUM_ZSTD_MAX_FRAMES, +} from "../../../src/domain/analysisResourceLimits.js"; import { pe } from "../../../src/domain/binaryTarget.fixture.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { gpuCodeObjectsPartialSchema } from "../../../src/domain/gpu/gpuCodeObjects.js"; import { GpuCodeObjectsProvider } from "../../../src/gpu/GpuCodeObjectsProvider.js"; +import { GpuInventory } from "../../../src/gpu/GpuInventory.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cudaFixture, @@ -36,6 +42,45 @@ const inspect = async ( onTestFinished(() => provider.close()); return provider.inspect({ path, analysis: "kernels", ...options }); }; +it("reports a Zstandard frame-count refusal as an adjustable GPU limit", async () => { + const frame = (byte: number) => + Buffer.from([0x28, 0xb5, 0x2f, 0xfd, 0x20, 0x01, 0x09, 0x00, 0x00, byte]); + const compressed = Buffer.concat([frame(0x61), frame(0x62)]); + const entry = Buffer.alloc(64 + Math.ceil(compressed.length / 8) * 8); + entry.writeUInt16LE(1, 0); + entry.writeUInt16LE(0x101, 2); + entry.writeUInt32LE(64, 4); + entry.writeBigUInt64LE(BigInt(entry.length - 64), 8); + entry.writeUInt32LE(compressed.length, 16); + entry.writeUInt32LE(80, 28); + entry.writeBigUInt64LE(0x11n | 0x8000n, 40); + entry.writeBigUInt64LE(2n, 56); + compressed.copy(entry, 64); + const header = Buffer.alloc(16); + header.writeUInt32LE(0xba55ed50); + header.writeUInt16LE(1, 4); + header.writeUInt16LE(16, 6); + header.writeBigUInt64LE(BigInt(entry.length), 8); + const result = await runWithAnalysisOperationSnapshot( + { revision: 1, overrides: { "zstd.max_frames": 1 } }, + () => inspect(Buffer.concat([header, entry])), + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + resource: "cpu", + limit: { + setting: "zstd.max_frames", + effective_value: 1, + source: "runtime", + minimum: MINIMUM_ZSTD_MAX_FRAMES, + maximum: MAXIMUM_ZSTD_MAX_FRAMES, + unit: "count", + }, + }); +}); + it.each(["none", "zstd", "lz4"] as const)( "reads selected CUDA metadata and PTX with %s compression", async (compression) => { @@ -70,6 +115,52 @@ it.each(["none", "zstd", "lz4"] as const)( ]); }, ); + +it("observes cancellation during a cubin kernel directory read", async () => { + const cubin = cubinFixture(); + const section = Number(cubin.readBigUInt64LE(40)) + 64; + const offset = Number(cubin.readBigUInt64LE(section + 24)); + const size = Number(cubin.readBigUInt64LE(section + 32)); + const names = cubin.subarray(offset, offset + size); + const compressionHeader = Buffer.alloc(24); + compressionHeader.writeUInt32LE(2); + compressionHeader.writeBigUInt64LE(BigInt(names.length), 8); + compressionHeader.writeBigUInt64LE(1n, 16); + const encodedNames = Buffer.concat([ + compressionHeader, + zstdCompressSync(names), + ]); + const bytes = Buffer.concat([cubin, encodedNames]); + bytes.writeBigUInt64LE(0x800n, section + 8); + bytes.writeBigUInt64LE(BigInt(cubin.length), section + 24); + bytes.writeBigUInt64LE(BigInt(encodedNames.length), section + 32); + + const controller = new AbortController(); + const inventory = new GpuInventory( + { path: "/artifacts/cubin", analysis: "kernels" }, + controller.signal, + ); + const cancelled = new Error("kernel directory read cancelled"); + let payloadReads = 0; + let pendingAbort: NodeJS.Immediate | undefined; + try { + await expect( + inventory.inspect({ + size: bytes.length, + containerId: null, + read: async (at, count) => { + // The selected payload is re-read into the kernel parser's buffer. + if (at === 0 && count === bytes.length && ++payloadReads === 1) + pendingAbort = setImmediate(() => controller.abort(cancelled)); + return bytes.subarray(at, at + count); + }, + }), + ).rejects.toMatchObject({ reason: "cancelled", cause: cancelled }); + expect(payloadReads).toBe(1); + } finally { + if (pendingAbort !== undefined) clearImmediate(pendingAbort); + } +}); it("decodes a CUDA 0x2000 LZ4 entry to the exact declared payload", async () => { const ptx = Buffer.from(ptxFixture), bytes = cudaFixture("lz4", [{ kind: 1, target: 80, bytes: ptx }]), diff --git a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts index 1916c635a4..fae8059031 100644 --- a/tests/boundary/il2cpp/il2cppBinaryFile.test.ts +++ b/tests/boundary/il2cpp/il2cppBinaryFile.test.ts @@ -1112,7 +1112,7 @@ it.each(["outside", "misaligned", "duplicate"] as const)( bytes.copy(bytes, 176, 120, 176); bytes.writeUInt16LE(3, 56); } - expect(() => readBinaryContainer(bytes)).not.toThrow(); + expect(() => readBinaryContainer(bytes, "elf-directory")).not.toThrow(); expect(() => link(bytes)).toThrow( shape === "duplicate" ? "multiple PT_DYNAMIC segments" diff --git a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts index 1a36c77c53..e1a0b82750 100644 --- a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts +++ b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts @@ -2,6 +2,7 @@ import { setImmediate } from "node:timers/promises"; import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { zstdCompressSync } from "node:zlib"; import { afterEach, expect, it, vi } from "vitest"; import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import { @@ -17,10 +18,12 @@ import { inspectIl2cppMetadataV108 } from "../../../src/il2cpp/Il2cppMetadataV10 import { Il2cppGhidraSeedWriter } from "../../../src/il2cpp/Il2cppGhidraSeedWriter.js"; import { Il2cppMetadataService } from "../../../src/application/il2cpp/Il2cppMetadataService.js"; import { il2cppMetadataSchema } from "../../../src/domain/il2cpp/il2cppMetadata.js"; +import * as zstandard from "../../../src/domain/compression/zstandardExactFrame.js"; import { createElfFixture, createIl2cppBinaryFixture, FIXTURE_GENERIC_INSTANCES, + FIXTURE_METHOD_ADDRESSES, } from "../../fixtures/il2cpp/binary.js"; import { createIl2cppMetadataFixture, @@ -755,6 +758,137 @@ it("accepts cancellation between binary linking batches and keeps the metadata s } }); +/** Add a compressed ELF64 .shstrtab outside the fixture's PT_LOAD. */ +const withCompressedElfNames = (image: Buffer): Buffer => { + const names = Buffer.from("\0.shstrtab\0"); + const compressionHeader = Buffer.alloc(24); + compressionHeader.writeUInt32LE(2, 0); // ELFCOMPRESS_ZSTD + compressionHeader.writeBigUInt64LE(BigInt(names.length), 8); + compressionHeader.writeBigUInt64LE(1n, 16); + const payload = Buffer.concat([compressionHeader, zstdCompressSync(names)]); + const padding = Buffer.alloc((8 - (image.length % 8)) % 8); + const tableOffset = image.length + padding.length; + const table = Buffer.alloc(128); + const namesHeader = table.subarray(64); + namesHeader.writeUInt32LE(1, 0); // ".shstrtab" offset in names + namesHeader.writeUInt32LE(3, 4); // SHT_STRTAB + namesHeader.writeBigUInt64LE(0x800n, 8); // SHF_COMPRESSED + namesHeader.writeBigUInt64LE(BigInt(tableOffset + table.length), 24); + namesHeader.writeBigUInt64LE(BigInt(payload.length), 32); + namesHeader.writeBigUInt64LE(1n, 48); + const bytes = Buffer.concat([image, padding, table, payload]); + bytes.writeBigUInt64LE(BigInt(tableOffset), 40); + bytes.writeUInt16LE(64, 58); + bytes.writeUInt16LE(2, 60); + bytes.writeUInt16LE(1, 62); + return bytes; +}; + +it("retains selected source evidence when cancelled decoding compressed ELF names", async () => { + const root = await createTestTempDirectory("rea-il2cpp-elf-names-cancel-"); + const path = join(root, "global-metadata.dat"); + const binary_path = join(root, "libil2cpp.so"); + const metadata = createLinkedIl2cppMetadataFixture().bytes; + const binary = withCompressedElfNames( + createElfFixture(createIl2cppBinaryFixture()), + ); + await Promise.all([ + writeFile(path, metadata), + writeFile(binary_path, binary), + ]); + + const controller = new AbortController(); + const reason = new Error("cancel while decoding ELF names"); + const decode = zstandard.decodeZstandardFramesCooperative; + const checkpoint = vi + .spyOn(zstandard, "decodeZstandardFramesCooperative") + .mockImplementation((bytes, options, pause) => + decode(bytes, options, async () => { + controller.abort(reason); + await pause(); + }), + ); + const provider = new Il2cppMetadataProvider(); + try { + const result = await provider.inspect( + { path, binary_path }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation during ELF decoding"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(result.error.partialObservation).toMatchObject({ + operation: "observe_il2cpp_metadata_source", + normalized_result: { + artifact: { path }, + selected_binary: { + path: binary_path, + identity_status: "observed", + bytes: binary.length, + sha256: createHash("sha256").update(binary).digest("hex"), + }, + inventory_status: "not-returned-as-complete", + }, + }); + checkpoint.mockRestore(); + const retry = await provider.inspect({ path, binary_path }); + if (!retry.ok) throw retry.error; + expect(retry.value.binary?.format).toBe("elf"); + expect(retry.value.method_addresses[0]?.address).toBe( + `0x${FIXTURE_METHOD_ADDRESSES[0].toString(16)}`, + ); + } finally { + await provider.close(); + } +}); + +it.each([false, true])( + "refuses unsupported generic metadata before reading an invalid ELF directory (signal=%s)", + async (withSignal) => { + const root = await createTestTempDirectory("rea-il2cpp-generic-layout-"); + const path = join(root, "global-metadata.dat"); + const binary_path = join(root, "libil2cpp.so"); + const metadata = createLinkedIl2cppMetadataFixture({ version: 39 }).bytes; + const binary = withCompressedElfNames( + createElfFixture(createIl2cppBinaryFixture()), + ); + binary.writeBigUInt64LE(BigInt(binary.length + 8), 40); + await Promise.all([ + writeFile(path, metadata), + writeFile(binary_path, binary), + ]); + const options = withSignal + ? { signal: new AbortController().signal } + : undefined; + const provider = new Il2cppMetadataProvider(); + try { + const result = await provider.inspect( + { path, binary_path, generic_instances: true }, + options, + ); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisUnsupportedTargetError", + reason: expect.stringContaining( + "generic method instances are not decoded for metadata version 39", + ), + }, + }); + const ordinary = await provider.inspect({ path, binary_path }, options); + expect(ordinary).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisInputError", + issues: [{ path: ["binary_path"], reason: "invalid_format" }], + }, + }); + } finally { + await provider.close(); + } + }, +); + it("refuses duplicate MethodDef tokens before publishing linked Evidence or Ghidra seeds", async () => { const root = await createTestTempDirectory( "rea-il2cpp-duplicate-method-token-", diff --git a/tests/boundary/rust/rustBinaryBudget.test.ts b/tests/boundary/rust/rustBinaryBudget.test.ts index 4e459d4b05..c6ed61bc3b 100644 --- a/tests/boundary/rust/rustBinaryBudget.test.ts +++ b/tests/boundary/rust/rustBinaryBudget.test.ts @@ -1,12 +1,14 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it } from "vitest"; +import { scheduler } from "node:timers/promises"; +import { expect, it, vi } from "vitest"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { RustBinaryProvider } from "../../../src/rust/RustBinaryProvider.js"; import { createRustImage, createRustHashOverflowImage, } from "../../fixtures/rust/image.js"; +import { elfFrameAllowanceFixture } from "../../fixtures/binaryDiagnostics/elfFrameAllowance.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; it("reports a recoverable compiler prefix without refusing the dependency report", async () => { @@ -204,3 +206,33 @@ it("reports byte trimming only when it removes a compiler path", async () => { await provider.close(); } }); + +it("cancels a many-frame compressed ELF section-name table", async () => { + const root = await createTestTempDirectory("rea-rust-zstd-cancel-"); + const path = join(root, "app"); + await writeFile(path, elfFrameAllowanceFixture()); + const reason = new Error("stop rust frames"); + const controller = new AbortController(); + const yieldImpl = scheduler.yield.bind(scheduler); + let yields = 0; + const spy = vi.spyOn(scheduler, "yield").mockImplementation(async () => { + yields += 1; + await yieldImpl(); + if (yields === 1) controller.abort(reason); + }); + try { + const result = await new RustBinaryProvider().inspect( + { path }, + { signal: controller.signal }, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "inspect_rust_binary", + cause: reason, + }); + } finally { + spy.mockRestore(); + } +}); diff --git a/tests/boundary/rust/rustBinaryImage.test.ts b/tests/boundary/rust/rustBinaryImage.test.ts index a5cc1ec126..209d1e8a49 100644 --- a/tests/boundary/rust/rustBinaryImage.test.ts +++ b/tests/boundary/rust/rustBinaryImage.test.ts @@ -48,7 +48,7 @@ it.each([ "retains read-only ELF section bytes with a %s name", (name, expectedName) => { const bytes = rustElfSectionNameFixture(name); - const container = readBinaryContainer(bytes); + const container = readBinaryContainer(bytes, "elf-directory"); expect(container.load_regions?.every(({ write }) => write)).toBe(true); expect(container.sections).toContainEqual( expect.objectContaining({ @@ -77,7 +77,9 @@ it.each([ const bytes = rustElfSectionNameFixture("empty"); bytes.writeUInt32LE(type, 384 + 4); bytes.writeBigUInt64LE(flags, 384 + 8); - expect(readBinaryContainer(bytes).sections).not.toContainEqual( + expect( + readBinaryContainer(bytes, "elf-directory").sections, + ).not.toContainEqual( expect.objectContaining({ offset: 2048, read_only: true }), ); expect(readRustBinaryImage(bytes).rustc_commit).toBeNull(); @@ -88,9 +90,9 @@ it("does not scan an unnamed ELF section whose declared range leaves the file", const bytes = rustElfSectionNameFixture("empty"); // The compiler path still exists at 2048, but its declared section no longer fits. bytes.writeBigUInt64LE(BigInt(bytes.length), 384 + 32); - expect(readBinaryContainer(bytes).sections).not.toContainEqual( - expect.objectContaining({ read_only: true }), - ); + expect( + readBinaryContainer(bytes, "elf-directory").sections, + ).not.toContainEqual(expect.objectContaining({ read_only: true })); expect(readRustBinaryImage(bytes).rustc_commit).toBeNull(); }); @@ -520,7 +522,7 @@ it("keeps the container's truncated failure and its facts unchanged", () => { } throw new Error("expected a container failure"); }; - const container = thrown(() => readBinaryContainer(bytes)); + const container = thrown(() => readBinaryContainer(bytes, "elf-directory")); const rust = thrown(() => readRustBinaryImage(bytes)); expect(rust).toBeInstanceOf(ParseFailureError); expect((rust as ParseFailureError).failure).toEqual( @@ -623,8 +625,8 @@ it.each([ } else { sectionName.copy(control, sectionOffset); } - expect(readBinaryContainer(bytes).load_regions).toEqual( - readBinaryContainer(control).load_regions, + expect(readBinaryContainer(bytes, "elf-directory").load_regions).toEqual( + readBinaryContainer(control, "elf-directory").load_regions, ); expect(() => readGoBinaryImage(bytes)).toThrow(/build-info|mapped/i); expect(readRustBinaryImage(bytes)).toMatchObject({ diff --git a/tests/fixtures/binaryDiagnostics/elfFrameAllowance.ts b/tests/fixtures/binaryDiagnostics/elfFrameAllowance.ts new file mode 100644 index 0000000000..7cbdfb6228 --- /dev/null +++ b/tests/fixtures/binaryDiagnostics/elfFrameAllowance.ts @@ -0,0 +1,17 @@ +import { zstdCompressSync } from "node:zlib"; +import { compressedElfNoteFixture } from "../../../src/native/elf/ElfNotes.fixture.js"; + +/** Two ordinary frames encode the original ELF note fixture's names. */ +export const elfFrameAllowanceFixture = () => { + const section = ".note.gnu.build-id"; + return compressedElfNoteFixture( + 64, + true, + 2, + section, + Buffer.concat([ + zstdCompressSync(Buffer.from(`\0${section}\0`)), + zstdCompressSync(Buffer.from(".shstrtab\0")), + ]), + ); +};