From cb4e5cfbff25c2dda8683ef8598c17cec8a73473 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:11:20 +0800 Subject: [PATCH] feat(analysis): make resource budgets recoverable across CLI and MCP (#9706) * refactor(analysis): centralize resource budgets and recovery facts * feat(analysis): expose ASAR and Ghidra runtime admission controls * feat(artifacts): expose reader admission and preserve limited recognition * feat(javascript): expose recoverable semantic precision budgets * feat(web): add recoverable source-map budgets and compact retained views * feat(web): make historical capture decoding budgets recoverable * fix(desktop): enforce selected view policies across capture and export * feat(runtime): make Frida and LLDB retention budgets selectable * feat(native): expose diagnostic budgets and preserve exhaustion causes * feat(managed): expose recoverable snapshot and representation budgets * feat(metadata): make Go and Rust inspection budgets recoverable * feat(wasm): expose representation and producer admissions * feat(evm): expose per-call worker resource budgets * feat(android): make resource and string-pool omissions recoverable * feat(producers): expose command and recorded-capture budgets * docs(analysis): record recovery contracts and integrated validation * fix(docs): link calibration oracle from published verification pages * fix(docs): restore desktop task anchors and receipt links * test(analysis): align fixtures with selectable budget contracts * fix(analysis): remove source-location projector dependency cycle * test(analysis): refresh budget recovery and verifier regressions * test(analysis): preserve format and publication recovery expectations * test(windows): align conformance with owned runtime admission * fix(desktop): preserve selected alias kinds in exported state * test(analysis): align provider fixtures with recoverable budgets * test(desktop): validate emitted alias metadata structurally * fix(ghidra): classify bounded Windows endpoint refusals * test(cli): assert selectable artifact admission recovery * fix(cli): preserve capture provenance in environment builds --- bridge/ghidra/ReaGhidraBridge.java | 122 +- bridge/mitmproxy/capture.py | 21 +- bridge/mitmproxy/test_capture.py | 8 +- bridge/native/rea_lldb_tracer.py | 60 +- bridge/native/test_rea_lldb_tracer.py | 72 +- bridge/pwndbg/core_context.py | 6 +- bridge/pwndbg/test_core_context.py | 13 +- bridge/pwntools/decoder_runtime.py | 21 +- docs/README.md | 4 + docs/bun-standalone.md | 21 +- docs/design/limits-and-recovery.md | 127 +- docs/desktop-application-capture.md | 19 +- docs/desktop-application-export.md | 16 +- docs/desktop-application-state.md | 59 + docs/desktop-application-tasks.md | 91 +- docs/dotnet-bundles.md | 15 +- docs/godot-pck.md | 27 +- .../android/apktool-resource-analysis.md | 42 +- docs/guides/android/flutter-analysis.md | 44 + docs/guides/data/packet-captures.md | 26 +- docs/guides/evm/evm-bytecode.md | 31 +- .../javascript-artifact-reconstruction.md | 39 +- docs/guides/managed/managed-code-analysis.md | 32 +- docs/guides/native/binary-diagnostics.md | 12 +- docs/guides/native/go-binaries.md | 60 +- docs/guides/native/native-investigation.md | 26 +- docs/guides/native/recorded-crashes.md | 10 +- docs/guides/runtime/frida-instrumentation.md | 2 +- docs/guides/signals/signal-recordings.md | 23 +- docs/guides/wasm/wasm-artifacts.md | 61 +- docs/guides/web/browser-observation.md | 2 +- docs/guides/web/web-network-captures.md | 53 +- docs/guides/web/web-source-location.md | 143 +- docs/kernel-modules.md | 19 +- docs/node-sea.md | 62 +- docs/providers/ghidra-provider.md | 34 + docs/reference/mcp-contracts.md | 5 +- docs/reverse-engineering-tools.md | 14 +- docs/rust-binaries.md | 20 +- .../2356-wasm-interface-comparison/spec.md | 4 +- .../analysis/recoverable-budgets/plan.md | 27 + .../analysis/recoverable-budgets/spec.md | 69 + .../analysis/recoverable-budgets/tasks.md | 137 ++ docs/specs/evm/producer-limits/plan.md | 6 + docs/specs/evm/producer-limits/spec.md | 7 + docs/specs/evm/producer-limits/tasks.md | 6 + docs/specs/native/go-string-literals/plan.md | 13 +- docs/specs/native/go-string-literals/spec.md | 31 +- docs/specs/native/go-string-literals/tasks.md | 5 + .../web/historical-capture-budgets/plan.md | 5 + .../web/historical-capture-budgets/spec.md | 9 + .../web/historical-capture-budgets/tasks.md | 6 + .../web/source-map-budget-recovery/plan.md | 9 + .../web/source-map-budget-recovery/spec.md | 35 + .../web/source-map-budget-recovery/tasks.md | 10 + .../flutter-budget-calibration.md | 135 ++ .../javascript-precision-calibration.md | 127 ++ ...pture-budget-qualification-2026-11-09.json | 168 ++ ...e-analysis-settings-ghidra-2026-11-09.json | 227 +++ ...ettings-ghidra-integration-2026-11-09.json | 232 +++ .../verification/runtime-analysis-settings.md | 111 +- docs/verification/source-location-delivery.md | 206 ++ .../source-map-budget-calibration.md | 109 ++ .../source-map-budgets-2026-11-09/acquire.py | 44 + .../artifacts.json | 252 +++ .../measurements.json | 370 ++++ .../source-map-budgets-2026-11-09/oracle.py | 62 + .../workload.mjs | 288 +++ .../fixtures/generate-mitmproxy-capture.py | 10 +- .../verify-real-ghidra-decompile-timeout.mjs | 394 ++-- scripts/verify-windows-native.mjs | 30 +- .../verify/analysis/budget-calibration-run.py | 214 +++ .../flutter-budget-calibration/aggregate.py | 45 + .../flutter-budget-calibration/configs.py | 37 + .../flutter-budget-calibration/manifest.json | 48 + .../measurements.json | 294 +++ .../flutter-budget-calibration/oracle.py | 94 + .../flutter-budget-calibration/task.mjs | 103 + .../aggregate.py | 59 + .../manifest.json | 16 + .../measurements.json | 323 ++++ .../prepare.py | 76 + .../javascript-precision-calibration/scan.mjs | 68 + .../source-inventory.json | 192 ++ .../javascript-precision-calibration/task.py | 70 + .../source-location-delivery/aggregate.py | 199 ++ .../measurements.json | 1706 +++++++++++++++++ .../source-location-delivery/task.mjs | 426 ++++ scripts/verify/architecture-guards.mjs | 4 +- scripts/verify/managed/config.mjs | 49 - scripts/verify/managed/conformance.mjs | 35 +- scripts/verify/managed/manifest.mjs | 48 +- scripts/verify/managed/oracles.mjs | 33 +- scripts/verify/managed/support.mjs | 21 - src/apktool/ApktoolProvider.test.ts | 58 + src/apktool/ApktoolProvider.ts | 3 +- src/apktool/ApktoolStringTables.test.ts | 55 +- src/apktool/ApktoolStringTables.ts | 121 +- src/apktool/ApktoolWorkspace.ts | 2 +- src/application/AnalysisSettings.test.ts | 39 +- src/application/BrowserEvidence.test.ts | 21 + src/application/BrowserEvidence.ts | 6 + .../JsonAuthenticationHeadroom.test.ts | 36 +- src/application/JsonAuthenticationHeadroom.ts | 24 + .../SetupRegistrationEnvironment.ts | 5 + src/application/WebNetworkCaptureService.ts | 9 + src/application/WebSourceLocationPorts.ts | 2 + src/application/WebSourceLocationService.ts | 25 +- .../analysisView/AnalysisViewService.ts | 4 +- .../WebSourceLocationViews.test.ts | 184 ++ .../binaryDiagnostics/BinaryLayoutService.ts | 3 + .../KernelModuleService.test.ts | 16 +- .../binaryDiagnostics/KernelModuleService.ts | 32 +- .../binaryDiagnostics/RecordedCrashService.ts | 3 + .../desktop/DesktopApplicationCapture.ts | 5 + .../desktop/DesktopApplicationExport.test.ts | 63 +- .../desktop/DesktopApplicationExport.ts | 10 +- .../desktop/DesktopApplicationState.test.ts | 30 + .../desktop/DesktopApplicationState.ts | 108 +- src/application/desktop/DesktopExportFiles.ts | 20 +- .../desktop/DesktopExportReaders.ts | 14 +- .../desktop/DesktopRetainedContent.ts | 3 +- src/application/desktop/DesktopSqliteState.ts | 32 +- src/application/desktop/DesktopTaskDraft.ts | 12 +- src/application/desktop/DesktopZipView.ts | 15 +- .../frida/FridaInstrumentationPort.ts | 11 +- .../frida/FridaInstrumentationService.test.ts | 4 +- .../frida/FridaInstrumentationService.ts | 10 +- src/application/go/GoBinaryService.test.ts | 14 +- .../JavaScriptApplicationEvidence.ts | 3 + .../JavaScriptApplicationService.ts | 17 +- .../JavaScriptArtifactGraphBuilder.ts | 29 +- .../JavaScriptArtifactReconstruction.ts | 3 + .../JavaScriptModuleRelationships.ts | 11 +- .../JavaScriptSemanticGraphBuilder.ts | 16 +- ...aScriptWrappedCallPathCancellation.test.ts | 4 +- .../managed/ManagedMemberComparisonService.ts | 68 +- .../reverse/ReverseEngineeringService.ts | 139 +- .../wasm/WasmArtifactService.test.ts | 41 +- src/application/wasm/WasmArtifactService.ts | 58 +- .../WasmInterfaceComparisonService.test.ts | 92 +- .../wasm/WasmInterfaceComparisonService.ts | 78 +- .../ArtifactEmbeddedFormatProbes.test.ts | 73 + src/artifacts/ArtifactEmbeddedFormatProbes.ts | 57 +- .../ArtifactOperationFailure.test.ts | 72 + src/artifacts/ArtifactOperationFailure.ts | 7 +- src/artifacts/ArtifactPaths.ts | 35 +- src/artifacts/ArtifactProvider.ts | 20 + src/artifacts/ArtifactReader.ts | 7 + src/artifacts/ArtifactReaderLimits.ts | 46 + src/artifacts/AsarArtifactReader.ts | 6 +- src/artifacts/AsarHeader.ts | 7 +- src/artifacts/AsarListing.ts | 19 +- src/artifacts/BunStandaloneArtifactReader.ts | 72 +- src/artifacts/DotnetBundleArtifactReader.ts | 6 + src/artifacts/GodotPckArtifactReader.test.ts | 74 + src/artifacts/GodotPckArtifactReader.ts | 192 +- src/artifacts/NodeSeaArtifactReader.test.ts | 196 +- src/artifacts/NodeSeaArtifactReader.ts | 74 +- src/artifacts/ZipArtifactReader.ts | 26 +- src/artifacts/artifactFormats.ts | 28 +- src/artifacts/bun/BunModuleGraph.test.ts | 98 +- src/artifacts/bun/BunModuleGraphLayout.ts | 239 ++- src/artifacts/bun/BunModuleGraphMembers.ts | 32 +- src/artifacts/bun/BunStandalone.fixture.ts | 16 +- .../bun/BunStandaloneArtifactReader.test.ts | 66 + .../deno/DenoStandaloneArtifactReader.test.ts | 32 + src/artifacts/dotnet/DotnetBundle.ts | 93 +- .../extraction/ArtifactExtraction.ts | 48 +- src/artifacts/inventory/ArtifactInventory.ts | 7 +- src/artifacts/inventory/classify.ts | 35 +- src/artifacts/inventory/reader.ts | 3 + src/artifacts/inventory/scanCanonical.ts | 11 +- src/artifacts/inventory/types.ts | 2 + .../javascript/JavaScriptSourceAnalysis.ts | 17 +- src/artifacts/nodeSea/NodeSeaAdmission.ts | 3 + src/artifacts/nodeSea/NodeSeaBlob.test.ts | 133 +- src/artifacts/nodeSea/NodeSeaBlob.ts | 87 +- src/artifacts/nodeSea/NodeSeaLocation.ts | 94 +- src/browser/IsolatedWebBundleAnalyzer.ts | 1 + src/browser/WebBundleAnalysisWorker.ts | 3 + .../WebBundleAnalysisWorkerProtocol.ts | 2 + src/browser/WebBundleAnalysisWorkerServer.ts | 10 +- src/browser/WebSourceMapFetcher.test.ts | 38 + src/browser/WebSourceMapFetcher.ts | 103 +- src/browser/assets/WebArtifactReads.ts | 13 +- .../assets/WebSourceLocationArtifacts.ts | 15 +- src/browser/history/CaptureBudgets.ts | 18 + src/browser/history/CaptureFailures.ts | 46 +- src/browser/history/CaptureJson.ts | 6 +- src/browser/history/HarCapture.test.ts | 7 +- src/browser/history/HarCapture.ts | 12 +- .../history/HistoricalCaptureBudgets.test.ts | 194 ++ .../history/HistoricalCaptureDecoder.ts | 127 +- src/browser/history/HistoricalCaptureReply.ts | 8 +- src/browser/history/NodeCaptureProcess.ts | 106 +- src/browser/history/QlogCapture.test.ts | 7 +- src/browser/history/SazCapture.ts | 12 +- src/browser/history/SazMetadata.ts | 6 +- src/browser/history/SazProjection.ts | 10 +- .../history/protobuf/ProtobufCapture.ts | 18 +- .../history/protobuf/ProtobufFraming.ts | 6 +- .../history/protobuf/ProtobufSchema.ts | 4 +- src/browser/history/protobuf/ProtobufWire.ts | 19 +- src/captures/TsharkCaptureProvider.ts | 23 +- src/cli/apktoolCommands.ts | 34 + src/cli/artifactCommands.ts | 21 +- src/cli/binaryDiagnosticsCommands.ts | 18 + src/cli/browserCommands.ts | 25 + src/cli/captureCommands.ts | 26 +- src/cli/desktopCaptureCommands.ts | 68 +- src/cli/electronCommands.ts | 13 + src/cli/environmentCommands.ts | 6 +- src/cli/evmCommands.ts | 10 +- src/cli/flutterCommands.ts | 23 + src/cli/fridaCommands.ts | 51 + src/cli/goCommands.ts | 60 +- src/cli/managedCommands.ts | 54 +- src/cli/nativeCallCommands.ts | 2 +- src/cli/producerLimitsOption.ts | 26 + src/cli/reverseEngineeringCommands.ts | 13 + src/cli/rustCommands.ts | 20 + src/cli/signalCommands.ts | 26 +- src/cli/sourceMapLimits.ts | 64 + src/cli/wasmCommands.ts | 30 + src/cli/webNetworkCaptureCommands.ts | 18 + src/cli/webSourceLocationCommands.ts | 47 +- src/cliObservationOptions.ts | 6 + .../javascriptAnalysisWorkerProcess.ts | 8 +- .../webBundleAnalysisWorkerProcess.ts | 4 +- src/config.test.ts | 17 + src/config/analysisSettings.ts | 10 + src/config/asarHeaderAdmission.ts | 3 + src/config/asarListingAdmission.ts | 42 + src/config/environment.ts | 12 + src/config/ghidraValueFlowLimits.ts | 18 + src/config/parseConfig.ts | 21 + src/config/runtimeAnalysisSettings.ts | 54 +- src/config/types.ts | 4 + src/contracts/analysisSettingsContracts.ts | 2 +- src/contracts/analysisViewToolContracts.ts | 2 +- src/contracts/apktool/apktoolToolContracts.ts | 11 +- src/contracts/artifactToolContracts.ts | 7 + src/contracts/evm/evmToolContracts.ts | 2 +- src/contracts/flutter/flutterToolContracts.ts | 2 +- src/contracts/fridaToolContracts.ts | 7 + src/contracts/go/goToolContracts.ts | 2 +- src/contracts/managed/managedToolContracts.ts | 2 + .../native/binaryDiagnosticsToolContracts.ts | 4 +- src/contracts/native/nativeToolContracts.ts | 2 +- .../reverseEngineeringToolContracts.ts | 4 +- src/contracts/wasm/wasmComparisonContracts.ts | 8 +- src/contracts/wasm/wasmToolContracts.ts | 2 +- src/contracts/webScriptToolContracts.ts | 36 +- src/desktop/DebianPackageInventory.ts | 3 + src/desktop/LibraryPackageInventory.test.ts | 58 + src/desktop/LibraryPackageInventory.ts | 21 +- src/desktop/LinuxApplicationInventory.ts | 6 + src/domain/analysisErrorPresentation.ts | 10 + src/domain/analysisErrorProjection.ts | 20 +- src/domain/analysisResourceLimits.ts | 13 + src/domain/analysisSettingDefinitions.ts | 96 +- src/domain/analysisSettings.ts | 7 +- src/domain/analysisView/analysisView.ts | 8 + .../analysisView/analysisViewRequest.ts | 2 + src/domain/analysisView/analysisViewResult.ts | 35 + .../analysisView/artifactInventoryView.ts | 5 + src/domain/analysisView/goBinaryView.ts | 35 +- src/domain/analysisView/rustBinaryView.ts | 15 +- .../analysisView/webSourceLocationView.ts | 317 +++ src/domain/apktool/apktoolResourceAnalysis.ts | 78 +- src/domain/artifactGraph.ts | 7 + src/domain/artifactInventoryEvidence.ts | 11 + src/domain/artifactOperationError.ts | 9 +- src/domain/artifactReaderLimits.ts | 210 ++ src/domain/captures/packetCapture.ts | 14 +- src/domain/captures/packetFields.ts | 8 + src/domain/commandObservation.ts | 10 + src/domain/desktop/appExport.test.ts | 35 + src/domain/desktop/appExport.ts | 9 +- src/domain/desktop/appExportPython.ts | 114 +- src/domain/desktop/appExportPythonZip.ts | 10 +- src/domain/desktop/desktopState.ts | 9 + src/domain/desktop/desktopViewPolicy.ts | 126 ++ src/domain/desktop/elfDependencies.test.ts | 29 +- src/domain/desktop/elfDependencies.ts | 90 +- src/domain/desktop/gzipDocument.test.ts | 48 +- src/domain/desktop/gzipDocument.ts | 35 +- src/domain/desktop/libraryPolicy.ts | 45 + src/domain/desktop/stateFormats.test.ts | 27 + src/domain/desktop/stateFormats.ts | 73 +- src/domain/desktop/zipView.ts | 35 +- src/domain/evm/evmInterface.ts | 43 +- src/domain/evm/evmInterfaceEvidence.ts | 3 + src/domain/explicitSensitiveFailure.ts | 15 +- src/domain/flutter/flutterBuildAnalysis.ts | 47 + src/domain/fridaCapturePolicy.ts | 57 + src/domain/fridaSessionStatus.ts | 2 + src/domain/go/goBinary.ts | 157 +- .../electronWrappedCallPaths.test.ts | 4 +- .../javascript/electronWrappedCallPaths.ts | 10 +- .../javascriptAnalysisResourceControls.ts | 13 + .../javascript/javascriptAstValues.test.ts | 13 +- src/domain/javascript/javascriptAstValues.ts | 44 +- .../javascriptBundledRequireOrigin.ts | 3 +- .../javascriptCommonJsExportReceivers.ts | 5 +- .../javascriptElectronAppPathChanges.ts | 3 +- .../javascriptSemanticAliasSelections.ts | 12 +- .../javascript/javascriptSemanticAnalysis.ts | 35 +- .../javascript/javascriptSemanticCalls.ts | 2 +- .../javascript/javascriptSemanticCoverage.ts | 72 +- .../javascriptSemanticDataEffects.ts | 2 +- .../javascriptSemanticDerivedAnalysis.ts | 6 +- .../javascriptSemanticExpressionShapes.ts | 3 +- .../javascriptSemanticGetterQueries.ts | 6 +- .../javascriptSemanticMemberMutations.ts | 7 +- .../javascriptSemanticPrecision.test.ts | 106 + .../javascript/javascriptSemanticPrecision.ts | 41 + .../javascriptSemanticPrimitives.ts | 17 +- .../javascriptSemanticProjectedSources.ts | 5 +- .../javascriptSemanticRequireOrigin.ts | 9 +- .../javascriptSemanticResourceLimits.ts | 86 +- .../javascriptSemanticReturnedReferences.ts | 5 +- .../javascript/javascriptSemanticState.ts | 2 + .../javascriptSemanticValues.memo.test.ts | 7 +- .../javascript/javascriptSemanticValues.ts | 101 +- .../javascript/javascriptSourceAnalysis.ts | 2 + .../javascript/javascriptStaticAnalysis.ts | 39 +- .../javascriptStaticAnalysisState.ts | 2 +- .../javascript/javascriptStaticCoverage.ts | 2 + .../javascript/javascriptSymbolicBindings.ts | 40 +- src/domain/managed/managedInspectionLimits.ts | 22 + src/domain/native/binaryLayout.ts | 2 + src/domain/native/kernelModule.ts | 37 +- src/domain/native/nativeCallCaptureLimits.ts | 64 + src/domain/native/nativeCallObservation.ts | 11 + .../native/nativeCallPartialObservation.ts | 1 + src/domain/native/nativeDeadlines.ts | 2 +- src/domain/native/nativeValueFlow.ts | 16 + src/domain/native/pwntoolsResourceLimits.ts | 43 + src/domain/native/recordedCrash.ts | 2 + src/domain/producerResourceLimits.ts | 59 + src/domain/reverseEngineering.ts | 28 +- src/domain/rust/rustBinary.ts | 18 + src/domain/rust/rustResourceLimits.ts | 24 + src/domain/signals/signalDecoders.ts | 5 + src/domain/signals/signalRecording.ts | 14 +- src/domain/sourceMapLimits.ts | 130 ++ src/domain/sqlite/sqliteDatabase.ts | 3 + src/domain/wasm/wasmArtifact.ts | 80 +- src/domain/wasm/wasmInterface.ts | 14 +- src/domain/wasm/wasmInterfaceBudget.ts | 41 + src/domain/webBundleAnalysis.ts | 21 +- src/domain/webBundleAnalyzer.ts | 4 +- src/domain/webBundleAnalyzerInspection.ts | 8 +- src/domain/webDeadlineControls.test.ts | 16 +- src/domain/webNetworkCapture.ts | 100 +- src/domain/webSourceLocation.ts | 28 +- src/dotnet/ManagedArtifactInspector.ts | 2 + src/dotnet/ManagedDecodeBudget.ts | 20 +- src/dotnet/ManagedMemberInspector.ts | 15 +- src/dotnet/ManagedMemberRecovery.ts | 1 + src/dotnet/ManagedMetadataLayout.ts | 3 +- src/dotnet/ManagedNativeBoundaryInspector.ts | 11 +- src/dotnet/ManagedSnapshot.ts | 51 +- src/dotnet/ManagedStaticProvider.ts | 74 +- src/evm/EvmProducerLimits.test.ts | 63 + src/evm/EvmWorkerLimits.test.ts | 31 + src/evm/EvmWorkerLimits.ts | 12 +- src/evm/EvmoleFailures.test.ts | 84 +- src/evm/EvmoleFailures.ts | 83 +- src/evm/EvmoleInterfaceProvider.ts | 67 +- src/evm/EvmoleInterfaceWorker.ts | 17 +- src/evm/EvmoleRelease.ts | 14 +- src/firmware/FirmwareRelease.ts | 2 +- src/flutter/FlutterBuildProvider.ts | 92 +- src/frida/FridaInstrumentationManager.test.ts | 160 +- src/frida/FridaInstrumentationManager.ts | 72 +- src/frida/fridaCaptureLimits.ts | 64 +- src/gdb/GdbSessionManager.ts | 87 +- src/ghidra/GhidraAnalysisProfile.ts | 1 + src/ghidra/GhidraClient.ts | 11 + src/ghidra/GhidraClientTypes.ts | 3 + src/ghidra/GhidraClientWire.test.ts | 18 +- src/ghidra/GhidraClientWire.ts | 6 + src/ghidra/GhidraProvider.behavior.test.ts | 96 +- src/ghidra/GhidraProviderCapabilities.ts | 12 +- src/ghidra/GhidraProviderClient.ts | 27 +- src/ghidra/GhidraRawProfileDigest.test.ts | 8 +- src/ghidra/GhidraSnapshotCache.test.ts | 37 +- src/ghidra/GhidraTransport.ts | 14 +- src/go/GoBinaryImage.ts | 117 +- src/go/GoBinaryProvider.ts | 201 +- src/go/GoStringLiterals.test.ts | 74 +- src/go/GoStringLiterals.ts | 39 +- .../analysis/JavaScriptAnalysisWorker.ts | 3 + .../JavaScriptAnalysisWorkerProtocol.ts | 2 + .../JavaScriptAnalysisWorkerSchemas.ts | 13 + .../JavaScriptAnalysisWorkerServer.ts | 3 + src/javascript/recovery/WakaruRelease.ts | 2 +- .../sourceMaps/SourceMapAdmission.test.ts | 77 + .../sourceMaps/SourceMapCodecProcess.ts | 28 +- .../sourceMaps/SourceMapCommand.test.ts | 10 +- src/javascript/sourceMaps/SourceMapCommand.ts | 67 +- src/javascript/sourceMaps/SourceMapDecoder.ts | 32 +- src/javascript/sourceMaps/SourceMapFormat.ts | 79 +- src/javascript/sourceMaps/SourceMapReply.ts | 7 +- src/javascript/sourceMaps/TraceSourceMap.ts | 15 +- src/native/LldbCallTracer.ts | 87 +- src/native/LldbRetainedObservations.ts | 22 +- src/native/NativeCallObservation.ts | 26 +- src/native/elf/KernelModuleParser.test.ts | 18 +- src/native/elf/KernelModuleParser.ts | 50 +- src/native/pwndbg/PwndbgCoreContext.ts | 25 +- src/native/pwntools/PwntoolsDecoder.ts | 91 +- src/native/pwntools/PwntoolsFailures.test.ts | 17 +- src/native/pwntools/PwntoolsFailures.ts | 56 +- .../pwntools/PwntoolsInputLimit.test.ts | 8 +- src/native/pwntools/PwntoolsInputLimit.ts | 4 +- .../pwntools/PwntoolsRecordedCrashProvider.ts | 2 + src/native/pwntools/PwntoolsRelease.ts | 16 - src/native/pwntools/PwntoolsResourceLimits.ts | 29 +- src/pcap/PcapNativeBound.test.ts | 2 +- src/pcap/PcapNativeBound.ts | 6 +- src/process/ArtifactAnalysisResources.ts | 32 +- src/process/LinuxResourceLimits.test.ts | 2 +- src/process/LinuxResourceLimits.ts | 10 - src/rust/RustAuditableReader.ts | 34 +- src/rust/RustBinaryImage.ts | 91 +- src/rust/RustBinaryProvider.ts | 150 +- src/server/createServer.ts | 11 +- src/server/registerFridaTools.ts | 11 + src/server/registerWebSourceLocationTool.ts | 21 +- src/server/toolResult.ts | 50 +- src/signals/SigrokRecordingProvider.ts | 32 +- src/sqlite/SqliteDatabaseLimits.ts | 4 +- src/wasm/WabtArtifactProvider.test.ts | 115 +- src/wasm/WabtArtifactProvider.ts | 20 +- src/wasm/WabtFailures.test.ts | 28 +- src/wasm/WabtFailures.ts | 82 +- src/wasm/WabtInterface.test.ts | 29 +- src/wasm/WabtInterface.ts | 95 +- src/wasm/WabtRelease.ts | 5 - src/wasm/WabtWat.ts | 4 +- src/wasm/WasmInterfaceBudget.test.ts | 41 +- src/wasm/WasmInterfaceBudget.ts | 19 - src/windows/WindowsPrivateRuntime.test.ts | 29 +- tests/acceptance/analysisSettingsMcp.test.ts | 69 + .../javascriptAnalysisLimits.test.ts | 5 + .../artifactReaderLimitsCli.test.ts | 328 ++++ tests/acceptance/bunPathBudgetCli.test.ts | 15 +- tests/acceptance/dotnetBundleJourney.test.ts | 24 +- tests/acceptance/godotPck.test.ts | 9 +- tests/acceptance/kernelModuleCli.test.ts | 37 + tests/acceptance/managedCliOutput.test.ts | 77 + tests/acceptance/modelArtifactCli.test.ts | 3 +- tests/acceptance/nodeSeaCli.test.ts | 18 +- tests/acceptance/rustBinaryCli.test.ts | 38 +- tests/acceptance/rustDependencyViews.test.ts | 50 + .../registrationRuntimeEnvironmentCli.test.ts | 5 + tests/acceptance/webSourceLocationCli.test.ts | 339 +++- .../pwntoolsLayoutProvider.test.ts | 32 +- .../browser/historicalCaptureDecoder.test.ts | 9 +- .../browser/webSourceMapFetcher.test.ts | 14 +- .../webSourceMapResponseIdentity.test.ts | 8 +- .../boundary/captures/captureProvider.test.ts | 59 +- .../cli/apktoolBooleanOptions.test.ts | 35 + .../boundary/cli/artifactInputLimits.test.ts | 8 +- .../cli/cliInputSchemaRequired.test.ts | 7 +- tests/boundary/cli/desktopCaptureCli.test.ts | 139 +- .../boundary/cli/environmentBuildCli.test.ts | 11 +- tests/boundary/cli/flutterCli.test.ts | 21 +- .../cli/managedMemberComparisonCli.test.ts | 18 +- .../cli/reverseEngineeringCommands.test.ts | 14 +- .../evm/evmoleInterfaceProvider.test.ts | 8 + .../evm/solidityMetadataWorker.test.ts | 2 + .../filesystem/asarListingAdmission.test.ts | 40 +- .../directAnalysisSnapshotReplay.test.ts | 32 +- .../filesystem/dotnetBundleArtifacts.test.ts | 119 +- .../filesystem/evidenceBundleFiles.test.ts | 11 +- .../filesystem/flutterBuildProvider.test.ts | 91 + .../javascriptArtifactReconstruction.test.ts | 8 +- tests/boundary/go/goBinaryImage.test.ts | 4 +- tests/boundary/go/goBinaryProvider.test.ts | 88 + .../javascriptSourceAnalysisWorker.test.ts | 55 + .../javascript/sourceMapDecoder.test.ts | 114 +- tests/boundary/mcp/dartAotSummaryMcp.test.ts | 48 +- tests/boundary/mcp/fridaLifecycleMcp.test.ts | 55 +- tests/boundary/mcp/ghidraEvidenceMcp.test.ts | 25 +- tests/boundary/mcp/ghidraNativeApiMcp.test.ts | 44 +- tests/boundary/mcp/goBinaryViews.test.ts | 18 + tests/boundary/mcp/kernelModuleMcp.test.ts | 60 + tests/boundary/mcp/nodeSeaMcp.test.ts | 66 +- tests/boundary/mcp/rustBinaryMcp.test.ts | 30 +- .../boundary/mcp/webSourceLocationMcp.test.ts | 40 + .../native/pwndbgReplyFinalization.test.ts | 2 + .../GdbSessionManagerCoverage.test.ts | 45 +- .../providers/desktopExportPython.test.ts | 148 +- .../providers/desktopExportSeedPolicy.test.ts | 152 ++ .../providers/desktopSqliteViewBudget.test.ts | 229 +++ .../providers/desktopTaskPython.test.ts | 7 +- .../desktopXmlWhitespacePython.test.ts | 28 +- .../managed/managedMemberComparison.test.ts | 10 +- .../managed/managedStaticProvider.test.ts | 83 +- .../native/lldbCallTracerFailures.test.ts | 132 +- .../native/lldbRetainedObservations.test.ts | 45 +- .../native/nativeCallObservation.test.ts | 23 +- tests/boundary/rust/rustBinaryBudget.test.ts | 87 +- tests/boundary/rust/rustBinaryImage.test.ts | 25 +- .../signals/recordingProvider.test.ts | 39 +- tests/composition/reverseEngineering.test.ts | 73 +- .../reverseEngineeringAdmission.test.ts | 180 +- .../reverseEngineeringEdgeCases.test.ts | 7 +- tests/fixtures/rust/image.ts | 13 +- .../browser/analyzeWebBundleFixture.ts | 2 +- 515 files changed, 21066 insertions(+), 3637 deletions(-) create mode 100644 docs/specs/analysis/recoverable-budgets/plan.md create mode 100644 docs/specs/analysis/recoverable-budgets/spec.md create mode 100644 docs/specs/analysis/recoverable-budgets/tasks.md create mode 100644 docs/specs/evm/producer-limits/plan.md create mode 100644 docs/specs/evm/producer-limits/spec.md create mode 100644 docs/specs/evm/producer-limits/tasks.md create mode 100644 docs/specs/web/historical-capture-budgets/plan.md create mode 100644 docs/specs/web/historical-capture-budgets/spec.md create mode 100644 docs/specs/web/historical-capture-budgets/tasks.md create mode 100644 docs/specs/web/source-map-budget-recovery/plan.md create mode 100644 docs/specs/web/source-map-budget-recovery/spec.md create mode 100644 docs/specs/web/source-map-budget-recovery/tasks.md create mode 100644 docs/verification/flutter-budget-calibration.md create mode 100644 docs/verification/javascript-precision-calibration.md create mode 100644 docs/verification/native-capture-budget-qualification-2026-11-09.json create mode 100644 docs/verification/runtime-analysis-settings-ghidra-2026-11-09.json create mode 100644 docs/verification/runtime-analysis-settings-ghidra-integration-2026-11-09.json create mode 100644 docs/verification/source-location-delivery.md create mode 100644 docs/verification/source-map-budget-calibration.md create mode 100644 docs/verification/source-map-budgets-2026-11-09/acquire.py create mode 100644 docs/verification/source-map-budgets-2026-11-09/artifacts.json create mode 100644 docs/verification/source-map-budgets-2026-11-09/measurements.json create mode 100644 docs/verification/source-map-budgets-2026-11-09/oracle.py create mode 100644 docs/verification/source-map-budgets-2026-11-09/workload.mjs create mode 100644 scripts/verify/analysis/budget-calibration-run.py create mode 100644 scripts/verify/analysis/flutter-budget-calibration/aggregate.py create mode 100644 scripts/verify/analysis/flutter-budget-calibration/configs.py create mode 100644 scripts/verify/analysis/flutter-budget-calibration/manifest.json create mode 100644 scripts/verify/analysis/flutter-budget-calibration/measurements.json create mode 100644 scripts/verify/analysis/flutter-budget-calibration/oracle.py create mode 100644 scripts/verify/analysis/flutter-budget-calibration/task.mjs create mode 100644 scripts/verify/analysis/javascript-precision-calibration/aggregate.py create mode 100644 scripts/verify/analysis/javascript-precision-calibration/manifest.json create mode 100644 scripts/verify/analysis/javascript-precision-calibration/measurements.json create mode 100644 scripts/verify/analysis/javascript-precision-calibration/prepare.py create mode 100644 scripts/verify/analysis/javascript-precision-calibration/scan.mjs create mode 100644 scripts/verify/analysis/javascript-precision-calibration/source-inventory.json create mode 100644 scripts/verify/analysis/javascript-precision-calibration/task.py create mode 100644 scripts/verify/analysis/source-location-delivery/aggregate.py create mode 100644 scripts/verify/analysis/source-location-delivery/measurements.json create mode 100644 scripts/verify/analysis/source-location-delivery/task.mjs create mode 100644 src/application/analysisView/WebSourceLocationViews.test.ts create mode 100644 src/artifacts/ArtifactReaderLimits.ts create mode 100644 src/browser/history/CaptureBudgets.ts create mode 100644 src/browser/history/HistoricalCaptureBudgets.test.ts create mode 100644 src/cli/producerLimitsOption.ts create mode 100644 src/cli/sourceMapLimits.ts create mode 100644 src/config/asarListingAdmission.ts create mode 100644 src/config/ghidraValueFlowLimits.ts create mode 100644 src/domain/analysisView/webSourceLocationView.ts create mode 100644 src/domain/artifactReaderLimits.ts create mode 100644 src/domain/desktop/desktopViewPolicy.ts create mode 100644 src/domain/desktop/libraryPolicy.ts create mode 100644 src/domain/fridaCapturePolicy.ts create mode 100644 src/domain/javascript/javascriptSemanticPrecision.test.ts create mode 100644 src/domain/javascript/javascriptSemanticPrecision.ts create mode 100644 src/domain/managed/managedInspectionLimits.ts create mode 100644 src/domain/native/nativeCallCaptureLimits.ts create mode 100644 src/domain/native/pwntoolsResourceLimits.ts create mode 100644 src/domain/producerResourceLimits.ts create mode 100644 src/domain/rust/rustResourceLimits.ts create mode 100644 src/domain/sourceMapLimits.ts create mode 100644 src/domain/wasm/wasmInterfaceBudget.ts create mode 100644 src/evm/EvmProducerLimits.test.ts delete mode 100644 src/wasm/WasmInterfaceBudget.ts create mode 100644 tests/acceptance/artifactReaderLimitsCli.test.ts create mode 100644 tests/boundary/providers/desktopExportSeedPolicy.test.ts create mode 100644 tests/boundary/providers/desktopSqliteViewBudget.test.ts diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 55ef0d1676..82d75c5cc2 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -119,9 +119,9 @@ import ghidra.program.util.VarnodeContext; public final class ReaGhidraBridge extends HeadlessScript { // At most 24 KiB after worst-case JSON escaping (six bytes per UTF-16 unit). private static final int PDB_ATTRIBUTE_UTF16_BUDGET = 4096; - private static final int MAX_HIGH_PCODE_OPS = 3000; - private static final int MAX_HIGH_PCODE_INPUTS_PER_OP = 64; - private static final int MAX_HIGH_PCODE_DEF_USE_EDGES = 12000; + private static final int DEFAULT_HIGH_PCODE_OPS = 3000; + private static final int DEFAULT_HIGH_PCODE_INPUTS_PER_OP = 64; + private static final int DEFAULT_HIGH_PCODE_DEF_USE_EDGES = 12000; private static final Gson GSON = new GsonBuilder().serializeNulls().create(); private final Map targetSourceIdentity = new IdentityHashMap<>(); private String admittedTargetSha256; @@ -184,6 +184,10 @@ public final class ReaGhidraBridge extends HeadlessScript { private String identityAdmissionFailure; /** Per-function native decompiler limit from the session; 0 leaves it unbounded. */ private int decompileTimeoutSeconds; + private Integer decompileMaxPayloadMiB; + // DecompileProcess converts MiB to bytes using a signed-int left shift. + private static final int MAX_DECOMPILE_PAYLOAD_MIB = Integer.MAX_VALUE >> 20; + private ValueFlowLimits valueFlowLimits = defaultValueFlowLimits(); // DecompileProcess converts seconds to an int millisecond timer delay. private static final int MAX_DECOMPILE_TIMEOUT_SECONDS = Integer.MAX_VALUE / 1000; private final ReaGhidraExtensions analysisExtensions = new ReaGhidraExtensions(); @@ -426,6 +430,8 @@ public final class ReaGhidraBridge extends HeadlessScript { } decompileTimeoutSeconds = request.decompileTimeoutSeconds() == null ? descriptor.decompileTimeoutSeconds() : request.decompileTimeoutSeconds(); + valueFlowLimits = request.valueFlowLimits(); + applyDecompilePayloadLimit(request.decompileMaxPayloadMiB()); writeSuccess(writer, request.id, handleRequest(request, descriptor)); } catch (RequestFailure failure) { @@ -1038,11 +1044,26 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } - private void initializeDecompiler() { + private DecompileOptions decompileOptions(Integer payloadMiB) { DecompileOptions options = new DecompileOptions(); options.setMaxInstructions(Integer.MAX_VALUE); + if (payloadMiB != null) options.setMaxPayloadMBytes(payloadMiB); + return options; + } + + private void applyDecompilePayloadLimit(Integer payloadMiB) { + if (java.util.Objects.equals(decompileMaxPayloadMiB, payloadMiB)) return; + // DecompInterface supports live setOptions: it updates the existing + // DecompileProcess result-size bound before sending native options. + if (!decompiler.setOptions(decompileOptions(payloadMiB))) + throw new RequestFailure("decompile_options_failed", "Ghidra could not apply the selected native decompiler payload limit"); + decompiler.flushCache(); + decompileMaxPayloadMiB = payloadMiB; + } + + private void initializeDecompiler() { decompiler = new DecompInterface(); - decompiler.setOptions(options); + decompiler.setOptions(decompileOptions(decompileMaxPayloadMiB)); decompiler.toggleCCode(true); decompiler.toggleSyntaxTree(true); decompiler.toggleJumpLoads(true); @@ -1856,7 +1877,7 @@ public final class ReaGhidraBridge extends HeadlessScript { while (operations.hasNext()) { monitor.checkCancelled(); PcodeOpAST operation = operations.next(); - if (selected.size() >= MAX_HIGH_PCODE_OPS) { + if (selected.size() >= valueFlowLimits.maxOperations()) { omittedOperationsLowerBound = 1; break; } @@ -1884,6 +1905,7 @@ public final class ReaGhidraBridge extends HeadlessScript { JsonArray effects = new JsonArray(); int omittedInputs = 0; int omittedEdges = 0; + boolean edgeLimitReached = false; for (PcodeOpAST operation : selected) { monitor.checkCancelled(); String operationId = operationIds.get(operation); @@ -1899,14 +1921,16 @@ public final class ReaGhidraBridge extends HeadlessScript { value.addProperty("block_membership", operation.getParent() == null ? "detached" : "member"); JsonArray inputs = new JsonArray(); int inputCount = operation.getNumInputs(); - int capturedInputs = Math.min(inputCount, MAX_HIGH_PCODE_INPUTS_PER_OP); + int capturedInputs = Math.min(inputCount, valueFlowLimits.maxInputsPerOperation()); for (int inputIndex = 0; inputIndex < capturedInputs; inputIndex += 1) { Varnode input = operation.getInput(inputIndex); inputs.add(varnodeValue(input)); if (input instanceof VarnodeAST astInput) { if (astInput.getHigh() instanceof HighParam parameter) { - if (parameterUses.size() >= MAX_HIGH_PCODE_DEF_USE_EDGES) omittedEdges++; - else { + if (parameterUses.size() >= valueFlowLimits.maxEdges()) { + edgeLimitReached = true; + omittedEdges++; + } else { JsonObject use = new JsonObject(); use.addProperty("ordinal", parameter.getSlot()); use.addProperty("use", operationId); @@ -1917,7 +1941,8 @@ public final class ReaGhidraBridge extends HeadlessScript { PcodeOp definition = astInput.getDef(); if (definition instanceof PcodeOpAST astDefinition) { String definitionId = operationIds.get(astDefinition); - if (definitionId == null || defUse.size() >= MAX_HIGH_PCODE_DEF_USE_EDGES) { + if (definitionId == null || defUse.size() >= valueFlowLimits.maxEdges()) { + if (defUse.size() >= valueFlowLimits.maxEdges()) edgeLimitReached = true; omittedEdges += 1; } else { JsonObject edge = new JsonObject(); @@ -1940,6 +1965,16 @@ public final class ReaGhidraBridge extends HeadlessScript { JsonObject result = new JsonObject(); result.addProperty("available", true); result.addProperty("provenance", "ghidra-high-pcode"); + JsonObject limits = new JsonObject(); + limits.addProperty("max_operations", valueFlowLimits.maxOperations()); + limits.addProperty("max_inputs_per_operation", valueFlowLimits.maxInputsPerOperation()); + limits.addProperty("max_edges", valueFlowLimits.maxEdges()); + result.add("limits", limits); + JsonArray limitsReached = new JsonArray(); + if (omittedOperationsLowerBound > 0) limitsReached.add("max_operations"); + if (omittedInputs > 0) limitsReached.add("max_inputs_per_operation"); + if (edgeLimitReached) limitsReached.add("max_edges"); + result.add("limits_reached", limitsReached); result.add("operations", operationValues); result.add("def_use", defUse); result.add("parameters", parameterValues); @@ -1968,15 +2003,35 @@ public final class ReaGhidraBridge extends HeadlessScript { "Memory read/write effects include stack and temporary memory operations; they do not identify persistent state or business meaning." ); limitations.add( - "Operation, input, and def-use limits can truncate large functions; inspect truncation, omitted lower bounds, and known omitted counts before drawing conclusions." + "Operation, input, and def-use limits can truncate large functions; inspect the effective limits, limits_reached and omitted counts before drawing conclusions." ); limitations.add( "Known omitted-edge counts cover links omitted from retained operations; omitted operations may contain additional uncounted links." ); + if (omittedOperationsLowerBound > 0) + limitations.add(valueFlowLimitRecovery("max_operations", valueFlowLimits.maxOperations())); + if (omittedInputs > 0) + limitations.add(valueFlowLimitRecovery("max_inputs_per_operation", valueFlowLimits.maxInputsPerOperation())); + if (edgeLimitReached) + limitations.add(valueFlowLimitRecovery("max_edges", valueFlowLimits.maxEdges())); result.add("limitations", limitations); return result; } + private static String valueFlowLimitRecovery(String field, int effective) { + String setting = "ghidra.value_flow_" + field; + String environment = "REA_GHIDRA_VALUE_FLOW_" + field.toUpperCase(Locale.ROOT); + String adjustment = effective < Integer.MAX_VALUE + ? "In MCP, use update_analysis_settings with setting " + setting + + ", action set and a larger integer value from 1 to " + Integer.MAX_VALUE + + ", then repeat the selected function analysis on this Program without restarting REA. " + + "For a one-shot CLI retry, set " + environment + + " to a larger count in the environment that starts the next rea invocation, then repeat the selected analysis; the previous invocation's Program is no longer active." + : "The setting is at its Java integer maximum; select a smaller function for this projection."; + return setting + " reached its effective value of " + effective + ". " + adjustment + + " These controls do not raise the decompile deadline, bridge transport or MCP client limits."; + } + private String pcodeOperationId(PcodeOpAST operation) { return canonicalAddress(operation.getSeqnum().getTarget()) + "#" + operation.getSeqnum().getTime(); @@ -3699,12 +3754,26 @@ public final class ReaGhidraBridge extends HeadlessScript { if (!results.decompileCompleted()) { throw new RequestFailure( "decompile_failed", - "Ghidra decompilation failed: " + diagnosticMessage(results.getErrorMessage()) + "Ghidra decompilation failed: " + diagnosticMessage(results.getErrorMessage()) + + ". The effective native decompiler payload allowance is " + + decompiler.getOptions().getMaxPayloadMBytes() + + " MiB. " + decompilePayloadRecovery() ); } return results; } + private String decompilePayloadRecovery() { + int effective = decompiler.getOptions().getMaxPayloadMBytes(); + if (effective >= MAX_DECOMPILE_PAYLOAD_MIB) + return "If the diagnostic reports payload exhaustion, the native payload allowance is already at its supported maximum; no higher value is accepted. Use analyze_function for available listing facts or explicitly select a smaller function for decompilation."; + return "If the diagnostic reports payload exhaustion, in MCP raise ghidra.decompile_max_payload_mib with update_analysis_settings " + + "(integer MiB from 1 to " + MAX_DECOMPILE_PAYLOAD_MIB + + ") and repeat the selected operation on this Program without restarting REA. " + + "For a one-shot CLI retry, set REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB to a larger MiB value in the environment that starts the next rea invocation, then repeat the selected operation; the previous invocation's Program is no longer active. " + + "Runtime null or an omitted launch seed restores the installed Ghidra default. This does not raise the decompile deadline, high-pcode projection budgets or MCP client limits."; + } + private static String decompiledText(DecompileResults results) { if (results == null) { return null; @@ -5156,7 +5225,28 @@ public final class ReaGhidraBridge extends HeadlessScript { Set keys = new HashSet<>(REQUEST_KEYS); if (parsed.isJsonObject() && parsed.getAsJsonObject().has("decompile_timeout_seconds")) keys.add("decompile_timeout_seconds"); + if (parsed.isJsonObject() && parsed.getAsJsonObject().has("value_flow_limits")) + keys.add("value_flow_limits"); + if (parsed.isJsonObject() && parsed.getAsJsonObject().has("decompile_max_payload_mib")) + keys.add("decompile_max_payload_mib"); JsonObject object = requireObject(parsed, keys); + Integer payloadMiB = null; + if (object.has("decompile_max_payload_mib") && !object.get("decompile_max_payload_mib").isJsonNull()) { + payloadMiB = requireInteger(object, "decompile_max_payload_mib"); + if (payloadMiB < 1 || payloadMiB > MAX_DECOMPILE_PAYLOAD_MIB) + throw new IllegalArgumentException("Invalid per-request native decompiler payload limit"); + } + ValueFlowLimits flowLimits = defaultValueFlowLimits(); + if (object.has("value_flow_limits")) { + JsonObject limits = requireObject(object.get("value_flow_limits"), + Set.of("max_operations", "max_inputs_per_operation", "max_edges")); + int operations = requireInteger(limits, "max_operations"); + int inputs = requireInteger(limits, "max_inputs_per_operation"); + int edges = requireInteger(limits, "max_edges"); + if (operations < 1 || inputs < 1 || edges < 1) + throw new IllegalArgumentException("Invalid per-request value-flow limits"); + flowLimits = new ValueFlowLimits(operations, inputs, edges); + } Integer timeout = null; if (object.has("decompile_timeout_seconds")) { JsonElement value = object.get("decompile_timeout_seconds"); @@ -5172,7 +5262,7 @@ public final class ReaGhidraBridge extends HeadlessScript { if (params == null || !params.isJsonObject()) { throw new IllegalArgumentException("Bridge request parameters are invalid"); } - return new Request(id, requireString(object, "method"), params.getAsJsonObject(), timeout); + return new Request(id, requireString(object, "method"), params.getAsJsonObject(), timeout, payloadMiB, flowLimits); } private static JsonObject requireObject(JsonElement element, Set keys) { @@ -5380,7 +5470,11 @@ public final class ReaGhidraBridge extends HeadlessScript { int decompileTimeoutSeconds, boolean existingProject ) {} - private record Request(int id, String method, JsonObject params, Integer decompileTimeoutSeconds) {} + private static ValueFlowLimits defaultValueFlowLimits() { + return new ValueFlowLimits(DEFAULT_HIGH_PCODE_OPS, DEFAULT_HIGH_PCODE_INPUTS_PER_OP, DEFAULT_HIGH_PCODE_DEF_USE_EDGES); + } + private record ValueFlowLimits(int maxOperations, int maxInputsPerOperation, int maxEdges) {} + private record Request(int id, String method, JsonObject params, Integer decompileTimeoutSeconds, Integer decompileMaxPayloadMiB, ValueFlowLimits valueFlowLimits) {} private static final class RequestFailure extends RuntimeException { private final String code; diff --git a/bridge/mitmproxy/capture.py b/bridge/mitmproxy/capture.py index c5ca704f69..910fa6e688 100644 --- a/bridge/mitmproxy/capture.py +++ b/bridge/mitmproxy/capture.py @@ -18,7 +18,6 @@ from mitmproxy import ctx, version from mitmproxy.io import tnetstring PROFILE = "12.2.3" -OUTPUT_BYTES = 96 * 1024 * 1024 CREDENTIALS = {b"authorization", b"proxy-authorization", b"cookie", b"set-cookie"} @@ -204,7 +203,7 @@ def without_userinfo(value, base_scheme=None): raise CaptureFailure("decoder", "Normalized URL authority could not be bound to its source.") -def project_record(state, secrets): +def project_record(state, secrets, max_depth): binaries, numbers, redactions = [], [], [] byte_secrets = [] for secret in secrets: @@ -251,8 +250,8 @@ def project_record(state, secrets): del redactions[before[2]:] def visit(value, path, depth=0, credential=False, transport_url=False, base_scheme=None): - if depth > 64: - raise CaptureFailure("input-limit", "Native capture exceeds the 64-level evidence nesting budget.", path) + if depth > max_depth: + raise CaptureFailure("input-limit", f"Native capture exceeds max_nesting_depth ({max_depth} levels).", path) if credential: validate_omitted(value, path, depth) redactions.append({"pointer": path, "reason": "transport-credential"}) @@ -337,12 +336,12 @@ def decode(request): except CaptureFailure: raise except RecursionError: - raise CaptureFailure("input-limit", "Native parser exceeded its recursion budget at byte offset " + str(start) + ".") from None + raise CaptureFailure("unsupported", "Native parser reached Python's recursion limit at byte offset " + str(start) + ". max_nesting_depth does not raise that upstream limit.") from None except (ValueError, IndexError, UnicodeError, EOFError): raise CaptureFailure("format", "Malformed native record at byte offset " + str(start) + ".") from None if not isinstance(state, dict): raise CaptureFailure("format", "Native flow record must be a dictionary at byte offset " + str(start) + ".") - records.append({"ordinal": len(records), "location": {"kind": "byte-range", "offset": start, "bytes": handle.tell() - start}, **project_record(state, request["sensitive_values"]), "limitations": ["Fields are original native states without FlowReader migration. Binary fields contain exact producer bytes; reported UTF-8 strings for those fields are derived display views. Unknown flow/version extensions are retained without interpretation."]}) + records.append({"ordinal": len(records), "location": {"kind": "byte-range", "offset": start, "bytes": handle.tell() - start}, **project_record(state, request["sensitive_values"], request["budgets"]["depth"]), "limitations": ["Fields are original native states without FlowReader migration. Binary fields contain exact producer bytes; reported UTF-8 strings for those fields are derived display views. Unknown flow/version extensions are retained without interpretation."]}) return {"decoder": {"id": "mitmproxy-native-tnetstring", "name": "REA offline native mitmproxy adapter", "version": PROFILE}, "container": {"reported": None, "numeric_literals": [], "redactions": [], "records_pointer": None}, "total_records": len(records), "records": records} @@ -350,6 +349,8 @@ def failure_reply(error): """Keep producer constraints, resource exhaustion and adapter defects distinct.""" if isinstance(error, CaptureFailure): return {"ok": False, "reason": error.reason, "message": str(error), "pointer": error.pointer} + if isinstance(error, RecursionError): + return {"ok": False, "reason": "unsupported", "message": "Native projection reached Python's recursion limit; max_nesting_depth does not raise that upstream limit.", "pointer": ""} if isinstance(error, MemoryError): return {"ok": False, "reason": "resource-limit", "message": "Native decoder exhausted its 768 MiB address-space budget.", "pointer": ""} return {"ok": False, "reason": "decoder", "message": "Offline native decoder failed unexpectedly: " + type(error).__name__ + ".", "pointer": ""} @@ -362,9 +363,13 @@ class OfflineCapture: def running(self): resource.setrlimit(resource.RLIMIT_AS, (768 * 1024 * 1024, 768 * 1024 * 1024)) resource.setrlimit(resource.RLIMIT_CPU, (30, 30)) - resource.setrlimit(resource.RLIMIT_FSIZE, (OUTPUT_BYTES, OUTPUT_BYTES)) resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) request = json.loads(Path(ctx.options.rea_request_path).read_text(encoding="utf-8")) + budgets = request["budgets"] + if any(type(budgets[key]) is not int or budgets[key] < 1 for key in ("inputBytes", "outputBytes", "depth")): + raise ValueError("Invalid admitted capture budgets") + output_bytes = budgets["outputBytes"] + resource.setrlimit(resource.RLIMIT_FSIZE, (output_bytes, output_bytes)) try: response = {"ok": True, "value": decode(request)} except Exception as error: @@ -374,7 +379,7 @@ class OfflineCapture: output = json.dumps(response, ensure_ascii=True, allow_nan=False, separators=(",", ":")).encode("ascii") except Exception as error: output = json.dumps(failure_reply(error), separators=(",", ":")).encode("ascii") - if len(output) > OUTPUT_BYTES: + if len(output) > output_bytes: output = b'{"ok":false,"reason":"limit","message":"Native reply exceeds the complete-evidence output budget.","pointer":""}' Path(request["reply_path"]).write_bytes(output) finally: diff --git a/bridge/mitmproxy/test_capture.py b/bridge/mitmproxy/test_capture.py index 3863312cb9..6b606f7997 100644 --- a/bridge/mitmproxy/test_capture.py +++ b/bridge/mitmproxy/test_capture.py @@ -36,7 +36,7 @@ def location(value, scheme=None): state = {"response": {"headers": [[b"Location", value]]}} if scheme is not None: state["request"] = {"scheme": scheme} - record = capture.project_record(state, []) + record = capture.project_record(state, [], 64) sidecar = next( field for field in record["binary_fields"] if field["pointer"] == "/response/headers/0/1" ) @@ -75,11 +75,11 @@ class TransportUserinfoTests(unittest.TestCase): def test_native_request_paths_preserve_origin_form_and_redact_absolute_form(self): for value in [b"/\\invoice@customer", b"//invoice@customer", b"\\\\invoice@customer"]: with self.subTest(value=value): - record = capture.project_record({"request": {"path": value}}, []) + record = capture.project_record({"request": {"path": value}}, [], 64) self.assertEqual(record["reported"]["request"]["path"], value.decode("ascii")) self.assertEqual(record["binary_fields"][0]["state"], "retained") self.assertEqual(record["redactions"], []) - record = capture.project_record({"request": {"path": b"https://user:pass@host/x"}}, []) + record = capture.project_record({"request": {"path": b"https://user:pass@host/x"}}, [], 64) self.assertEqual(record["reported"]["request"]["path"], "https://host/x") self.assertEqual(record["binary_fields"][0]["state"], "redacted") @@ -110,7 +110,7 @@ class TransportUserinfoTests(unittest.TestCase): "request": {"scheme": b"https"}, "response": {"headers": [[b"Location", b"https:invoice@customer"]]}, }, - }, []) + }, [], 64) self.assertEqual(record["reported"]["backup"]["response"]["headers"][0][1], "https:invoice@customer") self.assertEqual(record["redactions"], []) diff --git a/bridge/native/rea_lldb_tracer.py b/bridge/native/rea_lldb_tracer.py index b6f5fef42b..03eeaeebbd 100644 --- a/bridge/native/rea_lldb_tracer.py +++ b/bridge/native/rea_lldb_tracer.py @@ -25,24 +25,17 @@ ARGUMENT_REGISTERS = { "x86_64": ["rdi", "rsi", "rdx", "rcx", "r8", "r9"], "x86_64h": ["rdi", "rsi", "rdx", "rcx", "r8", "r9"], } -MAX_SELECTOR_BYTES = 1024 -MAX_OTHER_STOPS = 256 -MAX_TRACE_BYTES = 8 * 1024 * 1024 -MAX_TRACE_FRAMES = 65_536 -MAX_OBSERVATION_JOURNAL_BYTES = MAX_TRACE_BYTES -MAX_BREAKPOINT_LOCATION_BYTES = 8 * 1024 * 1024 -MAX_TARGET_OUTPUT_BYTES = 1024 * 1024 OUTPUT_DRAIN_BYTES_PER_PASS = 1024 * 1024 OUTPUT_EOF_WAIT_SECONDS = 1.0 OUTPUT_READER_JOIN_SECONDS = 0.5 LISTENER_NAME = "rea-native-call-observation" -def _can_retain_trace(current_bytes, current_frames, event_bytes, event_frames): +def _can_retain_trace(current_bytes, current_frames, event_bytes, event_frames, limits): """Whether a complete event fits both aggregate trace budgets.""" return ( - current_bytes + event_bytes <= MAX_TRACE_BYTES - and current_frames + event_frames <= MAX_TRACE_FRAMES + current_bytes + event_bytes <= limits["trace_bytes"] + and current_frames + event_frames <= limits["trace_frames"] ) @@ -71,7 +64,7 @@ def _fits_trace_budget(value, remaining_bytes): class _ObservationJournal: """Flush a bounded JSONL prefix so observations survive tracer failure.""" - def __init__(self, handle, max_bytes=MAX_OBSERVATION_JOURNAL_BYTES): + def __init__(self, handle, max_bytes): self.handle = handle self.max_bytes = max_bytes self.bytes = 0 @@ -371,7 +364,7 @@ def _record(context, thread, frame, index, sequence, location, remaining_bytes): error = lldb.SBError() text = process.ReadCStringFromMemory( values[names[1]], - MAX_SELECTOR_BYTES, + config["capture_limits"]["selector_bytes"], error, ) selector = text if error.Success() and text else None @@ -404,10 +397,16 @@ def _record(context, thread, frame, index, sequence, location, remaining_bytes): _TRACE_CONTEXT = None +def _resource_limit(context, limit): + context["resource_limit"] = limit + return "resource-limit" + + def _handle_breakpoint(context, frame, index): """Read one hit while LLDB's breakpoint callback owns the stopped frame.""" if not frame.IsValid() or frame.GetPC() == lldb.LLDB_INVALID_ADDRESS: raise RuntimeError("LLDB breakpoint callback did not supply a readable frame") + limits = context["config"]["capture_limits"] thread = frame.GetThread() events = context["events"] location = _location(frame.GetPCAddress(), context["target"]) @@ -424,49 +423,50 @@ def _handle_breakpoint(context, frame, index): context["retained_frames"], 0, event_frame_count, + limits, ): - return "resource-limit" + return _resource_limit(context, "trace_frames") try: - remaining_bytes = MAX_TRACE_BYTES - context["retained_trace_bytes"] + remaining_bytes = limits["trace_bytes"] - context["retained_trace_bytes"] record = _record( context, thread, frame, index, len(events), location, remaining_bytes ) except ValueError: - return "resource-limit" + return _resource_limit(context, "trace_bytes") if record is not None: if not _fits_trace_budget(record, remaining_bytes): - return "resource-limit" + return _resource_limit(context, "trace_bytes") record_bytes = len(json.dumps(record).encode("utf-8")) - if context["retained_trace_bytes"] + record_bytes > MAX_TRACE_BYTES: - return "resource-limit" + if context["retained_trace_bytes"] + record_bytes > limits["trace_bytes"]: + return _resource_limit(context, "trace_bytes") for depth in range(1, frame_count + 1): try: location = _location( thread.GetFrameAtIndex(depth).GetPCAddress(), context["target"] ) except ValueError: - return "resource-limit" + return _resource_limit(context, "trace_bytes") if ( _estimated_json_size(location) + record_bytes + 1 - > MAX_TRACE_BYTES - context["retained_trace_bytes"] + > limits["trace_bytes"] - context["retained_trace_bytes"] ): - return "resource-limit" + return _resource_limit(context, "trace_bytes") location_bytes = len(json.dumps(location).encode("utf-8")) addition = location_bytes + (1 if record["backtrace"] else 0) if ( context["retained_trace_bytes"] + record_bytes + addition - > MAX_TRACE_BYTES + > limits["trace_bytes"] ): - return "resource-limit" + return _resource_limit(context, "trace_bytes") record["backtrace"].append(location) record_bytes += addition journal = context.get("observation_journal") if journal is not None and not journal.append( {"kind": "event", "event": record} ): - return "resource-limit" + return _resource_limit(context, "trace_bytes") events.append(record) context["retained_frames"] += event_frame_count context["retained_trace_bytes"] += record_bytes @@ -498,9 +498,9 @@ def _handle_stop(context, other_stops): if journal is not None and not journal.append( {"kind": "other-stop", "reason": stop_description} ): - return "resource-limit" + return _resource_limit(context, "trace_bytes") other_stops.append(stop_description) - if len(other_stops) >= MAX_OTHER_STOPS: + if len(other_stops) >= context["config"]["capture_limits"]["other_stops"]: return "stop-limit" return None @@ -520,7 +520,7 @@ def _breakpoint_report(target, created, config): continue count += 1 remaining_bytes = ( - MAX_BREAKPOINT_LOCATION_BYTES - retained_location_bytes + config["capture_limits"]["breakpoint_location_bytes"] - retained_location_bytes ) if not _fits_trace_budget(location, remaining_bytes): locations_truncated = True @@ -651,7 +651,7 @@ def trace(debugger, config): for breakpoint in breakpoints: breakpoint_index[breakpoint.GetID()] = index listener = lldb.SBListener(LISTENER_NAME) - output_budget = config.get("max_output_bytes", MAX_TARGET_OUTPUT_BYTES) + output_budget = config["capture_limits"]["output_bytes"] stdout = _OutputPipe( config["stdout_path"], config["stdout_capture_path"], output_budget ) @@ -671,7 +671,8 @@ def trace(debugger, config): observation_path = config.get("observation_path") if observation_path: observation_journal = _ObservationJournal( - open(observation_path, "wb") + open(observation_path, "wb"), + config["capture_limits"]["trace_bytes"], ) process = target.Launch(_launch_info(target, config, listener), error) process_created = process.IsValid() @@ -862,6 +863,7 @@ def trace(debugger, config): "breakpoint_locations_truncated": breakpoint_locations_truncated, "events": events, "resource_limit_reached": outcome == "resource-limit", + "resource_limit": context.get("resource_limit"), "target_output": { "stdout_bytes": stdout.bytes, "stderr_bytes": stderr.bytes, diff --git a/bridge/native/test_rea_lldb_tracer.py b/bridge/native/test_rea_lldb_tracer.py index 5fd7c0d1f2..52bfa9af29 100644 --- a/bridge/native/test_rea_lldb_tracer.py +++ b/bridge/native/test_rea_lldb_tracer.py @@ -121,6 +121,7 @@ class NativeTraceBudgetTests(unittest.TestCase): result = tracer.trace(Debugger(), { "executable": "fixture", "architecture": "arm64", "expected_sha256": "expected", "breakpoints": [], "working_directory": "fixture/absent", + "capture_limits": {"output_bytes": 1024}, "stdout_path": "stdout", "stdout_capture_path": "stdout-capture", "stderr_path": "stderr", "stderr_capture_path": "stderr-capture", }) @@ -320,37 +321,44 @@ class NativeTraceBudgetTests(unittest.TestCase): return super().flush() stream = CountingStream() - journal = tracer._ObservationJournal(stream) + journal = tracer._ObservationJournal(stream, max_bytes=4096) journal.append({"kind": "event", "event": {"sequence": 0}}) journal.append({"kind": "other-stop", "reason": "signal"}) self.assertEqual((stream.writes, stream.flushes), (2, 2)) self.assertEqual(journal.bytes, len(stream.getvalue())) - def test_aggregate_trace_budget_admits_exact_boundary_and_rejects_next_record(self): - self.assertTrue( - tracer._can_retain_trace( - tracer.MAX_TRACE_BYTES - 10, - tracer.MAX_TRACE_FRAMES - 2, - 10, - 2, - ) - ) - self.assertFalse( - tracer._can_retain_trace( - tracer.MAX_TRACE_BYTES - 10, - tracer.MAX_TRACE_FRAMES - 2, - 11, - 2, - ) - ) - self.assertFalse( - tracer._can_retain_trace( - tracer.MAX_TRACE_BYTES - 10, - tracer.MAX_TRACE_FRAMES - 2, - 10, - 3, - ) + def test_aggregate_trace_budget_uses_the_selected_admission(self): + limits = {"trace_bytes": 100, "trace_frames": 10} + self.assertTrue(tracer._can_retain_trace(90, 8, 10, 2, limits)) + self.assertFalse(tracer._can_retain_trace(90, 8, 11, 2, limits)) + self.assertFalse(tracer._can_retain_trace(90, 8, 10, 3, limits)) + raised = {"trace_bytes": 101, "trace_frames": 11} + self.assertTrue(tracer._can_retain_trace(90, 8, 11, 3, raised)) + + def test_breakpoint_reports_exact_exhausted_budget_and_accepts_more_room(self): + frame = types.SimpleNamespace( + IsValid=lambda: True, GetPC=lambda: 1, + GetThread=lambda: types.SimpleNamespace(GetNumFrames=lambda: 1), + GetPCAddress=lambda: object(), ) + context = { + "target": object(), "events": [], "retained_frames": 1, + "retained_trace_bytes": 0, + "config": {"breakpoints": [{}], "backtrace_frames": 0, "max_events": 10, + "capture_limits": {"trace_bytes": 4096, "trace_frames": 1}}, + } + with patch.object(tracer, "_location", return_value={"symbol": "main"}), patch.object( + tracer, "_wanted", return_value=True + ), patch.object(tracer, "_record", return_value={"backtrace": []}): + self.assertEqual(tracer._handle_breakpoint(context, frame, 0), "resource-limit") + self.assertEqual(context["resource_limit"], "trace_frames") + context["config"]["capture_limits"] = {"trace_bytes": 8, "trace_frames": 2} + self.assertEqual(tracer._handle_breakpoint(context, frame, 0), "resource-limit") + self.assertEqual(context["resource_limit"], "trace_bytes") + self.assertEqual(context["events"], []) + context["config"]["capture_limits"]["trace_bytes"] = 4096 + self.assertIsNone(tracer._handle_breakpoint(context, frame, 0)) + self.assertEqual(context["events"], [{"backtrace": []}]) def test_oversized_producer_string_is_rejected_before_serialization(self): self.assertTrue(tracer._fits_trace_budget("method", 1024)) @@ -459,6 +467,7 @@ class NativeTraceBudgetTests(unittest.TestCase): object(), [[Breakpoint()]], { + "capture_limits": {"breakpoint_location_bytes": 4096}, "breakpoints": [ { "kind": "objc-method", @@ -493,7 +502,7 @@ class NativeTraceBudgetTests(unittest.TestCase): side_effect=lambda address, _target: {"address": address, "symbol": None}, ), patch.object(tracer, "_wanted", return_value=True): report, locations_truncated = tracer._breakpoint_report( - object(), [[Breakpoint()]], {"breakpoints": [{}]} + object(), [[Breakpoint()]], {"breakpoints": [{}], "capture_limits": {"breakpoint_location_bytes": 4096}} ) self.assertEqual(report[0]["location_count"], 65) @@ -574,9 +583,10 @@ class NativeTraceBudgetTests(unittest.TestCase): def GetLocationAtIndex(self, index): return Location() - breakpoint_count = 1000 + breakpoint_count = 30 created = [[Breakpoint()] for _ in range(breakpoint_count)] - config = {"breakpoints": [{"kind": "objc-method"}] * breakpoint_count} + config = {"breakpoints": [{"kind": "objc-method"}] * breakpoint_count, + "capture_limits": {"breakpoint_location_bytes": 512 * 1024}} for source_path in (None, "/tmp/" + "源🚀" * 3000 + "/calls.pas"): with self.subTest(source_path_present=source_path is not None): report, locations_truncated = tracer._breakpoint_report( @@ -590,13 +600,13 @@ class NativeTraceBudgetTests(unittest.TestCase): ) self.assertTrue(locations_truncated) self.assertEqual( - sum(row["location_count"] for row in report), 64_000 + sum(row["location_count"] for row in report), 1920 ) self.assertLess( - sum(len(row["locations"]) for row in report), 64_000 + sum(len(row["locations"]) for row in report), 1920 ) self.assertLessEqual( - retained_bytes, tracer.MAX_BREAKPOINT_LOCATION_BYTES + retained_bytes, 512 * 1024 ) if source_path is not None: self.assertTrue( diff --git a/bridge/pwndbg/core_context.py b/bridge/pwndbg/core_context.py index ffdbc72ab4..1caf147e42 100644 --- a/bridge/pwndbg/core_context.py +++ b/bridge/pwndbg/core_context.py @@ -5,7 +5,6 @@ from pathlib import Path import resource import gdb -OUTPUT_BYTES = 64 * 1024 * 1024 class ContextFailure(Exception): @@ -79,6 +78,7 @@ def failure_exit(request_path, status, marker): def main(): request_path = os.environ["REA_CORE_CONTEXT_REQUEST"] request = json.loads(Path(request_path).read_text(encoding="utf-8")) + output_bytes = request["producer_limits"]["output_bytes"] reserve = bytearray(1024 * 1024) try: reply = {"ok": True, "value": inspect(request)} @@ -94,8 +94,8 @@ def main(): reply = {"ok": False, "reason": "decoder", "message": type(error).__name__ + ": " + str(error)} try: encoded = json.dumps(reply, ensure_ascii=True, allow_nan=False).encode("utf-8") - if len(encoded) > OUTPUT_BYTES: - encoded = json.dumps({"ok": False, "reason": "output-limit", "message": "Complete debugger context exceeds the 64 MiB reply budget."}).encode("utf-8") + if len(encoded) > output_bytes: + encoded = json.dumps({"ok": False, "reason": "output-limit", "message": "Complete debugger context exceeds producer_limits.output_bytes=" + str(output_bytes) + "."}).encode("utf-8") descriptor = os.open(request["reply_path"], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(encoded) diff --git a/bridge/pwndbg/test_core_context.py b/bridge/pwndbg/test_core_context.py index 42f09d901e..c6afaba6c1 100644 --- a/bridge/pwndbg/test_core_context.py +++ b/bridge/pwndbg/test_core_context.py @@ -14,7 +14,7 @@ BRIDGE = Path(__file__).with_name("core_context.py") class CoreContextTests(unittest.TestCase): - def inspect(self, failure=None, connection="core", symbols=None, architecture="x86_64-little-endian", objfile="recorded-executable"): + def inspect(self, failure=None, connection="core", symbols=None, architecture="x86_64-little-endian", objfile="recorded-executable", output_bytes=16384): commands, reads, queries = [], [], [] state = {"executable": None, "symbols": None, "connection": None} gdb = types.ModuleType("gdb") @@ -63,7 +63,7 @@ class CoreContextTests(unittest.TestCase): plugin.write_text("") request = stage / "request.json" reply = stage / "reply.json" - request.write_text(json.dumps({"gdbinit": str(plugin), "architecture": architecture, "snapshot_path": str(root / "object.snapshot"), "reply_path": str(reply)})) + request.write_text(json.dumps({"gdbinit": str(plugin), "architecture": architecture, "snapshot_path": str(root / "object.snapshot"), "reply_path": str(reply), "producer_limits": {"output_bytes": output_bytes}})) with patch.dict(sys.modules, modules), patch.dict(os.environ, REA_CORE_CONTEXT_REQUEST=str(request)): runpy.run_path(str(BRIDGE)) result = json.loads(reply.read_text()) @@ -104,14 +104,15 @@ class CoreContextTests(unittest.TestCase): self.assertEqual(reads, []) self.assertEqual(queries, []) - def test_output_limit_message_separates_the_words_and_names_the_limit(self): - result, _, reads, queries = self.inspect(objfile="m" * (64 * 1024 * 1024)) + def test_selected_output_limit_reports_recovery_and_accepts_more_headroom(self): + result, _, reads, queries = self.inspect(objfile="m" * 2048, output_bytes=1024) self.assertFalse(result["ok"]) self.assertEqual(result["reason"], "output-limit") - self.assertNotIn("exceeds64", result["message"]) - self.assertIn("64 MiB", result["message"]) + self.assertIn("producer_limits.output_bytes=1024", result["message"]) self.assertEqual(reads, []) self.assertEqual(queries, ["maps"]) + recovered, _, _, _ = self.inspect(objfile="m" * 2048, output_bytes=4096) + self.assertTrue(recovered["ok"], recovered) def test_wrong_connection_or_remaining_symbols_never_queries_mappings(self): for options in [{"connection": "native"}, {"symbols": "unexpected-symbols"}]: diff --git a/bridge/pwntools/decoder_runtime.py b/bridge/pwntools/decoder_runtime.py index 6681dffee0..8d27e432ed 100644 --- a/bridge/pwntools/decoder_runtime.py +++ b/bridge/pwntools/decoder_runtime.py @@ -5,14 +5,12 @@ from pathlib import Path import resource import sys -OUTPUT_BYTES = 64 * 1024 * 1024 - -def lower_resource_limits(): +def lower_resource_limits(policy): limits = {} for name, kind, maximum in ( - ("address_space_bytes", resource.RLIMIT_AS, 3 * 1024**3), - ("cpu_seconds", resource.RLIMIT_CPU, 30), - ("file_size_bytes", resource.RLIMIT_FSIZE, OUTPUT_BYTES), + ("address_space_bytes", resource.RLIMIT_AS, policy["address_space_bytes"]), + ("cpu_seconds", resource.RLIMIT_CPU, policy["cpu_seconds"]), + ("file_size_bytes", resource.RLIMIT_FSIZE, policy["file_bytes"]), ): soft, hard = resource.getrlimit(kind) finite = [maximum] + [value for value in (soft, hard) if value != resource.RLIM_INFINITY] @@ -25,6 +23,11 @@ def lower_resource_limits(): def main(request_path, inspect, failure_type, profile): request = json.loads(Path(request_path).read_text(encoding="utf-8")) + policy = request["producer_limits"] + for key in ("input_bytes", "output_bytes", "diagnostic_bytes", "address_space_bytes", "file_bytes", "cpu_seconds"): + value = policy[key] + if type(value) is not int or not 1 <= value <= 9007199254740991: + raise ValueError("producer_limits." + key + " must be a positive safe integer") os.environ["PWNLIB_NOTERM"] = "1" os.environ["PWNLIB_CACHE_DIR"] = str(Path(request_path).parent / "cache") limits = None @@ -32,7 +35,7 @@ def main(request_path, inspect, failure_type, profile): # The outer exit-status contract also covers allocation failure during serialization. memory_reserve = bytearray(1024 * 1024) try: - limits = lower_resource_limits() + limits = lower_resource_limits(policy) descriptor = os.open(Path(request_path).parent / "limits.json", os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(descriptor, "w", encoding="utf-8") as handle: json.dump(limits, handle) @@ -51,8 +54,8 @@ def main(request_path, inspect, failure_type, profile): else: reply = {"ok": False, "reason": "decoder", "message": type(error).__name__ + ": " + str(error)} encoded = json.dumps(reply, ensure_ascii=True, allow_nan=False).encode("utf-8") - if len(encoded) > OUTPUT_BYTES: - encoded = json.dumps({"ok": False, "reason": "output-limit", "message": "Complete artifact result exceeds the 64 MiB reply budget."}).encode("utf-8") + if len(encoded) > policy["output_bytes"]: + encoded = json.dumps({"ok": False, "reason": "output-limit", "message": "Complete artifact result exceeds producer_limits.output_bytes=" + str(policy["output_bytes"]) + "."}).encode("utf-8") descriptor = os.open(request["reply_path"], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) with os.fdopen(descriptor, "wb") as handle: handle.write(encoded) diff --git a/docs/README.md b/docs/README.md index ddceb77cf7..b14c188077 100644 --- a/docs/README.md +++ b/docs/README.md @@ -111,6 +111,10 @@ product pages. The VitePress documentation source is this `docs/` tree. ## Development and verification - [Settings and recovery](design/limits-and-recovery.md) +- [Flutter budget calibration](verification/flutter-budget-calibration.md) +- [Source-map budget calibration](verification/source-map-budget-calibration.md) +- [Source-location delivery qualification](verification/source-location-delivery.md) +- [JavaScript precision calibration](verification/javascript-precision-calibration.md) - [MCP tool design](development/tool-design.md) - [Testing REA](development/testing.md) - [Client and agent verification](development/client-verification.md) diff --git a/docs/bun-standalone.md b/docs/bun-standalone.md index 5b6b915ab6..bd9db5d422 100644 --- a/docs/bun-standalone.md +++ b/docs/bun-standalone.md @@ -24,7 +24,7 @@ entry-point status, sizes and SHA-256. Sourcemaps, bytecode and module-info regi are separate children. Source names such as `/$bunfs/root/main.js` remain in metadata; extraction uses a relative path. Unsafe or non-UTF-8 names receive an indexed path and retain their source spelling or bytes. Virtual names deeper -than 256 logical path parts also use an indexed path with an explicit limitation, +than the selected `path_parts` allowance (default 256) also use an indexed path with an explicit limitation, preventing unbounded path-tree work while retaining the source fact. Duplicate names remain separate occurrences; ordinary extraction collision rules apply. @@ -43,12 +43,27 @@ compatible source files; bytecode is retained as evidence, not decoded. Module parsing reads exact metadata ranges rather than loading stored asset bodies or whole pages around small pointers. A cache retains at most 256 KiB and 1,024 ranges for one reader preparation. Cumulative uncached metadata reads -are limited to 256 MiB, checked before allocation or I/O. The module table, +default to 256 MiB, checked before allocation or I/O. The module table, decoded strings and combined logical path tree have separate admission checks; -at most 250,000 distinct path prefixes are admitted before inventory construction. +by default at most 250,000 distinct path prefixes are admitted before inventory construction. Stored bodies are range-streamed for hashing and extraction. Header/suffix observations are reused only for the same admitted file state, with selected-path checks before publication. +Both artifact operations accept `artifact_limits.bun`. Its independent controls +are `payload_bytes`, `module_count`, `representation_bytes`, `metadata_read_bytes`, +`recognition_table_bytes`, `recognition_load_command_bytes`, `path_parts` and +`path_nodes`. Failed recognition remains explicitly limited instead of proving +absence. ELF section names are examined at their actual offsets, including names +beyond a large section-name prefix. Select `artifact_limits.expanded_bytes` to +change the aggregate referenced-plaintext work allowance. + +For a reported metadata-read refusal, keep the same carrier and retry with +`--artifact-limits '{"bun":{"metadata_read_bytes":536870912}}'` or the equivalent +MCP `artifact_limits` object. Omitted fields keep defaults. The response names +the effective limit and exact repair; accepted overrides are reported separately +from caller selections. Changes apply on the next call without restarting REA. +Summary and selected extraction do not change the producer's admission. + See [limits and recovery](design/limits-and-recovery.md) for admission ceilings. Synthetic tests cover the carriers and malformed declarations. Run `node scripts/verify-bun-standalone.mjs` after `npm run build:cached` for local Bun diff --git a/docs/design/limits-and-recovery.md b/docs/design/limits-and-recovery.md index cb994326bc..b60778e761 100644 --- a/docs/design/limits-and-recovery.md +++ b/docs/design/limits-and-recovery.md @@ -172,17 +172,20 @@ checks all happen before publishing the replacement snapshot; a rejected batch cannot partially apply. Clients coordinating writes can supply `expected_revision`; without it, valid updates apply to the current state. -| Runtime setting | Application point | -| ------------------------------------------------------------- | ---------------------------------------------------------------------- | -| `asar.max_header_bytes` | Next archive operation, including full JavaScript application analysis | -| `ghidra.startup_timeout_ms`, `hopper.startup_timeout_ms` | Next provider startup attempt | -| `ghidra.decompile_timeout_seconds` | Next operation on the existing Ghidra Program; `null` means unbounded | -| `jadx.operation_timeout_ms` | Next JADX operation | -| `regex.matching_deadline_ms`, `regex.worker_heap_request_mib` | Next search / new regex worker | -| `capture.decoder_old_generation_mib` | Next owned capture decoder | -| `javascript.analysis_timeout_ms`, `javascript.max_heap_mb` | Next source analysis / new JavaScript worker | -| `wasm.command_timeout_ms` | Next WABT operation | -| `native.matching_representation_bytes` | Next `match_native_functions` request | +| Runtime setting | Application point | +| --------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| `asar.max_header_bytes` | Next archive operation, including full JavaScript application analysis | +| `asar.max_listing_bytes` | Next expanded ASAR listing, independently of packed header bytes | +| `ghidra.startup_timeout_ms`, `hopper.startup_timeout_ms` | Next provider startup attempt | +| `ghidra.decompile_timeout_seconds` | Next operation on the existing Ghidra Program; `null` means unbounded | +| `ghidra.decompile_max_payload_mib` | Next native decompile; `null` preserves the selected provider's payload default | +| `ghidra.value_flow_max_operations`, `ghidra.value_flow_max_inputs_per_operation`, `ghidra.value_flow_max_edges` | Next function value-flow projection on the existing Program | +| `jadx.operation_timeout_ms` | Next JADX operation | +| `regex.matching_deadline_ms`, `regex.worker_heap_request_mib` | Next search / new regex worker | +| `capture.decoder_old_generation_mib` | Next owned capture decoder | +| `javascript.analysis_timeout_ms`, `javascript.max_heap_mb` | Next source analysis / new JavaScript worker | +| `wasm.command_timeout_ms` | Next WABT operation | +| `native.matching_representation_bytes` | Next `match_native_functions` request | ### Precedence and lifetime @@ -202,6 +205,48 @@ for an unknown or omitted value elsewhere. A write does not extend active work, relaunch a provider, clear retained cleanup, or invalidate successful Evidence. +Expanded ASAR listing admission defaults to 64 MiB and has its own launch seed, +`REA_ASAR_MAX_LISTING_BYTES`. Raising `asar.max_header_bytes` alone cannot admit +expanded paths and entries that exceed this separate budget. The listing bound +uses exact safe-integer byte accounting; the host heap and transport constraints +remain independent. A lower runtime value is enforced on the next listing even +when the archive header was already examined. + +Ghidra value-flow controls default to 3,000 operations, 64 inputs per operation, +and 12,000 edges. They accept positive Java integer counts and use the respective +`REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS`, +`REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION`, and +`REA_GHIDRA_VALUE_FLOW_MAX_EDGES` launch seeds. Truncation names the exhausted +control. Completed Evidence remains accessible, but a new function dossier is +observed with the accepted controls instead of replaying an earlier projection. + +The native decompiler payload control is separate from those projections and +from decompile time. Its launch seed is +`REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB`. Accepted explicit values are 1..2047 MiB: +the supported Ghidra implementation shifts that value into a signed integer byte +capacity. Omission or runtime `null` keeps the provider's own default; REA does +not guess that default from a different provider version. Changing payload +options takes effect before the next native decompile. + +### Per-call resource policies and selection + +An agent can also use controls declared directly by an operation. These do not +require a runtime setter or restart. Keep omitted fields omitted in Evidence; +report the resolved policy separately where needed. See the owning guides for +[Apktool string pages](../guides/android/apktool-resource-analysis.md), +[source maps](../guides/web/web-source-location.md), +[historical captures](../guides/web/web-network-captures.md), +[JavaScript precision](../guides/javascript/javascript-application-graph.md), +[managed inspection](../guides/managed/managed-code-analysis.md), and +[Frida capture](../guides/runtime/frida-instrumentation.md). + +The same accepted policy must reach acquisition, parsing, worker requests, +serialization, cached-result admission and generated consumers where those +stages exist. Desktop state views and emitted AgentEnv Python share one policy; +delivery pagination never substitutes for producer selection. Capture limits +must be selected before an experiment: another experiment cannot recover bytes +that were dropped from a previous live observation. + Overrides belong to one REA runtime, survive target close/open, and disappear when REA exits. Connections sharing that runtime share its overrides. No files, ambient environment, registrations or upstream software are edited. Overrides @@ -419,17 +464,55 @@ editable setting or active worker telemetry. ## Bun standalone module graphs -`MAX_BUN_MODULE_COUNT` admits at most 100,000 module records. -`MAX_BUN_RETAINED_BYTES` admits at most 256 MiB of serialized graph payload and -separately bounds retained entry metadata. Physical metadata read work is also -limited to 256 MiB per preparation, including repeated uncached ranges. A -256 KiB / 1,024-entry exact-range cache avoids full-page amplification. The -combined logical path tree admits at most 250,000 distinct prefixes; this is -checked before adding nodes, with shared prefixes counted once. The shared -artifact expansion budget also bounds the sum of referenced member lengths. Refusals name the boundary. -There is no runtime override; additional host memory or `NODE_OPTIONS` does not -raise these ceilings. Use a smaller producer fixture or a separately reviewed -reader for larger graphs. No partial extraction is claimed on admission failure. +`inspect_artifact` and `extract_artifact` accept per-call `artifact_limits`. +The `bun` group defaults to 100,000 modules, independent 256 MiB payload, +retained-representation and physical metadata-read allowances, 256 path parts +and 250,000 logical path nodes. Recognition separately admits a 1 MiB ELF +section table and 16 MiB Mach-O load commands. Exhausted recognition remains +unknown rather than proving the carrier has no Bun graph. Inventory preserves +native-root facts or another confirmed pack and reports the exact unexamined +alternative inline; it does not assert Bun presence. ELF section names +are examined with exact bounded reads, including names beyond the former +prefix window. + +For example, keep the same complete carrier and retry a reported recognition +limitation with `{"artifact_limits":{"bun":{"recognition_table_bytes":2097152}}}`. +The CLI spelling is `--artifact-limits '{"bun":{"recognition_table_bytes":2097152}}'`. +No REA restart is needed. A smaller later selection is re-admitted; previously +parsed facts cannot bypass it. The 256 KiB / 1,024-entry exact-range cache +avoids full-page amplification and remains an internal optimization, not an agent +resource setting. Shared logical path prefixes count once before nodes are added. + +## Artifact reader admission + +The same request provides `dotnet_bundle`, `node_sea` and `godot` groups. Their +controls apply when recognition or inventory encounters that format; supplying +a policy does not assert the format is present. Unsupported keys are rejected. +Each group follows one domain-owned default policy. See [.NET bundles](../dotnet-bundles.md), +[Node SEA](../node-sea.md), [Godot PCK](../godot-pck.md) and +[Bun standalone](../bun-standalone.md) for format semantics. + +`artifact_limits.expanded_bytes` selects aggregate plaintext work for ZIP, Bun, +SEA and PCK readers, including repeated readable logical occurrences. Omission +retains each archive's encoded-size-derived default. The .NET bundle group's +own `expanded_bytes` controls its independently enforced bundle expansion. + +Failures preserve `details.reader_limit` with the exact parameter, effective +value, units, supported range, required value when known, next-call scope and +`can_increase`. When `can_increase` is true, raise that field and retry the same +complete artifact. At the supported maximum, or when the required representation +exceeds it, the failure is not retryable by increasing that budget; inspect the +declared dimensions and input validity, or explicitly choose a suitable reader. +Summary, pagination and +selected extraction do not bypass producing admission. Intrinsic format, +safe-integer, path validity and runtime representation constraints remain fixed. + +Evidence parameters retain the exact selected object, including omission or +`{}`. When an object was selected, inventory and extraction report the resolved +`effective_limits`; summaries preserve those facts. These are accepted policies, +not process-memory telemetry or a claim that every format reader ran. Complete +source identities and graph manifests remain content-addressed independently +of delivery selection. No partial extraction is claimed on admission failure. ## Deno compile sections diff --git a/docs/desktop-application-capture.md b/docs/desktop-application-capture.md index b7cca3554c..0a8de99b83 100644 --- a/docs/desktop-application-capture.md +++ b/docs/desktop-application-capture.md @@ -290,8 +290,8 @@ found nowhere are listed in not an ELF image (a launcher script, for example), `ldconfig`, `dpkg-query` or `apt-cache` is missing, `dpkg-query -S` could not tell which package owns a library (it exits 2 rather than 1, which only says no package owns the paths -it did not name), a library could not be read or names more than 512 -libraries, or the walk stopped after 512 libraries. A failed owner query does +it did not name), a library could not be read or exceeds the selected dependency +inventory budget, or the walk reached that budget. A failed owner query does not list the libraries it covered in `unpackaged_libraries`, since a lookup that could not finish is not proof that no package owns them. Completed batches retain their package and unowned-library facts. Paths containing package-query @@ -428,3 +428,18 @@ evidence for attribution, not proof. Clicks and keystrokes are not recorded. Window titles are sampled every half second, so short-lived titles can be missed. + +Dependency inventory headroom is selected for each capture with +`--library-policy '{"libraries":1024,"string_bytes":8192}'`. Unset fields retain +their defaults: `libraries: 512` bounds both the closure walk and distinct +DT_NEEDED offsets in each ELF image; `search_path_entries: 16` bounds distinct +DT_RUNPATH and DT_RPATH offsets per kind; `string_bytes: 4096` includes the NUL +terminator of each decoded string; `table_bytes: 1048576` bounds each program +header or dynamic table. Values must be positive safe integers. A budget refusal +names the field that needs headroom; partial package facts and unresolved +coverage remain visible. The policy applies during this capture's admission, +without restarting REA, and explicit selections are retained in Evidence. Raising +content retention or a worker heap does not raise these inventory budgets. +Choosing a new capture still launches the selected application; changing these +read budgets does not add execution authority or recover an earlier capture's +omitted inventory. diff --git a/docs/desktop-application-export.md b/docs/desktop-application-export.md index cc68e2baba..ec0e136bc7 100644 --- a/docs/desktop-application-export.md +++ b/docs/desktop-application-export.md @@ -560,7 +560,21 @@ or whose retained value nests more than 500 levels deep, gets an `error` instead REA. Duplicate keys keep their last value before the depth check. Text that is not one JSON document, such as JSON Lines, stays text. Format detection follows REA's rules. Files larger than 64 MiB get an `error` instead of a view -and are not read. XML is read with Python's `expat` twice: with namespace +and are not read. The selected `file_bytes` also applies to scrubbed seed task +views before admission, independently of how much seed data the export can +copy. SQLite table views have a shared `sqlite_view_bytes` allowance, default +16 MiB of compact UTF-8 JSON per table, checked before retaining each row. +An oversized table keeps its error while other tables remain readable. Raising +this view allowance does not raise the independent 16 MiB native SQLite +value/record limit or REA worker reply limit; worker facts include additional +schema and typed-cell metadata and may be refused earlier. The TypeScript +and exported Python readers count their respective native compact JSON encodings; +float and exponent spelling can make their exact byte thresholds differ even +when they report equal numeric facts. The Python byte counter quotes large +strings in bounded chunks before retention. +Exported SQLite readers require Python 3.11 or later for the native connection limits; the +AgentEnv compatibility workflow uses Python 3.12. +XML is read with Python's `expat` twice: with namespace processing, which refuses what REA's parser refuses, such as an undeclared prefix, then without it, so element and attribute names keep the prefixes the document spells and namespace declarations stay attributes, as in REA's view; diff --git a/docs/desktop-application-state.md b/docs/desktop-application-state.md index b069f913d6..530a150905 100644 --- a/docs/desktop-application-state.md +++ b/docs/desktop-application-state.md @@ -11,6 +11,65 @@ again. rea inspect-desktop-application-state ./mousepad-capture --format json ``` +Views use bounded defaults. When a view reports truncation or a byte-budget +refusal, retry the same capture with `--view-policy` JSON overrides: + +```bash +rea inspect-desktop-application-state ./mousepad-capture \ + --state-path /home/ada/Documents/notes.txt \ + --view-policy '{"text_lines":10000,"file_bytes":134217728}' --format json +``` + +`--state-path` selects an exact original captured path and can be repeated. +Omitting it examines every captured path. Selection preserves the captured +checkpoint and digest; it does not substitute the file's current contents. +A path absent from the capture is an input error. + +`--view-policy` accepts positive safe-integer overrides for `file_bytes`, +`sqlite_tables`, `sqlite_rows`, `sqlite_view_bytes`, `text_lines`, `xml_nodes`, `xml_depth`, +`parsed_bytes`, `json_depth`, `zip_entries`, `zip_entry_bytes`, `zip_bytes`, +and `zip_ratio`. Unspecified controls keep the defaults described below. +`xml_depth` admits at most 494 and `json_depth` at most 991: these bounds +follow the fixed Evidence JSON nesting limit, including the captured-version +and ZIP-member wrappers and XML's element/children representation. A request +above either bound is rejected before reading the capture. +Evidence parameters retain only the supplied `view_policy` overrides; omission +and an explicitly empty selection remain distinct. State inspection and task +drafts report the resolved policy separately as +`normalized_result.effective_view_policy`. Changes apply on this call, without +a restart. File and ZIP byte counts refer to retained file bytes or uncompressed +ZIP member bytes, not transport size. Gzip document views share `zip_entry_bytes` +and `zip_ratio` with ZIP entry views; `zip_bytes` bounds a ZIP archive, not a +gzip stream. These selections also apply to the decompressed document's format +view. `sqlite_view_bytes` instead bounds each +projected table's compact UTF-8 JSON, including column names, primary keys, +rows and truncation metadata; it defaults to 16 MiB. Rows are charged before +retention. A table that exceeds this allowance keeps an actionable `error`, +while other tables remain readable. Increase this control and retry the same +capture for a projected-view refusal. It does not raise the SQLite worker's +separate fixed value/record or complete-reply limits, which may refuse earlier. +Each reader counts its native compact JSON encoding. Float spelling can differ: +Python emits `16.0` where JavaScript emits `16`, and exponent spelling also +varies. Identical numeric facts can therefore cross this byte threshold at +different settings. This is not a common transport-byte guarantee. + +The same option is available on `describe-desktop-application-task` and +`export-desktop-application`. Export uses the accepted policy for task +validation and emits it into the Python readers used by the environment's +`data/get`; re-export with another policy to change those reader budgets. +Task checks apply `file_bytes` to the actual scrubbed seed bytes, including any +growth from replacing the capture home with `/home/user`. A seed over that +view allowance may still be exported under the independent acquisition limit, +but its unreadable view cannot establish a known task failure. +The TypeScript and exported Python readers share the policy's defaults. +These desktop capture workflows remain CLI-only. + +Raising a view budget cannot recover content the capture never retained, +change numeric exactness, enable encrypted or unsupported archive formats, +or raise SQLite worker, Evidence nesting, process-memory or transport limits. +Keep increases specific to the exhausted budget. Truncation and incomplete +comparison coverage remain visible until the requested facts were examined. + The result is Evidence with operation `inspect_desktop_application_state` and predicate type `rea.desktop-application-state`. Its confidence is `inferred`, because roles come from heuristics; the views themselves are read from the diff --git a/docs/desktop-application-tasks.md b/docs/desktop-application-tasks.md index 855c25c8c1..1ab4c9942f 100644 --- a/docs/desktop-application-tasks.md +++ b/docs/desktop-application-tasks.md @@ -3,11 +3,12 @@ A [desktop application export](desktop-application-export.md) can carry one task: a prompt for a computer-use agent and checks that grade what the agent left in the application's files. An agent, not REA, writes the prompt and the -checks from the change the capture recorded. REA validates the task against -the capture and keeps it only when its checks fail on the seed and all pass on -the captured end state. REA runs no model and no code the agent wrote: the -checks are declarative, and the exported verifier evaluates them the same way -REA does. +checks from the change the capture recorded. For file checks, REA validates +the task against the capture and keeps it only when its checks fail on the +seed and all pass on the captured end state. The checks are declarative; +optional [readers](#readers) run commands to interpret files, and +[screen judges](#screen-judges) use a model when no file shows the change. +The exported verifier uses the same checks and reader runner as REA. ```sh rea describe-desktop-application-task ./mousepad-capture --format json > draft.json @@ -518,7 +519,7 @@ and `CIRCLE`, and only `CIRCLE` going from 5 to 9 is the user's change. and hyphens); `prompt` is what the user asks for, naming the app, since a solving agent's own shell cannot see the computer's files and the app's name sends it to the screen; `checks` holds 1 to 32 -checks. Each check has a `kind`, a `file` given as a path under the home such as +checks. Each file check has a `kind`, a `file` given as a path under the home such as `Documents/todo.txt`, and an optional `id` (`check-1`, `check-2` and so on by position otherwise). A check reads the typed view `data/get` returns for that file, in the shapes of the state view. @@ -589,6 +590,82 @@ value, or one of only whitespace with `normalize_whitespace`, an empty content, such as a counter the app bumps on every save, is for the author to leave out. +## Readers + +For a file REA cannot interpret itself, such as an app's binary document, +the task can define `readers`: 1 to 8 commands using the app's own reading +capability. Each reader has a unique `id` of lowercase letters, digits and +hyphens, a home-relative `file`, an `argv` array and `output`, either `json` +or `lines`. Its `argv`, or optional `stdin` text, must contain `{file}`; +the runner replaces it with the path of a private copy of that file. +The command must be installed where the export runs and in the exported +computer where the task will be graded. + +The command prints its answer in lines starting with `REA `; other stdout +is ignored. With `output: "json"`, those marked lines join into one JSON +value for `value_equals`, `value_absent` and `member_count`. With +`output: "lines"`, they supply the lines for `lines_present`, `lines_absent` +and `line_count`. A check selects a reader with `"reader": "READER-ID"` +and must give that reader's file as its `file`. Every reader must be used by +at least one check. + +Readers execute commands, including any code in `argv` or `stdin`; review +them before exporting or running a task. The runner uses a private copy, a +fresh home, no session bus or computer control token, and no display unless +the reader requests `display: true`. This is execution isolation from the +running app's normal profile, not a filesystem sandbox. `timeout_s` defaults +to 60 seconds and accepts 1 to 300. A failed command, timeout, malformed +answer or missing marked output reports a reader error; it does not establish +that the document lacks the expected value. Correct the command or its output and retry; increase +`timeout_s` within its accepted range when the reader needs longer. + +Export runs readers on both retained ends, using the seed bytes as the bundle +actually ships them. The witness and exported verifier run them in the +computer after the app's files settle. Cleanup failures retain the reader's +reported resources: retry the check or close the build computer to retry +cleanup; when the command's outcome is unknown, confirm it stopped before +removing its folder. + +## Screen judges + +A task whose result exists only on screen can use `screen_judge` checks: + +```json +{ + "name": "show-list", + "prompt": "In Notes, show the shopping list.", + "checks": [ + { + "id": "shown", + "kind": "screen_judge", + "question": "Does the window show the shopping list?" + } + ] +} +``` + +Each question must be a nonempty yes-or-no question of at most 2,000 +characters, false on the seed and true when the task is done. A screen task +contains only screen checks and takes no readers. Export refuses it when an +owned file's readable change could be checked instead, or when the build +computer's seed or end screenshot is missing or exceeds 2 MiB. Capture those +checkpoints in a build computer before exporting. + +Export ships the two screenshots as labelled failing and passing examples; +it does not call the judge or establish that the questions distinguish them. +The [build witness](desktop-build-witness.md) runs the exported judge on a +fresh screenshot at each stage. The verifier takes a screenshot after the +solving agent stops. For each question, the judge sees both examples and +votes three times; a majority of yes answers passes the check. + +The judge uses `[model.roles] judge` in the runner's +`.agentenv/config.toml`, then `[model] default`, then +`anthropic/claude-sonnet-5-5`. Configure its endpoint with +`LITELLM_BASE_URL` and `LITELLM_API_KEY`, or `[model] base_url` and `api_key`. +`agent-env run --judge-model` does not reach this verifier. Without an +endpoint, grading raises and the run fails rather than scoring the agent; +configure the endpoint and rerun the witness or task. + ## Pointers and selectors An app that rewrites a document moves things: FreeCAD 1.0 records more @@ -796,7 +873,7 @@ that may hold only part of the array. ## Validation and export `rea export-desktop-application --task ` -evaluates the checks on the capture's state view: on each file as it was at +evaluates file checks on the capture's state view: on each file as it was at the seed checkpoint, which is the seed, as the bundle ships it with the capture host's name and user removed, and at the end checkpoint, the captured end state. Those are the capture's first and last checkpoints unless diff --git a/docs/dotnet-bundles.md b/docs/dotnet-bundles.md index 121d030f0e..ffc78488de 100644 --- a/docs/dotnet-bundles.md +++ b/docs/dotnet-bundles.md @@ -50,7 +50,8 @@ explicitly with the managed tools to establish PE/CLI facts. ## Admission and containment -Metadata is range-read with a 64 KiB cache. Policy budgets are 16 MiB of +Metadata is range-read with a cache of at most 64 KiB, bounded by the remaining +selected metadata allowance. Default policy budgets are 16 MiB of manifest metadata, 100,000 files, 64 KiB per UTF-8 string, 512 MiB of stored or expanded bytes per file, 256 path parts per member, 250,000 distinct in-memory path-tree nodes, and 2 GiB of aggregate expanded bytes. The node budget counts @@ -60,6 +61,18 @@ streams 64 KiB chunks and reuses completed probe facts during extraction; ordinary native files without a marker incur one extra read pass before host hashing. +Both artifact tools accept `artifact_limits.dotnet_bundle` to select +`metadata_bytes`, `entry_count`, `string_bytes`, `member_bytes`, +`expanded_bytes`, `path_parts`, and `path_nodes` for the next operation. The +string allowance applies to every manifest string, including the bundle ID. +For example, retry a complete bundle whose manifest exceeds the default with +`{"artifact_limits":{"dotnet_bundle":{"metadata_bytes":33554432}}}`. +The CLI accepts the same selection through `--artifact-limits` JSON. Omitted +fields preserve defaults; errors name the exact selected control, effective +value and required minimum when known. No REA restart is needed. Summary +delivery and paging do not change producer admission, and larger reader +budgets do not raise MCP response or Node representation limits. + Extraction hashes the whole host again through the same descriptor used to read members and compares it with the inventory's root digest before creating an output directory. This adds a full-host read pass to extraction, including when diff --git a/docs/godot-pck.md b/docs/godot-pck.md index 748fcf0d2d..5143861737 100644 --- a/docs/godot-pck.md +++ b/docs/godot-pck.md @@ -144,9 +144,34 @@ keys, decode Godot resources, decompile GDScript bytecode, analyze signals, apply patch/removal entries or execute targets. Readable logical member bytes, including repeated physical ranges, share the ZIP -archive expansion policy: at most the larger of 256 MiB or 500 times the PCK's +archive expansion policy by default: at most the larger of 256 MiB or 500 times the PCK's encoded byte span. Embedded carrier bytes do not enlarge that budget. The full directory is checked before any member is yielded, hashed or extracted. Encrypted members are not read and do not consume this plaintext-work allowance. Small bounded aliases remain valid. This is a processing/output resource bound, separate from the directory metadata retention budget and format validity. + +Directory retention defaults to 64 MiB of encoded metadata. Select +`artifact_limits.godot.directory_bytes` to change it; path/source facts reserve +worst-case JSON expansion before declared path bytes are read, and the reader +then charges actual retained metadata. A preallocation `required_value` is the +allowance needed by that admission check, not a measurement of final output size. +Select `artifact_limits.expanded_bytes` independently to change aggregate +plaintext work, including repeated readable logical occurrences. Both controls +apply to inspection and extraction, before any member is yielded. + +For example: + +```sh +rea inspect-artifact ./game.pck --artifact-limits '{"godot":{"directory_bytes":134217728},"expanded_bytes":1073741824}' --json +``` + +MCP uses the same object under `artifact_limits` on `inspect_artifact` or +`extract_artifact`. A refusal reports the exact `reader_limit` parameter, +effective value, unit, supported range and required value when known. Raise the +named admission and retry the same complete PCK on the next call; no REA restart +is needed. Summary or page size does not change producer admission. Omitted +policy selections stay omitted in Evidence parameters; a selected object records +the accepted selection and `effective_limits`, conditional on which readers are +encountered. Raising these policies does not admit malformed offsets, unsupported +flags, unsafe extraction paths or unavailable encrypted contents. diff --git a/docs/guides/android/apktool-resource-analysis.md b/docs/guides/android/apktool-resource-analysis.md index 07ea6f56f5..dbfc0491b5 100644 --- a/docs/guides/android/apktool-resource-analysis.md +++ b/docs/guides/android/apktool-resource-analysis.md @@ -66,6 +66,8 @@ rea decode-android-resources /targets/Example.apk --locales de --locales pt-BR rea decode-android-resources /targets/Example.apk --locales de --locales pt-BR --string-tables-only rea decode-android-resources /targets/Example.apk --no-strings rea decode-android-resources /targets/Example.apk --decode-timeout-ms 900000 +rea decode-android-resources /targets/Example.apk --string-offset 2000 --string-limit 100 --string-tables-only +rea decode-android-resources /targets/Example.apk --max-string-table-bytes 16777216 --max-total-string-table-bytes 67108864 ``` Equivalent MCP methods are `inspect_apktool_client` and @@ -87,6 +89,31 @@ responses and charges the aggregate output budget for one copy per table. Omitting the option preserves existing responses. `include_strings: false` still suppresses all string projection, including in this mode. +Use `string_offset` and `string_limit` (CLI `--string-offset` and +`--string-limit`) to select a page of valid parsed entries from each requested +table, in source order. Defaults are offset 0 and limit 2000. Explicit page or +budget controls include `string_tables` even for a single primary table; its +`page` gives the requested offset and limit, returned count, and `next_offset`. +Retry that offset with the same locale and confirm `target.sha256` matches. +Each call decodes a fresh owned snapshot; it does not retain a workspace or +promise identity across changed APKs. A null continuation means the page reaches +the end, while table coverage remains partial if earlier entries were omitted. +An offset at or beyond the end returns an empty, present table with known counts. +If the continuation equals the requested offset, no output entry fit: increase +`max_total_strings` or use `string_tables_only` before retrying. + +`max_total_strings` (CLI `--max-total-strings`, default 10000) bounds returned +string copies across all tables. Raise it, request fewer locales, or use +`string_tables_only` when aggregate output is exhausted. The legacy +multi-table shape charges both copies of primary strings. `max_string_table_bytes` +(CLI `--max-string-table-bytes`, default 8388608) bounds one XML source read; +`max_total_string_table_bytes` (CLI `--max-total-string-table-bytes`, default 33554432) bounds XML reads across distinct physical tables. Locale aliases +share the physical read and parsed page. Raise the relevant read budget to +retry an oversized table; a larger string limit or timeout does not raise it. +These controls take nonnegative safe-integer offsets and positive safe-integer +budgets, apply on the next decode, and require no REA restart. Process memory +and response transport capacity still apply. + ## Interpret the result - **An owned APK snapshot is digested and decoded.** `target.sha256` and @@ -117,8 +144,8 @@ still suppresses all string projection, including in this mode. Malformed XML and a wrong document root are whole-document projection failures: the result is partial, the failure reason is disclosed, and entry counts remain unknown. They are not counted as one unsupported string. - The default table is bounded to 2000 entries with partial coverage when more - exist. `locales` lists observed regular `strings.xml` tables whose exact + The default page returns up to 2000 entries with partial coverage when more + exist; its limitation names the next `string_offset`. `locales` lists observed regular `strings.xml` tables whose exact `values-` directory is selectable by the request's locale grammar. Locale suffixes allow up to 23 ASCII characters, derived from Apktool's language, script, region, and variant fields; for example, @@ -133,9 +160,10 @@ still suppresses all string projection, including in this mode. When multiple tables are requested with `locales`, `string_tables` records each projected table with its locale, status (`present`, `missing`, `oversized`, `malformed`, or `skipped`), entry counts, and coverage, - bounded at an aggregate budget of 10,000 strings and 32 MiB XML. When `locales` - is omitted and `string_tables_only` is not true, legacy single-table payloads are preserved - without adding `string_tables`. + with default aggregate budgets of 10,000 strings and 32 MiB XML. When `locales` + is omitted, `string_tables_only` is not true and no string page or budget + control is selected, legacy single-table payloads are preserved without adding + `string_tables`. - **Smali is skipped by design** (`apktool d --no-src`): Java-source recovery belongs to the JADX family. Resource smali edits and rebuild are a follow-up lane with mutation authority. @@ -156,8 +184,8 @@ still suppresses all string projection, including in this mode. - The version probe waits 20000 ms and retains 65536 bytes. This release has no runtime override for that probe. `decode_timeout_ms` does not raise it. - Manifest reads are bounded to 512 KiB before decoding UTF-8. Strings XML - reads are bounded to 8 MiB; larger documents are skipped with partial - coverage and an explicit limit, rather than reported as missing. + reads default to 8 MiB per table; larger documents are skipped with partial + coverage and an explicit read-budget recovery, rather than reported as missing. - `decoded_file_count` is capped at 100,000 files; the cap is reported in the limitations when hit. - An ordinary nonzero decode exit surfaces as `unsupported_target` naming diff --git a/docs/guides/android/flutter-analysis.md b/docs/guides/android/flutter-analysis.md index 7ffb31d2d6..5224a55382 100644 --- a/docs/guides/android/flutter-analysis.md +++ b/docs/guides/android/flutter-analysis.md @@ -217,3 +217,47 @@ The Gallery workflow and all 147 focused Flutter tests also pass on macOS Sonoma including the conflicting-load and note-range-limit CLI/MCP regressions. Windows verification runs under a non-administrator account. Verifiers use a shared temporary parent so each CLI/MCP process can own and release its own snapshots. + +## Selecting and recovering string-pool observations + +`inspect_dart_aot` accepts `pool_policy`; the CLI uses `--pool-policy` JSON: + +```sh +rea inspect-dart-aot app.apk --pool-policy '{"contains":"package:my_app/"}' +rea inspect-dart-aot app.apk --pool-policy '{"uri_limit":2000,"token_limit":4000,"run_bytes":4096,"token_bytes":256}' +``` + +A literal, case-sensitive `contains` filter runs before distinct collection, +so an omitted URI or identifier can be selected without returning every other +string. URI/token counts describe matching entries; `printable_run_count` counts +complete printable runs before filtering. Unset budgets preserve the defaults: +500 returned entries per URI category, 2000 identifier-like tokens, 1024 bytes +per printable run, and 128 bytes per identifier. Budget values must be positive +safe integers. Collection retains at most twice the selected category limit; +counts remain lower bounds when that capacity is reached. A run exceeding +`run_bytes` is omitted whole, rather than reported as a fabricated prefix. +The result names the budget reached, and `string_pool.policy` records the +selected filter and effective values. A larger budget or a narrower selection +applies to the next call without a REA restart; explicit choices stay in Evidence. + +For a larger package-URI inventory, select the category and its relevant budget: + +```sh +rea inspect-dart-aot app.apk --pool-policy '{"contains":"package:","uri_limit":2048}' +``` + +The equivalent MCP input is `pool_policy: {contains: "package:", uri_limit: 2048}`. +This recovers all bounded package-URI observations in the measured Gallery and +LocalSend APKs. Check the result's limits and coverage for other targets; long +runs and unsupported snapshot layouts remain separate limitations. The +[calibration report](../../verification/flutter-budget-calibration.md) +keeps the default at 500 because increasing every complete overview also delivers +more strings when they are unnecessary. Use `detail: "summary"` for the MCP +identity/count task described above. + +These controls do not raise APK, ELF, library or response budgets, deserialize +Dart objects, or establish typed class/function semantics. Retained +`inspect_analysis_view` pages cover the returned selection; exhausting those +pages does not recover facts that the producing pool scan omitted. Re-run the +producer with the reported selection or budget change to obtain a new +observation, checking its APK and library digests before comparing it. diff --git a/docs/guides/data/packet-captures.md b/docs/guides/data/packet-captures.md index aad2995eaa..706d989f97 100644 --- a/docs/guides/data/packet-captures.md +++ b/docs/guides/data/packet-captures.md @@ -117,8 +117,8 @@ are never passed to TShark. Installed libraries, global configuration and plugins remain trusted, are not fully fingerprinted, and can influence dissection. This is not an OS sandbox. -Input is limited to 32 MiB before reading, each command to 30 seconds, and -combined stdout/stderr to 4 MiB during collection. Output overflow fails instead +Default limits are 32 MiB input before reading, 30 seconds per command, and +4 MiB combined stdout/stderr during collection. Output overflow fails instead of returning truncated success. Before execution, `prlimit` enforces **1 GiB address space per process**, **64 MiB per file**, **30 CPU seconds** and no core dumps. Stricter inherited soft/hard limits are preserved; Evidence records the effective @@ -128,6 +128,23 @@ These are not aggregate process-memory or filesystem quotas. The selected limiter is trusted; its path, digest and arguments are retained in Evidence. No arbitrary process flags are accepted. +Each capture/recording operation accepts `producer_limits` with positive safe +integer overrides: `input_bytes`, `output_bytes`, `command_timeout_ms`, +`address_space_bytes`, `file_bytes`, and `cpu_seconds`. The wall-clock timeout +is at most 2,147,483,647 milliseconds, matching Node timers. Omitted fields keep +the defaults above. Discovery accepts the same controls except `input_bytes`, +since it does not read a capture or recording. Packet field discovery defaults +to 32 MiB output rather than 4 MiB. + +CLI supplies this object as JSON with `--producer-limits`, for example +`--producer-limits '{"input_bytes":67108864,"output_bytes":8388608,"cpu_seconds":60}'`. +Values apply to the next call without restarting REA. Effective native ceilings +still preserve any tighter inherited soft and hard limit. Errors identify the +exhausted producer control and retain partial output and cleanup status; follow +that repair after cleanup settles. Raising producer budgets does not raise +separate parser representations, transport buffers or external client deadlines. +Never truncate the selected artifact to fit an input budget. + ## Verification ```sh @@ -182,8 +199,9 @@ proof that the field appears in a selected recording. The adapter uses the documented `tshark -G fields` grammar under its existing private empty profile and Linux process limits. TShark 4.6.9's full glossary was -measured at 22,303,197 bytes; discovery admits at most **32 MiB** combined command -output (capture inspection remains at **4 MiB**). Parsing walks lines once and +measured at 22,303,197 bytes; discovery defaults to **32 MiB** combined command +output (capture inspection defaults to **4 MiB**); `producer_limits.output_bytes` +can change either request budget. Parsing walks lines once and constructs objects only for the selected page. Returned strings are copied so retained pages do not retain the complete glossary string. Before constructing a selected record it charges six times the source UTF-8 bytes plus 512 bytes against an diff --git a/docs/guides/evm/evm-bytecode.md b/docs/guides/evm/evm-bytecode.md index e215f0028e..5789bb0c45 100644 --- a/docs/guides/evm/evm-bytecode.md +++ b/docs/guides/evm/evm-bytecode.md @@ -118,16 +118,25 @@ CLI keeps its existing complete result. ## Resource and lifecycle limits -Inputs are limited to 4 MiB, complete replies to 16 MiB and combined diagnostic -output to 1 MiB. An owned worker has a 256 MiB JavaScript heap; its profile -maxima are 3 GiB virtual address space and 30 CPU seconds, with tighter inherited -limits retained. The worker command deadline is -`timeout_ms` (default 30000 ms, integer 1 through 2147483647) on the next call, -or `--timeout-ms` on `rea inspect-evm-interface`. Omission keeps 30000 ms. -Those byte, address-space, and CPU ceilings have no runtime override, and -`timeout_ms` does not raise them. An oversized selected carrier returns a -non-retryable `resource_constraint` with resource `file-size` and its fixed -input allowance in `reported_limits.maximum_input_bytes`. +Per-call `producer_limits` selects `input_bytes`, `output_bytes`, +`diagnostic_bytes`, `address_space_bytes`, `file_bytes` and `cpu_seconds`. +Omission keeps 4 MiB input, 16 MiB reply/file, 1 MiB diagnostics, 3 GiB address +space and 30 CPU seconds. Omitted `file_bytes` follows the selected +`output_bytes`, so raising the reply allowance does not silently meet the old +file-write allowance. Explicit file limits remain independent. These fields +accept positive safe integers; changes apply to the next call without a REA +restart. The CLI accepts the same exact policy as JSON, for example: + +```sh +rea inspect-evm-interface ./runtime.hex hex --producer-limits '{"input_bytes":8388608,"output_bytes":33554432}' +``` + +The owned worker's 256 MiB JavaScript heap remains separate. The worker command +deadline is `timeout_ms` (default 30000 ms, integer 1 through 2147483647), or +`--timeout-ms` on the CLI. Raising it does not raise producer, heap or client +limits. An oversized carrier reports non-retryable `resource_constraint` with +`reported_limits.maximum_input_bytes` and recovery naming +`producer_limits.input_bytes`. Complete-reply allowance refusals use the same non-retryable resource family, with `reported_limits.maximum_reply_bytes`; malformed JSON remains unreadable output. This refusal leaves no complete report to retrieve through another @@ -135,7 +144,7 @@ delivery channel. Virtual address space differs from resident memory. The worker soft/hard limits, reports its configured soft limits inline and leaves hard limits unchanged. WASM trap reservations are disabled so the engine can run within that virtual limit; actual memory is not equated to -its 3 GiB allowance. Limits fail without partial success. Stable snapshots, +the selected address-space allowance. Limits fail without partial success. Stable snapshots, private files and owned processes are cleaned independently of caller cancellation. Invalid-input, unsupported, cancellation, timeout and malformed-reply failures retain bounded stdout/stderr under `details.captured_output`, including whether diff --git a/docs/guides/javascript/javascript-artifact-reconstruction.md b/docs/guides/javascript/javascript-artifact-reconstruction.md index 460a62b0c5..fb6723f775 100644 --- a/docs/guides/javascript/javascript-artifact-reconstruction.md +++ b/docs/guides/javascript/javascript-artifact-reconstruction.md @@ -763,7 +763,7 @@ These safeguards are not caller-selectable output budgets. Source-map parsing reports truncation when its format or parser safety boundary is reached. The application graph has no aggregate node, edge, root, or observation prefix cap. -Semantic evaluation retains at most 256 distinct primitive candidates per +By default, semantic evaluation retains at most 256 distinct primitive candidates per expression and at most 1 MiB of worst-case JSON string bytes across a normalized primitive value's string candidates. The byte budget estimates six JSON bytes per UTF-16 code unit, the expansion bound for escaped strings, and bounds both @@ -783,13 +783,48 @@ available for endpoint queries, tracing and comparison. Each analysis records en later projections: a selected healthy sibling can remain exact while whole-file coverage reports the encountered bound. Projection preserves limiting values from the shared property-slot owner. Expression -evaluation and provenance walks stop after 256 nested levels. Union products +evaluation and provenance walks default to 256 nested levels. Union products and string lengths are checked before allocating combinations or concatenated strings. When a boundary is reached, the value stays unknown, semantic coverage becomes partial, and graph evidence retains the expression location and typed limiting reason. The number of unresolved alternatives is unknown; it is not reported as an exact omission count. +`semantic_precision` selects per-source evaluator budgets independently of +retained graph capacity, worker memory and deadlines. Both +`analyze_javascript_application` and `analyze_web_bundle` accept a partial object: + +| Field | Default | Unit | +| ---------------------- | ------: | ------------------------------------------- | +| `primitive_candidates` | 256 | alternatives per normalized primitive value | +| `primitive_bytes` | 1048576 | estimated JSON bytes per primitive value | +| `expression_depth` | 256 | nested expression/provenance levels | +| `getter_expansions` | 512 | scheduled states per precise getter query | +| `alias_selections` | 512 | additional precise selector states | +| `wrapper_visits` | 1000000 | wrapper visits/substitutions per source | +| `wrapper_depth` | 128 | active symbolic wrapper levels | + +`primitive_bytes` charges six estimated JSON bytes per UTF-16 code unit plus +quotes before creating concatenated strings. Ordinary ASCII can exhaust that +conservative estimate before its actual JSON encoding reaches the selected +byte count. The [calibration record](../../verification/javascript-precision-calibration.md) +keeps existing defaults for its measured literal-value tasks; it does not +establish optimal candidate, getter or alias budgets for larger applications. + +All fields accept positive safe integers; worker memory, execution and JSON +representation bounds remain independently enforced. Omission keeps the defaults; +invalid values fail validation. For example, retry a finite candidate-limit +observation with `semantic_precision: {"primitive_candidates": 512}`. The CLI +accepts `--semantic-precision '{"primitive_candidates":512}'` or a JSON file. +An override applies to that next analysis without restarting REA, remains in +Evidence parameters and effective coverage, and cannot change already retained +facts. Increasing these budgets does not raise graph retention, worker heap, +transfer admission, or parent result memory. Cyclic or unsupported evaluations +remain unknown. Wrapper limitations name visits and depth separately so a +caller can change the resource actually exhausted. Finite inline literal object +spreads use iterative syntax traversal and no longer inherit a duplicate +semantic-depth cut. + Byte, entry, path, and graph-shape bounds are hard limits. The parse deadline is checked before and between bounded parsing and traversal phases; the synchronous Babel and JSON parser calls cannot be preempted mid-call, so their input byte diff --git a/docs/guides/managed/managed-code-analysis.md b/docs/guides/managed/managed-code-analysis.md index 52fe21be53..15d9337bb3 100644 --- a/docs/guides/managed/managed-code-analysis.md +++ b/docs/guides/managed/managed-code-analysis.md @@ -264,12 +264,30 @@ is retained in the classification evidence list. ### Resource admission -The static provider admits regular input files up to 128 MiB before retaining -their bytes. Reads remain bounded if a file grows after admission. Metadata, -CIL decoding, and projected inspection records have a separate conservative -64 MiB representation budget; shared heap strings are cached, but repeated -output occurrences still consume that budget. These are inspection resource -policies, not PE format limits or a hard bound on total V8 heap use. +The static provider defaults to a 128 MiB eagerly retained PE snapshot and a +separate conservative 64 MiB budget for metadata, CIL decoding and projected +inspection records. All three static inspection tools accept +`max_snapshot_bytes` and `max_metadata_representation_bytes` as positive safe +integers in bytes. CLI inspection and path comparison expose +`--max-snapshot-bytes` and `--max-metadata-representation-bytes`. An omitted +value keeps its default; explicit values apply to that call without a restart. + +Snapshot refusal names its selected byte budget and tells the caller to raise +it for the same complete file. Representation exhaustion retains partial or +unavailable coverage and names the selected budget in its `resource-limit` +issue; raise that budget on the next inspection, or select fewer `type_names` +for members. For example: + +```bash +rea inspect-managed-members /absolute/path/Assembly.dll --max-snapshot-bytes 268435456 --max-metadata-representation-bytes 134217728 +``` + +Reads remain bounded if a file grows after admission. A cached snapshot is +rechecked against the current call's snapshot allowance and on-disk digest. +Shared heap strings are cached, but repeated output occurrences still consume +the representation budget. These are per-call resource policies, not PE format +limits or a bound on total V8 heap use. Raising one does not raise the other, +transport limits or available host memory. Path-based member comparison uses the same per-input snapshot policy and inspects its two selected paths sequentially, retaining each inspection rather @@ -326,7 +344,7 @@ original `resource-limit` coordinate. It does not return partially admitted member collections: empty fields, methods, references and edges are unexamined, not proof of absence; exported types, user strings and signature-resolution attestation remain null. Summary and retained pages keep this partial coverage. -The capacity limit is unchanged. If metadata or the type inventory itself cannot +The selected capacity applies to the metadata and member projection together. If metadata or the type inventory itself cannot be admitted, the result remains unavailable. Use the returned `types[].full_name` values to select declaring types before diff --git a/docs/guides/native/binary-diagnostics.md b/docs/guides/native/binary-diagnostics.md index dffd4cd09d..5f4da9d254 100644 --- a/docs/guides/native/binary-diagnostics.md +++ b/docs/guides/native/binary-diagnostics.md @@ -120,7 +120,13 @@ CLI output are unchanged. Complete results have a 32 MiB input, 64 MiB reply, 1 MiB combined diagnostics and 30-second owned command deadline. `REA_PWNTOOLS_MAX_INPUT_BYTES` raises the -input bound alone, up to 1 GiB; the reply and deadline bounds are unchanged. The +launch input default alone, up to 1 GiB. Per-call `producer_limits` selects +`input_bytes`, `output_bytes`, `diagnostic_bytes`, `address_space_bytes`, +`file_bytes`, and `cpu_seconds`; positive safe integers apply on the next call +without restart. `input_bytes` overrides the launch default. File admission +follows selected `output_bytes` unless `file_bytes` is explicit. The separate +`pwntools_timeout_ms` controls each command wall deadline. On CLI pass +`--producer-limits '{"cpu_seconds":60,"output_bytes":134217728}'`. The reply budget applies to the decoder record; the CLI/MCP Evidence envelope adds copies and encoding overhead. The pinned MCP SDK defaults to a 10 MiB receive buffer. Large complete results need @@ -128,8 +134,8 @@ a caller-configured `StdioClientTransport({ maxBufferSize: ... })` and a matchin server `REA_MCP_MAX_RESPONSE_BYTES`. See [MCP delivery budgets](../../reference/mcp-contracts.md#tool-results). These settings affect delivery, not decoder coverage or limits. The large extended-index verification uses 256 MiB and an explicit five-minute request -timeout in its separate opt-in MCP lane. REA does not silently truncate the result. Python lowers resource soft limits to at most 3 GiB virtual address space, -30 CPU seconds and 64 MiB file output. Inherited tighter soft/hard limits are +timeout in its separate opt-in MCP lane. REA does not silently truncate the result. Python lowers resource soft limits to the selected policy: defaults are 3 GiB +virtual address space, 30 CPU seconds and 64 MiB file output. Inherited tighter soft/hard limits are retained, and the effective values are returned in the result limitations. Virtual address space is not RSS; Unicorn needs a large virtual map. Limits fail with no partial success, and owned process/root cleanup completes independently of cancellation. diff --git a/docs/guides/native/go-binaries.md b/docs/guides/native/go-binaries.md index 2185643455..7df4ec5c7d 100644 --- a/docs/guides/native/go-binaries.md +++ b/docs/guides/native/go-binaries.md @@ -116,18 +116,22 @@ The result's limitations state this explicitly. A module-mode build with a main module record and no dependency records has an observed empty dependency list. When `include_string_literals: true` (CLI `--include-string-literals`, optional -`max_string_literals` / `--max-string-literals`, default `2000`, maximum `10000`) +`max_string_literals` / `--max-string-literals`, default `2000`) is selected, `string_literals` reports `evidence_kind: "inferred"` alongside deduplicated printable UTF-8 literals referenced by word-aligned two-word `(ptr, len)` string headers in non-executable initialized data. Because Go packs read-only string data contiguously without NUL terminators, slicing by `(ptr, len)` recovers exact literal boundaries instead of concatenated C-string runs. Each -target slice (`1`–`4096` bytes) must stay within a single read-only mapped +target slice (default `1`–`4096` bytes) must stay within a single read-only mapped section or segment, decode as printable UTF-8 (`\t`, `\n`, `\r`, joiners `U+200C` / `U+200D` and printable Unicode scalar values, excluding control and other format characters), and fit within -a 64 MiB header-scan budget and 512 KiB aggregate string budget -(`truncated: true` when capped). Compiler-inlined or +a default 64 MiB header-scan budget and 512 KiB aggregate string budget +(`truncated: true` when capped). Select `max_string_literal_bytes`, +`max_string_bytes`, `max_string_header_scan_bytes`, or `max_string_candidates` +to change these independent budgets. Their CLI flags use hyphens, for example +`--max-string-literal-bytes 16384`. `limits_reached` names exhausted controls; +increase those controls for a new inspection of the same complete artifact. Compiler-inlined or stack-constructed strings without static headers, unapplied relocation addends and Mach-O `LC_DYLD_CHAINED_FIXUPS` pointer encodings are noted in `limitations`. @@ -201,23 +205,39 @@ host permission denial, changed artifacts and cancellation retain distinct errors. The selected file is never modified; this inspection creates no temporary files, starts no subprocesses and accesses no network endpoint. -An input above 256 MiB returns `resource_constraint` with resource `file-size` -and `reported_limits.maximum_input_bytes`. Select a suitable input within that -fixed allowance; changing an output-write or transport budget does not admit it. +The existing default resource budgets are now explicit per-call controls: -Resource guards bound the selected file and cumulative header searches to 256 MiB -each, aggregate embedded string contents to 1 MiB (varint length prefixes are -framing and do not count toward this allowance), structural table decoding to -16 MiB and the complete normalized metadata report to 8 MiB. Inline strings must -also fit within their actual mapped source range. Identical search spans with -the same virtual alignment are searched once; partially overlapping spans -consume the cumulative search budget -and fail with a CPU resource constraint when it is exhausted. Oversized results -fail without partial success. MCP also applies -its [response transport budget](../../reference/mcp-contracts.md), which counts both text and -structured content; a report within the metadata limit can still require export -from retained Evidence. These limits are not operating-system memory or CPU -isolation. +| Parameter | Default | What it admits | +| ------------------------------ | ----------- | ------------------------------------------------------------- | +| `max_input_bytes` | 256 MiB | Complete stable input file | +| `max_report_bytes` | 8 MiB | Complete normalized metadata representation | +| `max_build_info_bytes` | 1 MiB | Aggregate embedded string contents, excluding length prefixes | +| `max_header_search_bytes` | 256 MiB | Cumulative build-info search work | +| `max_string_literals` | 2,000 | Returned inferred string rows | +| `max_string_literal_bytes` | 4,096 bytes | Each inferred string candidate | +| `max_string_bytes` | 512 KiB | Aggregate returned inferred text | +| `max_string_header_scan_bytes` | 64 MiB | String-header and candidate-byte examination | +| `max_string_candidates` | 100,000 | Distinct inferred candidates retained before projection | + +Every value is a positive safe integer. Input and string byte maxima follow the +runtime Buffer/string representation, with base64 expansion accounted for in +build-info admission; schemas advertise their exact maxima. CLI options use the same names with +hyphens, such as `--max-build-info-bytes 2097152`. Inferred-string controls +require `include_string_literals: true`. Omitted controls keep the defaults +and stay omitted in request Evidence. Error recovery names the exhausted +parameter and effective bytes; increase that parameter on the next call, +without restarting REA. The original complete artifact and its identity are +preserved. An unchanged retry cannot increase a deterministic allowance. + +The shared native container reader separately bounds structural table decoding +to 16 MiB. None of these Go controls changes that boundary. Inline strings must +also fit their actual mapped source range. Identical search spans with the same +virtual alignment are searched once; partially overlapping spans consume the +cumulative search budget. Oversized complete reports fail without partial +success and without allocating the full JSON serialization. MCP's separate +[response transport budget](../../reference/mcp-contracts.md) can still require +export or paging of successfully retained Evidence. These controls do not +establish operating-system memory or CPU isolation. Embedded metadata is evidence of file content. It does not authenticate a compiler, dependency, commit or supply chain, and it does not establish runtime diff --git a/docs/guides/native/native-investigation.md b/docs/guides/native/native-investigation.md index b1f6b864d3..afa10cd2cc 100644 --- a/docs/guides/native/native-investigation.md +++ b/docs/guides/native/native-investigation.md @@ -1247,7 +1247,7 @@ resource, REA releases that runtime and admits the next observation. Cancellation observed while output and Evidence are finalized returns partial cancellation Evidence, even if the target has already exited. -- the outcome, exit status and captured stdout/stderr (1 MiB kept per stream +- the outcome, exit status and captured stdout/stderr (1 MiB by default kept per stream while excess bytes are drained and counted without growing capture files). Each stream reports `complete`; when draining could not finish, its byte count is an observed lower bound and truncation is explicit; @@ -1260,16 +1260,32 @@ cancellation Evidence, even if the target has already exited. code makes coverage partial; - limitations. -The bridge limits retained event JSON to 8 MiB and caller frames to 65,536 -across the run. It checks admission before retaining another complete event; +The bridge defaults to 8 MiB retained event JSON and 65,536 caller frames +across the run. Select `capture_limits` before launch to adjust `trace_bytes`, +`trace_frames`, `breakpoint_location_bytes`, `other_stops`, `selector_bytes`, +and `output_bytes`. Defaults are respectively 8388608, 65536, 8388608, 256, +1024, and 1048576. Byte budgets accept positive safe integers up to the smaller of the running +Node Buffer and single-string representation ceilings; counts are positive safe +integers. These representability ceilings do not establish available memory or +raise complete Evidence/transport admission. Omitted fields keep their defaults. The journal shares +`trace_bytes`; completed and partial capture readers use the selected budgets. +For example, `"capture_limits":{"trace_bytes":16777216,"output_bytes":2097152}` +selects 16 MiB trace/journal and 2 MiB retained output per stream. These choices +do not extend `duration_ms`, `max_events`, the outer client deadline or MCP +receive allowance, and require no REA restart. A larger budget applies to a new +authorized observation; it cannot recover facts omitted from an earlier run. It checks admission before retaining another complete event; `resource-limit` reports which observations could not be completed without silently changing the requested event or frame settings. -Resolved breakpoint locations have a separate aggregate 8 MiB metadata budget. +Resolved breakpoint locations have a separate aggregate metadata budget, +default 8 MiB (`capture_limits.breakpoint_location_bytes`). If their details exceed it, `location_count` retains the full observed match count, coverage is partial, and `breakpoint_locations_truncated` is true. This post-run metadata limit does not imply that the target was killed. -Accepted entries and signal stops are also flushed to a bounded 8 MiB journal. +Accepted entries and signal stops are also flushed to the bounded trace journal. +When event admission stops, `coverage.capture_limit` identifies `trace_bytes` +or `trace_frames` when the bridge supplied that fact; historical reports that +lack it retain the combined limitation without guessing the exhausted resource. If cancellation, timeout, capture failure, or a later bridge error prevents a full result, the error retains this evidence in `details.partial_observation` with partial coverage. Available output prefixes remain inline; their byte diff --git a/docs/guides/native/recorded-crashes.md b/docs/guides/native/recorded-crashes.md index 4d571c3b9d..b08b17f897 100644 --- a/docs/guides/native/recorded-crashes.md +++ b/docs/guides/native/recorded-crashes.md @@ -105,8 +105,14 @@ Initial Linux x64 limits: 32 MiB input, 64 MiB decoder reply, 1 MiB diagnostics stage, 30 seconds owned wall deadline per stage, 3 GiB address-space soft ceiling and 30 seconds CPU soft ceiling. `REA_PWNTOOLS_MAX_INPUT_BYTES` raises the input bound alone, up to 1 GiB, and returns `provider_unavailable` when it is not a -decimal integer in range; the reply, diagnostic and deadline bounds stay fixed, so a -large core that outruns them still fails explicitly instead of partially. Tighter +decimal integer in range. Per-call `producer_limits` selects input/output/diagnostic, +file/address-space bytes and CPU seconds; positive safe integers apply on the +next call without restart. `input_bytes` overrides the launch default, and the +file allowance follows selected output unless `file_bytes` is explicit. Both +Python decoder and optional core-only debugger consume the same accepted policy. +Use CLI `--producer-limits '{"cpu_seconds":60,"output_bytes":134217728}'`. +`pwntools_timeout_ms` is the independent per-stage wall deadline. A large core +that still exceeds the selected complete-reply allowance fails explicitly. Tighter inherited soft/hard limits are retained; hard limits are not raised. Complete oversized results fail explicitly. These process limits do not constitute a sandbox for caller-supplied engine code. diff --git a/docs/guides/runtime/frida-instrumentation.md b/docs/guides/runtime/frida-instrumentation.md index 12cda750f7..49b32d0e22 100644 --- a/docs/guides/runtime/frida-instrumentation.md +++ b/docs/guides/runtime/frida-instrumentation.md @@ -26,6 +26,6 @@ Use `get_evidence_bundle` with `detail="summary"` and `filters={"evidence_id":"P Failed startup cleanup retains acquired Frida handles, including a paused spawn that has not attached, for later cleanup retries. Its remote connection stays leased until session detach and target resume complete. CLI commands await provider cleanup before returning; MCP retains these resources for cleanup when its server closes. Server shutdown stops admitting new Frida operations and waits for admitted startup, discovery, and queued work before releasing their resources. A failed cleanup remains reported and can be retried by closing the server again, rather than treating an unregistered session as fully released. -Frida scripts execute inside the selected target with that process's memory, filesystem, and network authority. Local device lookup waits 1000 ms. This release has no runtime override for that lookup; another REA timeout does not raise it. Remote endpoints do not use that lookup. Message capture is bounded to 16 MiB and reports truncation. Metadata, type, and text field caps, and the JSON node cap, are also fixed. Evidence names a cap when that observation actually dropped data. This release has no runtime override for those capture caps. Script source is not stored in Evidence; its selected path and SHA-256 digest are. For a file, `source_sha256` identifies the original selected bytes. Invalid UTF-8 continues to use replacement decoding; Evidence then records that limitation and `raw_result.source_representation` with the encoding, decoding mode, and `loaded_source_sha256` of the UTF-8 string passed to Frida. Valid UTF-8 files and inline scripts do not need this additional representation fact. Evidence retains non-authentication target selection such as device or endpoint, mode, program or PID, and spawn arguments. Authentication values remain excluded. Results report the observed session and messages, while unsupported platform metadata remains unknown. A request cancelled before a Frida operation starts has no provider effects. If cancellation arrives during a native Frida call, REA waits for that call to return, then closes any session it created and returns the observed session as a partial observation. A cancelled script load on an existing session releases only its new script before queued operations can run; the session and previously loaded scripts remain available. If cancellation is observed when the native call returns, partial Evidence retains the source identity and captured messages even if that call rejects, including messages emitted during awaited cleanup. The caller's cancellation reason remains primary and a native rejection is retained as a sanitized diagnostic. A failure already observed before cleanup stays primary if cancellation arrives during cleanup. The cancellation cutoff is REA's observation of native completion, rather than a claim about unobservable native event order. `script_id` is REA's load-attempt identity. When creation rejects without returning a handle, `script_cleanup: "no_handle"` means REA acquired no handle to unload; it does not prove that the engine performed no work. Once the provider commits a successful load, later request cancellation does not unload that script; use `unload_frida_script` to release it. If script cleanup fails, the script remains owned and a paused target cannot resume until the pending instrumentation is released. Queued script loads, resumes, and unloads check cancellation before dispatching native effects. Cancellation Evidence separately reports whether a target resume was observed, whether instrumentation cleanup completed, and that final target liveness remains unknown. REA does not kill the target. Remote discovery retains its observations and reports a separate cleanup error if the temporary connection cannot be removed. +Frida scripts execute inside the selected target with that process's memory, filesystem, and network authority. Local device lookup waits 1000 ms. This release has no runtime override for that lookup; another REA timeout does not raise it. Remote endpoints do not use that lookup. Message capture defaults to 16 MiB per session, 4 KiB per type, 32 KiB per error text field, 250,000 JSON nodes and depth 64. Select `capture_limits` on `start_frida_session` or `instrument_with_frida` before observing: `session_bytes`, `message_type_bytes`, `message_text_bytes`, `json_nodes`, and `json_depth`. Omitted fields keep these defaults. The session allowance includes JSON/base64 expansion, the envelope and metadata; a budget too small for one complete event retains no event. Depth is bounded to 992 levels, reserving eight enclosing message/status/Evidence containers within REA's shared JSON representation limit. These are retention budgets, independent of target authority, observation duration and MCP/client delivery limits. The selected policy is fixed for that session and returned as `capture_limits` in observations; no REA restart is required. The CLI exposes `--capture-session-bytes`, `--capture-message-type-bytes`, `--capture-message-text-bytes`, `--capture-json-nodes`, and `--capture-json-depth`. Evidence names the effective control when that observation dropped data. Changing a budget requires choosing a new session or experiment and cannot recover already discarded live observations. Script source is not stored in Evidence; its selected path and SHA-256 digest are. For a file, `source_sha256` identifies the original selected bytes. Invalid UTF-8 continues to use replacement decoding; Evidence then records that limitation and `raw_result.source_representation` with the encoding, decoding mode, and `loaded_source_sha256` of the UTF-8 string passed to Frida. Valid UTF-8 files and inline scripts do not need this additional representation fact. Evidence retains non-authentication target selection such as device or endpoint, mode, program or PID, and spawn arguments. Authentication values remain excluded. Results report the observed session and messages, while unsupported platform metadata remains unknown. A request cancelled before a Frida operation starts has no provider effects. If cancellation arrives during a native Frida call, REA waits for that call to return, then closes any session it created and returns the observed session as a partial observation. A cancelled script load on an existing session releases only its new script before queued operations can run; the session and previously loaded scripts remain available. If cancellation is observed when the native call returns, partial Evidence retains the source identity and captured messages even if that call rejects, including messages emitted during awaited cleanup. The caller's cancellation reason remains primary and a native rejection is retained as a sanitized diagnostic. A failure already observed before cleanup stays primary if cancellation arrives during cleanup. The cancellation cutoff is REA's observation of native completion, rather than a claim about unobservable native event order. `script_id` is REA's load-attempt identity. When creation rejects without returning a handle, `script_cleanup: "no_handle"` means REA acquired no handle to unload; it does not prove that the engine performed no work. Once the provider commits a successful load, later request cancellation does not unload that script; use `unload_frida_script` to release it. If script cleanup fails, the script remains owned and a paused target cannot resume until the pending instrumentation is released. Queued script loads, resumes, and unloads check cancellation before dispatching native effects. Cancellation Evidence separately reports whether a target resume was observed, whether instrumentation cleanup completed, and that final target liveness remains unknown. REA does not kill the target. Remote discovery retains its observations and reports a separate cleanup error if the temporary connection cannot be removed. The [Frida Core API](https://frida.re/docs/frida-core-api/) documents device management, sessions, and script loading; the [JavaScript API](https://frida.re/docs/javascript-api/) documents target-side instrumentation. [Luma](https://luma.frida.re/) is Frida's standalone GUI; REA does not automate or control it. `npm run verify:frida` requires a Linux host with the optional Frida binding and uses a short-lived local Node.js fixture created by the verifier. Add `-- --address HOST:PORT --pid PID` to run the separate remote attach lane against a caller-selected target. Mock tests do not establish real device/server compatibility. diff --git a/docs/guides/signals/signal-recordings.md b/docs/guides/signals/signal-recordings.md index 80a7a2c2ba..d2dbb38cef 100644 --- a/docs/guides/signals/signal-recordings.md +++ b/docs/guides/signals/signal-recordings.md @@ -103,10 +103,10 @@ failure; bounded post-run integrity checks finish before honoring a late abort. never passed to sigrok. Caller-supplied tools and their libraries/decoder modules are trusted, and hashes identify bytes rather than publisher authenticity. -The input copy is bounded at 32 MiB before reading, each command has a 30-second -deadline, and combined stdout/stderr is bounded at 4 MiB during collection. +By default, the input copy is bounded at 32 MiB before reading, each command has +a 30-second deadline, and combined stdout/stderr is bounded at 4 MiB during collection. Oversized output fails rather than becoming truncated success. These limits do -not form an OS sandbox. Before execution, `prlimit` enforces **1 GiB address +not form an OS sandbox. Before execution, `prlimit` defaults to **1 GiB address space per process**, **64 MiB per file**, **30 CPU seconds** and no core dumps. Stricter inherited soft/hard limits are preserved; Evidence records the effective ceilings. Both limits are set before execution, so decoder allocations and compressed @@ -114,6 +114,23 @@ recording expansion cannot exceed the per-process address-space cap. These are not aggregate process-memory or filesystem quotas. The selected limiter is trusted and its path, digest and arguments are retained in Evidence. +Each capture/recording operation accepts `producer_limits` with positive safe +integer overrides: `input_bytes`, `output_bytes`, `command_timeout_ms`, +`address_space_bytes`, `file_bytes`, and `cpu_seconds`. The wall-clock timeout +is at most 2,147,483,647 milliseconds, matching Node timers. Omitted fields keep +the defaults above. Discovery accepts the same controls except `input_bytes`, +since it does not read a capture or recording. Packet field discovery defaults +to 32 MiB output rather than 4 MiB. + +CLI supplies this object as JSON with `--producer-limits`, for example +`--producer-limits '{"input_bytes":67108864,"output_bytes":8388608,"cpu_seconds":60}'`. +Values apply to the next call without restarting REA. Effective native ceilings +still preserve any tighter inherited soft and hard limit. Errors identify the +exhausted producer control and retain partial output and cleanup status; follow +that repair after cleanup settles. Raising producer budgets does not raise +separate parser representations, transport buffers or external client deadlines. +Never truncate the selected artifact to fit an input budget. + ## Verification ```sh diff --git a/docs/guides/wasm/wasm-artifacts.md b/docs/guides/wasm/wasm-artifacts.md index 1bef747b31..ab679c3c86 100644 --- a/docs/guides/wasm/wasm-artifacts.md +++ b/docs/guides/wasm/wasm-artifacts.md @@ -64,10 +64,11 @@ Equivalent MCP input: ``` Each selected regular file is read through the existing stable artifact reader; -symlinks, changing files and files over 8 MiB fail closed. An oversized input -returns `resource_constraint` with resource `file-size` and its fixed byte -allowance in `reported_limits.maximum_input_bytes`; retrying the unchanged file -does not raise that allowance. WASM bytes are copied +symlinks, changing files and files over the selected `max_input_bytes` (default +8 MiB) fail closed. An oversized input returns `resource_constraint` with +resource `file-size` and the effective allowance in +`reported_limits.maximum_input_bytes`. Its recovery names `max_input_bytes`; +raise that per-call value before retrying the same complete file. WASM bytes are copied into a private workspace as read-only `module.wasm`; its digest is rechecked before and after each tool. WABT validates with `wasm-validate --enable-all --ignore-custom-section-errors`, reports sections @@ -137,12 +138,30 @@ the equivalent MCP input is `{"path":"/artifacts/module.wasm", "command_timeout_ms":60000,"detail":"summary"}`. Summary delivery retains the complete Evidence but does not reduce WABT analysis work. -The 8 MiB input allowance and 32 MiB combined command-output budget remain fixed; -increasing `command_timeout_ms`, host memory or the client deadline does not raise -them. Do not truncate required module bytes to fit. Select another suitable -complete artifact only if it answers the question, otherwise report the limit. -Inspection output past the budget fails with `resource_constraint`; an oversized -`--version` probe still reports `unreadable_output`. Both streams share the +The default input, executable, and combined command-output allowances remain +8 MiB, 64 MiB, and 32 MiB respectively. Select `max_input_bytes`, +`max_tool_bytes`, or `max_output_bytes` (CLI `--max-input-bytes`, +`--max-tool-bytes`, `--max-output-bytes`) to change one allowance for the next +inspection. Values are positive integer bytes; the advertised schema maximum +follows the runtime's single-string or Buffer representation ceiling. Raising a +budget does not reserve host memory or raise another budget, the command +deadline, or the client's receive buffer. No REA restart is required. Unset +values preserve the defaults, and explicit values remain in Evidence parameters. +The same effective input/tool limit is used for initial and integrity rereads. +Before cloning and committing complete Evidence, REA accounts for the actual +escaped JSON representation and authentication copies. A large command budget +does not guarantee that the complete report fits the runtime's single-string +ceiling or available heap. Those refusals name the distinct representation or +heap constraint; increasing `max_output_bytes` cannot repair them. WABT reports +use no redundant raw-result mirror, but exact import/export forms and derived +interface facts can still expand beyond their source WAT size. + +A limit failure names its effective byte value and exact per-call control. +Raise that control, resolve incomplete cleanup, then retry the same complete +artifact. Do not truncate required module bytes. If already at the runtime +maximum, report the limit or explicitly select another suitable complete input. +Output past the budget fails with `resource_constraint`, including a `--version` +probe; no truncated output is claimed as a successful inspection. Both streams share the budget, so a stream that starts after the other has filled it retains nothing. The captured output keeps up to the first 4,096 bytes that each stream retained, each stream's retained byte count, and the exit code and signal. Retained text @@ -238,3 +257,25 @@ Glue parsing accepts Node-supported WASM source-phase imports, including `await import.source("./module.wasm")`. Their string literals retain the same coordinates and local candidate associations as ordinary import literals; these are static source observations, without module instantiation or execution. + +Typed interface decoding separately charges `max_interface_bytes` (default +32 MiB; CLI `--max-interface-bytes`). The selected allowance applies to both +nested WAT parsing and retained typed entries, so increasing it is not blocked +by an independent fixed nested-parser cap. A partial interface preserves the +original WAT, admitted entries, exact exhausted allowance and retry guidance. +Inspect the same complete module again with a larger value to recover omitted +typed facts. + +`compare_wasm_interfaces` accepts its own `max_interface_bytes` in the JSON +input, also default 32 MiB. It admits selected entries and group metadata before +building the comparison inventory. A larger comparison budget cannot upgrade +an already partial inspection: first recover the producer Evidence, then compare +those exact replacement records. Both allowances retain independent runtime +serialization and heap boundaries. + +Auxiliary `candidate_paths` and `glue_paths` each retain a default admission of +16 paths. Set `max_artifact_paths` (CLI `--max-artifact-paths`) to admit a larger +explicit selection. Admission happens before any provider acquisition or command, +and errors report the selected list, requested count and required control. +Per-file byte budgets still apply to each admitted path; increasing the path +count does not raise those limits or reserve host memory. diff --git a/docs/guides/web/browser-observation.md b/docs/guides/web/browser-observation.md index bbcc1ca525..c0e7b95ca7 100644 --- a/docs/guides/web/browser-observation.md +++ b/docs/guides/web/browser-observation.md @@ -635,7 +635,7 @@ rejects it as unknown; other command rejections retain their reported failure. Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. Scripts are different: scripts the page parsed before REA attached are still reported under `scripts`. A page that loaded before attach can therefore list its scripts while `network.requests` lacks the requests that fetched them. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. -Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation uses `source_map_fetch_timeout_ms` (default 30000) and a 64 MiB aggregate streamed-response budget across redirects. The byte budget has no runtime override. A fetch overrun is an item limitation, not a thrown analysis timeout. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`. +Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation uses `source_map_fetch_timeout_ms` (default 30000) and a default 64 MiB aggregate streamed-response budget across redirects. Select `source_map_limits.fetch_bytes` to change byte admission for the next call. A fetch overrun is an item limitation, not a thrown analysis timeout. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding defaults to 64 MiB for raw resolved identities and 32 MiB for aggregate inline evidence; `source_map_limits.fetch_bytes` and `source_map_limits.output_bytes` own those independent admissions. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`. A fully read response that fails UTF-8, JSON, mapping, or structural validation retains `artifact: null`, empty original-source, module-edge, and mapping diff --git a/docs/guides/web/web-network-captures.md b/docs/guides/web/web-network-captures.md index 8d36c1e1cf..6882d50a1e 100644 --- a/docs/guides/web/web-network-captures.md +++ b/docs/guides/web/web-network-captures.md @@ -1114,44 +1114,21 @@ for the hint per reason. ## Resource and lifecycle boundaries -Complete evidence has a 32 MiB input, 96 MiB decoder reply and 64-level nesting -budget. Those budgets have no runtime override. `capture_timeout_ms` does not -raise them. Exceeding a budget returns no partial success. -Oversized selected captures return an input error identifying `capture_path` -and the byte budget; owned reply/output failures remain output or provider errors. -An explicit complete-reply refusal names the 100663296-byte allowance and its -lack of a runtime override. Record selection, paging, delivery mode and worker -memory do not increase that allowance; a smaller producer export is a narrower -observation, and the refusal leaves no completed capture Evidence to retrieve. -Cancellation during reply-file finalization retains validated completed capture -facts as partial observations. Malformed or unvalidated replies are not promoted. -HAR, SAZ and qlog run in an owned Node old-generation heap with one V8 worker. -`REA_CAPTURE_DECODER_OLD_GENERATION_MIB` is a decimal integer of mebibytes from -8 to 16384. Unset uses 192, the previous constant. Invalid values fail -configuration and name the setting; they do not fall back to 192. The setting -seeds the launch value. In MCP, set `capture.decoder_old_generation_mib` -through `update_analysis_settings` for decoders belonging to newly accepted -operations, without restarting REA. Changing the launch environment still -requires a new process. A memory failure names the effective mebibytes, known -source, default, floor, and maximum. The decoder -passes that value as `--max-old-space-size` and removes inherited -`NODE_OPTIONS`, so additional host memory and `NODE_OPTIONS` do not raise the -heap. The 8 MiB semi-space stays fixed; a smaller old generation aborts Node -during startup instead of enforcing a limit. Mitmproxy decoding applies -Linux limits of 768 MiB address space, 30 seconds CPU and 96 MiB file output; -both commands use `capture_timeout_ms` (default 30000) and independently supervised -process cleanup. Owned decoder diagnostics retain at most 1 MiB across stdout -and stderr; the collector drops overflow as it arrives, reports the limit -failure and verifies cleanup. Heap limits alone do not establish aggregate RSS limits. -SAZ and qlog refuse decoding when neither a Node heap limit nor a Windows Job Object bound can be enforced. Existing HAR availability is preserved. -When REA runs on Bun, the HAR/SAZ/qlog decoder runs on Bun, which ignores these V8 heap -flags; its limitations and resource failures then omit the V8 heap sizes. On -Windows, the decoder's owned Job Object instead limits its committed memory to -the effective `REA_CAPTURE_DECODER_OLD_GENERATION_MIB` value plus a 512 MiB -reserve for Bun itself. The default 192 MiB setting therefore yields 704 MiB. -The limitations state that limit, and reaching it returns `resource_constraint` -with `process_memory_limit_bytes`, whatever exit status Bun reports. Other Bun -hosts enforce no decoder heap limit, so raising the setting does not bound them. +Complete evidence defaults to a 32 MiB input, 96 MiB decoder reply and +64-level nesting allowance. Set `max_input_bytes`, `max_reply_bytes` or +`max_nesting_depth` on the next `inspect_web_network_capture` call to change +that operation's policy; CLI equivalents are `--max-input-bytes`, +`--max-reply-bytes` and `--max-nesting-depth`. No REA restart is required. +Byte allowances are positive safe integers; nesting is bounded by REA's +advertised maximum of 996 levels, derived by reserving four enclosing +reply/Evidence levels within the 1,000-level JSON representation. Input admission also covers expanded SAZ +members and protobuf descriptor/body reads. Reply admission covers the complete +child reply and protobuf interpretation work. These choices do not raise the +separately owned worker heap, CPU limits or client deadline/receive buffer. +Record selection takes place after complete decoding, so it cannot bypass +whole-capture admission. A refusal names the exhausted control; increase that +control and retry the same unchanged capture after cleanup. No completed capture +Evidence exists for a refused decode. Observed decoder memory exhaustion returns `resource_constraint`, with the selected capture, applicable limits and retained process diagnostics. Raising diff --git a/docs/guides/web/web-source-location.md b/docs/guides/web/web-source-location.md index a9e6a985a2..182e2601db 100644 --- a/docs/guides/web/web-source-location.md +++ b/docs/guides/web/web-source-location.md @@ -82,13 +82,13 @@ implement every current source-map extension. One owned Node subprocess decodes the map. Its private request directory is removed before success is returned. Only this child's inherited `NODE_OPTIONS` is removed; explicit V8 flags set a 192 MiB old-generation and 8 MiB semi-space -budget, one V8 worker and `decode_timeout_ms` (default 20000). The codec heap, map budget, and reply budget have no runtime override. Reported actual V8 heap limits +budget, one V8 worker and `decode_timeout_ms` (default 20000). These are the defaults: select `source_map_limits.codec_heap_mib` for a different heap request. Reported actual V8 heap limits are checked. These are heap limits, not an aggregate process RSS limit. When REA runs on Bun, the subprocess is Bun, which ignores these V8 flags: `runtime` records `id: "bun"`, the Bun version, its Node.js compatibility version and a null `v8_heap_limit_bytes`. On Windows, the codec's owned Job Object then -limits its committed memory to 704 MiB, the 192 MiB budget plus a 512 MiB -reserve for Bun itself; `process_memory_limit_bytes` records that limit, the +limits its committed memory to the selected codec heap request plus a 512 MiB +reserve for Bun itself (704 MiB with the default request); `process_memory_limit_bytes` records that limit, the limitations state it, and reaching it returns `resource_constraint` with `codec_process_memory_limit_bytes`. Elsewhere Bun has no enforced codec limit: `process_memory_limit_bytes` is null and the limitations state that no codec @@ -102,16 +102,143 @@ run token while process release is unconfirmed, and the directory until removal succeeds. A later successful retry settles every error bound to that command; historical failure diagnostics remain available. -Complete evidence is bounded by a 4 MiB map, 32 MiB codec reply and a shared +Complete evidence defaults to an 8 MiB map, 32 MiB codec reply and a shared 262144-record decoder budget. Generated rows, mapping segments, leaf maps, source and name declarations, `sourcesContent` entries and indexed sections all consume this budget. Generated rows and mapping segments also each have a -262144 limit; indexed maps allow at most 64 nested section levels. Those budgets and the -192 MiB codec heap have no runtime override. Exceeding a budget returns an explicit +262144 limit; indexed maps default to 64 nested section levels. Select per-call +`source_map_limits` to change these admissions. Exceeding a budget returns an explicit failure, without partial results. The existing 32 MiB manifest and 16 MiB selected script boundaries also apply. No global toolchain or browser configuration changes are made. +## Compact point delivery and retained source text + +Summary delivery is useful for large point answers. Complete output can answer +a small-map task with fewer calls. The +[delivery qualification](../../verification/source-location-delivery.md) +records measured task costs and first-use discovery tradeoffs. + +Complete delivery remains the default and includes raw map text and embedded source +content. Select `detail: "summary"` in MCP, or `--detail summary` with +`--evidence_output /analysis/source-point.json` in CLI, to retain the same complete +Evidence and return body-free point metadata. Summary includes source/map/manifest +identity, requested and matched generated positions, original source positions, +content digests and sizes, match count, runtime qualifications and limitations. +MCP returns the summary Evidence directly; its `normalized_result.parent_evidence_id` +identifies the complete parent. CLI returns `{ summary: , evidence_bundle: +{ path, bytes } }`; read `summary.normalized_result.parent_evidence_id` and use the +published bundle path for later views. The receipt byte count describes the written +complete Evidence bundle. +Delivery selection does not enter that parent's analysis parameters or change its ID. + +Summary returns the first metadata page (up to 64 matches and 1 MiB), with explicit +`coverage.total`, `coverage.next_offset` and `coverage.exhausted`. Every equal-position +alternative remains in the parent, including duplicate reported names, distinct +indexed-section/source indexes and generated-only segments. Page exhaustion describes +the retained lookup alternatives; execution and deployment authenticity remain unknown. +For another page, call `inspect_analysis_view` with the same retained parent: + +```json +{ + "source": { + "kind": "retained-evidence", + "evidence_id": "" + }, + "view": { + "kind": "source-location", + "collection": "matches", + "offset": 64, + "limit": 64 + } +} +``` + +Source text is explicit. Select `collection: "source-text"`, an original +`match_index` from the metadata page, and UTF-16 `offset`/`limit` (default 0/1024). +For a snippet around the mapped point, use its `original_position.offset` when +known. The result preserves content digest/size and total code units, returns only +the selected text, and reports text-page coverage. Unretained or generated-only +content returns `item: null` and unavailable coverage. An offset splitting a +surrogate pair is rejected; a page never splits a complete Unicode character. + +Both collections accept `max_bytes` (default 1048576), independently of their +row/code-unit limit and MCP response limits. A byte refusal names the allowance +and retained parent: select a smaller page or increase `max_bytes`, then retry +without decoding again. A large individual identity may require a larger view +allowance or complete export. Complete map text and diagnostics remain in the parent. +An optional `view` can accompany `detail: "summary"` on the initial trace, returning +that selected page immediately. `view` with complete delivery is rejected before +artifact acquisition. Summary also refuses before acquisition when session retention +or the CLI Evidence output is unavailable. + +These views read retained facts even after the original map is removed. CLI can read +the saved parent with `inspect-analysis-view` and an `evidence-file` source pointing +to the written bundle; select its `evidence_id` when the bundle has multiple records. +MCP can export the complete session with `export_evidence_bundle`. A complete MCP +response that exceeds transport admission retains its parent and supplies a +`source-location` matches continuation. + +## Selecting budgets and recovering the same map + +Both `trace_web_source_location` and `analyze_web_bundle` accept an optional +`source_map_limits` object. Omitted fields retain the defaults above and stay +omitted in Evidence parameters. An update applies on the next call without a +REA restart. Failure text names the exact field, effective value, supported +range and next action; increasing a deadline does not change any of these +budgets. + +| Field | Default | Scope | +| ------------------ | -------- | ------------------------------------------------------------------------------------- | +| `map_bytes` | 8388608 | Local selected map admission, enforced before reading and again by the codec | +| `output_bytes` | 33554432 | Complete local request/reply or aggregate fetched-map representation | +| `decoded_rows` | 262144 | Generated rows before upstream allocation | +| `decoded_segments` | 262144 | Mapping segments before upstream allocation | +| `decoded_records` | 262144 | Shared structural work across leaves, declarations, rows, segments and sections | +| `section_depth` | 64 | Indexed-section nesting | +| `codec_heap_mib` | 192 | Local codec old-generation request; process-memory enforcement remains host dependent | +| `fetch_bytes` | 67108864 | Aggregate response bytes and, independently, resolved-identity bytes | + +The local 8 MiB admission covers the pinned 4,284,329-byte main map and 5,529,840-byte +DateSelect map from Actual Budget without an admission retry. This is an artifact +compatibility allowance, not a context or latency optimization: complete Evidence +retains raw map text, and those two measured replies estimate 1.59 and 2.23 million +tokens. A smaller admission refuses the map; it does not compact successful +Evidence. MCP retains complete Evidence before encoding; an oversized response +returns its retained reference for `export_evidence_bundle`, or callers can use +complete CLI JSON. Select opt-in compact delivery above to keep complete Evidence accessible while returning only point metadata or an explicitly selected text page. +Browser collection keeps its separate conservative defaults; see the +[calibration record](../../verification/source-map-budget-calibration.md). + +Local tracing accepts all listed fields except `fetch_bytes`. Bundle fetching +accepts all except `map_bytes` and `codec_heap_mib`: it uses its existing +inspection process rather than launching the local codec. Unsupported fields +are rejected rather than silently ignored. Byte fields are bounded below +Node's single-string representation ceiling, record fields by the signed +32-bit decoder coordinate range, section depth by the same supported count range and shared structural admission, +and heap requests by the existing 128..16384 MiB worker envelope. These bounds +remain independent; a larger map can require a larger encoded request budget +and codec heap, and the outer MCP client still owns its receive buffer/deadline. + +For example, a valid 10 MiB selected local map can be retried with: + +```json +{ + "source_map_limits": { "map_bytes": 16777216 } +} +``` + +Keep the original manifest/map/point selection in that retry. The CLI spelling +is `--source-map-bytes 8388608`; the other local flags are +`--source-map-output-bytes`, `--source-map-decoded-rows`, +`--source-map-decoded-segments`, `--source-map-decoded-records`, +`--source-map-section-depth` and `--source-map-codec-heap-mib`. +`analyze-web-bundle` instead offers `--source-map-fetch-bytes` and the shared +output/record/depth flags. Raising only `decoded_records` does not raise the +independent row or segment budgets. Failed local decoding returns no partial +point evidence; completed browser map observations retain their existing +partial/unknown semantics. Wait for retained cleanup to settle before retrying. + ## Verification Browser bundle collection uses absolute section offsets for nested inline maps. @@ -130,10 +257,10 @@ its `original_sources` declaration, including repeated source URLs across indexe sections. Imports are deduplicated within each declaration. Older observations may omit the index; their edge declaration identity remains unknown. -Browser collection accepts up to 64 MiB of map responses per inspection and +Browser collection defaults to 64 MiB of map responses per inspection and preflights indexed sections, sources, names, generated rows and mapping segments against a 262144-record decode budget before upstream flattening. Indexed maps -are limited to 64 nested levels. These structural limits bound decoded object +default to 64 nested levels. These structural limits bound decoded object growth while preserving large embedded `sourcesContent` strings within the response-byte budget. It also bounds the expanded inline evidence to 32 MiB across the inspection, checking resolved source identities and mapping rows diff --git a/docs/kernel-modules.md b/docs/kernel-modules.md index 81a4275793..e6d9c66e5a 100644 --- a/docs/kernel-modules.md +++ b/docs/kernel-modules.md @@ -83,9 +83,22 @@ that `struct module`, symbol-table records, or certificate chains were decoded. The selected file is limited to 64 MiB by default (512 MiB maximum). At most 8192 `.modinfo` and `__versions` entries plus listed `__ksymtab*` sections are examined by default (65536 maximum); `coverage.examined_entries` counts all -three. Exceeding that budget fails with no partial success. Keys longer -than 256 bytes and values longer than 4096 bytes stay malformed per entry; -the field that fits is still copied. A `__ksymtab*` section name longer than +three. Exceeding that budget fails with no partial success. `.modinfo` keys +retain at most 256 bytes and values at most 4096 bytes by default. A larger +field becomes `null` and is named in the entry's `omitted_fields`; the field +that fits is still copied. Structural `status` and `reason` remain independent: +a terminated, nonempty `key=value` entry stays `observed` when its text is +omitted, while overall coverage becomes `partial`. Source range and digest +remain available. + +Raise `max_modinfo_key_bytes` or `max_modinfo_value_bytes` on the next MCP call +to retain the omitted text. The returned limitation names the required byte +budget. CLI flags are `--max-modinfo-key-bytes` and `--max-modinfo-value-bytes`. +Each accepts an integer from 1 through 536870912 bytes, matching the maximum +input size. No restart is required; file, entry and raw-hex budgets do not +change. These are per-call controls, not runtime settings. + +A `__ksymtab*` section name longer than 256 bytes keeps its range with `name: null` and `ksymtab_name_exceeds_limit`. Inline raw hex stops at 4096 bytes and the SHA-256 remains. diff --git a/docs/node-sea.md b/docs/node-sea.md index 85483d8fff..89a2ee4022 100644 --- a/docs/node-sea.md +++ b/docs/node-sea.md @@ -72,16 +72,60 @@ Carrier byte order is independent of blob byte order. Malformed magic, truncation, unknown flags/extensions, invalid UTF-8, duplicate raw asset keys and conflicting blob declarations fail closed with named reasons. -Zero padding after a complete blob is allowed. The blob read ceiling is 256 MiB; -PE carriers also use the shared 512 MiB input, 65,536 directory-entry and 8 MiB -resource-metadata ceilings. The decoder admits at most 100,000 assets whose -names total at most 8 MiB, and 4,096 exec arguments totaling at most 1 MiB of -text. These are fixed resource bounds, not Node format limits, checked before -decoding names or creating records; a hit fails as a limit with no runtime -override. Asset and argument decoding, padding scans, and inventory construction +Zero padding after a complete blob is allowed. Producer admission defaults to +a 256 MiB blob, 100,000 assets whose names total at most 8 MiB, and 4,096 exec +arguments totaling at most 1 MiB of text. These are resource policies, not Node +format limits. Select `artifact_limits.node_sea` on inspection or extraction to +change them before decoding names or creating records. Asset and argument +decoding, padding scans, and inventory construction yield in bounded batches and honor caller cancellation. Asset count and name -budgets bound retained metadata independently of stored member body sizes. The PE -name probe reads headers from the first 64 KiB of the carrier; a PE whose +budgets bound retained metadata independently of stored member body sizes. + +| `artifact_limits.node_sea` field | Default | Unit and admission | +| -------------------------------- | --------: | ------------------------------------------------------ | +| `blob_bytes` | 268435456 | Bytes in one blob, checked before reading and decoding | +| `asset_count` | 100000 | Asset records | +| `asset_name_bytes` | 8388608 | Aggregate raw UTF-8 asset-name bytes | +| `argument_count` | 4096 | Exec-argument records | +| `argument_bytes` | 1048576 | Aggregate raw UTF-8 exec-argument bytes | +| `recognition_header_count` | 4096 | ELF header rows admitted for recognition | +| `recognition_load_command_bytes` | 8388608 | Mach-O load-command bytes admitted for recognition | + +Exhausted recognition remains unknown, not examined absence. Inventory preserves +native-root facts or another positively recognized pack and records the exact +recognition limit inline. If extraction cannot establish a supported pack, it +refuses with that admission cause. Confirmed SEA blob admission still returns a +typed resource failure. ELF note-owner matching reads only the prefix needed to distinguish +`NODE_SEA_BLOB`; an unrelated long owner does not consume retained string space +or conceal a later SEA note. This does not change malformed framing checks. + +For example, retain the same active carrier and retry an argument-count refusal: + +```json +{ "artifact_limits": { "node_sea": { "argument_count": 8192 } } } +``` + +The CLI spelling is: + +```sh +rea inspect-artifact ./application --artifact-limits '{"node_sea":{"argument_count":8192}}' --json +``` + +Failures report `reader_limit`, including the exact parameter, effective value, +unit, supported range and required value when known. Changes apply on the next +call without restarting REA. Delivery summary, paging and extraction selection +cannot bypass whole-manifest admission. Omitted selections stay omitted in +Evidence parameters; selecting `artifact_limits` records the accepted selection +and `effective_limits`. Those are conditional policies for encountered readers, +not a claim that every listed format was examined. The independent aggregate +member-work override is `artifact_limits.expanded_bytes`; omission keeps the +encoded-size-derived work allowance. Runtime Buffer/string representation limits +and host memory remain independent of larger selected budgets. + +PE carriers additionally use the shared 512 MiB input, 65,536 directory-entry +and 8 MiB resource-metadata ceilings; the Node SEA selections do not raise those +shared PE-parser boundaries. The PE name probe reads headers from the first +64 KiB of the carrier; a PE whose headers extend beyond that is left to native classification. ELF note tables are read for at most one note per 12 carrier bytes in total, the most that disjoint tables can hold, so program headers that repeat the same notes do not diff --git a/docs/providers/ghidra-provider.md b/docs/providers/ghidra-provider.md index 15c43e64b8..f259fd84c2 100644 --- a/docs/providers/ghidra-provider.md +++ b/docs/providers/ghidra-provider.md @@ -96,6 +96,40 @@ and trace matches likewise retain Ghidra's external and thunk classification. The Ghidra analysis profile identifies this search projection, so older snapshots with unqualified search rows cannot replay under the new profile. +## Decompiler and value-flow budgets + +Use `inspect_analysis_settings` or a result's named recovery to inspect +`ghidra.value_flow_max_operations` (default 3000), +`ghidra.value_flow_max_inputs_per_operation` (default 64), and +`ghidra.value_flow_max_edges` (default 12000). Each accepts a positive Java +integer count. `update_analysis_settings` applies changes to the next operation +on the same Program without restarting REA. Launch seeds use the corresponding +`REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS`, +`REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION`, and +`REA_GHIDRA_VALUE_FLOW_MAX_EDGES` environment settings. +For a one-shot CLI retry, set the named seed in the environment that launches +the next `rea` invocation; the completed invocation's Program is no longer +active. Runtime MCP changes preserve the current Program. A changed launch seed +only affects a running MCP server after that server restarts. + +The returned `native_value_flow.limits` and `limits_reached` distinguish accepted +budgets from actual exhaustion. Truncation keeps omitted counts and names the +setting to adjust. The edge budget applies independently to def-use and +parameter-use collections. New function dossiers are reobserved rather than +replayed from snapshots; completed Evidence remains available for historical +inspection. Other observation caches keep their existing policy. + +`ghidra.decompile_max_payload_mib` controls the native decompiler's response +payload, separately from p-code projection and decompile time. Its launch seed +is `REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB`; explicit values are 1..2047 MiB to +avoid overflow in Ghidra's signed integer byte conversion. Omission or runtime +`null` preserves the installed provider's default. Changes apply before the +next native decompile, and native option changes clear its old cached function +state. A failed decompile preserves the original diagnostic and reports the +effective payload allowance; payload recovery applies only when that diagnostic +identifies payload exhaustion. Raising these controls does not raise the MCP +client's deadline or receive buffer. + ## PE debug-symbol coverage The post-analysis handshake reports Ghidra's PDB Program Information attributes: diff --git a/docs/reference/mcp-contracts.md b/docs/reference/mcp-contracts.md index 398f633a55..7d5e295fae 100644 --- a/docs/reference/mcp-contracts.md +++ b/docs/reference/mcp-contracts.md @@ -148,7 +148,10 @@ returned: `pseudocode`, `native_api` and `native_value_flow` are unavailable with the decompiler's failure as the reason, while the listing-backed facets are collected as usual. These failed decompiler observations and workflows derived from them retain Evidence but skip snapshot reuse, so a later request -can observe backend recovery; healthy dossiers keep their existing cache policy. +can observe backend recovery. Ghidra function dossiers also skip snapshot reuse +so each new operation honors its accepted value-flow budgets; completed source +Evidence remains accessible through retained views and bundle export. Other +healthy provider observations keep their existing cache policy. `procedure_pseudo_code` still fails for that function. Hopper's unavailable reference-kind metadata and reported unresolved calls still prevent equality conclusions for those dimensions. Analysis profiles version these declarations so older snapshots are not reused as current results. diff --git a/docs/reverse-engineering-tools.md b/docs/reverse-engineering-tools.md index 3cacd4602b..ba7b9454cb 100644 --- a/docs/reverse-engineering-tools.md +++ b/docs/reverse-engineering-tools.md @@ -43,14 +43,22 @@ rea inspect-with-objdump ./program dwarf rea inspect-with-objdump ./program dwarf --follow-debug-links ``` -Captured stdout and stderr are retained up to a 16 MiB combined limit; `output_truncated` marks output beyond that limit. That cap has no runtime override. +Captured stdout and stderr share `max_output_bytes` (CLI: `--max-output-bytes`), default 16777216 bytes (16 MiB). Select an integer from 1 to 44651859 bytes. The maximum leaves room for worst-case JSON escaping of both Evidence output mirrors under Node's single-string ceiling. The selected budget applies on the next call without a restart; `output_budget_bytes` reports it and `output_budget_source` distinguishes a default from an explicit request. On exhaustion REA retains a prefix, requests provider termination, and marks `output_truncated` with unknown completion. Narrow or page the native command, or raise the selected budget. Verify command effects and outstanding cleanup before deciding whether repeating it is safe. Client deadlines and receive buffers remain separate. Concurrent one-shot captures reserve their selected budgets within a 128 MiB aggregate admission and eight-operation capacity; wait for active work or select a smaller output budget when that capacity is occupied. -`inspect_with_objdump`, `rea list-objdump-architectures`, and one-shot `execute_rizin_command` accept `command_timeout_ms` (CLI: `--command-timeout-ms`). Omitted means 120000. The integer range is 1 to 2147483647. Target preflight shares one budget across stat and digest. The one-shot command receives that budget again. It applies on the next call; no restart is required. It does not raise the 16 MiB cap, a Ghidra or Hopper startup deadline, or a debugger session deadline. +`inspect_with_objdump`, `rea list-objdump-architectures`, and one-shot `execute_rizin_command` accept `command_timeout_ms` (CLI: `--command-timeout-ms`). Omitted means 120000. The integer range is 1 to 2147483647. Target preflight shares one budget across stat and digest. The one-shot command receives that budget again. It applies on the next call; no restart is required. It does not raise the separately selected `max_output_bytes`, a Ghidra or Hopper startup deadline, or a debugger session deadline. ## GDB Persistent GDB sessions, including the MCP workflow and one-shot `rea debug-with-gdb` CLI, are available only on Linux. REA verifies a live inferior's output descriptors through `/proc` before detaching it; it does not claim equivalent descriptor verification on macOS. Windows is also unsupported because owned Job Object cleanup can terminate debugger inferiors. On supported Linux hosts, `start_gdb_session`, `gdb_console`, `gdb_session_status`, and `close_gdb_session` manage a persistent GDB/MI session in MCP. The one-shot CLI requires a caller-selected durable TTY: `rea debug-with-gdb --inferior-tty /dev/pts/7 'file ./program'`. The TTY must remain available after GDB exits so a detached inferior can continue writing. MCP starts GDB with `--nx --quiet --interpreter=mi3`, retries initialization with MI2 if MI3 initialization returns an error and cleanup succeeds, waits for its prompt, and disables auto-loading. Pass optional `inferior_tty` to select the terminal that future inferior runs use; REA configures it before the session becomes ready and checks the exact value with GDB/MI. `gdb_session_status` reports the selected path, whether REA still considers the configuration verified, and whether invalid UTF-8 has been observed in MI bytes (`mi_invalid_utf8`) or stderr (`stderr_invalid_utf8`). Commands that can change the inferior TTY or redirect a run revoke that verification permanently for the session, so close retains GDB ownership if a live inferior remains. The complete built-in inspections `show inferior-tty` and `help inferior-tty` preserve verification; compound commands still use the conservative mutation checks. Closing a session with one live inferior and a verified TTY automatically detaches the target, which can resume execution; MCP clients receive destructive-operation annotations for this behavior. The console remains unrestricted and can run shell commands, change target state, access files, and use network in the REA process user's security context. Raw MI records and console/target/log streams are returned with Evidence; command output is retained up to the configured budget (16 MiB by default) and explicitly marked when truncated. A single console, target or log record over the 16 MiB MI line limit is dropped and the command is marked truncated, while an oversized result or async record still stops the session. GDB startup waits 10000 ms with no runtime override. Each MI command defaults to 120000 ms; its deadline, command-output budget, session capacity and pending-command queue use the [shared debugger launch settings](#debugger-launch-settings). `command_timeout_ms` does not change a persistent debugger session. +A truncated GDB command capture includes `output_limits` with the exact causes +and effective maxima: `command_bytes`, `command_records`, or `mi_line_bytes`. +Only byte-budget exhaustion recommends raising +`REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES`. The 16384-record retention ceiling and +16 MiB single-line protocol ceiling stay fixed; select a command with fewer or +smaller records when either is reached. These cause facts survive timeout, +cancellation and process-exit captures as well as successful commands. + With multiple live inferiors, or a live inferior without a verified TTY, manual close requests an explicit disposition and keeps the session ready. Use `gdb_console` to select and explicitly detach or terminate each remaining @@ -125,7 +133,7 @@ rea debug-with-rizin --inferior-tty /dev/pts/7 ./program dr rea execute-rizin-command ./program iI ``` -Captured stdout and stderr are retained up to a 16 MiB combined limit; `output_truncated` marks output beyond that limit. One-shot commands use the `command_timeout_ms` budget described above. That 16 MiB cap has no runtime override. +Captured stdout and stderr use the same `max_output_bytes` (CLI: `--max-output-bytes`) admission and recovery described above, default 16 MiB. One-shot commands independently use `command_timeout_ms`. Raising output admission cannot recover an earlier capture; command effects and cleanup must be checked before another invocation. ### Decoder and map inputs diff --git a/docs/rust-binaries.md b/docs/rust-binaries.md index 967e3b8c5c..53f34882fb 100644 --- a/docs/rust-binaries.md +++ b/docs/rust-binaries.md @@ -173,10 +173,26 @@ present. `architecture` on any other image is rejected rather than ignored. Package and dependency counts are checked against the minimum required JSON representation before graph allocation or schema cloning. Oversized graphs -report the output byte budget; the compiler-path capacity reports -`maximum_rustc_hashes` in hashes, independently of byte limits. +report the output byte budget. + +The inferred compiler-path facet keeps at most `max_rustc_hashes` distinct +hashes by default (64). Reaching that selection leaves the dependency report +available and adds `rustc_commit.coverage` with partial status and the effective +limit. Increase `max_rustc_hashes` (CLI `--max-rustc-hashes`) on the next call +to retain a larger prefix; no restart is required. The complete-report byte +allowance still applies. Optional compiler paths are shortened before they can +displace independently observed dependency facts. Reader architecture names remain verbatim in the report. Evidence subjects use the recognized canonical family: `386` maps to `x86`, `amd64` to `x86_64`, and `ppc` to `powerpc`. Unknown families such as `ppc64` remain unknown rather than being inferred from a related spelling. + +Input and complete-report admission are caller choices through `producer_limits`: +`input_bytes` defaults to 256 MiB and `output_bytes` to 8 MiB. Both accept positive +safe integers and apply to the next call without restarting REA. On CLI, use +`--producer-limits '{"input_bytes":536870912,"output_bytes":16777216}'`. +Graph expansion and compiler-path retention consume the same selected report +allowance. The independent compressed dependency limit (1 MiB), inflated JSON +limits (8 MiB and 64 times the consumed stream), parser nesting and transport +budgets remain enforced; increasing report admission does not raise those. diff --git a/docs/specs/2356-wasm-interface-comparison/spec.md b/docs/specs/2356-wasm-interface-comparison/spec.md index b4dfb7778d..419ae96c88 100644 --- a/docs/specs/2356-wasm-interface-comparison/spec.md +++ b/docs/specs/2356-wasm-interface-comparison/spec.md @@ -44,7 +44,9 @@ are outside admitted signature semantics and keep unknown mappings. ## Resource admission Typed producer rows, names and declaration tokenization are admitted before -allocation against the existing 32 MiB WABT output representation capacity. +allocation against the selected `max_interface_bytes` allowance (default 32 MiB). +Inspection and comparison select this allowance independently; comparison cannot +restore producer facts omitted under a smaller inspection allowance. Comparison uses linear grouping and cooperative transport yields; it never builds Cartesian duplicate pairings. CLI JSON input retains the shared file admission gate. Token usage and peak process-family memory remain unmeasured. diff --git a/docs/specs/analysis/recoverable-budgets/plan.md b/docs/specs/analysis/recoverable-budgets/plan.md new file mode 100644 index 0000000000..3f4af38a09 --- /dev/null +++ b/docs/specs/analysis/recoverable-budgets/plan.md @@ -0,0 +1,27 @@ +# Implementation plan + +Use the existing named domain contracts and runtime settings owner. Prefer +per-call selection or cohesive resource policies where they express analyst +intent. Extend the runtime registry for ASAR listing admission, which already +flows through the shared archive admission owner. + +Implement independent owner slices: kernel metadata; GDB/Ghidra; desktop and +generated Python; Apktool; JavaScript precision; source maps; historical +captures; artifacts/managed/Rust; live capture; and producer limits. Coordinate +shared schema/configuration files centrally and review cross-slice changes +before broad checks. + +Remove superseded literals, representations and misleading recovery rather +than adding downstream prose repairs. Use the repository's Result/error algebra, +existing process supervision, and existing CLI/MCP registration paths. + +Validate focused boundaries first, then build, type/lint checks and generated +documentation consistency. Document unavailable native/provider lanes. No +performance claim or benchmark is required for correctness and policy ownership. + +For default calibration, compare representative completed tasks, including +recovery calls and complete replies. Record artifact identity, an independent +coverage oracle, estimated tokens, elapsed/CPU time and sampled process-family +memory. Separate artifact admission from delivery size; do not raise defaults +merely to make a large response fit. Keep unmeasured defaults explicitly +provisional rather than presenting inherited numbers as optimal. diff --git a/docs/specs/analysis/recoverable-budgets/spec.md b/docs/specs/analysis/recoverable-budgets/spec.md new file mode 100644 index 0000000000..eff7431370 --- /dev/null +++ b/docs/specs/analysis/recoverable-budgets/spec.md @@ -0,0 +1,69 @@ +# Recoverable analysis budgets + +## Problem + +Several analysis workflows discard selected facts at fixed retention boundaries. +Existing runtime settings control only some resources. Increasing a deadline, +heap, or header allowance cannot repair an independent count or representation +limit. Desktop readers also duplicate policy in generated Python. + +Related discussions: [#6176](https://github.com/morluto/rea/issues/6176), +[#8434](https://github.com/morluto/rea/issues/8434), +[#5808](https://github.com/morluto/rea/issues/5808), and +[#8978](https://github.com/morluto/rea/issues/8978). + +## Requirements + +1. Preserve defaults unless representative completed-task measurements justify + a calibration; preserve caller selections, Evidence identity, omission, + unknown coverage, and cleanup authority. Changes must not grant new target + execution, injection, mutation, installation, or network authority. +2. Keep source validity independent of retention completeness. In particular, + an overlong but terminated kernel modinfo value is not malformed merely + because REA omitted its text. +3. Preserve exact exhaustion causes and give a next action addressing that + resource. GDB byte, record-count, and single-line limits must remain distinct. +4. Offer caller selection, continuation, or a bounded policy override for the + reviewed desktop projections, Apktool string tables, JavaScript semantic + precision, source maps, historical captures, provider capture retention, + artifact/managed representations, and producer admission/output limits. +5. Interpret each policy at its owner. Parent and child process checks consume + the same accepted values; generated Python consumes the desktop policy rather + than maintaining independent numeric defaults. Reject invalid values rather + than clamping or silently falling back. +6. Retain justified format, runtime, protocol, and ownership constraints. + Distinguish them from adjustable resource policy and external client budgets. + A larger delivery page cannot recover facts already discarded by a producer. +7. A runtime default update affects new accepted operations only. Explicit + per-call choices win. Report effective value, units, applicable control and + retry/restart scope without claiming active-worker telemetry. +8. Artifact recognition that cannot examine the selected carrier within its + accepted resource policy remains limited/unknown, never examined absence. + Preserve the exhausted cause through classification and retained probe facts. + Add cohesive per-call Bun, .NET bundle, Node SEA and Godot admission controls + at their producing owners, including preallocation and expansion checks. +9. Desktop task checks must admit the actual shipped seed representation under + the same selected view policy as the generated reader. Acquisition/export + budgets remain separate from runtime view admission. +10. Local source-location tracing offers opt-in compact retained delivery and + bounded access to mapping alternatives and embedded text. Preserve complete + Evidence, source identity, selected positions, unknowns and every equal-point + mapping; compact output must not repeat the raw map or entire source text. +11. Desktop SQLite projections share a configurable per-table encoded-view + allowance with generated readers and check it before retaining rows. Keep + the independent native SQLite value/record and worker-reply envelopes + explicit; a projected-view override does not raise those constraints. + +## Acceptance + +Exercise the exhausted boundary and its advertised repair through the strongest +practical public or production workflow. Use small lowered-budget fixtures when +they prove the same boundary without large allocations. Preserve independent +malformed-input, cancellation, identity, cleanup and fixed-protocol controls. +Record real-provider coverage separately from simulated process/protocol tests. + +## Non-goals + +Unbounded analysis, changing all numeric constants, treating format bit widths as +preferences, raising another application's client buffer/deadline, recovering +lost live observations retroactively, or adding new execution authority. diff --git a/docs/specs/analysis/recoverable-budgets/tasks.md b/docs/specs/analysis/recoverable-budgets/tasks.md new file mode 100644 index 0000000000..b8f5fbbe1d --- /dev/null +++ b/docs/specs/analysis/recoverable-budgets/tasks.md @@ -0,0 +1,137 @@ +# Tasks + +- [x] Separate source validity from retained modinfo fields. +- [x] Preserve GDB exhaustion causes and improve Ghidra value-flow recovery. +- [x] Consolidate desktop policy with emitted Python and expose caller controls. +- [x] Make Apktool string omissions reachable through caller selection. +- [x] Thread JavaScript semantic precision policy through actual evaluators. +- [x] Coordinate source-map parent, codec and fetch budgets. +- [x] Coordinate historical-capture parent and decoder budgets. +- [x] Expose independent artifact/managed admission and preserve partial Rust facts. +- [x] Expose capture retention policy without changing target authority. +- [x] Expose producer resource/output controls with exact recovery. +- [x] Integrate ASAR runtime setting, launch validation and setup forwarding. +- [x] Complete focused and integrated checks, update product docs and report gaps. +- [x] Repair reviewed managed CLI decimal parsing and selected-budget provenance. +- [x] Preserve Go/Rust owner-supplied exhaustion facts through summary views. +- [x] Preserve selected desktop policy separately from effective policy and bound + the emitted reader's initial probe during file growth. +- [x] Compare representative completed Flutter and source-map tasks before + calibrating defaults; retain reproducible inputs, oracles and receipts. +- [x] Preserve limited Bun recognition through inventory and probe facts. +- [x] Add per-call artifact admission policies and exact same-artifact recovery. +- [x] Align shipped desktop seed task admission with generated reader policy. +- [x] Add retained source-location summaries, paged matches and bounded text. +- [x] Verify compact point tracing on the pinned public maps and refresh native + provider recovery coverage; complete coordinated integration checks. + +## Validation + +Executed with the pinned Node 24.22.1 / npm 11.16.0 toolchain: + +- Runtime build and packaged skill generation through the focused-test + prerequisite. +- `npm run check:fast`: typecheck, lint and module boundaries pass. +- `npm run format:check`: passes. +- `npm run test:python`: bridge and analysis-efficiency suites pass. +- `npm run verify:test-discovery`: all 3,218 test files discovered exactly once. +- `npm run knip:production`: passes; advisory candidates were inspected rather + than removed without runtime/verifier ownership evidence. +- `npm run docs:generate` and `npm run docs:check`: pass, including the managed + conformance generator and README consistency checks. Removed obsolete ignored + managed-verifier argument bags instead of adding permissive budget fallbacks. + +Focused tests ran in batches across CLI, MCP, worker, parser, lifecycle and +provider boundaries. The primary 28-file run passed 726 tests, with six failures +and one host skip. Those failures were repaired and every affected file passed +its targeted rerun. The supplementary producer run passed 512 tests with five +failures and 120 platform skips; every affected file then passed its rerun. +The final capture-only run passed all 45 tests. This is a record of separate +executed runs, not an aggregate unique-test count or a claim that `check:pr` ran. + +Recovery checks include the same retained target after raised admission, +explicit selection reaching an omitted item, lower-budget cache re-admission, +worker-policy changes without stale results, generated Python parity, exact +exhaustion causes, and malformed/cancellation/cleanup controls. Integration +review also found and fixed XML/JSON view and historical-capture nesting budgets +that initially failed to reserve complete Evidence wrapper depth. + +The subsequent review fixes passed all 50 tests across six compiled acceptance, +CLI, MCP and emitted-Python boundary files. Typecheck and lint also passed after +those fixes. Default calibration records its separate measurements and +limitations in the [Flutter](../../../verification/flutter-budget-calibration.md) +and [source-map](../../../verification/source-map-budget-calibration.md) reports; +those measurements are not native-provider verification. +The calibrated source-map default and explicit smaller-budget recovery passed +all six compiled source-location CLI tests. The final formatting, type/lint, +generated-documentation and README consistency checks passed. The observer's +completed, timeout, surviving-child and signal-refusal controls retained +receipts and confirmed empty process groups. + +The final follow-up adds per-call artifact admission, limited recognition, +shipped-seed policy agreement, bounded SQLite views and retained source-location +delivery. The 12-file compiled CLI/MCP run passed 56 tests and exposed four stale +fixed-limit expectations; those owner expectations were updated to structured +resource facts and all five rerun files passed 60 tests, including the shared +analysis-view integration suite. The SEA MCP matrix separately passed both full +and compact schema profiles with same-session lower/raise/lower admission. +The final SQLite worker/emitted-reader checks passed all four scenarios. These +are separate executed runs, not an aggregate unique-test count. + +The [source-location delivery qualification](../../../verification/source-location-delivery.md) +retains 18 successful public MCP measurements with matching parent IDs, exact +mapping/snippet oracles and process-tree samples. The +[JavaScript precision calibration](../../../verification/javascript-precision-calibration.md) +retains 12 real CLI runs supporting unchanged numeric defaults for that narrow +corpus. Native capture qualification and the live same-Program Ghidra receipt +are linked from the [runtime verification report](../../../verification/runtime-analysis-settings.md). +The Python gate passed all 102 tests; architecture guards passed across 2,571 +source files. Formatting, type/lint, generated docs, README consistency and +production reachability checks passed. Reachability advisory candidates were +reviewed without deleting runtime/verifier-owned exports. The complete +`check:pr` gate was not run. + +## Integration onto current main for the PR + +Integrated onto main `8dfc47b107afc0cf4b53322a10b94c666fe9e18b`, preserving its +new desktop secret isolation, custom task readers, gzip support, Deno recognition, +signed bundle validation and chunked I/O. Incomplete Bun/SEA recognition now +keeps known native or other confirmed-pack observations with an explicit unknown +cause; it never caches unexamined data as absence. Main's malformed-directory +validation continues to take precedence over adjustable Godot retention. + +The fresh 11-file compiled integration run passed 118 tests with one host skip, +covering artifact admission, retained source locations, desktop seed/SQLite/CLI, +Go/Rust snapshots, runtime settings and setup forwarding. Separate owner checks +passed for new main's desktop secret/custom-reader/gzip and JavaScript +enumerable-copy/require-drain behavior. Source-map fetch-budget omission, empty +selection and partial selection now also remain distinct in browser Evidence. + +On this integrated branch, Python passed 102 tests; discovery found all 3,343 +test files exactly once; architecture guards passed across 2,696 source files. +The runtime/skill build, type/lint and generated-documentation checks passed. +The retained native Ghidra lane was rerun against the integrated code and again +confirmed live budget recovery, unchanged target bytes, Program continuity and +cleanup; see its [integration receipt](https://github.com/morluto/rea/blob/main/docs/verification/runtime-analysis-settings-ghidra-integration-2026-11-09.json). +Historical calibration and earlier receipts retain their original runtime pins; +they are not performance measurements of the integrated head. `check:pr` was +not run. + +## Verification limits + +The follow-up verified all packaged Java sources against installed Ghidra 12.1.4 +and exercised live operation/input/edge, native payload and deadline recovery on +the same Program with unchanged target bytes. The target was not executed. +The [dated native receipt](../../../verification/runtime-analysis-settings.md) +records exact versions, observations and verified cleanup; this does not qualify +Windows or every public transport/provider combination. Real local Frida and +LLDB producer-boundary budget checks also passed with explicitly controlled test +processes and confirmed cleanup. Full AgentEnv execution was not exercised; +desktop checks ran real SQLite workers and emitted reader helpers. Linux-only +native lanes and real Apktool, WABT, pwntools, pwndbg, sigrok and TShark remain +unverified. Source-owned codec/worker and simulated checks are labeled separately. +No native-provider performance improvement is claimed. Source-owned workflow +calibration measures a bounded public-artifact corpus, including complete +replies and recovery calls; it does not establish universally optimal defaults. +Ghidra function dossiers now reobserve accepted budgets while retaining the +native decompiler cache; the native latency cost of that change is unmeasured. diff --git a/docs/specs/evm/producer-limits/plan.md b/docs/specs/evm/producer-limits/plan.md new file mode 100644 index 0000000000..a28a731602 --- /dev/null +++ b/docs/specs/evm/producer-limits/plan.md @@ -0,0 +1,6 @@ +# Plan + +1. Give EVM defaults and per-call policy one domain owner using the shared producer schema. +2. Resolve the accepted policy before resource selection and carry byte limits to the worker request; retain tighter inherited soft and hard bounds. +3. Preserve exact budgets through read/reply/diagnostic/resource recovery, CLI and Evidence. Replace the failure owner's positional argument list with a typed context. +4. Verify policy recovery and existing malformed/cancellation/cleanup behavior with focused checks. diff --git a/docs/specs/evm/producer-limits/spec.md b/docs/specs/evm/producer-limits/spec.md new file mode 100644 index 0000000000..c7b4f4a954 --- /dev/null +++ b/docs/specs/evm/producer-limits/spec.md @@ -0,0 +1,7 @@ +# Per-call EVM producer budgets + +An analyst can raise or lower carrier, reply, diagnostic, address-space, file-output and CPU budgets without changing the selected artifact or restarting REA. Defaults stay 4 MiB carrier, 16 MiB reply/file, 1 MiB diagnostics, 3 GiB address space and 30 CPU seconds. `timeout_ms` remains the separate worker deadline. + +Use the shared typed `producer_limits` contract, advertising only fields enforced by EVM. If `file_bytes` is omitted, it follows the selected `output_bytes`; an explicit file budget stays independent. Linux soft/hard limits retain the tighter inherited value, and failures distinguish configured from unknown effective limits and signal causes. Resource guidance identifies the exhausted field, its selected value and whether inherited restrictions can prevent repair. The 256 MiB worker heap stays explicit and independent. + +Preserve carrier/decoded identities, metadata, target-free offline authority, cancellation, final cleanup and partial facts. Verify same-target recovery with a small selected allowance and narrower inherited-limit controls; report unavailable real Linux coverage. diff --git a/docs/specs/evm/producer-limits/tasks.md b/docs/specs/evm/producer-limits/tasks.md new file mode 100644 index 0000000000..90b500a8c8 --- /dev/null +++ b/docs/specs/evm/producer-limits/tasks.md @@ -0,0 +1,6 @@ +# Tasks + +- [x] Declare defaults and exact per-call policy. +- [x] Wire parent, worker and inherited resource enforcement. +- [x] Update exact recovery, CLI and Evidence. +- [x] Run focused regressions and report platform gaps. diff --git a/docs/specs/native/go-string-literals/plan.md b/docs/specs/native/go-string-literals/plan.md index 3055dd1270..4ee6646e2a 100644 --- a/docs/specs/native/go-string-literals/plan.md +++ b/docs/specs/native/go-string-literals/plan.md @@ -4,8 +4,8 @@ Extend `goBinarySchema` in `src/domain/go/goBinary.ts` with optional `string_literals` (`evidence_kind: "inferred"`, `items`, `discovered_count`, -`aggregate_bytes`, `max_items`, `truncated`), plus `include_string_literals` and -`max_string_literals` on `src/contracts/go/goToolContracts.ts` and +`aggregate_bytes`, `max_items`, `truncated`, optional `limits_reached`), plus +`include_string_literals` and the Go-owned per-call budget controls on `src/contracts/go/goToolContracts.ts` and `src/cli/goCommands.ts`. Add the `string_literals` view projection to `src/domain/analysisView/goBinaryView.ts` so callers can page recovered literals from retained Evidence without repeating file parsing. @@ -18,6 +18,9 @@ pointer alignment phase, merge overlaps, and sort by ascending file offset so `reference_count` counts each physical header once and `first_header_offset` is minimal. Deduplicate target slices in memory with bounded candidate and rejection caches before sorting deterministically by `(location.offset, location.bytes)`. +Use exact offset/length keys, so configurable string lengths cannot collide in +an arithmetic key encoding. Resolve budget defaults once in the image reader +and pass that policy to string recovery. ## Incremental work and validation @@ -37,4 +40,8 @@ caches before sorting deterministically by `(location.offset, location.bytes)`. Default `inspect_go_binary` calls omit `string_literals` and preserve historical Evidence compatibility. No external tool or target execution is introduced; only -bounded offline byte slicing over the already read `64 MiB` image buffer runs. +bounded offline byte slicing over the admitted stable image buffer runs. The +input default remains `256 MiB`; callers can select byte and work budgets within +the advertised runtime representability. Count normalized report bytes before +allocating its JSON serialization. Every exhausted limit carries its own next +request repair, independently of transport delivery. diff --git a/docs/specs/native/go-string-literals/spec.md b/docs/specs/native/go-string-literals/spec.md index 0cb5339c50..d4e6b79018 100644 --- a/docs/specs/native/go-string-literals/spec.md +++ b/docs/specs/native/go-string-literals/spec.md @@ -17,15 +17,30 @@ images so static string slices are recovered at their exact boundaries. `O(log R)` readable segment checks on Mach-O and sectionless/unnamed-ELF load-region fallbacks) in deduplicated ascending file-offset order. - Admit candidate `[ptr, ptr + len)` slices only inside single read-only - non-executable mapped spans with `1 <= len <= 4096`, non-overlapping with the + non-executable mapped spans with `1 <= len <= max_string_literal_bytes` + (default `4096`), non-overlapping with the header itself, and valid printable UTF-8 (allowing `\t`, `\n`, `\r`, `U+200C` and `U+200D` while rejecting control, surrogate, noncharacter, private-use and invisible/bidi `\p{Cf}` code points). -- Bound header-scan work (`64 MiB`), tracked candidates (`100,000`), negative - slice cache (`4,096`), returned items (`max_string_literals`, `1..10,000`, - default `2,000`), and aggregate returned UTF-8 bytes (`512 KiB`), reporting - `evidence_kind: "inferred"`, `discovered_count`, `aggregate_bytes` and - `truncated` honestly. +- Keep the existing defaults for header-scan work (`64 MiB`), tracked candidates + (`100,000`), returned items (`2,000`), and aggregate returned UTF-8 bytes + (`512 KiB`). Expose each independent boundary as the positive-safe-integer + per-call controls `max_string_header_scan_bytes`, `max_string_candidates`, + `max_string_literals`, `max_string_literal_bytes` and `max_string_bytes`. + String-byte maxima follow the runtime's string representability; there is no + separate fixed 10,000-item ceiling. The bounded negative-slice cache (`4,096`) + affects reuse only and must not discard valid later candidates. +- Report `evidence_kind: "inferred"`, `discovered_count`, `aggregate_bytes`, + `truncated` and the exact exhausted `limits_reached` controls honestly. + Increasing a named control and inspecting the same complete artifact can + recover omitted facts; output paging cannot recover facts never retained. +- Deduplicate candidates by their complete offset and length independently of + the selected literal-length allowance; raising that allowance cannot merge + distinct slices or inflate their reference counts. +- Share one Go analysis budget owner with build-info input, representation and + search controls. Omitted selections keep defaults and remain omitted from + request Evidence. Representation maxima account for base64 expansion; no + setting changes the separate shared native structural decoder or MCP budget. - Record limitation notes when Mach-O `LC_DYLD_CHAINED_FIXUPS` encodes rebases in pointer slots or when truncation occurs, and expose a focused `string_literals` projection in `inspect_analysis_view`. @@ -35,7 +50,9 @@ images so static string slices are recovered at their exact boundaries. Unit, boundary and CLI acceptance tests cover ELF/PE/Mach-O (32/64-bit, little and big endian), sectionless and unreadable-`.shstrtab` ELF images, overlapping spans, `U+200C`/`U+200D` versus disallowed `\p{Cf}` controls, negative-cache and -output caps, and 25,000-region Mach-O indexing/cancellation. The real Go binary +output defaults, named budget recovery, long-literal candidate identity, and +25,000-region Mach-O indexing/cancellation. Service and provider tests cover +malformed numeric controls and same-artifact retries with larger budgets. The real Go binary verifier validates CLI and advertised-schema MCP parity across compiled targets. ## Out of scope diff --git a/docs/specs/native/go-string-literals/tasks.md b/docs/specs/native/go-string-literals/tasks.md index a5d9273985..41a2294ecc 100644 --- a/docs/specs/native/go-string-literals/tasks.md +++ b/docs/specs/native/go-string-literals/tasks.md @@ -6,3 +6,8 @@ - [x] Add unit, provider, MCP view and CLI acceptance tests for multi-format recovery, Unicode `\p{Cf}` / ZWNJ / ZWJ filtering, caps, negative-cache bounds and 25,000-section Mach-O scaling. - [x] Verify CLI and `artifactMcpCall` advertised-schema MCP parity across compiled targets in `scripts/verify-go-binary.mjs`. - [x] Update the Go binary guide, skill reference and generated catalogs; pass documentation route and static checks. + +- [x] Consolidate Go-owned budgets and expose per-call input/report/build-info/search and inferred-string controls in CLI/MCP without changing defaults. +- [x] Preserve source validity and exact exhausted controls, remove the fixed 10,000-item schema ceiling, and deduplicate variable-length candidates with complete offset/length identity. +- [x] Run focused source string/service tests and production service budget-recovery checks over temporary offline fixture files. +- [ ] Run the compiled CLI/MCP boundary lanes and real Go verifier for the new budget selections; source checks do not establish those workflows. diff --git a/docs/specs/web/historical-capture-budgets/plan.md b/docs/specs/web/historical-capture-budgets/plan.md new file mode 100644 index 0000000000..152f7216b1 --- /dev/null +++ b/docs/specs/web/historical-capture-budgets/plan.md @@ -0,0 +1,5 @@ +# Implementation plan + +1. Declare caller budgets in the domain input and derive one immutable provider policy with existing defaults. +2. Carry that policy in private child requests and scope all parser/read/reply consumers to it without global mutable state. +3. Update exact recovery, CLI controls and Evidence parameters; verify selected-budget recovery and default regressions. diff --git a/docs/specs/web/historical-capture-budgets/spec.md b/docs/specs/web/historical-capture-budgets/spec.md new file mode 100644 index 0000000000..fb89865775 --- /dev/null +++ b/docs/specs/web/historical-capture-budgets/spec.md @@ -0,0 +1,9 @@ +# Historical capture admission budgets + +Allow an analyst to inspect the same retained HAR, SAZ, qlog or native mitmproxy capture after explicitly increasing an exhausted input, reply or nesting budget. Keep existing defaults, producer identity, sensitive-value handling and complete-or-failure semantics. + +The public contract accepts positive safe-integer `max_input_bytes` and `max_reply_bytes`, and `max_nesting_depth` up to REA's JSON representation depth minus the four enclosing record/reply/Evidence levels (996). Selected values apply to the admitted operation and its child decoder, source and reply readers, archive expansion and protobuf interpretation. They do not raise the separately owned worker heap, CPU or client transport limits. + +Failures name the exhausted parameter and value and the next-call repair. Omitted values retain 32 MiB input, 96 MiB reply and 64 nesting levels. Evidence preserves explicit selections and omission. + +Acceptance: the same valid capture rejected by a deliberately small selected budget succeeds after changing only that budget. A malformed control remains malformed. The child and parent enforce the same selected facts, and parallel operations keep distinct policies. diff --git a/docs/specs/web/historical-capture-budgets/tasks.md b/docs/specs/web/historical-capture-budgets/tasks.md new file mode 100644 index 0000000000..cdd84131db --- /dev/null +++ b/docs/specs/web/historical-capture-budgets/tasks.md @@ -0,0 +1,6 @@ +# Tasks + +- [x] Add input contracts and canonical policy selection. +- [x] Connect parent, Node/Python decoder, archive and protobuf boundaries. +- [x] Expose CLI controls and preserve request provenance. +- [x] Verify same-input recovery, malformed controls and parser regressions. diff --git a/docs/specs/web/source-map-budget-recovery/plan.md b/docs/specs/web/source-map-budget-recovery/plan.md new file mode 100644 index 0000000000..bdbac1aa96 --- /dev/null +++ b/docs/specs/web/source-map-budget-recovery/plan.md @@ -0,0 +1,9 @@ +# Plan + +Use one domain policy and schema for defaults, selection and recovery. Publish +local and fetch projections that reject fields their path does not consume. +Thread selected values through existing acquisition/codec/process/format owners, +including independently validated child request budgets. CLI exposes named +numeric options and MCP derives its inputs from canonical source contracts. +Existing guides own product behavior. Verify finite budget recovery through +production boundaries and the public CLI; keep resource/lifecycle regressions. diff --git a/docs/specs/web/source-map-budget-recovery/spec.md b/docs/specs/web/source-map-budget-recovery/spec.md new file mode 100644 index 0000000000..a27a763fc8 --- /dev/null +++ b/docs/specs/web/source-map-budget-recovery/spec.md @@ -0,0 +1,35 @@ +# Per-call source-map budget recovery + +## Problem + +Local tracing and bundle source-map collection discard or refuse useful facts +at independent fixed byte, structural and codec-heap ceilings. A longer timeout +cannot admit the same map. Related: #6176, #8434, #8978. + +## Requirements + +- Preserve defaults unless representative calibration justifies a change; + preserve selected artifact identities, source declarations, + unknown coverage and cleanup ownership. +- Admit explicit bounded per-call budgets at the producing owners, and propagate + the same policy through acquisition, structural preflight, codec child and + complete request/reply checks. +- Reject inapplicable fields; preserve omission in Evidence parameters. +- Explain the exact exhausted control, effective value, range, application point + and recovery. Retain real representation and client constraints. +- Keep upstream decoding unchanged and preflight rows/segments before allocation. + +## Acceptance + +A valid local map fails at an explicitly lowered admission, names map_bytes, +and succeeds for the same point through the public CLI after increasing that +budget. The calibrated 8 MiB default admits a complete map above 4 MiB while +preserving an explicit smaller selection; see the +[calibration record](../../../verification/source-map-budget-calibration.md). +A finite fetched map similarly recovers after increasing decoded_records. +Malformed maps, cancellation and failed cleanup retain their existing semantics. + +## Non-goals + +No source fetching beyond existing origin authority, application execution, +implicit retries, unbounded allocation or change to client deadlines. diff --git a/docs/specs/web/source-map-budget-recovery/tasks.md b/docs/specs/web/source-map-budget-recovery/tasks.md new file mode 100644 index 0000000000..5259b503f1 --- /dev/null +++ b/docs/specs/web/source-map-budget-recovery/tasks.md @@ -0,0 +1,10 @@ +# Tasks + +- [x] Define shared source-map policy with existing defaults and typed selections. +- [x] Propagate local budgets through acquisition, service, codec and process owner. +- [x] Propagate fetched response, record and representation budgets. +- [x] Expose exact CLI/MCP contracts and actionable recovery. +- [x] Add finite structural/fetch and public large-map recovery regressions. +- [x] Update source-location and browser guides. +- [x] Run compiled CLI/codec recovery and coordinated type/lint/documentation gates. +- [x] Calibrate local map admission against shipped maps; retain other budgets. diff --git a/docs/verification/flutter-budget-calibration.md b/docs/verification/flutter-budget-calibration.md new file mode 100644 index 0000000000..71c93932bb --- /dev/null +++ b/docs/verification/flutter-budget-calibration.md @@ -0,0 +1,135 @@ +# Flutter projection budget calibration + +Keep the default `pool_policy.uri_limit=500`. Two pinned real APKs show that +requesting 2048 immediately benefits a full bounded package-URI inventory, but +increases complete overview delivery when the task only needs identity and +snapshot facts. This is a task-specific selection, not evidence for a universal +2048 default. No production default changed in this experiment. + +## Tasks and correctness + +The inventory task retrieves every distinct complete printable ASCII run that +starts `package:` and has length 5–1024 in arm64 +`_kDartIsolateSnapshotData`. Both workflows select `contains:"package:"`. +The baseline starts at the default 500 and retries with 2048 when the returned +array is shorter than its count lower bound; the candidate starts at 2048. +Both final arrays exactly match an independent Python ZIP/ELF dynamic-symbol and +section-table parser plus byte-regex oracle. Gallery has 756 matches, LocalSend +1407; 1024 is insufficient for the latter. + +The overview control requires target/library identity, ABI and snapshot section +observations, without individual strings. It compares unfiltered complete +Evidence with URI 500 and URI 2048. All calls preserve `run_bytes=1024`, +`token_bytes=128`, `token_limit=2000` and existing admission budgets. Strings +remain byte-pattern observations, not recovered Dart types. Gallery has eight +printable runs over 1024 bytes and LocalSend one. Recovering every bounded +package URI does not make the overall string pool or snapshot coverage complete. + +## Measurements + +Three serial trials per case, 24 successful fresh source-service processes, +Node 24.22.1, macOS 26.3 arm64. Each starts `FlutterBuildAnalysisService` and closes +its provider. No app installation or execution occurred. OS caches were not +flushed; these are neither cold-cache nor warm-runtime-cache claims. No small +real Flutter artifact was available as a setup control. Eight task variants +across two APKs do not establish universal workload fit. + +Values below are medians of three runs; tokens were identical within each case. +The numeric receipts, artifact manifest and portable reproduction tools are in +[`scripts/verify/analysis/flutter-budget-calibration/`](https://github.com/morluto/rea/tree/main/scripts/verify/analysis/flutter-budget-calibration). + +| APK / completed task | Calls | cl100k tokens | Wall s | CPU s | Peak group RSS MiB | +| --------------------------------------- | ----: | ------------: | -----: | ----: | -----------------: | +| Gallery inventory, default + recovery | 2 | 14,488 | 0.836 | 1.028 | 258.656 | +| Gallery inventory, 2048 first | 1 | 8,631 | 0.480 | 0.601 | 238.859 | +| Gallery overview, default | 1 | 15,995 | 0.448 | 0.639 | 241.922 | +| Gallery overview, 2048 | 1 | 18,764 | 0.560 | 0.639 | 243.328 | +| LocalSend inventory, default + recovery | 2 | 22,360 | 0.405 | 0.519 | 221.328 | +| LocalSend inventory, 2048 first | 1 | 16,317 | 0.296 | 0.365 | 184.781 | +| LocalSend overview, default | 1 | 16,885 | 0.292 | 0.403 | 189.766 | +| LocalSend overview, 2048 | 1 | 27,158 | 0.293 | 0.388 | 189.641 | + +Tokens use tiktoken 0.12.0 `cl100k_base`, separately encoding compact Unicode JSON +for every logical request and complete reply, including retries. They are exact +counts for that encoding and those records, not model usage. CLI/MCP discovery, +adapter delivery, retained-resource paging, chat framing and model reasoning +are unmeasured. Path spelling can change replay counts. The driver also emits +separately labeled tokenx 1.3.0 estimates; the table uses cl100k counts. + +Wall time includes observer overhead. CPU is OS `wait4` user+system usage of the +waited subtree; it excludes sampler work and descendants not waited for. RSS is +the simultaneous process-group sum sampled every 20 ms: short peaks can be +missed, shared pages can be counted twice and descendants leaving the group +are excluded. The 1536 MiB observer stop is not a kernel hard limit. All 24 runs +exited 0 without a resource stop and left their groups empty. No timing or memory +claim is made for an unmeasured MCP summary workflow; the separate retained +local historical summary study (`experiments/analysis-efficiency/flutter-aot-summary.md`) measured that workflow historically. + +The frozen baseline is HEAD `e2a2ede5440df396e44b3406ef511a31a680290e` plus +tracked patch SHA256 `038b69c19bbd79a303c9ecd8d9d22a1251637e3886543ca4f5a4c1b70c724eec` +and copied untracked policy sources. HEAD alone does not identify the measured +runtime. Original driver/oracle hashes and per-run values are retained in +`measurements.json`; retained portable scripts change path resolution without +changing task semantics. Later observer cleanup fixes do not change the sampled +RSS algorithm or these successful, empty-group receipts. + +## Reproduce + +Use the pinned Node toolchain and installed locked dependencies. Download the two +URLs in `manifest.json` to an external input directory under their declared +filenames. `oracle.py` verifies APK sizes/digests, libapp digests, section offsets +and expected counts before generating exact arrays. Do not commit APKs or raw +reply records. Run comparisons serially; replace the paths below with absolute +paths to the selected source snapshot, Node 24 and external artifacts. + +```sh +REA_CAL_REPO=/absolute/path/to/rea +REA_CAL_NODE=/absolute/path/to/node24 +REA_CAL_INPUTS=/absolute/path/to/apks +REA_CAL_OUTPUT=/absolute/path/to/receipts +REA_CAL_ASSETS="$REA_CAL_REPO/scripts/verify/analysis/flutter-budget-calibration" +for trial in 1 2 3; do + python3 "$REA_CAL_ASSETS/oracle.py" "$REA_CAL_INPUTS" "$REA_CAL_OUTPUT/r$trial" + python3 "$REA_CAL_ASSETS/configs.py" "$REA_CAL_INPUTS" "$REA_CAL_OUTPUT/r$trial" + for config in "$REA_CAL_OUTPUT/r$trial/"*.config.json; do + python3 "$REA_CAL_REPO/scripts/verify/analysis/budget-calibration-run.py" \ + --output "${config%.config.json}.metrics.json" --cwd "$REA_CAL_REPO" \ + --timeout-seconds 90 --memory-mib 1536 -- \ + "$REA_CAL_NODE" --import "$REA_CAL_REPO/node_modules/tsx/dist/loader.mjs" \ + "$REA_CAL_ASSETS/task.mjs" "$REA_CAL_REPO" "$config" + done +done +python3 "$REA_CAL_ASSETS/aggregate.py" "$REA_CAL_OUTPUT" "$REA_CAL_OUTPUT/aggregate.json" +``` + +`aggregate.py` requires tiktoken 0.12.0. Install it separately if needed; the +calibration scripts do not change the project's dependencies or host software. +Replays on a later checkout are new measurements, not reproduction of the +frozen runtime merely because artifact hashes match. + +## Other defaults remain evidence-limited + +Keep Apktool strings 2000/table and 10000 aggregate: the retained real +three-APK native-decoder study (`experiments/analysis-efficiency/apktool-string-tables.md`, historical local output) has at most 1126 valid +strings in one table, and three selected locale tables fit the aggregate. +That does not calibrate all APKs or all-locales tasks. No fresh Apktool run was +performed here; the current host lacks the executable. + +Keep desktop sampling defaults pending a representative complete state-view +corpus. The pinned official QGIS GPKG used in the historical local query study +(`experiments/analysis-efficiency/sqlite-existing-controls.md`) has 26 declared tables, +240 countries and 4556 states/provinces according to an independent read-only +SQLite count. A 500-row sample covers the former and omits the latter. One +all-column geometry row can expand to 485277 bytes; selected tables/columns are +more appropriate than globally multiplying retained rows. Counts alone do not +calibrate CPU/memory or text/XML defaults. + +Keep JavaScript semantic candidate/depth/getter/alias defaults. They bound +analysis work and precision, rather than merely display projection. No +independent real-artifact sensitivity study crossing these defaults was found. +Synthetic recovery regressions establish behavior, not representative workload +fit. This calibration gap remains explicit. + +The supporting QGIS input is pinned in `manifest.json`; the row counts were +rechecked with Python SQLite in read-only mode. Historical local studies are +context, not new provider measurements or checked-in raw receipts. diff --git a/docs/verification/javascript-precision-calibration.md b/docs/verification/javascript-precision-calibration.md new file mode 100644 index 0000000000..3142bdaf68 --- /dev/null +++ b/docs/verification/javascript-precision-calibration.md @@ -0,0 +1,127 @@ +# JavaScript primitive-budget calibration — 2026-11-09 + +Keep the existing semantic precision defaults. A narrow real-source comparison +found no additional required facts when doubling `primitive_bytes`; it does not +justify a global increase or replacing the existing conservative admission rule. +The input contract now explains that rule, and its list of defaults derives from +the policy owner instead of repeating independent literals. + +## What was checked + +Two inert directories went through the actual compiled +`analyze-javascript-application` CLI, including its isolated source worker, +complete Evidence serialization and normal CLI cleanup: + +- An original `theme.ts` embedded in the pinned Actual Budget Theme source map. + The required task recovers all **237** literal CSS property/value pairs and + their source identity. A separate Python parser accepts only the complete + literal-only declaration and checks every name/value without using REA's + evaluator. A small package descriptor constructs the analysis directory; this + is one captured module, not the complete application or an authenticated + browser capture. +- The installed locked **ms 2.1.3** package's original `index.js` and package + descriptor. The requested small control recovers its `s = 1000` millisecond + base unit. Multiplication-derived coefficients remain unsupported and unknown; + larger byte admission does not implement arithmetic or prove those values. + +Input URLs/digests, a portable preparation/oracle script, CLI task driver and +all twelve compact numeric receipts are in +[the calibration assets](https://github.com/morluto/rea/tree/main/scripts/verify/analysis/javascript-precision-calibration). +No downloaded code or complete large replies are committed. The theme original +source SHA-256 is `05307dc6f927705a8c76e43579c910bf6415ce28b375eea19251dc51dd2b1726`. + +## Results + +Three fresh CLI/worker trials per case compare omitted precision with explicit +`semantic_precision: {"primitive_bytes": 2097152}`. Both requests hold the worker +and CLI host old-space allowance at 512 MiB. Each task makes one analysis call, +with no retries. All twelve exit successfully, verify the required facts and +leave the observer's process group empty. Theme has no semantic unknowns; ms +keeps the same 133 unrelated unknowns in both variants. This is complete +coverage of the requested facts, not complete semantic understanding of ms. + +| Task / allowance | Required facts | Estimated CLI task tokens | Wall s | CPU s | Sampled host-group RSS MiB | +| ----------------------------- | -------------: | ------------------------: | -----: | ----: | -------------------------: | +| Theme / default 1 MiB | 237 | 259,396 | 0.937 | 1.226 | 330.6 | +| Theme / explicit 2 MiB | 237 | 259,475 | 0.928 | 1.229 | 330.8 | +| ms base unit / default 1 MiB | 1 | 234,613 | 0.899 | 1.238 | 322.2 | +| ms base unit / explicit 2 MiB | 1 | 234,668 | 0.905 | 1.225 | 330.5 | + +Values are medians of three serial trials, in fixed default-then-double order. +Node 24.22.1 ran on macOS 26.3 arm64; OS caches were not flushed. Mixed small +timing/memory differences do not establish a performance improvement. + +Token estimates use tiktoken 0.12.0 `cl100k_base`: compact Unicode JSON of the +actual argv and complete parsed stdout Evidence, plus literal progress stderr. +The full complete replies are about 734 KB for Theme and 624 KB for ms. This +measurement includes the CLI entrypoint, worker, serialized output, and the +driver's correctness checks. It excludes artifact acquisition, one-off oracle +preparation, tokenizer computation, MCP discovery/protocol, chat framing and +model reasoning. These are not billed model tokens or a claim that full output +is an efficient agent workflow; existing retained summary/views can change +delivery independently of evaluator precision. Replays with other path spellings +can have different tokens and identities. + +CPU is OS `wait4` user+system usage of the waited subtree; sampler work and +unwaited descendants are excluded. RSS sums the host process group every 20 ms: +short peaks may be missed and shared pages counted more than once. **The isolated +JavaScript worker launches into another process group, so its peak memory is +unmeasured here.** The table is not total process-family peak memory. The +observer's 1536 MiB early-stop guard is not a kernel hard limit; its 90-second +deadline was not reached. Worker old-space flags are also not whole-process +memory limits. Compiled entrypoint and owner hashes are retained because HEAD +`e2a2ede5440df396e44b3406ef511a31a680290e` alone does not identify the measured +uncommitted runtime. Portable scripts revise path handling after measurement +without changing the requested facts; original driver/oracle hashes are retained. + +## Why retain the conservative rule + +`primitive_bytes` charges six estimated JSON bytes per UTF-16 code unit plus +quotes, before creating concatenations. At the 1 MiB default, 174,762 ASCII units +fit and one additional unit exhausts that estimate, even though actual ASCII +JSON occupies far fewer bytes. Control characters and lone surrogates can need +six escaped bytes, so this is a conservative precision/resource policy rather +than an assertion about observed encoded bytes. Existing regressions retain the +boundary and exact exhausted-control recovery. + +A bounded Babel inventory of the pinned generated Theme, main and DateSelect +scripts found their longest static strings/templates to be 152, 936 and 13,120 +UTF-16 units respectively. This inventory is not full semantic analysis and +does not bound dynamically composed values or candidate cross-products. It +nevertheless supplies no representative case requiring a higher string default. + +Exact preallocation accounting would need to inspect string contents across +every candidate concatenation, including pairs formed across piece boundaries, +while preserving work bounds. Without an observed task failing this estimate, +that added work and complexity are unjustified. The 256-candidate, 512-state and +other depth/work defaults remain provisional: these two tasks do not establish +their suitability for large unions, getters, aliases or adversarial sources. + +## Reproduce + +Download the pinned Theme map URL from `manifest.json` to external storage, use +the pinned Node toolchain and installed locked dependencies, then run: + +```sh +REA_JS_REPO=/absolute/path/to/rea +REA_JS_NODE=/absolute/path/to/node24 +REA_JS_DATA=/absolute/path/to/external-receipts +REA_JS_ASSETS="$REA_JS_REPO/scripts/verify/analysis/javascript-precision-calibration" +python3 "$REA_JS_ASSETS/prepare.py" "$REA_JS_REPO" /absolute/path/to/theme.js.map "$REA_JS_DATA" +for trial in 1 2 3; do + for task in theme ms; do + for budget in default double; do + python3 "$REA_JS_REPO/scripts/verify/analysis/budget-calibration-run.py" \ + --output "$REA_JS_DATA/r${trial}-${task}-${budget}.metrics.json" \ + --cwd "$REA_JS_REPO" --timeout-seconds 90 --memory-mib 1536 -- \ + python3 "$REA_JS_ASSETS/task.py" "$REA_JS_REPO" "$REA_JS_NODE" \ + "$REA_JS_DATA" "$task" "$budget" "$REA_JS_DATA/r${trial}-${task}-${budget}.replies.json" + done + done +done +python3 "$REA_JS_ASSETS/aggregate.py" "$REA_JS_DATA" "$REA_JS_DATA/aggregate.json" +``` + +`aggregate.py` needs the separately installed tokenizer version above. These +scripts neither install dependencies nor change host/provider configuration. +Later checkouts are new measurements even when target hashes still match. diff --git a/docs/verification/native-capture-budget-qualification-2026-11-09.json b/docs/verification/native-capture-budget-qualification-2026-11-09.json new file mode 100644 index 0000000000..ed5633dfa5 --- /dev/null +++ b/docs/verification/native-capture-budget-qualification-2026-11-09.json @@ -0,0 +1,168 @@ +{ + "schema": "rea-native-budget-qualification/v1", + "date": "2026-11-09", + "host": "macOS 26.3 arm64", + "node": "24.22.1", + "scope": "Real application service/provider boundaries, not public MCP/CLI recovery. Controlled owned fixtures only.", + "frida": { + "package_version": "17.23.2", + "cases": [ + { + "session_bytes": 1, + "messages": [], + "messages_truncated": true, + "effective_limits": { + "session_bytes": 1, + "message_type_bytes": 4096, + "message_text_bytes": 32768, + "json_nodes": 250000, + "json_depth": 64 + }, + "limitations": [ + "Artifact identity is unavailable for this observation.", + "The target operating system and architecture were not reported by this observation.", + "Frida capture reached capture_limits.session_bytes (1 bytes including JSON/base64 expansion and metadata). Select a larger capture_limits.session_bytes before starting a new explicitly chosen session or instrument_with_frida experiment; no REA restart is required. Already discarded observations cannot be recovered. Session bytes and MCP/client delivery budgets remain independent." + ], + "cleanup_error": null + }, + { + "session_bytes": 4096, + "messages": [ + { + "script_id": "2c901ff4-07b7-4a97-9784-bdc9f676c911", + "type": "send", + "payload": null, + "payload_truncated": true, + "payload_capture_limit": "payload_bytes" + } + ], + "messages_truncated": false, + "effective_limits": { + "session_bytes": 4096, + "message_type_bytes": 4096, + "message_text_bytes": 32768, + "json_nodes": 250000, + "json_depth": 64 + }, + "limitations": [ + "Artifact identity is unavailable for this observation.", + "The target operating system and architecture were not reported by this observation.", + "Frida capture reached capture_limits.session_bytes (4096 bytes including JSON/base64 expansion and metadata). Select a larger capture_limits.session_bytes before starting a new explicitly chosen session or instrument_with_frida experiment; no REA restart is required. Already discarded observations cannot be recovered. Session bytes and MCP/client delivery budgets remain independent." + ], + "cleanup_error": null + }, + { + "session_bytes": 262144, + "messages": [ + { + "script_id": "6283235c-f8c4-40db-ad61-3375003e4362", + "type": "send", + "payload": { + "rea_budget_probe": "ok" + } + } + ], + "messages_truncated": false, + "effective_limits": { + "session_bytes": 262144, + "message_type_bytes": 4096, + "message_text_bytes": 32768, + "json_nodes": 250000, + "json_depth": 64 + }, + "limitations": [ + "Artifact identity is unavailable for this observation.", + "The target operating system and architecture were not reported by this observation." + ], + "cleanup_error": null + } + ] + }, + "lldb": { + "version": "lldb-2100.0.17.108", + "cases": [ + { + "output_bytes": 2, + "target_sha256": "819c94d6cdf0aff72ff5cea9769a7caf900d223af23ef11d66a13baa8baf71aa", + "outcome": "exited", + "events": 1, + "stdout": { + "text": "re", + "bytes": 20, + "truncated": true, + "complete": true + }, + "stderr": { + "text": "", + "bytes": 0, + "truncated": false, + "complete": true + }, + "terminated": true + }, + { + "output_bytes": 64, + "target_sha256": "819c94d6cdf0aff72ff5cea9769a7caf900d223af23ef11d66a13baa8baf71aa", + "outcome": "exited", + "events": 1, + "stdout": { + "text": "rea_budget_probe_ok\n", + "bytes": 20, + "truncated": false, + "complete": true + }, + "stderr": { + "text": "", + "bytes": 0, + "truncated": false, + "complete": true + }, + "terminated": true + } + ], + "cleanup": { + "ok": true + } + }, + "working_controls": { + "frida_existing_smoke": { + "ok": true, + "provider": "frida", + "host_platform": "darwin", + "target_mode": "local_spawn", + "target_pid": 47641, + "target_name": "node", + "script_message_observed": true, + "cleanup_error": null + }, + "ghidra_java_compile": { + "ghidra_version": "12.1.4", + "javac": "javac 21.0.12.1", + "class_path_jars": 195, + "sources": [ + "bridge/ghidra/ReaGhidraAnalysisOptions.java", + "bridge/ghidra/ReaGhidraApplySeeds.java", + "bridge/ghidra/ReaGhidraBridge.java", + "bridge/ghidra/ReaGhidraBundleActivator.java", + "bridge/ghidra/ReaGhidraExtensions.java", + "bridge/ghidra/ReaGhidraMipsSoftFloat.java", + "bridge/ghidra/ReaGhidraNoReturnFix.java", + "bridge/ghidra/ReaGhidraPrepareCom.java", + "bridge/ghidra/ReaGhidraPrepareDol.java", + "bridge/ghidra/ReaGhidraPrepareRaw.java", + "bridge/ghidra/ReaGhidraRiscvCallingConvention.java" + ] + } + }, + "probe_source_sha256": { + "frida-budget-probe.mjs": "43dd424e0750bc8f60468210d99caedaa1e0fa16148db3c2290a8ff593254ccb", + "lldb-budget-probe.mjs": "f11e899698d6c219078b1e4a6b52d97bfd308ab9ee8fade6902cb2bdfd1826af" + }, + "limitations": [ + "Each raised capture budget starts a new controlled process; discarded live observations are not recovered retroactively.", + "Initial 4 KiB Frida marker assertion failed because a fixed 128 KiB metadata reserve leaves no payload allowance; tagged truncation is preserved.", + "Initial 1-second LLDB observation elapsed before any event; a 10-second-window rerun isolated output retention.", + "Successful observer receipts showed empty owned process groups; this is not a kernel process-family memory or performance qualification.", + "Windows/Linux, full AgentEnv, remote Frida and public native-tool transport recovery are unverified." + ] +} diff --git a/docs/verification/runtime-analysis-settings-ghidra-2026-11-09.json b/docs/verification/runtime-analysis-settings-ghidra-2026-11-09.json new file mode 100644 index 0000000000..b178bf9de4 --- /dev/null +++ b/docs/verification/runtime-analysis-settings-ghidra-2026-11-09.json @@ -0,0 +1,227 @@ +{ + "status": "passed", + "platform": "darwin", + "architecture": "arm64", + "node": "v24.22.1", + "fixture": { + "kind": "volatile-straight-line", + "statements": 5400, + "source_bytes": 297679 + }, + "target_sha256": "4004126020cc5517d75c47cacabc68422a12b3260ccd19c105f9a7df087c69d1", + "source_sha256": "8bc279d5a4881ca87d1629fc26d73d191986d2a2fb014f77a8d711ea0eca9426", + "compiler": { + "command": "cc", + "arguments": [ + "-dynamiclib", + "-fPIC", + "-O0", + "/var/folders/xl/60j5sd4j6b99tn8q082l9t8r0000gn/T/rea-ghidra-decompile-timeout-c0NNHy/slow-decompile.c", + "-o", + "/var/folders/xl/60j5sd4j6b99tn8q082l9t8r0000gn/T/rea-ghidra-decompile-timeout-c0NNHy/slow-decompile.dylib" + ] + }, + "provider": { + "id": "ghidra", + "name": "Ghidra", + "version": "12.1.4" + }, + "analysis_profile_digest": "43b27fd852aa4e8b9b08c60928f06e9961e04f17410d0246ca771c3d526248c4", + "configured_jvm_max_heap": "1G", + "elapsed_seconds": 16.066160042, + "stages": [ + { + "stage": "operations_low", + "available": true, + "operations": 1, + "largest_input_count": 2, + "def_use_edges": 0, + "parameter_uses": 1, + "truncated": true, + "limits": { + "max_operations": 1, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": ["max_operations"], + "omitted_operations_lower_bound": 1, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "operations_recovered", + "available": true, + "operations": 8, + "largest_input_count": 2, + "def_use_edges": 6, + "parameter_uses": 1, + "truncated": false, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": [], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "inputs_low", + "available": true, + "operations": 8, + "largest_input_count": 1, + "def_use_edges": 4, + "parameter_uses": 1, + "truncated": true, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 1, + "max_edges": 12000 + }, + "limits_reached": ["max_inputs_per_operation"], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 5, + "known_omitted_edges": 0 + }, + { + "stage": "inputs_recovered", + "available": true, + "operations": 8, + "largest_input_count": 2, + "def_use_edges": 6, + "parameter_uses": 1, + "truncated": false, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": [], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "edges_low", + "available": true, + "operations": 8, + "largest_input_count": 2, + "def_use_edges": 1, + "parameter_uses": 1, + "truncated": true, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 64, + "max_edges": 1 + }, + "limits_reached": ["max_edges"], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 0, + "known_omitted_edges": 5 + }, + { + "stage": "edges_recovered", + "available": true, + "operations": 8, + "largest_input_count": 2, + "def_use_edges": 6, + "parameter_uses": 1, + "truncated": false, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": [], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "operations_lowered", + "available": true, + "operations": 2, + "largest_input_count": 2, + "def_use_edges": 1, + "parameter_uses": 1, + "truncated": true, + "limits": { + "max_operations": 2, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": ["max_operations"], + "omitted_operations_lower_bound": 1, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "payload_1_mib", + "seconds": 1.7718680000000004, + "code": "execution_failure", + "message": "Ghidra decompilation failed: Exception while decompiling 00000414: Response buffer size exceeded for: target-4004126020cc.dylib . The effective native decompiler payload allowance is 1 MiB. If the diagnostic reports payload exhaustion, in MCP raise ghidra.decompile_max_payload_mib with update_analysis_settings (integer MiB from 1 to 2047) and repeat the selected operation on this Program without restarting REA. For a one-shot CLI retry, set REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB to a larger MiB value in the environment that starts the next rea invocation, then repeat the selected operation; the previous invocation's Program is no longer active. Runtime null or an omitted launch seed restores the installed Ghidra default. This does not raise the decompile deadline, high-pcode projection budgets or MCP client limits. Only the decompiler failed for this procedure. The Ghidra session remains usable." + }, + { + "stage": "payload_64_mib_recovered", + "seconds": 2.11943175, + "pseudocode_bytes": 261131 + }, + { + "stage": "deadline_partial_dossier", + "seconds": 2.3652059170000013, + "pseudocode": { + "available": false, + "reason": "Ghidra decompilation of 0x414 exceeded the 1-second configured decompilation limit (ghidra.decompile_timeout_seconds; launch seed REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS)" + }, + "assembly_lines": 49912, + "basic_blocks": 1, + "cache_policy": "live" + }, + { + "stage": "deadline_pseudocode", + "seconds": 1.2178966249999994, + "code": "provider_timeout", + "message": "Ghidra decompilation of 0x414 exceeded the 1-second configured decompilation limit (ghidra.decompile_timeout_seconds; launch seed REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS). Only this procedure's decompilation was stopped. The Ghidra session remains usable.", + "limit": { + "setting": "ghidra.decompile_timeout_seconds", + "effective_value": 1, + "unit": "seconds", + "minimum": 1, + "maximum": 2147483, + "applies_at": "next_operation", + "agent_editable": true, + "scope": "rea-runtime", + "rea_restart_required": false + }, + "remediation": { + "action": "Use analyze_function (CLI: `rea function`) for listing facts. Call update_analysis_settings with setting ghidra.decompile_timeout_seconds and action set to raise the per-function deadline for the next operation on this Program. For CLI or future REA processes, set REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS. Set the MCP tool-call deadline to cover startup and the full call; earlier cancellation closes the session. `rea setup` estimates deadlines for new Codex/Kimi registrations and preserves existing positive values; edit existing deadlines yourself." + } + }, + { + "stage": "deadline_recovered_payload_null", + "seconds": 2.0052460419999987, + "selected_timeout_seconds": 20, + "selected_payload_mib": null, + "pseudocode_bytes": 261131 + } + ], + "initial_session_run_id": "c03f277d-d679-4e85-965b-690474ec4e9c", + "final_session_run_id": "c03f277d-d679-4e85-965b-690474ec4e9c", + "same_session_next_decompile": "succeeded", + "verifier_run": { + "run_id": "7fea6f67-fb82-4c6a-b129-8bbe3de8b5b2", + "verifier_pid": 52515, + "parent_pid": 2850, + "process_lineage": { + "status": "verified", + "observed_at": "2026-11-09T16:50:38.401Z", + "launcher_pid": 52515, + "launcher_parent_pid": 2850, + "process_group_id": 52515, + "descendants": [] + } + }, + "scope": "Real Ghidra analysis of generated Mach-O/ELF through authenticated loopback: independent value-flow budgets, lowered admission, native payload exhaustion and recovery, natural decompile timeout with listing-backed partial facts, and deadline recovery on the same Program. No target execution. Windows and public CLI/MCP transport recovery are unverified. Timings are observations, not performance comparisons; CPU, peak process-family memory and agent tokens are unmeasured.", + "cleanup": "complete" +} diff --git a/docs/verification/runtime-analysis-settings-ghidra-integration-2026-11-09.json b/docs/verification/runtime-analysis-settings-ghidra-integration-2026-11-09.json new file mode 100644 index 0000000000..209853f037 --- /dev/null +++ b/docs/verification/runtime-analysis-settings-ghidra-integration-2026-11-09.json @@ -0,0 +1,232 @@ +{ + "integration_base": "8dfc47b107afc0cf4b53322a10b94c666fe9e18b", + "tested_feature_head": "329df8b331ece4201e7ea0849643c4481611ac86", + "qualification": "Post-main-integration correctness run; not a performance comparison.", + "receipt": { + "status": "passed", + "platform": "darwin", + "architecture": "arm64", + "node": "v24.22.1", + "fixture": { + "kind": "volatile-straight-line", + "statements": 5400, + "source_bytes": 297679 + }, + "target_sha256": "0eabbda46fcb6bbb33a84df09f8a2f1b2e4621aa5c7b8c7216921188e3b55ad8", + "source_sha256": "8bc279d5a4881ca87d1629fc26d73d191986d2a2fb014f77a8d711ea0eca9426", + "compiler": { + "command": "cc", + "arguments": [ + "-dynamiclib", + "-fPIC", + "-O0", + "/var/folders/xl/60j5sd4j6b99tn8q082l9t8r0000gn/T/rea-ghidra-decompile-timeout-8ppfRJ/slow-decompile.c", + "-o", + "/var/folders/xl/60j5sd4j6b99tn8q082l9t8r0000gn/T/rea-ghidra-decompile-timeout-8ppfRJ/slow-decompile.dylib" + ] + }, + "provider": { + "id": "ghidra", + "name": "Ghidra", + "version": "12.1.4" + }, + "analysis_profile_digest": "a70b06ba1d73ceb5b6e7ee2d9f7587583c17e076134ebb96f0169f6a8b95956a", + "configured_jvm_max_heap": "1G", + "elapsed_seconds": 16.838551541999998, + "stages": [ + { + "stage": "operations_low", + "available": true, + "operations": 1, + "largest_input_count": 2, + "def_use_edges": 0, + "parameter_uses": 1, + "truncated": true, + "limits": { + "max_operations": 1, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": ["max_operations"], + "omitted_operations_lower_bound": 1, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "operations_recovered", + "available": true, + "operations": 8, + "largest_input_count": 2, + "def_use_edges": 6, + "parameter_uses": 1, + "truncated": false, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": [], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "inputs_low", + "available": true, + "operations": 8, + "largest_input_count": 1, + "def_use_edges": 4, + "parameter_uses": 1, + "truncated": true, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 1, + "max_edges": 12000 + }, + "limits_reached": ["max_inputs_per_operation"], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 5, + "known_omitted_edges": 0 + }, + { + "stage": "inputs_recovered", + "available": true, + "operations": 8, + "largest_input_count": 2, + "def_use_edges": 6, + "parameter_uses": 1, + "truncated": false, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": [], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "edges_low", + "available": true, + "operations": 8, + "largest_input_count": 2, + "def_use_edges": 1, + "parameter_uses": 1, + "truncated": true, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 64, + "max_edges": 1 + }, + "limits_reached": ["max_edges"], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 0, + "known_omitted_edges": 5 + }, + { + "stage": "edges_recovered", + "available": true, + "operations": 8, + "largest_input_count": 2, + "def_use_edges": 6, + "parameter_uses": 1, + "truncated": false, + "limits": { + "max_operations": 3000, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": [], + "omitted_operations_lower_bound": 0, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "operations_lowered", + "available": true, + "operations": 2, + "largest_input_count": 2, + "def_use_edges": 1, + "parameter_uses": 1, + "truncated": true, + "limits": { + "max_operations": 2, + "max_inputs_per_operation": 64, + "max_edges": 12000 + }, + "limits_reached": ["max_operations"], + "omitted_operations_lower_bound": 1, + "known_omitted_inputs": 0, + "known_omitted_edges": 0 + }, + { + "stage": "payload_1_mib", + "seconds": 1.6774305839999988, + "code": "execution_failure", + "message": "Ghidra decompilation failed: Exception while decompiling 00000414: Response buffer size exceeded for: target-0eabbda46fcb.dylib . The effective native decompiler payload allowance is 1 MiB. If the diagnostic reports payload exhaustion, in MCP raise ghidra.decompile_max_payload_mib with update_analysis_settings (integer MiB from 1 to 2047) and repeat the selected operation on this Program without restarting REA. For a one-shot CLI retry, set REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB to a larger MiB value in the environment that starts the next rea invocation, then repeat the selected operation; the previous invocation's Program is no longer active. Runtime null or an omitted launch seed restores the installed Ghidra default. This does not raise the decompile deadline, high-pcode projection budgets or MCP client limits. Only the decompiler failed for this procedure. The Ghidra session remains usable." + }, + { + "stage": "payload_64_mib_recovered", + "seconds": 2.1426570829999982, + "pseudocode_bytes": 261131 + }, + { + "stage": "deadline_partial_dossier", + "seconds": 2.4800411670000013, + "pseudocode": { + "available": false, + "reason": "Ghidra decompilation of 0x414 exceeded the 1-second configured decompilation limit (ghidra.decompile_timeout_seconds; launch seed REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS)" + }, + "assembly_lines": 49912, + "basic_blocks": 1, + "cache_policy": "live" + }, + { + "stage": "deadline_pseudocode", + "seconds": 1.2235721249999987, + "code": "provider_timeout", + "message": "Ghidra decompilation of 0x414 exceeded the 1-second configured decompilation limit (ghidra.decompile_timeout_seconds; launch seed REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS). Only this procedure's decompilation was stopped. The Ghidra session remains usable.", + "limit": { + "setting": "ghidra.decompile_timeout_seconds", + "effective_value": 1, + "unit": "seconds", + "minimum": 1, + "maximum": 2147483, + "applies_at": "next_operation", + "agent_editable": true, + "scope": "rea-runtime", + "rea_restart_required": false + }, + "remediation": { + "action": "Use analyze_function (CLI: `rea function`) for listing facts. Call update_analysis_settings with setting ghidra.decompile_timeout_seconds and action set to raise the per-function deadline for the next operation on this Program. For CLI or future REA processes, set REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS. Set the MCP tool-call deadline to cover startup and the full call; earlier cancellation closes the session. `rea setup` estimates deadlines for new Codex/Kimi registrations and preserves existing positive values; edit existing deadlines yourself." + } + }, + { + "stage": "deadline_recovered_payload_null", + "seconds": 2.361492542, + "selected_timeout_seconds": 20, + "selected_payload_mib": null, + "pseudocode_bytes": 261131 + } + ], + "initial_session_run_id": "89aac416-1a46-4eeb-9ddf-9652ba17262b", + "final_session_run_id": "89aac416-1a46-4eeb-9ddf-9652ba17262b", + "same_session_next_decompile": "succeeded", + "verifier_run": { + "run_id": "1c4699a5-d090-4cad-8271-961cdd41d23e", + "verifier_pid": 9490, + "parent_pid": 2850, + "process_lineage": { + "status": "verified", + "observed_at": "2026-11-09T21:55:25.629Z", + "launcher_pid": 9490, + "launcher_parent_pid": 2850, + "process_group_id": 9490, + "descendants": [] + } + }, + "scope": "Real Ghidra analysis of generated Mach-O/ELF through authenticated loopback: independent value-flow budgets, lowered admission, native payload exhaustion and recovery, natural decompile timeout with listing-backed partial facts, and deadline recovery on the same Program. No target execution. Windows and public CLI/MCP transport recovery are unverified. Timings are observations, not performance comparisons; CPU, peak process-family memory and agent tokens are unmeasured.", + "cleanup": "complete" + } +} diff --git a/docs/verification/runtime-analysis-settings.md b/docs/verification/runtime-analysis-settings.md index ee3757fed9..c3fc6c2635 100644 --- a/docs/verification/runtime-analysis-settings.md +++ b/docs/verification/runtime-analysis-settings.md @@ -2,11 +2,16 @@ Historical report added in [commit f3ed1657f82188fb90914f745a08849537342a23](https://github.com/morluto/rea/commit/f3ed1657f82188fb90914f745a08849537342a23) -on 2026-10-25. The exact source revisions used for the checks and measurements -below were not recorded here. This record does not establish current verification. +on 2026-10-25. The exact source revisions used for the initial checks and +measurements were not recorded here. Those results do not establish current +verification. The initial checks describe the runtime-settings introduction; +dated follow-ups below preserve their own receipts and coverage limitations. +Rebasing the implementation does not requalify a later runtime against these +historical measurements. -Verified on macOS with Node 24.22.1, Java 21.0.12.1 and Ghidra 12.1.4. -This change adds two MCP tools and 11 runtime defaults without a new dependency. +The initial verification used macOS with Node 24.22.1, Java 21.0.12.1 and +Ghidra 12.1.4. That implementation added two MCP tools and 11 runtime defaults +without a new dependency. ## Behavioral checks @@ -86,3 +91,101 @@ and regex setting integration have source/boundary coverage, not a new real provider run for every setting. Linux/Windows engines and Bun are unverified for this change. Client deadlines and receive buffers remain externally owned; active worker values are explicitly unobserved in the settings inspection. + +## Ghidra budget recovery — 2026-11-09 + +The existing +[`verify-real-ghidra-decompile-timeout.mjs`](https://github.com/morluto/rea/blob/main/scripts/verify-real-ghidra-decompile-timeout.mjs) +now covers p-code projection budgets and native payload admission in the same +real Program as deadline recovery. It replaces the branch fixture in that lane +with 5,400 straight-line volatile statements. A preliminary 1,800-statement +calibration completed under one second, so it did not reproduce deadline +exhaustion; this was fixture calibration, not a failed provider result. + +The retained lane ran once on macOS arm64 with Node 24.22.1, Ghidra 12.1.4 and +Homebrew OpenJDK 21.0.12.1. It compiled its own Mach-O dylib using +`cc -dynamiclib -fPIC -O0`; the target was never executed. The lane requested +a 1 GiB JVM heap, bounded compiler execution to 30 seconds, native recovery +decompiles to 20 seconds, and the overall run to 180 seconds. The JVM request +does not bound the separate native decompiler's memory. + +Reproduce after the runtime build with an existing Ghidra/JDK installation: + +```sh +GHIDRA_INSTALL_DIR=/path/to/ghidra_12.1.4_PUBLIC \ +JAVA_HOME=/path/to/jdk-21 \ +node scripts/verify-real-ghidra-decompile-timeout.mjs +``` + +`REA_DECOMPILE_TIMEOUT_STATEMENTS` selects another source-bounded fixture size +when calibrating a different host. The verifier refuses a scenario that does +not naturally exhaust the selected allowance instead of claiming a reproduced +timeout or payload failure. + +The [compact receipt](https://github.com/morluto/rea/blob/main/docs/verification/runtime-analysis-settings-ghidra-2026-11-09.json) retains +the source/target hashes, profile digest, exact setting facts, native failure +diagnostics, timings and cleanup lineage. Observed results were: + +| Selected control | Observed failure or omission | Recovery in the same Program | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| P-code operations: 1 | One operation retained, omitted-operation lower bound 1 | Raising to 3000 returned eight operations without truncation; lowering to 2 subsequently returned two | +| Inputs per operation: 1 | Five known inputs omitted | Restoring 64 returned complete input coverage for the selected function | +| Def-use edges: 1 | One edge retained, five omitted | Restoring 12000 returned six edges without truncation | +| Native payload: 1 MiB | Ghidra reported `Response buffer size exceeded` | Raising to 64 MiB returned 261131 bytes of pseudocode | +| Native deadline: 1 second | The dossier retained 49912 assembly lines and one basic block with unavailable decompiler facets; direct pseudocode returned typed `provider_timeout` | Raising to 20 seconds and selecting payload `null` returned 261131 bytes of pseudocode; the next small function also decompiled | + +The complete lane took 16.07 seconds. Initial and final authenticated session +run IDs both equal `c03f277d-d679-4e85-965b-690474ec4e9c`; the target SHA-256 +remained `4004126020cc5517d75c47cacabc68422a12b3260ccd19c105f9a7df087c69d1`. +Provider close succeeded, token-verified lineage reported no live descendants, +and the fixture workspace was removed. Ghidra may replace its native decompiler +after a timeout; this check establishes Program/session continuity, not native +child-process continuity. + +This lane uses the real provider and authenticated loopback bridge with the +shared runtime setting store. It does not exercise public CLI or MCP transport +recovery, Windows, another Ghidra release, or every parameter-use edge boundary. +The successful payload-null call does not measure the installed default's +effective ceiling. Timings are observations, not a before/after performance +comparison. CPU time, peak process-family memory and agent token usage were not +measured in the retained run. + +## Native capture qualification — 2026-11-09 + +The installed Frida Node SDK and system LLDB also exercised the new capture +controls through their real application service/provider boundaries on macOS +arm64. The [compact receipt](https://github.com/morluto/rea/blob/main/docs/verification/native-capture-budget-qualification-2026-11-09.json) +records the selected/effective policies, observations and cleanup. + +Frida's existing local-spawn verifier observed its marker and reported no cleanup +error. Additional controlled runs selected session allowances of 1 byte, 4 KiB +and 256 KiB. They respectively retained no event, a tagged payload-truncated +event, and the complete marker. The 4 KiB case exposes conservative accounting: +the existing fixed 128 KiB metadata reservation leaves no payload allowance. +This is explicit truncation, not evidence that the nominal 4 KiB is filled with +retained bytes. Its first marker assertion failed for that explained reason; +the subsequent larger-budget experiment does not recover discarded observations. + +The production LLDB tracer ran a newly compiled arm64 C fixture with one known +function entry and a 20-byte stdout line. With `output_bytes: 2`, it retained +`re`, reported 20 drained bytes and truncation. A new run of the same unchanged +binary with `output_bytes: 64` returned the complete line, one expected event and +normal exit. Both targets were confirmed terminated and tracer cleanup succeeded. +An initial one-second observation elapsed during startup before any event; the +ten-second-window rerun isolated output retention. No caller target was run. + +These checks establish real local producer behavior and cleanup, not public +MCP/CLI recovery, full AgentEnv execution, remote Frida, Windows/Linux behavior, +or optimal default sizes. Group cleanup observations are not whole-family peak +memory measurements. The live Ghidra lane above separately establishes same-Program +recovery for read-only static analysis. + +## Main integration qualification — 2026-11-09 + +After integration onto main `8dfc47b107afc0cf4b53322a10b94c666fe9e18b`, the +retained Ghidra lane ran again against the integrated feature code. Its +[integration receipt](https://github.com/morluto/rea/blob/main/docs/verification/runtime-analysis-settings-ghidra-integration-2026-11-09.json) +records the new source/target identities and observations. Payload and deadline +recovery again used one Program with unchanged target bytes, and owned cleanup +completed. This is a fresh correctness qualification, not a new performance +comparison or a replacement for the historical measurements above. diff --git a/docs/verification/source-location-delivery.md b/docs/verification/source-location-delivery.md new file mode 100644 index 0000000000..44a4598e3f --- /dev/null +++ b/docs/verification/source-location-delivery.md @@ -0,0 +1,206 @@ +# Source-location delivery qualification — 2026-11-09 + +Opt-in summary delivery substantially reduces the measured main-map task's +serialized transcript and its median elapsed time, CPU and sampled memory. The +small Theme control exposes a tradeoff: summary adds a call and becomes more +expensive when a client must first load the complete `inspect_analysis_view` +definition. Complete delivery remains the compatibility default. These results +support explicit summary selection for large point answers or already available +views; they do not establish a uniform whole-task token or latency improvement. + +The [product guide](../guides/web/web-source-location.md#compact-point-delivery-and-retained-source-text) +explains summary, matches pages, explicit UTF-16 source text selection and retained +complete Evidence. This record measures the public compiled stdio MCP workflow. +CLI publication performance is unmeasured here. + +## Task, oracle and execution + +The task returns every alternative at one generated point, preserving source/map +identity and original source positions, plus a 512 UTF-16-code-unit snippet around +each mapped point. Inputs are the pinned public Actual Budget Theme, main +`index.CMrE7aQb`, and `DateSelect.BVYaCNUe` maps and generated scripts in the existing +[artifact manifest](https://github.com/morluto/rea/blob/main/docs/verification/source-map-budgets-2026-11-09/artifacts.json). Acquisition used +verified cached public bytes and the existing acquisition helper's explicitly +incomplete analyst-manifest construction. No application or browser was executed. +The independent [VLQ oracle](https://github.com/morluto/rea/blob/main/docs/verification/source-map-budgets-2026-11-09/oracle.py) supplies the +expected source declarations, positions, alternatives and content digests. + +Both variants use the same compiled runtime, files and portable driver: + +- **Complete:** request complete Evidence. Theme and main return it inline; the + driver selects the required snippet locally from the returned source content. + DateSelect exceeds the default 10 MiB transport allowance, then follows its + retained matches continuation and requests the selected source text: three tool + calls, including the failed delivery. +- **Summary:** request `detail: summary`, then explicitly request the selected + source-text page: two tool calls. Match paging remains part of the driver when + required. All three selected real points have one alternative. + +This compares current **complete delivery with bounded retained-view recovery** +against opt-in summary. It does not compare against the former export-only fallback. +The measurement includes driver artifact hashing/reference parsing, initialization, +full catalog discovery, trace, necessary continuation/text calls, answer validation +and shutdown. Export is not needed to answer this task and was not included. + +Three fresh-process runs per case/variant completed. Trial two reversed variant +order; OS caches were not flushed. All 18 runs exited zero, with empty owned groups +and no resource stop. Aggregation independently verifies all snippets and pinned +map/generated-script/content hashes, identical answers and parent Evidence IDs +across variants/trials, and exact raw-map SHA-256 whenever the initial complete +response includes raw text. Runtime execution and deployment authenticity remain +unknown. + +## Serialized token estimates + +Counts use tiktoken 0.12.0 `cl100k_base` on compact Unicode JSON-RPC request/reply +lines, including initialization, the initialized notification, `tools/list`, failed +calls, continuations and selected text. Replies include both MCP text and structured +content. These are serialized-transcript estimates, **not actual model usage or +billing**; client presentation can omit or transform those representations. + +The observed full catalog uses compact input schemas and full output schemas. +`tools/list` returns 4,487,850 wire bytes. Initialization, notification and catalog +account for **1,145,053 estimated tokens** in every run. Do not add selected +contracts again to the catalog-included totals. Post-discovery counts exclude only +that discovery prefix, so they include all task retries and subsequent calls. + +| Map / delivery | Tool calls | Catalog-included task tokens | Post-discovery tokens | +| ----------------------------------- | ---------: | ---------------------------: | --------------------: | +| Theme / complete | 1 | 1,196,400 | 51,347 | +| Theme / summary | 2 | 1,154,878 | 9,825 | +| Main / complete | 1 | 4,354,894 | 3,209,841 | +| Main / summary | 2 | 1,154,851 | 9,798 | +| DateSelect / complete with recovery | 3 | 1,155,932 | 10,879 | +| DateSelect / summary | 2 | 1,154,974 | 9,921 | + +Selected complete wired definitions cost **8,408 tokens** for +`trace_web_source_location` and **180,576** for `inspect_analysis_view`. When those +definitions are uncached and a client loads only the tools it uses, Theme complete +costs 8,408 + 51,347 = **59,755**, whereas summary costs +8,408 + 180,576 + 9,825 = **198,809**. Summary loses in this first-use accounting +scenario. The measured full-catalog scenario already loads the view definition, +so its Theme total instead falls from 1,196,400 to 1,154,878. Main still improves +with selected complete definitions: 3,218,249 to 198,782. + +As a separate **hypothetical client projection**, retaining only each definition's +`{name, description, inputSchema}` costs 548 and 3,533 tokens respectively. Adding +those selected projected definitions to the same post-discovery transcripts gives: + +| Case | Complete → summary, selected complete wired definitions | Complete → summary, hypothetical input-only definitions | +| ---------- | ------------------------------------------------------: | ------------------------------------------------------: | +| Theme | 59,755 → 198,809 | 51,895 → 13,906 | +| Main | 3,218,249 → 198,782 | 3,210,389 → 13,879 | +| DateSelect | 199,863 → 198,905 | 14,960 → 14,002 | + +This projection is an offline accounting calculation, not an observed client/model +transformation or an alternate measured discovery workflow. Cached or deferred tool +loading, output-schema profiles and model-facing reply presentation can change the +costs. No actual model run or billing comparison was performed. + +## Elapsed time, CPU and sampled memory + +Values are medians of three runs on macOS 26.3 arm64 with Node v24.22.1. The small +control has a slight elapsed/CPU increase; DateSelect memory is similar across +variants. Three runs do not establish general performance guarantees. + +| Map / delivery | Wall seconds | CPU seconds, waited subtree | Sampled group peak MiB | Sampled tree peak MiB | +| ----------------------------------- | -----------: | --------------------------: | ---------------------: | --------------------: | +| Theme / complete | 0.972 | 1.128 | 498.0 | 555.8 | +| Theme / summary | 0.985 | 1.143 | 483.9 | 542.1 | +| Main / complete | 1.625 | 1.815 | 776.5 | 776.5 | +| Main / summary | 1.134 | 1.294 | 549.7 | 661.4 | +| DateSelect / complete with recovery | 1.199 | 1.351 | 580.6 | 695.6 | +| DateSelect / summary | 1.150 | 1.326 | 588.6 | 694.2 | + +The observer samples simultaneous `ps` RSS sums at nominal 20 ms intervals, plus +sampling work. **Tree RSS includes +codec descendants outside the root PGID while their PPID ancestry remains linked.** +Short peaks, exited/reparented/orphaned descendants can be missed; shared pages can +be counted repeatedly. These are sampled sums, not kernel peak or private-memory +measurements. CPU is OS `wait4` user + system for the waited subtree, excluding +sampler CPU and unwaited/orphaned descendants. Wall time includes observer overhead. + +The 90-second deadline, 1536 MiB observed memory early stop and cleanup signals apply +only to the **owned process group**. Observational tree membership grants no signal +authority. The group guard does not enforce a tree-wide or kernel memory hard cap; +empty-group cleanup does not prove that an unobserved orphan cannot exist. + +A preliminary 18-run completed-task matrix used group-only RSS and a temporary +driver that decoded UTF-8 separately per incoming chunk. It is superseded: codec +processes can change PGID, and chunk boundaries can split UTF-8. Its local method +receipts are preserved; **none of its result statistics appear in these tables**. +All final runs use the retained driver's UTF-8 stream decoder, raw-map digest +check and the observer's additional read-only tree RSS measurement. The earlier +[map-admission calibration](source-map-budget-calibration.md) is a separate study. + +## Retained evidence and reproduction + +The compact [measurement record](https://github.com/morluto/rea/blob/main/scripts/verify/analysis/source-location-delivery/measurements.json) +contains 18 numeric records, per-call bytes/token estimates, oracle/answer checks, +full SHA-256 pins and the compiled-tree digest. Large replies, artifacts and model +transcripts are not committed. Runtime pins include `dist/mcp/main.js` +(`9bc4c070f63adc9963f276da20b36ffbbcc77b4079585cadf90d792dd2cc04dd`) +and the 2,921-file compiled `dist` tree +(`a8a510499dabc07b3cdf6b164d02e55f2fc6917b696dcc4d23449510e91e80fa`). +The recorded CLI digest is separately bound to `dist/cli.js`. These pins identify +the measured build and selected source-location owners. A later unrelated generated +desktop SQLite diagnostic change and rebuild are outside this snapshot; later +whole-tree hashes do not replace the measured one. + +The [portable driver](https://github.com/morluto/rea/blob/main/scripts/verify/analysis/source-location-delivery/task.mjs) +has SHA-256 `b21ec9d5315682337eb55a4ad208e2145a7397d8d567cdb6653facd3f0bec90e`. +The [observer](https://github.com/morluto/rea/blob/main/scripts/verify/analysis/budget-calibration-run.py) +has SHA-256 `c18a06c44715c97ec9cbc76248dfbcd862d0e5ca3cd0c159d868bc7fa1fa9e9f`. +Use the repository's built runtime and pinned Node. Acquisition refuses changed +publisher bytes; the existing acquisition helper creates `artifacts.json` and +analyst manifests under the selected artifact directory. + +Create a fresh receipt directory and its `runtime.json` before measuring. Keep +source files and the compiled runtime unchanged throughout all 18 runs and +aggregation; do not build or edit between them. The compiled-tree hash collected +afterward identifies the measured runtime under that condition. Aggregate before +any subsequent rebuild. + +```sh +python3 docs/verification/source-map-budgets-2026-11-09/acquire.py \ + docs/verification/source-map-budgets-2026-11-09/artifacts.json /tmp/rea-point-artifacts +python3 - <<'PY' +import hashlib, json +from pathlib import Path +receipts = Path('/tmp/rea-source-location-measurements') +receipts.mkdir() +paths = { + 'driver_sha256': Path('scripts/verify/analysis/source-location-delivery/task.mjs'), + 'observer_sha256': Path('scripts/verify/analysis/budget-calibration-run.py'), + 'cli_sha256': Path('dist/cli.js'), +} +metadata = {key: hashlib.sha256(path.read_bytes()).hexdigest() + for key, path in paths.items() if key != 'cli_sha256'} +cli = paths['cli_sha256'] +metadata['cli_sha256'] = hashlib.sha256(cli.read_bytes()).hexdigest() if cli.is_file() else None +(receipts / 'runtime.json').write_text(json.dumps(metadata, indent=2) + '\n') +PY +python3 scripts/verify/analysis/budget-calibration-run.py \ + --output /tmp/rea-source-location-measurements/r1-index-summary.metrics.json --cwd "$PWD" \ + --timeout-seconds 90 --memory-mib 1536 -- \ + node scripts/verify/analysis/source-location-delivery/task.mjs "$PWD" \ + '{"artifactDir":"/tmp/rea-point-artifacts","select":"index.","detail":"summary","resultPath":"/tmp/rea-source-location-measurements/r1-index-summary.replies.json"}' +``` + +Run Theme, main and DateSelect with both detail modes serially, three times, reversing +variant order for trial two. Use unique receipt/result paths named +`r1-index-summary.metrics.json` / `.replies.json` (and the corresponding case/trial +names), preserving the runtime metadata. After measurements, the reusable +[aggregator](https://github.com/morluto/rea/blob/main/scripts/verify/analysis/source-location-delivery/aggregate.py) +requires Python with tiktoken 0.12.0 and regenerates the compact record: + +```sh +python3 scripts/verify/analysis/source-location-delivery/aggregate.py \ + /tmp/rea-source-location-measurements --repo-root "$PWD" --output /tmp/point-measurements.json +``` + +The aggregator hashes the compiled tree after performance runs, verifies every +answer and successful complete raw-map response, and excludes large payloads from +its output. Analysis still decodes and retains complete Evidence; this study does +not qualify arbitrary larger maps, batch source-map fetching, browser execution or +disk-cold behavior. diff --git a/docs/verification/source-map-budget-calibration.md b/docs/verification/source-map-budget-calibration.md new file mode 100644 index 0000000000..964f7e2ff6 --- /dev/null +++ b/docs/verification/source-map-budget-calibration.md @@ -0,0 +1,109 @@ +# Source-map admission calibration — 2026-11-09 + +The local map default changes from 4 to **8 MiB** to admit the pinned Actual +Budget main and DateSelect maps on the first call. Other source-map defaults +remain unchanged. This is a compatibility improvement for the measured artifacts, +not an established latency, memory or context optimization or a universal optimum. + +[Issue #8978](https://github.com/morluto/rea/issues/8978) identifies Actual Budget +v26.11.0. We acquired eight public shipped maps from +[the public demo](https://demo.actualbudget.org/) without running the application. +These eight preload-selected maps total 18,319,814 bytes and 834,659 structural +records; they are not the complete eleven-map scope of the issue. Pinned URLs, +SHA-256 values, counts and independently decoded lookup expectations are in +[artifacts.json](https://github.com/morluto/rea/blob/main/docs/verification/source-map-budgets-2026-11-09/artifacts.json). + +The completed point task reads and verifies a selected generated script, analyst +manifest and map, runs the actual owned source codec, checks every equal-position +match and embedded-source digest, and creates and serializes complete Evidence. +The acquisition manifest is constructed and explicitly incomplete: it establishes +no browser capture, runtime execution or deployment authenticity. The independent +[Python VLQ oracle](https://github.com/morluto/rea/blob/main/docs/verification/source-map-budgets-2026-11-09/oracle.py) derives mapping positions +and counts directly from artifact bytes, separately from REA's decoder. + +## Admission-only measurements before compact delivery + +Each cell below is the median of three fresh-process runs. A baseline call uses +4 MiB admission and, when refused, retries the same complete map with 8 MiB. The +candidate selects 8 MiB on its first call. Other policies are identical. All +18 point runs exit successfully, verify the oracle and leave their process groups +empty. The baseline source snapshot and individual measurements/digests are in +[measurements.json](https://github.com/morluto/rea/blob/main/docs/verification/source-map-budgets-2026-11-09/measurements.json). + +| Map / workflow | Calls | Estimated task tokens | Wall s | CPU s | Sampled host-group RSS MiB | +| ---------------------------- | ----: | --------------------: | -----: | ----: | -------------------------: | +| Theme / default | 1 | 25,103 | 0.368 | 0.359 | 98.6 | +| Theme / 8 MiB | 1 | 25,126 | 0.367 | 0.391 | 99.3 | +| Main / default + retry | 2 | 1,590,278 | 0.553 | 0.573 | 174.0 | +| Main / 8 MiB | 1 | 1,589,746 | 0.539 | 0.559 | 181.5 | +| DateSelect / default + retry | 2 | 2,232,370 | 0.582 | 0.586 | 205.0 | +| DateSelect / 8 MiB | 1 | 2,231,835 | 0.599 | 0.617 | 192.4 | + +Tokens are tiktoken 0.12.0 `cl100k_base` estimates of compact UTF-8 JSON arguments +and complete replies, including failed calls and retries; they are not actual +model usage. Catalog discovery was not performed. Catalog/tool schemas, JSON-RPC +envelopes, network acquisition and actual model billing are unmeasured. Complete +Evidence retains raw map text: eliminating a retry saves only about 532–535 +estimated tokens against replies exceeding 1.5–2.2 million tokens. A map admission +cap is therefore not a suitable agent context budget. These measurements do not +qualify a smaller public reply projection or a live browser investigation. +Source inspection confirms that the production MCP handler records complete +Evidence before response encoding. Overflow recovery names its retained +reference and offers session export or complete CLI JSON; facts are not lost. +At the time of this admission-only experiment, local tracing had no summary, +paged matches view or native analysis-view continuation. This experiment did not +exercise a large reply through a live MCP client. The subsequent +[compact-delivery qualification](source-location-delivery.md) measures real +public MCP summaries, continuations and source-text selection separately. + +Node v24.22.1 ran on macOS 26.3 arm64. Workers/processes were fresh; OS caches were +not flushed, and no cold-cache comparison is claimed. Wall time includes the +observer. CPU is OS `wait4` user + +system usage for the waited subtree; orphaned/unwaited descendants and sampler +CPU are excluded. RSS is a simultaneous process-group sum sampled every 20 ms; +short peaks and descendants leaving the group can be missed and shared pages can +be counted repeatedly. In particular, the owned codec launches in a separate +group, so its peak is unmeasured in this older table; this is not whole-family +memory. The 1536 MiB observer threshold is an early-stop guard, +not a kernel memory limit. Mixed timing/memory differences do not establish a +performance improvement. + +## Unsuccessful full-fetch sizing + +An eight-map sizing attempt selected 128 MiB expanded output and 1,048,576 +records/segments. It generated a 123 MB full-Evidence file, but the observer's +signal race failed while the sampled process group exceeded its 1536 MiB guard. +No valid complete measurement receipt was produced; this is an unsuccessful, +unmeasured sizing attempt, not evidence of a safe completed task or a calibrated +fetch default. The observer was subsequently repaired. A small Theme fetch +control completed, but one control does not justify raising collection defaults. +Full-fetch output, records, heap and response-byte defaults stay conservative and +configurable. Local tracing retains its existing 262,144-record budget; the +largest measured individual map consumes 260,561 records, so broader map coverage +remains unqualified. + +## Reproduction + +No downloaded artifacts or complete large replies are committed. Publisher URLs +may change; acquisition refuses a digest mismatch. Use a separate baseline source +snapshot identified in `measurements.json`, then select the same explicit policies. +The retained driver uses production source through the pinned tsx loader and runs +the real source-owned codec. Fetch mode replays pinned HTTP artifact bytes through +the fetcher and includes schema/Evidence/serialization costs, while excluding +browser capture and generated-script analysis. + +```sh +python3 docs/verification/source-map-budgets-2026-11-09/acquire.py \ + docs/verification/source-map-budgets-2026-11-09/artifacts.json /tmp/rea-map-calibration +python3 docs/verification/source-map-budgets-2026-11-09/oracle.py \ + /tmp/rea-map-calibration/artifacts.json +node --import ./node_modules/tsx/dist/loader.mjs \ + docs/verification/source-map-budgets-2026-11-09/workload.mjs "$PWD" \ + '{"artifactDir":"/tmp/rea-map-calibration","mode":"point","select":"index.","strategy":"recover","policy":{"map_bytes":8388608},"resultPath":"/tmp/map-replies.json"}' +``` + +For candidate-first, omit `strategy`. Use `select: "theme."` or `"DateSelect."` +for the other point cases. Wrap each invocation with +[the process-group observer](https://github.com/morluto/rea/blob/main/scripts/verify/analysis/budget-calibration-run.py) for bounded process-group +observation; keep runs serial. After the default change, reconstruct the baseline +snapshot or explicitly select `map_bytes: 4194304` for the initial baseline call. diff --git a/docs/verification/source-map-budgets-2026-11-09/acquire.py b/docs/verification/source-map-budgets-2026-11-09/acquire.py new file mode 100644 index 0000000000..f684757601 --- /dev/null +++ b/docs/verification/source-map-budgets-2026-11-09/acquire.py @@ -0,0 +1,44 @@ +"""Acquire pinned public artifacts read-only and construct explicitly incomplete analyst manifests.""" +import hashlib +import json +from pathlib import Path +import sys +import urllib.request + +manifest_path, destination = map(Path, sys.argv[1:]) +destination.mkdir(parents=True, exist_ok=True) +manifest_bytes = manifest_path.read_bytes() +manifest = json.loads(manifest_bytes) +(destination / 'artifacts.json').write_bytes(manifest_bytes) +for item in manifest: + for artifact in [item, *([item['generated_script']] if 'generated_script' in item else [])]: + with urllib.request.urlopen(artifact['url'], timeout=30) as response: + data = response.read(16 * 1024 * 1024) + assert len(data) == artifact['bytes'] + assert hashlib.sha256(data).hexdigest() == artifact['sha256'], artifact['url'] + (destination / artifact['path']).write_bytes(data) + if 'generated_script' not in item: + continue + script = item['generated_script'] + (destination / 'files').mkdir(exist_ok=True) + (destination / 'files' / script['path']).write_bytes((destination / script['path']).read_bytes()) + context = { + 'capture_path': str((destination / 'artifacts.json').resolve()), + 'capture_sha256': hashlib.sha256(manifest_bytes).hexdigest(), + 'capture_kind': 'browser-scenario', + 'source_evidence_id': None, + 'capture_completeness': { + 'status': 'incomplete', 'equality_eligible': False, + 'missing_sections': ['action', 'screenshot', 'dom', 'accessibility', 'url', 'history', 'storage', 'events', 'frames', 'workers', 'popups', 'websockets'], + 'truncated_sections': [], + }, + 'output_directory': str(destination.resolve()), + 'analysis_input': {'input_path': str(destination.resolve()), 'format': 'directory'}, + 'limitations': ['Constructed analyst manifest over pinned public HTTP bytes. capture_path identifies the artifact manifest, not a browser capture. No capture, deployment authenticity or execution is asserted.'], + 'scripts': [{ + 'url': script['url'], + 'source': {'kind': 'scenario-response', 'transaction_id': 'read-only-public-http', 'request_sequence': 1, 'response_sequence': 2, 'status': 200}, + 'content': {'state': 'exported', 'relative_path': script['path'], 'layout': 'isolated', 'layout_reason': 'explicit-public-artifact-selection', 'sha256': script['sha256'], 'bytes': script['bytes'], 'media_type': 'text/javascript', 'redacted': False, 'representation': 'browser-decoded-response-bytes'}, + }], + } + (destination / (script['path'] + '.manifest.json')).write_text(json.dumps(context)) diff --git a/docs/verification/source-map-budgets-2026-11-09/artifacts.json b/docs/verification/source-map-budgets-2026-11-09/artifacts.json new file mode 100644 index 0000000000..7aab2128f6 --- /dev/null +++ b/docs/verification/source-map-budgets-2026-11-09/artifacts.json @@ -0,0 +1,252 @@ +[ + { + "url": "https://demo.actualbudget.org/static/js/index.CMrE7aQb.js.map", + "path": "index.CMrE7aQb.js.map", + "bytes": 4284329, + "sha256": "f0de6886f6a49758f53c7f2de0b77fbff6d72c719a7c32ac23d13453ebe7b3e8", + "sources": 790, + "names": 1050, + "generated_lines": 97, + "headers": { + "Content-Type": "application/octet-stream", + "Etag": "\"e9cabe1eb63edd2868b94f9e315c2b15-ssl\"" + }, + "segment_count": 201344, + "record_count": 204072, + "point": { + "line": 2, + "column": 15579 + }, + "expected_matches": [ + { + "source_index": 0, + "source": "../../../../../node_modules/absurd-sql/dist/indexeddb-main-thread.js", + "original_line": 10, + "original_column": 0, + "content_sha256": "d6bf964929e31ceeb301a174d85f4f58de4d4a73f65336488aac7b50fce0a1cd" + } + ], + "mapped_count": 201344, + "generated_script": { + "path": "index.CMrE7aQb.js", + "url": "https://demo.actualbudget.org/static/js/index.CMrE7aQb.js", + "bytes": 2078745, + "sha256": "937f792d1849b69a6e0741234428b22b4884ec0157ea31dd6949dd1bacc46456" + } + }, + { + "url": "https://demo.actualbudget.org/static/js/theme.EvD-JhP6.chunk.js.map", + "path": "theme.EvD-JhP6.chunk.js.map", + "bytes": 49264, + "sha256": "ceef8194eaca339e55f35cf2decf7ceffa121c7744ee1fffd976a07d39477b32", + "sources": 7, + "names": 0, + "generated_lines": 1, + "headers": { + "Content-Type": "application/octet-stream", + "Etag": "\"f9829244cb0a7c333e7bbe83b110c2ab-ssl\"" + }, + "segment_count": 2339, + "record_count": 2355, + "point": { + "line": 1, + "column": 1643 + }, + "expected_matches": [ + { + "source_index": 0, + "source": "../../../../../node_modules/react/cjs/react.production.js", + "original_line": 12, + "original_column": 0, + "content_sha256": "6b28feb9dbe9781ab1e1b64dea7b29a7c70f03d2dc10eaa966363053d8a9b4e5" + } + ], + "mapped_count": 2339, + "generated_script": { + "path": "theme.EvD-JhP6.chunk.js", + "url": "https://demo.actualbudget.org/static/js/theme.EvD-JhP6.chunk.js", + "bytes": 29718, + "sha256": "10ecf4c66ae103222a7c146ad9b945dc83e78b6c78ce05021296e3c2f799bd5b" + } + }, + { + "url": "https://demo.actualbudget.org/static/js/months.CGmeIa7I.chunk.js.map", + "path": "months.CGmeIa7I.chunk.js.map", + "bytes": 1630058, + "sha256": "f18a2a2f4d6360b944a8c57432e3f3898314f1cf39d69ba5d049233ab2f39e8d", + "sources": 267, + "names": 208, + "generated_lines": 7, + "headers": { + "Content-Type": "application/octet-stream", + "Etag": "\"8e5c8614ccacde6bd659f89916deba34-ssl\"" + }, + "segment_count": 65705, + "record_count": 66455, + "point": { + "line": 1, + "column": 230 + }, + "expected_matches": [ + { + "source_index": 0, + "source": "../../../src/build-shims.js", + "original_line": 1, + "original_column": 6, + "content_sha256": "1039d33f95ace766ed5731b2775e957198f2c9ad102989ca09cbd297d94b2863" + } + ], + "mapped_count": 65705 + }, + { + "url": "https://demo.actualbudget.org/static/js/DateSelect.BVYaCNUe.chunk.js.map", + "path": "DateSelect.BVYaCNUe.chunk.js.map", + "bytes": 5529840, + "sha256": "e1939ef36214ff46263d3bdaeb99d60a46c75714a6b6220ed27c9be2d24d6cc8", + "sources": 1001, + "names": 3893, + "generated_lines": 116, + "headers": { + "Content-Type": "application/octet-stream", + "Etag": "\"407e8912b56411e3b2b9a55824a6bb7b-ssl\"" + }, + "segment_count": 254549, + "record_count": 260561, + "point": { + "line": 1, + "column": 3866 + }, + "expected_matches": [ + { + "source_index": 0, + "source": "../../../../../node_modules/date-fns/isSameWeek.js", + "original_line": 40, + "original_column": 0, + "content_sha256": "436cf6fa9365c1cbf783a422c8da3a3b82e55edeac248f987cca22c0ba950902" + } + ], + "mapped_count": 254549, + "generated_script": { + "path": "DateSelect.BVYaCNUe.chunk.js", + "url": "https://demo.actualbudget.org/static/js/DateSelect.BVYaCNUe.chunk.js", + "bytes": 2191517, + "sha256": "9e45139dd0b510ad7014188a5e9d11b307c2ac15dfdc761050388e76e94759bf" + } + }, + { + "url": "https://demo.actualbudget.org/static/js/useLocalPref.CMSDl-Rj.chunk.js.map", + "path": "useLocalPref.CMSDl-Rj.chunk.js.map", + "bytes": 443520, + "sha256": "341c7d72797abdda122b935c0a5fc80718a79457e02d315fed2cf32f4bbd6f38", + "sources": 32, + "names": 9, + "generated_lines": 1, + "headers": { + "Content-Type": "application/octet-stream", + "Etag": "\"ffaaea8a6a08415032b4cba4f266624f-ssl\"" + }, + "segment_count": 8976, + "record_count": 9051, + "point": { + "line": 1, + "column": 352 + }, + "expected_matches": [ + { + "source_index": 0, + "source": "../../../../../node_modules/react-router/dist/production/lib/router/url.js", + "original_line": 12, + "original_column": 0, + "content_sha256": "493e1c3475d74a528af0c1aaae7574e841fe21019fb9e801dc8fe60f212570c7" + } + ], + "mapped_count": 8976 + }, + { + "url": "https://demo.actualbudget.org/static/js/Value.FdYRYOLz.chunk.js.map", + "path": "Value.FdYRYOLz.chunk.js.map", + "bytes": 4367424, + "sha256": "8428453a8a2aa4366a719e37f6aeb93e75f70aea2e8713529ec6f898d4cddcfa", + "sources": 538, + "names": 717, + "generated_lines": 52, + "headers": { + "Content-Type": "application/octet-stream", + "Etag": "\"6adb2ff5f177a9236937e770097bbeef-ssl\"" + }, + "segment_count": 181768, + "record_count": 183614, + "point": { + "line": 2, + "column": 7624 + }, + "expected_matches": [ + { + "source_index": 0, + "source": "../../../../loot-core/src/shared/query.ts", + "original_line": 22, + "original_column": 13, + "content_sha256": "e4b93dea21ab9d73af8eeffcee818935c00c9a10427549a643ac133be0dfc685" + } + ], + "mapped_count": 181768 + }, + { + "url": "https://demo.actualbudget.org/static/js/columns.BR9M2Oqz.chunk.js.map", + "path": "columns.BR9M2Oqz.chunk.js.map", + "bytes": 1549581, + "sha256": "39f0c008302908e9648f492f073570e76f03a978db25c7b527192e4936cc255f", + "sources": 346, + "names": 626, + "generated_lines": 53, + "headers": { + "Content-Type": "application/octet-stream", + "Etag": "\"c23d08378a207ac086f12075af6703b4-ssl\"" + }, + "segment_count": 83166, + "record_count": 84538, + "point": { + "line": 1, + "column": 843 + }, + "expected_matches": [ + { + "source_index": 0, + "source": "../../../../../node_modules/reselect/dist/reselect.mjs", + "original_line": 63, + "original_column": 0, + "content_sha256": "ce215e8758151f87b03d651c8ebdd097a8b658ffbe56e0162e0154bd00b92f12" + } + ], + "mapped_count": 83166 + }, + { + "url": "https://demo.actualbudget.org/static/js/TransactionEdit.C3L723rq.chunk.js.map", + "path": "TransactionEdit.C3L723rq.chunk.js.map", + "bytes": 465798, + "sha256": "cf04088c6542a020fadc284264cb3cb52763de9f352b2e630fbe10278ceab4c8", + "sources": 69, + "names": 227, + "generated_lines": 1, + "headers": { + "Content-Type": "application/octet-stream", + "Etag": "\"35e992601467cfecec49b2eeb6925d92-ssl\"" + }, + "segment_count": 23646, + "record_count": 24013, + "point": { + "line": 1, + "column": 6411 + }, + "expected_matches": [ + { + "source_index": 0, + "source": "../../../../../node_modules/react-aria-components/dist/private/Checkbox.mjs", + "original_line": 46, + "original_column": 6, + "content_sha256": "8319abfad740fe6665fc33f1ca8790251fb58d144497b856f7073c10eef62974" + } + ], + "mapped_count": 23646 + } +] diff --git a/docs/verification/source-map-budgets-2026-11-09/measurements.json b/docs/verification/source-map-budgets-2026-11-09/measurements.json new file mode 100644 index 0000000000..857e1f9107 --- /dev/null +++ b/docs/verification/source-map-budgets-2026-11-09/measurements.json @@ -0,0 +1,370 @@ +{ + "tokenizer": "tiktoken0.12.0 cl100k_base; compact UTF8 JSON args and complete replies; estimates not model usage; catalog and protocol unmeasured", + "rows": [ + { + "name": "point-r1-DateSelect-default-retry", + "elapsed_s": 0.5818042080063606, + "cpu_s": 0.586226, + "rss_bytes": 214908928, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 2, + "tokens": { + "arguments": 193, + "replies": 2232177 + }, + "reply_bytes": 5775790, + "measurement_receipt_sha256": "5f7c3a7b1bbcacb046f0caf136d41b6afa43a805d14f85365f265df9ac0015ae", + "reply_trace_sha256": "04b9aa60f9843ce5441ea30abf0c4d2e803a2764949ca9cf01136d5d720b424d" + }, + { + "name": "point-r1-DateSelect-map8", + "elapsed_s": 0.5986265420069685, + "cpu_s": 0.616543, + "rss_bytes": 203620352, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 102, + "replies": 2231733 + }, + "reply_bytes": 5774116, + "measurement_receipt_sha256": "85b6a677a2c542ca55ef156ed2cfb9a1af92fc6783921d22c118d0a28b2c4866", + "reply_trace_sha256": "ec9abc70ca71f5e91911147879f97df2d999ad2c4370e6c975b081577fdd5969" + }, + { + "name": "point-r1-index-default-retry", + "elapsed_s": 0.5493043329915963, + "cpu_s": 0.573376, + "rss_bytes": 184139776, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 2, + "tokens": { + "arguments": 181, + "replies": 1590097 + }, + "reply_bytes": 4428697, + "measurement_receipt_sha256": "cab96c90a933d2ccf8a854308b2de0b2593a678413afd5d95dd9d1628df277b0", + "reply_trace_sha256": "d613c618414f970b94b17ffee783768cedf45e3c414162dc4c408943522ac585" + }, + { + "name": "point-r1-index-map8", + "elapsed_s": 0.5393435839941958, + "cpu_s": 0.559231, + "rss_bytes": 190300160, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 96, + "replies": 1589650 + }, + "reply_bytes": 4427067, + "measurement_receipt_sha256": "abf76eb747e00c26eab5807944224ed55d5fd3b08bc4c8b147732e70180dceca", + "reply_trace_sha256": "abc7c820c6d627553f3e2d3549899d4ff7160271a0b6a937a5719d87420190bf" + }, + { + "name": "point-r1-theme-default-retry", + "elapsed_s": 0.6525920830026735, + "cpu_s": 0.72032, + "rss_bytes": 167591936, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 88, + "replies": 25015 + }, + "reply_bytes": 77129, + "measurement_receipt_sha256": "c108ab41544816f7706e6343584264bf3db938a97dfd51b52d4495f579efeb30", + "reply_trace_sha256": "d086b3579fbd099df1a32c7d1e54e2af6ea6a5b244747d1a012fa19ed541d4e5" + }, + { + "name": "point-r1-theme-map8", + "elapsed_s": 0.3669995829986874, + "cpu_s": 0.391014, + "rss_bytes": 104103936, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 99, + "replies": 25027 + }, + "reply_bytes": 77171, + "measurement_receipt_sha256": "146b3d877d36fea192e0442f3ae06d9e05cfbaf8c75902d8b1db914cea818cf2", + "reply_trace_sha256": "6a905573af8e55b34bbbd1618b48aec7d3cc12860f59643f28024002e81fcf00" + }, + { + "name": "point-r2-DateSelect-default-retry", + "elapsed_s": 0.5365884999919217, + "cpu_s": 0.5738759999999999, + "rss_bytes": 206454784, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 2, + "tokens": { + "arguments": 193, + "replies": 2232177 + }, + "reply_bytes": 5775790, + "measurement_receipt_sha256": "415deab64b1490e8300b63fa926a517f8630f68117c5735c126f7a8f8f02e635", + "reply_trace_sha256": "ae1055e47e9b2e3ef6ded1aa124950d220ba0bbe7a27afe68b9f46039299ce1e" + }, + { + "name": "point-r2-DateSelect-map8", + "elapsed_s": 0.5848246670066146, + "cpu_s": 0.5841609999999999, + "rss_bytes": 196968448, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 102, + "replies": 2231733 + }, + "reply_bytes": 5774116, + "measurement_receipt_sha256": "18c256638988915c3029f1cf6c19b1ce6cde5e2b7e046a93d47d0afbf5c74c6f", + "reply_trace_sha256": "ee27a03f5b3580d54ad8ad33bc7f22f552c724c9e1674f8c39aac4dd7b8fffab" + }, + { + "name": "point-r2-index-default-retry", + "elapsed_s": 0.5531810840038816, + "cpu_s": 0.5698139999999999, + "rss_bytes": 182484992, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 2, + "tokens": { + "arguments": 181, + "replies": 1590097 + }, + "reply_bytes": 4428697, + "measurement_receipt_sha256": "e97c8b561949f6e6615f9684e8187c7eb2a47833e836532891631b56ac26526a", + "reply_trace_sha256": "35a4f9f5f868083cd1c45767282e5d7555f07a2962fbdf647de538a07452ae02" + }, + { + "name": "point-r2-index-map8", + "elapsed_s": 0.4954192500008503, + "cpu_s": 0.53103, + "rss_bytes": 176865280, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 96, + "replies": 1589650 + }, + "reply_bytes": 4427067, + "measurement_receipt_sha256": "d9f3c788ee7de0508ba42f3a37fea9df282d195a634e4e523b673eed6f813b3a", + "reply_trace_sha256": "e19bbc0e4534d60d9577c8c5871243604439f1e84f6e6e1c5a763cf5001665f1" + }, + { + "name": "point-r2-theme-default-retry", + "elapsed_s": 0.36803100000543054, + "cpu_s": 0.358402, + "rss_bytes": 103415808, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 88, + "replies": 25015 + }, + "reply_bytes": 77129, + "measurement_receipt_sha256": "ad7ff37773f7a383a2ba1376863358451959a535d802c3300495dba4d8c0ef2e", + "reply_trace_sha256": "4b9afba56e44a75d3e1f674b70ddd2b030ace630ad8bf8bde2950cf866890880" + }, + { + "name": "point-r2-theme-map8", + "elapsed_s": 0.38679324999975506, + "cpu_s": 0.40970799999999996, + "rss_bytes": 104267776, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 99, + "replies": 25027 + }, + "reply_bytes": 77171, + "measurement_receipt_sha256": "3958afd848f3cc5529bb1a56f8ba8ea82eea34818713c8b045d327ea0aae9f65", + "reply_trace_sha256": "241cf2e478a29f069c1a17fdb5b14f35e3430ee9c276d818189a7de789608c89" + }, + { + "name": "point-r3-DateSelect-default-retry", + "elapsed_s": 0.6513657079922268, + "cpu_s": 0.613527, + "rss_bytes": 220790784, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 2, + "tokens": { + "arguments": 193, + "replies": 2232177 + }, + "reply_bytes": 5775790, + "measurement_receipt_sha256": "19e779011951101b141f48e84b427161bf86f41998e79a2fb368d9e129aa6b00", + "reply_trace_sha256": "7d1916049662bf0a278ef0bf341874e35a317182846d4ac4eca1a09a166a3e58" + }, + { + "name": "point-r3-DateSelect-map8", + "elapsed_s": 0.6560826670029201, + "cpu_s": 0.686476, + "rss_bytes": 201785344, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 102, + "replies": 2231733 + }, + "reply_bytes": 5774116, + "measurement_receipt_sha256": "32af3c6720329c991dc434844878aba283e0c6001087ed0f93a1bfaabbdecc67", + "reply_trace_sha256": "5feb55a1ff8c433f5dee38a7c3ae877b9260d1fbd0d5ef8501ed38d8e5d46d60" + }, + { + "name": "point-r3-index-default-retry", + "elapsed_s": 0.5955084579909453, + "cpu_s": 0.6005269999999999, + "rss_bytes": 182140928, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 2, + "tokens": { + "arguments": 181, + "replies": 1590097 + }, + "reply_bytes": 4428697, + "measurement_receipt_sha256": "ab049ddfe685398068198b13ed317eb6c51aff104d88d9d11a0cc7dab6fab900", + "reply_trace_sha256": "7fd9f40f0e16debbffb65c9b5bff8043e586b7aa493daf376eefb0348676fd1e" + }, + { + "name": "point-r3-index-map8", + "elapsed_s": 0.6677014579909155, + "cpu_s": 0.6370119999999999, + "rss_bytes": 194920448, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 96, + "replies": 1589650 + }, + "reply_bytes": 4427067, + "measurement_receipt_sha256": "fa78f3707dda0387d07a8cfee9f4591d64d5a79b49f097c4501de143e32f89e5", + "reply_trace_sha256": "184a5536859edd8e990e89a785048200847aff85de28008fb6f972ec3455e491" + }, + { + "name": "point-r3-theme-default-retry", + "elapsed_s": 0.3368863329960732, + "cpu_s": 0.358882, + "rss_bytes": 102940672, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 88, + "replies": 25015 + }, + "reply_bytes": 77129, + "measurement_receipt_sha256": "ab076949d4ba9ffc11e1f677e6af805d8dae976ee642a1344fdf252c710c5ec6", + "reply_trace_sha256": "ae3742c091752393a0fae881f1203287fdbb2bde08838edf989e809ae278e246" + }, + { + "name": "point-r3-theme-map8", + "elapsed_s": 0.3349004170013359, + "cpu_s": 0.36136099999999993, + "rss_bytes": 103153664, + "exit_code": 0, + "cleanup": "group_empty", + "calls": 1, + "tokens": { + "arguments": 99, + "replies": 25027 + }, + "reply_bytes": 77171, + "measurement_receipt_sha256": "eff22ab422018535d88a0e0d3c135139a6930a18a24a3d32f9c68defaeae8dce", + "reply_trace_sha256": "4af542e52f3377ad3410a069a60c64cf4db76771a0bc686075ebea31c2733032" + } + ], + "medians": { + "DateSelect-default-retry": { + "runs": 3, + "elapsed_s": 0.5818042080063606, + "cpu_s": 0.586226, + "rss_mib": 204.953125, + "calls": 2, + "argument_tokens": 193, + "reply_tokens": 2232177, + "reply_bytes": 5775790 + }, + "DateSelect-map8": { + "runs": 3, + "elapsed_s": 0.5986265420069685, + "cpu_s": 0.616543, + "rss_mib": 192.4375, + "calls": 1, + "argument_tokens": 102, + "reply_tokens": 2231733, + "reply_bytes": 5774116 + }, + "index-default-retry": { + "runs": 3, + "elapsed_s": 0.5531810840038816, + "cpu_s": 0.573376, + "rss_mib": 174.03125, + "calls": 2, + "argument_tokens": 181, + "reply_tokens": 1590097, + "reply_bytes": 4428697 + }, + "index-map8": { + "runs": 3, + "elapsed_s": 0.5393435839941958, + "cpu_s": 0.559231, + "rss_mib": 181.484375, + "calls": 1, + "argument_tokens": 96, + "reply_tokens": 1589650, + "reply_bytes": 4427067 + }, + "theme-default-retry": { + "runs": 3, + "elapsed_s": 0.36803100000543054, + "cpu_s": 0.358882, + "rss_mib": 98.625, + "calls": 1, + "argument_tokens": 88, + "reply_tokens": 25015, + "reply_bytes": 77129 + }, + "theme-map8": { + "runs": 3, + "elapsed_s": 0.3669995829986874, + "cpu_s": 0.391014, + "rss_mib": 99.28125, + "calls": 1, + "argument_tokens": 99, + "reply_tokens": 25027, + "reply_bytes": 77171 + } + }, + "baseline": { + "baseline_worktree": "/var/folders/xl/60j5sd4j6b99tn8q082l9t8r0000gn/T/rea-budget-calibration-baseline-2rg1qcde", + "head": "e2a2ede5440df396e44b3406ef511a31a680290e", + "tracked_patch_sha256": "038b69c19bbd79a303c9ecd8d9d22a1251637e3886543ca4f5a4c1b70c724eec", + "source_map_policy_sha256": "6d28cba1ff7936dee640474a4d5e1e456a3acb3c63432ea6ab4698181a53ca05" + }, + "conditions": { + "host": "macOS26.3 arm64", + "node": "v24.22.1", + "workers": "fresh process and owned codec each run; OS caches warm, not flushed", + "rss": "sampled simultaneous process-group RSS sum at20ms, shared pages may count twice; short peaks and descendants leaving group excluded", + "cpu": "OS wait4 user+system waited subtree; orphaned or unwaited descendants excluded; sampler excluded", + "guard": "1536MiB observed early stop, not kernel hard limit", + "catalog": "not counted; no catalog discovery in replay; tool schemas, JSON-RPC envelopes, actual billed tokens, network, browser capture, and generated-script analysis unmeasured" + } +} diff --git a/docs/verification/source-map-budgets-2026-11-09/oracle.py b/docs/verification/source-map-budgets-2026-11-09/oracle.py new file mode 100644 index 0000000000..e0cafebfcb --- /dev/null +++ b/docs/verification/source-map-budgets-2026-11-09/oracle.py @@ -0,0 +1,62 @@ +"""Independent source-map VLQ artifact oracle; standard library only.""" +import hashlib +import json +from pathlib import Path + +ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + +def values(segment): + output, value, shift = [], 0, 0 + for character in segment: + digit = ALPHABET.index(character) + value += (digit & 31) << shift + if digit & 32: + shift += 5 + else: + output.append(-(value >> 1) if value & 1 else value >> 1) + value, shift = 0, 0 + assert shift == 0 + return output + +def inspect(path): + data = Path(path).read_bytes() + raw = json.loads(data) + source = original_line = original_column = 0 + count, mapped_count, first, matches = 0, 0, None, [] + for line, row in enumerate(raw['mappings'].split(';'), 1): + column = 0 + for segment in row.split(','): + if not segment: + continue + count += 1 + fields = values(segment) + column += fields[0] + if len(fields) > 1: + assert len(fields) in (4, 5) + mapped_count += 1 + source += fields[1] + original_line += fields[2] + original_column += fields[3] + if first is None and len(fields) > 1: + first = {'line': line, 'column': column} + if first == {'line': line, 'column': column}: + if len(fields) == 1: + matches.append({'generated_only': True}) + else: + text = raw['sourcesContent'][source] + matches.append({'source_index':source,'source':raw['sources'][source], + 'original_line':original_line+1,'original_column':original_column, + 'content_sha256':hashlib.sha256(text.encode()).hexdigest() if text is not None else None}) + lines = raw['mappings'].count(';')+1 + return {'sha256':hashlib.sha256(data).hexdigest(), 'bytes':len(data), + 'segment_count':count, 'mapped_count':mapped_count, 'record_count':1+len(raw['sources'])+len(raw['names'])+len(raw['sourcesContent'])+lines+count, + 'point':first,'expected_matches':matches} + +if __name__ == '__main__': + import sys + manifest = json.loads(Path(sys.argv[1]).read_text()) + for source in manifest: + observed = inspect(Path(sys.argv[1]).resolve().parent / source['path']) + for key, value in observed.items(): + assert source[key] == value, (source['path'], key, source[key], value) + print(json.dumps({'verified_maps':len(manifest), 'sha256s':[source['sha256'] for source in manifest]})) diff --git a/docs/verification/source-map-budgets-2026-11-09/workload.mjs b/docs/verification/source-map-budgets-2026-11-09/workload.mjs new file mode 100644 index 0000000000..0e37e5bb11 --- /dev/null +++ b/docs/verification/source-map-budgets-2026-11-09/workload.mjs @@ -0,0 +1,288 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; +const [projectRoot, configText] = process.argv.slice(2); +const config = JSON.parse(configText); +const require = createRequire(join(projectRoot, "package.json")); +assert( + config.artifactDir, + "Select artifactDir containing the acquired artifacts.json", +); +const manifest = JSON.parse( + await readFile(join(config.artifactDir, "artifacts.json"), "utf8"), +).map((source) => ({ ...source, path: join(config.artifactDir, source.path) })); +const selected = + config.select === "all" + ? manifest + : manifest.filter((x) => x.path.includes(config.select)); +assert(selected.length > 0); +const load = (path) => import(pathToFileURL(join(projectRoot, path)).href); +const { projectAnalysisError } = await load( + "src/domain/analysisErrorProjection.ts", +); +const calls = []; +let finalResult; +const digest = (value) => createHash("sha256").update(value).digest("hex"); +const start = performance.now(); +if (config.mode === "point") { + const { SourceMapDecoder } = await load( + "src/javascript/sourceMaps/SourceMapDecoder.ts", + ); + const { WebSourceLocationService } = await load( + "src/application/WebSourceLocationService.ts", + ); + const { LocalWebSourceLocationArtifacts } = await load( + "src/browser/assets/WebSourceLocationArtifacts.ts", + ); + const { spawnOwnedProviderProcess } = await load( + "src/process/ProviderProcess.ts", + ); + const decoder = new SourceMapDecoder({ + launcher: (options) => + spawnOwnedProviderProcess({ + ...options, + arguments: [ + "--import", + require.resolve("tsx"), + ...options.arguments.map((x) => + x.endsWith("/SourceMapCodecProcess.js") + ? join( + projectRoot, + "src/javascript/sourceMaps/SourceMapCodecProcess.ts", + ) + : x, + ), + ], + }), + }); + const service = new WebSourceLocationService( + new LocalWebSourceLocationArtifacts(), + decoder, + ); + try { + for (const source of selected) { + const text = await readFile(source.path, "utf8"); + const policies = + config.strategy === "recover" + ? [{}, config.policy] + : [config.policy ?? {}]; + let completed = false; + for (const policy of policies) { + const args = { + manifest_path: source.path.slice(0, -4) + ".manifest.json", + script_index: 0, + source_map: { path: source.path, url: source.url }, + generated_position: source.point, + ...(Object.keys(policy).length ? { source_map_limits: policy } : {}), + }; + const began = performance.now(); + const result = await service.trace(args); + calls.push({ + operation: "trace_web_source_location", + args, + reply: result.ok ? result.value : projectAnalysisError(result.error), + elapsed_ms: performance.now() - began, + }); + if (!result.ok) continue; + const report = result.value.normalized_result; + assert.equal(report.source_map_sha256, source.sha256); + assert.equal(report.matches.length, source.expected_matches.length); + for (let i = 0; i < source.expected_matches.length; i++) { + const expected = source.expected_matches[i], + actual = report.matches[i]; + assert.equal( + actual.state, + expected.generated_only ? "generated-only" : "mapped", + ); + if (expected.generated_only) continue; + assert.equal(actual.source_index, expected.source_index); + assert.equal(actual.reported_source, expected.source); + assert.equal(actual.original_position.line, expected.original_line); + assert.equal( + actual.original_position.column, + expected.original_column, + ); + assert.equal(actual.content.state, "embedded"); + assert.equal(actual.content.utf8_sha256, expected.content_sha256); + } + finalResult = result.value; + completed = true; + break; + } + assert( + completed, + `point incomplete for ${source.path}: ${JSON.stringify(calls.at(-1)?.reply)}`, + ); + } + } finally { + await service.close(); + } +} else { + const { fetchWebSourceMaps } = await load( + "src/browser/WebSourceMapFetcher.ts", + ); + const { analyzeWebBundleInputSchema, webBundleAnalysisSchema } = await load( + "src/domain/webBundleAnalysis.ts", + ); + const { jsonValueSchema } = await load("src/domain/jsonValue.ts"); + const { createBrowserEvidence } = await load( + "src/application/BrowserEvidence.ts", + ); + const requests = selected.map((s, i) => ({ + scriptKey: `scr_${String(i + 1).padStart(64, "0")}`, + declaredUrl: s.url, + fetchUrl: s.url, + })); + const policies = + config.strategy === "recover" ? [{}, config.policy] : [config.policy ?? {}]; + const byUrl = new Map(selected.map((s) => [s.url, s])); + let completed = false; + for (const policy of policies) { + const input = analyzeWebBundleInputSchema.parse({ + cdp_endpoint: "http://127.0.0.1:9222", + allowed_origins: ["https://demo.actualbudget.org"], + target_id: "calibration-artifact-replay", + fetch_source_maps: true, + ...(Object.keys(policy).length ? { source_map_limits: policy } : {}), + }); + const began = performance.now(); + const result = await fetchWebSourceMaps(requests, input, undefined, { + fetch: async (url) => + new Response(await readFile(byUrl.get(String(url)).path), { + status: 200, + headers: { "content-type": "application/source-map+json" }, + }), + }); + let reply = result.ok ? result.value : projectAnalysisError(result.error); + if (result.ok && config.evidence !== false) { + // Isolate source-map retention cost: script capture/analysis are explicitly outside this replay. + const bundle = webBundleAnalysisSchema.parse({ + capture: { + target_url: "https://demo.actualbudget.org/", + target_origin: "https://demo.actualbudget.org", + scripts_observed: 0, + scripts_analyzed: 0, + source_artifacts: [], + }, + observations: { + chunks: { nodes: [], edges: [] }, + routes: [], + endpoints: [], + webmcp_declarations: [], + source_maps: result.value, + }, + inferences: [], + unknowns: [], + completeness: { + status: result.value.status === "included" ? "complete" : "partial", + parsed_scripts: 0, + parse_failures: 0, + visited_ast_nodes: 0, + }, + limitations: [ + "Pinned public source-map transport replay. Browser capture and generated-script static analysis were not performed.", + ], + }); + jsonValueSchema.parse(bundle); + const evidence = createBrowserEvidence( + "analyze_web_bundle", + input, + bundle, + { + id: "source-map-calibration-replay", + name: "Pinned public source-map replay", + version: "1", + }, + ); + // Include actual Evidence envelope cost rather than only the producer report. + reply = evidence; + } + calls.push({ + operation: "analyze_web_bundle.source_maps", + args: { + source_urls: selected.map((s) => s.url), + source_sha256s: selected.map((s) => s.sha256), + ...(Object.keys(policy).length ? { source_map_limits: policy } : {}), + }, + reply, + elapsed_ms: performance.now() - began, + }); + if ( + !result.ok || + result.value.items.length !== selected.length || + result.value.items.some( + (item) => !["included", "partial"].includes(item.status), + ) + ) + continue; + for (let i = 0; i < selected.length; i++) { + const original = selected[i], + item = result.value.items[i], + raw = JSON.parse(await readFile(original.path, "utf8")); + assert(["included", "partial"].includes(item.status)); + assert.equal(item.mappings.length, original.mapped_count); + assert.equal(item.artifact.sha256, original.sha256); + assert.equal(item.original_sources.length, raw.sources.length); + const expected = original.expected_matches.filter( + (x) => !x.generated_only, + ); + const actual = item.mappings.filter( + (x) => + x.generated_line === original.point.line && + x.generated_column === original.point.column, + ); + assert.equal(actual.length, expected.length); + for (let j = 0; j < expected.length; j++) { + assert.equal(actual[j].source_index, expected[j].source_index); + assert.equal(actual[j].original_line, expected[j].original_line); + assert.equal(actual[j].original_column, expected[j].original_column); + } + for (let j = 0; j < raw.sources.length; j++) { + assert.equal(item.original_sources[j].source_index, j); + if (raw.sourcesContent[j] !== null) + assert.equal( + item.original_sources[j].artifact.sha256, + digest(raw.sourcesContent[j]), + ); + } + } + finalResult = result.value; + completed = true; + break; + } + if (!completed && config.allowIncomplete !== true) + throw Error( + `fetch incomplete: ${JSON.stringify(calls.at(-1)?.reply?.items?.map((i) => ({ status: i.status, limitation: i.limitation })))}`, + ); +} +if (config.resultPath) + await writeFile(config.resultPath, JSON.stringify({ config, calls })); +const rows = calls.map((c) => { + const report = + c.reply.normalized_result?.observations?.source_maps ?? c.reply; + return { + operation: c.operation, + elapsed_ms: c.elapsed_ms, + reply_bytes: Buffer.byteLength(JSON.stringify(c.reply)), + state: report.status ?? (c.reply.code ? "error" : "complete"), + error: c.reply.code ? c.reply.message : undefined, + items: report.items?.map((i) => ({ + status: i.status, + sources: i.original_sources?.length, + mappings: i.mappings?.length, + limitation: i.limitation, + })), + }; +}); +console.log( + JSON.stringify({ + config, + operation_ms: performance.now() - start, + calls: rows, + result_sha256: finalResult ? digest(JSON.stringify(finalResult)) : null, + resource_usage: process.resourceUsage(), + }), +); diff --git a/scripts/fixtures/generate-mitmproxy-capture.py b/scripts/fixtures/generate-mitmproxy-capture.py index cf539a65f5..7bea41228e 100644 --- a/scripts/fixtures/generate-mitmproxy-capture.py +++ b/scripts/fixtures/generate-mitmproxy-capture.py @@ -30,31 +30,31 @@ class Generator: assert "payload must not enter diagnostics" not in json.dumps(reply) for state, expected in [({"metadata": {1: b"bad-key"}}, "format"), ({"metadata": object()}, "decoder")]: try: - adapter.project_record(state, []) + adapter.project_record(state, [], 64) except adapter.CaptureFailure as error: assert error.reason == expected, (error.reason, expected) assert error.pointer == "/metadata", error.pointer else: raise AssertionError("Invalid native representation was accepted") - non_utf8 = adapter.project_record({"request": {"headers": [(b"Referer", b"https://byte-user:byte-secret@example.test/path\xff")] }}, []) + non_utf8 = adapter.project_record({"request": {"headers": [(b"Referer", b"https://byte-user:byte-secret@example.test/path\xff")] }}, [], 64) assert non_utf8["reported"]["request"]["headers"][0][1] is None, non_utf8 assert non_utf8["binary_fields"][-1]["state"] == "redacted", non_utf8 for value in [" \tHTTPS://ows-user:ows-secret@example.test/path\t ", b" \tHTTPS://ows-user:ows-secret@example.test/path\t "]: - projected = adapter.project_record({"request": {"headers": [(b"Referer", value)]}}, []) + projected = adapter.project_record({"request": {"headers": [(b"Referer", value)]}}, [], 64) assert projected["reported"]["request"]["headers"][0][1] == " \tHTTPS://example.test/path\t ", projected assert "ows-secret" not in json.dumps(projected), projected if isinstance(value, bytes): assert projected["binary_fields"][-1]["state"] == "redacted", projected body = b"A" * 12345 state = {"response": {"content": body}} - control = adapter.project_record(state, []) + control = adapter.project_record(state, [], 64) field = control["binary_fields"][0] assert field["state"] == "retained" and field["bytes"] == len(body), control for literal in ["12345", "2345"]: # Only the generated byte count matches this exclusion. assert literal not in control["reported"]["response"]["content"] assert literal not in field["content_base64"] and literal not in field["sha256"] - projected = adapter.project_record(state, [literal]) + projected = adapter.project_record(state, [literal], 64) assert projected["reported"]["response"]["content"] is None, projected assert projected["binary_fields"] == [{"pointer": "/response/content", "representation": "producer-bytes", "state": "redacted", "content_base64": None, "bytes": None, "sha256": None}], projected assert {"pointer": "/response/content", "reason": "explicit-sensitive-value"} in projected["redactions"], projected diff --git a/scripts/verify-real-ghidra-decompile-timeout.mjs b/scripts/verify-real-ghidra-decompile-timeout.mjs index 5e7f33448e..1378990fdd 100644 --- a/scripts/verify-real-ghidra-decompile-timeout.mjs +++ b/scripts/verify-real-ghidra-decompile-timeout.mjs @@ -24,41 +24,71 @@ import { createVerifierRun, } from "./lib/verifier-run.mjs"; -// A natural slow decompile: one function with many independent branches on -// volatile state. Ghidra decompiles it correctly but takes far longer than the -// configured limit, so the limit, not an injected result, stops it. +// Volatile straight-line work produces a natural native response above 1 MiB +// and a decompile above one second, without the old branch-heavy fixture. +// Admit the generated source before allocating its line array. const LIMIT_SECONDS = 1; -// This verifier budgets one MiB for its generated C source. Bound the number -// of lines before allocating them, including the largest admitted integer's -// decimal representation; the joined source and line array remain bounded. +const RECOVERY_SECONDS = 20; const FIXTURE_SOURCE_BYTES = 1024 * 1024; const FIXTURE_PREFIX = [ - "volatile unsigned rea_state[64];", + "volatile unsigned rea_state[128];", + "__attribute__((noinline, used)) unsigned rea_small(unsigned x) {", + " unsigned a = x + rea_state[0];", + " unsigned b = a ^ rea_state[1];", + " return b + (a * 3u);", + "}", "__attribute__((noinline, used)) unsigned rea_slow(unsigned x) {", " unsigned a = x;", ]; -const FIXTURE_SUFFIX = [ - " return a;", - "}", - "__attribute__((noinline, used)) int rea_small(int v) {", - " return v + (int)rea_state[0];", - "}", - "", -]; -const MAX_BRANCH_LINE_BYTES = Buffer.byteLength( - ` if (rea_state[63] & 1073741824u) a = a * 31u + ${Number.MAX_SAFE_INTEGER}u; else a ^= rea_state[63];\n`, +const FIXTURE_SUFFIX = [" return a;", "}", ""]; +const MAX_STATEMENT_BYTES = Buffer.byteLength( + ` a += rea_state[127] ^ (x + ${Number.MAX_SAFE_INTEGER}u); rea_state[127] = a;\n`, ); const FIXTURE_FIXED_BYTES = Buffer.byteLength( [...FIXTURE_PREFIX, ...FIXTURE_SUFFIX].join("\n"), ); -const MAX_BRANCHES = Math.floor( - (FIXTURE_SOURCE_BYTES - FIXTURE_FIXED_BYTES) / MAX_BRANCH_LINE_BYTES, +const MAX_STATEMENTS = Math.floor( + (FIXTURE_SOURCE_BYTES - FIXTURE_FIXED_BYTES) / MAX_STATEMENT_BYTES, ); -const BRANCHES = Number(process.env.REA_DECOMPILE_TIMEOUT_BRANCHES ?? 6000); +const STATEMENTS = Number(process.env.REA_DECOMPILE_TIMEOUT_STATEMENTS ?? 5400); assert.ok( - Number.isSafeInteger(BRANCHES) && BRANCHES > 0 && BRANCHES <= MAX_BRANCHES, - `REA_DECOMPILE_TIMEOUT_BRANCHES must be an integer from 1 to ${MAX_BRANCHES} (generated source budget: ${FIXTURE_SOURCE_BYTES} bytes)`, + Number.isSafeInteger(STATEMENTS) && + STATEMENTS > 0 && + STATEMENTS <= MAX_STATEMENTS, + `REA_DECOMPILE_TIMEOUT_STATEMENTS must be an integer from 1 to ${MAX_STATEMENTS} (generated source budget: ${FIXTURE_SOURCE_BYTES} bytes)`, ); +const fixtureSource = () => { + const lines = [...FIXTURE_PREFIX]; + for (let index = 0; index < STATEMENTS; index += 1) + lines.push( + ` a += rea_state[${index % 128}] ^ (x + ${index}u); rea_state[${(index * 7) % 128}] = a;`, + ); + lines.push(...FIXTURE_SUFFIX); + const source = lines.join("\n"); + assert.ok(Buffer.byteLength(source) <= FIXTURE_SOURCE_BYTES); + return source; +}; +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const value = (result) => { + if (!result.ok) throw result.error; + return result.value; +}; +const flowSummary = (flow) => ({ + available: flow.available, + operations: flow.operations?.length, + largest_input_count: flow.operations?.reduce( + (largest, operation) => Math.max(largest, operation.inputs.length), + 0, + ), + def_use_edges: flow.def_use?.length, + parameter_uses: flow.parameter_uses?.length, + truncated: flow.truncated, + limits: flow.limits, + limits_reached: flow.limits_reached, + omitted_operations_lower_bound: flow.omitted_operations_lower_bound, + known_omitted_inputs: flow.known_omitted_inputs, + known_omitted_edges: flow.known_omitted_edges, +}); const run = createVerifierRun(); assert.ok( @@ -67,36 +97,30 @@ assert.ok( ); const environment = { ...process.env, - REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: String(LIMIT_SECONDS), + GHIDRA_HEADLESS_MAXMEM: + process.env.GHIDRA_HEADLESS_MAXMEM || process.env.GHIDRA_MAXMEM || "1G", + REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: String(RECOVERY_SECONDS), }; const settings = new AnalysisSettings(environment); -const config = parseConfig(environment); -if (!config.ok) throw config.error; -assert.equal(config.value.ghidraDecompileTimeoutSeconds, LIMIT_SECONDS); +const config = value(parseConfig(environment)); const installation = await inspectVerifierGhidraInstallation({ environment, - installDir: config.value.ghidraInstallDir, - javaHome: config.value.ghidraJavaHome, + installDir: config.ghidraInstallDir, + javaHome: config.ghidraJavaHome, }); assert.equal(installation.status, "available", JSON.stringify(installation)); assert.equal(installation.providerVersion, SUPPORTED_GHIDRA_VERSION); -const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); -const fixtureSource = () => { - const lines = [...FIXTURE_PREFIX]; - for (let index = 0; index < BRANCHES; index += 1) - lines.push( - ` if (rea_state[${index % 64}] & ${(1 << (index % 31)) >>> 0}u) a = a * 31u + ${index}u; else a ^= rea_state[${(index * 7) % 64}];`, - ); - lines.push(...FIXTURE_SUFFIX); - const source = lines.join("\n"); - assert.ok(Buffer.byteLength(source) <= FIXTURE_SOURCE_BYTES); - return source; -}; - const workspace = await mkdtemp( join(tmpdir(), "rea-ghidra-decompile-timeout-"), ); +const began = performance.now(); +const controller = new AbortController(); +const outerDeadline = setTimeout( + () => controller.abort(new Error("Ghidra verifier exceeded 180000 ms")), + 180_000, +); +outerDeadline.unref(); let cleaned = true; let report; try { @@ -124,7 +148,11 @@ try { try { await runOwnedCommand( { ...compiler, runId: run.run_id, expectedCommand: null }, - { timeoutMs: 300_000, diagnosticBytes: 1024 * 1024 }, + { + timeoutMs: 30_000, + diagnosticBytes: 64 * 1024, + signal: controller.signal, + }, ); cleaned = true; } catch (cause) { @@ -133,17 +161,13 @@ try { throw cause; } const targetDigest = digest(await readFile(targetPath)); - - // One real provider session over the read-only authenticated TCP bridge. let bridgeClient; const provider = new GhidraProvider( - runtimeAnalysisConfig(config.value, settings.read), + runtimeAnalysisConfig(config, settings.read), silentLogger, environment, undefined, (options) => { - // This verifier has no caller-selected run ID: each new launch receives - // a fresh owner-generated identity, so equality detects session restarts. assert.equal(options.runId, undefined); bridgeClient = new GhidraClient({ ...options, @@ -152,113 +176,196 @@ try { return bridgeClient; }, ); - const target = await parseBinaryTarget(targetPath); - if (!target.ok) throw target.error; - const profile = await provider.resolveAnalysisProfile(target.value); - if (!profile.ok) throw profile.error; - assert.ok(profile.value.profile); - const client = provider.createClient(target.value, profile.value.profile); - cleaned = false; + const target = value(await parseBinaryTarget(targetPath)); + const profile = value(await provider.resolveAnalysisProfile(target)); + assert.ok(profile.profile); + const client = provider.createClient(target, profile.profile); + const execute = (operation, input) => + settings.run(() => + client.execute(operation, input, { signal: controller.signal }), + ); + const update = (changes) => + value( + settings.update({ + changes: changes.map(([setting, selected]) => ({ + setting, + action: "set", + value: selected, + })), + }), + ); + const observed = { stages: [] }; + const record = (stage, facts) => observed.stages.push({ stage, ...facts }); const failures = []; - const observed = {}; + cleaned = false; try { - // Mach-O symbols carry a leading underscore; address the functions by - // their entry instead of guessing the recovered name. - const listed = await client.execute("list_procedures", {}); - if (!listed.ok) throw listed.error; + const listed = value(await execute("list_procedures", {})); const entry = (name) => { - const found = listed.value.result.find( + const found = listed.result.find( (item) => item.value === name || item.value === `_${name}`, ); assert.ok(found, `Missing fixture function ${name}`); return found.address; }; - const slowEntry = entry("rea_slow"); const smallEntry = entry("rea_small"); - assert.ok(bridgeClient); - const initialSession = await bridgeClient.ping(); - if (!initialSession.ok) throw initialSession.error; - assert.equal(initialSession.value.target.sha256, targetDigest); - observed.initial_session_run_id = initialSession.value.run_id; - const limitText = `exceeded the ${String(LIMIT_SECONDS)}-second configured decompilation limit (ghidra.decompile_timeout_seconds; launch seed REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS)`; + const slowEntry = entry("rea_slow"); + const initial = value( + await bridgeClient.ping({ signal: controller.signal }), + ); + assert.equal(initial.target.sha256, targetDigest); + observed.initial_session_run_id = initial.run_id; + const flow = async () => + value(await execute("analyze_function", { procedure: smallEntry })).result + .native_value_flow; + + update([["ghidra.value_flow_max_operations", 1]]); + const operationsLow = await flow(); + assert.equal(operationsLow.operations.length, 1); + assert.ok(operationsLow.limits_reached.includes("max_operations")); + record("operations_low", flowSummary(operationsLow)); + update([["ghidra.value_flow_max_operations", 3000]]); + const complete = await flow(); + assert.equal(complete.truncated, false); + assert.ok(complete.operations.length > 2); + record("operations_recovered", flowSummary(complete)); + + update([["ghidra.value_flow_max_inputs_per_operation", 1]]); + const inputsLow = await flow(); + assert.ok(inputsLow.known_omitted_inputs > 0); + assert.ok(inputsLow.limits_reached.includes("max_inputs_per_operation")); + record("inputs_low", flowSummary(inputsLow)); + update([["ghidra.value_flow_max_inputs_per_operation", 64]]); + const inputsRecovered = await flow(); + assert.equal(inputsRecovered.truncated, false); + record("inputs_recovered", flowSummary(inputsRecovered)); + + update([["ghidra.value_flow_max_edges", 1]]); + const edgesLow = await flow(); + assert.ok(edgesLow.known_omitted_edges > 0); + assert.ok(edgesLow.limits_reached.includes("max_edges")); + record("edges_low", flowSummary(edgesLow)); + update([["ghidra.value_flow_max_edges", 12000]]); + const edgesRecovered = await flow(); + assert.equal(edgesRecovered.truncated, false); + assert.ok(edgesRecovered.def_use.length > 1); + record("edges_recovered", flowSummary(edgesRecovered)); + + update([["ghidra.value_flow_max_operations", 2]]); + const lowered = await flow(); + assert.equal(lowered.operations.length, 2); + assert.ok(lowered.limits_reached.includes("max_operations")); + record("operations_lowered", flowSummary(lowered)); + update([ + ["ghidra.value_flow_max_operations", 3000], + ["ghidra.decompile_max_payload_mib", 1], + ]); + let started = performance.now(); - const slow = await client.execute("analyze_function", { + const payloadLow = await execute("procedure_pseudo_code", { procedure: slowEntry, }); - observed.analyze_seconds = (performance.now() - started) / 1000; - if (!slow.ok) throw slow.error; - const dossier = slow.value.result; - assert.equal(dossier.facet_availability.pseudocode.available, false); - assert.ok( - dossier.facet_availability.pseudocode.reason.endsWith(limitText), - dossier.facet_availability.pseudocode.reason, + assert.equal( + payloadLow.ok, + false, + "Fixture did not exhaust the 1 MiB payload allowance; calibrate REA_DECOMPILE_TIMEOUT_STATEMENTS rather than reporting a provider failure.", ); + const payloadError = projectAnalysisError(payloadLow.error); + assert.ok( + payloadError.message.includes("Response buffer size exceeded"), + payloadError.message, + ); + assert.ok( + payloadError.message.includes("ghidra.decompile_max_payload_mib"), + ); + record("payload_1_mib", { + seconds: (performance.now() - started) / 1000, + code: payloadError.code, + message: payloadError.message, + }); + update([["ghidra.decompile_max_payload_mib", 64]]); + started = performance.now(); + const payloadRecovered = value( + await execute("procedure_pseudo_code", { procedure: slowEntry }), + ); + assert.ok(String(payloadRecovered.result).includes("rea_state")); + record("payload_64_mib_recovered", { + seconds: (performance.now() - started) / 1000, + pseudocode_bytes: Buffer.byteLength(String(payloadRecovered.result)), + }); + + update([["ghidra.decompile_timeout_seconds", LIMIT_SECONDS]]); + const limitText = `exceeded the ${LIMIT_SECONDS}-second configured decompilation limit`; + started = performance.now(); + const degraded = value( + await execute("analyze_function", { procedure: slowEntry }), + ); + const dossier = degraded.result; + assert.equal( + dossier.facet_availability.pseudocode.available, + false, + "Fixture completed within the one-second native deadline; calibrate REA_DECOMPILE_TIMEOUT_STATEMENTS rather than reporting a provider failure.", + ); + assert.ok(dossier.facet_availability.pseudocode.reason.includes(limitText)); assert.equal(dossier.native_api.available, false); assert.equal(dossier.native_value_flow.available, false); - assert.ok(dossier.assembly.length > BRANCHES); + assert.ok(dossier.assembly.length > STATEMENTS); assert.ok(dossier.basic_blocks.length > 0); - assert.equal(slow.value.cachePolicy, "live"); - observed.degraded = { + assert.equal(degraded.cachePolicy, "live"); + record("deadline_partial_dossier", { + seconds: (performance.now() - started) / 1000, pseudocode: dossier.facet_availability.pseudocode, assembly_lines: dossier.assembly.length, basic_blocks: dossier.basic_blocks.length, - cache_policy: slow.value.cachePolicy, - }; + cache_policy: degraded.cachePolicy, + }); started = performance.now(); - const pseudocode = await client.execute("procedure_pseudo_code", { + const timedOut = await execute("procedure_pseudo_code", { procedure: slowEntry, }); - observed.pseudocode_seconds = (performance.now() - started) / 1000; - assert.equal(pseudocode.ok, false); - const projected = projectAnalysisError(pseudocode.error); - assert.equal(pseudocode.error._tag, "AnalysisTimeoutError"); - assert.equal(projected.code, "provider_timeout"); - assert.equal(projected.retryable, true); - assert.equal(projected.details.timeout_ms, LIMIT_SECONDS * 1000); - assert.ok(projected.message.includes(limitText), projected.message); - assert.ok( - projected.remediation.action.includes( - "REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS", - ), + assert.equal(timedOut.ok, false); + assert.equal(timedOut.error._tag, "AnalysisTimeoutError"); + const timeout = projectAnalysisError(timedOut.error); + assert.equal(timeout.code, "provider_timeout"); + assert.equal(timeout.retryable, true); + assert.equal(timeout.details.timeout_ms, 1000); + assert.equal( + timeout.details.limit.setting, + "ghidra.decompile_timeout_seconds", ); - observed.pseudocode_error = { - code: projected.code, - retryable: projected.retryable, - message: projected.message, - }; - - const changed = settings.update({ - changes: [ - { - setting: "ghidra.decompile_timeout_seconds", - action: "set", - value: 60, - }, - ], + assert.equal(timeout.details.limit.scope, "rea-runtime"); + assert.ok(timeout.message.includes(limitText), timeout.message); + record("deadline_pseudocode", { + seconds: (performance.now() - started) / 1000, + code: timeout.code, + message: timeout.message, + limit: timeout.details.limit, + remediation: timeout.remediation, }); - assert.equal(changed.ok, true); - const recovered = await settings.run(() => - client.execute("procedure_pseudo_code", { procedure: slowEntry }), + + update([ + ["ghidra.decompile_timeout_seconds", RECOVERY_SECONDS], + ["ghidra.decompile_max_payload_mib", null], + ]); + started = performance.now(); + const recovered = value( + await execute("procedure_pseudo_code", { procedure: slowEntry }), ); - if (!recovered.ok) throw recovered.error; - assert.ok(String(recovered.value.result).includes("rea_state")); - observed.runtime_recovery = { - timeout_seconds: 60, - pseudocode_bytes: Buffer.byteLength(String(recovered.value.result)), - }; - - // The same session decompiles the next function with a fresh decompiler. - const small = await client.execute("procedure_pseudo_code", { - procedure: smallEntry, + assert.ok(String(recovered.result).includes("rea_state")); + record("deadline_recovered_payload_null", { + seconds: (performance.now() - started) / 1000, + selected_timeout_seconds: RECOVERY_SECONDS, + selected_payload_mib: null, + pseudocode_bytes: Buffer.byteLength(String(recovered.result)), }); - if (!small.ok) throw small.error; - assert.ok(String(small.value.result).includes("rea_state")); - const continuedSession = await bridgeClient.ping(); - if (!continuedSession.ok) throw continuedSession.error; - assert.equal(continuedSession.value.run_id, initialSession.value.run_id); - assert.equal(continuedSession.value.target.sha256, targetDigest); - observed.final_session_run_id = continuedSession.value.run_id; + const small = value( + await execute("procedure_pseudo_code", { procedure: smallEntry }), + ); + assert.ok(String(small.result).includes("rea_state")); + const final = value(await bridgeClient.ping({ signal: controller.signal })); + assert.equal(final.run_id, initial.run_id); + assert.equal(final.target.sha256, targetDigest); + observed.final_session_run_id = final.run_id; observed.same_session_next_decompile = "succeeded"; } catch (cause) { failures.push(cause); @@ -266,33 +373,38 @@ try { const closed = await client.close(); if (!closed.ok) failures.push(closed.error); if (failures.length > 0) - throw new AggregateError(failures, "Decompile timeout verification failed"); + throw new AggregateError( + failures, + "Ghidra runtime budget verification failed", + ); assert.equal(digest(await readFile(targetPath)), targetDigest); const verifierRun = await completeVerifierRun(run); - try { - assertVerifierCleanupComplete(verifierRun.process_lineage); - } catch (cause) { - throw new Error( - `Verifier cleanup unresolved; retained ${workspace}; run ${run.run_id}`, - { cause }, - ); - } + assertVerifierCleanupComplete(verifierRun.process_lineage); cleaned = true; report = { status: "passed", - limit_seconds: LIMIT_SECONDS, - branches: BRANCHES, + platform: process.platform, + architecture: process.arch, + node: process.version, + fixture: { + kind: "volatile-straight-line", + statements: STATEMENTS, + source_bytes: Buffer.byteLength(source), + }, target_sha256: targetDigest, source_sha256: digest(Buffer.from(source)), compiler, - provider: profile.value.profile.provider, - analysis_profile_digest: profile.value.profile.digest, + provider: profile.profile.provider, + analysis_profile_digest: profile.profile.digest, + configured_jvm_max_heap: environment.GHIDRA_HEADLESS_MAXMEM, + elapsed_seconds: (performance.now() - began) / 1000, ...observed, verifier_run: verifierRun, scope: - "Real Ghidra ELF or Mach-O analysis of a generated slow function through the read-only TCP provider: the decompile limit stops it, analyze_function degrades, procedure_pseudo_code names the setting, a runtime update recovers the same slow function on the same Program, and the same session decompiles the next function. No target execution; Windows is unverified.", + "Real Ghidra analysis of generated Mach-O/ELF through authenticated loopback: independent value-flow budgets, lowered admission, native payload exhaustion and recovery, natural decompile timeout with listing-backed partial facts, and deadline recovery on the same Program. No target execution. Windows and public CLI/MCP transport recovery are unverified. Timings are observations, not performance comparisons; CPU, peak process-family memory and agent tokens are unmeasured.", }; } finally { + clearTimeout(outerDeadline); if (cleaned) await rm(workspace, { recursive: true, force: true }); } console.log(JSON.stringify({ ...report, cleanup: "complete" })); diff --git a/scripts/verify-windows-native.mjs b/scripts/verify-windows-native.mjs index 4d605eef0e..a3eedf3660 100644 --- a/scripts/verify-windows-native.mjs +++ b/scripts/verify-windows-native.mjs @@ -543,10 +543,12 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe "readback-", ); try { - readbackRuntime.writeFile("written.txt", "runtime-write-readback"); + const written = "runtime-write-readback"; + const sourceBytes = await readFile(source); + readbackRuntime.writeFile("written.txt", written); assert.equal( - readbackRuntime.readFile("written.txt"), - "runtime-write-readback", + readbackRuntime.readFile("written.txt", Buffer.byteLength(written)), + written, ); const readbackSnapshot = await readbackRuntime.snapshot( source, @@ -554,21 +556,19 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe ); assert.equal( readbackSnapshot.sha256, - createHash("sha256") - .update(await readFile(source)) - .digest("hex"), + createHash("sha256").update(sourceBytes).digest("hex"), ); assert.equal( - readbackRuntime.readFile("snapshot.bin"), - await readFile(source, "utf8"), + readbackRuntime.readFile("snapshot.bin", sourceBytes.byteLength), + sourceBytes.toString("utf8"), ); const writtenPath = join(readbackRuntime.observation.path, "written.txt"); await assert.rejects(writeFile(writtenPath, "replace")); await assert.rejects(rename(writtenPath, `${writtenPath}.renamed`)); await assert.rejects(rm(writtenPath)); assert.equal( - readbackRuntime.readFile("written.txt"), - "runtime-write-readback", + readbackRuntime.readFile("written.txt", Buffer.byteLength(written)), + written, ); const snapshotPath = join( readbackRuntime.observation.path, @@ -590,8 +590,8 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe await assert.rejects(rename(snapshotPath, `${snapshotPath}.renamed`)); await assert.rejects(rm(snapshotPath)); assert.equal( - readbackRuntime.readFile("snapshot.bin"), - await readFile(source, "utf8"), + readbackRuntime.readFile("snapshot.bin", sourceBytes.byteLength), + sourceBytes.toString("utf8"), ); report.controls = { ...report.controls, @@ -622,7 +622,11 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe assert.equal(await exists(heldPath), true); assert.throws( () => retryRuntime.writeFile("after-close.txt", "forbidden"), - /cleanup is in progress or complete/u, + /Windows private runtime cleanup is in progress or complete/u, + ); + assert.equal( + await exists(join(retryRuntime.observation.path, "after-close.txt")), + false, ); holder.child.stdin.end("\n"); const [exitCode] = await holder.closed; diff --git a/scripts/verify/analysis/budget-calibration-run.py b/scripts/verify/analysis/budget-calibration-run.py new file mode 100644 index 0000000000..23f39fd0ed --- /dev/null +++ b/scripts/verify/analysis/budget-calibration-run.py @@ -0,0 +1,214 @@ +"""Observe one selected experiment's process group and descendant tree. + +macOS has no cgroup peak counter. RSS is a sampled sum, not an exact kernel +peak or aggregate memory hard limit. Only the owned group is stopped early; +descendant-tree sampling is observational and does not confer signal authority. +""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import signal +import subprocess +import time + + +def process_snapshot(): + rows = subprocess.run( + ["ps", "-axo", "pid=,ppid=,pgid=,rss="], + text=True, capture_output=True, check=True, + ).stdout.splitlines() + return [tuple(map(int, row.split())) for row in rows] + + +def group_rss(group, snapshot=None): + if snapshot is None: + snapshot = process_snapshot() + return [(pid, kib * 1024) for pid, _, pgid, kib in snapshot if pgid == group] + + +def tree_rss(root, snapshot): + children = {} + for pid, ppid, _, _ in snapshot: + children.setdefault(ppid, []).append(pid) + pending = [root] + descendants = set() + while pending: + pid = pending.pop() + if pid not in descendants: + descendants.add(pid) + pending.extend(children.get(pid, [])) + return [(pid, kib * 1024) for pid, _, _, kib in snapshot if pid in descendants] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", required=True) + parser.add_argument("--cwd", required=True) + parser.add_argument("--timeout-seconds", type=float, default=90) + parser.add_argument("--memory-mib", type=int, default=1536) + parser.add_argument("command", nargs=argparse.REMAINDER) + options = parser.parse_args() + command = options.command + if command[:1] == ["--"]: + command = command[1:] + if not command or options.timeout_seconds <= 0 or options.memory_mib <= 0: + parser.error("Select an argv target and positive observation bounds") + output = Path(options.output).resolve() + output.parent.mkdir(parents=True, exist_ok=True) + stdout_path = output.with_suffix(".stdout") + stderr_path = output.with_suffix(".stderr") + peak = 0 + maximum_members = 0 + tree_peak = 0 + maximum_tree_members = 0 + samples = 0 + stopped = None + usage = None + status = None + signal_errors = [] + observer_error = None + started = time.monotonic() + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + process = subprocess.Popen( + command, cwd=options.cwd, stdout=stdout, stderr=stderr, + start_new_session=True, + ) + + def reap(): + nonlocal status, usage + if process.returncode is not None: + return True + pid, observed_status, observed_usage = os.wait4(process.pid, os.WNOHANG) + if not pid: + return False + status, usage = observed_status, observed_usage + process.returncode = os.waitstatus_to_exitcode(status) + return True + + def root_exited(): + # Keep the session leader waitable until group cleanup finishes. + # Its reserved PID prevents our process-group identity being reused. + return os.waitid( + os.P_PID, process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT, + ) is not None + + def group_has_work(): + exited = root_exited() + return any( + pid != process.pid or not exited + for pid, _ in group_rss(process.pid) + ) + + def signal_owned_group(selected_signal): + # A sample and the following signal are not atomic. A completed + # root may still own live children. Do not reap it before their + # cleanup, and preserve receipts when the OS refuses a signal. + if not group_has_work(): + return + try: + os.killpg(process.pid, selected_signal) + except ProcessLookupError: + pass + except OSError as error: + signal_errors.append({ + "signal": selected_signal.name, + "errno": error.errno, + "reason": str(error), + }) + + def stop_owned_group(): + signal_owned_group(signal.SIGTERM) + time.sleep(0.2) + signal_owned_group(signal.SIGKILL) + deadline = time.monotonic() + 1 + while group_has_work() and time.monotonic() < deadline: + time.sleep(0.02) + reap() + + try: + while True: + if root_exited(): + if group_has_work(): + stopped = "target_exited_with_group_members" + stop_owned_group() + else: + reap() + break + # One OS snapshot supplies both scopes. Ancestry is an + # observation, never permission to signal detached children. + snapshot = process_snapshot() + members = group_rss(process.pid, snapshot) + tree_members = tree_rss(process.pid, snapshot) + samples += 1 + peak = max(peak, sum(rss for _, rss in members)) + maximum_members = max(maximum_members, len(members)) + tree_peak = max(tree_peak, sum(rss for _, rss in tree_members)) + maximum_tree_members = max(maximum_tree_members, len(tree_members)) + elapsed = time.monotonic() - started + if peak > options.memory_mib * 1024 * 1024: + stopped = "sampled_group_rss_early_stop" + elif elapsed > options.timeout_seconds: + stopped = "wall_deadline" + if stopped: + stop_owned_group() + break + time.sleep(0.02) + except BaseException as error: + observer_error = {"type": type(error).__name__, "reason": str(error)} + stopped = stopped or "observer_error" + stop_owned_group() + remaining = group_rss(process.pid) + receipt = { + "schema": "rea-budget-calibration/v1", + "argv": command, + "cwd": options.cwd, + "platform": platform.platform(), + "elapsed_seconds": time.monotonic() - started, + "cpu_seconds_waited_subtree": usage.ru_utime + usage.ru_stime if usage else None, + "peak_process_group_rss_bytes_sampled": peak, + "maximum_group_members_sampled": maximum_members, + "peak_process_tree_rss_bytes_sampled": tree_peak, + "maximum_tree_members_sampled": maximum_tree_members, + "sample_count": samples, + "sample_interval_seconds": 0.02, + "memory_early_stop_bytes": options.memory_mib * 1024 * 1024, + "memory_early_stop_scope": "owned_process_group", + "wall_deadline_seconds": options.timeout_seconds, + "wall_deadline_signal_scope": "owned_process_group", + "stop_reason": stopped, + "signal_errors": signal_errors, + "observer_error": observer_error, + "exit_code": process.returncode, + "cleanup": "group_empty" if not remaining else "group_members_remaining", + "cleanup_scope": "owned_process_group", + "remaining_group_pids": [pid for pid, _ in remaining], + "stdout": str(stdout_path), + "stderr": str(stderr_path), + "stdout_sha256": hashlib.sha256(stdout_path.read_bytes()).hexdigest(), + "stderr_sha256": hashlib.sha256(stderr_path.read_bytes()).hexdigest(), + "limitations": [ + "RSS scopes are simultaneous sums within each ps snapshot; short peaks can be missed.", + "Shared pages can be counted in multiple members' RSS.", + "Group RSS excludes descendants that leave this group; tree RSS includes detached descendants while PPID ancestry remains linked to the waitable root.", + "Tree samples can miss exited, reparented, or orphaned descendants; no cumulative PID list or descendant signal authority is inferred.", + "CPU is OS wait4 user+system usage of the waited subtree; sampler CPU is excluded.", + "CPU excludes descendants the target did not wait for, including orphaned children.", + "Memory early stop and cleanup signals apply only to the owned group, not the observed tree; neither RSS metric is a kernel hard limit.", + "Wall time includes observation overhead; OS caches are not flushed.", + ], + } + output.write_text(json.dumps(receipt, indent=2) + "\n") + print(json.dumps({key: receipt[key] for key in [ + "elapsed_seconds", "cpu_seconds_waited_subtree", + "peak_process_group_rss_bytes_sampled", "exit_code", "cleanup", "stop_reason", + "peak_process_tree_rss_bytes_sampled", + ]})) + return process.returncode if not stopped and not remaining and process.returncode is not None else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify/analysis/flutter-budget-calibration/aggregate.py b/scripts/verify/analysis/flutter-budget-calibration/aggregate.py new file mode 100644 index 0000000000..d5cc8c4604 --- /dev/null +++ b/scripts/verify/analysis/flutter-budget-calibration/aggregate.py @@ -0,0 +1,45 @@ +"""Join serial observer receipts with exact tokenization of every logical call. + +Requires tiktoken 0.12.0 for comparison with the retained measurements. +Token counts exclude chat framing, catalogs, adapters, retained delivery and +model reasoning. Artifact and repository path spelling can affect tokens. +""" + +import argparse +import json +from pathlib import Path + +import tiktoken + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('receipts', type=Path) +parser.add_argument('output', type=Path) +options = parser.parse_args() +encoding = tiktoken.get_encoding('cl100k_base') +rows = [] +for path in sorted(options.receipts.rglob('*.metrics.json')): + metrics = json.loads(path.read_text()) + key = str(path.relative_to(options.receipts)).removesuffix('.metrics.json') + # The observer argv ends with the selected configuration file. + config = json.loads(Path(metrics['argv'][-1]).read_text()) + reply = json.loads(Path(config['reply_receipt']).read_text()) + assert metrics['exit_code'] == 0 and metrics['stop_reason'] is None, path + assert metrics['cleanup'] == 'group_empty' and reply['success'], path + tokens = sum( + len(encoding.encode(json.dumps(call[field], ensure_ascii=False, separators=(',', ':')))) + for call in reply['calls'] + for field in ['request', 'reply'] + ) + rows.append({ + 'key': key, + 'calls': reply['call_count'], + 'tokens_cl100k_base': tokens, + 'seconds': metrics['elapsed_seconds'], + 'cpu': metrics['cpu_seconds_waited_subtree'], + 'peak_mib': metrics['peak_process_group_rss_bytes_sampled'] / 1024**2, + 'delivered': reply['package_uris_delivered_final'], + 'cleanup': metrics['cleanup'], + 'scope': reply['boundary'], + }) +assert rows, 'No *.metrics.json receipts found' +options.output.write_text(json.dumps(rows, indent=2) + '\n') diff --git a/scripts/verify/analysis/flutter-budget-calibration/configs.py b/scripts/verify/analysis/flutter-budget-calibration/configs.py new file mode 100644 index 0000000000..cb7ea757dd --- /dev/null +++ b/scripts/verify/analysis/flutter-budget-calibration/configs.py @@ -0,0 +1,37 @@ +"""Generate eight task configs after oracle.py has verified the pinned APKs.""" + +import argparse +import json +from pathlib import Path + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('inputs', type=Path) +parser.add_argument('output', type=Path) +options = parser.parse_args() +options.output.mkdir(parents=True, exist_ok=True) +manifest = json.loads((Path(__file__).parent / 'manifest.json').read_text()) +cases = [ + ('inventory-default-retry', 'package-inventory', None, True), + ('inventory-2048', 'package-inventory', 2048, False), + ('overview-default', 'overview', None, False), + ('overview-2048', 'overview', 2048, False), +] +for item in manifest['artifacts']: + stem = Path(item['file']).stem + oracle = (options.output / f'{stem}.oracle.json').resolve() + if not oracle.exists(): + parser.error(f'Run oracle.py first: {oracle}') + for case, task, limit, recover in cases: + config = { + 'artifact': str((options.inputs / item['file']).resolve()), + 'oracle': str(oracle), + 'task': task, + 'recover': recover, + 'recovery_uri_limit': 2048, + 'reply_receipt': str((options.output / f'{stem}.{case}.replies.json').resolve()), + } + if limit is not None: + config['first_uri_limit'] = limit + (options.output / f'{stem}.{case}.config.json').write_text( + json.dumps(config, indent=2) + '\n' + ) diff --git a/scripts/verify/analysis/flutter-budget-calibration/manifest.json b/scripts/verify/analysis/flutter-budget-calibration/manifest.json new file mode 100644 index 0000000000..13180e5f1c --- /dev/null +++ b/scripts/verify/analysis/flutter-budget-calibration/manifest.json @@ -0,0 +1,48 @@ +{ + "artifacts": [ + { + "file": "flutter_gallery_Android.apk", + "url": "https://github.com/flutter-team-archive/gallery/releases/download/v2.9.2/flutter_gallery_Android.apk", + "bytes": 117817550, + "sha256": "659b5fc6a4319dfe4f117d00ad4e7ec004915ebcadcb96628797fb3544040e29", + "expected_libapp_sha256": "0ae30443b66dcf579d024d2f121f77204639bda9897116a8b2f66a6385189fd9", + "expected_package_count_1024": 756, + "expected_dart_count_1024": 201, + "expected_isolate_section": { + "name": "_kDartIsolateSnapshotData", + "offset": 10420224, + "size": 14263200 + } + }, + { + "file": "LocalSend-1.18.2-android-arm64v8.apk", + "url": "https://github.com/localsend/localsend/releases/download/v1.18.2/LocalSend-1.18.2-android-arm64v8.apk", + "bytes": 46558706, + "sha256": "82ec3568fba2aa5295b9aae8b76f701d7a4703d86b9f8bad749472038fbaeab3", + "expected_libapp_sha256": "8f46efd1f78d65ebc0ff630655f274f3c36be89aa13316fb7322a3dda8a497d6", + "expected_package_count_1024": 1407, + "expected_dart_count_1024": 216, + "expected_isolate_section": { + "name": "_kDartIsolateSnapshotData", + "offset": 19264, + "size": 6742144 + } + } + ], + "supporting_artifacts": [ + { + "file": "qgis-world-map.gpkg", + "url": "https://raw.githubusercontent.com/qgis/QGIS/e1223b8e9e71fc50ee995ed556f711c54ccc101b/resources/data/world_map.gpkg", + "bytes": 12914688, + "sha256": "01d293159d1b77bd9212b5252587853f61aa9e25b2387135f34afaba8d057190", + "independent_sqlite_row_counts": { + "declared_tables_including_virtual_and_shadow": 26, + "countries": 240, + "states_provinces": 4556, + "disputed_borders": 46, + "layer_styles": 4 + }, + "role": "Supporting counts only, not a new timed desktop calibration" + } + ] +} diff --git a/scripts/verify/analysis/flutter-budget-calibration/measurements.json b/scripts/verify/analysis/flutter-budget-calibration/measurements.json new file mode 100644 index 0000000000..eaeeac43bb --- /dev/null +++ b/scripts/verify/analysis/flutter-budget-calibration/measurements.json @@ -0,0 +1,294 @@ +{ + "schema": "rea-flutter-budget-calibration/v1", + "baseline": { + "head": "e2a2ede5440df396e44b3406ef511a31a680290e", + "tracked_patch_sha256": "038b69c19bbd79a303c9ecd8d9d22a1251637e3886543ca4f5a4c1b70c724eec", + "description": "Frozen in-progress source snapshot; HEAD alone does not identify this runtime." + }, + "host": "macOS-26.3-arm64-arm-64bit-Mach-O", + "node": "24.22.1", + "driver_original_sha256": "b696f10ddef5bab2dc966587bf362a4f93e7a5c662db1f4c12d107b090e78cba", + "oracle_original_sha256": "82e7e68bea71929ebfce3e607d13bea88fb6bd0259892bfa3655d2cd17a1a49a", + "tokenizer": { + "encoding": "cl100k_base", + "tiktoken_version": "0.12.0", + "scope": "logical request and reply JSON, including retries; no chat framing/catalog/adapters/model usage", + "serialization": "json.dumps(call[field], ensure_ascii=False, separators=(',', ':')) for request and reply separately on every call" + }, + "method": { + "trials": 3, + "cases": 8, + "cache": "fresh process per case; OS caches not flushed; no cold-cache or warm-runtime-cache claims", + "process_group_rss_sample_seconds": 0.02, + "cpu": "wait4 waited-subtree user+system; sampler excluded", + "memory_early_stop_mib": 1536, + "wall_deadline_seconds": 90, + "memory_limit_note": "sampled early-stop threshold, not kernel hard limit" + }, + "runs": [ + { + "key": "r1-LocalSend-1.18.2-android-arm64v8.inventory-2048", + "calls": 1, + "tokens_cl100k_base": 16317, + "seconds": 0.299053416994866, + "cpu": 0.374338, + "peak_mib": 184.78125, + "delivered": 1407, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r1-LocalSend-1.18.2-android-arm64v8.inventory-default-retry", + "calls": 2, + "tokens_cl100k_base": 22360, + "seconds": 0.40635258400288876, + "cpu": 0.515262, + "peak_mib": 221.328125, + "delivered": 1407, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r1-LocalSend-1.18.2-android-arm64v8.overview-2048", + "calls": 1, + "tokens_cl100k_base": 27158, + "seconds": 0.29257937498914544, + "cpu": 0.388501, + "peak_mib": 189.515625, + "delivered": 1407, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r1-LocalSend-1.18.2-android-arm64v8.overview-default", + "calls": 1, + "tokens_cl100k_base": 16885, + "seconds": 0.2934171250090003, + "cpu": 0.413556, + "peak_mib": 189.625, + "delivered": 500, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r1-flutter_gallery_Android.inventory-2048", + "calls": 1, + "tokens_cl100k_base": 8631, + "seconds": 0.5524000000004889, + "cpu": 0.598673, + "peak_mib": 238.859375, + "delivered": 756, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r1-flutter_gallery_Android.inventory-default-retry", + "calls": 2, + "tokens_cl100k_base": 14488, + "seconds": 1.01431600000069, + "cpu": 1.250315, + "peak_mib": 299.453125, + "delivered": 756, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r1-flutter_gallery_Android.overview-2048", + "calls": 1, + "tokens_cl100k_base": 18764, + "seconds": 0.5649452920042677, + "cpu": 0.639156, + "peak_mib": 243.328125, + "delivered": 756, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r1-flutter_gallery_Android.overview-default", + "calls": 1, + "tokens_cl100k_base": 15995, + "seconds": 0.5576133750000736, + "cpu": 0.6388879999999999, + "peak_mib": 240.71875, + "delivered": 500, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r2-LocalSend-1.18.2-android-arm64v8.inventory-2048", + "calls": 1, + "tokens_cl100k_base": 16317, + "seconds": 0.2958210000069812, + "cpu": 0.36518599999999996, + "peak_mib": 183.703125, + "delivered": 1407, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r2-LocalSend-1.18.2-android-arm64v8.inventory-default-retry", + "calls": 2, + "tokens_cl100k_base": 22360, + "seconds": 0.4046505839942256, + "cpu": 0.530767, + "peak_mib": 216.390625, + "delivered": 1407, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r2-LocalSend-1.18.2-android-arm64v8.overview-2048", + "calls": 1, + "tokens_cl100k_base": 27158, + "seconds": 0.3230182090046583, + "cpu": 0.38625, + "peak_mib": 189.640625, + "delivered": 1407, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r2-LocalSend-1.18.2-android-arm64v8.overview-default", + "calls": 1, + "tokens_cl100k_base": 16885, + "seconds": 0.289660875001573, + "cpu": 0.400553, + "peak_mib": 189.765625, + "delivered": 500, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r2-flutter_gallery_Android.inventory-2048", + "calls": 1, + "tokens_cl100k_base": 8631, + "seconds": 0.47969675000058487, + "cpu": 0.606711, + "peak_mib": 241.25, + "delivered": 756, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r2-flutter_gallery_Android.inventory-default-retry", + "calls": 2, + "tokens_cl100k_base": 14488, + "seconds": 0.8361371669889195, + "cpu": 1.028465, + "peak_mib": 258.65625, + "delivered": 756, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r2-flutter_gallery_Android.overview-2048", + "calls": 1, + "tokens_cl100k_base": 18764, + "seconds": 0.5597699580102926, + "cpu": 0.633868, + "peak_mib": 241.53125, + "delivered": 756, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r2-flutter_gallery_Android.overview-default", + "calls": 1, + "tokens_cl100k_base": 15995, + "seconds": 0.4476122909982223, + "cpu": 0.641888, + "peak_mib": 245.84375, + "delivered": 500, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r3-LocalSend-1.18.2-android-arm64v8.inventory-2048", + "calls": 1, + "tokens_cl100k_base": 16317, + "seconds": 0.29069650000019465, + "cpu": 0.362932, + "peak_mib": 185.859375, + "delivered": 1407, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r3-LocalSend-1.18.2-android-arm64v8.inventory-default-retry", + "calls": 2, + "tokens_cl100k_base": 22360, + "seconds": 0.3999212499911664, + "cpu": 0.519303, + "peak_mib": 230.84375, + "delivered": 1407, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r3-LocalSend-1.18.2-android-arm64v8.overview-2048", + "calls": 1, + "tokens_cl100k_base": 27158, + "seconds": 0.2895570419932483, + "cpu": 0.388245, + "peak_mib": 189.984375, + "delivered": 1407, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r3-LocalSend-1.18.2-android-arm64v8.overview-default", + "calls": 1, + "tokens_cl100k_base": 16885, + "seconds": 0.29197387500607874, + "cpu": 0.402588, + "peak_mib": 190.25, + "delivered": 500, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r3-flutter_gallery_Android.inventory-2048", + "calls": 1, + "tokens_cl100k_base": 8631, + "seconds": 0.447617124998942, + "cpu": 0.600641, + "peak_mib": 238.6875, + "delivered": 756, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r3-flutter_gallery_Android.inventory-default-retry", + "calls": 2, + "tokens_cl100k_base": 14488, + "seconds": 0.8252949169982458, + "cpu": 1.0068, + "peak_mib": 254.765625, + "delivered": 756, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r3-flutter_gallery_Android.overview-2048", + "calls": 1, + "tokens_cl100k_base": 18764, + "seconds": 0.4492089169943938, + "cpu": 0.6431779999999999, + "peak_mib": 249.65625, + "delivered": 756, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + }, + { + "key": "r3-flutter_gallery_Android.overview-default", + "calls": 1, + "tokens_cl100k_base": 15995, + "seconds": 0.44429433399636764, + "cpu": 0.6304479999999999, + "peak_mib": 241.921875, + "delivered": 500, + "cleanup": "group_empty", + "scope": "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning" + } + ] +} diff --git a/scripts/verify/analysis/flutter-budget-calibration/oracle.py b/scripts/verify/analysis/flutter-budget-calibration/oracle.py new file mode 100644 index 0000000000..cc6ecb082c --- /dev/null +++ b/scripts/verify/analysis/flutter-budget-calibration/oracle.py @@ -0,0 +1,94 @@ +"""Independently enumerate bounded printable runs in two pinned Flutter APKs. + +This intentionally small ELF64LE parser is for these verified artifacts, not a +replacement for production parsing. It uses no REA implementation code. +""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import struct +import zipfile + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('inputs', type=Path) +parser.add_argument('output', type=Path) +options = parser.parse_args() +options.output.mkdir(parents=True, exist_ok=True) +manifest = json.loads((Path(__file__).parent / 'manifest.json').read_text()) +for item in manifest['artifacts']: + apk = options.inputs / item['file'] + assert apk.stat().st_size == item['bytes'], apk + with apk.open('rb') as source: + artifact_digest = hashlib.file_digest(source, 'sha256').hexdigest() + assert artifact_digest == item['sha256'], apk + with zipfile.ZipFile(apk) as archive: + data = archive.read('lib/arm64-v8a/libapp.so') + assert data[:6] == b'\x7fELF\x02\x01', apk + section_offset = struct.unpack_from(' run_bytes for run in runs), + } + assert oracle['libapp_sha256'] == item['expected_libapp_sha256'] + assert oracle['section'] == item['expected_isolate_section'] + assert oracle['policies']['1024']['package_count'] == item['expected_package_count_1024'] + assert oracle['policies']['1024']['dart_count'] == item['expected_dart_count_1024'] + target = options.output / (apk.stem + '.oracle.json') + target.write_text(json.dumps(oracle, indent=2) + '\n') + print(json.dumps({ + 'oracle': str(target), + 'artifact_sha256': artifact_digest, + 'libapp_sha256': oracle['libapp_sha256'], + 'section': oracle['section'], + 'policies': { + key: {name: value for name, value in policy.items() if not isinstance(value, list)} + for key, policy in oracle['policies'].items() + }, + })) diff --git a/scripts/verify/analysis/flutter-budget-calibration/task.mjs b/scripts/verify/analysis/flutter-budget-calibration/task.mjs new file mode 100644 index 0000000000..ed7f9f2cda --- /dev/null +++ b/scripts/verify/analysis/flutter-budget-calibration/task.mjs @@ -0,0 +1,103 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { join } from "node:path"; +import assert from "node:assert/strict"; +const [repository, configPath] = process.argv.slice(2); +const config = JSON.parse(await readFile(configPath, "utf8")); +const { FlutterBuildAnalysisService } = await import( + pathToFileURL( + join(repository, "src/application/flutter/FlutterBuildAnalysisService.ts"), + ) +); +const { createFlutterBuildAnalysisProvider } = await import( + pathToFileURL(join(repository, "src/composition/flutter.ts")) +); +const { estimateTokenCount } = await import( + pathToFileURL(join(repository, "node_modules/tokenx/dist/index.mjs")) +); +const oracle = JSON.parse(await readFile(config.oracle, "utf8")); +const calls = []; +const provider = createFlutterBuildAnalysisProvider(); +const service = new FlutterBuildAnalysisService(provider); +try { + let uriLimit = config.first_uri_limit; + for (let attempt = 0; attempt < 5; attempt++) { + const input = { + path: config.artifact, + abi: "arm64-v8a", + pool_policy: { + ...(uriLimit === undefined ? {} : { uri_limit: uriLimit }), + ...(config.task === "package-inventory" + ? { contains: "package:" } + : {}), + }, + }; + const request = { operation: "inspect_dart_aot", input }; + const reply = await service.execute(request.operation, request.input); + calls.push({ + request, + reply, + request_token_estimate: estimateTokenCount(JSON.stringify(request)), + reply_token_estimate: estimateTokenCount(JSON.stringify(reply)), + }); + assert.equal(reply.ok, true, JSON.stringify(reply)); + const result = reply.value.normalized_result; + assert.equal(result.target.sha256, oracle.artifact_sha256); + assert.equal(result.libapp.sha256, oracle.libapp_sha256); + assert.equal(result.abi, oracle.abi); + assert.equal(result.string_pool.policy.run_bytes, 1024); + assert.equal(result.string_pool.policy.token_bytes, 128); + assert.equal(result.string_pool.policy.token_limit, 2000); + if (config.task === "package-inventory") { + const expected = oracle.policies["1024"].package_uris; + if ( + result.string_pool.package_uris.length < + result.string_pool.package_uri_count_at_least + ) { + assert.equal( + config.recover, + true, + "Candidate failed inventory completeness", + ); + uriLimit = config.recovery_uri_limit; + continue; + } + assert.deepEqual([...result.string_pool.package_uris].sort(), expected); + } else { + // Overview controls verify identity, ABI and snapshot section identities; + // they deliberately do not require any printable string values. + assert.ok(result.libapp.sections.length > 0); + } + const tokenEstimate = calls.reduce( + (n, call) => n + call.request_token_estimate + call.reply_token_estimate, + 0, + ); + const receipt = { + task: config.task, + boundary: + "shared application service; excludes CLI/MCP discovery, adapter delivery, retained resource overhead and model reasoning", + token_estimator: + "tokenx 1.3.0, estimate only; all logical requests and replies including retries", + token_estimate: tokenEstimate, + call_count: calls.length, + success: true, + pool_coverage: result.string_pool.coverage, + package_uris_delivered_final: result.string_pool.package_uris.length, + package_uri_count_at_least_final: + result.string_pool.package_uri_count_at_least, + config, + calls, + }; + await writeFile( + config.reply_receipt, + JSON.stringify(receipt, null, 2) + "\n", + ); + process.stdout.write( + JSON.stringify({ ...receipt, calls: undefined }) + "\n", + ); + break; + } + assert.ok(calls.length < 5, "Recovery did not complete"); +} finally { + await provider.close(); +} diff --git a/scripts/verify/analysis/javascript-precision-calibration/aggregate.py b/scripts/verify/analysis/javascript-precision-calibration/aggregate.py new file mode 100644 index 0000000000..7968b316a2 --- /dev/null +++ b/scripts/verify/analysis/javascript-precision-calibration/aggregate.py @@ -0,0 +1,59 @@ +"""Aggregate retained observer receipts and complete CLI records; needs tiktoken.""" + +import argparse +import json +from pathlib import Path +import statistics + +import tiktoken + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('inputs', type=Path) + parser.add_argument('output', type=Path) + options = parser.parse_args() + encoder = tiktoken.get_encoding('cl100k_base') + def compact(value): + return json.dumps(value, ensure_ascii=False, separators=(',', ':')) + rows = [] + for path in sorted(options.inputs.glob('*.metrics.json')): + metrics = json.loads(path.read_text()) + record = json.loads(path.with_name(path.name.replace('.metrics.json', '.replies.json')).read_text()) + summary = json.loads(path.with_suffix('.stdout').read_text()) + tokens = { + 'arguments': len(encoder.encode(compact(record['argv']))), + 'reply': len(encoder.encode(compact(json.loads(record['stdout'])))), + 'progress_stderr': len(encoder.encode(record['stderr'])), + } + rows.append({ + 'name': path.stem.removesuffix('.metrics'), + 'task': summary['case'], 'budget': summary['budget'], + 'verified_facts': summary['verified_facts'], 'unknowns': summary['unknowns'], + 'wall_s': metrics['elapsed_seconds'], + 'cpu_s': metrics['cpu_seconds_waited_subtree'], + 'host_group_rss_bytes': metrics['peak_process_group_rss_bytes_sampled'], + 'tokens': tokens, 'reply_bytes': summary['reply_bytes'], + 'exit_code': metrics['exit_code'], 'cleanup': metrics['cleanup'], + 'stop_reason': metrics['stop_reason'], + }) + medians = [] + for task in ['theme', 'ms']: + for budget in ['default', 'double']: + selected = [row for row in rows if row['task'] == task and row['budget'] == budget] + if not selected: + continue + medians.append({ + 'task': task, 'budget': budget, 'trials': len(selected), + 'facts': selected[0]['verified_facts'], 'unknowns': selected[0]['unknowns'], + 'wall_s': statistics.median(row['wall_s'] for row in selected), + 'cpu_s': statistics.median(row['cpu_s'] for row in selected), + 'host_group_rss_mib': statistics.median(row['host_group_rss_bytes'] / 1048576 for row in selected), + 'tokens': statistics.median(sum(row['tokens'].values()) for row in selected), + 'reply_bytes': selected[0]['reply_bytes'], + }) + options.output.write_text(json.dumps({'rows': rows, 'medians': medians}, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/scripts/verify/analysis/javascript-precision-calibration/manifest.json b/scripts/verify/analysis/javascript-precision-calibration/manifest.json new file mode 100644 index 0000000000..ea758ba6bc --- /dev/null +++ b/scripts/verify/analysis/javascript-precision-calibration/manifest.json @@ -0,0 +1,16 @@ +{ + "theme": { + "url": "https://demo.actualbudget.org/static/js/theme.EvD-JhP6.chunk.js.map", + "map_sha256": "ceef8194eaca339e55f35cf2decf7ceffa121c7744ee1fffd976a07d39477b32", + "reported_source": "../../../../component-library/src/theme.ts", + "source_sha256": "05307dc6f927705a8c76e43579c910bf6415ce28b375eea19251dc51dd2b1726" + }, + "ms": { + "version": "2.1.3", + "source": "locked installed node_modules/ms", + "files": { + "index.js": "e5f0b6a946a9b2b356a28557728410717df54ea2f599edb619f9839df6b7b0e9", + "package.json": "1a6b4d9739790c0b94ab96c8cc0507e281c164c311ff4fbf5e57fb8d26290b40" + } + } +} diff --git a/scripts/verify/analysis/javascript-precision-calibration/measurements.json b/scripts/verify/analysis/javascript-precision-calibration/measurements.json new file mode 100644 index 0000000000..3f5d43c40e --- /dev/null +++ b/scripts/verify/analysis/javascript-precision-calibration/measurements.json @@ -0,0 +1,323 @@ +{ + "method": "tiktoken0.12.0 cl100k_base compactUnicodeJSON argv and full parsedstdoutEvidence plusliteralprogressstderr; no MCPcatalog/chat/modelbilling", + "runtime_hashes": { + "scripts/rea.mjs": "11120fdc741c2eeca9ce903de5fd18cd1c7c3fffa15ae2beccca750f161d9ee3", + "dist/cli.js": "acd23e0f186188c99fb905cba36f81e11f92d7f280acd684a9ee93ccc75eb9ce", + "dist/domain/javascript/javascriptSemanticResourceLimits.js": "567ecef0cdb593c75ac156075a43c0e3fc8fb6b5ade3991614f58feb256d075c", + "dist/domain/javascript/javascriptSemanticPrecision.js": "56cec74b6fbe0c2dd6e4fb058811f35635d63ae3235f74d2b408e1b7c6dcc366", + "dist/domain/javascript/javascriptSemanticAnalysis.js": "c21da46e866ce49f69a45e0b5ab923fcb8eb6746a50e8f6c8b77ee61d0ca0bbd", + "dist/javascript/analysis/JavaScriptAnalysisWorker.js": "132f199041291d70ae304474a86143fb4bba3e8df4d0114607bce0f5c3f2fe61", + "dist/cli-runtime/cli.js": "b24d2408a52483a8e9b93435110c243deb53bb7ef94757c802a0c6246d8df20c", + "dist/cli-runtime/cliBuiltinArguments.js": "7b0c81902f568cdd7f8003b7baadcb4f493828b1e472ee613940168213e15090", + "./dist/composition/javascriptAnalysisWorkerProcess.js": "ab927845ccfe917549198b5438d4d206c3ecfb087b0e79e431519bbf4ef1f9fd" + }, + "rows": [ + { + "name": "r1-ms-default", + "task": "ms", + "budget": "default", + "verified_facts": 1, + "unknowns": 133, + "wall_s": 0.9129875000071479, + "cpu_s": 1.239214, + "host_group_rss_bytes": 344621056, + "tokens": { + "arguments": 67, + "reply": 233873, + "progress_stderr": 673 + }, + "reply_bytes": 624004, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r1-ms-double", + "task": "ms", + "budget": "double", + "verified_facts": 1, + "unknowns": 133, + "wall_s": 0.9050752910115989, + "cpu_s": 1.2302469999999999, + "host_group_rss_bytes": 348717056, + "tokens": { + "arguments": 81, + "reply": 233914, + "progress_stderr": 673 + }, + "reply_bytes": 624244, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r1-theme-default", + "task": "theme", + "budget": "default", + "verified_facts": 237, + "unknowns": 0, + "wall_s": 0.9374774580064695, + "cpu_s": 1.221098, + "host_group_rss_bytes": 346652672, + "tokens": { + "arguments": 67, + "reply": 258656, + "progress_stderr": 673 + }, + "reply_bytes": 733950, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r1-theme-double", + "task": "theme", + "budget": "double", + "verified_facts": 237, + "unknowns": 0, + "wall_s": 0.9283738329977496, + "cpu_s": 1.236008, + "host_group_rss_bytes": 346423296, + "tokens": { + "arguments": 81, + "reply": 258721, + "progress_stderr": 673 + }, + "reply_bytes": 734190, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r2-ms-default", + "task": "ms", + "budget": "default", + "verified_facts": 1, + "unknowns": 133, + "wall_s": 0.8994753339939052, + "cpu_s": 1.211395, + "host_group_rss_bytes": 326647808, + "tokens": { + "arguments": 67, + "reply": 233873, + "progress_stderr": 673 + }, + "reply_bytes": 624004, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r2-ms-double", + "task": "ms", + "budget": "double", + "verified_facts": 1, + "unknowns": 133, + "wall_s": 0.9041235420008888, + "cpu_s": 1.206674, + "host_group_rss_bytes": 326893568, + "tokens": { + "arguments": 81, + "reply": 233914, + "progress_stderr": 673 + }, + "reply_bytes": 624244, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r2-theme-default", + "task": "theme", + "budget": "default", + "verified_facts": 237, + "unknowns": 0, + "wall_s": 0.9415311659977306, + "cpu_s": 1.231308, + "host_group_rss_bytes": 348160000, + "tokens": { + "arguments": 67, + "reply": 258656, + "progress_stderr": 673 + }, + "reply_bytes": 733950, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r2-theme-double", + "task": "theme", + "budget": "double", + "verified_facts": 237, + "unknowns": 0, + "wall_s": 0.9384306669962825, + "cpu_s": 1.228734, + "host_group_rss_bytes": 350060544, + "tokens": { + "arguments": 81, + "reply": 258721, + "progress_stderr": 673 + }, + "reply_bytes": 734190, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r3-ms-default", + "task": "ms", + "budget": "default", + "verified_facts": 1, + "unknowns": 133, + "wall_s": 0.8975688329956029, + "cpu_s": 1.2376939999999998, + "host_group_rss_bytes": 337887232, + "tokens": { + "arguments": 67, + "reply": 233873, + "progress_stderr": 673 + }, + "reply_bytes": 624004, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r3-ms-double", + "task": "ms", + "budget": "double", + "verified_facts": 1, + "unknowns": 133, + "wall_s": 0.9211560420080787, + "cpu_s": 1.22497, + "host_group_rss_bytes": 346587136, + "tokens": { + "arguments": 81, + "reply": 233914, + "progress_stderr": 673 + }, + "reply_bytes": 624244, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r3-theme-default", + "task": "theme", + "budget": "default", + "verified_facts": 237, + "unknowns": 0, + "wall_s": 0.9006596249964787, + "cpu_s": 1.225968, + "host_group_rss_bytes": 340787200, + "tokens": { + "arguments": 67, + "reply": 258656, + "progress_stderr": 673 + }, + "reply_bytes": 733950, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + }, + { + "name": "r3-theme-double", + "task": "theme", + "budget": "double", + "verified_facts": 237, + "unknowns": 0, + "wall_s": 0.9200200000050245, + "cpu_s": 1.2206029999999999, + "host_group_rss_bytes": 346898432, + "tokens": { + "arguments": 81, + "reply": 258721, + "progress_stderr": 673 + }, + "reply_bytes": 734190, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null + } + ], + "medians": [ + { + "task": "theme", + "budget": "default", + "trials": 3, + "facts": 237, + "unknowns": 0, + "wall_s": 0.9374774580064695, + "cpu_s": 1.225968, + "host_group_rss_mib": 330.59375, + "tokens": 259396, + "reply_bytes": 733950 + }, + { + "task": "theme", + "budget": "double", + "trials": 3, + "facts": 237, + "unknowns": 0, + "wall_s": 0.9283738329977496, + "cpu_s": 1.228734, + "host_group_rss_mib": 330.828125, + "tokens": 259475, + "reply_bytes": 734190 + }, + { + "task": "ms", + "budget": "default", + "trials": 3, + "facts": 1, + "unknowns": 133, + "wall_s": 0.8994753339939052, + "cpu_s": 1.2376939999999998, + "host_group_rss_mib": 322.234375, + "tokens": 234613, + "reply_bytes": 624004 + }, + { + "task": "ms", + "budget": "double", + "trials": 3, + "facts": 1, + "unknowns": 133, + "wall_s": 0.9050752910115989, + "cpu_s": 1.22497, + "host_group_rss_mib": 330.53125, + "tokens": 234668, + "reply_bytes": 624244 + } + ], + "baseline": { + "head": "e2a2ede5440df396e44b3406ef511a31a680290e", + "working_tree": "uncommitted runtime; relevant compiled entrypoint/owner hashes retained; HEAD alone does not identify it", + "original_task_sha256": "a17526f4b4325139896488b1d64106e5f955f7a5f998a062cb6d319425136782", + "original_oracle_sha256": "1f4070c8814a1cd45b5ab205efd9cbf797f4b82aff195abb41877f79642c5388" + }, + "measurement_scope": { + "observer": "sampled simultaneous process-group RSS; not a kernel hard cap", + "excluded_memory": "isolated JavaScript worker launches detached into another process group; its peak memory is unmeasured", + "cpu": "wait4 waited-subtree user+system; sampler and unwaited descendants excluded", + "cache": "fresh CLI/worker per trial; OS caches not flushed; fixed default-then-double ordering", + "node": "24.22.1", + "host": "macOS26.3 arm64", + "heap_mib": "CLI and isolated worker both held512 via NODE_OPTIONS and --max-heap-mb", + "deadline_seconds": 90, + "group_early_stop_mib": 1536, + "calls_per_task": 1, + "retries": 0 + }, + "unknown_equivalence": { + "theme": { + "unknown_meaning_sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "equivalent_trials": 6 + }, + "ms": { + "unknown_meaning_sha256": "9868f30b3b2ffc97cea3b26908a54673ff0137970a41c8c6ab95356ae48a9f19", + "equivalent_trials": 6 + } + } +} diff --git a/scripts/verify/analysis/javascript-precision-calibration/prepare.py b/scripts/verify/analysis/javascript-precision-calibration/prepare.py new file mode 100644 index 0000000000..a14b1c062b --- /dev/null +++ b/scripts/verify/analysis/javascript-precision-calibration/prepare.py @@ -0,0 +1,76 @@ +"""Verify pinned inputs and prepare inert application directories and an oracle.""" + +import argparse +import hashlib +import json +from pathlib import Path +import re +import shutil + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def theme_values(source): + prefix, suffix = 'export const theme = {', '};\n' + if not source.startswith(prefix) or not source.endswith(suffix): + raise ValueError('Expected the pinned literal-only theme declaration') + body = source[len(prefix):-len(suffix)] + pair = re.compile(r"\s*([A-Za-z_]\w*)\s*:\s*'([^'\\]*)'\s*,") + values = {} + position = 0 + while body[position:].strip(): + item = pair.match(body, position) + if item is None or item[1] in values: + raise ValueError('Oracle only accepts distinct literal CSS property declarations') + values[item[1]] = item[2] + position = item.end() + if len(values) != 237: + raise ValueError('The pinned source must declare 237 theme properties') + return values + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('repo', type=Path) + parser.add_argument('source_map', type=Path) + parser.add_argument('output', type=Path) + options = parser.parse_args() + manifest = json.loads((Path(__file__).parent / 'manifest.json').read_text()) + pinned = manifest['theme'] + if sha256(options.source_map) != pinned['map_sha256']: + parser.error('Source-map digest does not match the pinned artifact') + document = json.loads(options.source_map.read_text()) + matching = [text for name, text in zip(document['sources'], document['sourcesContent']) + if name == pinned['reported_source']] + if len(matching) != 1 or matching[0] is None: + parser.error('Expected one embedded original theme source') + source = matching[0] + if hashlib.sha256(source.encode()).hexdigest() != pinned['source_sha256']: + parser.error('Embedded source digest mismatch') + theme = options.output / 'cases/theme' + control = options.output / 'cases/ms' + theme.mkdir(parents=True, exist_ok=True) + control.mkdir(parents=True, exist_ok=True) + (theme / 'theme.ts').write_text(source) + (theme / 'package.json').write_text(json.dumps({ + 'name': 'rea-captured-theme-calibration', 'type': 'module', 'main': 'theme.ts', + })) + for name, digest in manifest['ms']['files'].items(): + path = options.repo / 'node_modules/ms' / name + if sha256(path) != digest: + parser.error(f'Installed ms input digest mismatch: {name}') + shutil.copyfile(path, control / name) + oracle = { + 'theme': {'source_file': 'theme.ts', 'source_sha256': pinned['source_sha256'], + 'expected': theme_values(source)}, + 'ms': {'source_file': 'index.js', + 'source_sha256': manifest['ms']['files']['index.js'], 'expected': {'s': 1000}}, + } + options.output.mkdir(parents=True, exist_ok=True) + (options.output / 'oracle.json').write_text(json.dumps(oracle, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/scripts/verify/analysis/javascript-precision-calibration/scan.mjs b/scripts/verify/analysis/javascript-precision-calibration/scan.mjs new file mode 100644 index 0000000000..b9bcc8c08a --- /dev/null +++ b/scripts/verify/analysis/javascript-precision-calibration/scan.mjs @@ -0,0 +1,68 @@ +import { readFile, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { parse } from "@babel/parser"; +const [output, ...files] = process.argv.slice(2); +const rows = []; +for (const file of files) { + const text = await readFile(file, "utf8"); + const ast = parse(text, { sourceType: "unambiguous" }); + const strings = []; + let nodes = 0; + const stack = [ast]; + while (stack.length) { + const n = stack.pop(); + if (!n || typeof n !== "object") continue; + if (Array.isArray(n)) { + stack.push(...n); + continue; + } + if (typeof n.type === "string") nodes++; + if (n.type === "StringLiteral") { + let oracle = false; + try { + oracle = JSON.parse(text.slice(n.start, n.end)) === n.value; + } catch {} + strings.push({ + start: n.start, + end: n.end, + units: n.value.length, + jsonBytes: Buffer.byteLength(JSON.stringify(n.value)), + oracle, + }); + } + if ( + n.type === "TemplateLiteral" && + n.expressions.length === 0 && + n.quasis[0].value.cooked !== null + ) { + const value = n.quasis[0].value.cooked; + const raw = text.slice(n.start + 1, n.end - 1); + strings.push({ + start: n.start, + end: n.end, + units: value.length, + jsonBytes: Buffer.byteLength(JSON.stringify(value)), + oracle: !raw.includes("\\") && raw === value, + template: true, + }); + } + for (const [k, v] of Object.entries(n)) + if ( + !["loc", "extra", "comments", "tokens"].includes(k) && + v && + typeof v === "object" + ) + stack.push(v); + } + strings.sort((a, b) => b.units - a.units); + rows.push({ + file, + bytes: Buffer.byteLength(text), + sha256: createHash("sha256").update(text).digest("hex"), + nodes, + strings: strings.length, + largest: strings.slice(0, 5), + }); +} +await writeFile(output, JSON.stringify(rows, null, 2)); +console.log(JSON.stringify(rows)); diff --git a/scripts/verify/analysis/javascript-precision-calibration/source-inventory.json b/scripts/verify/analysis/javascript-precision-calibration/source-inventory.json new file mode 100644 index 0000000000..629f4261cc --- /dev/null +++ b/scripts/verify/analysis/javascript-precision-calibration/source-inventory.json @@ -0,0 +1,192 @@ +[ + { + "file": "/tmp/rea-source-map-calibration/theme.EvD-JhP6.chunk.js", + "bytes": 29718, + "sha256": "10ecf4c66ae103222a7c146ad9b945dc83e78b6c78ce05021296e3c2f799bd5b", + "nodes": 4022, + "strings": 349, + "largest": [ + { + "start": 1434, + "end": 1589, + "units": 152, + "jsonBytes": 155, + "oracle": true + }, + { + "start": 3123, + "end": 3227, + "units": 102, + "jsonBytes": 104, + "oracle": true, + "template": true + }, + { + "start": 7255, + "end": 7330, + "units": 73, + "jsonBytes": 75, + "oracle": true, + "template": true + }, + { + "start": 18225, + "end": 18297, + "units": 70, + "jsonBytes": 72, + "oracle": true, + "template": true + }, + { + "start": 10083, + "end": 10154, + "units": 69, + "jsonBytes": 71, + "oracle": true, + "template": true + } + ] + }, + { + "file": "/tmp/rea-source-map-calibration/index.CMrE7aQb.js", + "bytes": 2078745, + "sha256": "937f792d1849b69a6e0741234428b22b4884ec0157ea31dd6949dd1bacc46456", + "nodes": 460754, + "strings": 15897, + "largest": [ + { + "start": 78956, + "end": 79894, + "units": 936, + "jsonBytes": 938, + "oracle": true, + "template": true + }, + { + "start": 104891, + "end": 105550, + "units": 657, + "jsonBytes": 659, + "oracle": true, + "template": true + }, + { + "start": 1752523, + "end": 1753169, + "units": 644, + "jsonBytes": 662, + "oracle": true, + "template": true + }, + { + "start": 1751921, + "end": 1752324, + "units": 401, + "jsonBytes": 413, + "oracle": true, + "template": true + }, + { + "start": 583954, + "end": 584343, + "units": 387, + "jsonBytes": 391, + "oracle": true, + "template": true + } + ] + }, + { + "file": "/tmp/rea-source-map-calibration/DateSelect.BVYaCNUe.chunk.js", + "bytes": 2191517, + "sha256": "9e45139dd0b510ad7014188a5e9d11b307c2ac15dfdc761050388e76e94759bf", + "nodes": 363578, + "strings": 29000, + "largest": [ + { + "start": 34808, + "end": 47930, + "units": 13120, + "jsonBytes": 13122, + "oracle": true, + "template": true + }, + { + "start": 21787, + "end": 34790, + "units": 13001, + "jsonBytes": 13003, + "oracle": true, + "template": true + }, + { + "start": 8827, + "end": 21772, + "units": 12943, + "jsonBytes": 12945, + "oracle": true, + "template": true + }, + { + "start": 47947, + "end": 50426, + "units": 2477, + "jsonBytes": 2479, + "oracle": true, + "template": true + }, + { + "start": 50458, + "end": 52394, + "units": 1934, + "jsonBytes": 1936, + "oracle": true, + "template": true + } + ] + }, + { + "file": "node_modules/ms/index.js", + "bytes": 3024, + "sha256": "e5f0b6a946a9b2b356a28557728410717df54ea2f599edb619f9839df6b7b0e9", + "nodes": 427, + "strings": 48, + "largest": [ + { + "start": 702, + "end": 757, + "units": 53, + "jsonBytes": 55, + "oracle": false + }, + { + "start": 1851, + "end": 1865, + "units": 12, + "jsonBytes": 14, + "oracle": false + }, + { + "start": 1876, + "end": 1889, + "units": 11, + "jsonBytes": 13, + "oracle": false + }, + { + "start": 1745, + "end": 1754, + "units": 7, + "jsonBytes": 9, + "oracle": false + }, + { + "start": 1639, + "end": 1648, + "units": 7, + "jsonBytes": 9, + "oracle": false + } + ] + } +] diff --git a/scripts/verify/analysis/javascript-precision-calibration/task.py b/scripts/verify/analysis/javascript-precision-calibration/task.py new file mode 100644 index 0000000000..a4c0488412 --- /dev/null +++ b/scripts/verify/analysis/javascript-precision-calibration/task.py @@ -0,0 +1,70 @@ +"""Run one complete CLI task and independently check the required source facts.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('repo', type=Path) + parser.add_argument('node') + parser.add_argument('inputs', type=Path) + parser.add_argument('case', choices=['theme', 'ms']) + parser.add_argument('budget', choices=['default', 'double']) + parser.add_argument('output', type=Path) + options = parser.parse_args() + oracle = json.loads((options.inputs / 'oracle.json').read_text())[options.case] + target = options.inputs / 'cases' / options.case + source = target / oracle['source_file'] + if hashlib.sha256(source.read_bytes()).hexdigest() != oracle['source_sha256']: + raise ValueError('Selected source digest does not match the oracle') + argv = [options.node, str(options.repo / 'scripts/rea.mjs'), + 'analyze-javascript-application', str(target), '--max-heap-mb', '512', + '--format', 'json'] + if options.budget == 'double': + argv += ['--semantic-precision', '{"primitive_bytes":2097152}'] + environment = {**os.environ, 'NODE_OPTIONS': '--max-old-space-size=512'} + result = subprocess.run(argv, cwd=options.repo, env=environment, capture_output=True, + text=True, timeout=45) + options.output.parent.mkdir(parents=True, exist_ok=True) + options.output.write_text(json.dumps({ + 'argv': argv, 'stdout': result.stdout, 'stderr': result.stderr, + 'exit_code': result.returncode, + }) + '\n') + if result.returncode != 0: + raise RuntimeError(f'CLI failed: {result.stderr[:500]}') + evidence = json.loads(result.stdout) + graph = evidence['normalized_result']['semantic_graph'] + nodes = {node['node_id']: node for node in graph['nodes']} + if any(node['identity']['artifact_sha256'] != oracle['source_sha256'] + for node in nodes.values() + if node['identity']['module_path'] == oracle['source_file']): + raise ValueError('Selected graph source identity differs from the oracle') + facts = {} + for edge in graph['relations']: + if edge['relation'] != 'defines': + continue + left, right = nodes[edge['source_node_id']], nodes[edge['target_node_id']] + if left['kind'] != 'literal': + continue + if options.case == 'theme' and right['kind'] == 'property-slot': + facts[right['properties']['name']] = left['properties']['value'] + if options.case == 'ms' and right['kind'] == 'binding' and right['label'] == 's': + facts['s'] = left['properties']['value'] + if facts != oracle['expected']: + raise ValueError('Required source facts differ from the independent oracle') + if any('primitive-bytes' in json.dumps(unknown) for unknown in graph['unknowns']): + raise ValueError('Selected byte allowance unexpectedly limited the task') + print(json.dumps({ + 'case': options.case, 'budget': options.budget, 'verified_facts': len(facts), + 'source_sha256': oracle['source_sha256'], + 'reply_bytes': len(result.stdout.encode()), 'unknowns': len(graph['unknowns']), + })) + + +if __name__ == '__main__': + main() diff --git a/scripts/verify/analysis/source-location-delivery/aggregate.py b/scripts/verify/analysis/source-location-delivery/aggregate.py new file mode 100644 index 0000000000..6bbfb2e02e --- /dev/null +++ b/scripts/verify/analysis/source-location-delivery/aggregate.py @@ -0,0 +1,199 @@ +"""Validate and summarize bounded public source-location task receipts. + +Requires tiktoken 0.12.0. Full replies stay in the caller-selected measurement +directory; the output retains only identities, hashes, numeric measurements and +per-call serialized JSON-RPC token/byte estimates. +""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import statistics + +import tiktoken + + +def compact(value, *, ordered=False): + return json.dumps(value, ensure_ascii=False, separators=(',', ':'), sort_keys=ordered) + + +def sha256(path): + digest = hashlib.sha256() + with path.open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(chunk) + return digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('measurement_dir', type=Path) + parser.add_argument('--repo-root', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + repo = args.repo_root.resolve() + assert tiktoken.__version__ == '0.12.0', 'Use the recorded tokenizer version' + encoder = tiktoken.get_encoding('cl100k_base') + token_cache = {} + + def tokens(value, *, line=False): + document = compact(value) + ('\n' if line else '') + digest = hashlib.sha256(document.encode()).hexdigest() + if digest not in token_cache: + token_cache[digest] = len(encoder.encode(document, disallowed_special=())) + return token_cache[digest] + + pins_path = repo / 'docs/verification/source-map-budgets-2026-11-09/artifacts.json' + artifacts = json.loads(pins_path.read_text()) + records, semantic_answers, evidence_ids = [], {}, {} + definitions = None + input_only_definitions = None + artifact_cache = {} + receipts = sorted(args.measurement_dir.glob('r*-*.metrics.json'), key=lambda p: p.stat().st_mtime_ns) + assert len(receipts) == 18, f'Expected 18 final receipts, found {len(receipts)}' + for path in receipts: + name = path.name.removesuffix('.metrics.json') + selected = re.fullmatch(r'r([123])-(theme|index|DateSelect)-(complete|summary)', name) + assert selected, name + trial, case, detail = int(selected[1]), selected[2], selected[3] + metric = json.loads(path.read_text()) + trace_path = path.with_name(name + '.replies.json') + trace = json.loads(trace_path.read_text()) + assert metric['exit_code'] == 0 and metric['cleanup'] == 'group_empty' + assert metric['stop_reason'] is None and metric['observer_error'] is None + assert metric['signal_errors'] == [] and metric['remaining_group_pids'] == [] + assert trace['config']['select'] == case + '.' and trace['config']['detail'] == detail + answer = trace['answer'] + artifact = next(item for item in artifacts if case + '.' in item['path']) + assert answer['source_map_sha256'] == artifact['sha256'] + assert answer['lookup']['requested'] == artifact['point'] + assert answer['lookup']['matched_generated_position'] == artifact['point'] + assert len(answer['matches']) == len(artifact['expected_matches']) + if case not in artifact_cache: + raw_path = Path(trace['config']['artifactDir']) / artifact['path'] + assert sha256(raw_path) == artifact['sha256'] + artifact_cache[case] = json.loads(raw_path.read_text()) + raw = artifact_cache[case] + for index, (actual, expected) in enumerate(zip(answer['matches'], artifact['expected_matches'])): + assert actual['state'] == ('generated-only' if expected.get('generated_only') else 'mapped') + if expected.get('generated_only'): + continue + assert actual['match_index'] == index + assert actual['source_index'] == expected['source_index'] + assert actual['reported_source'] == expected['source'] + assert actual['original_position']['line'] == expected['original_line'] + assert actual['original_position']['column'] == expected['original_column'] + body = raw['sourcesContent'][expected['source_index']] + assert hashlib.sha256(body.encode()).hexdigest() == expected['content_sha256'] + assert actual['content']['utf8_sha256'] == expected['content_sha256'] + snippet = actual['content']['snippet'] + encoded = body.encode('utf-16-le') + start, length = snippet['offset_utf16'], snippet['length_utf16'] + assert snippet['text'] == encoded[start * 2:(start + length) * 2].decode('utf-16-le') + assert length == 512, 'These pinned tasks require a 512-unit snippet' + assert start <= actual['original_position']['offset'] <= start + length + assert actual['content']['code_units'] == len(encoded) // 2 + semantic = {key: answer[key] for key in ['evidence_id', 'source_map_sha256', 'lookup', 'matches']} + semantic_digest = hashlib.sha256(compact(semantic, ordered=True).encode()).hexdigest() + if case in semantic_answers: + assert semantic_digest == semantic_answers[case], f'Answer mismatch in {name}' + assert answer['evidence_id'] == evidence_ids[case] + else: + semantic_answers[case], evidence_ids[case] = semantic_digest, answer['evidence_id'] + catalog = next(call['reply']['result'] for call in trace['calls'] if call['request']['method'] == 'tools/list') + chosen_definitions = {tool: next(item for item in catalog['tools'] if item['name'] == tool) for tool in ['trace_web_source_location', 'inspect_analysis_view']} + definition_counts = {tool: tokens(definition) for tool, definition in chosen_definitions.items()} + input_only_counts = {tool: tokens({key: definition[key] for key in ['name', 'description', 'inputSchema']}) for tool, definition in chosen_definitions.items()} + if definitions is None: + definitions = definition_counts + input_only_definitions = input_only_counts + else: + assert definitions == definition_counts + assert input_only_definitions == input_only_counts + rows, seen_catalog = [], False + used_tools = set() + for call in trace['calls']: + request, reply = call['request'], call['reply'] + operation = request.get('params', {}).get('name', request['method']) + request_count = tokens(request, line=True) + reply_count = tokens(reply, line=True) if reply is not None else 0 + post_discovery = seen_catalog + if request['method'] == 'tools/list': + seen_catalog = True + if request['method'] == 'tools/call': + used_tools.add(operation) + if reply is not None and request['method'] == 'tools/call' and not reply['result'].get('isError'): + normalized = reply['result']['structuredContent']['normalized_result'] + source_report = normalized.get('source_location', normalized.get('summary', normalized)) + assert source_report['source']['sha256'] == artifact['generated_script']['sha256'] + assert source_report['source_map']['sha256'] == artifact['sha256'] + rows.append({ + 'operation': operation, 'post_discovery': post_discovery, + 'request_tokens_estimated': request_count, 'reply_tokens_estimated': reply_count, + 'request_bytes_compact_json_line': len((compact(request) + '\n').encode()), + 'reply_bytes_observed_wire': call['wire_bytes'], + 'reply_bytes_reserialized_json_line': len((compact(reply) + '\n').encode()) if reply is not None else 0, + 'elapsed_ms': call['elapsed_ms'], + 'is_error': reply is not None and reply.get('result', {}).get('isError', False), + }) + if operation == 'trace_web_source_location' and reply is not None: + result = reply['result'] + if not result.get('isError') and detail == 'complete': + text = result['structuredContent']['raw_result']['source_map_text'] + assert hashlib.sha256(text.encode()).hexdigest() == artifact['sha256'] + total = sum(row['request_tokens_estimated'] + row['reply_tokens_estimated'] for row in rows) + post = sum(row['request_tokens_estimated'] + row['reply_tokens_estimated'] for row in rows if row['post_discovery']) + assert used_tools <= chosen_definitions.keys() + records.append({ + 'name': name, 'trial': trial, 'artifact': case, 'detail': detail, + 'evidence_id': answer['evidence_id'], 'map_sha256': artifact['sha256'], 'generated_script_sha256': artifact['generated_script']['sha256'], 'answer_sha256': semantic_digest, + 'receipt_sha256': sha256(path), 'reply_trace_sha256': sha256(trace_path), + 'task_tokens_catalog_included_estimated': total, 'post_discovery_tokens_estimated': post, + 'used_tool_definitions_tokens_estimated': sum(definitions[tool] for tool in used_tools), + 'post_discovery_plus_used_definitions_tokens_estimated': post + sum(definitions[tool] for tool in used_tools), + 'hypothetical_used_input_only_definitions_tokens_estimated': sum(input_only_definitions[tool] for tool in used_tools), + 'hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated': post + sum(input_only_definitions[tool] for tool in used_tools), + 'tool_calls': sum(call['request']['method'] == 'tools/call' for call in trace['calls']), + 'elapsed_seconds': metric['elapsed_seconds'], 'cpu_seconds_waited_subtree': metric['cpu_seconds_waited_subtree'], + 'peak_process_group_rss_bytes_sampled': metric['peak_process_group_rss_bytes_sampled'], + 'peak_process_tree_rss_bytes_sampled': metric['peak_process_tree_rss_bytes_sampled'], + 'maximum_group_members_sampled': metric['maximum_group_members_sampled'], + 'maximum_tree_members_sampled': metric['maximum_tree_members_sampled'], + 'sample_count': metric['sample_count'], 'exit_code': metric['exit_code'], 'cleanup': metric['cleanup'], 'stop_reason': metric['stop_reason'], + 'calls': rows, + }) + groups = {} + for record in records: + groups.setdefault(record['artifact'] + '-' + record['detail'], []).append(record) + fields = ['post_discovery_plus_used_definitions_tokens_estimated', 'hypothetical_used_input_only_definitions_tokens_estimated', 'hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated', 'task_tokens_catalog_included_estimated', 'post_discovery_tokens_estimated', 'used_tool_definitions_tokens_estimated', 'tool_calls', 'elapsed_seconds', 'cpu_seconds_waited_subtree', 'peak_process_group_rss_bytes_sampled', 'peak_process_tree_rss_bytes_sampled'] + medians = {name: {'runs': len(group), **{field: statistics.median(record[field] for record in group) for field in fields}} for name, group in groups.items()} + assert all(len(group) == 3 for group in groups.values()) and len(groups) == 6 + recorded_runtime = json.loads((args.measurement_dir / 'runtime.json').read_text()) + paths = ['scripts/verify/analysis/source-location-delivery/task.mjs', 'scripts/verify/analysis/budget-calibration-run.py', 'scripts/rea.mjs', 'dist/cli.js', 'dist/mcp/main.js', 'src/application/WebSourceLocationService.ts', 'src/domain/analysisView/webSourceLocationView.ts', 'src/application/analysisView/AnalysisViewService.ts', 'src/contracts/webScriptToolContracts.ts', 'src/server/ToolResultDelivery.ts', 'src/server/toolResult.ts', 'src/domain/sourceMapLimits.ts', 'src/server/registerWebSourceLocationTool.ts', 'src/cli/webSourceLocationCommands.ts', 'scripts/verify/analysis/source-location-delivery/aggregate.py', 'package-lock.json', 'docs/verification/source-map-budgets-2026-11-09/artifacts.json'] + hashes = {path: sha256(repo / path) for path in paths} + assert hashes[paths[0]] == recorded_runtime['driver_sha256'] + assert hashes[paths[1]] == recorded_runtime['observer_sha256'] + if recorded_runtime.get('cli_sha256') is not None: + assert hashes['dist/cli.js'] == recorded_runtime['cli_sha256'] + tree = [(path.relative_to(repo / 'dist').as_posix(), sha256(path)) for path in sorted((repo / 'dist').rglob('*')) if path.is_file()] + tree_digest = hashlib.sha256(''.join(path + '\0' + digest + '\n' for path, digest in tree).encode()).hexdigest() + output = { + 'schema': 'rea-source-location-delivery-calibration/v1', + 'token_accounting': {'library': 'tiktoken', 'version': tiktoken.__version__, 'encoding': 'cl100k_base', 'representation': 'compact Unicode JSON-RPC request/reply envelopes with trailing newline; initialized notification has no reply; estimates, not model billing', 'whole_catalog_included': 'initialize + initialized notification + tools/list + every trace/continuation/text call', 'post_discovery': 'all requests/replies after tools/list; excludes discovery only, not recovery', 'selected_definitions': 'complete serialized trace_web_source_location and inspect_analysis_view definitions counted separately; do not add them again to whole-catalog totals; client cached/deferred tool presentation may differ'}, + 'runtime': {'recorded_runtime': recorded_runtime, 'sha256': hashes, 'compiled_tree': {'root': 'dist', 'files': len(tree), 'sha256': tree_digest, 'encoding': 'sorted relative UTF-8 filename + NUL + lowercase SHA256 + newline for every regular file, then SHA256'}}, + 'conditions': {'host': metric['platform'], 'node': 'v24.22.1', 'runs': 'three fresh-process runs; second trial reverses variant order; OS caches not flushed; no disk-cold claim', 'task': 'all equal-position mappings and exact source/map identity plus 512 UTF-16 units around each mapped point', 'comparison': 'current complete delivery including bounded retained-view overflow recovery versus opt-in summary; not the older export-only fallback', 'profiles': {'tool_catalog': 'full', 'input_schema': 'compact', 'output_schema': 'full', 'response_budget_bytes': 10485760}, 'observer': {'interval_seconds': 0.02, 'wall_deadline_seconds': 90, 'memory_early_stop_bytes': 1536 * 1024 * 1024, 'signal_scope': 'owned process group only', 'rss': 'simultaneous ps-snapshot group/tree sums; tree includes linked-PPID codec descendants outside root PGID; can miss short peaks/exited/reparented/orphaned descendants and double-count shared pages', 'cpu': 'wait4 user+system waited subtree, excluding sampler and unwaited/orphaned descendants', 'memory_guard': 'group early stop; observational tree RSS is not signal authority or a hard/kernel peak limit'}}, + 'selected_definition_tokens_estimated': definitions, + 'hypothetical_input_only_definition_tokens_estimated': input_only_definitions, + 'hypothetical_input_only_definition_projection': 'Only {name, description, inputSchema} from captured definitions; not an observed client/model transformation or a measured alternate discovery workflow', + 'observed_completion_order': [record['name'] for record in records], + 'validation': {'completed_runs': len(records), 'same_answer_and_parent_id_across_variants_and_trials': True, 'independent_artifact_oracle_and_snippets_verified': True, 'successful_complete_initial_raw_map_sha256_verified': True, 'all_owned_groups_empty': True, 'all_resource_stops': None}, + 'medians': medians, 'records': records, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(output, ensure_ascii=False, indent=2) + '\n') + print(json.dumps({'medians': medians, 'selected_definitions': definitions, 'compiled_tree': output['runtime']['compiled_tree']}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/verify/analysis/source-location-delivery/measurements.json b/scripts/verify/analysis/source-location-delivery/measurements.json new file mode 100644 index 0000000000..3f1f0dfa47 --- /dev/null +++ b/scripts/verify/analysis/source-location-delivery/measurements.json @@ -0,0 +1,1706 @@ +{ + "schema": "rea-source-location-delivery-calibration/v1", + "token_accounting": { + "library": "tiktoken", + "version": "0.12.0", + "encoding": "cl100k_base", + "representation": "compact Unicode JSON-RPC request/reply envelopes with trailing newline; initialized notification has no reply; estimates, not model billing", + "whole_catalog_included": "initialize + initialized notification + tools/list + every trace/continuation/text call", + "post_discovery": "all requests/replies after tools/list; excludes discovery only, not recovery", + "selected_definitions": "complete serialized trace_web_source_location and inspect_analysis_view definitions counted separately; do not add them again to whole-catalog totals; client cached/deferred tool presentation may differ" + }, + "runtime": { + "recorded_runtime": { + "driver_sha256": "b21ec9d5315682337eb55a4ad208e2145a7397d8d567cdb6653facd3f0bec90e", + "observer_sha256": "c18a06c44715c97ec9cbc76248dfbcd862d0e5ca3cd0c159d868bc7fa1fa9e9f", + "cli_sha256": "acd23e0f186188c99fb905cba36f81e11f92d7f280acd684a9ee93ccc75eb9ce" + }, + "sha256": { + "scripts/verify/analysis/source-location-delivery/task.mjs": "b21ec9d5315682337eb55a4ad208e2145a7397d8d567cdb6653facd3f0bec90e", + "scripts/verify/analysis/budget-calibration-run.py": "c18a06c44715c97ec9cbc76248dfbcd862d0e5ca3cd0c159d868bc7fa1fa9e9f", + "scripts/rea.mjs": "11120fdc741c2eeca9ce903de5fd18cd1c7c3fffa15ae2beccca750f161d9ee3", + "dist/cli.js": "acd23e0f186188c99fb905cba36f81e11f92d7f280acd684a9ee93ccc75eb9ce", + "dist/mcp/main.js": "9bc4c070f63adc9963f276da20b36ffbbcc77b4079585cadf90d792dd2cc04dd", + "src/application/WebSourceLocationService.ts": "da91fc2f53b3487a06cf89209b8b5cbe64891bbee0323c51e322fb1e1ee69f5f", + "src/domain/analysisView/webSourceLocationView.ts": "eeb2994886cd132771b8675659ca80b42c697ac2ec3c07a2ebe2fb265278e5e3", + "src/application/analysisView/AnalysisViewService.ts": "ec61f3ff1faadb0266716f6ac7dfb4af53d366e485cdc133d3f201f5aad16fb6", + "src/contracts/webScriptToolContracts.ts": "a62048a28591c9b44df50fe1e57bd3e5e8c453f9e0c1d449a462b1da669799a7", + "src/server/ToolResultDelivery.ts": "1c281b45b0efa317d1a7ca7cb296b12ab10d6174003affa0a0114c7d45ba2788", + "src/server/toolResult.ts": "6c7a602a96a1a1cf3d5c4fb4605d3b3642a53eacfede407808df9e0972418eeb", + "src/domain/sourceMapLimits.ts": "39072c5fd5e4892e75fdd313e63bf1da923ed717988192bbe80a3ef3f24aeeec", + "src/server/registerWebSourceLocationTool.ts": "1e0d4087fe057dfb21b8e08a33c1d6e08eed7483edfc02346cfab26d422e7999", + "src/cli/webSourceLocationCommands.ts": "30851411a099ca7f58e0bf7af217d33681435672ca69f81db5202829642398a7", + "scripts/verify/analysis/source-location-delivery/aggregate.py": "bfe061035756010006181cbcbc9153ec642ac3a56e83b668e601ee6980fa0c68", + "package-lock.json": "186c7cd2b7f87f409c799c39bb8f18e6d1a8d529c08772251c743da5f374c027", + "docs/verification/source-map-budgets-2026-11-09/artifacts.json": "aedfc0905cdb1f024c0a6e7cba66a5e3197bf77fcc18a9918c7d7b76b4552cd4" + }, + "compiled_tree": { + "root": "dist", + "files": 2921, + "sha256": "a8a510499dabc07b3cdf6b164d02e55f2fc6917b696dcc4d23449510e91e80fa", + "encoding": "sorted relative UTF-8 filename + NUL + lowercase SHA256 + newline for every regular file, then SHA256" + } + }, + "conditions": { + "host": "macOS-26.3-arm64-arm-64bit-Mach-O", + "node": "v24.22.1", + "runs": "three fresh-process runs; second trial reverses variant order; OS caches not flushed; no disk-cold claim", + "task": "all equal-position mappings and exact source/map identity plus 512 UTF-16 units around each mapped point", + "comparison": "current complete delivery including bounded retained-view overflow recovery versus opt-in summary; not the older export-only fallback", + "profiles": { + "tool_catalog": "full", + "input_schema": "compact", + "output_schema": "full", + "response_budget_bytes": 10485760 + }, + "observer": { + "interval_seconds": 0.02, + "wall_deadline_seconds": 90, + "memory_early_stop_bytes": 1610612736, + "signal_scope": "owned process group only", + "rss": "simultaneous ps-snapshot group/tree sums; tree includes linked-PPID codec descendants outside root PGID; can miss short peaks/exited/reparented/orphaned descendants and double-count shared pages", + "cpu": "wait4 user+system waited subtree, excluding sampler and unwaited/orphaned descendants", + "memory_guard": "group early stop; observational tree RSS is not signal authority or a hard/kernel peak limit" + } + }, + "selected_definition_tokens_estimated": { + "trace_web_source_location": 8408, + "inspect_analysis_view": 180576 + }, + "hypothetical_input_only_definition_tokens_estimated": { + "trace_web_source_location": 548, + "inspect_analysis_view": 3533 + }, + "hypothetical_input_only_definition_projection": "Only {name, description, inputSchema} from captured definitions; not an observed client/model transformation or a measured alternate discovery workflow", + "observed_completion_order": [ + "r1-theme-complete", + "r1-theme-summary", + "r1-index-complete", + "r1-index-summary", + "r1-DateSelect-complete", + "r1-DateSelect-summary", + "r2-theme-summary", + "r2-theme-complete", + "r2-index-summary", + "r2-index-complete", + "r2-DateSelect-summary", + "r2-DateSelect-complete", + "r3-theme-complete", + "r3-theme-summary", + "r3-index-complete", + "r3-index-summary", + "r3-DateSelect-complete", + "r3-DateSelect-summary" + ], + "validation": { + "completed_runs": 18, + "same_answer_and_parent_id_across_variants_and_trials": true, + "independent_artifact_oracle_and_snippets_verified": true, + "successful_complete_initial_raw_map_sha256_verified": true, + "all_owned_groups_empty": true, + "all_resource_stops": null + }, + "medians": { + "theme-complete": { + "runs": 3, + "post_discovery_plus_used_definitions_tokens_estimated": 59755, + "hypothetical_used_input_only_definitions_tokens_estimated": 548, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 51895, + "task_tokens_catalog_included_estimated": 1196400, + "post_discovery_tokens_estimated": 51347, + "used_tool_definitions_tokens_estimated": 8408, + "tool_calls": 1, + "elapsed_seconds": 0.9717148750060005, + "cpu_seconds_waited_subtree": 1.128418, + "peak_process_group_rss_bytes_sampled": 522207232, + "peak_process_tree_rss_bytes_sampled": 582778880 + }, + "theme-summary": { + "runs": 3, + "post_discovery_plus_used_definitions_tokens_estimated": 198809, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 13906, + "task_tokens_catalog_included_estimated": 1154878, + "post_discovery_tokens_estimated": 9825, + "used_tool_definitions_tokens_estimated": 188984, + "tool_calls": 2, + "elapsed_seconds": 0.9846506659960141, + "cpu_seconds_waited_subtree": 1.1426440000000002, + "peak_process_group_rss_bytes_sampled": 507412480, + "peak_process_tree_rss_bytes_sampled": 568426496 + }, + "index-complete": { + "runs": 3, + "post_discovery_plus_used_definitions_tokens_estimated": 3218249, + "hypothetical_used_input_only_definitions_tokens_estimated": 548, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 3210389, + "task_tokens_catalog_included_estimated": 4354894, + "post_discovery_tokens_estimated": 3209841, + "used_tool_definitions_tokens_estimated": 8408, + "tool_calls": 1, + "elapsed_seconds": 1.6246519580017775, + "cpu_seconds_waited_subtree": 1.8145129999999998, + "peak_process_group_rss_bytes_sampled": 814170112, + "peak_process_tree_rss_bytes_sampled": 814170112 + }, + "index-summary": { + "runs": 3, + "post_discovery_plus_used_definitions_tokens_estimated": 198782, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 13879, + "task_tokens_catalog_included_estimated": 1154851, + "post_discovery_tokens_estimated": 9798, + "used_tool_definitions_tokens_estimated": 188984, + "tool_calls": 2, + "elapsed_seconds": 1.1343032500008121, + "cpu_seconds_waited_subtree": 1.293817, + "peak_process_group_rss_bytes_sampled": 576421888, + "peak_process_tree_rss_bytes_sampled": 693518336 + }, + "DateSelect-complete": { + "runs": 3, + "post_discovery_plus_used_definitions_tokens_estimated": 199863, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 14960, + "task_tokens_catalog_included_estimated": 1155932, + "post_discovery_tokens_estimated": 10879, + "used_tool_definitions_tokens_estimated": 188984, + "tool_calls": 3, + "elapsed_seconds": 1.1992472090059891, + "cpu_seconds_waited_subtree": 1.351098, + "peak_process_group_rss_bytes_sampled": 608813056, + "peak_process_tree_rss_bytes_sampled": 729399296 + }, + "DateSelect-summary": { + "runs": 3, + "post_discovery_plus_used_definitions_tokens_estimated": 198905, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 14002, + "task_tokens_catalog_included_estimated": 1154974, + "post_discovery_tokens_estimated": 9921, + "used_tool_definitions_tokens_estimated": 188984, + "tool_calls": 2, + "elapsed_seconds": 1.1498196669999743, + "cpu_seconds_waited_subtree": 1.325962, + "peak_process_group_rss_bytes_sampled": 617168896, + "peak_process_tree_rss_bytes_sampled": 727924736 + } + }, + "records": [ + { + "name": "r1-theme-complete", + "trial": 1, + "artifact": "theme", + "detail": "complete", + "evidence_id": "ev_4793658938c5d3e27870a764dc1f5a1211d12ec5e86f86bf072cf856db72b5dc", + "map_sha256": "ceef8194eaca339e55f35cf2decf7ceffa121c7744ee1fffd976a07d39477b32", + "generated_script_sha256": "10ecf4c66ae103222a7c146ad9b945dc83e78b6c78ce05021296e3c2f799bd5b", + "answer_sha256": "b75f60d46b92295b8d2e3c432b5b128a330734d4513feaff0b626ed3fc17d6e1", + "receipt_sha256": "140312e203cb550357ae45f7092b731f65048ea56e778f3b2f3e4222088cf35a", + "reply_trace_sha256": "5dc04db326b8b5d275fde36a2038441baf2ace49b638941e4b11fbcd0ce4aff1", + "task_tokens_catalog_included_estimated": 1196400, + "post_discovery_tokens_estimated": 51347, + "used_tool_definitions_tokens_estimated": 8408, + "post_discovery_plus_used_definitions_tokens_estimated": 59755, + "hypothetical_used_input_only_definitions_tokens_estimated": 548, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 51895, + "tool_calls": 1, + "elapsed_seconds": 0.9717148750060005, + "cpu_seconds_waited_subtree": 1.130636, + "peak_process_group_rss_bytes_sampled": 508133376, + "peak_process_tree_rss_bytes_sampled": 587300864, + "maximum_group_members_sampled": 2, + "maximum_tree_members_sampled": 3, + "sample_count": 22, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 117.104125, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 139.29, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 135, + "reply_tokens_estimated": 51212, + "request_bytes_compact_json_line": 474, + "reply_bytes_observed_wire": 159113, + "reply_bytes_reserialized_json_line": 159113, + "elapsed_ms": 583.2275830000001, + "is_error": false + } + ] + }, + { + "name": "r1-theme-summary", + "trial": 1, + "artifact": "theme", + "detail": "summary", + "evidence_id": "ev_4793658938c5d3e27870a764dc1f5a1211d12ec5e86f86bf072cf856db72b5dc", + "map_sha256": "ceef8194eaca339e55f35cf2decf7ceffa121c7744ee1fffd976a07d39477b32", + "generated_script_sha256": "10ecf4c66ae103222a7c146ad9b945dc83e78b6c78ce05021296e3c2f799bd5b", + "answer_sha256": "b75f60d46b92295b8d2e3c432b5b128a330734d4513feaff0b626ed3fc17d6e1", + "receipt_sha256": "f988085b5e6f3c7d37f117fc89fc85a83ecbc4bbcf031882dbdad85914247576", + "reply_trace_sha256": "6fce7b5e47f4928d680c7878a84bec0707315807798c8cdec1f93ac084434a99", + "task_tokens_catalog_included_estimated": 1154878, + "post_discovery_tokens_estimated": 9825, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 198809, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 13906, + "tool_calls": 2, + "elapsed_seconds": 0.9732366249954794, + "cpu_seconds_waited_subtree": 1.1426440000000002, + "peak_process_group_rss_bytes_sampled": 515080192, + "peak_process_tree_rss_bytes_sampled": 583434240, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 22, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 118.352541, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 140.63200000000003, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 139, + "reply_tokens_estimated": 4562, + "request_bytes_compact_json_line": 493, + "reply_bytes_observed_wire": 17846, + "reply_bytes_reserialized_json_line": 17846, + "elapsed_ms": 585.7748750000001, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 107, + "reply_tokens_estimated": 5017, + "request_bytes_compact_json_line": 325, + "reply_bytes_observed_wire": 19501, + "reply_bytes_reserialized_json_line": 19501, + "elapsed_ms": 5.605625000000032, + "is_error": false + } + ] + }, + { + "name": "r1-index-complete", + "trial": 1, + "artifact": "index", + "detail": "complete", + "evidence_id": "ev_7385676797553c25b8fb5b1428caa24b9a1f943a6fc4f69462612bad8240043c", + "map_sha256": "f0de6886f6a49758f53c7f2de0b77fbff6d72c719a7c32ac23d13453ebe7b3e8", + "generated_script_sha256": "937f792d1849b69a6e0741234428b22b4884ec0157ea31dd6949dd1bacc46456", + "answer_sha256": "c8a0e46117f873873c1756daf5f44b4c9f855335c0beb5758e51c257282234c7", + "receipt_sha256": "ff1ce5e06213bbdf966c184476f139c3810335c42f279762b05033601c99c245", + "reply_trace_sha256": "207c7a5e42ca2588b76914662ffe8b4c349aa3f34d7bc5282dc5f2dbc6b70618", + "task_tokens_catalog_included_estimated": 4354894, + "post_discovery_tokens_estimated": 3209841, + "used_tool_definitions_tokens_estimated": 8408, + "post_discovery_plus_used_definitions_tokens_estimated": 3218249, + "hypothetical_used_input_only_definitions_tokens_estimated": 548, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 3210389, + "tool_calls": 1, + "elapsed_seconds": 1.5899935420020483, + "cpu_seconds_waited_subtree": 1.8149629999999999, + "peak_process_group_rss_bytes_sampled": 857866240, + "peak_process_tree_rss_bytes_sampled": 857866240, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 37, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 115.822708, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 141.78362499999997, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 132, + "reply_tokens_estimated": 3209709, + "request_bytes_compact_json_line": 457, + "reply_bytes_observed_wire": 9121044, + "reply_bytes_reserialized_json_line": 9121044, + "elapsed_ms": 1197.73825, + "is_error": false + } + ] + }, + { + "name": "r1-index-summary", + "trial": 1, + "artifact": "index", + "detail": "summary", + "evidence_id": "ev_7385676797553c25b8fb5b1428caa24b9a1f943a6fc4f69462612bad8240043c", + "map_sha256": "f0de6886f6a49758f53c7f2de0b77fbff6d72c719a7c32ac23d13453ebe7b3e8", + "generated_script_sha256": "937f792d1849b69a6e0741234428b22b4884ec0157ea31dd6949dd1bacc46456", + "answer_sha256": "c8a0e46117f873873c1756daf5f44b4c9f855335c0beb5758e51c257282234c7", + "receipt_sha256": "7957f1267de410dfd403ab39db1b0ab53b447ab01165463783babbec4ccccd48", + "reply_trace_sha256": "eb57733efc2f26ea39945f3f01da7bd98b9a1c6f887eb48a03b4c084860bb024", + "task_tokens_catalog_included_estimated": 1154851, + "post_discovery_tokens_estimated": 9798, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 198782, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 13879, + "tool_calls": 2, + "elapsed_seconds": 1.1343032500008121, + "cpu_seconds_waited_subtree": 1.2970629999999999, + "peak_process_group_rss_bytes_sampled": 576421888, + "peak_process_tree_rss_bytes_sampled": 662896640, + "maximum_group_members_sampled": 4, + "maximum_tree_members_sampled": 4, + "sample_count": 26, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 115.92387499999998, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 140.296041, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 136, + "reply_tokens_estimated": 4566, + "request_bytes_compact_json_line": 476, + "reply_bytes_observed_wire": 17760, + "reply_bytes_reserialized_json_line": 17760, + "elapsed_ms": 735.846625, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 105, + "reply_tokens_estimated": 4991, + "request_bytes_compact_json_line": 327, + "reply_bytes_observed_wire": 19381, + "reply_bytes_reserialized_json_line": 19381, + "elapsed_ms": 5.44987500000002, + "is_error": false + } + ] + }, + { + "name": "r1-DateSelect-complete", + "trial": 1, + "artifact": "DateSelect", + "detail": "complete", + "evidence_id": "ev_9b82f05c94b00018545c9b60c492cdbcba8f5bf1428c22f9ee243ee046877386", + "map_sha256": "e1939ef36214ff46263d3bdaeb99d60a46c75714a6b6220ed27c9be2d24d6cc8", + "generated_script_sha256": "9e45139dd0b510ad7014188a5e9d11b307c2ac15dfdc761050388e76e94759bf", + "answer_sha256": "24064ff1089893082998aba4a03cda18bd7e010663921436858fd40efb94516d", + "receipt_sha256": "51a53c27b6da6603f92ccd824bc6606f2d87fcfaa9fe282eb6fbf1ff85af1908", + "reply_trace_sha256": "3cc424daee85efe2823b11cb9e4f20b198c3ef82a852ec504f4eb43392d1b4c4", + "task_tokens_catalog_included_estimated": 1155932, + "post_discovery_tokens_estimated": 10879, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 199863, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 14960, + "tool_calls": 3, + "elapsed_seconds": 1.1978024580021156, + "cpu_seconds_waited_subtree": 1.348827, + "peak_process_group_rss_bytes_sampled": 612401152, + "peak_process_tree_rss_bytes_sampled": 729399296, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 27, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 116.440875, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 141.418792, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 138, + "reply_tokens_estimated": 776, + "request_bytes_compact_json_line": 489, + "reply_bytes_observed_wire": 3310, + "reply_bytes_reserialized_json_line": 3310, + "elapsed_ms": 792.3197090000001, + "is_error": true + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 107, + "reply_tokens_estimated": 4647, + "request_bytes_compact_json_line": 324, + "reply_bytes_observed_wire": 18288, + "reply_bytes_reserialized_json_line": 18288, + "elapsed_ms": 6.202125000000024, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 106, + "reply_tokens_estimated": 5105, + "request_bytes_compact_json_line": 327, + "reply_bytes_observed_wire": 19577, + "reply_bytes_reserialized_json_line": 19577, + "elapsed_ms": 1.421291999999994, + "is_error": false + } + ] + }, + { + "name": "r1-DateSelect-summary", + "trial": 1, + "artifact": "DateSelect", + "detail": "summary", + "evidence_id": "ev_9b82f05c94b00018545c9b60c492cdbcba8f5bf1428c22f9ee243ee046877386", + "map_sha256": "e1939ef36214ff46263d3bdaeb99d60a46c75714a6b6220ed27c9be2d24d6cc8", + "generated_script_sha256": "9e45139dd0b510ad7014188a5e9d11b307c2ac15dfdc761050388e76e94759bf", + "answer_sha256": "24064ff1089893082998aba4a03cda18bd7e010663921436858fd40efb94516d", + "receipt_sha256": "d7f05810c887057bef2d5eddeafb6bb2be107876874ec09834afd95ef25c8d4f", + "reply_trace_sha256": "876357acaf0c387a81610d1643136122ecd59dd171504a67bbf83984d1135efe", + "task_tokens_catalog_included_estimated": 1154974, + "post_discovery_tokens_estimated": 9921, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 198905, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 14002, + "tool_calls": 2, + "elapsed_seconds": 1.1498196669999743, + "cpu_seconds_waited_subtree": 1.325962, + "peak_process_group_rss_bytes_sampled": 617168896, + "peak_process_tree_rss_bytes_sampled": 727924736, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 26, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 118.07429200000001, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 140.234916, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 142, + "reply_tokens_estimated": 4568, + "request_bytes_compact_json_line": 508, + "reply_bytes_observed_wire": 17948, + "reply_bytes_reserialized_json_line": 17948, + "elapsed_ms": 777.3016249999998, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 106, + "reply_tokens_estimated": 5105, + "request_bytes_compact_json_line": 327, + "reply_bytes_observed_wire": 19577, + "reply_bytes_reserialized_json_line": 19577, + "elapsed_ms": 5.165166999999883, + "is_error": false + } + ] + }, + { + "name": "r2-theme-summary", + "trial": 2, + "artifact": "theme", + "detail": "summary", + "evidence_id": "ev_4793658938c5d3e27870a764dc1f5a1211d12ec5e86f86bf072cf856db72b5dc", + "map_sha256": "ceef8194eaca339e55f35cf2decf7ceffa121c7744ee1fffd976a07d39477b32", + "generated_script_sha256": "10ecf4c66ae103222a7c146ad9b945dc83e78b6c78ce05021296e3c2f799bd5b", + "answer_sha256": "b75f60d46b92295b8d2e3c432b5b128a330734d4513feaff0b626ed3fc17d6e1", + "receipt_sha256": "9aeb47885d5d02e7a002a95ecd5ed042e31f4c7087737da267b48e8e2e1df213", + "reply_trace_sha256": "cb68a9b3f817c58e75e45a547746a30e94cc08b2c4d54a47dd96d947c5fc237c", + "task_tokens_catalog_included_estimated": 1154878, + "post_discovery_tokens_estimated": 9825, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 198809, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 13906, + "tool_calls": 2, + "elapsed_seconds": 0.9931398749904474, + "cpu_seconds_waited_subtree": 1.150327, + "peak_process_group_rss_bytes_sampled": 507412480, + "peak_process_tree_rss_bytes_sampled": 568426496, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 23, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 117.2985, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 139.67425, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 139, + "reply_tokens_estimated": 4562, + "request_bytes_compact_json_line": 493, + "reply_bytes_observed_wire": 17846, + "reply_bytes_reserialized_json_line": 17846, + "elapsed_ms": 592.3672079999999, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 107, + "reply_tokens_estimated": 5017, + "request_bytes_compact_json_line": 325, + "reply_bytes_observed_wire": 19501, + "reply_bytes_reserialized_json_line": 19501, + "elapsed_ms": 5.385041000000001, + "is_error": false + } + ] + }, + { + "name": "r2-theme-complete", + "trial": 2, + "artifact": "theme", + "detail": "complete", + "evidence_id": "ev_4793658938c5d3e27870a764dc1f5a1211d12ec5e86f86bf072cf856db72b5dc", + "map_sha256": "ceef8194eaca339e55f35cf2decf7ceffa121c7744ee1fffd976a07d39477b32", + "generated_script_sha256": "10ecf4c66ae103222a7c146ad9b945dc83e78b6c78ce05021296e3c2f799bd5b", + "answer_sha256": "b75f60d46b92295b8d2e3c432b5b128a330734d4513feaff0b626ed3fc17d6e1", + "receipt_sha256": "66c72b8090b1a16318f71b78eff8b09cc031178b78869775e0ddce1bd1e5244f", + "reply_trace_sha256": "c2a695ba9364be19dd94e3104cb31a08779ef1cf4cf7eabdbf88d7a58dcc68ec", + "task_tokens_catalog_included_estimated": 1196400, + "post_discovery_tokens_estimated": 51347, + "used_tool_definitions_tokens_estimated": 8408, + "post_discovery_plus_used_definitions_tokens_estimated": 59755, + "hypothetical_used_input_only_definitions_tokens_estimated": 548, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 51895, + "tool_calls": 1, + "elapsed_seconds": 0.9780228329909733, + "cpu_seconds_waited_subtree": 1.125509, + "peak_process_group_rss_bytes_sampled": 522207232, + "peak_process_tree_rss_bytes_sampled": 571965440, + "maximum_group_members_sampled": 2, + "maximum_tree_members_sampled": 3, + "sample_count": 22, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 117.50962499999999, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 140.177416, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 135, + "reply_tokens_estimated": 51212, + "request_bytes_compact_json_line": 474, + "reply_bytes_observed_wire": 159113, + "reply_bytes_reserialized_json_line": 159113, + "elapsed_ms": 583.343208, + "is_error": false + } + ] + }, + { + "name": "r2-index-summary", + "trial": 2, + "artifact": "index", + "detail": "summary", + "evidence_id": "ev_7385676797553c25b8fb5b1428caa24b9a1f943a6fc4f69462612bad8240043c", + "map_sha256": "f0de6886f6a49758f53c7f2de0b77fbff6d72c719a7c32ac23d13453ebe7b3e8", + "generated_script_sha256": "937f792d1849b69a6e0741234428b22b4884ec0157ea31dd6949dd1bacc46456", + "answer_sha256": "c8a0e46117f873873c1756daf5f44b4c9f855335c0beb5758e51c257282234c7", + "receipt_sha256": "a90a6d59dc69ae1e51e4ce663b078edc2fc76208eadcc4c01273f62f2225efb4", + "reply_trace_sha256": "0df82d35ab341d2a3daccc75145f8c19c7a7aee0d36693877927b482b5b65ea1", + "task_tokens_catalog_included_estimated": 1154851, + "post_discovery_tokens_estimated": 9798, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 198782, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 13879, + "tool_calls": 2, + "elapsed_seconds": 1.1378968750068452, + "cpu_seconds_waited_subtree": 1.293817, + "peak_process_group_rss_bytes_sampled": 568672256, + "peak_process_tree_rss_bytes_sampled": 693518336, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 26, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 117.28758300000001, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 140.665125, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 136, + "reply_tokens_estimated": 4566, + "request_bytes_compact_json_line": 476, + "reply_bytes_observed_wire": 17760, + "reply_bytes_reserialized_json_line": 17760, + "elapsed_ms": 739.5446670000001, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 105, + "reply_tokens_estimated": 4991, + "request_bytes_compact_json_line": 327, + "reply_bytes_observed_wire": 19381, + "reply_bytes_reserialized_json_line": 19381, + "elapsed_ms": 5.537582999999927, + "is_error": false + } + ] + }, + { + "name": "r2-index-complete", + "trial": 2, + "artifact": "index", + "detail": "complete", + "evidence_id": "ev_7385676797553c25b8fb5b1428caa24b9a1f943a6fc4f69462612bad8240043c", + "map_sha256": "f0de6886f6a49758f53c7f2de0b77fbff6d72c719a7c32ac23d13453ebe7b3e8", + "generated_script_sha256": "937f792d1849b69a6e0741234428b22b4884ec0157ea31dd6949dd1bacc46456", + "answer_sha256": "c8a0e46117f873873c1756daf5f44b4c9f855335c0beb5758e51c257282234c7", + "receipt_sha256": "c564d0376ec35023fa3f7d8d4d8f7604d33f4b1334f375e18ef0798c9f3e5a20", + "reply_trace_sha256": "cfa4cf8a5b8410212accf79c96ecda36ed59e8010932661ad11c16d310d014ba", + "task_tokens_catalog_included_estimated": 4354894, + "post_discovery_tokens_estimated": 3209841, + "used_tool_definitions_tokens_estimated": 8408, + "post_discovery_plus_used_definitions_tokens_estimated": 3218249, + "hypothetical_used_input_only_definitions_tokens_estimated": 548, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 3210389, + "tool_calls": 1, + "elapsed_seconds": 1.6278382080054143, + "cpu_seconds_waited_subtree": 1.8145129999999998, + "peak_process_group_rss_bytes_sampled": 814170112, + "peak_process_tree_rss_bytes_sampled": 814170112, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 37, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 118.947125, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 140.20320800000002, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 132, + "reply_tokens_estimated": 3209709, + "request_bytes_compact_json_line": 457, + "reply_bytes_observed_wire": 9121044, + "reply_bytes_reserialized_json_line": 9121044, + "elapsed_ms": 1211.5193749999999, + "is_error": false + } + ] + }, + { + "name": "r2-DateSelect-summary", + "trial": 2, + "artifact": "DateSelect", + "detail": "summary", + "evidence_id": "ev_9b82f05c94b00018545c9b60c492cdbcba8f5bf1428c22f9ee243ee046877386", + "map_sha256": "e1939ef36214ff46263d3bdaeb99d60a46c75714a6b6220ed27c9be2d24d6cc8", + "generated_script_sha256": "9e45139dd0b510ad7014188a5e9d11b307c2ac15dfdc761050388e76e94759bf", + "answer_sha256": "24064ff1089893082998aba4a03cda18bd7e010663921436858fd40efb94516d", + "receipt_sha256": "a28c9ad829c7f117135489149167dc4b3f0209ea9dbb1d601b64fd3871b2bcca", + "reply_trace_sha256": "f4ac36282b210340aef86666f619f360cc6ae0f6f63e08f0af4b20de249d0b74", + "task_tokens_catalog_included_estimated": 1154974, + "post_discovery_tokens_estimated": 9921, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 198905, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 14002, + "tool_calls": 2, + "elapsed_seconds": 1.1604734580032527, + "cpu_seconds_waited_subtree": 1.341315, + "peak_process_group_rss_bytes_sampled": 629407744, + "peak_process_tree_rss_bytes_sampled": 732086272, + "maximum_group_members_sampled": 2, + "maximum_tree_members_sampled": 3, + "sample_count": 26, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 117.813166, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 140.96874999999997, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 142, + "reply_tokens_estimated": 4568, + "request_bytes_compact_json_line": 508, + "reply_bytes_observed_wire": 17948, + "reply_bytes_reserialized_json_line": 17948, + "elapsed_ms": 780.4921250000001, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 106, + "reply_tokens_estimated": 5105, + "request_bytes_compact_json_line": 327, + "reply_bytes_observed_wire": 19577, + "reply_bytes_reserialized_json_line": 19577, + "elapsed_ms": 5.372749999999996, + "is_error": false + } + ] + }, + { + "name": "r2-DateSelect-complete", + "trial": 2, + "artifact": "DateSelect", + "detail": "complete", + "evidence_id": "ev_9b82f05c94b00018545c9b60c492cdbcba8f5bf1428c22f9ee243ee046877386", + "map_sha256": "e1939ef36214ff46263d3bdaeb99d60a46c75714a6b6220ed27c9be2d24d6cc8", + "generated_script_sha256": "9e45139dd0b510ad7014188a5e9d11b307c2ac15dfdc761050388e76e94759bf", + "answer_sha256": "24064ff1089893082998aba4a03cda18bd7e010663921436858fd40efb94516d", + "receipt_sha256": "983ff78e144fc17758321d84e0bc5b8bb778153fcc976e502521a8b865c0c6c9", + "reply_trace_sha256": "5a8451f63a4f376e07a1311e7dd6a99b6c92e799e84192b371c49c1c4cf4fad0", + "task_tokens_catalog_included_estimated": 1155932, + "post_discovery_tokens_estimated": 10879, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 199863, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 14960, + "tool_calls": 3, + "elapsed_seconds": 1.2022515409917105, + "cpu_seconds_waited_subtree": 1.362353, + "peak_process_group_rss_bytes_sampled": 608813056, + "peak_process_tree_rss_bytes_sampled": 729481216, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 27, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 117.905917, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 140.61375, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 138, + "reply_tokens_estimated": 776, + "request_bytes_compact_json_line": 489, + "reply_bytes_observed_wire": 3310, + "reply_bytes_reserialized_json_line": 3310, + "elapsed_ms": 798.153916, + "is_error": true + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 107, + "reply_tokens_estimated": 4647, + "request_bytes_compact_json_line": 324, + "reply_bytes_observed_wire": 18288, + "reply_bytes_reserialized_json_line": 18288, + "elapsed_ms": 6.067457999999988, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 106, + "reply_tokens_estimated": 5105, + "request_bytes_compact_json_line": 327, + "reply_bytes_observed_wire": 19577, + "reply_bytes_reserialized_json_line": 19577, + "elapsed_ms": 1.393000000000029, + "is_error": false + } + ] + }, + { + "name": "r3-theme-complete", + "trial": 3, + "artifact": "theme", + "detail": "complete", + "evidence_id": "ev_4793658938c5d3e27870a764dc1f5a1211d12ec5e86f86bf072cf856db72b5dc", + "map_sha256": "ceef8194eaca339e55f35cf2decf7ceffa121c7744ee1fffd976a07d39477b32", + "generated_script_sha256": "10ecf4c66ae103222a7c146ad9b945dc83e78b6c78ce05021296e3c2f799bd5b", + "answer_sha256": "b75f60d46b92295b8d2e3c432b5b128a330734d4513feaff0b626ed3fc17d6e1", + "receipt_sha256": "cf31dd97d8cd028b2fbd609e24929423e976285bfd0a04619589550d3d325e9b", + "reply_trace_sha256": "1b9f35ce69024eeaf3069d27c2d69524ddc0977d36b359794e9a0b55798bb086", + "task_tokens_catalog_included_estimated": 1196400, + "post_discovery_tokens_estimated": 51347, + "used_tool_definitions_tokens_estimated": 8408, + "post_discovery_plus_used_definitions_tokens_estimated": 59755, + "hypothetical_used_input_only_definitions_tokens_estimated": 548, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 51895, + "tool_calls": 1, + "elapsed_seconds": 0.9534965420025401, + "cpu_seconds_waited_subtree": 1.128418, + "peak_process_group_rss_bytes_sampled": 543539200, + "peak_process_tree_rss_bytes_sampled": 582778880, + "maximum_group_members_sampled": 4, + "maximum_tree_members_sampled": 4, + "sample_count": 22, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 117.75683399999998, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 139.257292, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 135, + "reply_tokens_estimated": 51212, + "request_bytes_compact_json_line": 474, + "reply_bytes_observed_wire": 159113, + "reply_bytes_reserialized_json_line": 159113, + "elapsed_ms": 590.0055830000001, + "is_error": false + } + ] + }, + { + "name": "r3-theme-summary", + "trial": 3, + "artifact": "theme", + "detail": "summary", + "evidence_id": "ev_4793658938c5d3e27870a764dc1f5a1211d12ec5e86f86bf072cf856db72b5dc", + "map_sha256": "ceef8194eaca339e55f35cf2decf7ceffa121c7744ee1fffd976a07d39477b32", + "generated_script_sha256": "10ecf4c66ae103222a7c146ad9b945dc83e78b6c78ce05021296e3c2f799bd5b", + "answer_sha256": "b75f60d46b92295b8d2e3c432b5b128a330734d4513feaff0b626ed3fc17d6e1", + "receipt_sha256": "0f2f8aa6cc82e51cce00f9bb73cfb150bcabb12b1d096a9d0f657dcb8215f889", + "reply_trace_sha256": "3a93a35b0753cbdd47431bdc222cd9e0a5dc872efa971838bbec195dbcce90e5", + "task_tokens_catalog_included_estimated": 1154878, + "post_discovery_tokens_estimated": 9825, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 198809, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 13906, + "tool_calls": 2, + "elapsed_seconds": 0.9846506659960141, + "cpu_seconds_waited_subtree": 1.1376249999999999, + "peak_process_group_rss_bytes_sampled": 499728384, + "peak_process_tree_rss_bytes_sampled": 562528256, + "maximum_group_members_sampled": 2, + "maximum_tree_members_sampled": 3, + "sample_count": 23, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 118.36849999999998, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 139.385667, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 139, + "reply_tokens_estimated": 4562, + "request_bytes_compact_json_line": 493, + "reply_bytes_observed_wire": 17846, + "reply_bytes_reserialized_json_line": 17846, + "elapsed_ms": 588.1954169999999, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 107, + "reply_tokens_estimated": 5017, + "request_bytes_compact_json_line": 325, + "reply_bytes_observed_wire": 19501, + "reply_bytes_reserialized_json_line": 19501, + "elapsed_ms": 5.76400000000001, + "is_error": false + } + ] + }, + { + "name": "r3-index-complete", + "trial": 3, + "artifact": "index", + "detail": "complete", + "evidence_id": "ev_7385676797553c25b8fb5b1428caa24b9a1f943a6fc4f69462612bad8240043c", + "map_sha256": "f0de6886f6a49758f53c7f2de0b77fbff6d72c719a7c32ac23d13453ebe7b3e8", + "generated_script_sha256": "937f792d1849b69a6e0741234428b22b4884ec0157ea31dd6949dd1bacc46456", + "answer_sha256": "c8a0e46117f873873c1756daf5f44b4c9f855335c0beb5758e51c257282234c7", + "receipt_sha256": "fedb8ce284efb8298a44a5c42a09b337a077efa64748459636d640367d0ef990", + "reply_trace_sha256": "be8243c00103732524b21a1abbf06ffc4fe17ac8e100a15f09c90fc4304ebcf8", + "task_tokens_catalog_included_estimated": 4354894, + "post_discovery_tokens_estimated": 3209841, + "used_tool_definitions_tokens_estimated": 8408, + "post_discovery_plus_used_definitions_tokens_estimated": 3218249, + "hypothetical_used_input_only_definitions_tokens_estimated": 548, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 3210389, + "tool_calls": 1, + "elapsed_seconds": 1.6246519580017775, + "cpu_seconds_waited_subtree": 1.813149, + "peak_process_group_rss_bytes_sampled": 806813696, + "peak_process_tree_rss_bytes_sampled": 806813696, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 38, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 119.61375000000001, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 139.36666599999998, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 132, + "reply_tokens_estimated": 3209709, + "request_bytes_compact_json_line": 457, + "reply_bytes_observed_wire": 9121044, + "reply_bytes_reserialized_json_line": 9121044, + "elapsed_ms": 1213.944167, + "is_error": false + } + ] + }, + { + "name": "r3-index-summary", + "trial": 3, + "artifact": "index", + "detail": "summary", + "evidence_id": "ev_7385676797553c25b8fb5b1428caa24b9a1f943a6fc4f69462612bad8240043c", + "map_sha256": "f0de6886f6a49758f53c7f2de0b77fbff6d72c719a7c32ac23d13453ebe7b3e8", + "generated_script_sha256": "937f792d1849b69a6e0741234428b22b4884ec0157ea31dd6949dd1bacc46456", + "answer_sha256": "c8a0e46117f873873c1756daf5f44b4c9f855335c0beb5758e51c257282234c7", + "receipt_sha256": "4268023730f752a394d23aed93d6ad5f6e2972a012f9ea1c3133ea8aa052e349", + "reply_trace_sha256": "4c107c7599221a2d066d139b22e9343d2313a2876665cc8bbf4afcfa23af0c33", + "task_tokens_catalog_included_estimated": 1154851, + "post_discovery_tokens_estimated": 9798, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 198782, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 13879, + "tool_calls": 2, + "elapsed_seconds": 1.1233145829901332, + "cpu_seconds_waited_subtree": 1.289125, + "peak_process_group_rss_bytes_sampled": 610041856, + "peak_process_tree_rss_bytes_sampled": 694534144, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 25, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 116.732417, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 139.42274999999998, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 136, + "reply_tokens_estimated": 4566, + "request_bytes_compact_json_line": 476, + "reply_bytes_observed_wire": 17760, + "reply_bytes_reserialized_json_line": 17760, + "elapsed_ms": 739.232125, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 105, + "reply_tokens_estimated": 4991, + "request_bytes_compact_json_line": 327, + "reply_bytes_observed_wire": 19381, + "reply_bytes_reserialized_json_line": 19381, + "elapsed_ms": 5.22625000000005, + "is_error": false + } + ] + }, + { + "name": "r3-DateSelect-complete", + "trial": 3, + "artifact": "DateSelect", + "detail": "complete", + "evidence_id": "ev_9b82f05c94b00018545c9b60c492cdbcba8f5bf1428c22f9ee243ee046877386", + "map_sha256": "e1939ef36214ff46263d3bdaeb99d60a46c75714a6b6220ed27c9be2d24d6cc8", + "generated_script_sha256": "9e45139dd0b510ad7014188a5e9d11b307c2ac15dfdc761050388e76e94759bf", + "answer_sha256": "24064ff1089893082998aba4a03cda18bd7e010663921436858fd40efb94516d", + "receipt_sha256": "12ed80861a944f6ad7843397aa21ba2deba0ed95c6a7ebd5381acdc9994c6350", + "reply_trace_sha256": "8fe553747e9fb003023b30d29f72cc83321b370e1c4cb9bd373a8ec2b6e2578a", + "task_tokens_catalog_included_estimated": 1155932, + "post_discovery_tokens_estimated": 10879, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 199863, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 14960, + "tool_calls": 3, + "elapsed_seconds": 1.1992472090059891, + "cpu_seconds_waited_subtree": 1.351098, + "peak_process_group_rss_bytes_sampled": 605306880, + "peak_process_tree_rss_bytes_sampled": 723009536, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 27, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 117.77824999999999, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 139.39975, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 138, + "reply_tokens_estimated": 776, + "request_bytes_compact_json_line": 489, + "reply_bytes_observed_wire": 3310, + "reply_bytes_reserialized_json_line": 3310, + "elapsed_ms": 794.9873750000002, + "is_error": true + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 107, + "reply_tokens_estimated": 4647, + "request_bytes_compact_json_line": 324, + "reply_bytes_observed_wire": 18288, + "reply_bytes_reserialized_json_line": 18288, + "elapsed_ms": 6.362374999999929, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 106, + "reply_tokens_estimated": 5105, + "request_bytes_compact_json_line": 327, + "reply_bytes_observed_wire": 19577, + "reply_bytes_reserialized_json_line": 19577, + "elapsed_ms": 1.4037920000000668, + "is_error": false + } + ] + }, + { + "name": "r3-DateSelect-summary", + "trial": 3, + "artifact": "DateSelect", + "detail": "summary", + "evidence_id": "ev_9b82f05c94b00018545c9b60c492cdbcba8f5bf1428c22f9ee243ee046877386", + "map_sha256": "e1939ef36214ff46263d3bdaeb99d60a46c75714a6b6220ed27c9be2d24d6cc8", + "generated_script_sha256": "9e45139dd0b510ad7014188a5e9d11b307c2ac15dfdc761050388e76e94759bf", + "answer_sha256": "24064ff1089893082998aba4a03cda18bd7e010663921436858fd40efb94516d", + "receipt_sha256": "98bb7ee1c73f7b394416de1bc2f5854dcc558ad08b1c0bfff57e738b85791966", + "reply_trace_sha256": "90eb38c7fdd80bbdefa148348d9b83df3cf12a6f818128025285f54d5865d7bb", + "task_tokens_catalog_included_estimated": 1154974, + "post_discovery_tokens_estimated": 9921, + "used_tool_definitions_tokens_estimated": 188984, + "post_discovery_plus_used_definitions_tokens_estimated": 198905, + "hypothetical_used_input_only_definitions_tokens_estimated": 4081, + "hypothetical_post_discovery_plus_used_input_only_definitions_tokens_estimated": 14002, + "tool_calls": 2, + "elapsed_seconds": 1.1433994160033762, + "cpu_seconds_waited_subtree": 1.323411, + "peak_process_group_rss_bytes_sampled": 584908800, + "peak_process_tree_rss_bytes_sampled": 702742528, + "maximum_group_members_sampled": 3, + "maximum_tree_members_sampled": 3, + "sample_count": 26, + "exit_code": 0, + "cleanup": "group_empty", + "stop_reason": null, + "calls": [ + { + "operation": "initialize", + "post_discovery": false, + "request_tokens_estimated": 48, + "reply_tokens_estimated": 640, + "request_bytes_compact_json_line": 176, + "reply_bytes_observed_wire": 3149, + "reply_bytes_reserialized_json_line": 3149, + "elapsed_ms": 119.06162500000002, + "is_error": false + }, + { + "operation": "notifications/initialized", + "post_discovery": false, + "request_tokens_estimated": 14, + "reply_tokens_estimated": 0, + "request_bytes_compact_json_line": 55, + "reply_bytes_observed_wire": 0, + "reply_bytes_reserialized_json_line": 0, + "elapsed_ms": 0, + "is_error": false + }, + { + "operation": "tools/list", + "post_discovery": false, + "request_tokens_estimated": 21, + "reply_tokens_estimated": 1144330, + "request_bytes_compact_json_line": 59, + "reply_bytes_observed_wire": 4487850, + "reply_bytes_reserialized_json_line": 4487850, + "elapsed_ms": 138.797542, + "is_error": false + }, + { + "operation": "trace_web_source_location", + "post_discovery": true, + "request_tokens_estimated": 142, + "reply_tokens_estimated": 4568, + "request_bytes_compact_json_line": 508, + "reply_bytes_observed_wire": 17948, + "reply_bytes_reserialized_json_line": 17948, + "elapsed_ms": 775.976625, + "is_error": false + }, + { + "operation": "inspect_analysis_view", + "post_discovery": true, + "request_tokens_estimated": 106, + "reply_tokens_estimated": 5105, + "request_bytes_compact_json_line": 327, + "reply_bytes_observed_wire": 19577, + "reply_bytes_reserialized_json_line": 19577, + "elapsed_ms": 5.66579200000001, + "is_error": false + } + ] + } + ] +} diff --git a/scripts/verify/analysis/source-location-delivery/task.mjs b/scripts/verify/analysis/source-location-delivery/task.mjs new file mode 100644 index 0000000000..76045aefc5 --- /dev/null +++ b/scripts/verify/analysis/source-location-delivery/task.mjs @@ -0,0 +1,426 @@ +import { spawn } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import assert from "node:assert/strict"; +import { isAbsolute, join, relative, resolve, sep } from "node:path"; + +// Acquire artifactDir with docs/verification/source-map-budgets-2026-11-09/acquire.py. +// This driver runs the public compiled stdio workflow; no target application is executed. +const [rootArgument, configText, ...extra] = process.argv.slice(2); +assert( + rootArgument && configText && extra.length === 0, + "Usage: node task.mjs REPO_ROOT CONFIG_JSON", +); +const root = resolve(rootArgument); +const config = JSON.parse(configText); +assert( + config && typeof config === "object" && !Array.isArray(config), + "CONFIG_JSON must be an object", +); +const allowed = new Set([ + "artifactDir", + "select", + "detail", + "resultPath", + "exportPath", + "maxResponseBytes", +]); +assert( + Object.keys(config).every((key) => allowed.has(key)), + "Unknown task configuration field", +); +assert( + typeof config.artifactDir === "string" && isAbsolute(config.artifactDir), + "artifactDir must be an absolute acquired-artifact directory", +); +assert( + typeof config.select === "string" && + config.select.length > 0 && + config.select.length <= 256, + "select must be a nonempty artifact filename selector, at most 256 characters", +); +assert( + config.detail === "complete" || config.detail === "summary", + "detail must be complete or summary", +); +for (const key of ["resultPath", "exportPath"]) { + assert( + config[key] === undefined || + (typeof config[key] === "string" && isAbsolute(config[key])), + `${key} must be an absolute path when selected`, + ); +} +const MAX_LINE_BYTES = 32 * 1024 * 1024; +const RPC_TIMEOUT_MS = 45000; +assert( + config.maxResponseBytes === undefined || + (Number.isSafeInteger(config.maxResponseBytes) && + config.maxResponseBytes >= 10 * 1024 * 1024 && + config.maxResponseBytes <= MAX_LINE_BYTES), + "maxResponseBytes must fit the driver receive guard: integer 10..32 MiB", +); +const sources = JSON.parse( + await readFile(join(config.artifactDir, "artifacts.json"), "utf8"), +); +assert( + Array.isArray(sources), + "artifacts.json must be the retained artifact list", +); +const selected = sources.filter( + (source) => + typeof source.path === "string" && source.path.includes(config.select), +); +assert.equal( + selected.length, + 1, + "select must identify exactly one pinned artifact", +); +const source = selected[0]; +assert( + source.generated_script, + "Selected point task requires an acquired generated script and analyst manifest", +); +const localFile = (path) => { + assert( + typeof path === "string" && !isAbsolute(path), + "Artifact manifest paths must be relative", + ); + const resolved = resolve(config.artifactDir, path); + assert( + !relative(config.artifactDir, resolved).startsWith(".." + sep) && + resolved !== resolve(config.artifactDir, ".."), + "Artifact path must stay within artifactDir", + ); + return resolved; +}; +const mapPath = localFile(source.path); +const bytes = await readFile(mapPath); +assert.equal(bytes.length, source.bytes); +assert.equal(createHash("sha256").update(bytes).digest("hex"), source.sha256); +const artifact = JSON.parse(bytes.toString("utf8")); +const manifestPath = localFile(source.generated_script.path + ".manifest.json"); +const child = spawn(process.execPath, [join(root, "scripts/rea.mjs"), "mcp"], { + cwd: root, + env: { + ...process.env, + REA_LOG_LEVEL: "silent", + REA_MCP_TOOL_CATALOG_PROFILE: "full", + REA_MCP_INPUT_SCHEMA_PROFILE: "compact", + REA_MCP_OUTPUT_SCHEMA_PROFILE: "full", + ...(config.maxResponseBytes === undefined + ? {} + : { REA_MCP_MAX_RESPONSE_BYTES: String(config.maxResponseBytes) }), + HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper", + GHIDRA_INSTALL_DIR: process.execPath, + }, + stdio: ["pipe", "pipe", "pipe"], +}); +let buffer = "", + sequence = 0, + stderr = ""; +const pending = new Map(), + calls = []; +const failPending = (error) => { + for (const request of pending.values()) request.reject(error); + pending.clear(); +}; +child.stdout.setEncoding("utf8"); +child.stderr.setEncoding("utf8"); +child.stderr.on("data", (chunk) => { + stderr = (stderr + chunk).slice(-8192); +}); +child.once("error", failPending); +child.once("exit", (code, signal) => + failPending( + new Error( + `MCP server exited (${String(code)}, ${String(signal)}): ${stderr}`, + ), + ), +); +child.stdin.on("error", failPending); +child.stdout.on("data", (chunk) => { + try { + buffer += chunk; + assert( + Buffer.byteLength(buffer) < MAX_LINE_BYTES, + "32 MiB RPC line guard exceeded", + ); + let end; + while ((end = buffer.indexOf("\n")) >= 0) { + const line = buffer.slice(0, end); + buffer = buffer.slice(end + 1); + if (!line) continue; + const payload = JSON.parse(line), + request = pending.get(payload.id); + if (request) { + pending.delete(payload.id); + request.resolve({ payload, wireBytes: Buffer.byteLength(line) + 1 }); + } + } + } catch (error) { + failPending(error); + } +}); +const rpc = async (method, params = {}) => { + const request = { jsonrpc: "2.0", id: ++sequence, method, params }, + start = performance.now(); + const waiting = new Promise((resolveReply, rejectReply) => { + const timer = setTimeout(() => { + pending.delete(request.id); + rejectReply(new Error(`RPC timeout ${method}: ${stderr}`)); + }, RPC_TIMEOUT_MS); + pending.set(request.id, { + resolve: (value) => { + clearTimeout(timer); + resolveReply(value); + }, + reject: (error) => { + clearTimeout(timer); + rejectReply(error); + }, + }); + }); + child.stdin.write(JSON.stringify(request) + "\n"); + const { payload, wireBytes } = await waiting; + assert(!payload.error, JSON.stringify(payload.error)); + calls.push({ + request, + reply: payload, + wire_bytes: wireBytes, + elapsed_ms: performance.now() - start, + }); + return payload.result; +}; +const errorOf = (r) => + r.isError + ? JSON.parse(r.content.find((c) => c.type === "text").text).error + : undefined; +const contentOf = (r) => { + assert(!r.isError, JSON.stringify(errorOf(r))); + return r.structuredContent; +}; +try { + await rpc("initialize", { + protocolVersion: "2025-03-26", + capabilities: {}, + clientInfo: { name: "source-location-completed-task", version: "1" }, + }); + const initialized = { jsonrpc: "2.0", method: "notifications/initialized" }; + child.stdin.write(JSON.stringify(initialized) + "\n"); + calls.push({ + request: initialized, + reply: null, + wire_bytes: 0, + elapsed_ms: 0, + }); + const catalog = await rpc("tools/list"); + const definition = catalog.tools.find( + (t) => t.name === "trace_web_source_location", + ); + assert(definition); + const args = { + manifest_path: manifestPath, + script_index: 0, + source_map: { path: mapPath, url: source.url }, + generated_position: source.point, + ...(config.detail === "summary" ? { detail: "summary" } : {}), + }; + const initial = await rpc("tools/call", { + name: "trace_web_source_location", + arguments: args, + }); + const error = errorOf(initial); + let evidenceId, + report, + metadata = [], + coverage; + if (error) { + assert.equal(error.code, "resource_constraint"); + const limits = error.details.reported_limits; + evidenceId = limits.evidence_id; + assert(evidenceId); + const continuation = limits.continuation; + assert(continuation, "new source-location continuation required"); + const first = contentOf( + await rpc("tools/call", { + name: continuation.name, + arguments: continuation.arguments, + }), + ).normalized_result; + report = first.source_location; + metadata = [...first.items]; + coverage = first.coverage; + } else { + const evidence = contentOf(initial), + result = evidence.normalized_result; + if (config.detail === "summary") { + evidenceId = result.parent_evidence_id; + report = result.summary; + metadata = [...report.matches]; + coverage = result.coverage; + } else { + assert.equal( + createHash("sha256") + .update(evidence.raw_result.source_map_text) + .digest("hex"), + source.sha256, + "Complete initial MCP response must preserve exact raw map bytes", + ); + evidenceId = evidence.evidence_id; + report = result; + metadata = [...report.matches]; + coverage = { exhausted: true }; + } + } + const reference = { kind: "retained-evidence", evidence_id: evidenceId }; + while (!coverage.exhausted) { + const page = contentOf( + await rpc("tools/call", { + name: "inspect_analysis_view", + arguments: { + source: reference, + view: { + kind: "source-location", + collection: "matches", + offset: coverage.next_offset, + limit: 64, + }, + }, + }), + ).normalized_result; + metadata.push(...page.items); + coverage = page.coverage; + } + assert.equal(report.source_map.sha256, source.sha256); + assert.equal(metadata.length, source.expected_matches.length); + const answer = []; + for (let index = 0; index < metadata.length; index++) { + const match = metadata[index], + expected = source.expected_matches[index]; + assert.equal( + match.state, + expected.generated_only ? "generated-only" : "mapped", + ); + if (match.state === "generated-only") { + answer.push(match); + continue; + } + assert.equal(match.source_index, expected.source_index); + assert.equal(match.reported_source, expected.source); + assert.equal(match.original_position.line, expected.original_line); + assert.equal(match.original_position.column, expected.original_column); + assert.equal(match.content.utf8_sha256, expected.content_sha256); + const full = artifact.sourcesContent[expected.source_index], + point = match.original_position.offset; + assert(point !== null); + let offset = Math.max(0, point - 256); + if ( + offset > 0 && + full.charCodeAt(offset) >= 0xdc00 && + full.charCodeAt(offset) <= 0xdfff + ) + offset--; + let snippet; + if ("text" in match.content) { + assert.equal( + createHash("sha256").update(match.content.text).digest("hex"), + expected.content_sha256, + ); + let end = Math.min(full.length, offset + 512); + if ( + end < full.length && + full.charCodeAt(end - 1) >= 0xd800 && + full.charCodeAt(end - 1) <= 0xdbff + ) + end--; + snippet = match.content.text.slice(offset, end); + } else { + const text = contentOf( + await rpc("tools/call", { + name: "inspect_analysis_view", + arguments: { + source: reference, + view: { + kind: "source-location", + collection: "source-text", + match_index: match.match_index, + offset, + limit: 512, + }, + }, + }), + ).normalized_result; + assert.equal(text.match.content.utf8_sha256, expected.content_sha256); + snippet = text.item.text; + } + assert.equal(snippet, full.slice(offset, offset + snippet.length)); + assert(offset <= point && offset + snippet.length >= point); + const { text: _text, ...identity } = match.content; + answer.push({ + ...match, + match_index: index, + content: { + ...identity, + code_units: full.length, + snippet: { + offset_utf16: offset, + length_utf16: snippet.length, + text: snippet, + }, + }, + }); + } + let exported = false; + if (config.exportPath) { + const result = await rpc("tools/call", { + name: "export_evidence_bundle", + arguments: { path: config.exportPath }, + }); + assert(!result.isError); + const bundle = JSON.parse(await readFile(config.exportPath, "utf8")); + const parent = bundle.records.find((r) => r.evidence_id === evidenceId); + assert(parent); + assert.equal( + createHash("sha256") + .update(parent.raw_result.source_map_text) + .digest("hex"), + source.sha256, + ); + exported = true; + } + const result = { + config, + evidence_id: evidenceId, + source_map_sha256: source.sha256, + lookup: report.lookup, + matches: answer, + export_verified: exported, + calls: calls.map((c) => ({ + method: c.request.method, + tool: c.request.params?.name, + wire_bytes: c.wire_bytes, + elapsed_ms: c.elapsed_ms, + isError: c.reply?.result?.isError ?? false, + })), + }; + if (config.resultPath) + await writeFile( + config.resultPath, + JSON.stringify({ config, definition, calls, answer: result }), + { flag: "wx" }, + ); + console.log(JSON.stringify(result)); +} finally { + child.stdin.end(); + if (child.exitCode === null && child.signalCode === null) + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + child.kill("SIGTERM"); + reject(Error("MCP close timeout")); + }, 5000); + child.once("exit", () => { + clearTimeout(timer); + resolve(); + }); + }); +} diff --git a/scripts/verify/architecture-guards.mjs b/scripts/verify/architecture-guards.mjs index 07a51ce615..821073913a 100644 --- a/scripts/verify/architecture-guards.mjs +++ b/scripts/verify/architecture-guards.mjs @@ -140,10 +140,10 @@ check( // inputBytes values with a different policy (an overridable pwntools admission, // historical capture intake) are not this budget. check( - "inline resident-input budget (use BOUNDED_PRODUCER_LIMITS from src/process/LinuxResourceLimits)", + "inline resident-input budget (use BOUNDED_PRODUCER_LIMITS from src/domain/producerResourceLimits)", files, /input_bytes:\s*32 \* 1024 \* 1024/, - ["src/process/LinuxResourceLimits.ts"], + ["src/domain/producerResourceLimits.ts"], ); // Fixed-arity backtick keys audited 2026-10-10: every prefix field is NUL-free diff --git a/scripts/verify/managed/config.mjs b/scripts/verify/managed/config.mjs index 8c9906e32c..a7e3984dd2 100644 --- a/scripts/verify/managed/config.mjs +++ b/scripts/verify/managed/config.mjs @@ -1,52 +1,3 @@ -export const inspectionLimits = { - referenceOffset: 0, - referenceLimit: 100, - resourceOffset: 0, - resourceLimit: 100, - attributeOffset: 0, - attributeLimit: 100, - maxMetadataBytes: 1024 * 1024, - maxTableRows: 1_000, - maxHeapItemBytes: 1024 * 1024, -}; - -export const memberLimits = { - typeOffset: 0, - typeLimit: 100, - methodOffset: 0, - methodLimit: 100, - fieldOffset: 0, - fieldLimit: 100, - memberRefOffset: 0, - memberRefLimit: 100, - edgeOffset: 0, - edgeLimit: 100, - instructionAnchorLimit: 100, - maxMetadataBytes: 1024 * 1024, - maxTableRows: 1_000, - maxHeapItemBytes: 1024 * 1024, - maxMethodBodyBytes: 1024 * 1024, - maxMethodInstructions: 1_000, -}; - -export const nativeBoundaryLimits = { - moduleRefOffset: 0, - moduleRefLimit: 100, - importOffset: 0, - importLimit: 100, - implementationOffset: 0, - implementationLimit: 100, - maxMetadataBytes: 1024 * 1024, - maxTableRows: 1_000, - maxHeapItemBytes: 1024 * 1024, -}; - -export const comparisonLimits = { - max_method_matches: 100, - max_field_matches: 100, - max_candidates: 20, -}; - export const defaultIlBody = Buffer.from([ 0x32, 0x02, 0x7b, 0x01, 0x00, 0x00, 0x04, 0x28, 0x01, 0x00, 0x00, 0x0a, 0x2a, ]); diff --git a/scripts/verify/managed/conformance.mjs b/scripts/verify/managed/conformance.mjs index a7a35cb96a..38bd5340d7 100644 --- a/scripts/verify/managed/conformance.mjs +++ b/scripts/verify/managed/conformance.mjs @@ -19,13 +19,7 @@ import { buildNativePeFixture, } from "../../fixtures/managed/pe.mjs"; import { createManagedConformanceSupport } from "./support.mjs"; -import { - comparisonLimits, - defaultIlBody, - inspectionLimits, - memberLimits, - nativeBoundaryLimits, -} from "./config.mjs"; +import { defaultIlBody } from "./config.mjs"; import { createManagedCompletionReport } from "./completion-report.mjs"; import { completeVerifierRun, @@ -45,8 +39,6 @@ const { sha256, } = createManagedConformanceSupport({ workspace, - inspectionLimits, - memberLimits, defaultIlBody, }); @@ -58,7 +50,6 @@ try { const modernArtifact = inspectManagedArtifactBytes( modern.bytes, modern.target, - inspectionLimits, ); assert.equal(modernArtifact.classification.status, "managed"); assert.equal(modernArtifact.classification.runtime_family, "modern-dotnet"); @@ -83,7 +74,6 @@ try { const frameworkArtifact = inspectManagedArtifactBytes( framework.bytes, framework.target, - inspectionLimits, ); assert.equal( frameworkArtifact.classification.runtime_family, @@ -93,7 +83,6 @@ try { const frameworkBoundaries = inspectManagedNativeBoundariesBytes( framework.bytes, framework.target, - nativeBoundaryLimits, ); assert.equal(frameworkBoundaries.pinvoke_imports.length, 1); assert.equal( @@ -108,14 +97,13 @@ try { const frameworkMembers = inspectManagedMembersBytes( framework.bytes, framework.target, - memberLimits, ); const frameworkArtifactEvidence = createEvidence( framework.target, MANAGED_STATIC_PROVIDER, { operation: "inspect_managed_artifact", - parameters: inspectionLimits, + parameters: {}, result: frameworkArtifact, rawResult: null, limitations: frameworkArtifact.limitations, @@ -127,7 +115,7 @@ try { MANAGED_STATIC_PROVIDER, { operation: "inspect_managed_members", - parameters: memberLimits, + parameters: {}, result: frameworkMembers, rawResult: null, limitations: frameworkMembers.limitations, @@ -139,7 +127,7 @@ try { MANAGED_STATIC_PROVIDER, { operation: "inspect_managed_native_boundaries", - parameters: nativeBoundaryLimits, + parameters: {}, result: frameworkBoundaries, rawResult: null, limitations: frameworkBoundaries.limitations, @@ -221,7 +209,6 @@ try { const r2rArtifact = inspectManagedArtifactBytes( readyToRun.bytes, readyToRun.target, - inspectionLimits, ); assert.equal(r2rArtifact.pe.architecture, "x86_64"); assert.equal( @@ -231,7 +218,6 @@ try { const r2rBoundaries = inspectManagedNativeBoundariesBytes( readyToRun.bytes, readyToRun.target, - nativeBoundaryLimits, ); assert.equal(r2rBoundaries.summary.ready_to_run, true); assert.equal(r2rBoundaries.native_implementations[0]?.code_type, "native"); @@ -248,7 +234,6 @@ try { const obfuscatedMembers = inspectManagedMembersBytes( obfuscated.bytes, obfuscated.target, - memberLimits, ); assert.equal(obfuscatedMembers.types[0]?.full_name, "Fixture.ꙮType"); assert.equal(obfuscatedMembers.methods[0]?.name, "λ⛧"); @@ -309,11 +294,6 @@ try { const comparison = await compareManagedMemberPaths({ leftPath: left.path, rightPath: right.path, - memberLimits: { - maxFileBytes: 1024 * 1024, - ...memberLimits, - }, - comparisonLimits, }); assert.equal(comparison.ok, true); const comparisonResult = comparison.value.normalized_result; @@ -341,11 +321,6 @@ try { const partialBodyComparison = await compareManagedMemberPaths({ leftPath: unavailableBodyLeft.path, rightPath: unavailableBodyRight.path, - memberLimits: { - maxFileBytes: 1024 * 1024, - ...memberLimits, - }, - comparisonLimits, }); assert.equal(partialBodyComparison.ok, true); const partialBodyMethod = @@ -363,7 +338,6 @@ try { const nativeResult = inspectManagedArtifactBytes( nativeOnly.bytes, nativeOnly.target, - inspectionLimits, ); assert.equal(nativeResult.classification.status, "not-managed"); @@ -373,7 +347,6 @@ try { const malformedResult = inspectManagedArtifactBytes( malformed.bytes, malformed.target, - inspectionLimits, ); assert.equal(malformedResult.classification.status, "malformed"); diff --git a/scripts/verify/managed/manifest.mjs b/scripts/verify/managed/manifest.mjs index 693daf1bca..4132f4fbeb 100644 --- a/scripts/verify/managed/manifest.mjs +++ b/scripts/verify/managed/manifest.mjs @@ -14,13 +14,7 @@ import { parseManagedAppManifest, } from "./manifest-parser.mjs"; -export const createManagedManifestVerifier = (context) => ({ - verifyManagedAppManifest: (rawManifest, manifestPath) => - verifyManagedAppManifest(context, rawManifest, manifestPath), -}); - -async function verifyManagedAppManifest(context, rawManifest, manifestPath) { - const { appManifestInspectionLimits } = context; +export async function verifyManagedAppManifest(rawManifest, manifestPath) { const manifest = parseManagedAppManifest(rawManifest); const targetPath = isAbsolute(manifest.target.path) ? manifest.target.path @@ -36,25 +30,15 @@ async function verifyManagedAppManifest(context, rawManifest, manifestPath) { `target sha256 mismatch: expected ${manifest.target.sha256}, observed ${target.sha256}`, ); const bytes = await readFile(targetPath); - const artifact = inspectManagedArtifactBytes( - bytes, - target, - appManifestInspectionLimits, - ); + const artifact = inspectManagedArtifactBytes(bytes, target); verifyManifestTarget(manifest, artifact); - const methodVerification = verifyManifestMethods( - context, - manifest, - bytes, - target, - ); + const methodVerification = verifyManifestMethods(manifest, bytes, target); const { methods, inspectedMethods } = methodVerification; let assertions = 1 + methodVerification.assertions; const applicationGraph = manifest.application_graph === undefined ? undefined : verifyManagedAppManifestApplicationGraph({ - context, graphManifest: manifest.application_graph, bytes, target, @@ -109,17 +93,12 @@ function verifyManifestTarget(manifest, artifact) { ); } -function verifyManifestMethods(context, manifest, bytes, target) { +function verifyManifestMethods(manifest, bytes, target) { const methods = []; let assertions = 0; const inspectedMethods = new Map(); for (const expectedMethod of manifest.methods) { - const method = inspectManagedMethodByToken( - context, - bytes, - target, - expectedMethod, - ); + const method = inspectManagedMethodByToken(bytes, target, expectedMethod); ensure( method.signature.raw_sha256 === expectedMethod.signature_sha256, `method ${expectedMethod.token} signature sha256 mismatch: expected ${expectedMethod.signature_sha256}, observed ${method.signature.raw_sha256}`, @@ -154,7 +133,6 @@ function verifyManifestMethods(context, manifest, bytes, target) { } function verifyManagedAppManifestApplicationGraph({ - context, graphManifest, bytes, target, @@ -164,7 +142,6 @@ function verifyManagedAppManifestApplicationGraph({ }) { const graphByToken = new Map(); const projectionState = { - context, graphByToken, inspectedMethods, bytes, @@ -238,8 +215,6 @@ function verifyManagedAppManifestApplicationGraph({ } const projectManifestMethod = (state, token) => { - const { appManifestInspectionLimits, appManifestMemberLimits } = - state.context; const existing = state.graphByToken.get(token); if (existing !== undefined) return existing; const method = state.inspectedMethods.get(token); @@ -248,7 +223,6 @@ const projectManifestMethod = (state, token) => { `application graph method ${token} must also be declared in manifest.methods`, ); const members = inspectManagedMethodPageByToken( - state.context, state.bytes, state.target, token, @@ -258,7 +232,7 @@ const projectManifestMethod = (state, token) => { MANAGED_STATIC_PROVIDER, { operation: "inspect_managed_artifact", - parameters: appManifestInspectionLimits, + parameters: {}, result: state.artifact, rawResult: null, limitations: state.artifact.limitations, @@ -268,7 +242,7 @@ const projectManifestMethod = (state, token) => { const memberEvidence = createEvidence(state.target, MANAGED_STATIC_PROVIDER, { operation: "inspect_managed_members", parameters: { - ...appManifestMemberLimits, + methodLimit: 1, methodOffset: methodTokenRow(token) - 1, }, result: members, @@ -298,9 +272,8 @@ const projectManifestMethod = (state, token) => { return projection; }; -function inspectManagedMethodByToken(context, bytes, target, expectedMethod) { +function inspectManagedMethodByToken(bytes, target, expectedMethod) { const members = inspectManagedMethodPageByToken( - context, bytes, target, expectedMethod.token, @@ -319,11 +292,10 @@ function inspectManagedMethodByToken(context, bytes, target, expectedMethod) { return method; } -function inspectManagedMethodPageByToken(context, bytes, target, token) { - const { appManifestMemberLimits } = context; +function inspectManagedMethodPageByToken(bytes, target, token) { const row = methodTokenRow(token); return inspectManagedMembersBytes(bytes, target, { - ...appManifestMemberLimits, + methodLimit: 1, methodOffset: row - 1, }); } diff --git a/scripts/verify/managed/oracles.mjs b/scripts/verify/managed/oracles.mjs index caac6b7302..6f717fd8a6 100644 --- a/scripts/verify/managed/oracles.mjs +++ b/scripts/verify/managed/oracles.mjs @@ -10,13 +10,12 @@ import { inspectManagedMembersBytes } from "../../../dist/dotnet/ManagedMemberIn import { importManagedReconstructionEvidence } from "../../../dist/application/managed/ManagedReconstructionService.js"; import { createEvidence } from "../../../dist/domain/evidence.js"; import { MANAGED_STATIC_PROVIDER } from "../../../dist/domain/investigationProviders.js"; -import { createManagedManifestVerifier } from "./manifest.mjs"; +import { verifyManagedAppManifest } from "./manifest.mjs"; const execFileAsync = promisify(execFile); export const createManagedConformanceOracleSupport = (context) => { - const verifier = createManagedManifestVerifier(context); - const support = { ...context, ...verifier }; + const support = { ...context, verifyManagedAppManifest }; return { runManagedAppManifestSelfTest: () => runManagedAppManifestSelfTest(support), runOptionalManagedAppManifest: () => runOptionalManagedAppManifest(support), @@ -29,16 +28,8 @@ async function runManagedAppManifestSelfTest(context) { methodName: "PinnedSemanticSlice", ilBody: context.defaultIlBody, }); - const artifact = inspectManagedArtifactBytes( - sample.bytes, - sample.target, - context.inspectionLimits, - ); - const members = inspectManagedMembersBytes( - sample.bytes, - sample.target, - context.memberLimits, - ); + const artifact = inspectManagedArtifactBytes(sample.bytes, sample.target); + const members = inspectManagedMembersBytes(sample.bytes, sample.target); const method = members.methods[0]; assert.ok(method); const summary = await context.verifyManagedAppManifest( @@ -132,16 +123,8 @@ async function runOptionalIlspyOracle(context) { const sample = await context.fixture("ilspy-oracle.exe", { methodName: "PinnedSemanticSlice", }); - const artifact = inspectManagedArtifactBytes( - sample.bytes, - sample.target, - context.inspectionLimits, - ); - const members = inspectManagedMembersBytes( - sample.bytes, - sample.target, - context.memberLimits, - ); + const artifact = inspectManagedArtifactBytes(sample.bytes, sample.target); + const members = inspectManagedMembersBytes(sample.bytes, sample.target); const method = members.methods[0]; ensureIlspy(method !== undefined, "source-owned fixture has no method"); const listOutput = await runIlspy( @@ -167,7 +150,6 @@ async function runOptionalIlspyOracle(context) { "ilspycmd C# output exceeded the managed reconstruction import bound", ); const reconstructionImport = importIlspyReconstruction({ - context, sample, method, members, @@ -210,7 +192,6 @@ async function runOptionalIlspyOracle(context) { } function importIlspyReconstruction({ - context, sample, method, members, @@ -223,7 +204,7 @@ function importIlspyReconstruction({ MANAGED_STATIC_PROVIDER, { operation: "inspect_managed_members", - parameters: context.memberLimits, + parameters: {}, result: members, rawResult: null, limitations: members.limitations, diff --git a/scripts/verify/managed/support.mjs b/scripts/verify/managed/support.mjs index 18a200e338..979f60230a 100644 --- a/scripts/verify/managed/support.mjs +++ b/scripts/verify/managed/support.mjs @@ -17,32 +17,11 @@ export const createManagedConformanceSupport = (context) => { const fixture = (name, options) => fixtureBytes(name, buildManagedPeFixture(options)); const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex"); - const appManifestInspectionLimits = { - ...context.inspectionLimits, - maxMetadataBytes: 64 * 1024 * 1024, - maxTableRows: 250_000, - maxHeapItemBytes: 16 * 1024 * 1024, - }; - const appManifestMemberLimits = { - ...context.memberLimits, - typeLimit: 1, - methodLimit: 1, - fieldLimit: 1, - memberRefLimit: 1, - edgeLimit: 1, - maxMetadataBytes: 64 * 1024 * 1024, - maxTableRows: 250_000, - maxHeapItemBytes: 16 * 1024 * 1024, - maxMethodBodyBytes: 4 * 1024 * 1024, - maxMethodInstructions: 20_000, - }; const base = { ...context, fixture, fixtureBytes, sha256, - appManifestInspectionLimits, - appManifestMemberLimits, }; return { ...base, ...createManagedConformanceOracleSupport(base) }; }; diff --git a/src/apktool/ApktoolProvider.test.ts b/src/apktool/ApktoolProvider.test.ts index 1897a18260..e3fe9146f4 100644 --- a/src/apktool/ApktoolProvider.test.ts +++ b/src/apktool/ApktoolProvider.test.ts @@ -177,6 +177,64 @@ const apkFixture = async ( }; }; +it("recovers later string pages and an exhausted XML admission on the same APK", async () => { + const stringsXml = `${Array.from({ length: 2001 }, (_, index) => `value_${String(index)}`).join("")}`; + const { command, root } = await writeApktoolStub(decodeStub({ stringsXml })); + const apk = await apkFixture(root); + const provider = providerOver(command); + onTestFinished(() => provider.close()); + const service = new ApktoolResourceAnalysisService(provider); + const inspect = async (input: Record) => { + const outcome = await service.execute("decode_android_resources", { + path: apk.path, + ...input, + }); + if (!outcome.ok) throw outcome.error; + expect(outcome.value.subject).toMatchObject({ + digest: { sha256: apk.sha256 }, + }); + return apktoolResultSchemas.decode_android_resources.parse( + outcome.value.normalized_result, + ); + }; + + const first = await inspect({ string_tables_only: true, string_limit: 2000 }); + expect(first.strings).toEqual([]); + expect(first.string_tables?.[0]).toMatchObject({ + total_count: 2001, + unparsed_count: 0, + page: { offset: 0, returned_count: 2000, next_offset: 2000 }, + coverage: "partial", + }); + const next = await inspect({ + string_tables_only: true, + string_offset: first.string_tables?.[0]?.page?.next_offset, + string_limit: 2000, + }); + expect(next.string_tables?.[0]).toMatchObject({ + strings: [{ name: "s_2000", value: "value_2000" }], + total_count: 2001, + page: { offset: 2000, returned_count: 1, next_offset: null }, + coverage: "partial", + }); + + const refused = await inspect({ max_string_table_bytes: 100 }); + expect(refused.string_tables?.[0]).toMatchObject({ + status: "oversized", + total_count: null, + }); + const recovered = await inspect({ + string_limit: 2001, + max_string_table_bytes: Buffer.byteLength(stringsXml) + 1, + }); + expect(recovered.strings).toHaveLength(2001); + expect(recovered.string_tables?.[0]).toMatchObject({ + total_count: 2001, + page: { next_offset: null }, + coverage: "complete", + }); +}); + it("returns table-only facts for a single selected primary table", async () => { const { command, root } = await writeApktoolStub(decodeStub()); const apk = await apkFixture(root); diff --git a/src/apktool/ApktoolProvider.ts b/src/apktool/ApktoolProvider.ts index 0bb7ef28fc..5bab56c021 100644 --- a/src/apktool/ApktoolProvider.ts +++ b/src/apktool/ApktoolProvider.ts @@ -31,6 +31,7 @@ import { providerConfigurationRefusal } from "../domain/providerConfigurationRef import { createAnalysisExecutionEvidence } from "../domain/analysisObservation.js"; import { apktoolResultSchemas, + apktoolStringTablesRequested, DEFAULT_APKTOOL_DECODE_TIMEOUT_MS, MAX_APKTOOL_DECODE_TIMEOUT_MS, type ApktoolRequest, @@ -666,7 +667,7 @@ export class ApktoolProvider { : "complete"; completedDecode.strings = [...projectedStrings.strings]; - if (input.locales !== undefined || input.string_tables_only === true) + if (apktoolStringTablesRequested(input)) completedDecode.string_tables = [...projectedStrings.tables]; completedDecode.coverage = coverage; if (!projection.ok) throw projection.error.cause; diff --git a/src/apktool/ApktoolStringTables.test.ts b/src/apktool/ApktoolStringTables.test.ts index 969fc95bf0..d5d97c672e 100644 --- a/src/apktool/ApktoolStringTables.test.ts +++ b/src/apktool/ApktoolStringTables.test.ts @@ -3,8 +3,8 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { - MAX_AGGREGATE_STRINGS, - MAX_STRINGS, + DEFAULT_APKTOOL_TOTAL_STRINGS, + DEFAULT_APKTOOL_STRING_LIMIT, } from "../domain/apktool/apktoolResourceAnalysis.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { projectApktoolStringTables } from "./ApktoolStringTables.js"; @@ -24,7 +24,7 @@ it("charges table-only output without reserving a duplicate primary copy", async { async readText(path) { paths.push(path); - return textRead(xml(MAX_STRINGS)); + return textRead(xml(DEFAULT_APKTOOL_STRING_LIMIT)); }, }, "decoded", @@ -42,7 +42,7 @@ it("charges table-only output without reserving a duplicate primary copy", async ); expect( outcome.value.tables.reduce((sum, table) => sum + table.strings.length, 0), - ).toBe(MAX_AGGREGATE_STRINGS); + ).toBe(DEFAULT_APKTOOL_TOTAL_STRINGS); expect(outcome.value.coverage).toBe("complete"); }); @@ -115,12 +115,43 @@ describe("Apktool locale alias table selection", () => { }); describe("physical Apktool string facts", () => { + it("applies the selected page before aggregate allocation while preserving alias counts", async () => { + const result = await projectApktoolStringTables( + { readText: async () => textRead(xml(4)) }, + "decoded", + { + include_strings: true, + locale: "pt-BR", + locales: ["pt-rBR"], + string_tables_only: true, + string_offset: 1, + string_limit: 2, + max_total_strings: 3, + }, + ); + if (!result.ok) throw result.error.cause; + expect(result.value.tables).toMatchObject([ + { + strings: [{ name: "s_1" }, { name: "s_2" }], + total_count: 4, + page: { offset: 1, returned_count: 2, next_offset: 3 }, + coverage: "partial", + }, + { + strings: [{ name: "s_1" }], + total_count: 4, + page: { offset: 1, returned_count: 1, next_offset: 2 }, + coverage: "partial", + }, + ]); + }); + it("counts complete source entries beyond the retained table prefix", async () => { const outcome = await projectApktoolStringTables( { async readText() { return textRead( - xml(MAX_STRINGS + 1).replace( + xml(DEFAULT_APKTOOL_STRING_LIMIT + 1).replace( "", 'omitted', ), @@ -133,13 +164,13 @@ describe("physical Apktool string facts", () => { if (!outcome.ok) throw outcome.error.cause; const result = outcome.value; expect(result.tables[0]).toMatchObject({ - total_count: MAX_STRINGS + 1, + total_count: DEFAULT_APKTOOL_STRING_LIMIT + 1, unparsed_count: 1, coverage: "partial", }); - expect(result.strings).toHaveLength(MAX_STRINGS); + expect(result.strings).toHaveLength(DEFAULT_APKTOOL_STRING_LIMIT); expect(result.limitations).toContain( - `strings reports the first ${String(MAX_STRINGS)} of ${String(MAX_STRINGS + 1)} entries`, + `strings reports the first ${String(DEFAULT_APKTOOL_STRING_LIMIT)} of ${String(DEFAULT_APKTOOL_STRING_LIMIT + 1)} entries`, ); }); @@ -151,7 +182,7 @@ describe("physical Apktool string facts", () => { paths.push(path); if (path.includes("values-fr")) throw new Error("Unexamined table was read"); - return textRead(xml(MAX_STRINGS)); + return textRead(xml(DEFAULT_APKTOOL_STRING_LIMIT)); }, }, "decoded", @@ -171,7 +202,7 @@ describe("physical Apktool string facts", () => { expect( result.strings.length + result.tables.reduce((count, table) => count + table.strings.length, 0), - ).toBe(MAX_AGGREGATE_STRINGS); + ).toBe(DEFAULT_APKTOOL_TOTAL_STRINGS); }); it("keeps a known empty alias present after the output budget is spent", async () => { @@ -181,7 +212,9 @@ describe("physical Apktool string facts", () => { async readText(path) { paths.push(path); return textRead( - xml(path.includes("values-pt-rBR") ? 0 : MAX_STRINGS), + xml( + path.includes("values-pt-rBR") ? 0 : DEFAULT_APKTOOL_STRING_LIMIT, + ), ); }, }, diff --git a/src/apktool/ApktoolStringTables.ts b/src/apktool/ApktoolStringTables.ts index 779277f937..34f0a8c4f5 100644 --- a/src/apktool/ApktoolStringTables.ts +++ b/src/apktool/ApktoolStringTables.ts @@ -2,9 +2,11 @@ import { join } from "node:path"; import { ok, err, type Result } from "../domain/result.js"; import { - AGGREGATE_STRINGS_XML_MAX_BYTES, - MAX_AGGREGATE_STRINGS, - MAX_STRINGS, + DEFAULT_APKTOOL_STRING_LIMIT, + DEFAULT_APKTOOL_TOTAL_STRINGS, + DEFAULT_APKTOOL_STRING_TABLE_BYTES, + DEFAULT_APKTOOL_TOTAL_STRING_TABLE_BYTES, + apktoolStringTablesRequested, androidLocaleDirectorySuffix, type ApktoolRequest, type StringTable, @@ -16,10 +18,17 @@ import { } from "./ApktoolDecodeOutput.js"; import type { ApktoolWorkspace } from "./ApktoolWorkspace.js"; -const STRINGS_XML_MAX_BYTES = 8 * 1024 * 1024; type Selection = Pick< Extract["input"], - "include_strings" | "locale" | "locales" | "string_tables_only" + | "include_strings" + | "locale" + | "locales" + | "string_tables_only" + | "string_offset" + | "string_limit" + | "max_total_strings" + | "max_string_table_bytes" + | "max_total_string_table_bytes" >; type PresentTable = { readonly status: "present"; @@ -52,6 +61,8 @@ const readTable = async ( resources: Pick, path: string, budget: number, + offset: number, + limit: number, signal?: AbortSignal, ): Promise<{ readonly fact: PhysicalTable; readonly bytes: number }> => { const source = await resources.readText(path, budget, signal); @@ -86,9 +97,13 @@ const readTable = async ( bytes, fact: { status: "present", - // Future aliases need counts and at most one table's output, not every - // parsed entry. The complete source was examined before this retention cut. - entries: parsed.value.entries.slice(0, MAX_STRINGS), + // Cache only the requested page for aliases. Counts describe the whole + // examined table, independently of selection and output allocation. + entries: parsed.value.entries.slice( + offset, + offset + + Math.min(limit, Math.max(0, parsed.value.entries.length - offset)), + ), totalCount: parsed.value.entries.length, unparsedCount: parsed.value.unparsedCount, }, @@ -106,9 +121,12 @@ const unavailableTableLimitation = ( ? "No default res/values/strings.xml was found in the decoded workspace" : `No strings.xml was found for locale ${locale}`; if (fact.status === "oversized") - return input.locales === undefined || locale === null - ? `strings.xml exceeds the ${String(fact.byteBudget)}-byte projection budget` - : `strings.xml for locale ${locale} exceeds the ${String(fact.byteBudget)}-byte projection budget`; + return ( + (input.locales === undefined || locale === null + ? `strings.xml exceeds the ${String(fact.byteBudget)}-byte projection budget` + : `strings.xml for locale ${locale} exceeds the ${String(fact.byteBudget)}-byte projection budget`) + + "; raise max_string_table_bytes and max_total_string_table_bytes as needed on the next decode, or select fewer locales. No REA restart is required; decode_timeout_ms does not raise these read budgets." + ); return `String table ${JSON.stringify(`res/${directory}/strings.xml`)} could not be projected: ${fact.message}; entry counts are unknown`; }; @@ -139,18 +157,44 @@ export const projectApktoolStringTables = async ( coverage: "complete", }); const primary = input.locale ?? null; + const offset = input.string_offset ?? 0; + const limit = input.string_limit ?? DEFAULT_APKTOOL_STRING_LIMIT; + const maxTotalStrings = + input.max_total_strings ?? DEFAULT_APKTOOL_TOTAL_STRINGS; + const tableBytes = + input.max_string_table_bytes ?? DEFAULT_APKTOOL_STRING_TABLE_BYTES; + const totalTableBytes = + input.max_total_string_table_bytes ?? + DEFAULT_APKTOOL_TOTAL_STRING_TABLE_BYTES; + const includeTables = apktoolStringTablesRequested(input); + const includePage = + input.string_offset !== undefined || + input.string_limit !== undefined || + input.max_total_strings !== undefined || + input.max_string_table_bytes !== undefined || + input.max_total_string_table_bytes !== undefined; const locales = [...new Set([primary, ...(input.locales ?? [])])]; const physical = new Map(); const tables: StringTable[] = []; const limitations = new Set(); let xmlBytes = 0; let outputEntries = 0; - let aggregateCapped = false; + const exhausted = new Set< + "max_total_strings" | "max_total_string_table_bytes" + >(); const projection = (): ApktoolStringProjection => { - if (aggregateCapped) + for (const setting of exhausted) { + const value = + setting === "max_total_strings" ? maxTotalStrings : totalTableBytes; + const unit = + setting === "max_total_strings" + ? "returned string copies" + : "XML source bytes"; + const source = input[setting] === undefined ? "default" : "request"; limitations.add( - "Aggregate string table projection stopped due to resource limits", + `Aggregate string table projection reached ${setting}=${String(value)} ${unit} (source ${source}). Raise ${setting} or select fewer locales for the next decode. ${setting === "max_total_strings" ? "string_tables_only avoids a duplicate primary copy. " : "max_string_table_bytes independently bounds each XML source read. "}No REA restart is required.`, ); + } return { strings: input.string_tables_only === true ? [] : (tables[0]?.strings ?? []), @@ -179,11 +223,8 @@ export const projectApktoolStringTables = async ( : `res/${directory}/strings.xml`; let fact = physical.get(directory); if (fact === undefined) { - const budget = Math.min( - STRINGS_XML_MAX_BYTES, - AGGREGATE_STRINGS_XML_MAX_BYTES - xmlBytes, - ); - if (budget <= 0 || outputEntries >= MAX_AGGREGATE_STRINGS) { + const budget = Math.min(tableBytes, totalTableBytes - xmlBytes); + if (budget <= 0 || outputEntries >= maxTotalStrings) { tables.push({ locale, status: "skipped", @@ -195,19 +236,23 @@ export const projectApktoolStringTables = async ( limitations.add( `Projection for ${label} skipped: aggregate resource limit reached`, ); - aggregateCapped = true; + if (budget <= 0) exhausted.add("max_total_string_table_bytes"); + if (outputEntries >= maxTotalStrings) + exhausted.add("max_total_strings"); continue; } const read = await readTable( resources, join(workspace, "res", directory, "strings.xml"), budget, + offset, + limit, signal, ); fact = read.fact; xmlBytes += read.bytes; - if (fact.status === "oversized" && budget < STRINGS_XML_MAX_BYTES) - aggregateCapped = true; + if (fact.status === "oversized" && budget < tableBytes) + exhausted.add("max_total_string_table_bytes"); physical.set(directory, fact); } if (fact.status !== "present") { @@ -227,17 +272,15 @@ export const projectApktoolStringTables = async ( // The default batch shape repeats primary entries in strings and tables; // table-only output charges one copy. Account before selecting entries. const copies = - index === 0 && - input.locales !== undefined && - input.string_tables_only !== true + index === 0 && includeTables && input.string_tables_only !== true ? 2 : 1; - const remaining = Math.max(0, MAX_AGGREGATE_STRINGS - outputEntries); + const remaining = Math.max(0, maxTotalStrings - outputEntries); const selected = fact.entries.slice(0, Math.floor(remaining / copies)); outputEntries += selected.length * copies; - const skipped = selected.length === 0 && fact.totalCount > 0; - const limited = selected.length < Math.min(fact.totalCount, MAX_STRINGS); - if (limited) aggregateCapped = true; + const skipped = selected.length === 0 && fact.entries.length > 0; + const limited = selected.length < fact.entries.length; + if (limited) exhausted.add("max_total_strings"); tables.push({ locale, status: skipped ? "skipped" : "present", @@ -245,6 +288,20 @@ export const projectApktoolStringTables = async ( // Output exhaustion cannot erase counts from an already examined table. total_count: fact.totalCount, unparsed_count: fact.unparsedCount, + ...(includePage + ? { + page: { + offset, + limit, + returned_count: selected.length, + next_offset: + offset < fact.totalCount && + selected.length < fact.totalCount - offset + ? offset + selected.length + : null, + }, + } + : {}), coverage: selected.length < fact.totalCount || fact.unparsedCount > 0 ? "partial" @@ -264,12 +321,16 @@ export const projectApktoolStringTables = async ( limitations.add( `Locale ${locale ?? "default"} strings reports ${String(selected.length)} entries due to aggregate strings limit`, ); - else if (fact.totalCount > MAX_STRINGS && !skipped) + else if (fact.totalCount > limit && offset === 0 && !skipped) limitations.add( input.locales === undefined || locale === null ? `strings reports the first ${String(selected.length)} of ${String(fact.totalCount)} entries` : `Locale ${locale} strings reports the first ${String(selected.length)} of ${String(fact.totalCount)} entries`, ); + if (selected.length < fact.totalCount && !skipped) + limitations.add( + `String page for ${label} starts at string_offset=${String(offset)} and returns ${String(selected.length)} of ${String(fact.totalCount)} valid entries. ${offset + selected.length < fact.totalCount ? `Continue with string_offset=${String(offset + selected.length)} and the same APK digest and locale;` : "This page reaches the end of the table;"} string_limit=${String(limit)} controls each page and max_total_strings=${String(maxTotalStrings)} controls returned copies. Table coverage describes the entire source table.`, + ); } } catch (cause) { limitations.add( diff --git a/src/apktool/ApktoolWorkspace.ts b/src/apktool/ApktoolWorkspace.ts index 693876740f..176a7c5147 100644 --- a/src/apktool/ApktoolWorkspace.ts +++ b/src/apktool/ApktoolWorkspace.ts @@ -181,7 +181,7 @@ export class ApktoolWorkspace { signal, async ({ handle: file, initial }) => { admitted = true; - const bytes = Buffer.alloc(budget + 1); + const bytes = Buffer.alloc(Math.min(initial.size, budget) + 1); let length = 0; while (length < bytes.length) { assertArtifactReadActive(signal); diff --git a/src/application/AnalysisSettings.test.ts b/src/application/AnalysisSettings.test.ts index d6fa2d8bff..7572c4c4b4 100644 --- a/src/application/AnalysisSettings.test.ts +++ b/src/application/AnalysisSettings.test.ts @@ -75,6 +75,10 @@ it("keeps accepted operations and their typed provider views stable across concu const pending = store.run(async () => { await barrier; expect(asar.maximumBytes).toBe(16777216); + expect(asar.listing?.maximumBytes).toBe(67108864); + expect(live.ghidraValueFlowMaxOperations).toBe(3000); + expect(live.ghidraValueFlowMaxInputsPerOperation).toBe(64); + expect(live.ghidraValueFlowMaxEdges).toBe(12000); expect(live.ghidraStartupTimeoutMs).toBe(330000); expect(live.regexMatchingDeadline.deadlineMs).toBe(5000); expect(store.read().revision).toBe(0); @@ -83,6 +87,18 @@ it("keeps accepted operations and their typed provider views stable across concu const changed = store.update({ changes: [ { setting: "asar.max_header_bytes", action: "set", value: 33554432 }, + { setting: "asar.max_listing_bytes", action: "set", value: 134217728 }, + { + setting: "ghidra.value_flow_max_operations", + action: "set", + value: 6000, + }, + { + setting: "ghidra.value_flow_max_inputs_per_operation", + action: "set", + value: 128, + }, + { setting: "ghidra.value_flow_max_edges", action: "set", value: 24000 }, { setting: "ghidra.startup_timeout_ms", action: "set", value: 999999 }, { setting: "regex.matching_deadline_ms", action: "set", value: 10000 }, ], @@ -95,17 +111,25 @@ it("keeps accepted operations and their typed provider views stable across concu await pending; store.run(() => { expect(asar.maximumBytes).toBe(33554432); + expect(asar.listing?.maximumBytes).toBe(134217728); + expect(live.ghidraValueFlowMaxOperations).toBe(6000); + expect(live.ghidraValueFlowMaxInputsPerOperation).toBe(128); + expect(live.ghidraValueFlowMaxEdges).toBe(24000); expect(live.ghidraStartupTimeoutMs).toBe(999999); expect(live.regexMatchingDeadline.deadlineMs).toBe(10000); }); }); it("exposes shared provider controls and preserves the explicit unbounded decompile value", () => { - const environment = { REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "7" }; + const environment = { + REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: "7", + REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB: "64", + }; const store = new AnalysisSettings(environment); const parsed = parseConfig(environment); if (!parsed.ok) throw parsed.error; const config = runtimeAnalysisConfig(parsed.value, store.read); + expect(config.ghidraDecompileMaxPayloadMiB).toBe(64); expect( store .inspect({ provider: "ghidra", operation: "search_strings" }) @@ -120,7 +144,7 @@ it("exposes shared provider controls and preserves the explicit unbounded decomp store .inspect({ operation: "inventory_artifact" }) .settings.map(({ setting }) => setting), - ).toEqual(["asar.max_header_bytes"]); + ).toEqual(["asar.max_listing_bytes", "asar.max_header_bytes"]); expect( store.update({ changes: [ @@ -129,16 +153,27 @@ it("exposes shared provider controls and preserves the explicit unbounded decomp action: "set", value: null, }, + { + setting: "ghidra.decompile_max_payload_mib", + action: "set", + value: null, + }, ], }).ok, ).toBe(true); expect(config.ghidraDecompileTimeoutSeconds).toBeUndefined(); + expect(config.ghidraDecompileMaxPayloadMiB).toBeUndefined(); expect( store.inspect({ settings: ["ghidra.decompile_timeout_seconds"] }) .settings[0], ).toMatchObject({ value: null, launch_value: 7, source: "runtime" }); for (const [setting, value] of [ ["asar.max_header_bytes", null], + ["asar.max_listing_bytes", 0], + ["asar.max_listing_bytes", Number.MAX_SAFE_INTEGER + 1], + ["ghidra.value_flow_max_operations", 0], + ["ghidra.value_flow_max_inputs_per_operation", 2147483648], + ["ghidra.decompile_max_payload_mib", 2048], ["asar.max_header_bytes", 268435457], ["javascript.max_heap_mb", 127], ["regex.matching_deadline_ms", 1.5], diff --git a/src/application/BrowserEvidence.test.ts b/src/application/BrowserEvidence.test.ts index a48d36a062..27e965f8e3 100644 --- a/src/application/BrowserEvidence.test.ts +++ b/src/application/BrowserEvidence.test.ts @@ -110,3 +110,24 @@ describe("web bundle Evidence origin scope", () => { expect(evidence.parameters.allowed_origins).toEqual([]); }); }); + +it.each([undefined, {}, { decoded_records: 1024 }])( + "preserves the selected source-map admission policy %j without resolving defaults", + (selection) => { + const selected = analyzeWebBundleInputSchema.parse({ + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "page-1", + fetch_source_maps: true, + ...(selection === undefined ? {} : { source_map_limits: selection }), + }); + const evidence = createBrowserEvidence( + "analyze_web_bundle", + selected, + result, + provider, + ); + if (selection === undefined) + expect(evidence.parameters).not.toHaveProperty("source_map_limits"); + else expect(evidence.parameters.source_map_limits).toEqual(selection); + }, +); diff --git a/src/application/BrowserEvidence.ts b/src/application/BrowserEvidence.ts index d664dd7aa3..57fb6d6b48 100644 --- a/src/application/BrowserEvidence.ts +++ b/src/application/BrowserEvidence.ts @@ -195,6 +195,12 @@ const browserParameters = ( ...("fetch_source_maps" in input ? { fetch_source_maps: input.fetch_source_maps, + ...(input.source_map_limits === undefined + ? {} + : { source_map_limits: input.source_map_limits }), + ...(input.semantic_precision === undefined + ? {} + : { semantic_precision: input.semantic_precision }), ...(input.max_heap_mb === undefined ? {} : { max_heap_mb: input.max_heap_mb }), diff --git a/src/application/JsonAuthenticationHeadroom.test.ts b/src/application/JsonAuthenticationHeadroom.test.ts index ac1429a8bc..3cbf994adc 100644 --- a/src/application/JsonAuthenticationHeadroom.test.ts +++ b/src/application/JsonAuthenticationHeadroom.test.ts @@ -19,9 +19,43 @@ it("stops a bounded representation scan before visiting an over-budget tail", () }); expect(measured.valueBytes).toBeGreaterThan(512); }); +it("checks escaped complete JSON against serialization capacity independently of heap", () => { + const value = { + normalized_result: { wat: '\0"\\😀'.repeat(30), imports: ["\0"] }, + raw_result: null, + }; + const encodedLength = JSON.stringify(value).length; + const options = { + operation: "inspect_wasm_artifact", + path: "/module.wasm", + description: "Complete WASM Evidence cannot be assembled", + availableBytes: Number.MAX_SAFE_INTEGER, + }; + const rejected = requireAuthenticationHeadroom(value, { + ...options, + serializedCodeUnitLimit: encodedLength - 1, + }); + expect(rejected).toMatchObject({ + ok: false, + error: { + retryable: false, + reportedLimits: { + serialized_json_code_units: encodedLength, + maximum_serialized_json_code_units: encodedLength - 1, + }, + }, + }); + expect( + requireAuthenticationHeadroom(value, { + ...options, + serializedCodeUnitLimit: encodedLength, + }), + ).toEqual({ ok: true, value: undefined }); +}); + describe("JSON authentication headroom", () => { it("measures source IDs without first constructing their array", () => { - const values = ["ev_123", null, 'escaped\"\n😀']; + const values = ["ev_123", null, 'escaped"\n😀']; const empty = estimateJsonStorage({ evidence_links: [] }); const incremental = values.reduce((total, value, index) => { const entry = estimateJsonArrayScalarEntry(value, index > 0); diff --git a/src/application/JsonAuthenticationHeadroom.ts b/src/application/JsonAuthenticationHeadroom.ts index 1b17c79b63..e42fb26607 100644 --- a/src/application/JsonAuthenticationHeadroom.ts +++ b/src/application/JsonAuthenticationHeadroom.ts @@ -89,6 +89,8 @@ export const requireAuthenticationHeadroom = ( readonly path: string; readonly description: string; readonly availableBytes?: number; + /** A caller's real serialization ceiling, checked before allocating JSON text. */ + readonly serializedCodeUnitLimit?: number; readonly additional?: { readonly valueBytes: number; readonly serializedUtf16Bytes: number; @@ -103,6 +105,28 @@ export const requireAuthenticationHeadroom = ( const serializedUtf16Bytes = estimate.serializedUtf16Bytes + (options.additional?.serializedUtf16Bytes ?? 0); + const serializedCodeUnits = serializedUtf16Bytes / 2; + if ( + options.serializedCodeUnitLimit !== undefined && + serializedCodeUnits > options.serializedCodeUnitLimit + ) + return err( + new AnalysisResourceConstraintError( + options.operation, + "memory", + `${options.description}: expanded JSON exceeds the single-string serialization limit.`, + { + input_path: options.path, + serialized_json_code_units: serializedCodeUnits, + maximum_serialized_json_code_units: options.serializedCodeUnitLimit, + }, + { + retryable: false, + remediationAction: + "Reduce the selected facts before serialization. If the question requires the complete representation above this ceiling, report this unsupported size instead of repeating the unchanged request. Increasing heap or command-output admission does not raise the serialization ceiling. A delivery summary cannot repair an Evidence record that could not be assembled.", + }, + ), + ); // Canonical hashing and manifest normalization can still encode scalars or // metadata; account for their escaped representations before allocating them. const requiredBytes = diff --git a/src/application/SetupRegistrationEnvironment.ts b/src/application/SetupRegistrationEnvironment.ts index dc4c9948f7..b9356dd2f3 100644 --- a/src/application/SetupRegistrationEnvironment.ts +++ b/src/application/SetupRegistrationEnvironment.ts @@ -54,6 +54,11 @@ const SELECTED_REGISTRATION_SETTINGS = [ "REA_KUNA_JOBS", "REA_ANALYSIS_PROVIDER", "REA_ASAR_MAX_HEADER_BYTES", + "REA_ASAR_MAX_LISTING_BYTES", + "REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS", + "REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION", + "REA_GHIDRA_VALUE_FLOW_MAX_EDGES", + "REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB", "REA_BRIDGE_MAX_REQUEST_BYTES", "REA_CAPTURE_DECODER_OLD_GENERATION_MIB", "REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS", diff --git a/src/application/WebNetworkCaptureService.ts b/src/application/WebNetworkCaptureService.ts index 3b16ad32c7..3d5d68b50c 100644 --- a/src/application/WebNetworkCaptureService.ts +++ b/src/application/WebNetworkCaptureService.ts @@ -243,6 +243,15 @@ export class WebNetworkCaptureService { }, }), sensitive_value_count: input.sensitive_values.length, + ...(input.max_input_bytes === undefined + ? {} + : { max_input_bytes: input.max_input_bytes }), + ...(input.max_reply_bytes === undefined + ? {} + : { max_reply_bytes: input.max_reply_bytes }), + ...(input.max_nesting_depth === undefined + ? {} + : { max_nesting_depth: input.max_nesting_depth }), ...(input.capture_timeout_ms === undefined ? {} : { capture_timeout_ms: input.capture_timeout_ms }), diff --git a/src/application/WebSourceLocationPorts.ts b/src/application/WebSourceLocationPorts.ts index e7ad3d4fd3..978a91c180 100644 --- a/src/application/WebSourceLocationPorts.ts +++ b/src/application/WebSourceLocationPorts.ts @@ -1,3 +1,4 @@ +import type { SourceMapLimitSelection } from "../domain/sourceMapLimits.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { Result } from "../domain/result.js"; import type { @@ -39,6 +40,7 @@ export interface WebSourceMapPort { readonly url: string; readonly position: WebSourceLocationInput["generated_position"]; readonly decodeTimeoutMs?: number; + readonly limits?: SourceMapLimitSelection; }, options?: ExecutionOptions, ): Promise>; diff --git a/src/application/WebSourceLocationService.ts b/src/application/WebSourceLocationService.ts index eeb0c334f1..0c20b4cec9 100644 --- a/src/application/WebSourceLocationService.ts +++ b/src/application/WebSourceLocationService.ts @@ -1,3 +1,7 @@ +import { + selectedSourceMapLimits, + type SourceMapLimits, +} from "../domain/sourceMapLimits.js"; import { WEB_SOURCE_LOCATION_PROVIDER } from "../domain/webTraceProviders.js"; import { closeOwnedWebResources } from "./closeOwnedWebResources.js"; import { createHash } from "node:crypto"; @@ -19,7 +23,6 @@ import { } from "../domain/webScriptArtifacts.js"; import { DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS, - WEB_SOURCE_MAP_LIMITS, webSourceLocationInputSchema, webSourceLocationResultSchema, webSourceMapReportSchema, @@ -56,16 +59,17 @@ const LIMITATIONS = [ "Lookup returns all segments at the greatest generated position not greater than the selected point, including earlier lines and generated-only segments. A mapped point does not establish an executed range or UI causality.", "Regular maps and inline indexed sections are supported. External section URLs are unsupported; unknown extensions are retained in raw map text and are not interpreted.", ]; -const codecBudget = (timeoutMs: number): string => - `Complete evidence has a 4 MiB map, 32 MiB reply and a ${String(WEB_SOURCE_MAP_LIMITS.decodedRecords)}-record shared decoder budget for generated rows, mapping segments, leaf maps, source and name declarations, sourcesContent entries, and indexed sections. Generated rows and mapping segments have separate ${String(WEB_SOURCE_MAP_LIMITS.decodedRows)} and ${String(WEB_SOURCE_MAP_LIMITS.decodedSegments)} limits, and indexed maps have a ${String(WEB_SOURCE_MAP_LIMITS.sectionDepth)}-level section budget. The owned codec has a ${String(timeoutMs)} ms decode_timeout_ms deadline`; +const codecBudget = (timeoutMs: number, limits: SourceMapLimits): string => + `Complete evidence has a ${String(limits.mapBytes)}-byte map, ${String(limits.outputBytes)}-byte reply and a ${String(limits.decodedRecords)}-record shared decoder budget for generated rows, mapping segments, leaf maps, source and name declarations, sourcesContent entries, and indexed sections. Generated rows and mapping segments have separate ${String(limits.decodedRows)} and ${String(limits.decodedSegments)} limits, and indexed maps have a ${String(limits.sectionDepth)}-level section budget. The owned codec has a ${String(timeoutMs)} ms decode_timeout_ms deadline`; /** Bun accepts but ignores the codec's V8 heap flags; Windows enforces a commit limit instead. */ const codecBudgetLimitation = ( runtime: WebSourceMapReport["runtime"], timeoutMs: number, + limits: SourceMapLimits, ): string => { - const budget = codecBudget(timeoutMs); + const budget = codecBudget(timeoutMs, limits); if (runtime.id === "node-v8") - return `${budget}, ${String(WEB_SOURCE_MAP_LIMITS.codecHeapMiB)} MiB old-generation heap and one V8 worker; cleanup may extend the deadline. V8 heap limits do not establish aggregate RSS limits.`; + return `${budget}, ${String(limits.codecHeapMiB)} MiB old-generation heap and one V8 worker; cleanup may extend the deadline. V8 heap limits do not establish aggregate RSS limits.`; return runtime.process_memory_limit_bytes === null ? `${budget}; cleanup may extend the deadline. The codec ran on Bun, which does not enforce REA's V8 heap flags, so no codec heap limit is established.` : `${budget}; cleanup may extend the deadline. The codec ran on Bun, which does not enforce REA's V8 heap flags; a Windows Job Object limited its committed memory to ${String(runtime.process_memory_limit_bytes)} bytes, which is not a heap or RSS limit.`; @@ -158,6 +162,9 @@ export class WebSourceLocationService { path: map.file.path, url: map.url, position: input.generated_position, + ...(input.source_map_limits === undefined + ? {} + : { limits: input.source_map_limits }), ...(input.decode_timeout_ms === undefined ? {} : { decodeTimeoutMs: input.decode_timeout_ms }), @@ -171,7 +178,11 @@ export class WebSourceLocationService { input.decode_timeout_ms ?? DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS; const limitations = [ ...LIMITATIONS, - codecBudgetLimitation(report.value.runtime, decodeTimeoutMs), + codecBudgetLimitation( + report.value.runtime, + decodeTimeoutMs, + selectedSourceMapLimits(input.source_map_limits), + ), ]; const normalized = webSourceLocationResultSchema.safeParse({ ...report.value, @@ -285,7 +296,7 @@ const verifyPairing = ( map.url !== input.source_map.url || Buffer.byteLength(map.text) !== map.file.bytes || createHash("sha256").update(map.text).digest("hex") !== map.file.sha256 || - map.file.bytes > WEB_SOURCE_MAP_LIMITS.mapBytes + map.file.bytes > selectedSourceMapLimits(input.source_map_limits).mapBytes ) return invalidOutput( "Artifact port changed the selected source-map byte identity or URL context.", diff --git a/src/application/analysisView/AnalysisViewService.ts b/src/application/analysisView/AnalysisViewService.ts index 936a17e8d6..243bb323b1 100644 --- a/src/application/analysisView/AnalysisViewService.ts +++ b/src/application/analysisView/AnalysisViewService.ts @@ -328,12 +328,14 @@ export const projectRetainedAnalysis = ( if (!retained.ok) return retained; // Duplicate semantic records retain their original ledger metadata. Detail // selection changes presentation, not the historical lookup or payload owner. - return projectResolvedAnalysisView( + const projected = projectResolvedAnalysisView( { source, view }, prepareAnalysisViewProjection(view), retained.value, evidenceSubjectTarget(evidence.subject), ); + if (!projected.ok) projected.error.retainPartialObservation(source); + return projected; }; const resolveAnalysisViewSource = ( diff --git a/src/application/analysisView/WebSourceLocationViews.test.ts b/src/application/analysisView/WebSourceLocationViews.test.ts new file mode 100644 index 0000000000..8c0c03d465 --- /dev/null +++ b/src/application/analysisView/WebSourceLocationViews.test.ts @@ -0,0 +1,184 @@ +import { EvidenceLedger } from "../investigation/EvidenceLedger.js"; +import { expect, it } from "vitest"; +import { + webSourceLocationArgs, + webSourceLocationFixture, + webSourceMapDecoderFixture, +} from "../../../tests/fixtures/webSourceLocation.js"; +import { WebSourceLocationService } from "../WebSourceLocationService.js"; +import { ok } from "../../domain/result.js"; +import { + inspectAnalysisView, + projectRetainedAnalysis, +} from "./AnalysisViewService.js"; +import { analysisViewRequestSchema } from "../../domain/analysisView/analysisView.js"; +import type { Evidence } from "../../domain/evidence.js"; + +const trace = async ( + mappings = "AAAA,ACAA,A", + sourcesContent: (string | null)[] = ["a😀b", null], +) => { + const fixture = webSourceLocationFixture( + JSON.stringify({ + version: 3, + sources: ["one.ts", "one.ts"], + sourcesContent, + names: [], + mappings, + }), + ); + const service = new WebSourceLocationService( + { load: async () => ok(fixture) }, + webSourceMapDecoderFixture, + ); + const result = await service.trace(webSourceLocationArgs); + if (!result.ok) throw result.error; + return result.value; +}; +const viewOf = (evidence: Evidence, view: unknown) => + inspectAnalysisView( + { + source: { kind: "retained-evidence", evidence_id: evidence.evidence_id }, + view, + }, + (id) => (id === evidence.evidence_id ? evidence : undefined), + ); + +it("keeps every equal-position alternative and original indices without embedded text", async () => { + const parent = await trace(); + const summary = await viewOf(parent, { kind: "summary" }); + if (!summary.ok) throw summary.error; + expect(summary.value.normalized_result).toMatchObject({ + parent_evidence_id: parent.evidence_id, + summary: { + match_count: 3, + matches: [ + { + match_index: 0, + source_index: 0, + reported_source: "one.ts", + content: { state: "embedded", code_units: 4 }, + }, + { + match_index: 1, + source_index: 1, + reported_source: "one.ts", + content: { state: "unavailable" }, + }, + { match_index: 2, state: "generated-only" }, + ], + }, + coverage: { total: 3, examined: 3, exhausted: true }, + }); + expect(JSON.stringify(summary.value)).not.toContain('"text":'); + const page = await viewOf(parent, { + kind: "source-location", + collection: "matches", + offset: 1, + limit: 1, + }); + if (!page.ok) throw page.error; + expect(page.value.normalized_result).toMatchObject({ + items: [{ match_index: 1, source_index: 1 }], + coverage: { total: 3, examined: 1, next_offset: 2, exhausted: false }, + }); +}); + +it("reports an empty lookup without inventing a mapping", async () => { + const summary = await viewOf(await trace(""), { kind: "summary" }); + if (!summary.ok) throw summary.error; + expect(summary.value.normalized_result).toMatchObject({ + summary: { match_count: 0, matches: [] }, + coverage: { status: "empty", total: 0, examined: 0, exhausted: true }, + }); +}); + +it("selects exact UTF-16 content pages and preserves unavailable content", async () => { + const parent = await trace(); + const view = { + kind: "source-location", + collection: "source-text", + match_index: 0, + offset: 1, + limit: 2, + }; + const text = await viewOf(parent, view); + if (!text.ok) throw text.error; + expect(text.value.normalized_result).toMatchObject({ + match: { match_index: 0, content: { code_units: 4 } }, + item: { text: "😀", unit: "utf16-code-units" }, + coverage: { total: 4, examined: 2, next_offset: 3 }, + }); + expect((await viewOf(parent, { ...view, offset: 2 })).ok).toBe(false); + expect((await viewOf(parent, { ...view, limit: 1 })).ok).toBe(false); + const unavailable = await viewOf(parent, { ...view, match_index: 1 }); + if (!unavailable.ok) throw unavailable.error; + expect(unavailable.value.normalized_result).toMatchObject({ + item: null, + match: { content: { state: "unavailable" } }, + coverage: { status: "unavailable", total: null, exhausted: false }, + }); +}); + +it("returns recoverable byte admission and rejects invalid indices", async () => { + const parent = await trace(); + const selected = { + kind: "source-location", + collection: "matches", + max_bytes: 1, + }; + const refused = await viewOf(parent, selected); + if (refused.ok) throw new Error("expected view byte refusal"); + expect(refused.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + reportedLimits: { max_bytes: 1 }, + remediationAction: expect.stringContaining("view.max_bytes"), + }); + expect((await viewOf(parent, { ...selected, max_bytes: 65536 })).ok).toBe( + true, + ); + expect( + ( + await viewOf(parent, { + kind: "source-location", + collection: "source-text", + match_index: 3, + }) + ).ok, + ).toBe(false); +}); + +it("preserves already retained parent recovery when a selected projection fails", async () => { + const parent = await trace("AAAA", ["a".repeat(10000), null]); + const records = new EvidenceLedger(); + const retention = { + recordEvidence: (evidence: Evidence) => records.record(evidence), + evidenceById: (id: string) => records.forAnalysis(id), + }; + const view = { + kind: "source-location", + collection: "source-text", + match_index: 0, + limit: 10000, + max_bytes: 2000, + }; + const refused = projectRetainedAnalysis( + parent, + retention, + analysisViewRequestSchema.parse(view), + ); + if (refused.ok) throw new Error("expected projection admission failure"); + expect(refused.error.partialObservation).toEqual({ + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }); + expect(records.forAnalysis(parent.evidence_id)?.evidence_id).toBe( + parent.evidence_id, + ); + const recovered = await viewOf(parent, { ...view, max_bytes: 20000 }); + if (!recovered.ok) throw recovered.error; + const output = recovered.value.normalized_result as { + item: { text: string }; + }; + expect(output.item.text.length).toBe(10000); +}); diff --git a/src/application/binaryDiagnostics/BinaryLayoutService.ts b/src/application/binaryDiagnostics/BinaryLayoutService.ts index 757ab7eb6b..689e3a7582 100644 --- a/src/application/binaryDiagnostics/BinaryLayoutService.ts +++ b/src/application/binaryDiagnostics/BinaryLayoutService.ts @@ -93,6 +93,9 @@ export class BinaryLayoutService { operation: OPERATION, parameters: { path: input.value.path, + ...(input.value.producer_limits === undefined + ? {} + : { producer_limits: input.value.producer_limits }), ...(input.value.pwntools_timeout_ms === undefined ? {} : { pwntools_timeout_ms: input.value.pwntools_timeout_ms }), diff --git a/src/application/binaryDiagnostics/KernelModuleService.test.ts b/src/application/binaryDiagnostics/KernelModuleService.test.ts index 8a80ac2540..612ba4d2fc 100644 --- a/src/application/binaryDiagnostics/KernelModuleService.test.ts +++ b/src/application/binaryDiagnostics/KernelModuleService.test.ts @@ -67,7 +67,7 @@ const report: KernelModule = { limitations: ["Kernel module metadata was not executed."], }; -it("rejects a relative path and extra arguments before reading", async () => { +it("rejects relative paths, extra arguments and invalid field budgets before reading", async () => { let calls = 0; const service = new KernelModuleService({ identity, @@ -85,6 +85,20 @@ it("rejects a relative path and extra arguments before reading", async () => { expect(extra.ok).toBe(false); if (!relative.ok) expect(relative.error).toBeInstanceOf(AnalysisInputError); if (!extra.ok) expect(extra.error).toBeInstanceOf(AnalysisInputError); + for (const budget of ["max_modinfo_key_bytes", "max_modinfo_value_bytes"]) { + for (const value of [0, 536870913]) { + const invalid = await service.inspect({ + path: report.artifact.path, + [budget]: value, + }); + expect(invalid.ok).toBe(false); + if (!invalid.ok) + expect(invalid.error).toMatchObject({ + _tag: "AnalysisInputError", + issues: [expect.objectContaining({ path: [budget] })], + }); + } + } expect(calls).toBe(0); }); diff --git a/src/application/binaryDiagnostics/KernelModuleService.ts b/src/application/binaryDiagnostics/KernelModuleService.ts index c86c474f04..6c80f7d801 100644 --- a/src/application/binaryDiagnostics/KernelModuleService.ts +++ b/src/application/binaryDiagnostics/KernelModuleService.ts @@ -13,6 +13,7 @@ import { MODULE_SIGNATURE_MAGIC, MODULE_SIGNATURE_PKCS7_ID_TYPE, type KernelModule, + type InspectKernelModuleInput, } from "../../domain/native/kernelModule.js"; import { BINARY_ARCHITECTURES } from "../../domain/binaryTargetTypes.js"; import { elfArchitecture } from "../../domain/elfFixedHeader.js"; @@ -49,11 +50,7 @@ export class KernelModuleService { if ( !report.success || report.data.artifact.path !== input.value.path || - !validReport( - report.data, - input.value.max_entries, - input.value.max_file_bytes, - ) + !validReport(report.data, input.value) ) return err( new AnalysisOutputError( @@ -124,6 +121,7 @@ const rangesOf = (report: KernelModule): Range[] => { const modinfoConsistent = ( modinfo: NonNullable, + input: InspectKernelModuleInput, ): boolean => { const closed = modinfo.status === "malformed" || modinfo.status === "unsupported"; @@ -145,6 +143,17 @@ const modinfoConsistent = ( return false; return modinfo.entries.every((entry, index) => { if (entry.index !== index) return false; + if (new Set(entry.omitted_fields).size !== entry.omitted_fields.length) + return false; + for (const field of ["key", "value"] as const) { + const text = entry[field]; + if (entry.omitted_fields.includes(field) && text !== null) return false; + const maximum = + field === "key" + ? input.max_modinfo_key_bytes + : input.max_modinfo_value_bytes; + if (text !== null && text.length > maximum) return false; + } const rawFits = entry.location.bytes <= MAX_KERNEL_MODULE_RAW_BYTES; if (rawFits) { if ( @@ -155,7 +164,9 @@ const modinfoConsistent = ( } else if (entry.raw_hex !== null) return false; if (entry.status === "observed") return ( - entry.reason === null && entry.key !== null && entry.value !== null + entry.reason === null && + (entry.key !== null || entry.omitted_fields.includes("key")) && + (entry.value !== null || entry.omitted_fields.includes("value")) ); return entry.reason !== null; }); @@ -274,12 +285,11 @@ const metadataAbsent = (report: KernelModule): boolean => const validReport = ( report: KernelModule, - maxEntries: number, - maxBytes: number, + input: InspectKernelModuleInput, ): boolean => { if ( - report.artifact.bytes > maxBytes || - report.coverage.examined_entries > maxEntries || + report.artifact.bytes > input.max_file_bytes || + report.coverage.examined_entries > input.max_entries || !rangesOf(report).every(inside(report.artifact.bytes)) ) return false; @@ -342,7 +352,7 @@ const validReport = ( return false; if (report.type !== 1) return false; return ( - modinfoConsistent(report.modinfo) && + modinfoConsistent(report.modinfo, input) && versionsConsistent(report.versions) && signatureConsistent(report.signature) && sizedConsistent(report.init_text) && diff --git a/src/application/binaryDiagnostics/RecordedCrashService.ts b/src/application/binaryDiagnostics/RecordedCrashService.ts index 7713a336b4..d866d2ac5d 100644 --- a/src/application/binaryDiagnostics/RecordedCrashService.ts +++ b/src/application/binaryDiagnostics/RecordedCrashService.ts @@ -95,6 +95,9 @@ export class RecordedCrashService { parameters: { path: input.value.path, include_debugger_context: input.value.include_debugger_context, + ...(input.value.producer_limits === undefined + ? {} + : { producer_limits: input.value.producer_limits }), ...(input.value.pwntools_timeout_ms === undefined ? {} : { pwntools_timeout_ms: input.value.pwntools_timeout_ms }), diff --git a/src/application/desktop/DesktopApplicationCapture.ts b/src/application/desktop/DesktopApplicationCapture.ts index 9375160fb9..1a6de5f776 100644 --- a/src/application/desktop/DesktopApplicationCapture.ts +++ b/src/application/desktop/DesktopApplicationCapture.ts @@ -1,3 +1,4 @@ +import type { DesktopLibraryPolicyInput } from "../../domain/desktop/libraryPolicy.js"; import { arch, hostname, userInfo } from "node:os"; import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; @@ -57,6 +58,7 @@ const POLL_INTERVAL_MS = 500; /** What the caller asked to capture, after CLI parsing. */ export interface DesktopCaptureRequest { + readonly libraryPolicy?: DesktopLibraryPolicyInput; readonly desktopEntry: string | undefined; readonly executable: string | undefined; readonly arguments: readonly string[]; @@ -176,6 +178,9 @@ const captureEvidence = ( state_path_count: request.statePaths.length, exclude_count: request.excludes.length, package_file: request.packageFile !== undefined, + ...(request.libraryPolicy === undefined + ? {} + : { library_policy: request.libraryPolicy }), }, result: capture, confidence: "observed", diff --git a/src/application/desktop/DesktopApplicationExport.test.ts b/src/application/desktop/DesktopApplicationExport.test.ts index 58b21b8c7f..69153edb09 100644 --- a/src/application/desktop/DesktopApplicationExport.test.ts +++ b/src/application/desktop/DesktopApplicationExport.test.ts @@ -125,6 +125,35 @@ const exists = (path: string) => const execute = promisify(execFile); +/** Read the exported module's literal alias metadata without importing the desktop SDK. */ +const exportedAliases = async (directory: string) => { + const { stdout } = await execute( + process.platform === "win32" ? "python" : "python3", + [ + "-I", + "-B", + "-c", + `import ast, json, sys +from pathlib import Path +module = ast.parse(Path(sys.argv[1]).read_text(encoding="utf-8")) +assignments = [node.value for node in module.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "STATE_ALIASES" for target in node.targets)] +assert len(assignments) == 1 +print(json.dumps(ast.literal_eval(assignments[0])))`, + join(directory, "envs/notes/notes_app.py"), + ], + ); + return z + .array( + z.tuple([ + z.string(), + z.string(), + z.enum(["file", "directory"]).nullable(), + ]), + ) + .parse(JSON.parse(stdout)); +}; + afterEach(() => { vi.restoreAllMocks(); }); @@ -311,11 +340,9 @@ it("seeds the app's state at its own path and retains a selected alias", async ( "utf8", ), ).toBe("milk\n"); - const module = await readFile( - join(directory, "envs/notes/notes_app.py"), - "utf8", - ); - expect(module).toContain('("settings", ".config/notes", "directory")'); + expect(await exportedAliases(directory)).toEqual([ + ["settings", ".config/notes", "directory"], + ]); }); it("restores a selected alias when the state directory has no files", async () => { @@ -377,11 +404,9 @@ it.each(["settings/notes", "chosen/notes"])( "utf8", ), ).toBe("[ui]\nwrap=false\n"); - const module = await readFile( - join(directory, "envs/notes/notes_app.py"), - "utf8", - ); - expect(module).toContain(`("${selected}", ".config/notes", "directory")`); + expect(await exportedAliases(directory)).toEqual([ + [selected, ".config/notes", "directory"], + ]); }, ); @@ -426,18 +451,14 @@ it.each([true, false])( }, }); if (result instanceof Error) throw result; - const module = await readFile( - join(directory, "envs/notes/notes_app.py"), - "utf8", - ); - expect(module).toContain('("settings", ".config/notes", "directory")'); - expect(module).not.toContain( - '("settings/state/notes", ".config/notes/state/notes", "directory")', - ); + const aliases = [["settings", ".config/notes", "directory"]]; if (!redundant) - expect(module).toContain( - '("settings/state/notes", ".local/share/separate", "directory")', - ); + aliases.push([ + "settings/state/notes", + ".local/share/separate", + "directory", + ]); + expect(await exportedAliases(directory)).toEqual(aliases); }, ); diff --git a/src/application/desktop/DesktopApplicationExport.ts b/src/application/desktop/DesktopApplicationExport.ts index 0d97683362..7aa0751bcf 100644 --- a/src/application/desktop/DesktopApplicationExport.ts +++ b/src/application/desktop/DesktopApplicationExport.ts @@ -1,3 +1,4 @@ +import type { DesktopViewPolicyInput } from "../../domain/desktop/desktopViewPolicy.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { DownloadInstall } from "../../domain/desktop/appDownloadInstall.js"; import { @@ -14,13 +15,11 @@ import { type HomePathCollision, type SeedScrubReport, } from "../../domain/desktop/appExport.js"; -import { INSTALL_SCRIPT } from "../../domain/desktop/appExportInstall.js"; import type { WithheldFile } from "../../domain/desktop/appWithheld.js"; import { trashedFiles, runtimeSeeds, unownedChangedFolders, - unownedFoldersLimitation, unreadDconfLimitation, } from "../../domain/desktop/appExportUnowned.js"; import { installLimitations } from "../../domain/desktop/appExportInstallLimitations.js"; @@ -32,7 +31,6 @@ import { } from "../../domain/desktop/desktopState.js"; import { isWithinAny } from "../../domain/desktop/desktopStateDiff.js"; import type { HostIdentity } from "../../domain/desktop/hostIdentity.js"; -import { recoveryListLimitation } from "../../domain/desktop/seedRecoveryList.js"; import { scrubDconf } from "../../domain/desktop/seedScrub.js"; import { createEvidence, type Evidence } from "../../domain/evidence.js"; import type { JsonValue } from "../../domain/jsonValue.js"; @@ -80,6 +78,7 @@ const SEED_MAX_BYTES = 1024 * 1024 * 1024; /** What the caller asked to export, after CLI parsing. */ export interface DesktopExportRequest extends WithheldRequest { + readonly viewPolicy?: DesktopViewPolicyInput | undefined; readonly capture: string; readonly directory: string; readonly name: string | undefined; @@ -235,6 +234,7 @@ const planExport = async ( }); const launch = planLaunch(capture, electron, seeds, files); const plan: DesktopAppExportPlan = { + viewPolicy: request.viewPolicy, name: source, className: className(source), title: application.desktop_entry?.name ?? source, @@ -349,6 +349,7 @@ const runExport = async ( request.capture, readers, signal, + { viewPolicy: request.viewPolicy }, ); if (stateEvidence instanceof AnalysisError) throw stateEvidence; const state = desktopApplicationStateSchema.parse( @@ -477,6 +478,9 @@ export const exportDesktopApplication = async ( withhold_to: request.withhold?.to ?? null, withhold_paths: [...(request.withhold?.paths ?? [])], seed_withheld: request.seedWithheld === true, + ...(request.viewPolicy === undefined + ? {} + : { view_policy: { ...request.viewPolicy } }), }, result: { ...result, diff --git a/src/application/desktop/DesktopApplicationState.test.ts b/src/application/desktop/DesktopApplicationState.test.ts index 17e847f6fd..8ccc3d3b0f 100644 --- a/src/application/desktop/DesktopApplicationState.test.ts +++ b/src/application/desktop/DesktopApplicationState.test.ts @@ -98,6 +98,36 @@ it("treats both explicitly selected files as documents when one ends in -journal }); }); +describe("desktop state view admission", () => { + it.each([{ text_lines: 0 }, { xml_depth: 495 }, { json_depth: 992 }])( + "rejects an invalid view policy before reading a capture: %j", + async (viewPolicy) => { + const refused = await inspectDesktopApplicationState( + "/must-not-read", + readers, + undefined, + { viewPolicy }, + ); + expect(refused).toMatchObject({ _tag: "AnalysisInputError" }); + }, + ); + + it("preserves an unresolved captured path instead of reading the current file", async () => { + const folder = await writeCaptureFolder(capture()); + const missing = `${HOME}/Documents/missing.txt`; + const refused = await inspectDesktopApplicationState( + folder, + readers, + undefined, + { paths: [missing] }, + ); + expect(refused).toMatchObject({ + _tag: "AnalysisInputError", + userMessage: expect.stringContaining(missing), + }); + }); +}); + describe("inspectDesktopApplicationState", () => { it("assigns roles and reads each version into typed views and changes", async () => { const folder = await writeCaptureFolder(capture()); diff --git a/src/application/desktop/DesktopApplicationState.ts b/src/application/desktop/DesktopApplicationState.ts index 15a3326cc1..0b1dcf517d 100644 --- a/src/application/desktop/DesktopApplicationState.ts +++ b/src/application/desktop/DesktopApplicationState.ts @@ -1,9 +1,16 @@ +import { + desktopViewPolicy, + desktopViewPolicyInputSchema, + type DesktopViewPolicy, + type DesktopViewPolicyInput, +} from "../../domain/desktop/desktopViewPolicy.js"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { desktopApplicationCaptureSchema, type DesktopApplicationCapture, @@ -63,6 +70,7 @@ export interface DesktopStateReaders { readonly sqlite: ( database: string, signal?: AbortSignal, + policy?: DesktopViewPolicy, ) => Promise<{ ok: true; view: JsonValue } | { ok: false; error: string }>; readonly dconf: ( database: string, @@ -130,6 +138,7 @@ interface ViewContext { readonly names: ReadonlySet; readonly signal: AbortSignal | undefined; readonly index: StateIndex; + readonly policy: DesktopViewPolicy; } const dconfGroups = (text: string, names: ReadonlySet) => { @@ -242,7 +251,11 @@ const sqliteView = async ( staged.push({ name: `state.db${sidecar.suffix}`, bytes: bytes.value }); } return withStaged(staged, async ([database]) => { - const read = await context.readers.sqlite(database ?? "", context.signal); + const read = await context.readers.sqlite( + database ?? "", + context.signal, + context.policy, + ); return read.ok ? { view: read.view, error: null } : failed(read.error); }); }; @@ -273,9 +286,11 @@ const viewOf = async ( case "json": case "keyfile": case "xml": - return outcomeOf(textualView(format, input.bytes)); + return outcomeOf(textualView(format, input.bytes, context.policy)); case "zip": - return outcomeOf(await readZipView(input.bytes, context.signal)); + return outcomeOf( + await readZipView(input.bytes, context.signal, context.policy), + ); case "dconf": return dconfView(input, context); case "sqlite": @@ -341,7 +356,7 @@ const readVersion = async ( version.view_error = `retained content ${bytes.error}`; return; } - const document = openStateDocument(path, bytes.value); + const document = openStateDocument(path, bytes.value, context.policy); version.format = document.format; if (document.compression !== undefined) version.compression = document.compression; @@ -404,14 +419,15 @@ const describePath = async ( }; }; -const STATE_LIMITATIONS = [ +const stateLimitations = (policy: DesktopViewPolicy) => [ "Roles are inferred from state directories, open files, window titles, selected paths and dconf key names; a path can be misattributed.", "Views cover retained content only; a file that existed before the capture but outside the content roots has no starting view.", - "SQLite views list at most 500 rows of at most 64 ordinary tables, text views at most 5,000 lines and XML views at most 5,000 nodes; changes between truncated views, or views with an unread table, are marked incomplete.", - "JSON and key file views read files of at most 4 MiB, JSON nested at most 500 levels and only numbers JavaScript holds exactly; other files of those formats have a view_error.", + `SQLite views list at most ${String(policy.sqlite_rows)} rows of ${String(policy.sqlite_tables)} ordinary tables, text views ${String(policy.text_lines)} lines and XML views ${String(policy.xml_nodes)} nodes; changes between truncated views, or views with an unread table, are marked incomplete. Increase the corresponding --view-policy budget and retry the same capture; no restart is needed.`, + `Each projected SQLite table view is bounded at sqlite_view_bytes=${String(policy.sqlite_view_bytes)} UTF-8 JSON bytes. A refused table keeps its error; other tables remain readable. Raise sqlite_view_bytes in --view-policy on the next call for that refusal. The separate SQLite worker reply and value/record limits may refuse earlier and are not raised by view policy.`, + `JSON and key file views read files of at most ${String(policy.parsed_bytes)} bytes and JSON nested at most ${String(policy.json_depth)} levels; only numbers JavaScript holds exactly are admitted. Increase parsed_bytes or json_depth in --view-policy for a budget refusal; byte budgets do not change numeric exactness or Evidence nesting.`, "dconf views keep only groups whose path names the application.", - "Zip views list at most 1,000 entries and read an entry only when it is unencrypted, stored or deflated, at most 16 MiB and at most 100 times its compressed size, up to 64 MiB per archive; databases and archives inside an archive are not read.", - "A gzip file's view is its document's, read only when it is at most 16 MiB and at most 100 times the file's size; a database inside one is not read, nor a gzip file inside an archive.", + `Zip views list at most ${String(policy.zip_entries)} entries and read an entry only when unencrypted, stored or deflated, at most ${String(policy.zip_entry_bytes)} bytes and ${String(policy.zip_ratio)} times its compressed size, up to ${String(policy.zip_bytes)} bytes per archive. Increase the corresponding --view-policy budget for a resource refusal; databases and archives inside an archive are not read.`, + `A gzip file's view is its document's, read only when it is at most ${String(policy.zip_entry_bytes)} bytes and at most ${String(policy.zip_ratio)} times the file's size; a database inside one is not read, nor a gzip file inside an archive.`, ]; /** Read a capture folder into the application's state view. */ @@ -419,20 +435,46 @@ const inspectState = async ( folder: string, readers: DesktopStateReaders, signal: AbortSignal | undefined, + options: DesktopStateViewOptions, ): Promise => { + const policy = desktopViewPolicy(options.viewPolicy); const { evidence, capture } = await readDesktopCapture(folder, signal); const names = applicationNames(capture); const facts = roleFacts(capture); const index = indexState(capture); - const paths = await withRetainedContent(folder, async (content) => { - const context: ViewContext = { content, readers, names, signal, index }; - const described: DesktopStatePath[] = []; - for (const path of statePaths(index)) - described.push(await describePath(capture, path, facts, context)); - return described; - }); + const candidates = statePaths(index); + for (const selected of options.paths ?? []) + if (!candidates.includes(selected)) + throw new AnalysisInputError( + OPERATION, + { + userMessage: `Captured path ${JSON.stringify(selected)} was not found.`, + remediationAction: + "Use an exact path from this capture's state view; no current filesystem path is substituted.", + }, + [{ path: ["paths"], reason: "invalid_value" }], + ); + const paths = await withRetainedContent( + folder, + async (content) => { + const context: ViewContext = { + content, + readers, + names, + signal, + index, + policy, + }; + const described: DesktopStatePath[] = []; + for (const path of candidates) + if (options.paths === undefined || options.paths.includes(path)) + described.push(await describePath(capture, path, facts, context)); + return described; + }, + policy.file_bytes, + ); const limitations = [ - ...STATE_LIMITATIONS, + ...stateLimitations(policy), ...(capture.state.state_directories === undefined ? [ "The capture does not record its state directories as walked, so files in a symlinked state directory may be classed as documents.", @@ -440,6 +482,7 @@ const inspectState = async ( : []), ]; const result: DesktopApplicationState = { + effective_view_policy: policy, capture: { evidence_id: evidence.evidence_id, home: capture.host.home, @@ -457,7 +500,13 @@ const inspectState = async ( return createEvidence(undefined, DESKTOP_PROVIDER, { predicateType: "rea.desktop-application-state", operation: OPERATION, - parameters: { capture_evidence_id: evidence.evidence_id }, + parameters: { + capture_evidence_id: evidence.evidence_id, + ...(options.viewPolicy === undefined + ? {} + : { view_policy: { ...options.viewPolicy } }), + ...(options.paths === undefined ? {} : { paths: [...options.paths] }), + }, result, confidence: "inferred", authority: "analyst-inference", @@ -465,14 +514,35 @@ const inspectState = async ( }); }; +export interface DesktopStateViewOptions { + readonly viewPolicy?: DesktopViewPolicyInput | undefined; + /** Exact original captured paths; omission examines every retained path. */ + readonly paths?: readonly string[] | undefined; +} + /** Inspect one capture folder, returning its state view Evidence or a failure. */ export const inspectDesktopApplicationState = async ( folder: string, readers: DesktopStateReaders, signal?: AbortSignal, + options: DesktopStateViewOptions = {}, ): Promise => { try { - return await inspectState(folder, readers, signal); + const admitted = desktopViewPolicyInputSchema.safeParse( + options.viewPolicy === undefined ? {} : options.viewPolicy, + ); + if (!admitted.success) + return analysisInputErrorFromIssues( + OPERATION, + admitted.error.issues, + options.viewPolicy, + ); + return await inspectState(folder, readers, signal, { + ...options, + ...(options.viewPolicy === undefined + ? {} + : { viewPolicy: admitted.data }), + }); } catch (cause: unknown) { return desktopOperationFailure( cause, diff --git a/src/application/desktop/DesktopExportFiles.ts b/src/application/desktop/DesktopExportFiles.ts index 6bd5ddd1d7..7cda6f352c 100644 --- a/src/application/desktop/DesktopExportFiles.ts +++ b/src/application/desktop/DesktopExportFiles.ts @@ -1,3 +1,8 @@ +import { + DESKTOP_VIEW_DEFAULTS, + desktopViewFileLimitMessage, + type DesktopViewPolicy, +} from "../../domain/desktop/desktopViewPolicy.js"; import { createHash } from "node:crypto"; import { Readable } from "node:stream"; @@ -138,6 +143,7 @@ export const shippedSeedViews = async ( copies: readonly RetainedCopy[], identity: HostIdentity, signal: AbortSignal | undefined, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, ): Promise> => { const views = new Map(); for (const copy of copies) { @@ -150,15 +156,23 @@ export const shippedSeedViews = async ( const bytes = await retainedBytes(content, copy, signal); const shipped = shippedSeed(bytes, seed, identity).bytes; if (shipped === undefined) continue; - const document = openStateDocument(seed.home, shipped); + if (shipped.length > policy.file_bytes) { + views.set(seed.home, { + format: null, + view: null, + error: desktopViewFileLimitMessage(policy), + }); + continue; + } + const document = openStateDocument(seed.home, shipped, policy); const { format } = document; const view = "error" in document ? err(document.error) : format === "zip" - ? await readZipView(document.bytes, signal) + ? await readZipView(document.bytes, signal, policy) : isTextualFormat(format) - ? textualView(format, document.bytes) + ? textualView(format, document.bytes, policy) : undefined; views.set(seed.home, { format, diff --git a/src/application/desktop/DesktopExportReaders.ts b/src/application/desktop/DesktopExportReaders.ts index a5c1eda76d..014fd7df5a 100644 --- a/src/application/desktop/DesktopExportReaders.ts +++ b/src/application/desktop/DesktopExportReaders.ts @@ -1,3 +1,7 @@ +import { + desktopViewPolicy, + type DesktopViewPolicyInput, +} from "../../domain/desktop/desktopViewPolicy.js"; import { readFile, stat } from "node:fs/promises"; import { join } from "node:path"; @@ -29,6 +33,7 @@ import type { DesktopExportRequest } from "./DesktopApplicationExport.js"; /** What the export reads a reader's file from: the retained content, the seeds it ships and each file's versions. */ export interface ReaderSources { readonly capture: string; + readonly viewPolicy?: DesktopViewPolicyInput | undefined; readonly copies: readonly RetainedCopy[]; readonly files: ReadonlyMap; readonly ends: StateEnds; @@ -107,14 +112,18 @@ const exportReaderViews = /** What a task's checks read beyond the state view: the text seeds' views as the bundle ships them, its readers' and its screenshots. */ export const taskReads = async ( - request: Pick, + request: Pick< + DesktopExportRequest, + "capture" | "tasks" | "screens" | "viewPolicy" + >, maxBytes: number, - found: Omit, + found: Omit, ) => { const sources = { ...found, capture: request.capture, screens: request.screens, + viewPolicy: request.viewPolicy, maxBytes, }; return readsOf(request.tasks.length, sources); @@ -132,6 +141,7 @@ const readsOf = async (tasks: number, sources: ReaderSources) => ({ sources.copies, sources.identity, sources.signal, + desktopViewPolicy(sources.viewPolicy), ), sources.maxBytes, ), diff --git a/src/application/desktop/DesktopRetainedContent.ts b/src/application/desktop/DesktopRetainedContent.ts index a24bbddead..826dfd7574 100644 --- a/src/application/desktop/DesktopRetainedContent.ts +++ b/src/application/desktop/DesktopRetainedContent.ts @@ -1,3 +1,4 @@ +import { DESKTOP_VIEW_DEFAULTS } from "../../domain/desktop/desktopViewPolicy.js"; import { createHash } from "node:crypto"; import { join } from "node:path"; @@ -22,7 +23,7 @@ import { const MIB = 1024 * 1024; /** Largest retained file the state view reads. */ -const DEFAULT_MAX_BYTES = 64 * MIB; +const DEFAULT_MAX_BYTES = DESKTOP_VIEW_DEFAULTS.file_bytes; type FailedRelease = Extract< ArtifactCleanupAttempt, diff --git a/src/application/desktop/DesktopSqliteState.ts b/src/application/desktop/DesktopSqliteState.ts index 1472432a71..7b85929262 100644 --- a/src/application/desktop/DesktopSqliteState.ts +++ b/src/application/desktop/DesktopSqliteState.ts @@ -2,6 +2,7 @@ import type { z } from "zod"; import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import type { JsonValue } from "../../domain/jsonValue.js"; +import { jsonValueByteLength } from "../../domain/jsonValueByteLength.js"; import { sqliteDatabaseSchema, sqliteValueSchema, @@ -10,8 +11,11 @@ import { import { plainRecord } from "../../domain/desktop/stateFormats.js"; import type { SqliteDatabaseService } from "../sqlite/SqliteDatabaseService.js"; -const ROW_LIMIT = 500; -const MAX_TABLES = 64; +import { + DESKTOP_VIEW_DEFAULTS, + desktopSqliteViewLimitMessage, + type DesktopViewPolicy, +} from "../../domain/desktop/desktopViewPolicy.js"; /** A database's ordinary tables as JSON rows, or why it could not be read. */ export type SqliteStateView = @@ -79,6 +83,7 @@ export const readSqliteStateView = async ( service: SqliteReader, database: string, signal?: AbortSignal, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, ): Promise => { const inventory = await report( service, @@ -90,10 +95,10 @@ export const readSqliteStateView = async ( const ordinary = inventory.schema.tables.filter( (table) => table.kind === "table", ); - for (const table of ordinary.slice(0, MAX_TABLES)) { + for (const table of ordinary.slice(0, policy.sqlite_tables)) { const selected = await report( service, - { path: database, table: table.name, row_limit: ROW_LIMIT }, + { path: database, table: table.name, row_limit: policy.sqlite_rows }, signal, ); if (typeof selected === "string") { @@ -101,7 +106,7 @@ export const readSqliteStateView = async ( continue; } const rows = selected.rows; - tables[table.name] = { + const view = { columns: rows?.columns ?? [], primary_key: table.columns .filter((column) => column.primary_key_position > 0) @@ -110,12 +115,25 @@ export const readSqliteStateView = async ( left.primary_key_position - right.primary_key_position, ) .map((column) => column.name), - rows: (rows?.values ?? []).map((row) => row.map(cellValue)), + rows: [] as JsonValue[][], truncated: rows?.truncated ?? false, }; + let bytes = jsonValueByteLength(view, policy.sqlite_view_bytes); + for (const row of rows?.values ?? []) { + if (bytes > policy.sqlite_view_bytes) break; + const projected = row.map(cellValue); + bytes += + (view.rows.length === 0 ? 0 : 1) + + jsonValueByteLength(projected, policy.sqlite_view_bytes - bytes); + if (bytes <= policy.sqlite_view_bytes) view.rows.push(projected); + } + tables[table.name] = + bytes > policy.sqlite_view_bytes + ? { error: desktopSqliteViewLimitMessage(policy) } + : view; } return { ok: true, - view: { tables, tables_truncated: ordinary.length > MAX_TABLES }, + view: { tables, tables_truncated: ordinary.length > policy.sqlite_tables }, }; }; diff --git a/src/application/desktop/DesktopTaskDraft.ts b/src/application/desktop/DesktopTaskDraft.ts index 04fffcb406..08f3f8c84d 100644 --- a/src/application/desktop/DesktopTaskDraft.ts +++ b/src/application/desktop/DesktopTaskDraft.ts @@ -1,3 +1,4 @@ +import type { DesktopViewPolicyInput } from "../../domain/desktop/desktopViewPolicy.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { exportedStateFiles } from "../../domain/desktop/appExport.js"; import { DRAFT_PROMPT } from "../../domain/desktop/appTask.js"; @@ -56,12 +57,14 @@ const draft = async ( readers: DesktopStateReaders, signal: AbortSignal | undefined, selection: DraftSelection, + viewPolicy: DesktopViewPolicyInput | undefined, ): Promise => { const { evidence, capture } = await readDesktopCapture(folder, signal); const stateEvidence = await inspectDesktopApplicationState( folder, readers, signal, + { viewPolicy }, ); if (stateEvidence instanceof AnalysisError) throw stateEvidence; const state = desktopApplicationStateSchema.parse( @@ -114,6 +117,9 @@ const draft = async ( }); const environment = exportEnvironmentName(capture, undefined); const result = { + ...(state.effective_view_policy === undefined + ? {} + : { effective_view_policy: state.effective_view_policy }), application: { environment, title: capture.application.desktop_entry?.name ?? environment, @@ -160,6 +166,9 @@ const draft = async ( capture_evidence_id: evidence.evidence_id, seed_checkpoint: selection.seed ?? null, end_checkpoint: selection.end ?? null, + ...(stateEvidence.parameters.view_policy === undefined + ? {} + : { view_policy: stateEvidence.parameters.view_policy }), }, result, confidence: "inferred", @@ -177,9 +186,10 @@ export const draftDesktopApplicationTask = async ( readers: DesktopStateReaders, signal?: AbortSignal, selection: DraftSelection = {}, + viewPolicy?: DesktopViewPolicyInput, ): Promise => { try { - return await draft(folder, readers, signal, selection); + return await draft(folder, readers, signal, selection, viewPolicy); } catch (cause: unknown) { return desktopOperationFailure( cause, diff --git a/src/application/desktop/DesktopZipView.ts b/src/application/desktop/DesktopZipView.ts index d83bb70bc1..855bcf8e04 100644 --- a/src/application/desktop/DesktopZipView.ts +++ b/src/application/desktop/DesktopZipView.ts @@ -1,3 +1,7 @@ +import { + DESKTOP_VIEW_DEFAULTS, + type DesktopViewPolicy, +} from "../../domain/desktop/desktopViewPolicy.js"; import { Reader, Uint8ArrayWriter, @@ -12,7 +16,6 @@ import { } from "../../artifacts/ZipArtifactReader.js"; import { DAMAGED_ZIP_ENTRY, - MAX_ZIP_ENTRIES, UNREADABLE_ZIP, unreadZipEntry, zipEntry, @@ -63,6 +66,7 @@ const readEntry = async ( entry: FileEntry, header: ZipEntryHeader, signal: AbortSignal | undefined, + policy: DesktopViewPolicy, ): Promise => { let bytes: Uint8Array; try { @@ -74,7 +78,7 @@ const readEntry = async ( signal?.throwIfAborted(); return unreadZipEntry(header, DAMAGED_ZIP_ENTRY); } - return zipEntry(header, bytes); + return zipEntry(header, bytes, policy); }; /** @@ -86,9 +90,10 @@ const readEntry = async ( export const readZipView = async ( bytes: Uint8Array, signal?: AbortSignal, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, ): Promise> => { // The directory is scanned first, so zip.js sees only the records the view lists. - const directory = scanZipDirectory(bytes, MAX_ZIP_ENTRIES); + const directory = scanZipDirectory(bytes, policy.zip_entries); if (directory === undefined) return err(UNREADABLE_ZIP); const reader = new ZipReader( new BoundedArchiveReader( @@ -113,11 +118,11 @@ export const readZipView = async ( method: entry.compressionMethod, encrypted: entry.encrypted, }; - const refusal = zipEntryRefusal(header, spent); + const refusal = zipEntryRefusal(header, spent, policy); if (refusal === undefined) spent += header.size; entries.push( refusal === undefined - ? await readEntry(entry, header, signal) + ? await readEntry(entry, header, signal, policy) : unreadZipEntry(header, refusal), ); } diff --git a/src/application/frida/FridaInstrumentationPort.ts b/src/application/frida/FridaInstrumentationPort.ts index 19bf5feab4..69f803cc19 100644 --- a/src/application/frida/FridaInstrumentationPort.ts +++ b/src/application/frida/FridaInstrumentationPort.ts @@ -1,3 +1,7 @@ +import type { + FridaCapturePolicy, + FridaCapturePolicyInput, +} from "../../domain/fridaCapturePolicy.js"; import type { AnalysisError, AnalysisErrorOptions, @@ -32,8 +36,9 @@ export interface FridaProcessObservation { readonly identifier: string | null; } -export type StartFridaSessionInput = FridaDeviceSelector & - ( +export type StartFridaSessionInput = FridaDeviceSelector & { + readonly captureLimits?: FridaCapturePolicyInput; +} & ( | { readonly mode: "attach"; readonly pid: number } | { readonly mode: "spawn"; @@ -47,6 +52,7 @@ export type FridaScriptSource = | { readonly sourceKind: "file"; readonly path: string }; export interface FridaSessionObservation { + readonly captureLimits?: FridaCapturePolicy; readonly sessionId: string; readonly deviceId: string; readonly target: string; @@ -56,6 +62,7 @@ export interface FridaSessionObservation { } export interface FridaScriptObservation { + readonly captureLimits?: FridaCapturePolicy; readonly scriptId: string; readonly sourceKind: "inline" | "file"; readonly sourcePath: string | null; diff --git a/src/application/frida/FridaInstrumentationService.test.ts b/src/application/frida/FridaInstrumentationService.test.ts index e3447f759a..bea7b7d50e 100644 --- a/src/application/frida/FridaInstrumentationService.test.ts +++ b/src/application/frida/FridaInstrumentationService.test.ts @@ -128,7 +128,9 @@ it("keeps default Frida status identity and messages and refreshes every summary expect(parent?.limitations).toEqual( expect.arrayContaining([ expect.stringContaining("16777216"), - expect.stringContaining("no runtime override"), + expect.stringContaining( + "Already discarded observations cannot be recovered", + ), ]), ); expect(parent?.authority).toBe("external-service"); diff --git a/src/application/frida/FridaInstrumentationService.ts b/src/application/frida/FridaInstrumentationService.ts index b615fda1a7..e6ece825d0 100644 --- a/src/application/frida/FridaInstrumentationService.ts +++ b/src/application/frida/FridaInstrumentationService.ts @@ -11,7 +11,6 @@ import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorClean import { AnalysisAccessDeniedError, AnalysisCancelledError, - AnalysisCapabilityUnavailableError, AnalysisInputError, } from "../../domain/analysisErrorCore.js"; import { err, ok, type Result } from "../../domain/result.js"; @@ -548,6 +547,9 @@ const fridaParameters = ( }, }), mode: input.mode, + ...(input.captureLimits === undefined + ? {} + : { capture_limits: input.captureLimits }), ...(input.mode === "attach" ? { pid: input.pid } : { @@ -563,6 +565,9 @@ const sourceOutput = (script: FridaScriptObservation) => ({ }); const scriptOutput = (script: FridaScriptObservation) => ({ + ...(script.captureLimits === undefined + ? {} + : { capture_limits: script.captureLimits }), script_id: script.scriptId, ...sourceOutput(script), messages: script.messages, @@ -570,6 +575,9 @@ const scriptOutput = (script: FridaScriptObservation) => ({ }); const sessionStatusOutput = (status: FridaSessionStatus) => ({ + ...(status.captureLimits === undefined + ? {} + : { capture_limits: status.captureLimits }), session_id: status.sessionId, device_id: status.deviceId, state: status.state, diff --git a/src/application/go/GoBinaryService.test.ts b/src/application/go/GoBinaryService.test.ts index aff5f4badb..a13a8e03c4 100644 --- a/src/application/go/GoBinaryService.test.ts +++ b/src/application/go/GoBinaryService.test.ts @@ -290,7 +290,7 @@ it("validates string_literals input options and output invariants", async () => { path: report.artifact.path, include_string_literals: true, - max_string_literals: 10_001, + max_string_literals: Number.MAX_SAFE_INTEGER + 1, }, ]) { const refused = await service.inspect(invalidInput); @@ -298,6 +298,18 @@ it("validates string_literals input options and output invariants", async () => if (!refused.ok) expect(refused.error).toBeInstanceOf(AnalysisInputError); } expect(calls).toBe(0); + const admitted = await service.inspect({ + path: report.artifact.path, + include_string_literals: true, + max_string_literals: 10_001, + }); + if (!admitted.ok) throw admitted.error; + expect(admitted.value.parameters).toEqual({ + path: report.artifact.path, + include_string_literals: true, + max_string_literals: 10_001, + }); + expect(calls).toBe(1); const baseLiterals: NonNullable = { evidence_kind: "inferred", diff --git a/src/application/javascript/JavaScriptApplicationEvidence.ts b/src/application/javascript/JavaScriptApplicationEvidence.ts index 560fb6ae7a..bc24688ced 100644 --- a/src/application/javascript/JavaScriptApplicationEvidence.ts +++ b/src/application/javascript/JavaScriptApplicationEvidence.ts @@ -90,6 +90,9 @@ const parameters = ( input: AnalyzeJavaScriptApplicationInput, ): EvidenceObservation["parameters"] => ({ input_path: input.input_path, + ...(input.semantic_precision === undefined + ? {} + : { semantic_precision: input.semantic_precision }), format: input.format, integrity_policy: input.integrity_policy, ...(input.max_semantic_nodes === undefined diff --git a/src/application/javascript/JavaScriptApplicationService.ts b/src/application/javascript/JavaScriptApplicationService.ts index ff4f25a4d0..fad0217392 100644 --- a/src/application/javascript/JavaScriptApplicationService.ts +++ b/src/application/javascript/JavaScriptApplicationService.ts @@ -3,6 +3,7 @@ import { ASAR_HEADER_ADMISSION_SETTING, parseAsarHeaderAdmission, } from "../../config/asarHeaderAdmission.js"; +import { parseAsarListingAdmission } from "../../config/asarListingAdmission.js"; import { z } from "zod"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; @@ -69,12 +70,22 @@ export const analyzeJavaScriptApplicationValidated = async ( options.environment?.[ASAR_HEADER_ADMISSION_SETTING], ); if (!asarHeaderAdmission.ok) return asarHeaderAdmission; + const listingAdmission = parseAsarListingAdmission( + options.environment?.REA_ASAR_MAX_LISTING_BYTES, + ); + if (!listingAdmission.ok) return listingAdmission; const overrides = options.settings?.().overrides; const header = overrides?.["asar.max_header_bytes"]; - const acceptedAdmission = - header === undefined + const listing = overrides?.["asar.max_listing_bytes"]; + const acceptedAdmission = { + ...(header === undefined ? asarHeaderAdmission.value - : { maximumBytes: header, source: "runtime" as const }; + : { maximumBytes: header, source: "runtime" as const }), + listing: + listing === undefined + ? listingAdmission.value + : { maximumBytes: listing, source: "runtime" as const }, + }; let reconstructed: JavaScriptArtifactReconstructionResult | undefined; let result: JavaScriptApplicationAnalysisResult | undefined; let evidence: Evidence | undefined; diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts index 926e6e35fb..1e1e5ecab5 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts @@ -317,7 +317,9 @@ const graphCoverage = (context: JavaScriptArtifactGraphContext) => { const limits = [ ...new Map( [ - ...semanticResourceLimitCoverage(resourceLimits), + ...semanticResourceBounds(context).flatMap(({ limit, precision }) => + semanticResourceLimitCoverage([limit], precision), + ), ...executionLimits, ...(inlineScriptsTruncated ? [HTML_INLINE_SCRIPT_LOCATION_COVERAGE_LIMIT] @@ -340,17 +342,24 @@ const graphCoverage = (context: JavaScriptArtifactGraphContext) => { return completeApplicationCoverage(); }; +const semanticResourceBounds = (context: JavaScriptArtifactGraphContext) => [ + ...new Map( + context.analysis.files.flatMap(({ semantic }) => + semantic === null + ? [] + : semanticCoverageResourceLimits(semantic.ir.coverage).map((limit) => { + const fact = { limit, precision: semantic.ir.coverage.precision }; + return [JSON.stringify(fact), fact] as const; + }), + ), + ).values(), +]; + const semanticResourceLimits = ( context: JavaScriptArtifactGraphContext, ): JavaScriptSemanticResourceLimit[] => [ - ...new Set( - context.analysis.files.flatMap(({ semantic }) => - semantic === null - ? [] - : semanticCoverageResourceLimits(semantic.ir.coverage), - ), - ), + ...new Set(semanticResourceBounds(context).map(({ limit }) => limit)), ].sort(); const graphLimitations = ( @@ -426,7 +435,9 @@ const graphLimitations = ( "Sender, frame, URL, and origin checks are observations only; REA does not claim that they enforce a complete authorization policy.", ] : []), - ...semanticResourceLimits(context).map(semanticResourceLimitReason), + ...semanticResourceBounds(context).map(({ limit, precision }) => + semanticResourceLimitReason(limit, precision), + ), ...(context.analysis.files.some( ({ javascript }) => (javascript?.electron.native_addon_bindings.length ?? 0) > 0, diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts index 11e993d88e..8a89d439fc 100644 --- a/src/application/javascript/JavaScriptArtifactReconstruction.ts +++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts @@ -182,6 +182,9 @@ export const reconstructJavaScriptArtifact = async ( : { fileNodes: input.max_semantic_nodes_per_file }), }); const sourceLimits = { + ...(input.semantic_precision === undefined + ? {} + : { semanticPrecision: input.semantic_precision }), heapMb: input.max_heap_mb ?? overrides?.["javascript.max_heap_mb"] ?? diff --git a/src/application/javascript/JavaScriptModuleRelationships.ts b/src/application/javascript/JavaScriptModuleRelationships.ts index 63b3b31bf4..eb4ce77804 100644 --- a/src/application/javascript/JavaScriptModuleRelationships.ts +++ b/src/application/javascript/JavaScriptModuleRelationships.ts @@ -518,7 +518,11 @@ const relationshipEvidence = ( limitations: [ ...input.semantic.ir.limitations, ...semanticCoverageResourceLimits(input.semantic.ir.coverage).map( - semanticResourceLimitReason, + (limit) => + semanticResourceLimitReason( + limit, + input.semantic.ir.coverage.precision, + ), ), ...limitations, ], @@ -531,7 +535,10 @@ const semanticCoverage = ( if (semantic.ir.coverage.status === "complete" && resourceLimits.length === 0) return completeApplicationCoverage(); return partialApplicationCoverage( - semanticResourceLimitCoverage(resourceLimits), + semanticResourceLimitCoverage( + resourceLimits, + semantic.ir.coverage.precision, + ), semantic.ir.coverage.omittedCount, ); }; diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index bb14d397f3..ea29bb7cbf 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -723,15 +723,17 @@ const projectFileResourceLimits = ( family: "data-flow", relation_kinds: ["defines"], reason: "resource-limit", - detail: semanticResourceLimitReason(limit), + detail: semanticResourceLimitReason(limit, context.ir.coverage.precision), candidate_node_ids: [], evidence: { ...unknownSemanticEvidence(context.file, null), coverage: partialApplicationCoverage( - semanticResourceLimitCoverage([limit]), + semanticResourceLimitCoverage([limit], context.ir.coverage.precision), null, ), - limitations: [semanticResourceLimitReason(limit)], + limitations: [ + semanticResourceLimitReason(limit, context.ir.coverage.precision), + ], }, }); } @@ -859,15 +861,17 @@ function* createCallSiteNodes( family: "call-flow", relation_kinds: ["argument-to-parameter"], reason: "resource-limit", - detail: semanticResourceLimitReason(limit), + detail: semanticResourceLimitReason(limit, ir.coverage.precision), candidate_node_ids: [], evidence: { ...unknownSemanticEvidence(file, call.location), coverage: partialApplicationCoverage( - semanticResourceLimitCoverage([limit]), + semanticResourceLimitCoverage([limit], ir.coverage.precision), null, ), - limitations: [semanticResourceLimitReason(limit)], + limitations: [ + semanticResourceLimitReason(limit, ir.coverage.precision), + ], }, }); } diff --git a/src/application/javascript/JavaScriptWrappedCallPathCancellation.test.ts b/src/application/javascript/JavaScriptWrappedCallPathCancellation.test.ts index b13fbf0dfe..df1b605b37 100644 --- a/src/application/javascript/JavaScriptWrappedCallPathCancellation.test.ts +++ b/src/application/javascript/JavaScriptWrappedCallPathCancellation.test.ts @@ -42,7 +42,7 @@ it("cancels actual call-path work while its first seed indexes a long block", as ).rejects.toMatchObject({ reason: "cancelled", cause: reason }); expect(work.next()).toEqual({ done: true, value: undefined }); expect(accumulator.ipc).toEqual([]); - expect(accumulator.electronCallPathLimitReached).toBeUndefined(); + expect(accumulator.electronCallPathLimits).toBeUndefined(); // A new analysis owns a new index and can still complete the same source. const completed = createJavaScriptAnalysisAccumulator(source.length); @@ -57,5 +57,5 @@ it("cancels actual call-path work while its first seed indexes a long block", as ), ), ).toContainEqual(expect.objectContaining({ entry: "first", conditions: [] })); - expect(completed.electronCallPathLimitReached).toBeUndefined(); + expect(completed.electronCallPathLimits).toBeUndefined(); }); diff --git a/src/application/managed/ManagedMemberComparisonService.ts b/src/application/managed/ManagedMemberComparisonService.ts index d3ac5bad64..c337097c20 100644 --- a/src/application/managed/ManagedMemberComparisonService.ts +++ b/src/application/managed/ManagedMemberComparisonService.ts @@ -97,14 +97,18 @@ interface ManagedMemberComparisonPathDependencies { readonly readSnapshot: typeof readManagedSnapshot; } +interface ManagedMemberComparisonPathInput { + readonly leftPath: string; + readonly rightPath: string; + /** Applied to both sides with inspect_managed_members selection semantics. */ + readonly typeNames?: readonly string[]; + readonly maxSnapshotBytes?: number; + readonly maxRepresentationBytes?: number; +} + /** Inspect two local PE artifacts and return one derived comparison Evidence. */ export const compareManagedMemberPaths = async ( - input: { - readonly leftPath: string; - readonly rightPath: string; - /** Applied to both sides with inspect_managed_members selection semantics. */ - readonly typeNames?: readonly string[]; - }, + input: ManagedMemberComparisonPathInput, dependencies: Partial = {}, resources?: ArtifactResourceScope, ): Promise> => { @@ -146,6 +150,9 @@ export const compareManagedMemberPaths = async ( let snapshot: Awaited>; try { snapshot = await readSnapshot(path, operation, { + ...(input.maxSnapshotBytes === undefined + ? {} + : { maxBytes: input.maxSnapshotBytes }), expectedSha256: target.sha256, ...(resources === undefined ? {} : { resources }), }); @@ -161,6 +168,9 @@ export const compareManagedMemberPaths = async ( try { inspection = inspectManagedMembersBytes(snapshot.bytes, target, { userStrings: false, + ...(input.maxRepresentationBytes === undefined + ? {} + : { maxRepresentationBytes: input.maxRepresentationBytes }), ...(input.typeNames === undefined ? {} : { typeNames: input.typeNames }), @@ -177,10 +187,19 @@ export const compareManagedMemberPaths = async ( } const evidence = createEvidence(target, MANAGED_STATIC_PROVIDER, { operation: "inspect_managed_members", - parameters: - input.typeNames === undefined + parameters: { + ...(input.typeNames === undefined ? {} - : { type_names: [...input.typeNames] }, + : { type_names: [...input.typeNames] }), + ...(input.maxSnapshotBytes === undefined + ? {} + : { max_snapshot_bytes: input.maxSnapshotBytes }), + ...(input.maxRepresentationBytes === undefined + ? {} + : { + max_metadata_representation_bytes: input.maxRepresentationBytes, + }), + }, result: inspection, rawResult: null, limitations: inspection.limitations, @@ -223,11 +242,7 @@ const createManagedMemberComparisonEvidence = ( left: Evidence, right: Evidence, result: ManagedMemberComparisonResult, - sourcePaths?: { - readonly leftPath: string; - readonly rightPath: string; - readonly typeNames?: readonly string[]; - }, + pathRequest?: ManagedMemberComparisonPathInput, ): Evidence => createEvidence(undefined, MANAGED_WORKFLOW_PROVIDER, { predicateType: "rea.managed-member-comparison", @@ -235,15 +250,24 @@ const createManagedMemberComparisonEvidence = ( parameters: { left_evidence_id: left.evidence_id, right_evidence_id: right.evidence_id, - ...(sourcePaths === undefined + ...(pathRequest === undefined ? {} : { - left_path: sourcePaths.leftPath, - right_path: sourcePaths.rightPath, + left_path: pathRequest.leftPath, + right_path: pathRequest.rightPath, }), - ...(sourcePaths?.typeNames === undefined + ...(pathRequest?.typeNames === undefined ? {} - : { type_names: [...sourcePaths.typeNames] }), + : { type_names: [...pathRequest.typeNames] }), + ...(pathRequest?.maxSnapshotBytes === undefined + ? {} + : { max_snapshot_bytes: pathRequest.maxSnapshotBytes }), + ...(pathRequest?.maxRepresentationBytes === undefined + ? {} + : { + max_metadata_representation_bytes: + pathRequest.maxRepresentationBytes, + }), }, result, rawResult: null, @@ -252,11 +276,11 @@ const createManagedMemberComparisonEvidence = ( environment: null, limitations: result.limitations, locations: - sourcePaths === undefined + pathRequest === undefined ? [] : [ - { kind: "artifact-path", path: sourcePaths.leftPath }, - { kind: "artifact-path", path: sourcePaths.rightPath }, + { kind: "artifact-path", path: pathRequest.leftPath }, + { kind: "artifact-path", path: pathRequest.rightPath }, ], evidenceLinks: result.evidence_links, }); diff --git a/src/application/reverse/ReverseEngineeringService.ts b/src/application/reverse/ReverseEngineeringService.ts index 42a948a950..19e2d515ea 100644 --- a/src/application/reverse/ReverseEngineeringService.ts +++ b/src/application/reverse/ReverseEngineeringService.ts @@ -28,6 +28,8 @@ import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js"; import { err, ok, type Result } from "../../domain/result.js"; import { DEFAULT_REVERSE_COMMAND_TIMEOUT_MS, + DEFAULT_REVERSE_COMMAND_OUTPUT_BYTES, + MAX_REVERSE_COMMAND_OUTPUT_BYTES, MAX_REVERSE_COMMAND_TIMEOUT_MS, objdumpInputSchema, rizinInputSchema, @@ -88,10 +90,16 @@ interface CommandLaunchOutcome { succeeded: number; } -const MAX_COMBINED_OUTPUT_BYTES = 16 * 1024 * 1024; -export const REVERSE_COMMAND_OUTPUT_LIMITATION = `REA stopped collecting output at its ${String(MAX_COMBINED_OUTPUT_BYTES)}-byte cap and requested termination of the provider process; the captured stdout and stderr are a prefix and command completion is unknown. Narrow the command or page its output.`; -// Eight concurrent captures bound retained provider output to 128 MiB. +const reverseCommandOutputLimitation = ( + bytes: number, + source: "default" | "request", +): string => + `REA stopped collecting output at max_output_bytes=${String(bytes)} bytes (source ${source}, default ${String(DEFAULT_REVERSE_COMMAND_OUTPUT_BYTES)}, range 1..${String(MAX_REVERSE_COMMAND_OUTPUT_BYTES)}) and requested termination of the provider process; captured stdout and stderr are a prefix and command completion is unknown. Narrow or page the provider command, or increase max_output_bytes (--max-output-bytes on the CLI) for the next call; no restart is required. Confirm command effects and cleanup before deciding whether it is safe to repeat. Client receive limits are separate.`; +// Bound simultaneous provider process families independently of output admission. const MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS = 8; +const MAX_ACTIVE_CAPTURE_BYTES = + MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS * + DEFAULT_REVERSE_COMMAND_OUTPUT_BYTES; export interface ReverseEngineeringServiceOptions { readonly environment?: Readonly; @@ -140,6 +148,17 @@ export const resolveRizinCommandRequest = ( ); }; +/** One accepted command, including the controls retained through admission and cleanup. */ +interface ReverseCommandObservation { + readonly operation: ReverseEngineeringOperation; + readonly profile: ReverseEngineeringProviderCommand; + readonly path: string | undefined; + readonly parameters: Readonly>; + readonly signal: AbortSignal | undefined; + readonly timeoutMs: number; + readonly outputBudgetBytes: number; +} + /** Shared application workflow for caller-selected local analysis commands. */ export class ReverseEngineeringService { readonly #environment: Readonly; @@ -159,6 +178,7 @@ export class ReverseEngineeringService { AbortController, Promise> >(); + #reservedOutputBytes = 0; #closing = false; #closePromise: Promise | undefined; #dispatched = 0; @@ -297,6 +317,7 @@ export class ReverseEngineeringService { const { detail, command_timeout_ms: commandTimeoutMs, + max_output_bytes: maxOutputBytes, ...input } = parsed.value; if (detail === "summary") { @@ -314,14 +335,17 @@ export class ReverseEngineeringService { } else parameters = input; if (commandTimeoutMs !== undefined) parameters = { ...parameters, command_timeout_ms: commandTimeoutMs }; - const observed = await this.#observe( - "inspect_with_objdump", - command, - input.operation === "architectures" ? undefined : input.path, + if (maxOutputBytes !== undefined) + parameters = { ...parameters, max_output_bytes: maxOutputBytes }; + const observed = await this.#observe({ + operation: "inspect_with_objdump", + profile: command, + path: input.operation === "architectures" ? undefined : input.path, parameters, - options.signal, - commandTimeoutMs ?? DEFAULT_REVERSE_COMMAND_TIMEOUT_MS, - ); + signal: options.signal, + timeoutMs: commandTimeoutMs ?? DEFAULT_REVERSE_COMMAND_TIMEOUT_MS, + outputBudgetBytes: maxOutputBytes ?? DEFAULT_REVERSE_COMMAND_OUTPUT_BYTES, + }); return observed.ok && detail === "summary" ? summarizeRetainedAnalysis(observed.value, retention) : observed; @@ -342,26 +366,27 @@ export class ReverseEngineeringService { request: RizinInput, options: { readonly signal?: AbortSignal } = {}, ): Promise> { - const { command_timeout_ms: commandTimeoutMs, ...input } = request; + const { + command_timeout_ms: commandTimeoutMs, + max_output_bytes: maxOutputBytes, + ...input + } = request; const command = this.#commands.rizin(input); - return this.#observe( - "execute_rizin_command", - command, - input.path, - commandTimeoutMs === undefined ? input : request, - options.signal, - commandTimeoutMs ?? DEFAULT_REVERSE_COMMAND_TIMEOUT_MS, - ); + return this.#observe({ + operation: "execute_rizin_command", + profile: command, + path: input.path, + parameters: request, + signal: options.signal, + timeoutMs: commandTimeoutMs ?? DEFAULT_REVERSE_COMMAND_TIMEOUT_MS, + outputBudgetBytes: maxOutputBytes ?? DEFAULT_REVERSE_COMMAND_OUTPUT_BYTES, + }); } #observe( - operation: ReverseEngineeringOperation, - profile: ReverseEngineeringProviderCommand, - path: string | undefined, - parameters: Readonly>, - signal: AbortSignal | undefined, - timeoutMs: number, + request: ReverseCommandObservation, ): Promise> { + const { operation, signal, outputBudgetBytes } = request; if (signal?.aborted) return Promise.resolve( err(new AnalysisCancelledError(operation, { cause: signal.reason })), @@ -377,21 +402,26 @@ export class ReverseEngineeringService { ); if ( this.#activeOperations.size >= - MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS + MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS || + this.#reservedOutputBytes + outputBudgetBytes > MAX_ACTIVE_CAPTURE_BYTES ) return Promise.resolve( err( new AnalysisResourceConstraintError( operation, "transport", - `Concurrent reverse-engineering operation capacity reached (${String(MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS)} operations). This release has no runtime override.`, + `Reverse-engineering capture capacity reached (${String(MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS)} operations or ${String(MAX_ACTIVE_CAPTURE_BYTES)} reserved output bytes).`, { max_concurrent_operations: MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS, active_operations: this.#activeOperations.size, + max_active_capture_bytes: MAX_ACTIVE_CAPTURE_BYTES, + reserved_output_bytes: this.#reservedOutputBytes, + requested_output_bytes: outputBudgetBytes, }, { - remediationAction: `Wait for an active objdump or rizin command to finish, then retry. This capacity of ${String(MAX_CONCURRENT_REVERSE_ENGINEERING_OPERATIONS)} operations has no runtime override, and command_timeout_ms does not raise it.`, + retryable: true, + remediationAction: `Wait for active objdump or rizin work to finish, then retry, or lower max_output_bytes to fit the ${String(MAX_ACTIVE_CAPTURE_BYTES)}-byte aggregate capture admission. This live capacity has no runtime override; command_timeout_ms does not raise it. No restart is required.`, }, ), ), @@ -406,30 +436,32 @@ export class ReverseEngineeringService { signal === undefined || signal instanceof AbortSignal ? controller.signal : signal; - const running = this.#observeProvider( - operation, - profile, - path, - parameters, - operationSignal, - timeoutMs, - ); + this.#reservedOutputBytes += outputBudgetBytes; + const running = this.#observeProvider({ + ...request, + signal: operationSignal, + }); const tracked = running.finally(() => { signal?.removeEventListener("abort", relayAbort); this.#activeOperations.delete(controller); + this.#reservedOutputBytes -= outputBudgetBytes; }); this.#activeOperations.set(controller, tracked); return tracked; } async #observeProvider( - operation: ReverseEngineeringOperation, - profile: ReverseEngineeringProviderCommand, - path: string | undefined, - parameters: Readonly>, - signal: AbortSignal | undefined, - timeoutMs: number, + request: ReverseCommandObservation, ): Promise> { + const { + operation, + profile, + path, + parameters, + signal, + timeoutMs, + outputBudgetBytes, + } = request; const { executable: command, arguments: args } = profile; if (signal?.aborted) return err( @@ -540,7 +572,7 @@ export class ReverseEngineeringService { signal, timeout: timeoutMs, stopSignal: "SIGTERM", - maxBuffer: MAX_COMBINED_OUTPUT_BYTES, + maxBuffer: outputBudgetBytes, }); stdout = output.stdout; stderr = output.stderr; @@ -718,8 +750,6 @@ export class ReverseEngineeringService { : { cleanupOwner: darwinFailure.cleanupOwner }), }); } else { - const provider = - operation === "inspect_with_objdump" ? "objdump" : "rizin"; const setting = operation === "inspect_with_objdump" ? "REA_OBJDUMP_COMMAND" @@ -994,11 +1024,15 @@ export class ReverseEngineeringService { : digestAfter === null || digest === undefined ? null : digestAfter !== digest; + const outputBudgetSource = + parameters.max_output_bytes === undefined ? "default" : "request"; const result = { stdout, stderr, exit_code: exitCode, signal: signalName, + output_budget_bytes: outputBudgetBytes, + output_budget_source: outputBudgetSource, output_truncated: outputTruncated, completion_status: completionStatus, artifact_sha256_after: absolutePath === undefined ? null : digestAfter, @@ -1043,7 +1077,12 @@ export class ReverseEngineeringService { ] : []), ...(outputLimitReached - ? [REVERSE_COMMAND_OUTPUT_LIMITATION] + ? [ + reverseCommandOutputLimitation( + outputBudgetBytes, + outputBudgetSource, + ), + ] : outputTruncated ? [ "Captured provider output is partial because bytes were omitted; process completion is reported separately.", @@ -1092,6 +1131,9 @@ export class ReverseEngineeringService { ...(absolutePath === undefined ? { operation: "architectures", + ...(typeof parameters.max_output_bytes === "number" + ? { max_output_bytes: parameters.max_output_bytes } + : {}), ...(typeof parameters.command_timeout_ms === "number" ? { command_timeout_ms: parameters.command_timeout_ms } : {}), @@ -1212,7 +1254,8 @@ const runOwnedCommandOutput = }, { timeoutMs: options.timeout ?? DEFAULT_REVERSE_COMMAND_TIMEOUT_MS, - diagnosticBytes: MAX_COMBINED_OUTPUT_BYTES, + diagnosticBytes: + options.maxBuffer ?? DEFAULT_REVERSE_COMMAND_OUTPUT_BYTES, acceptNonZeroExit: true, }, { signal: options.signal }, @@ -1349,7 +1392,7 @@ const reverseCommandTimeoutGuidance = ( timeoutMs < MAX_REVERSE_COMMAND_TIMEOUT_MS ? `Increase command_timeout_ms above ${String(timeoutMs)} (integer milliseconds from 1 to ${String(MAX_REVERSE_COMMAND_TIMEOUT_MS)}; omitted means ${String(DEFAULT_REVERSE_COMMAND_TIMEOUT_MS)}), or use --command-timeout-ms on the CLI.` : `command_timeout_ms is already at the maximum ${String(MAX_REVERSE_COMMAND_TIMEOUT_MS)} ms; no higher deadline is supported.` - } The new value applies on the next call; no restart is required. The ${String(MAX_COMBINED_OUTPUT_BYTES)} byte (16 MiB) combined output cap has no runtime override. Allow execution, cleanup and delivery headroom in the outer MCP client deadline. Confirm outstanding work and cleanup before retrying.`, + } The new value applies on the next call; no restart is required. The combined output allowance is separately selected by max_output_bytes (default ${String(DEFAULT_REVERSE_COMMAND_OUTPUT_BYTES)} bytes). Allow execution, cleanup and delivery headroom in the outer MCP client deadline. Confirm outstanding work and cleanup before retrying.`, }); export const digestFile = async ( diff --git a/src/application/wasm/WasmArtifactService.test.ts b/src/application/wasm/WasmArtifactService.test.ts index 8b5180c0fe..def72a7344 100644 --- a/src/application/wasm/WasmArtifactService.test.ts +++ b/src/application/wasm/WasmArtifactService.test.ts @@ -20,6 +20,18 @@ it.each([ path: "/module", command_timeout_ms, })), + ...[ + "max_input_bytes", + "max_tool_bytes", + "max_output_bytes", + "max_interface_bytes", + "max_artifact_paths", + ].flatMap((name) => + [0, -1, 1.5, Number.MAX_SAFE_INTEGER + 1, "1000", null].map((value) => ({ + path: "/module", + [name]: value, + })), + ), ])("rejects invalid selection before effects: %j", async (input) => { const service = new WasmArtifactService({ close: () => Promise.resolve(), @@ -52,19 +64,22 @@ it("preserves the selected command budget in provider input and Evidence", async expect(result.value.normalized_result).toEqual(value); }); -it("rejects a provider report that silently changes the selected command budget", async () => { - const value = wasmArtifactFixture(); - const service = new WasmArtifactService({ - close: () => Promise.resolve(), - inspect: async () => ok(wasmArtifactExecution(value)), - }); - expect( - await service.inspect({ - path: value.artifact.path, - command_timeout_ms: 42_000, - }), - ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); -}); +it.each([{ command_timeout_ms: 42_000 }, { max_output_bytes: 1024 }])( + "rejects a provider report that silently changes the selected budget: %j", + async (selection) => { + const value = wasmArtifactFixture(); + const service = new WasmArtifactService({ + close: () => Promise.resolve(), + inspect: async () => ok(wasmArtifactExecution(value)), + }); + expect( + await service.inspect({ + path: value.artifact.path, + ...selection, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }, +); it.each(["path", "subject-path", "digest", "wat", "candidates", "glue"])( "rejects mismatched upstream observation: %s", async (problem) => { diff --git a/src/application/wasm/WasmArtifactService.ts b/src/application/wasm/WasmArtifactService.ts index b6f1619b8f..a31bf8db82 100644 --- a/src/application/wasm/WasmArtifactService.ts +++ b/src/application/wasm/WasmArtifactService.ts @@ -1,19 +1,30 @@ import type { AnalysisSettingsReader } from "../../domain/analysisSettings.js"; +import { constants as bufferConstants } from "node:buffer"; import type { ExecutionOptions } from "../AnalysisProvider.js"; -import { createAnalysisExecutionEvidence } from "../../domain/analysisObservation.js"; +import { + analysisExecutionEnvelopeForAdmission, + createAnalysisExecutionEvidence, +} from "../../domain/analysisObservation.js"; import type { WasmArtifactPort } from "./WasmArtifactPort.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; -import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../../domain/analysisErrorCore.js"; import type { Evidence } from "../../domain/evidence.js"; import { inspectWasmArtifactInputSchema, wasmArtifactSchema, wasmArtifactMatchesSubject, DEFAULT_WASM_COMMAND_TIMEOUT_MS, + DEFAULT_WASM_ARTIFACT_PATHS, + MAX_WASM_ARTIFACT_PATHS, + wasmResourceLimits, } from "../../domain/wasm/wasmArtifact.js"; import { err, type Result } from "../../domain/result.js"; import { completeProviderEvidence } from "../ProviderOperationExecution.js"; import { parseAnalysisInput } from "../AnalysisInput.js"; +import { requireAuthenticationHeadroom } from "../JsonAuthenticationHeadroom.js"; const OPERATION = "inspect_wasm_artifact"; @@ -41,6 +52,32 @@ export class WasmArtifactService { options?.signal, ); if (!input.ok) return input; + const maximumPaths = + input.value.max_artifact_paths ?? DEFAULT_WASM_ARTIFACT_PATHS; + for (const field of ["glue_paths", "candidate_paths"] as const) { + if (input.value[field].length > maximumPaths) + return err( + new AnalysisResourceConstraintError( + OPERATION, + "capacity", + `${field} exceeds max_artifact_paths (${String(maximumPaths)} paths)`, + { + maximum_paths: maximumPaths, + default_paths: DEFAULT_WASM_ARTIFACT_PATHS, + source: + input.value.max_artifact_paths === undefined + ? "default" + : "request", + selected_paths: input.value[field].length, + input_field: field, + }, + { + retryable: false, + remediationAction: `Raise max_artifact_paths to at least ${String(input.value[field].length)} (maximum ${String(MAX_WASM_ARTIFACT_PATHS)}), or use --max-artifact-paths on the CLI; then retry the same selection. The value applies on the next inspection without restart. File and command byte budgets remain independent.`, + }, + ), + ); + } const timeout = input.value.command_timeout_ms ?? this.settings?.().overrides["wasm.command_timeout_ms"] ?? @@ -51,12 +88,29 @@ export class WasmArtifactService { }); if (!inspected.ok) return inspected; const execution = inspected.value; + const admission = requireAuthenticationHeadroom( + analysisExecutionEnvelopeForAdmission( + OPERATION, + input.value, + execution, + "observed", + ), + { + operation: OPERATION, + path: input.value.path, + description: "Complete WASM Evidence cannot be assembled", + serializedCodeUnitLimit: bufferConstants.MAX_STRING_LENGTH, + }, + ); + if (!admission.ok) return admission; const report = wasmArtifactSchema.safeParse(execution.result); if ( !report.success || report.data.artifact.path !== input.value.path || execution.subject.path !== input.value.path || report.data.tool_profile.timeout_ms !== timeout || + report.data.tool_profile.output_budget_bytes !== + wasmResourceLimits(input.value).outputBytes || !wasmArtifactMatchesSubject(report.data.artifact, execution.subject) || JSON.stringify(report.data.glue.map((item) => item.artifact.path)) !== JSON.stringify(input.value.glue_paths) || diff --git a/src/application/wasm/WasmInterfaceComparisonService.test.ts b/src/application/wasm/WasmInterfaceComparisonService.test.ts index 478377fb5e..64bca0279f 100644 --- a/src/application/wasm/WasmInterfaceComparisonService.test.ts +++ b/src/application/wasm/WasmInterfaceComparisonService.test.ts @@ -6,11 +6,7 @@ import { wasmInterfaceComparisonSchema } from "../../contracts/wasm/wasmComparis import { compareWasmInterfaces } from "./WasmInterfaceComparisonService.js"; import { retainAnalysisResultSources } from "../investigation/recordEvidenceSources.js"; import { ok } from "../../domain/result.js"; -import { - WASM_INTERFACE_REPRESENTATION_BYTES, - wasmInterfaceEntryRepresentationBytes, - type WasmInterface, -} from "../../domain/wasm/wasmInterface.js"; +import type { WasmInterface } from "../../domain/wasm/wasmInterface.js"; import { wasmArtifactFixture } from "../../../tests/fixtures/wasm/artifact.js"; const facts = ( @@ -240,69 +236,43 @@ it("resolves exact retained references and permits cancellation between comparis expect(cancelled.error.partialObservation).toBeUndefined(); }); -/** One inventory under the comparison cap; two of them exceed it together. */ -const admittedInventory = (label: string): WasmInterface => { - const base = facts(label).entries[0]!; - const minimum = Math.floor(WASM_INTERFACE_REPRESENTATION_BYTES / 2) + 4096; - let low = 0; - let high = 4_000_000; - let best = 0; - while (low <= high) { - const mid = Math.floor((low + high) / 2); - const name = `${label}-${"n".repeat(mid)}`; - const bytes = wasmInterfaceEntryRepresentationBytes({ - ...base, - name, - wat_form: `(export "${name}" (func 0))`, - }); - if (bytes >= minimum) { - best = mid; - high = mid - 1; - } else low = mid + 1; - } - const name = `${label}-${"n".repeat(best)}`; - const entry = { - ...base, - name, - wat_form: `(export "${name}" (func 0))`, - }; - const bytes = wasmInterfaceEntryRepresentationBytes(entry); - if ( - bytes >= WASM_INTERFACE_REPRESENTATION_BYTES || - bytes + 512 >= WASM_INTERFACE_REPRESENTATION_BYTES - ) - throw new Error("Sized inventory is not individually admitted"); - return { state: "complete", issues: [], entries: [entry] }; -}; - -it("refuses two admitted inventories that together exceed the comparison cap", async () => { - const left = admittedInventory("left"); - const right = admittedInventory("right"); - const empty: WasmInterface = { state: "complete", issues: [], entries: [] }; +it("admits each selected inventory independently and recovers a combined comparison with more capacity", async () => { + const left = evidence(facts("left")); + const right = evidence(facts("right")); + const empty = evidence({ state: "complete", issues: [], entries: [] }); const alone = await compareWasmInterfaces({ - left: evidence(left), - right: evidence(empty), + left, + right: empty, scope: "both", + max_interface_bytes: 2048, }); expect(alone.ok).toBe(true); + const selected = { left, right, scope: "both" as const }; const combined = await compareWasmInterfaces({ - left: evidence(left), - right: evidence(right), - scope: "both", + ...selected, + max_interface_bytes: 2048, }); - if (combined.ok) throw new Error("Expected comparison allowance refusal"); + if (combined.ok) + throw new Error("Expected combined comparison capacity refusal"); const projected = projectAnalysisError(combined.error); - expect(projected.retryable).toBe(false); - expect(projected.code).toBe("resource_constraint"); - expect(combined.error).toMatchObject({ resource: "capacity" }); - expect(projected.remediation.action).toContain( - "The WASM interface comparison inventory", - ); - expect(projected.remediation.action).toContain("fixed byte allowance"); - expect(projected.remediation.action).not.toContain("fixed count"); - expect(projected.remediation.action).not.toMatch( - /Wait for an active analysis/u, - ); + expect(projected).toMatchObject({ + code: "resource_constraint", + retryable: false, + details: { reported_limits: { maximum_bytes: 2048, source: "request" } }, + }); + expect(projected.remediation.action).toContain("max_interface_bytes"); + const recovered = await compareWasmInterfaces({ + ...selected, + max_interface_bytes: 8192, + }); + if (!recovered.ok) throw recovered.error; + expect(recovered.value.evidence.normalized_result).toMatchObject({ + coverage: "complete-within-inputs", + summary: { added: 1, removed: 1 }, + }); + expect(recovered.value.evidence.parameters).toMatchObject({ + max_interface_bytes: 8192, + }); }); it.each([false, true])( diff --git a/src/application/wasm/WasmInterfaceComparisonService.ts b/src/application/wasm/WasmInterfaceComparisonService.ts index d7689cdbe1..d0e2b49f17 100644 --- a/src/application/wasm/WasmInterfaceComparisonService.ts +++ b/src/application/wasm/WasmInterfaceComparisonService.ts @@ -1,14 +1,17 @@ -import { - WASM_INTERFACE_REPRESENTATION_BYTES, - wasmInterfaceEntryRepresentationBytes, -} from "../../domain/wasm/wasmInterface.js"; +import { wasmInterfaceEntryRepresentationBytes } from "../../domain/wasm/wasmInterface.js"; import { setImmediate } from "node:timers/promises"; +import { constants as bufferConstants } from "node:buffer"; +import { requireAuthenticationHeadroom } from "../JsonAuthenticationHeadroom.js"; import { ARTIFACT_COMPARISON_PROVIDER } from "../../domain/comparisonEvidenceIdentity.js"; import { compareWasmInterfacesInputSchema, wasmInterfaceComparisonSchema, } from "../../contracts/wasm/wasmComparisonContracts.js"; -import { createEvidence, type Evidence } from "../../domain/evidence.js"; +import { + createEvidence, + evidenceEnvelopeForAdmission, + type Evidence, +} from "../../domain/evidence.js"; import { wasmArtifactSchema, wasmArtifactMatchesSubject, @@ -18,9 +21,8 @@ import type { WasmInterfaceEntry } from "../../domain/wasm/wasmInterface.js"; import { AnalysisCancelledError, AnalysisInputError, - AnalysisResourceConstraintError, } from "../../domain/analysisErrorCore.js"; -import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; +import { WasmInterfaceBudget } from "../../domain/wasm/wasmInterfaceBudget.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../../domain/result.js"; import { @@ -131,6 +133,10 @@ export const compareWasmInterfaces = async ( entry: WasmInterfaceEntry; } >(); + const budget = new WasmInterfaceBudget( + resolved.value.input.max_interface_bytes, + OPERATION, + ); let work = 0; for (const [side, inspection] of [ ["left", left], @@ -142,9 +148,11 @@ export const compareWasmInterfaces = async ( assertActive(); } if (!selected(entry)) continue; + budget.reserve(wasmInterfaceEntryRepresentationBytes(entry)); const key = JSON.stringify([entry.direction, entry.module, entry.name]); let group = groups.get(key); if (group === undefined) { + budget.reserve(512); group = { left: [], right: [], entry }; groups.set(key, group); } @@ -162,29 +170,7 @@ export const compareWasmInterfaces = async ( unchanged: 0, unknown: 0, }; - let remaining = WASM_INTERFACE_REPRESENTATION_BYTES; for (const group of groups.values()) { - remaining -= - 512 + - group.left.reduce( - (sum, entry) => sum + wasmInterfaceEntryRepresentationBytes(entry), - 0, - ) + - group.right.reduce( - (sum, entry) => sum + wasmInterfaceEntryRepresentationBytes(entry), - 0, - ); - if (remaining < 0) - throw new AnalysisResourceConstraintError( - OPERATION, - "capacity", - "WASM comparison exceeds typed interface representation capacity", - { maximum_bytes: WASM_INTERFACE_REPRESENTATION_BYTES }, - fixedAllowanceOptions( - "The WASM interface comparison inventory", - "bytes", - ), - ); if (++work % 256 === 0) { await setImmediate(); assertActive(); @@ -251,7 +237,7 @@ export const compareWasmInterfaces = async ( "Exact declared interface equality is not runtime/ABI compatibility, successful linking or original-source identity.", "Typed facts come from the selected validated WABT decoding; unsupported signature semantics and duplicate names remain unknown.", ]; - const report = wasmInterfaceComparisonSchema.parse({ + const observedReport = { left: { evidence_id: left.evidence.evidence_id, artifact: left.result.artifact, @@ -265,20 +251,42 @@ export const compareWasmInterfaces = async ( changes, summary, limitations, - }); - const evidence = createEvidence(undefined, ARTIFACT_COMPARISON_PROVIDER, { + }; + const observation = { operation: OPERATION, predicateType: "rea.wasm-interface-comparison", parameters: { left_evidence_id: left.evidence.evidence_id, right_evidence_id: right.evidence.evidence_id, scope, + ...(resolved.value.input.max_interface_bytes === undefined + ? {} + : { max_interface_bytes: resolved.value.input.max_interface_bytes }), }, - result: report, - confidence: "derived", - authority: "analyst-inference", + result: observedReport, + confidence: "derived" as const, + authority: "analyst-inference" as const, limitations, evidenceLinks: [left.evidence.evidence_id, right.evidence.evidence_id], + }; + const admission = requireAuthenticationHeadroom( + evidenceEnvelopeForAdmission( + undefined, + ARTIFACT_COMPARISON_PROVIDER, + observation, + ), + { + operation: OPERATION, + path: "selected WASM Evidence", + description: "Complete WASM comparison Evidence cannot be assembled", + serializedCodeUnitLimit: bufferConstants.MAX_STRING_LENGTH, + }, + ); + if (!admission.ok) return admission; + const report = wasmInterfaceComparisonSchema.parse(observedReport); + const evidence = createEvidence(undefined, ARTIFACT_COMPARISON_PROVIDER, { + ...observation, + result: report, }); await setImmediate(); const completed = completeProviderEvidence(evidence, options?.signal); diff --git a/src/artifacts/ArtifactEmbeddedFormatProbes.test.ts b/src/artifacts/ArtifactEmbeddedFormatProbes.test.ts index 2030a34817..2e9b7a870d 100644 --- a/src/artifacts/ArtifactEmbeddedFormatProbes.test.ts +++ b/src/artifacts/ArtifactEmbeddedFormatProbes.test.ts @@ -11,6 +11,79 @@ import { dotnetBundleFixture } from "./dotnet/DotnetBundle.fixture.js"; import { denoMachO } from "./deno/DenoStandalone.fixture.js"; import * as denoLocation from "./deno/DenoStandaloneLocation.js"; import * as pckLocation from "./GodotPckLocation.js"; +import { resolveArtifactReaderLimits } from "../domain/artifactReaderLimits.js"; +import { nodeSeaElf } from "./nodeSea/NodeSea.fixture.js"; + +it("readmits cached SEA recognition when the accepted policy changes", async () => { + const path = join( + await createTestTempDirectory("rea-probe-policy-"), + "carrier", + ); + await writeFile(path, nodeSeaElf({ fuse: "set", ownership: "section" })); + await withOwnedRegularFile(path, {}, async (source) => { + const original = new ArtifactEmbeddedFormatProbes(path, source); + const location = await original.nodeSea(); + expect(location).toBeDefined(); + const lower = new ArtifactEmbeddedFormatProbes( + path, + source, + undefined, + original.completedFacts(), + resolveArtifactReaderLimits({ node_sea: { blob_bytes: 1 } }), + ); + await expect(lower.nodeSea()).rejects.toMatchObject({ + reason: "limit", + readerLimit: { + parameter: "artifact_limits.node_sea.blob_bytes", + effective_value: 1, + }, + }); + expect(Object.hasOwn(lower.completedFacts(), "nodeSea")).toBe(false); + const restored = new ArtifactEmbeddedFormatProbes( + path, + source, + undefined, + lower.completedFacts(), + resolveArtifactReaderLimits(), + ); + expect(await restored.nodeSea()).toEqual(location); + const recognitionLimited = new ArtifactEmbeddedFormatProbes( + path, + source, + undefined, + restored.completedFacts(), + resolveArtifactReaderLimits({ + node_sea: { recognition_header_count: 1 }, + }), + ); + await expect(recognitionLimited.nodeSea()).resolves.toBeUndefined(); + expect(Object.hasOwn(recognitionLimited.completedFacts(), "nodeSea")).toBe( + false, + ); + expect( + recognitionLimited.completedFacts().nodeSeaRecognitionFailure, + ).toMatchObject({ + readerLimit: { + parameter: "artifact_limits.node_sea.recognition_header_count", + effective_value: 1, + required_value: 3, + }, + }); + const recognitionRestored = new ArtifactEmbeddedFormatProbes( + path, + source, + undefined, + recognitionLimited.completedFacts(), + resolveArtifactReaderLimits({ + node_sea: { recognition_header_count: 3 }, + }), + ); + expect(await recognitionRestored.nodeSea()).toEqual(location); + expect( + recognitionRestored.completedFacts().nodeSeaRecognitionFailure, + ).toBeUndefined(); + }); +}); it("distinguishes examined absence from unexamined formats and honors cancellation on a cached result", async () => { const root = await createTestTempDirectory("rea-embedded-probe-facts-"); diff --git a/src/artifacts/ArtifactEmbeddedFormatProbes.ts b/src/artifacts/ArtifactEmbeddedFormatProbes.ts index 70c2150d05..75a526b5dd 100644 --- a/src/artifacts/ArtifactEmbeddedFormatProbes.ts +++ b/src/artifacts/ArtifactEmbeddedFormatProbes.ts @@ -6,6 +6,10 @@ import { locateDenoStandalone, type DenoStandaloneProbe, } from "./deno/DenoStandaloneLocation.js"; +import { + resolveArtifactReaderLimits, + type ArtifactReaderLimits, +} from "../domain/artifactReaderLimits.js"; import type { StableRegularFileDescriptor, RegularFileState, @@ -28,13 +32,15 @@ import { type PyInstallerCookieObservation, } from "./pyinstaller/CArchiveLayout.js"; -/** Completed observations only; omission means unexamined, and null is examined absence. */ +/** Completed observations and explicit recognition limits; null alone is examined absence. */ export interface ArtifactEmbeddedFormatProbeFacts { readonly sourceState: RegularFileState; + readonly readerLimits?: ArtifactReaderLimits; readonly bunStandalone?: BunStandaloneProbe; readonly denoStandalone?: DenoStandaloneProbe; readonly godotPck?: GodotPckLocation | null; readonly nodeSea?: NodeSeaLocation | null; + readonly nodeSeaRecognitionFailure?: ArtifactReaderFailure; readonly dotnetBundleHeaderOffset?: number | null; readonly pyInstaller?: PyInstallerCookieObservation; } @@ -48,6 +54,7 @@ export class ArtifactEmbeddedFormatProbes { private readonly source: StableRegularFileDescriptor, private readonly signal?: AbortSignal, previous?: ArtifactEmbeddedFormatProbeFacts, + private readonly limits: ArtifactReaderLimits = resolveArtifactReaderLimits(), ) { abortIfNeeded(signal); if ( @@ -58,7 +65,22 @@ export class ArtifactEmbeddedFormatProbes { "integrity", `Root artifact changed before inventory: ${path}`, ); - this.#facts = previous ?? { sourceState: regularFileState(source.initial) }; + const reusable = + previous !== undefined && + JSON.stringify(previous.readerLimits) === JSON.stringify(limits); + if (previous !== undefined && !reusable) { + const { + bunStandalone: _bun, + nodeSea: _sea, + nodeSeaRecognitionFailure: _seaLimit, + ...facts + } = previous; + this.#facts = { ...facts, readerLimits: limits }; + } else + this.#facts = previous ?? { + sourceState: regularFileState(source.initial), + readerLimits: limits, + }; } async godotPck(prefix?: Uint8Array): Promise { @@ -79,13 +101,29 @@ export class ArtifactEmbeddedFormatProbes { abortIfNeeded(this.signal); if (this.#facts.nodeSea !== undefined) return this.#facts.nodeSea ?? undefined; - const location = await locateNodeSea( - this.source.handle, - this.source.initial.size, - this.signal, - ); - this.#facts = { ...this.#facts, nodeSea: location ?? null }; - return location; + if (this.#facts.nodeSeaRecognitionFailure !== undefined) return undefined; + try { + const location = await locateNodeSea( + this.source.handle, + this.source.initial.size, + this.signal, + this.limits.node_sea, + ); + this.#facts = { ...this.#facts, nodeSea: location ?? null }; + return location; + } catch (cause) { + if ( + cause instanceof ArtifactReaderFailure && + (cause.readerLimit?.parameter === + "artifact_limits.node_sea.recognition_header_count" || + cause.readerLimit?.parameter === + "artifact_limits.node_sea.recognition_load_command_bytes") + ) { + this.#facts = { ...this.#facts, nodeSeaRecognitionFailure: cause }; + return undefined; + } + throw cause; + } } async pyInstaller(): Promise { @@ -131,6 +169,7 @@ export class ArtifactEmbeddedFormatProbes { this.source.handle, this.source.initial.size, this.signal, + this.limits.bun, ); abortIfNeeded(this.signal); this.#facts = { ...this.#facts, bunStandalone: observed }; diff --git a/src/artifacts/ArtifactOperationFailure.test.ts b/src/artifacts/ArtifactOperationFailure.test.ts index f64b0f4c5c..89d1575c9d 100644 --- a/src/artifacts/ArtifactOperationFailure.test.ts +++ b/src/artifacts/ArtifactOperationFailure.test.ts @@ -3,6 +3,78 @@ import { expect, it } from "vitest"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; import { translateArtifactFailure } from "./ArtifactOperationFailure.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js"; +import { constants as bufferConstants } from "node:buffer"; + +it.each([ + { effective: 1, required: bufferConstants.MAX_STRING_LENGTH + 1 }, + { effective: bufferConstants.MAX_STRING_LENGTH, required: undefined }, +])( + "does not recommend a larger budget when no supported increase can repair it: %o", + ({ effective, required }) => { + const failure = artifactReaderLimitFailure( + "artifact_limits.godot.directory_bytes", + effective, + "bytes", + required, + ); + const projected = projectAnalysisError( + translateArtifactFailure("inspect_artifact", failure), + ); + expect(projected).toMatchObject({ + retryable: false, + details: { + reader_limit: { + can_increase: false, + maximum: bufferConstants.MAX_STRING_LENGTH, + }, + }, + }); + expect(projected.remediation.action).toContain("No supported increase"); + }, +); + +it("preserves adjustable reader admission facts and gives cleanup priority", () => { + const failure = artifactReaderLimitFailure( + "artifact_limits.node_sea.argument_count", + 1, + "count", + 2, + ); + const adjustable = projectAnalysisError( + translateArtifactFailure("inspect_artifact", failure), + ); + expect(adjustable).toMatchObject({ + code: "truncated", + retryable: true, + details: { + reader_limit: { + parameter: "artifact_limits.node_sea.argument_count", + effective_value: 1, + required_value: 2, + unit: "count", + applies_at: "next_operation", + rea_restart_required: false, + }, + }, + }); + expect(adjustable.remediation.action).toContain( + "retry the same complete artifact", + ); + const wrapped = ArtifactReaderFailure.withCleanup(failure, { + reason: "Reader descriptor release failed", + resources: ["owned input descriptor"], + }); + const projected = projectAnalysisError( + translateArtifactFailure("inspect_artifact", wrapped), + ); + expect(projected.retryable).toBe(false); + expect(projected.details).toMatchObject({ + reader_limit: adjustable.details?.reader_limit, + cleanup: "incomplete", + resources: ["owned input descriptor"], + }); +}); it.each(["capacity", "file-size", "memory"] as const)( "retains the producing %s limit through public error projection", diff --git a/src/artifacts/ArtifactOperationFailure.ts b/src/artifacts/ArtifactOperationFailure.ts index 38fdb29662..4b4f008764 100644 --- a/src/artifacts/ArtifactOperationFailure.ts +++ b/src/artifacts/ArtifactOperationFailure.ts @@ -34,12 +34,17 @@ export const translateArtifactFailure = ( ? {} : { admission: cause.admission }), ...(cause.boundary === undefined ? {} : { boundary: cause.boundary }), + ...(cause.readerLimit === undefined + ? {} + : { readerLimit: cause.readerLimit }), }, ); if (cause.admission !== undefined) error.bindLimit({ ...analysisSettingLimit( - "asar.max_header_bytes", + cause.admission.kind === "asar-header" + ? "asar.max_header_bytes" + : "asar.max_listing_bytes", cause.admission.maximumBytes, cause.admission.source, ), diff --git a/src/artifacts/ArtifactPaths.ts b/src/artifacts/ArtifactPaths.ts index af40a88e1b..1c30156935 100644 --- a/src/artifacts/ArtifactPaths.ts +++ b/src/artifacts/ArtifactPaths.ts @@ -104,6 +104,11 @@ export class ArtifactPathRegistry { private readonly budget?: { readonly maxParts: number; readonly maxNodes: number; + readonly limitFailure?: ( + resource: "path_parts" | "path_nodes", + effectiveValue: number, + requiredValue: number, + ) => ArtifactReaderFailure; }, ) {} @@ -136,10 +141,17 @@ export class ArtifactPathRegistry { #visit(path: string, kind?: DestinationEntryKind): void { const parts = path.split("/"); if (this.budget !== undefined && parts.length > this.budget.maxParts) - throw artifactFixedLimitFailure( - "Artifact path depth", - this.budget.maxParts, - "path parts", + throw ( + this.budget.limitFailure?.( + "path_parts", + this.budget.maxParts, + parts.length, + ) ?? + artifactFixedLimitFailure( + "Artifact path depth", + this.budget.maxParts, + "path parts", + ) ); if (this.budget !== undefined) { let observed = this.#root; @@ -147,10 +159,17 @@ export class ArtifactPathRegistry { const existing = observed.children.get(part); if (existing === undefined) { if (parts.length - index > this.budget.maxNodes - this.#nodeCount) - throw artifactFixedLimitFailure( - "Artifact path tree", - this.budget.maxNodes, - "path nodes", + throw ( + this.budget.limitFailure?.( + "path_nodes", + this.budget.maxNodes, + this.#nodeCount + parts.length - index, + ) ?? + artifactFixedLimitFailure( + "Artifact path tree", + this.budget.maxNodes, + "path nodes", + ) ); break; } diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index 499cddf3d9..0059551db1 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -1,4 +1,8 @@ import type { EspFlashSelection } from "../domain/firmware/espFlashLocation.js"; +import { + resolveArtifactReaderLimits, + type ArtifactReaderLimitsInput, +} from "../domain/artifactReaderLimits.js"; import type { AnalysisSettingsReader } from "../domain/analysisSettings.js"; import { runtimeAsarHeaderAdmission } from "../config/runtimeAnalysisSettings.js"; import { artifactTargetRefusal } from "../domain/apple/artifactTargetApplicability.js"; @@ -348,6 +352,13 @@ class ArtifactClient implements AnalysisClient { this.settings, ), integrityPolicy: parsed.integrity_policy, + ...(parsed.artifact_limits === undefined + ? {} + : { + readerLimits: resolveArtifactReaderLimits( + parsed.artifact_limits, + ), + }), resourceScope: this.#resourceScope, }, options?.signal, @@ -487,6 +498,9 @@ class ArtifactClient implements AnalysisClient { const parsed = artifactInventoryInputSchema.parse(parameters); const inventoryParameters = { integrity_policy: parsed.integrity_policy, + ...(parsed.artifact_limits === undefined + ? {} + : { artifact_limits: parsed.artifact_limits }), }; await options?.progress?.report({ phase: "inspect_artifact.inventory", @@ -536,6 +550,7 @@ class ArtifactClient implements AnalysisClient { private inventory( parsed: { readonly integrity_policy: "fail" | "record-and-continue"; + readonly artifact_limits?: ArtifactReaderLimitsInput | undefined; }, options?: ExecutionOptions, ) { @@ -554,6 +569,11 @@ class ArtifactClient implements AnalysisClient { espFlashSelection: espFlashSelection(this.target), ...(options?.signal === undefined ? {} : { signal: options.signal }), integrity: parsed.integrity_policy, + ...(parsed.artifact_limits === undefined + ? {} + : { + readerLimits: resolveArtifactReaderLimits(parsed.artifact_limits), + }), }); } } diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index dc9dc6da19..422b940be0 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -1,4 +1,5 @@ import { fixedAllowanceRemediation } from "../domain/fixedAllowanceRemediation.js"; +import type { ArtifactReaderLimitFact } from "../domain/artifactReaderLimits.js"; import type { ArtifactAdmissionLimit, ArtifactLimitBoundary, @@ -37,6 +38,7 @@ export interface ArtifactReaderFailureOptions extends ErrorOptions { readonly boundary?: ArtifactLimitBoundary; /** Named admission whose recovery must not be inferred from other limit text. */ readonly admission?: ArtifactAdmissionLimit; + readonly readerLimit?: ArtifactReaderLimitFact; } /** Archive-neutral entry metadata. Reader adapters never choose output paths. */ @@ -243,6 +245,7 @@ export class ArtifactReaderFailure extends Error { readonly boundary: ArtifactLimitBoundary | undefined; /** Named admission for a limit whose public recovery is owned by the producer. */ readonly admission: ArtifactAdmissionLimit | undefined; + readonly readerLimit: ArtifactReaderLimitFact | undefined; constructor( readonly reason: @@ -271,6 +274,7 @@ export class ArtifactReaderFailure extends Error { this.partialObservation = options?.partialObservation; this.boundary = options?.boundary; this.admission = options?.admission; + this.readerLimit = options?.readerLimit; } /** Retain completed facts when a later artifact step fails. */ @@ -326,6 +330,9 @@ export class ArtifactReaderFailure extends Error { ...(primary.admission === undefined ? {} : { admission: primary.admission }), + ...(primary.readerLimit === undefined + ? {} + : { readerLimit: primary.readerLimit }), ...(mergedObservation === undefined ? {} : { partialObservation: mergedObservation }), diff --git a/src/artifacts/ArtifactReaderLimits.ts b/src/artifacts/ArtifactReaderLimits.ts new file mode 100644 index 0000000000..858978c0e3 --- /dev/null +++ b/src/artifacts/ArtifactReaderLimits.ts @@ -0,0 +1,46 @@ +import type { ArtifactLimitBoundary } from "../domain/artifactOperationError.js"; +import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import { artifactReaderLimitMaximum } from "../domain/artifactReaderLimits.js"; + +/** Construct exact next-call recovery at the enforcing resource owner. */ +export const artifactReaderLimitFailure = ( + parameter: string, + effectiveValue: number, + unit: "bytes" | "count", + requiredValue?: number | bigint, + boundary: ArtifactLimitBoundary = "capacity", +): ArtifactReaderFailure => { + const required = + typeof requiredValue === "bigint" + ? requiredValue <= BigInt(Number.MAX_SAFE_INTEGER) + ? Number(requiredValue) + : requiredValue.toString() + : requiredValue; + const maximum = artifactReaderLimitMaximum(parameter, unit); + const canIncrease = + effectiveValue < maximum && + (requiredValue === undefined || + (typeof requiredValue === "bigint" + ? requiredValue <= BigInt(maximum) + : requiredValue <= maximum)); + const recovery = canIncrease + ? `Select a larger ${parameter} (1..${String(maximum)} ${unit}) and retry the same complete artifact. Changes apply on the next call without restarting REA.` + : `No supported increase to ${parameter} can admit this observation; the maximum is ${String(maximum)} ${unit}. Keep the complete input and inspect its declared dimensions and validity. If those declarations are valid, use an explicitly selected format-specific reader that supports the required representation.`; + const detail = `Artifact admission reached ${parameter} (${String(effectiveValue)} ${unit})${required === undefined ? "." : `; requires ${String(required)} ${unit}.`} ${recovery} Delivery paging does not raise this producer admission.`; + return new ArtifactReaderFailure("limit", detail, { + boundary, + userMessage: detail, + readerLimit: { + parameter, + effective_value: effectiveValue, + unit, + minimum: 1, + maximum, + ...(required === undefined ? {} : { required_value: required }), + applies_at: "next_operation", + agent_editable: true, + can_increase: canIncrease, + rea_restart_required: false, + }, + }); +}; diff --git a/src/artifacts/AsarArtifactReader.ts b/src/artifacts/AsarArtifactReader.ts index 27542d1f23..4dd8bf328c 100644 --- a/src/artifacts/AsarArtifactReader.ts +++ b/src/artifacts/AsarArtifactReader.ts @@ -79,7 +79,11 @@ export class AsarArtifactReader implements ArtifactReader { }; assertAsarReadActive(signal); try { - return await readAsarListing(admitted.header, signal); + return await readAsarListing( + admitted.header, + signal, + this.options.headerAdmission?.listing, + ); } catch (cause: unknown) { // Listing names members relative to this archive; keep which archive. if ( diff --git a/src/artifacts/AsarHeader.ts b/src/artifacts/AsarHeader.ts index 816a0bf8ae..5a40c9b528 100644 --- a/src/artifacts/AsarHeader.ts +++ b/src/artifacts/AsarHeader.ts @@ -15,6 +15,7 @@ import { DEFAULT_ASAR_MAX_HEADER_BYTES, MAXIMUM_ASAR_MAX_HEADER_BYTES, } from "../domain/analysisResourceLimits.js"; +import { parseAsarListingAdmission } from "../config/asarListingAdmission.js"; /** * Resolve the admission from a composition root's selected environment; an @@ -28,7 +29,11 @@ export const resolveAsarHeaderAdmission = ( environment[ASAR_HEADER_ADMISSION_SETTING], ); if (!parsed.ok) throw parsed.error; - return parsed.value; + const listing = parseAsarListingAdmission( + environment.REA_ASAR_MAX_LISTING_BYTES, + ); + if (!listing.ok) throw listing.error; + return { ...parsed.value, listing: listing.value }; }; const MEBIBYTE = 1024 * 1024; diff --git a/src/artifacts/AsarListing.ts b/src/artifacts/AsarListing.ts index f4744ec1c7..b534b97fb2 100644 --- a/src/artifacts/AsarListing.ts +++ b/src/artifacts/AsarListing.ts @@ -5,9 +5,10 @@ import { asarDeclaredSizeSchema, asarNodeSchema } from "./AsarNode.js"; import { ArtifactReaderFailure, type ArtifactEntry } from "./ArtifactReader.js"; import { assertAsarReadActive } from "./AsarFailure.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; - -/** Capacity for expanded paths and listing records, independent of packed bytes. */ -const MAX_LISTING_BYTES = 64 * 1024 * 1024; +import { + DEFAULT_ASAR_LISTING_ADMISSION, + type AsarListingAdmission, +} from "../config/asarListingAdmission.js"; const BLOCK_INTEGRITY_LIMITATION = "ASAR block integrity declarations were not validated. Whole-file digest comparison does not establish block integrity or Electron runtime acceptance."; /** Source text quoted in listing diagnostics; longer values keep their length. */ @@ -104,6 +105,7 @@ function* ownChildren(files: Record) { export const readAsarListing = async ( header: unknown, signal?: AbortSignal, + admission: AsarListingAdmission = DEFAULT_ASAR_LISTING_ADMISSION, ): Promise => { const parsedRoot = asarListingNodeSchema.safeParse(header); if (!parsedRoot.success) @@ -116,10 +118,17 @@ export const readAsarListing = async ( let retainedBytes = 0; const admitRetainedBytes = (bytes: number): void => { retainedBytes += bytes; - if (retainedBytes > MAX_LISTING_BYTES) + if (retainedBytes > admission.maximumBytes) throw new ArtifactReaderFailure( "limit", - "ASAR expanded listing exceeds its 64 MiB representation budget", + `ASAR expanded listing exceeds its ${String(admission.maximumBytes)} byte representation budget (source ${admission.source}). On MCP, set asar.max_listing_bytes with update_analysis_settings, then retry the same archive; the change applies to the next operation without restarting REA. For CLI, set REA_ASAR_MAX_LISTING_BYTES in the invocation environment. The budget must be a positive safe integer in bytes. Header admission and worker heaps do not raise this independent retention budget.`, + { + admission: { + kind: "asar-listing", + ...admission, + requiredBytes: retainedBytes, + }, + }, ); }; let examined = 0; diff --git a/src/artifacts/BunStandaloneArtifactReader.ts b/src/artifacts/BunStandaloneArtifactReader.ts index 0cfe43f55e..f229d9780a 100644 --- a/src/artifacts/BunStandaloneArtifactReader.ts +++ b/src/artifacts/BunStandaloneArtifactReader.ts @@ -1,3 +1,8 @@ +import { + DEFAULT_BUN_READER_LIMITS, + type BunReaderLimits, +} from "../domain/artifactReaderLimits.js"; +import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js"; import type { BunStandaloneAdmission } from "./bun/BunStandaloneAdmission.js"; import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; import { setImmediate } from "node:timers/promises"; @@ -30,7 +35,6 @@ import { import { ArtifactFileSource } from "./ArtifactFileSource.js"; import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; import { - MAX_BUN_RETAINED_BYTES, bunFormatFailure, bunMetadataBytesLimit, probeBunStandaloneGraph, @@ -44,8 +48,6 @@ import { const REGIONS = ["contents", "sourcemap", "bytecode", "module-info"] as const; type BunRegion = (typeof REGIONS)[number]; -const MAX_LOGICAL_PATH_PARTS = 256; -const MAX_LOGICAL_PATH_NODES = 250_000; interface BunModuleDestination { readonly path: string; readonly sourcePath?: string; @@ -72,6 +74,8 @@ export class BunStandaloneArtifactReader implements ArtifactReader { constructor( path: string, admitted?: StableRegularFileDescriptor | BunStandaloneAdmission, + private readonly limits: BunReaderLimits = DEFAULT_BUN_READER_LIMITS, + private readonly expandedBytes?: number, ) { if (admitted !== undefined && "kind" in admitted) { this.#admission = admitted; @@ -229,6 +233,7 @@ export class BunStandaloneArtifactReader implements ArtifactReader { source.handle, source.initial.size, signal, + this.limits, )); if (probe.status === "absent") throw bunFormatFailure("no Bun section or trailer"); @@ -239,16 +244,19 @@ export class BunStandaloneArtifactReader implements ArtifactReader { source.handle, probe.claim, signal, + this.limits, ); if (located.status === "absent") throw bunFormatFailure("no Bun section or trailer"); if (located.failure !== undefined) throw located.failure; if (located.graph === undefined) throw bunFormatFailure("graph data was not admitted"); - const decoded = await decodeBunModuleGraph(located.graph); + const decoded = await decodeBunModuleGraph(located.graph, this.limits); const entryModule = decoded.modules[decoded.entryPointId]; const entryDestination = - entryModule === undefined ? null : logicalModulePath(entryModule); + entryModule === undefined + ? null + : logicalModulePath(entryModule, this.limits.path_parts); const header: BunStandaloneMetadata = { kind: "bun-standalone", layout_revision: decoded.revision, @@ -263,33 +271,45 @@ export class BunStandaloneArtifactReader implements ArtifactReader { compile_exec_argv_hex: decoded.compileExecArgv.hex, flags: decoded.flags, }; - let retainedBytes = jsonValueByteLength(header, MAX_BUN_RETAINED_BYTES); - if (retainedBytes > MAX_BUN_RETAINED_BYTES) - throw bunMetadataBytesLimit(retainedBytes); + let retainedBytes = jsonValueByteLength( + header, + this.limits.representation_bytes, + ); + if (retainedBytes > this.limits.representation_bytes) + throw bunMetadataBytesLimit(retainedBytes, this.limits); const entries = new Map(); let declaredPlaintext = 0n; const expansionBudget = BigInt( - artifactExpansionBudgetBytes(located.graph.payloadLength), + this.expandedBytes ?? + artifactExpansionBudgetBytes(located.graph.payloadLength), ); const paths = new ArtifactPathRegistry({ - maxParts: MAX_LOGICAL_PATH_PARTS, - maxNodes: MAX_LOGICAL_PATH_NODES, + maxParts: this.limits.path_parts, + maxNodes: this.limits.path_nodes, + limitFailure: (field, maximum, required) => + artifactReaderLimitFailure( + `artifact_limits.bun.${field}`, + maximum, + "count", + required, + ), }); for (const module of decoded.modules) { assertArtifactReadActive(signal); const destination = module.index === decoded.entryPointId && entryDestination !== null ? entryDestination - : logicalModulePath(module); + : logicalModulePath(module, this.limits.path_parts); for (const region of REGIONS) { const range = regionRange(module, region); if (range === null) continue; declaredPlaintext += BigInt(range.length); if (declaredPlaintext > expansionBudget) - throw new ArtifactReaderFailure( - "limit", - `Bun standalone readable entries declare ${String(declaredPlaintext)} plaintext bytes, exceeding this graph's ${String(expansionBudget)}-byte expansion budget. This release has no runtime override. Host memory and NODE_OPTIONS do not raise it.`, - { boundary: "capacity" }, + throw artifactReaderLimitFailure( + "artifact_limits.expanded_bytes", + Number(expansionBudget), + "bytes", + declaredPlaintext, ); const entry = moduleEntry( located.graph, @@ -301,10 +321,10 @@ export class BunStandaloneArtifactReader implements ArtifactReader { paths.reservePath(entry.path); retainedBytes += jsonValueByteLength( entryMetadataJson(entry), - Math.max(0, MAX_BUN_RETAINED_BYTES - retainedBytes), + Math.max(0, this.limits.representation_bytes - retainedBytes), ); - if (retainedBytes > MAX_BUN_RETAINED_BYTES) - throw bunMetadataBytesLimit(retainedBytes); + if (retainedBytes > this.limits.representation_bytes) + throw bunMetadataBytesLimit(retainedBytes, this.limits); entries.set(entry.adapterKey, entry); } } @@ -387,20 +407,26 @@ const moduleEntry = ( * `sourcePath`. Unsafe, drive, and invalid UTF-8 names fall back to * `bun-modules/` so inventory can still retain the module. */ -const logicalModulePath = (module: BunDecodedModule): BunModuleDestination => { - const fallback = `bun-modules/${String(module.index)}`; +const logicalModulePath = ( + module: BunDecodedModule, + maximumParts: number, +): BunModuleDestination => { + const fallback = + maximumParts === 1 + ? `bun-module-${String(module.index)}` + : `bun-modules/${String(module.index)}`; const text = module.nameText.text; if (text === null || text.length === 0) return { path: fallback }; const candidate = text.startsWith("/") ? text.slice(1) : text; // Preserve virtual source facts without admitting an unbounded logical path tree. let parts = 1; for (let index = 0; index < candidate.length; index++) - if (candidate.charCodeAt(index) === 47 && ++parts > MAX_LOGICAL_PATH_PARTS) + if (candidate.charCodeAt(index) === 47 && ++parts > maximumParts) return { path: fallback, sourcePath: text, limitations: [ - `The virtual name exceeds the fixed ${String(MAX_LOGICAL_PATH_PARTS)}-part logical path allowance; an indexed path is used and the original name is retained.`, + `The virtual name exceeds artifact_limits.bun.path_parts (${String(maximumParts)} parts); an indexed path is used and the original name is retained.`, ], }; diff --git a/src/artifacts/DotnetBundleArtifactReader.ts b/src/artifacts/DotnetBundleArtifactReader.ts index 51b0a89a91..3cb88f68d6 100644 --- a/src/artifacts/DotnetBundleArtifactReader.ts +++ b/src/artifacts/DotnetBundleArtifactReader.ts @@ -1,6 +1,10 @@ import { Readable } from "node:stream"; import { finished, pipeline } from "node:stream/promises"; import { createInflateRaw } from "node:zlib"; +import { + DEFAULT_DOTNET_BUNDLE_READER_LIMITS, + type DotnetBundleReaderLimits, +} from "../domain/artifactReaderLimits.js"; import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; import { ArtifactFileSource } from "./ArtifactFileSource.js"; @@ -39,6 +43,7 @@ export class DotnetBundleArtifactReader implements ArtifactReader { constructor( path: string, admitted?: StableRegularFileDescriptor | DotnetBundleAdmission, + private readonly limits: DotnetBundleReaderLimits = DEFAULT_DOTNET_BUNDLE_READER_LIMITS, ) { if (admitted !== undefined && "kind" in admitted) { this.#admission = admitted; @@ -83,6 +88,7 @@ export class DotnetBundleArtifactReader implements ArtifactReader { source.initial.size, header, signal, + this.limits, ); this.#admission = undefined; } diff --git a/src/artifacts/GodotPckArtifactReader.test.ts b/src/artifacts/GodotPckArtifactReader.test.ts index 4b40dd5dee..f4d76ce9c9 100644 --- a/src/artifacts/GodotPckArtifactReader.test.ts +++ b/src/artifacts/GodotPckArtifactReader.test.ts @@ -21,6 +21,7 @@ import { openRegularFile } from "../filesystem/RegularFile.js"; import type { ArtifactEntry } from "./ArtifactReader.js"; import { ArtifactEmbeddedFormatProbes } from "./ArtifactEmbeddedFormatProbes.js"; import { GodotPckArtifactReader } from "./GodotPckArtifactReader.js"; +import { DEFAULT_GODOT_READER_LIMITS } from "../domain/artifactReaderLimits.js"; const writePck = async (bytes: Buffer): Promise => { const path = join(await createTestTempDirectory("rea-godot-pck-"), "game"); @@ -54,6 +55,79 @@ const members: readonly GodotPckMember[] = [ { path: "data/é.json", data: '{"session":"Session1"}' }, ]; +it("recovers the same PCK directory after raising encoded metadata admission", async () => { + const bytes = buildGodotPck(4, [ + { path: "res://main.gd", data: "extends Node\n" }, + ]); + const path = await writePck(bytes); + const refused = new GodotPckArtifactReader(path, undefined, { + ...DEFAULT_GODOT_READER_LIMITS, + directory_bytes: 128, + }); + try { + await expect(list(refused)).rejects.toMatchObject({ + reason: "limit", + boundary: "memory", + readerLimit: { + parameter: "artifact_limits.godot.directory_bytes", + effective_value: 128, + }, + }); + } finally { + await refused.close(); + } + const recovered = new GodotPckArtifactReader(path, undefined, { + ...DEFAULT_GODOT_READER_LIMITS, + directory_bytes: 4096, + }); + try { + const entries = await list(recovered); + expect(entries).toMatchObject([{ path: "main.gd", declaredSize: 13 }]); + expect(await text(await recovered.open(entries[0]!))).toBe( + "extends Node\n", + ); + } finally { + await recovered.close(); + } +}); + +it("applies selected aggregate work before yielding Godot members", async () => { + const path = await writePck( + buildGodotPck(4, [{ path: "res://main.gd", data: "extends Node\n" }]), + ); + const refused = new GodotPckArtifactReader( + path, + undefined, + DEFAULT_GODOT_READER_LIMITS, + 12, + ); + try { + await expect(list(refused)).rejects.toMatchObject({ + reason: "limit", + readerLimit: { + parameter: "artifact_limits.expanded_bytes", + effective_value: 12, + required_value: 13, + }, + }); + } finally { + await refused.close(); + } + const recovered = new GodotPckArtifactReader( + path, + undefined, + DEFAULT_GODOT_READER_LIMITS, + 13, + ); + try { + expect(await list(recovered)).toMatchObject([ + { path: "main.gd", declaredSize: 13 }, + ]); + } finally { + await recovered.close(); + } +}); + describe("GodotPckArtifactReader format 1", () => { it.each([false, true])( "reads Godot 3 format 1 directory and absolute member offsets (embedded=%s)", diff --git a/src/artifacts/GodotPckArtifactReader.ts b/src/artifacts/GodotPckArtifactReader.ts index 438c7ae8ad..05d081adcd 100644 --- a/src/artifacts/GodotPckArtifactReader.ts +++ b/src/artifacts/GodotPckArtifactReader.ts @@ -1,4 +1,9 @@ import { readArtifactFileBytes } from "./ArtifactFileRange.js"; +import { + DEFAULT_GODOT_READER_LIMITS, + type GodotReaderLimits, +} from "../domain/artifactReaderLimits.js"; +import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js"; import { readGodotPckHeaderFacts } from "../domain/godotPckFormat.js"; import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; import type { FileHandle } from "node:fs/promises"; @@ -24,8 +29,6 @@ const PACK_REL_FILEBASE = 2; const PACK_SPARSE_BUNDLE = 4; const PACK_FILE_ENCRYPTED = 1; const DIRECTORY_CHUNK_BYTES = 64 * 1024; -/** Encoded retained entry metadata, including escaped paths and source facts. */ -const MAX_DIRECTORY_METADATA_BYTES = 64 * 1024 * 1024; const UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); /** One descriptor, one validated directory, and bounded positional member reads. */ @@ -37,6 +40,8 @@ export class GodotPckArtifactReader extends PositionalArtifactReader { constructor( path: string, admitted?: StableRegularFileDescriptor | GodotPckAdmission, + private readonly limits: GodotReaderLimits = DEFAULT_GODOT_READER_LIMITS, + private readonly expandedBytes?: number, ) { const admission = admitted !== undefined && "kind" in admitted ? admitted : undefined; @@ -79,99 +84,115 @@ export class GodotPckArtifactReader extends PositionalArtifactReader { handle, header.directory_start + 4, header.pack_end, - signal, + { signal, limits: this.limits }, ); const entries: PositionalArtifactEntry[] = []; let retainedBytes = 0; let declaredPlaintextBytes = 0n; const expansionBudget = BigInt( - artifactExpansionBudgetBytes(header.pack_end - header.pack_start), + this.expandedBytes ?? + artifactExpansionBudgetBytes(header.pack_end - header.pack_start), ); for (let index = 0; index < header.declared_file_count; index++) { if (index % 256 === 0) await setImmediate(); assertArtifactReadActive(signal); - const pathBytes = await cursor.uint32(); - cursor.requireBytes(pathBytes + (header.format_version === 1 ? 32 : 36)); - // Check worst-case JSON escaping before decoding or allocating the path. - // The logical path, raw source path, and metadata source path can each - // need six JSON bytes per input byte before retained facts are charged. - if (pathBytes * 18 > MAX_DIRECTORY_METADATA_BYTES - retainedBytes) - throw metadataLimit(); - const rawPath = await cursor.bytes(pathBytes); - const offset = await cursor.uint64(); - const size = await cursor.uint64(); - const md5 = (await cursor.bytes(16)).toString("hex"); - // Godot 3's format 1 has neither per-file flags nor a file-base field. - const flags = header.format_version === 1 ? 0 : await cursor.uint32(); - if ((flags & ~PACK_FILE_ENCRYPTED) !== 0) - throw new ArtifactReaderFailure( - "unavailable", - `Godot patch or unknown member flags are unsupported: 0x${flags.toString(16)}`, - ); - const sourcePath = pckMemberPath(rawPath); - if (header.format_version === 1 && offset === 0n) - throw new ArtifactReaderFailure( - "unavailable", - `Godot PCK patch removal entry is unsupported: ${sourcePath}`, - ); - const start = toSafeOffset( - offset, - header.file_base, - header.pack_end, - sourcePath, + const metadata = await this.#readEntry( + cursor, + header, + index, + retainedBytes, ); - if (size > BigInt(header.pack_end - start)) - throw formatFailure(`member exceeds the pack: ${sourcePath}`); // Aliases are valid, but each path causes another hash/copy. Admit the // complete directory before yielding anything, including all-zero claims. - if ((flags & PACK_FILE_ENCRYPTED) === 0) { - declaredPlaintextBytes += size; + if (!metadata.encrypted) { + declaredPlaintextBytes += BigInt(metadata.declaredSize); if (declaredPlaintextBytes > expansionBudget) - throw new ArtifactReaderFailure( - "limit", - `Godot PCK readable entries declare ${String(declaredPlaintextBytes)} plaintext bytes, exceeding this pack's ${String(expansionBudget)}-byte expansion budget`, + throw artifactReaderLimitFailure( + "artifact_limits.expanded_bytes", + Number(expansionBudget), + "bytes", + declaredPlaintextBytes, + "memory", ); } - const metadata = { - path: sourcePath.startsWith("res://") - ? sourcePath.slice(6) - : sourcePath, - sourcePath, - kind: "file" as const, - declaredSize: Number(size), - compressedSize: null, - executable: false, - encrypted: (flags & PACK_FILE_ENCRYPTED) !== 0, - byteOffset: start, - declaredIntegrity: /^0{32}$/u.test(md5) - ? null - : { algorithm: "md5" as const, digest: md5 }, - unpacked: false, - limitations: /^0{32}$/u.test(md5) - ? ["Godot PCK declares an all-zero MD5; source integrity is unknown."] - : [], - adapterKey: `godot-pck-entry:${String(index)}`, - formatMetadata: { - kind: "godot-pck-entry" as const, - index, - source_path: sourcePath, - declared_offset: Number(offset), - flags, - md5_status: /^0{32}$/u.test(md5) - ? ("all-zero" as const) - : ("declared" as const), - }, - } satisfies PositionalArtifactEntry; retainedBytes += jsonValueByteLength( metadata, - MAX_DIRECTORY_METADATA_BYTES - retainedBytes, + this.limits.directory_bytes - retainedBytes, ); - if (retainedBytes > MAX_DIRECTORY_METADATA_BYTES) throw metadataLimit(); + if (retainedBytes > this.limits.directory_bytes) + throw metadataLimit(this.limits, retainedBytes); entries.push(metadata); } await this.verifyCarrier(signal); return { entries, coverage: undefined }; } + async #readEntry( + cursor: PckCursor, + header: GodotPckMetadata, + index: number, + retainedBytes: number, + ) { + const pathBytes = await cursor.uint32(); + cursor.requireBytes(pathBytes + (header.format_version === 1 ? 32 : 36)); + // Check worst-case JSON escaping before decoding or allocating the path. + // The logical path, raw source path, and metadata source path can each + // need six JSON bytes per input byte before retained facts are charged. + if (pathBytes * 18 > this.limits.directory_bytes - retainedBytes) + throw metadataLimit(this.limits, retainedBytes + pathBytes * 18); + const rawPath = await cursor.bytes(pathBytes); + const offset = await cursor.uint64(); + const size = await cursor.uint64(); + const md5 = (await cursor.bytes(16)).toString("hex"); + // Godot 3's format 1 has neither per-file flags nor a file-base field. + const flags = header.format_version === 1 ? 0 : await cursor.uint32(); + if ((flags & ~PACK_FILE_ENCRYPTED) !== 0) + throw new ArtifactReaderFailure( + "unavailable", + `Godot patch or unknown member flags are unsupported: 0x${flags.toString(16)}`, + ); + const sourcePath = pckMemberPath(rawPath); + if (header.format_version === 1 && offset === 0n) + throw new ArtifactReaderFailure( + "unavailable", + `Godot PCK patch removal entry is unsupported: ${sourcePath}`, + ); + const start = toSafeOffset( + offset, + header.file_base, + header.pack_end, + sourcePath, + ); + if (size > BigInt(header.pack_end - start)) + throw formatFailure(`member exceeds the pack: ${sourcePath}`); + return { + path: sourcePath.startsWith("res://") ? sourcePath.slice(6) : sourcePath, + sourcePath, + kind: "file" as const, + declaredSize: Number(size), + compressedSize: null, + executable: false, + encrypted: (flags & PACK_FILE_ENCRYPTED) !== 0, + byteOffset: start, + declaredIntegrity: /^0{32}$/u.test(md5) + ? null + : { algorithm: "md5" as const, digest: md5 }, + unpacked: false, + limitations: /^0{32}$/u.test(md5) + ? ["Godot PCK declares an all-zero MD5; source integrity is unknown."] + : [], + adapterKey: `godot-pck-entry:${String(index)}`, + formatMetadata: { + kind: "godot-pck-entry" as const, + index, + source_path: sourcePath, + declared_offset: Number(offset), + flags, + md5_status: /^0{32}$/u.test(md5) + ? ("all-zero" as const) + : ("declared" as const), + }, + } satisfies PositionalArtifactEntry; + } } /** Decode header fields once; unsupported content does not erase observed headers. */ @@ -273,7 +294,10 @@ class PckCursor { private readonly handle: FileHandle, private position: number, private readonly end: number, - private readonly signal?: AbortSignal, + private readonly options: { + readonly signal: AbortSignal | undefined; + readonly limits: GodotReaderLimits; + }, ) {} requireBytes(length: number): void { if (length > this.end - this.position) @@ -281,7 +305,8 @@ class PckCursor { } async bytes(length: number): Promise { this.requireBytes(length); - if (length > MAX_DIRECTORY_METADATA_BYTES / 18) throw metadataLimit(); + if (length > this.options.limits.directory_bytes / 18) + throw metadataLimit(this.options.limits, length * 18); if (length > DIRECTORY_CHUNK_BYTES) { // A large valid virtual path is bounded by retained representation, not // the host filesystem's PATH_MAX. Only this path receives a larger read. @@ -290,7 +315,7 @@ class PckCursor { this.position, length, this.end, - this.signal, + this.options.signal, ); this.position += length; return value; @@ -302,7 +327,7 @@ class PckCursor { this.position, Math.min(DIRECTORY_CHUNK_BYTES, this.end - this.position), this.end, - this.signal, + this.options.signal, ); this.#bufferStart = this.position; } @@ -330,9 +355,14 @@ const toSafeOffset = ( }; const formatFailure = (detail: string): ArtifactReaderFailure => new ArtifactReaderFailure("format", `Godot PCK is malformed: ${detail}`); -const metadataLimit = (): ArtifactReaderFailure => - new ArtifactReaderFailure( - "limit", - "Godot PCK directory exceeds the 64 MiB encoded metadata budget", - { boundary: "memory" }, +const metadataLimit = ( + limits: GodotReaderLimits, + required: number, +): ArtifactReaderFailure => + artifactReaderLimitFailure( + "artifact_limits.godot.directory_bytes", + limits.directory_bytes, + "bytes", + required, + "memory", ); diff --git a/src/artifacts/NodeSeaArtifactReader.test.ts b/src/artifacts/NodeSeaArtifactReader.test.ts index 9a815d8d91..b3f525db68 100644 --- a/src/artifacts/NodeSeaArtifactReader.test.ts +++ b/src/artifacts/NodeSeaArtifactReader.test.ts @@ -3,7 +3,13 @@ import { once } from "node:events"; import { writeFileSync } from "node:fs"; import { buffer } from "node:stream/consumers"; import { createReader } from "./inventory/reader.js"; -import { open, readFile, writeFile, type FileHandle } from "node:fs/promises"; +import { + open, + readFile, + stat, + writeFile, + type FileHandle, +} from "node:fs/promises"; import { join } from "node:path"; import { setImmediate } from "node:timers"; @@ -12,6 +18,7 @@ import { expect, it, vi } from "vitest"; import * as artifactPaths from "./ArtifactPaths.js"; import { NodeSeaArtifactReader } from "./NodeSeaArtifactReader.js"; +import type { ArtifactEntry } from "./ArtifactReader.js"; import { artifactFormatDescriptor } from "./artifactFormats.js"; import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; import { extractArtifact } from "./extraction/ArtifactExtraction.js"; @@ -31,6 +38,8 @@ import { MAX_INSPECT_PE_FILE_BYTES } from "../domain/native/peResources.js"; import { openRegularFile } from "../filesystem/RegularFile.js"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js"; +import { classifyAndHashRootForInventory } from "./inventory/classify.js"; +import { DEFAULT_NODE_SEA_READER_LIMITS } from "../domain/artifactReaderLimits.js"; const writeCarrier = async (name: string, bytes: Buffer): Promise => { const directory = await createTestTempDirectory("rea-node-sea-"); @@ -39,6 +48,14 @@ const writeCarrier = async (name: string, bytes: Buffer): Promise => { return path; }; +const seaEntries = async ( + reader: NodeSeaArtifactReader, +): Promise => { + const entries: ArtifactEntry[] = []; + for await (const entry of reader.entries()) entries.push(entry); + return entries; +}; + const script = 'console.log("rea-node-sea")\n'; const populated = { codePath: "app/main.js", @@ -50,6 +67,124 @@ const populated = { fuse: "set" as const, }; +it("rechecks a lower SEA policy instead of reusing an earlier admitted decode", async () => { + const path = await writeCarrier( + "sea-policy", + nodeSeaElf({ execArgv: ["one", "two"], fuse: "set" }), + ); + const scope = new ArtifactResourceScope(); + try { + await scope.run(async () => { + const classified = await classifyAndHashRootForInventory( + path, + false, + await stat(path), + { resourceScope: scope }, + ); + const source = classified.rootSource; + if (source !== undefined) scope.addCloseHook(() => source.owner.close()); + const admitted = classified.readerAdmission; + if ( + admitted === undefined || + !("kind" in admitted) || + admitted.kind !== "node-sea" + ) + throw new Error("Expected a production Node SEA admission"); + const refused = new NodeSeaArtifactReader(path, admitted, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + argument_count: 1, + }); + try { + await expect(seaEntries(refused)).rejects.toMatchObject({ + reason: "limit", + readerLimit: { + parameter: "artifact_limits.node_sea.argument_count", + effective_value: 1, + required_value: 2, + }, + }); + } finally { + await refused.close(); + } + const recovered = new NodeSeaArtifactReader(path, admitted, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + argument_count: 2, + }); + try { + await seaEntries(recovered); + expect(recovered.formatMetadata()?.exec_argv).toEqual(["one", "two"]); + } finally { + await recovered.close(); + } + }); + } finally { + await scope.close(); + } +}); + +it("locates SEA after a long unrelated ELF note owner without admitting its text", async () => { + const notes = Buffer.concat([ + elfNote("vendor".repeat(700), 1, Buffer.from("unrelated")), + elfNote( + "NODE_SEA_BLOB", + 0, + buildNodeSeaBlob({ main: script, codePath: "sea.js" }), + ), + ]); + const path = await writeCarrier( + "long-note-owner", + elfNoteFixture({ notes, ownership: "program", type: 2 }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + expect(snapshot.manifest.root_format).toBe("node-sea"); + expect(snapshot.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: "sea.js", + declared_size: Buffer.byteLength(script), + }), + ); +}); + +it("applies selected aggregate work before yielding SEA members", async () => { + const path = await writeCarrier( + "sea-expanded", + nodeSeaElf({ + main: "hello", + assets: [{ name: "data", body: "world" }], + fuse: "set", + }), + ); + const refused = new NodeSeaArtifactReader( + path, + undefined, + DEFAULT_NODE_SEA_READER_LIMITS, + 9, + ); + try { + await expect(seaEntries(refused)).rejects.toMatchObject({ + reason: "limit", + readerLimit: { + parameter: "artifact_limits.expanded_bytes", + effective_value: 9, + required_value: 10, + }, + }); + } finally { + await refused.close(); + } + const recovered = new NodeSeaArtifactReader( + path, + undefined, + DEFAULT_NODE_SEA_READER_LIMITS, + 10, + ); + try { + expect(await seaEntries(recovered)).toHaveLength(2); + } finally { + await recovered.close(); + } +}); + it.each([ ["mach-o", nodeSeaMachO(populated)], ["elf", nodeSeaElf(populated)], @@ -452,6 +587,65 @@ const repeatedNoteElf = (count: number, notes: Buffer): Buffer => { return bytes; }; +it("reports limited SEA recognition rather than absence above the ELF header policy", async () => { + const blob = buildNodeSeaBlob({ main: script, codePath: "sea.js" }); + const bytes = repeatedNoteElf(4097, elfNote("NODE_SEA_BLOB", 0, blob)); + // Only the first header claims the note; the others are valid PT_NULL rows. + for (let index = 1; index < 4097; index++) + bytes.writeUInt32LE(0, 64 + index * 56); + const path = await writeCarrier("sea-header-count", bytes); + const handle = await open(path, "r"); + try { + await expect(locateNodeSea(handle, bytes.length)).rejects.toMatchObject({ + reason: "limit", + readerLimit: { + parameter: "artifact_limits.node_sea.recognition_header_count", + effective_value: 4096, + required_value: 4097, + }, + }); + await expect( + locateNodeSea(handle, bytes.length, undefined, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + recognition_header_count: 4097, + }), + ).resolves.toMatchObject({ carrier: "elf", blobLength: blob.length }); + } finally { + await handle.close(); + } +}); + +it("recovers SEA recognition after raising Mach-O load-command admission", async () => { + const bytes = nodeSeaMachO({ main: script, codePath: "sea.js" }); + const required = bytes.readUInt32LE(20); + const handle = await open( + await writeCarrier("sea-load-commands", bytes), + "r", + ); + try { + await expect( + locateNodeSea(handle, bytes.length, undefined, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + recognition_load_command_bytes: required - 1, + }), + ).rejects.toMatchObject({ + reason: "limit", + readerLimit: { + parameter: "artifact_limits.node_sea.recognition_load_command_bytes", + required_value: required, + }, + }); + await expect( + locateNodeSea(handle, bytes.length, undefined, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + recognition_load_command_bytes: required, + }), + ).resolves.toMatchObject({ carrier: "mach-o" }); + } finally { + await handle.close(); + } +}); + it("reads repeated PT_NOTE declarations at most once per note the file holds", async () => { // 4 KiB of zero bytes is 341 empty 12-byte notes. const empty = Buffer.alloc(4096); diff --git a/src/artifacts/NodeSeaArtifactReader.ts b/src/artifacts/NodeSeaArtifactReader.ts index bc86c50a1a..ef2f0b07ba 100644 --- a/src/artifacts/NodeSeaArtifactReader.ts +++ b/src/artifacts/NodeSeaArtifactReader.ts @@ -1,4 +1,8 @@ import type { NodeSeaAdmission } from "./nodeSea/NodeSeaAdmission.js"; +import { + DEFAULT_NODE_SEA_READER_LIMITS, + type NodeSeaReaderLimits, +} from "../domain/artifactReaderLimits.js"; import { setImmediate } from "node:timers/promises"; import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; @@ -8,9 +12,8 @@ import { normalizeArtifactPath, } from "./ArtifactPaths.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js"; import { - NODE_SEA_BLOB_TOO_LARGE, - NODE_SEA_MAX_BLOB_BYTES, decodeNodeSeaBlob, type DecodedNodeSeaBlob, type NodeSeaByteRange, @@ -19,7 +22,6 @@ import { locateNodeSea, readNodeSeaBlob, scanNodeSeaFuse, - type NodeSeaLocation, } from "./nodeSea/NodeSeaLocation.js"; import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; import { @@ -44,6 +46,8 @@ export class NodeSeaArtifactReader extends PositionalArtifactReader { constructor( path: string, admitted?: StableRegularFileDescriptor | NodeSeaAdmission, + private readonly limits: NodeSeaReaderLimits = DEFAULT_NODE_SEA_READER_LIMITS, + private readonly expandedBytes?: number, ) { const admission = admitted !== undefined && "kind" in admitted ? admitted : undefined; @@ -66,34 +70,48 @@ export class NodeSeaArtifactReader extends PositionalArtifactReader { this.#admission = undefined; } + /** Reuse only the decode admitted under this accepted policy. */ + async #readBlob( + source: StableRegularFileDescriptor, + signal?: AbortSignal, + ): Promise> { + const admittedLimits = this.#admission?.limits; + if ( + this.#admission !== undefined && + admittedLimits !== undefined && + Object.entries(this.limits).every( + ([field, value]) => Reflect.get(admittedLimits, field) === value, + ) + ) + return this.#admission; + const location = await locateNodeSea( + source.handle, + source.initial.size, + signal, + this.limits, + ); + if (location === undefined) throw formatFailure("blob was not found"); + const decoded = await decodeNodeSeaBlob( + await readNodeSeaBlob(source, location, signal, this.limits), + signal, + this.limits, + ); + return { location, decoded }; + } + protected override async prepareEntries( signal?: AbortSignal, ): Promise { const { handle, initial } = await this.verifyCarrier(signal); - // Freshness remains owned by ArtifactFileSource even when classification - // already decoded this exact borrowed descriptor's blob. - let location: NodeSeaLocation; - let decoded: DecodedNodeSeaBlob; - if (this.#admission !== undefined) { - ({ location, decoded } = this.#admission); - } else { - const found = await locateNodeSea(handle, initial.size, signal); - if (found === undefined) throw formatFailure("blob was not found"); - location = found; - if (location.blobLength > NODE_SEA_MAX_BLOB_BYTES) - throw new ArtifactReaderFailure("limit", NODE_SEA_BLOB_TOO_LARGE, { - boundary: "memory", - }); - decoded = await decodeNodeSeaBlob( - await readNodeSeaBlob(handle, location, initial.size, signal), - signal, - ); - } + const { location, decoded } = await this.#readBlob( + { handle, initial }, + signal, + ); const fuse = await scanNodeSeaFuse(handle, initial.size, signal); const entries: PositionalArtifactEntry[] = []; let declaredBytes = 0n; const expansionBudget = BigInt( - artifactExpansionBudgetBytes(location.blobLength), + this.expandedBytes ?? artifactExpansionBudgetBytes(location.blobLength), ); let index = 0; for (const member of seaMembers(decoded)) { @@ -101,10 +119,12 @@ export class NodeSeaArtifactReader extends PositionalArtifactReader { this.assertOpen(signal); declaredBytes += BigInt(member.range.length); if (declaredBytes > expansionBudget) - throw new ArtifactReaderFailure( - "limit", - `Node SEA readable entries declare ${String(declaredBytes)} bytes, exceeding this blob's ${String(expansionBudget)}-byte expansion budget`, - { boundary: "memory" }, + throw artifactReaderLimitFailure( + "artifact_limits.expanded_bytes", + Number(expansionBudget), + "bytes", + declaredBytes, + "memory", ); const start = location.blobOffset + member.range.offset; if (start > initial.size || member.range.length > initial.size - start) diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index e8d0a61137..791da08573 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -39,11 +39,8 @@ import { type ArtifactReader, } from "./ArtifactReader.js"; import type { ZipPackageFormat } from "../domain/zipPackageFormat.js"; -import { - ARTIFACT_EXPANSION_RATIO, - artifactExpansionBudgetBytes, - MINIMUM_ARTIFACT_EXPANDED_BYTES, -} from "../domain/artifactExpansionBudget.js"; +import { artifactExpansionBudgetBytes } from "../domain/artifactExpansionBudget.js"; +import { artifactReaderLimitFailure } from "./ArtifactReaderLimits.js"; import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; import { ArtifactFileSource } from "./ArtifactFileSource.js"; import { readFileHandleRange } from "../filesystem/readFileHandleRange.js"; @@ -189,6 +186,7 @@ export class ZipArtifactReader implements ArtifactReader { format: ZipPackageFormat, maximumMetadataReadBytes?: number, admitted?: StableRegularFileDescriptor, + private readonly expandedBytes?: number, ) { this.format = format; this.#source = new NodeFileReader(path, maximumMetadataReadBytes, admitted); @@ -199,7 +197,7 @@ export class ZipArtifactReader implements ArtifactReader { this.#assertOpen(signal); let index = 0; // A traversal owns its accounting, including overlapping or repeated passes. - let declaredFileBytes = 0; + let declaredFileBytes = 0n; const enumeration = typedZipFailures( this.#reader.getEntriesGenerator({ decodeText: decodeZipEntryText }), ); @@ -220,12 +218,16 @@ export class ZipArtifactReader implements ArtifactReader { Number.isSafeInteger(metadata.declaredSize) && metadata.declaredSize >= 0 ) { - declaredFileBytes += metadata.declaredSize; - const budget = artifactExpansionBudgetBytes(this.#source.size); - if (declaredFileBytes > budget) - throw new ArtifactReaderFailure( - "limit", - `ZIP entries declare ${String(declaredFileBytes)} uncompressed bytes, exceeding this archive's ${String(budget)}-byte decompression budget (${String(MINIMUM_ARTIFACT_EXPANDED_BYTES)}-byte minimum, ${String(ARTIFACT_EXPANSION_RATIO)}x archive size); a small archive cannot expand this far`, + declaredFileBytes += BigInt(metadata.declaredSize); + const budget = + this.expandedBytes ?? + artifactExpansionBudgetBytes(this.#source.size); + if (declaredFileBytes > BigInt(budget)) + throw artifactReaderLimitFailure( + "artifact_limits.expanded_bytes", + budget, + "bytes", + declaredFileBytes, ); } const stored = this.#entries.get(metadata.adapterKey); diff --git a/src/artifacts/artifactFormats.ts b/src/artifacts/artifactFormats.ts index 0399d590c3..29a9222994 100644 --- a/src/artifacts/artifactFormats.ts +++ b/src/artifacts/artifactFormats.ts @@ -49,6 +49,9 @@ import { ZipArtifactReader } from "./ZipArtifactReader.js"; export type AbsentInventory = "platform" | "universal" | "unavailable" | "root"; export interface ArtifactReaderConstruction { + readonly readerLimits?: + | import("../domain/artifactReaderLimits.js").ArtifactReaderLimits + | undefined; readonly environment: Readonly; readonly unreadableDirectories: "fail" | "omit"; readonly admission?: ArtifactReaderAdmission | undefined; @@ -162,7 +165,13 @@ const zipDescriptor = (id: ZipPackageFormat) => id, ZIP_FAMILY_CAPABILITIES, (path, options) => - new ZipArtifactReader(path, id, undefined, sourceOf(options)), + new ZipArtifactReader( + path, + id, + undefined, + sourceOf(options), + options.readerLimits?.expanded_bytes, + ), ); function described( @@ -226,6 +235,7 @@ const NON_ZIP_FORMATS = { new DotnetBundleArtifactReader( path, admittedAs("dotnet-bundle", options), + options.readerLimits?.dotnet_bundle, ), }), "bun-standalone": container("bun-standalone", { @@ -235,6 +245,8 @@ const NON_ZIP_FORMATS = { new BunStandaloneArtifactReader( path, admittedAs("bun-standalone", options), + options.readerLimits?.bun, + options.readerLimits?.expanded_bytes, ), }), "deno-standalone": container("deno-standalone", { @@ -292,7 +304,12 @@ const NON_ZIP_FORMATS = { readerLimitation: "PCK inventory establishes packaged bytes and metadata; scenes, binary resources, GDScript bytecode and runtime signal behavior were not decoded.", createReader: (path, options) => - new GodotPckArtifactReader(path, admittedAs("godot-pck", options)), + new GodotPckArtifactReader( + path, + admittedAs("godot-pck", options), + options.readerLimits?.godot, + options.readerLimits?.expanded_bytes, + ), }), unityfs: container("unityfs", { roleContainer: true, @@ -368,7 +385,12 @@ const NON_ZIP_FORMATS = { readerLimitation: "Node SEA inventory establishes the injected blob, flags, fuse sentinel, main script, optional code cache, snapshot, and assets. V8 snapshots and code cache are not deserialized, and the application is not executed.", createReader: (path, options) => - new NodeSeaArtifactReader(path, admittedAs("node-sea", options)), + new NodeSeaArtifactReader( + path, + admittedAs("node-sea", options), + options.readerLimits?.node_sea, + options.readerLimits?.expanded_bytes, + ), }), "intel-hex": container("intel-hex", { roleContainer: true, diff --git a/src/artifacts/bun/BunModuleGraph.test.ts b/src/artifacts/bun/BunModuleGraph.test.ts index 91c4fd2cd6..03fb1b9fde 100644 --- a/src/artifacts/bun/BunModuleGraph.test.ts +++ b/src/artifacts/bun/BunModuleGraph.test.ts @@ -1,10 +1,13 @@ +import { + DEFAULT_BUN_READER_LIMITS, + type BunReaderLimits, +} from "../../domain/artifactReaderLimits.js"; import { open, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { probeBunStandaloneGraph, - bunModuleCountLimit, BUN_GRAPH_SUFFIX_BYTES, } from "./BunModuleGraphLayout.js"; import { decodeBunModuleGraph } from "./BunModuleGraphMembers.js"; @@ -52,7 +55,7 @@ async function locate(bytes: Buffer) { } /** Probe while counting bytes read through the handle. */ -async function probe(bytes: Buffer) { +async function probe(bytes: Buffer, limits?: BunReaderLimits) { const path = join(await createTestTempDirectory("rea-bun-parser-"), "app"); await writeFile(path, bytes); const handle = await open(path, "r"); @@ -70,7 +73,12 @@ async function probe(bytes: Buffer) { }); try { return { - result: await probeBunStandaloneGraph(counted, bytes.length), + result: await probeBunStandaloneGraph( + counted, + bytes.length, + undefined, + limits, + ), bytesRead, }; } finally { @@ -308,7 +316,7 @@ it.each([ }, ); -it("matches ELF section names in bounded time when headers share one long name", async () => { +it("reads a shared long ELF section name once without materializing its string table", async () => { const shnum = 16_383; const stringOffset = 64; const strings = Buffer.alloc(1024 * 1024, 0x61); @@ -325,9 +333,9 @@ it("matches ELF section names in bounded time when headers share one long name", bytes.writeBigUInt64LE(BigInt(stringOffset), header + 24); bytes.writeBigUInt64LE(BigInt(strings.length), header + 32); } - const started = performance.now(); - expect((await probe(bytes)).result).toEqual({ status: "absent" }); - expect(performance.now() - started).toBeLessThan(2_000); + const observed = await probe(bytes); + expect(observed.result).toEqual({ status: "absent" }); + expect(observed.bytesRead).toBeLessThan(shnum * 64 + 1024); }); it("refuses an excessive table count before reading or allocating the module table", async () => { @@ -340,7 +348,6 @@ it("refuses an excessive table count before reading or allocating the module tab if (result.status !== "present" || result.graph === undefined) throw new Error("Missing graph"); const read = vi.fn(result.graph.data.read); - const expected = bunModuleCountLimit(count); await expect( decodeBunModuleGraph({ ...result.graph, @@ -349,7 +356,80 @@ it("refuses an excessive table count before reading or allocating the module tab ).rejects.toMatchObject({ reason: "limit", boundary: "capacity", - message: expected.message, + readerLimit: { + parameter: "artifact_limits.bun.module_count", + effective_value: 100_000, + required_value: 100_001, + unit: "count", + }, }); expect(read).not.toHaveBeenCalled(); }); + +it.each([ + { + carrier: "ELF", + encode: elfBunSection, + field: "recognition_table_bytes", + needed: 192, + }, + { + carrier: "Mach-O", + encode: machoBunSection, + field: "recognition_load_command_bytes", + needed: 152, + }, +] as const)( + "keeps $carrier recognition unknown at its selected boundary and recovers the same graph", + async ({ encode, field, needed }) => { + const bytes = encode( + bunPrefixedPayload( + bunModuleGraphPayload({ + modules: [{ name: "main.js", contents: Buffer.from("retained") }], + }), + ), + ); + const limited = await probe(bytes, { + ...DEFAULT_BUN_READER_LIMITS, + [field]: needed - 1, + }); + expect(limited.result).toMatchObject({ + status: "limited", + failure: { + reason: "limit", + readerLimit: { + parameter: `artifact_limits.bun.${field}`, + effective_value: needed - 1, + required_value: needed, + }, + }, + }); + const recovered = await probe(bytes, { + ...DEFAULT_BUN_READER_LIMITS, + [field]: needed, + }); + expect(recovered.result).toMatchObject({ + status: "present", + claim: { + carrier: encode === elfBunSection ? "elf-section" : "mach-o-section", + }, + }); + }, +); + +it("recognizes an ELF Bun name beyond the former string-table prefix without reading the prefix", async () => { + const bytes = elfBunSection( + bunPrefixedPayload( + bunModuleGraphPayload({ + modules: [{ name: "main.js", contents: Buffer.from("retained") }], + }), + ), + 1024 * 1024 + 8, + ); + const observed = await probe(bytes); + expect(observed.result).toMatchObject({ + status: "present", + claim: { carrier: "elf-section" }, + }); + expect(observed.bytesRead).toBeLessThan(1024); +}); diff --git a/src/artifacts/bun/BunModuleGraphLayout.ts b/src/artifacts/bun/BunModuleGraphLayout.ts index 2b9270205f..dec848b418 100644 --- a/src/artifacts/bun/BunModuleGraphLayout.ts +++ b/src/artifacts/bun/BunModuleGraphLayout.ts @@ -8,12 +8,13 @@ import { } from "../../domain/apple/machoLoadCommands.js"; import type { FileHandle } from "node:fs/promises"; -import { fixedAllowanceRemediation } from "../../domain/fixedAllowanceRemediation.js"; -import { abortIfNeeded } from "../ArtifactHash.js"; import { - ArtifactReaderFailure, - artifactFixedLimitFailure, -} from "../ArtifactReader.js"; + DEFAULT_BUN_READER_LIMITS, + type BunReaderLimits, +} from "../../domain/artifactReaderLimits.js"; +import { artifactReaderLimitFailure } from "../ArtifactReaderLimits.js"; +import { abortIfNeeded } from "../ArtifactHash.js"; +import { ArtifactReaderFailure } from "../ArtifactReader.js"; /** Exact trailer written by Bun's standalone module graph serializer. */ export const BUN_TRAILER = Buffer.from("\n---- Bun! ----\n"); @@ -27,20 +28,6 @@ export const BUN_RECORD_BYTES_1_3_3 = 36; /** Bun 1.3.9 through at least 1.4.4 module records. */ export const BUN_RECORD_BYTES_1_3_9 = 52; -/** - * Module-table admission. This release has no runtime override; host memory - * and NODE_OPTIONS do not raise it. - */ -export const MAX_BUN_MODULE_COUNT = 100_000; -/** - * Serialized graph payload admission, in bytes. A claimed length above this - * ceiling is refused before the payload is read. This release has no runtime - * override; host memory and NODE_OPTIONS do not raise it. - */ -export const MAX_BUN_RETAINED_BYTES = 256 * 1024 * 1024; - -const MAX_LOAD_COMMAND_BYTES = 16 * 1024 * 1024; -const MAX_SECTION_TABLE_BYTES = 1024 * 1024; const MAX_PE_SECTIONS = 96; const LC_SEGMENT = 0x1; const LC_SEGMENT_64 = 0x19; @@ -91,6 +78,7 @@ export interface BunGraphClaim { export type BunStandaloneProbe = | { readonly status: "absent" } + | { readonly status: "limited"; readonly failure: ArtifactReaderFailure } | { readonly status: "present"; readonly claim: BunGraphClaim; @@ -104,6 +92,7 @@ export type BunStandaloneProbe = export type BunStandaloneRecognition = | { readonly status: "absent" } + | { readonly status: "limited"; readonly failure: ArtifactReaderFailure } | { readonly status: "present"; readonly graph: LocatedBunGraph; @@ -115,6 +104,13 @@ export type BunStandaloneRecognition = readonly graph?: never; }; +interface BunProbeSource { + readonly handle: FileHandle; + readonly fileSize: number; + readonly signal: AbortSignal | undefined; + readonly limits: BunReaderLimits; +} + interface SectionWindow { readonly carrier: Exclude; readonly fileOffset: number; @@ -133,17 +129,26 @@ export const probeBunStandaloneGraph = async ( handle: FileHandle, fileSize: number, signal?: AbortSignal, + limits: BunReaderLimits = DEFAULT_BUN_READER_LIMITS, ): Promise => { abortIfNeeded(signal); - const claims = [ - ...(await machOClaims(handle, fileSize, 0, fileSize, signal)), - ...(await peClaims(handle, fileSize, signal)), - ...(await elfClaims(handle, fileSize, signal)), - ]; + const source = { handle, fileSize, signal, limits }; + let claims: readonly SectionWindow[]; + try { + claims = [ + ...(await machOClaims(source, 0, fileSize)), + ...(await peClaims(source)), + ...(await elfClaims(source)), + ]; + } catch (cause) { + if (!(cause instanceof ArtifactReaderFailure) || cause.reason !== "limit") + throw cause; + return { status: "limited", failure: cause }; + } const real: BunStandaloneProbe[] = []; for (const claim of claims) { abortIfNeeded(signal); - const evaluated = await evaluateSection(handle, fileSize, claim, signal); + const evaluated = await evaluateSection(source, claim); if (evaluated.status === "present") { if (real.length > 0) return presentFailure( @@ -157,7 +162,7 @@ export const probeBunStandaloneGraph = async ( } const only = real[0]; if (only !== undefined) return only; - return evaluateAppendedTrailer(handle, fileSize, signal); + return evaluateAppendedTrailer(source); }; /** Read a probed payload once and validate its suffix against the claim. */ @@ -165,14 +170,20 @@ export const loadBunStandaloneGraph = async ( handle: FileHandle, claim: BunGraphClaim, signal?: AbortSignal, + limits: BunReaderLimits = DEFAULT_BUN_READER_LIMITS, ): Promise => { + if (claim.payloadLength > limits.payload_bytes) + return { + status: "present", + failure: bunPayloadBytesLimit(claim.payloadLength, limits), + }; const suffix = await readExact( handle, claim.payloadOffset + claim.payloadLength - BUN_GRAPH_SUFFIX_BYTES, BUN_GRAPH_SUFFIX_BYTES, signal, ); - const failure = suffixFailure(suffix, claim.payloadLength); + const failure = suffixFailure(suffix, claim.payloadLength, limits); if (failure !== undefined) return { status: "present", failure }; const offsets = suffix.subarray(0, BUN_OFFSETS_BYTES); return { @@ -184,6 +195,7 @@ export const loadBunStandaloneGraph = async ( claim.payloadOffset, claim.payloadLength - BUN_GRAPH_SUFFIX_BYTES, signal, + limits.metadata_read_bytes, ), modules: readPointer(offsets, 8), entryPointId: offsets.readUInt32LE(16), @@ -193,7 +205,6 @@ export const loadBunStandaloneGraph = async ( }; }; -const MAX_METADATA_READ_BYTES = 256 * 1024 * 1024; const METADATA_CACHE_BYTES = 256 * 1024; const METADATA_CACHE_ENTRIES = 1024; @@ -206,7 +217,8 @@ class BunGraphRangeReader implements BunGraphData { private readonly handle: FileHandle, private readonly start: number, readonly length: number, - private readonly signal?: AbortSignal, + private readonly signal: AbortSignal | undefined, + private readonly maximumReadBytes: number, ) {} async read(offset: number, length: number): Promise { @@ -228,11 +240,12 @@ class BunGraphRangeReader implements BunGraphData { this.#ranges.set(key, existing); return Buffer.from(existing); } - if (length > MAX_METADATA_READ_BYTES - this.#readBytes) - throw artifactFixedLimitFailure( - "Bun metadata read work", - MAX_METADATA_READ_BYTES, + if (length > this.maximumReadBytes - this.#readBytes) + throw artifactReaderLimitFailure( + "artifact_limits.bun.metadata_read_bytes", + this.maximumReadBytes, "bytes", + this.#readBytes + length, ); // Charge physical work before allocation or I/O, including uncached large names. this.#readBytes += length; @@ -257,29 +270,37 @@ class BunGraphRangeReader implements BunGraphData { } } -const bunByteLimit = ( - claimed: number | bigint, - scope: "payload" | "metadata", -): ArtifactReaderFailure => { - const allowance = `Bun standalone ${scope} allowance of ${String(MAX_BUN_RETAINED_BYTES)} bytes`; - const message = `The ${scope} claims ${String(claimed)} bytes. ${fixedAllowanceRemediation(allowance, "parser")} This release has no runtime override.`; - return new ArtifactReaderFailure("limit", message, { - boundary: scope === "payload" ? "file-size" : "memory", - userMessage: message, - }); -}; export const bunPayloadBytesLimit = ( claimed: number | bigint, -): ArtifactReaderFailure => bunByteLimit(claimed, "payload"); + limits: BunReaderLimits = DEFAULT_BUN_READER_LIMITS, +): ArtifactReaderFailure => + artifactReaderLimitFailure( + "artifact_limits.bun.payload_bytes", + limits.payload_bytes, + "bytes", + claimed, + "file-size", + ); export const bunMetadataBytesLimit = ( - claimed: number | bigint, -): ArtifactReaderFailure => bunByteLimit(claimed, "metadata"); - -export const bunModuleCountLimit = (count: number): ArtifactReaderFailure => - new ArtifactReaderFailure( - "limit", - `Bun standalone module graph claims ${String(count)} modules, exceeding MAX_BUN_MODULE_COUNT (${String(MAX_BUN_MODULE_COUNT)} modules). This release has no runtime override. Host memory and NODE_OPTIONS do not raise it.`, - { boundary: "capacity" }, + claimed: number, + limits: BunReaderLimits = DEFAULT_BUN_READER_LIMITS, +): ArtifactReaderFailure => + artifactReaderLimitFailure( + "artifact_limits.bun.representation_bytes", + limits.representation_bytes, + "bytes", + claimed, + "memory", + ); +export const bunModuleCountLimit = ( + count: number, + limits: BunReaderLimits = DEFAULT_BUN_READER_LIMITS, +): ArtifactReaderFailure => + artifactReaderLimitFailure( + "artifact_limits.bun.module_count", + limits.module_count, + "count", + count, ); export const bunFormatFailure = (detail: string): ArtifactReaderFailure => @@ -293,11 +314,10 @@ const presentFailure = ( ): BunStandaloneProbe => ({ status: "present", failure }); const evaluateSection = async ( - handle: FileHandle, - fileSize: number, + source: BunProbeSource, claim: SectionWindow, - signal?: AbortSignal, ): Promise => { + const { handle, signal, limits } = source; if (claim.truncated) return presentFailure( bunFormatFailure("section extends past the end of the file"), @@ -337,8 +357,8 @@ const evaluateSection = async ( } } if (payloadLength === 0n) return { status: "placeholder" }; - if (payloadLength > BigInt(MAX_BUN_RETAINED_BYTES)) - return presentFailure(bunPayloadBytesLimit(payloadLength)); + if (payloadLength > BigInt(limits.payload_bytes)) + return presentFailure(bunPayloadBytesLimit(payloadLength, limits)); if (payloadLength < BigInt(BUN_GRAPH_SUFFIX_BYTES)) return presentFailure(bunFormatFailure("trailer is truncated")); const length = Number(payloadLength); @@ -351,7 +371,7 @@ const evaluateSection = async ( BUN_GRAPH_SUFFIX_BYTES, signal, ); - const failure = suffixFailure(suffix, length); + const failure = suffixFailure(suffix, length, limits); if (failure !== undefined) return presentFailure(failure); return { status: "present", @@ -364,10 +384,9 @@ const evaluateSection = async ( * trailer has to be the exact final bytes; this does not scan backward. */ const evaluateAppendedTrailer = async ( - handle: FileHandle, - fileSize: number, - signal?: AbortSignal, + source: BunProbeSource, ): Promise => { + const { handle, fileSize, signal, limits } = source; if (fileSize < BUN_GRAPH_SUFFIX_BYTES) return { status: "absent" }; let graphEnd = fileSize; let trailer = await readExact( @@ -399,12 +418,9 @@ const evaluateAppendedTrailer = async ( signal, ); const byteCount = offsets.readBigUInt64LE(0); - if ( - byteCount + BigInt(BUN_GRAPH_SUFFIX_BYTES) > - BigInt(MAX_BUN_RETAINED_BYTES) - ) + if (byteCount + BigInt(BUN_GRAPH_SUFFIX_BYTES) > BigInt(limits.payload_bytes)) return presentFailure( - bunPayloadBytesLimit(byteCount + BigInt(BUN_GRAPH_SUFFIX_BYTES)), + bunPayloadBytesLimit(byteCount + BigInt(BUN_GRAPH_SUFFIX_BYTES), limits), ); const payloadLength = byteCount + BigInt(BUN_GRAPH_SUFFIX_BYTES); if (payloadLength > BigInt(graphEnd)) @@ -424,14 +440,15 @@ const evaluateAppendedTrailer = async ( const suffixFailure = ( suffix: Buffer, payloadLength: number, + limits: BunReaderLimits, ): ArtifactReaderFailure | undefined => { if (suffix.length < BUN_GRAPH_SUFFIX_BYTES) return bunFormatFailure("trailer is truncated"); if (!suffix.subarray(BUN_OFFSETS_BYTES).equals(BUN_TRAILER)) return bunFormatFailure("trailer does not match the Bun sentinel"); const byteCount = suffix.readBigUInt64LE(0); - if (byteCount > BigInt(MAX_BUN_RETAINED_BYTES)) - return bunPayloadBytesLimit(byteCount); + if (byteCount > BigInt(limits.payload_bytes)) + return bunPayloadBytesLimit(byteCount, limits); if (byteCount + BigInt(BUN_GRAPH_SUFFIX_BYTES) !== BigInt(payloadLength)) return bunFormatFailure("byte_count does not match the located payload"); return undefined; @@ -446,18 +463,17 @@ export const readPointer = ( }); const machOClaims = async ( - handle: FileHandle, - fileSize: number, + source: BunProbeSource, imageOffset: number, imageLimit: number, - signal?: AbortSignal, ): Promise => { + const { handle, fileSize, signal, limits } = source; if (imageOffset < 0 || imageOffset >= fileSize) return []; const header = await tryRead(handle, imageOffset, 12, fileSize, signal); if (header === null || header.length < 4) return []; const facts = readMachoHeaderPrefixFacts(header); if (facts.identity === null) - return imageOffset === 0 ? fatClaims(handle, fileSize, header, signal) : []; + return imageOffset === 0 ? fatClaims(source, header) : []; const full = await tryRead( handle, imageOffset, @@ -474,7 +490,13 @@ const machOClaims = async ( headerSize: headerBytes, commandBytes: sizeofcmds, } = table; - if (sizeofcmds > MAX_LOAD_COMMAND_BYTES) return []; + if (sizeofcmds > limits.recognition_load_command_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.bun.recognition_load_command_bytes", + limits.recognition_load_command_bytes, + "bytes", + sizeofcmds, + ); const commands = await tryRead( handle, imageOffset + headerBytes, @@ -530,7 +552,7 @@ const segmentClaims = ( claims.push({ carrier: "mach-o-section", fileOffset: imageOffset + offset, - declaredSize: MAX_BUN_RETAINED_BYTES + 1, + declaredSize: Number.MAX_SAFE_INTEGER, available: 0, truncated: false, }); @@ -544,11 +566,10 @@ const segmentClaims = ( }; const fatClaims = async ( - handle: FileHandle, - fileSize: number, + source: BunProbeSource, prefix: Buffer, - signal?: AbortSignal, ): Promise => { + const { handle, fileSize, signal } = source; let parsed = readFatContainerHeader(prefix, fileSize); if (parsed.state === "needs-data") { const table = await tryRead( @@ -566,24 +587,15 @@ const fatClaims = async ( for (const slice of parsed.slices) { abortIfNeeded(signal); if (parsed.rangeIssues[slice.index] !== null) continue; - claims.push( - ...(await machOClaims( - handle, - fileSize, - slice.offset, - slice.size, - signal, - )), - ); + claims.push(...(await machOClaims(source, slice.offset, slice.size))); } return claims; }; const peClaims = async ( - handle: FileHandle, - fileSize: number, - signal?: AbortSignal, + source: BunProbeSource, ): Promise => { + const { handle, fileSize, signal } = source; const dos = await tryRead(handle, 0, 64, fileSize, signal); if (dos === null || dos.length < 64 || dos[0] !== 0x4d || dos[1] !== 0x5a) return []; @@ -619,10 +631,9 @@ const peClaims = async ( }; const elfClaims = async ( - handle: FileHandle, - fileSize: number, - signal?: AbortSignal, + source: BunProbeSource, ): Promise => { + const { handle, fileSize, signal, limits } = source; const header = await tryRead(handle, 0, 64, fileSize, signal); if ( header === null || @@ -646,17 +657,22 @@ const elfClaims = async ( : readU32(header, shoffOffset, littleEndian); const shnum = readU16(header, shnumOffset, littleEndian); const shstrndx = readU16(header, shstrndxOffset, littleEndian); - // Extended numbering and unreadable tables stay absent so a native ELF is - // not reclassified just because its section table could not be walked. + // Unsupported extended numbering does not establish a Bun graph. if ( shoff === undefined || shoff === 0 || shnum === 0 || shnum === 0xffff || - shstrndx >= shnum || - shnum * shentsize > MAX_SECTION_TABLE_BYTES + shstrndx >= shnum ) return []; + if (shnum * shentsize > limits.recognition_table_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.bun.recognition_table_bytes", + limits.recognition_table_bytes, + "bytes", + shnum * shentsize, + ); const table = await tryRead( handle, shoff, @@ -667,20 +683,31 @@ const elfClaims = async ( if (table === null) return []; const strings = elfSectionBounds(table, shstrndx, wide, littleEndian); if (strings === null) return []; - const stringTable = await tryRead( - handle, - strings.offset, - Math.min(strings.size, MAX_SECTION_TABLE_BYTES), - fileSize, - signal, - ); - if (stringTable === null) return []; + // A section name lookup needs only `.bun` and its terminator. Read the + // referenced range, rather than treating a truncated string-table prefix as + // evidence that later names are absent. Duplicate offsets share one lookup. + const names = new Map(); const claims: SectionWindow[] = []; for (let index = 0; index < shnum; index += 1) { const bounds = elfSectionBounds(table, index, wide, littleEndian); if (bounds === null) continue; const nameOffset = readU32(table, index * shentsize, littleEndian); - if (!elfNameIsBun(stringTable, nameOffset)) continue; + let isBun = names.get(nameOffset); + if (isBun === undefined) { + const name = + nameOffset >= strings.size + ? null + : await tryRead( + handle, + strings.offset + nameOffset, + Math.min(ELF_BUN_NAME.length + 1, strings.size - nameOffset), + fileSize, + signal, + ); + isBun = name !== null && elfNameIsBun(name, 0); + names.set(nameOffset, isBun); + } + if (!isBun) continue; claims.push(windowFor("elf-section", bounds.offset, bounds.size, fileSize)); } return claims; diff --git a/src/artifacts/bun/BunModuleGraphMembers.ts b/src/artifacts/bun/BunModuleGraphMembers.ts index 88b6dfa785..cb9dab9962 100644 --- a/src/artifacts/bun/BunModuleGraphMembers.ts +++ b/src/artifacts/bun/BunModuleGraphMembers.ts @@ -1,10 +1,12 @@ +import { + DEFAULT_BUN_READER_LIMITS, + type BunReaderLimits, +} from "../../domain/artifactReaderLimits.js"; import { setImmediate } from "node:timers/promises"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; import { BUN_RECORD_BYTES_1_3_3, BUN_RECORD_BYTES_1_3_9, - MAX_BUN_MODULE_COUNT, - MAX_BUN_RETAINED_BYTES, bunMetadataBytesLimit, bunFormatFailure, bunModuleCountLimit, @@ -90,6 +92,7 @@ export interface BunDecodedGraph { */ export const decodeBunModuleGraph = async ( graph: LocatedBunGraph, + limits: BunReaderLimits = DEFAULT_BUN_READER_LIMITS, ): Promise => { const { data } = graph; const table = graph.modules; @@ -102,8 +105,8 @@ export const decodeBunModuleGraph = async ( `record table length ${String(table.length)} matches no known revision`, ); // Count admission precedes table allocation, independently of payload size. - if (candidates.every((size) => table.length / size > MAX_BUN_MODULE_COUNT)) - throw bunModuleCountLimit(table.length / Math.max(...candidates)); + if (candidates.every((size) => table.length / size > limits.module_count)) + throw bunModuleCountLimit(table.length / Math.max(...candidates), limits); const records = await data.read(table.offset, table.length); let recordBytes = candidates[0]!; if (candidates.length > 1) { @@ -111,12 +114,7 @@ export const decodeBunModuleGraph = async ( for (const recordBytes of candidates) assessments.push({ recordBytes, - failure: await assessLayout( - data, - records, - recordBytes, - graph.entryPointId, - ), + failure: await assessLayout(graph, records, recordBytes, limits), }); const valid = assessments.filter((item) => item.failure === null); if (valid.length > 1) @@ -148,7 +146,8 @@ export const decodeBunModuleGraph = async ( // Bound repeated pointer expansion before decoding strings. JSON escaping and // UTF-16 strings can expand each source byte; aliased names count per record. let retained = graph.compileExecArgv.length * 6; - if (retained > MAX_BUN_RETAINED_BYTES) throw bunMetadataBytesLimit(retained); + if (retained > limits.representation_bytes) + throw bunMetadataBytesLimit(retained, limits); for (let index = 0; index < count; index += 1) { const record = records.subarray( index * recordBytes, @@ -157,8 +156,8 @@ export const decodeBunModuleGraph = async ( retained += 512 + readPointer(record, 0).length * 6; if (recordBytes === BUN_RECORD_BYTES_1_3_9) retained += readPointer(record, 40).length * 6; - if (retained > MAX_BUN_RETAINED_BYTES) - throw bunMetadataBytesLimit(retained); + if (retained > limits.representation_bytes) + throw bunMetadataBytesLimit(retained, limits); if (index % 256 === 0) await setImmediate(); if (candidates.length === 1) { const failure = await recordPointerFailure( @@ -191,13 +190,14 @@ const recordSizes = (length: number): readonly number[] => { }; const assessLayout = async ( - data: BunGraphData, + graph: LocatedBunGraph, records: Buffer, recordBytes: number, - entryPointId: number, + limits: BunReaderLimits, ): Promise => { + const { data, entryPointId } = graph; const count = records.length / recordBytes; - if (count > MAX_BUN_MODULE_COUNT) return bunModuleCountLimit(count); + if (count > limits.module_count) return bunModuleCountLimit(count, limits); for (let index = 0; index < count; index += 1) { const record = records.subarray( index * recordBytes, diff --git a/src/artifacts/bun/BunStandalone.fixture.ts b/src/artifacts/bun/BunStandalone.fixture.ts index bc73f6b5db..553d8bb775 100644 --- a/src/artifacts/bun/BunStandalone.fixture.ts +++ b/src/artifacts/bun/BunStandalone.fixture.ts @@ -17,7 +17,7 @@ export const locateBunGraphFixture = async ( fileSize: number, ): Promise => { const probed = await probeBunStandaloneGraph(handle, fileSize); - if (probed.status === "absent" || probed.claim === undefined) return probed; + if (probed.status !== "present" || probed.claim === undefined) return probed; return loadBunStandaloneGraph(handle, probed.claim); }; @@ -216,8 +216,10 @@ export const elfAppendedBunPayload = (payload: Buffer): Buffer => Buffer.concat([elfPrefix(), payload]); /** ELF64 whose `.bun` section holds the length-prefixed graph. */ -export const elfBunSection = (section: Buffer): Buffer => { - const shstr = Buffer.from("\0.shstrtab\0.bun\0"); +export const elfBunSection = (section: Buffer, nameOffset = 11): Buffer => { + const shstr = Buffer.alloc(nameOffset + 5); + shstr.write("\0.shstrtab\0"); + shstr.write(".bun\0", nameOffset); const shstrOffset = 64; const sectionOffset = shstrOffset + shstr.length; const tableOffset = sectionOffset + section.length; @@ -231,7 +233,13 @@ export const elfBunSection = (section: Buffer): Buffer => { shstr.copy(bytes, shstrOffset); section.copy(bytes, sectionOffset); writeElfSection(bytes, tableOffset + 64, 1, shstrOffset, shstr.length); - writeElfSection(bytes, tableOffset + 128, 11, sectionOffset, section.length); + writeElfSection( + bytes, + tableOffset + 128, + nameOffset, + sectionOffset, + section.length, + ); return bytes; }; diff --git a/src/artifacts/bun/BunStandaloneArtifactReader.test.ts b/src/artifacts/bun/BunStandaloneArtifactReader.test.ts index fe82e9748d..1914901cd0 100644 --- a/src/artifacts/bun/BunStandaloneArtifactReader.test.ts +++ b/src/artifacts/bun/BunStandaloneArtifactReader.test.ts @@ -1,12 +1,19 @@ import { open, writeFile, type FileHandle } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { join } from "node:path"; import { buffer } from "node:stream/consumers"; import { expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import * as artifactPaths from "../ArtifactPaths.js"; import { BunStandaloneArtifactReader } from "../BunStandaloneArtifactReader.js"; +import { inventoryArtifact } from "../inventory/ArtifactInventory.js"; +import { ArtifactResourceScope } from "../ArtifactResourceScope.js"; +import { resolveArtifactReaderLimits } from "../../domain/artifactReaderLimits.js"; import { bunModuleGraphPayload, + bunPrefixedPayload, + elfBunSection, + elfPrefix, elfAppendedBunPayload, } from "./BunStandalone.fixture.js"; @@ -58,6 +65,65 @@ it("keeps duplicate and unsafe names distinct while extracting exact bytes", asy }); }); +it("keeps exhausted Bun recognition distinct from a native ELF inventory and recovers the same carrier", async () => { + const path = join(await createTestTempDirectory("rea-bun-inventory-"), "app"); + const bytes = elfBunSection( + bunPrefixedPayload( + bunModuleGraphPayload({ + modules: [{ name: "main.js", contents: Buffer.from("retained") }], + }), + ), + ); + await writeFile(path, bytes); + const rootSha256 = createHash("sha256").update(bytes).digest("hex"); + const scan = async (recognitionTableBytes: number) => { + const scope = new ArtifactResourceScope(); + try { + return await inventoryArtifact(path, { + resourceScope: scope, + readerLimits: resolveArtifactReaderLimits({ + bun: { recognition_table_bytes: recognitionTableBytes }, + }), + }); + } finally { + await scope.close(); + } + }; + const limited = await scan(191); + expect(limited.manifest).toMatchObject({ + root_format: "elf", + root_sha256: rootSha256, + occurrence_count: 1, + }); + expect(limited.limitations).toContainEqual( + expect.stringContaining("Bun graph recognition remains unknown:"), + ); + expect(limited.limitations).toContainEqual( + expect.stringContaining( + "artifact_limits.bun.recognition_table_bytes (191 bytes); requires 192 bytes", + ), + ); + const recovered = await scan(192); + expect(recovered.manifest).toMatchObject({ + root_format: "bun-standalone", + root_sha256: rootSha256, + occurrence_count: 2, + }); + expect(recovered.occurrences.map((entry) => entry.logical_path)).toContain( + "main.js", + ); + expect((await scan(191)).limitations).toEqual(limited.limitations); + + const nativeElf = elfPrefix(); + await writeFile(path, nativeElf); + const ordinary = await scan(1); + expect(ordinary.manifest).toMatchObject({ + root_format: "elf", + root_sha256: createHash("sha256").update(nativeElf).digest("hex"), + occurrence_count: 1, + }); +}); + async function collect(values: AsyncIterable): Promise { const result: T[] = []; for await (const value of values) result.push(value); diff --git a/src/artifacts/deno/DenoStandaloneArtifactReader.test.ts b/src/artifacts/deno/DenoStandaloneArtifactReader.test.ts index 3c1f511ee1..252958c7af 100644 --- a/src/artifacts/deno/DenoStandaloneArtifactReader.test.ts +++ b/src/artifacts/deno/DenoStandaloneArtifactReader.test.ts @@ -34,6 +34,38 @@ import { machOWithSegment, } from "./DenoStandalone.fixture.js"; import { MAX_DENO_PATH_COMPONENT_BYTES } from "./DenoVirtualFs.js"; +import { resolveArtifactReaderLimits } from "../../domain/artifactReaderLimits.js"; + +it("keeps a confirmed Deno inventory when another pack recognizer is limited", async () => { + const original = denoElf(); + // Add a valid empty section table without changing the existing PT_NOTE. + const bytes = Buffer.concat([original, Buffer.alloc(3 * 64)]); + bytes.writeBigUInt64LE(BigInt(original.length), 40); + bytes.writeUInt16LE(64, 58); + bytes.writeUInt16LE(3, 60); + bytes.writeUInt16LE(0, 62); + const path = await writeCarrier("deno-with-sections", bytes); + const inventory = await scanCanonicalArtifactInventory(path, { + readerLimits: resolveArtifactReaderLimits({ + bun: { recognition_table_bytes: 191 }, + }), + }); + expect(inventory.manifest).toMatchObject({ + root_format: "deno-standalone", + root_sha256: createHash("sha256").update(bytes).digest("hex"), + }); + expect(inventory.limitations).toContainEqual( + expect.stringContaining("Bun graph recognition remains unknown:"), + ); + expect(inventory.limitations).toContainEqual( + expect.stringContaining( + "artifact_limits.bun.recognition_table_bytes (191 bytes)", + ), + ); + expect(inventory.occurrences[0]?.limitations).toContainEqual( + expect.stringContaining("Bun graph recognition remains unknown:"), + ); +}); const secret = "super-secret-value"; diff --git a/src/artifacts/dotnet/DotnetBundle.ts b/src/artifacts/dotnet/DotnetBundle.ts index 6368b3aaad..775116dfd4 100644 --- a/src/artifacts/dotnet/DotnetBundle.ts +++ b/src/artifacts/dotnet/DotnetBundle.ts @@ -1,4 +1,9 @@ import type { FileHandle } from "node:fs/promises"; +import { + DEFAULT_DOTNET_BUNDLE_READER_LIMITS, + type DotnetBundleReaderLimits, +} from "../../domain/artifactReaderLimits.js"; +import { artifactReaderLimitFailure } from "../ArtifactReaderLimits.js"; import { readArtifactFileChunks } from "../readArtifactFileChunks.js"; import { @@ -17,16 +22,6 @@ export const DOTNET_BUNDLE_SIGNATURE = Buffer.from( "hex", ); -// These are resource budgets, not format limits. Metadata stays bounded even -// when malicious manifests alias the same bytes or declare many empty files. -const MAX_METADATA_BYTES = 16 * 1024 * 1024; -const MAX_ENTRIES = 100_000; -const MAX_PATH_BYTES = 64 * 1024; -const MAX_PATH_PARTS = 256; -const MAX_PATH_NODES = 250_000; -const MAX_BUNDLE_ENTRY_BYTES = 512 * 1024 * 1024; -const MAX_EXPANDED_BYTES = 2 * 1024 * 1024 * 1024; - export interface DotnetBundleFile { readonly path: string; readonly offset: number; @@ -85,8 +80,9 @@ export const readDotnetBundle = async ( size: number, headerOffset: number, signal?: AbortSignal, + limits: DotnetBundleReaderLimits = DEFAULT_DOTNET_BUNDLE_READER_LIMITS, ): Promise => { - const cursor = new BundleCursor(handle, size, headerOffset, signal); + const cursor = new BundleCursor(handle, size, headerOffset, limits, signal); const majorVersion = await cursor.u32(); const minorVersion = await cursor.u32(); if (![1, 2, 6].includes(majorVersion) || minorVersion !== 0) @@ -96,11 +92,12 @@ export const readDotnetBundle = async ( ); const count = await cursor.i32(); if (count < 0) fail(".NET bundle entry count is negative"); - if (count > MAX_ENTRIES) - throw artifactFixedLimitFailure( - ".NET bundle entry count", - MAX_ENTRIES, - "entries", + if (count > limits.entry_count) + throw artifactReaderLimitFailure( + "artifact_limits.dotnet_bundle.entry_count", + limits.entry_count, + "count", + count, ); await cursor.string(); // Bundle ID is not an authentication or content digest. const configuration: { offset: number; size: number; type: number }[] = []; @@ -124,10 +121,17 @@ export const readDotnetBundle = async ( } const files: DotnetBundleFile[] = []; const paths = new ArtifactPathRegistry({ - maxParts: MAX_PATH_PARTS, - maxNodes: MAX_PATH_NODES, + maxParts: limits.path_parts, + maxNodes: limits.path_nodes, + limitFailure: (field, maximum, required) => + artifactReaderLimitFailure( + `artifact_limits.dotnet_bundle.${field}`, + maximum, + "count", + required, + ), }); - let expandedBytes = 0; + let expandedBytes = 0n; for (let index = 0; index < count; index++) { abortIfNeeded(signal); const offset = await cursor.nonnegativeI64(); @@ -138,21 +142,20 @@ export const readDotnetBundle = async ( const path = await cursor.string(); const normalized = normalizeArtifactPath(path); paths.add(normalized, "file"); - if ( - fileSize > MAX_BUNDLE_ENTRY_BYTES || - compressedSize > MAX_BUNDLE_ENTRY_BYTES - ) - throw artifactFixedLimitFailure( - ".NET bundle member stored or expanded size", - MAX_BUNDLE_ENTRY_BYTES, + if (fileSize > limits.member_bytes || compressedSize > limits.member_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.dotnet_bundle.member_bytes", + limits.member_bytes, "bytes", + Math.max(fileSize, compressedSize), ); - expandedBytes += fileSize; - if (expandedBytes > MAX_EXPANDED_BYTES) - throw artifactFixedLimitFailure( - ".NET bundle aggregate expanded size", - MAX_EXPANDED_BYTES, + expandedBytes += BigInt(fileSize); + if (expandedBytes > BigInt(limits.expanded_bytes)) + throw artifactReaderLimitFailure( + "artifact_limits.dotnet_bundle.expanded_bytes", + limits.expanded_bytes, "bytes", + expandedBytes, ); const storedSize = compressedSize || fileSize; if (offset > headerOffset || storedSize > headerOffset - offset) @@ -194,6 +197,7 @@ class BundleCursor { private readonly handle: FileHandle, private readonly size: number, private readonly start: number, + private readonly limits: DotnetBundleReaderLimits, private readonly signal?: AbortSignal, ) { this.#position = start; @@ -201,11 +205,13 @@ class BundleCursor { async take(length: number): Promise { abortIfNeeded(this.signal); - if (this.#position - this.start + length > MAX_METADATA_BYTES) - throw artifactFixedLimitFailure( - ".NET bundle metadata", - MAX_METADATA_BYTES, + const consumed = this.#position - this.start; + if (consumed + length > this.limits.metadata_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.dotnet_bundle.metadata_bytes", + this.limits.metadata_bytes, "bytes", + consumed + length, ); if (length > this.size - this.#position) fail("Truncated .NET bundle manifest"); @@ -213,7 +219,11 @@ class BundleCursor { if (cached < 0 || cached + length > this.#cache.length) { this.#cacheOffset = this.#position; this.#cache = Buffer.alloc( - Math.min(Math.max(length, 64 * 1024), this.size - this.#position), + Math.min( + Math.max(length, 64 * 1024), + this.size - this.#position, + this.limits.metadata_bytes - consumed, + ), ); let read = 0; while (read < this.#cache.length) { @@ -264,11 +274,12 @@ class BundleCursor { if (shift === 28 && byte > 15) fail("Invalid .NET bundle string length"); length += (byte & 127) * 2 ** shift; if ((byte & 128) !== 0) continue; - if (length > MAX_PATH_BYTES) - throw artifactFixedLimitFailure( - ".NET bundle decoded string", - MAX_PATH_BYTES, - "UTF-8 bytes", + if (length > this.limits.string_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.dotnet_bundle.string_bytes", + this.limits.string_bytes, + "bytes", + length, ); // Source failures are not encoding failures. Acquire bytes outside the // decoder boundary so I/O and cancellation retain their original cause. diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 4afa291382..c84a6a1b6f 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -1,4 +1,5 @@ import { selectArtifactExtractionOccurrences } from "./ArtifactExtractionSelection.js"; +import type { ArtifactReaderLimits } from "../../domain/artifactReaderLimits.js"; import { assertAdmittedRootDigest } from "../inventory/classify.js"; import { assertArtifactFileSelectedSource } from "../ArtifactFileSource.js"; import { @@ -64,6 +65,7 @@ import type { ArtifactIntegrityPolicyName } from "../../domain/artifactIntegrity /** Local extraction input with the output root chosen by the adapter. */ export interface ArtifactExtractionInput { + readonly readerLimits?: ArtifactReaderLimits; readonly asarHeaderAdmission?: AsarHeaderAdmission; /** Caller inputs when the target was explicitly selected as an ESP flash image. */ readonly espFlashSelection?: EspFlashSelection | undefined; @@ -84,7 +86,12 @@ export const extractArtifact = async ( input: ArtifactExtractionInput, signal?: AbortSignal, ): Promise => - input.resourceScope.run(() => extractArtifactInScope(input, signal)); + input.resourceScope.run(async () => { + const result = await extractArtifactInScope(input, signal); + return input.readerLimits === undefined + ? result + : { ...result, effective_limits: input.readerLimits }; + }); const extractArtifactInScope = async ( input: ArtifactExtractionInput, @@ -110,6 +117,7 @@ const extractArtifactInScope = async ( resolveAsarHeaderAdmission(input.environment), espFlashSelection: input.espFlashSelection, integrity: input.integrityPolicy, + readerLimits: input.readerLimits, }, probeFacts, ); @@ -627,6 +635,27 @@ const createExtractionResult = ( * The target kind, not the host, is unsupported by extraction. The error keeps * the caller's spelling of the path; I/O uses the canonical path. */ +const hasExecutablePack = async ( + probes: ArtifactEmbeddedFormatProbes, +): Promise => { + if ( + (await probes.nodeSea()) !== undefined || + (await probes.pyInstaller()) !== undefined + ) + return true; + const bun = await probes.bunStandalone(); + if ( + bun.status === "present" || + (await probes.denoStandalone()).status === "present" || + (await probes.dotnetBundle()) !== undefined + ) + return true; + if (bun.status === "limited") throw bun.failure; + const seaFailure = probes.completedFacts().nodeSeaRecognitionFailure; + if (seaFailure !== undefined) throw seaFailure; + return false; +}; + const requireExtractableFormat = async ( path: string, input: Pick< @@ -636,6 +665,7 @@ const requireExtractableFormat = async ( | "inputFormat" | "resourceScope" | "expectedRootSha256" + | "readerLimits" >, signal?: AbortSignal, ): Promise => { @@ -650,15 +680,17 @@ const requireExtractableFormat = async ( input.resourceScope, signal, async (source) => { - const probes = new ArtifactEmbeddedFormatProbes(path, source, signal); + const probes = new ArtifactEmbeddedFormatProbes( + path, + source, + signal, + undefined, + input.readerLimits, + ); const pack = (await probes.godotPck()) !== undefined || ((format === "elf" || format === "pe" || format === "mach-o") && - ((await probes.nodeSea()) !== undefined || - (await probes.pyInstaller()) !== undefined || - (await probes.bunStandalone()).status === "present" || - (await probes.denoStandalone()).status === "present" || - (await probes.dotnetBundle()) !== undefined)); + (await hasExecutablePack(probes))); if (pack && input.expectedRootSha256 !== undefined) { const digest = await hashStableRootArtifactHandle( path, @@ -697,6 +729,7 @@ const createReader = async ( | "asarHeaderAdmission" | "resourceScope" | "espFlashSelection" + | "readerLimits" >, manifest: ArtifactGraphManifest, transferredSource?: ActiveArtifactInventorySource, @@ -757,6 +790,7 @@ const createReader = async ( ), admission: transferredSource?.admission, espFlashSelection: input.espFlashSelection, + readerLimits: input.readerLimits, asarHeaderAdmission: containerFormat === "asar" ? (input.asarHeaderAdmission ?? diff --git a/src/artifacts/inventory/ArtifactInventory.ts b/src/artifacts/inventory/ArtifactInventory.ts index d3913bfb85..2e880b4931 100644 --- a/src/artifacts/inventory/ArtifactInventory.ts +++ b/src/artifacts/inventory/ArtifactInventory.ts @@ -23,7 +23,12 @@ export const inventoryArtifact = async ( const snapshot = await options.resourceScope.run(() => scanArtifactInventoryInScope(inputPath, options), ); - return artifactInventoryResultSchema.parse(snapshot); + return artifactInventoryResultSchema.parse({ + ...snapshot, + ...(options.readerLimits === undefined + ? {} + : { effective_limits: options.readerLimits }), + }); }; export const scanArtifactInventoryInScope = async ( diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 53c6ea58bd..8b932dd0ce 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -4,6 +4,10 @@ import { parseInesLayout, } from "../../domain/inesContainer.js"; import type { GodotPckAdmission } from "../GodotPckLocation.js"; +import { + resolveArtifactReaderLimits, + type ArtifactReaderLimits, +} from "../../domain/artifactReaderLimits.js"; import type { PyInstallerAdmission } from "../pyinstaller/PyInstallerAdmission.js"; import type { ArtifactReaderAdmission, @@ -71,6 +75,7 @@ interface RootClassification { type RootInventoryClassification = RootClassification & { readonly rootSource: RootInventorySource | undefined; readonly readerAdmission: ArtifactReaderAdmission | undefined; + readonly recognitionLimitations?: readonly string[]; }; /** The admitted descriptor and the single owner that can confirm its close. */ @@ -91,6 +96,7 @@ export const classifyAndHashRootForInventory = async ( | "selectedPath" | "expectedRootSha256" | "espFlashSelection" + | "readerLimits" >, probeFacts?: ArtifactEmbeddedFormatProbeFacts, ): Promise => { @@ -143,6 +149,7 @@ export const classifyAndHashRootForInventory = async ( source, options.signal, probeFacts, + options.readerLimits, ); const admittedDigest = options.expectedRootSha256 === undefined @@ -167,6 +174,7 @@ export const classifyAndHashRootForInventory = async ( options.asarHeaderAdmission, options.selectedPath, options.espFlashSelection, + options.readerLimits, ); const format = typeof classified === "string" ? classified : classified.kind; @@ -180,11 +188,26 @@ export const classifyAndHashRootForInventory = async ( )); const retainSource = artifactFormatDescriptor(format)?.retainClassifiedSource === true; + const bunRecognition = probes.completedFacts().bunStandalone; + const seaRecognition = probes.completedFacts().nodeSeaRecognitionFailure; + const recognitionLimitations = [ + ...(bunRecognition?.status === "limited" + ? [ + `Bun graph recognition remains unknown: ${bunRecognition.failure.message}`, + ] + : []), + ...(seaRecognition === undefined + ? [] + : [`Node SEA recognition remains unknown: ${seaRecognition.message}`]), + ]; outcome = { kind: "completed", value: { format, digest, + ...(recognitionLimitations.length === 0 + ? {} + : { recognitionLimitations }), rootSource: retainSource ? source : undefined, readerAdmission: typeof classified !== "string" @@ -220,6 +243,7 @@ const classifyRootFormat = async ( asarHeaderAdmission?: AsarHeaderAdmission, selectedPath?: string, espFlashSelection?: EspFlashSelection, + readerLimits: ArtifactReaderLimits = resolveArtifactReaderLimits(), ): Promise => { const { handle } = source; const probe = createArtifactClassificationProbe(asarHeaderAdmission); @@ -354,10 +378,17 @@ const classifyRootFormat = async ( abortIfNeeded(signal); if (sea !== undefined) { const decoded = await decodeNodeSeaBlob( - await readNodeSeaBlob(handle, sea, archiveBytes, signal), + await readNodeSeaBlob(source, sea, signal, readerLimits.node_sea), signal, + readerLimits.node_sea, ); - return { kind: "node-sea", source, location: sea, decoded }; + return { + kind: "node-sea", + source, + location: sea, + decoded, + limits: readerLimits.node_sea, + }; } const bun = await embeddedProbes.bunStandalone(); if (bun.status === "present") diff --git a/src/artifacts/inventory/reader.ts b/src/artifacts/inventory/reader.ts index b2bda7feed..d7afc1d366 100644 --- a/src/artifacts/inventory/reader.ts +++ b/src/artifacts/inventory/reader.ts @@ -1,4 +1,5 @@ import type { ArtifactReaderAdmission } from "../ArtifactReaderAdmission.js"; +import type { ArtifactReaderLimits } from "../../domain/artifactReaderLimits.js"; import type { AsarHeaderAdmission } from "../../config/asarHeaderAdmission.js"; import type { ArtifactReader, @@ -21,6 +22,7 @@ export const createReader = ( readerAdmission?: ArtifactReaderAdmission, asarHeaderAdmission?: AsarHeaderAdmission, selectedPath?: string, + readerLimits?: ArtifactReaderLimits, ): ArtifactReader | undefined => { if ( artifactFormatDescriptor(format)?.requiresDarwin === true && @@ -33,6 +35,7 @@ export const createReader = ( admission: readerAdmission, asarHeaderAdmission, selectedPath, + readerLimits, }); }; diff --git a/src/artifacts/inventory/scanCanonical.ts b/src/artifacts/inventory/scanCanonical.ts index 2d024c1d9c..bb9ee99fdb 100644 --- a/src/artifacts/inventory/scanCanonical.ts +++ b/src/artifacts/inventory/scanCanonical.ts @@ -125,6 +125,7 @@ const scanCanonical = async ( digest: rootDigest, rootSource, readerAdmission, + recognitionLimitations, } = await classifyAndHashRootForInventory( path, metadata.isDirectory(), @@ -142,6 +143,7 @@ const scanCanonical = async ( readerAdmission, options.asarHeaderAdmission, selectedPath, + options.readerLimits, ); } catch (cause: unknown) { readerCreationFailure = { cause }; @@ -187,6 +189,7 @@ const scanCanonical = async ( unknownParentOccurrences, traversal, rootContradiction, + recognitionLimitations, }), }; } catch (cause: unknown) { @@ -406,6 +409,7 @@ const cleanupInventoryOwners = async ({ }; interface SnapshotBuildInput { + readonly recognitionLimitations?: readonly string[] | undefined; readonly resourceScope: ArtifactInventoryOptions["resourceScope"]; readonly path: string; readonly metadata: Stats; @@ -443,6 +447,7 @@ const buildInventorySnapshot = async ( path, formatMetadata: reader?.formatMetadata?.(), }); + rootOccurrence.limitations.push(...(input.recognitionLimitations ?? [])); for (const occurrence of input.occurrences) if ( occurrence.parent_occurrence_id === null && @@ -537,7 +542,11 @@ const buildInventorySnapshot = async ( integrity_contradictions: integrityContradictions, coverage, limitations: buildLimitations( - [...traversal.limitations, ...edgeResult.limitations], + [ + ...traversal.limitations, + ...edgeResult.limitations, + ...(input.recognitionLimitations ?? []), + ], integrityContradictions, input.caseCollisionLimitation, ), diff --git a/src/artifacts/inventory/types.ts b/src/artifacts/inventory/types.ts index 2d6f00b476..5938f64404 100644 --- a/src/artifacts/inventory/types.ts +++ b/src/artifacts/inventory/types.ts @@ -2,9 +2,11 @@ import type { EspFlashSelection } from "../../domain/firmware/espFlashLocation.j import type { ArtifactIntegrityPolicyName } from "../../domain/artifactIntegrityPolicy.js"; import type { AsarHeaderAdmission } from "../../config/asarHeaderAdmission.js"; import type { ArtifactResourceScope } from "../ArtifactResourceScope.js"; +import type { ArtifactReaderLimits } from "../../domain/artifactReaderLimits.js"; /** Options shared by artifact inventory scans. */ export interface ArtifactInventoryOptions { + readonly readerLimits?: ArtifactReaderLimits | undefined; readonly resourceScope: ArtifactResourceScope; /** Identity of a previously admitted file root; directory roots have no file digest. */ readonly expectedRootSha256?: string | undefined; diff --git a/src/artifacts/javascript/JavaScriptSourceAnalysis.ts b/src/artifacts/javascript/JavaScriptSourceAnalysis.ts index e34916f626..69bcffde7d 100644 --- a/src/artifacts/javascript/JavaScriptSourceAnalysis.ts +++ b/src/artifacts/javascript/JavaScriptSourceAnalysis.ts @@ -1,3 +1,4 @@ +import { resolveJavaScriptSemanticPrecision } from "../../domain/javascript/javascriptSemanticPrecision.js"; import type { JavaScriptSourceAnalysisLimits, JavaScriptSourceAnalysisPort, @@ -128,8 +129,20 @@ const registerJavaScriptSourceAnalysisClose = ( const sameLimits = ( left: JavaScriptSourceAnalysisLimits, right: JavaScriptSourceAnalysisLimits, -): boolean => - left.heapMb === right.heapMb && left.timeoutMs === right.timeoutMs; +): boolean => { + if (left.heapMb !== right.heapMb || left.timeoutMs !== right.timeoutMs) + return false; + const leftPrecision = resolveJavaScriptSemanticPrecision( + left.semanticPrecision, + ); + const rightPrecision = resolveJavaScriptSemanticPrecision( + right.semanticPrecision, + ); + return Object.entries(leftPrecision).every( + ([key, value]) => + rightPrecision[key as keyof typeof rightPrecision] === value, + ); +}; /** * Park a healthy worker on its scope. The next checkout with the same limits diff --git a/src/artifacts/nodeSea/NodeSeaAdmission.ts b/src/artifacts/nodeSea/NodeSeaAdmission.ts index b5855fc20d..2b5612c720 100644 --- a/src/artifacts/nodeSea/NodeSeaAdmission.ts +++ b/src/artifacts/nodeSea/NodeSeaAdmission.ts @@ -1,6 +1,7 @@ import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js"; import type { DecodedNodeSeaBlob } from "./NodeSeaBlob.js"; import type { NodeSeaLocation } from "./NodeSeaLocation.js"; +import type { NodeSeaReaderLimits } from "../../domain/artifactReaderLimits.js"; /** Validated blob facts borrowed with the exact descriptor that produced them. */ export interface NodeSeaAdmission { @@ -8,4 +9,6 @@ export interface NodeSeaAdmission { readonly source: StableRegularFileDescriptor; readonly location: NodeSeaLocation; readonly decoded: DecodedNodeSeaBlob; + /** Accepted producing policy; older admissions require fresh admission. */ + readonly limits?: NodeSeaReaderLimits; } diff --git a/src/artifacts/nodeSea/NodeSeaBlob.test.ts b/src/artifacts/nodeSea/NodeSeaBlob.test.ts index 8d15ababcf..3e76582e20 100644 --- a/src/artifacts/nodeSea/NodeSeaBlob.test.ts +++ b/src/artifacts/nodeSea/NodeSeaBlob.test.ts @@ -2,7 +2,7 @@ import { setImmediate } from "node:timers/promises"; import { expect, it } from "vitest"; -import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { DEFAULT_NODE_SEA_READER_LIMITS } from "../../domain/artifactReaderLimits.js"; import { NODE_SEA_MAGIC, decodeNodeSeaBlob, @@ -134,80 +134,83 @@ it.each([ }, ); -const limitOf = async (bytes: Buffer): Promise => { - try { - await decodeNodeSeaBlob(bytes); - } catch (cause) { - if (cause instanceof ArtifactReaderFailure) return cause; - throw cause; - } - throw new Error("expected a limit failure"); -}; - -it("refuses an asset table over its count bound before creating records", async () => { - const assets = Array.from({ length: 100_001 }, (_, index) => ({ - name: `asset-${String(index)}`, - body: "", - })); - const failure = await limitOf(buildNodeSeaBlob({ assets })); - expect(failure).toMatchObject({ - reason: "limit", - message: expect.stringMatching( - /asset count.*100000.*no runtime override/iu, - ), - }); -}); - -it("refuses asset names over the decoded name budget", async () => { - const name = "n".repeat(4 * 1024 * 1024 + 1); - const failure = await limitOf( - buildNodeSeaBlob({ - assets: [ - { name: `a${name}`, body: "" }, - { name: `b${name}`, body: "" }, - ], +it("rejects impossible asset rows before applying resource admission", async () => { + const prefix = buildNodeSeaBlob({ assets: [{ name: "a", body: "" }] }); + const count = Buffer.alloc(8); + count.writeBigUInt64LE(2n); + const bytes = Buffer.concat([prefix.subarray(0, -25), count]); + await expect( + decodeNodeSeaBlob(bytes, undefined, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + asset_count: 1, }), - ); - expect(failure).toMatchObject({ - reason: "limit", - message: expect.stringMatching( - /asset names.*8388608.*no runtime override/iu, - ), + ).rejects.toMatchObject({ + reason: "format", + message: expect.stringMatching(/Node SEA.*truncated/iu), + readerLimit: undefined, }); }); it.each([ - { - label: "count", - accepted: () => Array.from({ length: 4096 }, () => "x"), - args: () => Array.from({ length: 4097 }, () => "x"), - boundary: "capacity", - warning: /exec argv count.*4096.*no runtime override/iu, - }, - { - label: "text", - accepted: () => ["x".repeat(1024 * 1024)], - args: () => ["x".repeat(1024 * 1024 + 1)], - boundary: "memory", - warning: /exec argv text.*1048576.*no runtime override/iu, - }, + { field: "asset_count" as const, limit: 1, raised: 2 }, + { field: "asset_name_bytes" as const, limit: 2, raised: 3 }, + { field: "argument_count" as const, limit: 1, raised: 2 }, + { field: "argument_bytes" as const, limit: 2, raised: 3 }, ])( - "admits exec argument $label at its bound and refuses more", - async ({ accepted, args, boundary, warning }) => { - const atLimit = accepted(); - expect( - (await decodeNodeSeaBlob(buildNodeSeaBlob({ execArgv: atLimit }))) - .execArgv, - ).toEqual(atLimit); - const failure = await limitOf(buildNodeSeaBlob({ execArgv: args() })); - expect(failure).toMatchObject({ - reason: "limit", - boundary, - message: expect.stringMatching(warning), + "recovers the same blob after raising $field", + async ({ field, limit, raised }) => { + const bytes = buildNodeSeaBlob({ + assets: [ + { name: "a", body: "one" }, + { name: "bb", body: "two" }, + ], + execArgv: ["x", "yz"], }); + await expect( + decodeNodeSeaBlob(bytes, undefined, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + [field]: limit, + }), + ).rejects.toMatchObject({ + reason: "limit", + readerLimit: { + parameter: `artifact_limits.node_sea.${field}`, + effective_value: limit, + }, + }); + const recovered = await decodeNodeSeaBlob(bytes, undefined, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + [field]: raised, + }); + expect(recovered.assets.map((asset) => asset.name)).toEqual(["a", "bb"]); + expect(recovered.execArgv).toEqual(["x", "yz"]); }, ); +it("applies blob byte admission even when called without a carrier reader", async () => { + const bytes = buildNodeSeaBlob({ main: "hello" }); + await expect( + decodeNodeSeaBlob(bytes, undefined, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + blob_bytes: bytes.length - 1, + }), + ).rejects.toMatchObject({ + reason: "limit", + readerLimit: { + parameter: "artifact_limits.node_sea.blob_bytes", + required_value: bytes.length, + }, + }); + expect( + ( + await decodeNodeSeaBlob(bytes, undefined, { + ...DEFAULT_NODE_SEA_READER_LIMITS, + blob_bytes: bytes.length, + }) + ).mainScript?.length, + ).toBe(5); +}); + it("rejects non-zero trailing bytes and keeps zero padding", async () => { const padded = buildNodeSeaBlob({ trailing: Buffer.alloc(8) }); expect((await decodeNodeSeaBlob(padded)).codePath).toBe("sea.js"); diff --git a/src/artifacts/nodeSea/NodeSeaBlob.ts b/src/artifacts/nodeSea/NodeSeaBlob.ts index a737c222ad..7a9c92eba2 100644 --- a/src/artifacts/nodeSea/NodeSeaBlob.ts +++ b/src/artifacts/nodeSea/NodeSeaBlob.ts @@ -2,13 +2,15 @@ import { setImmediate } from "node:timers/promises"; import { assertArtifactReadActive } from "../ArtifactFileOperation.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { + DEFAULT_NODE_SEA_READER_LIMITS, + type NodeSeaReaderLimits, +} from "../../domain/artifactReaderLimits.js"; +import { artifactReaderLimitFailure } from "../ArtifactReaderLimits.js"; /** Node `SeaBlob::kMagic` from `src/node_sea.h` (v24). */ export const NODE_SEA_MAGIC = 0x143da20; -/** In-memory cap for one injected blob. Larger declarations fail closed. */ -export const NODE_SEA_MAX_BLOB_BYTES = 256 * 1024 * 1024; - const FLAG_DISABLE_EXPERIMENTAL_SEA_WARNING = 1 << 0; const FLAG_USE_SNAPSHOT = 1 << 1; const FLAG_USE_CODE_CACHE = 1 << 2; @@ -21,15 +23,6 @@ const KNOWN_FLAGS = FLAG_INCLUDE_ASSETS | FLAG_INCLUDE_EXEC_ARGV; -// Resource bounds checked before records are created, not Node format limits. -// Asset names are budgeted separately because empty bodies make each record -// cost far more retained memory than its blob bytes. -const MAX_ASSET_COUNT = 100_000; -const MAX_ASSET_NAME_BYTES = 8 * 1024 * 1024; -const MAX_EXEC_ARGV = 4096; -const MAX_EXEC_ARGV_TEXT_BYTES = 1024 * 1024; -const NO_OVERRIDE = "This release has no runtime override."; - const NODE_SEA_BAD_MAGIC = "Node SEA blob magic is not 0x143da20"; export const NODE_SEA_TRUNCATED = "Node SEA blob is truncated"; const NODE_SEA_UNKNOWN_FLAGS = "Node SEA blob flags contain unknown bits"; @@ -54,11 +47,6 @@ const NODE_SEA_FUSE_UNSET = const NODE_SEA_FUSE_CONFLICT = "Node SEA fuse sentinels disagree"; const NODE_SEA_EMPTY_CODE_PATH = "Node SEA code path is empty"; const NODE_SEA_INVALID_UTF8 = "Node SEA string is not valid UTF-8"; -const NODE_SEA_ASSET_COUNT = `Node SEA asset count exceeds the ${String(MAX_ASSET_COUNT)}-asset bound. ${NO_OVERRIDE}`; -const NODE_SEA_ASSET_NAME_BYTES = `Node SEA asset names exceed the ${String(MAX_ASSET_NAME_BYTES)}-byte name budget. ${NO_OVERRIDE}`; -const NODE_SEA_EXEC_ARGV_COUNT = `Node SEA exec argv count exceeds the ${String(MAX_EXEC_ARGV)}-argument bound. ${NO_OVERRIDE}`; -const NODE_SEA_EXEC_ARGV_TEXT = `Node SEA exec argv text exceeds the ${String(MAX_EXEC_ARGV_TEXT_BYTES)}-byte bound. ${NO_OVERRIDE}`; -export const NODE_SEA_BLOB_TOO_LARGE = `Node SEA blob exceeds the ${String(NODE_SEA_MAX_BLOB_BYTES)}-byte in-memory bound`; const FUSE_PREFIX = "NODE_SEA_FUSE_"; @@ -98,12 +86,6 @@ export interface NodeSeaFuseObservation { readonly fuseContradiction: string | null; } -const limitFailure = ( - message: string, - boundary: "capacity" | "memory", -): ArtifactReaderFailure => - new ArtifactReaderFailure("limit", message, { boundary }); - const formatFailure = ( message: string, cause?: unknown, @@ -123,9 +105,18 @@ const formatFailure = ( export const decodeNodeSeaBlob = async ( bytes: Buffer, signal?: AbortSignal, + limits: NodeSeaReaderLimits = DEFAULT_NODE_SEA_READER_LIMITS, ): Promise => { assertArtifactReadActive(signal); - const cursor = new BlobCursor(bytes); + if (bytes.length > limits.blob_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.node_sea.blob_bytes", + limits.blob_bytes, + "bytes", + bytes.length, + "memory", + ); + const cursor = new BlobCursor(bytes, limits); if (cursor.remaining < 4) throw formatFailure(NODE_SEA_TRUNCATED); const magic = cursor.u32(); if (magic !== NODE_SEA_MAGIC) throw formatFailure(NODE_SEA_BAD_MAGIC); @@ -212,7 +203,10 @@ const extensionName = (value: number): NodeSeaExecArgvExtension => { class BlobCursor { #offset = 0; - constructor(private readonly bytes: Buffer) {} + constructor( + private readonly bytes: Buffer, + private readonly limits: NodeSeaReaderLimits, + ) {} get remaining(): number { return this.bytes.length - this.#offset; @@ -241,6 +235,12 @@ class BlobCursor { raw, ); } catch (cause: unknown) { + if (cause instanceof RangeError) + throw new ArtifactReaderFailure( + "limit", + "Node SEA UTF-8 text exceeds this runtime's string or memory capacity. Admission budgets do not raise runtime representation limits or reserve host memory.", + { boundary: "memory", cause }, + ); throw formatFailure(NODE_SEA_INVALID_UTF8, cause); } if (!allowEmpty && text.length === 0) throw formatFailure(emptyMessage); @@ -255,14 +255,20 @@ class BlobCursor { } async assets(signal?: AbortSignal): Promise { - const count = this.#count(MAX_ASSET_COUNT, 16, NODE_SEA_ASSET_COUNT); + const count = this.#count("asset_count", 16); const seen = new Set(); const assets: NodeSeaAsset[] = []; let nameBytes = 0; const charge = (bytes: number): void => { nameBytes += bytes; - if (nameBytes > MAX_ASSET_NAME_BYTES) - throw limitFailure(NODE_SEA_ASSET_NAME_BYTES, "memory"); + if (nameBytes > this.limits.asset_name_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.node_sea.asset_name_bytes", + this.limits.asset_name_bytes, + "bytes", + nameBytes, + "memory", + ); }; for (let index = 0; index < count; index += 1) { if (index % 256 === 0) { @@ -279,13 +285,19 @@ class BlobCursor { } async execArgv(signal?: AbortSignal): Promise { - const count = this.#count(MAX_EXEC_ARGV, 8, NODE_SEA_EXEC_ARGV_COUNT); + const count = this.#count("argument_count", 8); const values: string[] = []; let textBytes = 0; const charge = (bytes: number): void => { textBytes += bytes; - if (textBytes > MAX_EXEC_ARGV_TEXT_BYTES) - throw limitFailure(NODE_SEA_EXEC_ARGV_TEXT, "memory"); + if (textBytes > this.limits.argument_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.node_sea.argument_bytes", + this.limits.argument_bytes, + "bytes", + textBytes, + "memory", + ); }; for (let index = 0; index < count; index += 1) { if (index % 256 === 0) { @@ -324,13 +336,22 @@ class BlobCursor { return Number(value); } - #count(limit: number, minimumRowBytes: number, message: string): number { + #count( + field: "asset_count" | "argument_count", + minimumRowBytes: number, + ): number { if (this.remaining < 8) throw formatFailure(NODE_SEA_TRUNCATED); const value = this.#demand(8).readBigUInt64LE(0); if (value > BigInt(Math.floor(this.remaining / minimumRowBytes))) throw formatFailure(NODE_SEA_TRUNCATED); const count = Number(value); - if (count > limit) throw limitFailure(message, "capacity"); + if (count > this.limits[field]) + throw artifactReaderLimitFailure( + `artifact_limits.node_sea.${field}`, + this.limits[field], + "count", + count, + ); return count; } diff --git a/src/artifacts/nodeSea/NodeSeaLocation.ts b/src/artifacts/nodeSea/NodeSeaLocation.ts index 0fcddec37b..e2796f60b4 100644 --- a/src/artifacts/nodeSea/NodeSeaLocation.ts +++ b/src/artifacts/nodeSea/NodeSeaLocation.ts @@ -4,6 +4,7 @@ import { readElfSectionHeader } from "../../domain/elfSectionHeader.js"; import { readElfProgramHeader } from "../../domain/elfProgramHeader.js"; import { readElfRawTableDeclarations } from "../../domain/elfRawTableDeclarations.js"; import { createHash } from "node:crypto"; +import { constants as bufferConstants } from "node:buffer"; import { parsePeResources } from "../../native/pe/PeResourceParser.js"; import { hasPeResourceName, @@ -14,9 +15,15 @@ import { MAX_INSPECT_PE_DIRECTORY_ENTRIES, } from "../../domain/native/peResources.js"; import type { FileHandle } from "node:fs/promises"; +import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js"; import { abortIfNeeded } from "../ArtifactHash.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; +import { + DEFAULT_NODE_SEA_READER_LIMITS, + type NodeSeaReaderLimits, +} from "../../domain/artifactReaderLimits.js"; +import { artifactReaderLimitFailure } from "../ArtifactReaderLimits.js"; import { readFatContainerHeader, readFatHeaderPrefix, @@ -30,14 +37,12 @@ import { readElfNoteHeader, } from "../../domain/native/elfNoteLayout.js"; import { - NODE_SEA_BLOB_TOO_LARGE, NODE_SEA_DUPLICATE_BLOB, NODE_SEA_ELF_DESCRIPTOR_TRUNCATED, NODE_SEA_ELF_NOTES_REPEATED, NODE_SEA_ELF_TABLE_TRUNCATED, NODE_SEA_MACHO_COMMANDS_TRUNCATED, NODE_SEA_MACHO_SEGMENT_TRUNCATED, - NODE_SEA_MAX_BLOB_BYTES, NODE_SEA_TRUNCATED, observeNodeSeaFuseMatches, type NodeSeaFuseObservation, @@ -50,9 +55,6 @@ const SHT_NOTE = 7; const RT_RCDATA = 10; const RESOURCE_NAME = "NODE_SEA_BLOB"; const SEA_SECTION = "__NODE_SEA_BLOB"; -const MAX_COMMAND_BYTES = 8 * 1024 * 1024; -const MAX_ELF_HEADERS = 4096; -const MAX_NOTE_NAME_BYTES = 4096; const FUSE_LENGTH = "NODE_SEA_FUSE_".length + 32 + 2; // PE header structures must fit this prefix for the bounded name probe. const PE_HEADER_PROBE_BYTES = 64 * 1024; @@ -73,6 +75,7 @@ interface Carrier { readonly handle: FileHandle; readonly fileSize: number; readonly signal: AbortSignal | undefined; + readonly limits?: NodeSeaReaderLimits; } const formatFailure = (message: string): ArtifactReaderFailure => @@ -81,6 +84,22 @@ const formatFailure = (message: string): ArtifactReaderFailure => const limitFailure = (message: string): ArtifactReaderFailure => new ArtifactReaderFailure("limit", message, { boundary: "memory" }); +const admitBlobBytes = (carrier: Carrier, required: number): void => { + const limits = carrier.limits ?? DEFAULT_NODE_SEA_READER_LIMITS; + if (required > limits.blob_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.node_sea.blob_bytes", + limits.blob_bytes, + "bytes", + required, + "memory", + ); + if (required > bufferConstants.MAX_LENGTH) + throw limitFailure( + `Node SEA blob requires ${String(required)} bytes, exceeding this runtime's ${String(bufferConstants.MAX_LENGTH)}-byte Buffer representation limit; increasing artifact_limits.node_sea.blob_bytes cannot raise that runtime limit`, + ); +}; + /** * Find a postject `NODE_SEA_BLOB` slot. Mach-O accepts segment `NODE_SEA` * (the postject name) and `__NODE_SEA`. ELF matches the full note owner @@ -93,9 +112,10 @@ export const locateNodeSea = async ( handle: FileHandle, fileSize: number, signal?: AbortSignal, + limits: NodeSeaReaderLimits = DEFAULT_NODE_SEA_READER_LIMITS, ): Promise => { if (!Number.isSafeInteger(fileSize) || fileSize < 4) return undefined; - const carrier: Carrier = { handle, fileSize, signal }; + const carrier: Carrier = { handle, fileSize, signal, limits }; const prefix = await readExact(carrier, 0, Math.min(fileSize, 64)); if (prefix === undefined || prefix.length < 4) return undefined; const thin = readMachoHeaderPrefixFacts(prefix).format; @@ -120,13 +140,16 @@ export const locateNodeSea = async ( /** Read a located blob, refusing declarations above the in-memory cap. */ export const readNodeSeaBlob = async ( - handle: FileHandle, + source: StableRegularFileDescriptor, location: NodeSeaLocation, - fileSize: number, signal?: AbortSignal, + limits: NodeSeaReaderLimits = DEFAULT_NODE_SEA_READER_LIMITS, ): Promise => { - if (location.blobLength > NODE_SEA_MAX_BLOB_BYTES) - throw limitFailure(NODE_SEA_BLOB_TOO_LARGE); + const { + handle, + initial: { size: fileSize }, + } = source; + admitBlobBytes({ handle, fileSize, signal, limits }, location.blobLength); const bytes = await readExact( { handle, fileSize, signal }, location.blobOffset, @@ -256,7 +279,15 @@ const locateThinMachO = async ( commandCount: ncmds, commandBytes: sizeofcmds, } = framing.value; - if (sizeofcmds > MAX_COMMAND_BYTES) return []; + const limits = carrier.limits ?? DEFAULT_NODE_SEA_READER_LIMITS; + if (sizeofcmds > limits.recognition_load_command_bytes) + throw artifactReaderLimitFailure( + "artifact_limits.node_sea.recognition_load_command_bytes", + limits.recognition_load_command_bytes, + "bytes", + sizeofcmds, + "memory", + ); const commands = await readExact(carrier, base + headerSize, sizeofcmds); if (commands === undefined) return []; const found: BlobRange[] = []; @@ -309,8 +340,11 @@ const locateThinMachO = async ( if (cString(record.nameBytes, 0, 16) !== SEA_SECTION) continue; const length = Number(record.size); const fileOffset = record.offset; - if (!Number.isSafeInteger(length) || length > NODE_SEA_MAX_BLOB_BYTES) - throw limitFailure(NODE_SEA_BLOB_TOO_LARGE); + if (!Number.isSafeInteger(length)) + throw limitFailure( + "Node SEA blob length exceeds exact-number representation", + ); + admitBlobBytes(carrier, length); if (fileOffset > sliceSize || length > sliceSize - fileOffset) throw formatFailure(NODE_SEA_TRUNCATED); found.push({ offset: base + fileOffset, length }); @@ -468,22 +502,30 @@ const readElfTables = async ( if (table.count === 0) return []; if ( table.offset === undefined || - table.count > MAX_ELF_HEADERS || table.entrySize < table.minimumEntry || - table.entrySize > 1024 || - table.offset > carrier.fileSize + table.offset > carrier.fileSize || + table.count > + Math.floor((carrier.fileSize - table.offset) / table.entrySize) ) return undefined; + const limits = carrier.limits ?? DEFAULT_NODE_SEA_READER_LIMITS; + if (table.count > limits.recognition_header_count) + throw artifactReaderLimitFailure( + "artifact_limits.node_sea.recognition_header_count", + limits.recognition_header_count, + "count", + table.count, + ); const entries: Buffer[] = []; for (let index = 0; index < table.count; index += 1) { if (index % 128 === 0) abortIfNeeded(carrier.signal); const offset = table.offset + index * table.entrySize; if ( offset > carrier.fileSize || - table.entrySize > carrier.fileSize - offset + table.minimumEntry > carrier.fileSize - offset ) return undefined; - const entry = await readExact(carrier, offset, table.entrySize); + const entry = await readExact(carrier, offset, table.minimumEntry); if (entry === undefined) return undefined; entries.push(entry); } @@ -533,8 +575,6 @@ const scanElfNoteTable = async ( if (!parsedHeader.ok) return failNoteTable(alreadyFound || found.length > 0); const { namesz, descsz } = parsedHeader.value; - if (namesz > MAX_NOTE_NAME_BYTES) - return failNoteTable(alreadyFound || found.length > 0); if (cursor > note.length - 12 || namesz > note.length - cursor - 12) return failNoteTable(alreadyFound || found.length > 0); let layout: ReturnType; @@ -549,7 +589,13 @@ const scanElfNoteTable = async ( layout.next < 12 ) return failNoteTable(alreadyFound || found.length > 0); - const name = await readExact(carrier, note.offset + cursor + 12, namesz); + // Only the exact SEA owner matters here. A bounded prefix distinguishes + // it without retaining an unrelated vendor's potentially long name. + const name = await readExact( + carrier, + note.offset + cursor + 12, + Math.min(namesz, RESOURCE_NAME.length + 1), + ); if (name === undefined) return failNoteTable(alreadyFound || found.length > 0); const owner = cString(name, 0, name.length); @@ -559,8 +605,7 @@ const scanElfNoteTable = async ( if (owner === RESOURCE_NAME) { if (!descriptorFits) throw formatFailure(NODE_SEA_ELF_DESCRIPTOR_TRUNCATED); - if (descsz > NODE_SEA_MAX_BLOB_BYTES) - throw limitFailure(NODE_SEA_BLOB_TOO_LARGE); + admitBlobBytes(carrier, descsz); const blobOffset = note.offset + cursor + layout.descriptor; if ( blobOffset > carrier.fileSize || @@ -644,8 +689,7 @@ const locatePeResources = async (carrier: Carrier): Promise => { ) return []; const range = resource.payload.location; - if (range.bytes > NODE_SEA_MAX_BLOB_BYTES) - throw limitFailure(NODE_SEA_BLOB_TOO_LARGE); + admitBlobBytes(carrier, range.bytes); return [{ offset: range.offset, length: range.bytes }]; }); }; diff --git a/src/browser/IsolatedWebBundleAnalyzer.ts b/src/browser/IsolatedWebBundleAnalyzer.ts index c34abe2132..12930f6e41 100644 --- a/src/browser/IsolatedWebBundleAnalyzer.ts +++ b/src/browser/IsolatedWebBundleAnalyzer.ts @@ -215,6 +215,7 @@ export class IsolatedWebBundleAnalyzer { script, input.analysis_timeout_ms ?? DEFAULT_WEB_BUNDLE_SCRIPT_TIMEOUT_MS, deadline.signal, + input.semantic_precision, ); if (analyzed.ok) results.push(analyzed.value); else { diff --git a/src/browser/WebBundleAnalysisWorker.ts b/src/browser/WebBundleAnalysisWorker.ts index 9bf2696a83..34de81b81e 100644 --- a/src/browser/WebBundleAnalysisWorker.ts +++ b/src/browser/WebBundleAnalysisWorker.ts @@ -1,3 +1,4 @@ +import type { JavaScriptSemanticPrecisionInput } from "../domain/javascript/javascriptSemanticPrecision.js"; import { analysisErrorWithCancellation } from "../domain/analysisErrorCancellation.js"; import { WorkerFrameSignal } from "../process/WorkerFrameSignal.js"; import { createHash, randomUUID } from "node:crypto"; @@ -273,6 +274,7 @@ export class WebBundleAnalysisWorker { script: IncludedScript, timeoutMs: number, signal?: AbortSignal, + precision?: JavaScriptSemanticPrecisionInput, ): Promise> { let deadline: ProviderStartupDeadline | undefined; const id = randomUUID(); @@ -324,6 +326,7 @@ export class WebBundleAnalysisWorker { ); const request = webBundleWorkerRequestSchema.parse({ kind: "analyze", + ...(precision === undefined ? {} : { semantic_precision: precision }), id, source, result_budget_bytes: resultBudget, diff --git a/src/browser/WebBundleAnalysisWorkerProtocol.ts b/src/browser/WebBundleAnalysisWorkerProtocol.ts index 7485c0482a..6a098e8759 100644 --- a/src/browser/WebBundleAnalysisWorkerProtocol.ts +++ b/src/browser/WebBundleAnalysisWorkerProtocol.ts @@ -1,3 +1,4 @@ +import { javaScriptSemanticPrecisionSchema } from "../domain/javascript/javascriptSemanticPrecision.js"; import { z } from "zod"; import { JAVASCRIPT_STATIC_FACT_BYTES } from "../domain/javascript/javascriptAnalysisResourceControls.js"; import { digestSchema } from "../domain/digests.js"; @@ -35,6 +36,7 @@ export const includedWebScriptSchema = browserScriptSchema export const webBundleWorkerRequestSchema = z.strictObject({ kind: z.literal("analyze"), + semantic_precision: javaScriptSemanticPrecisionSchema.exactOptional(), id: z.uuid(), source: z.strictObject({ bytes: z.number().int().min(0).max(WEB_BUNDLE_WORKER_LIMITS.sourceBytes), diff --git a/src/browser/WebBundleAnalysisWorkerServer.ts b/src/browser/WebBundleAnalysisWorkerServer.ts index e2259e7ff6..130817fdcd 100644 --- a/src/browser/WebBundleAnalysisWorkerServer.ts +++ b/src/browser/WebBundleAnalysisWorkerServer.ts @@ -1,3 +1,4 @@ +import type { JavaScriptSemanticPrecisionInput } from "../domain/javascript/javascriptSemanticPrecision.js"; import { fitJavaScriptRouteSourceFacts } from "../domain/javascript/javascriptRouteResultBudget.js"; import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -26,7 +27,10 @@ import { } from "./WebBundleAnalysisWorkerProtocol.js"; export interface WebBundleScriptAnalyzer { - (script: IncludedScript): WebBundleScriptAnalysis; + ( + script: IncludedScript, + precision?: JavaScriptSemanticPrecisionInput, + ): WebBundleScriptAnalysis; } class WebBundleWorkerInputError extends Error {} @@ -127,7 +131,9 @@ const analyzeJob = async ( await send({ kind: "phase", id: request.id, phase: "analyze" }); let result: WebBundleScriptAnalysis; try { - result = webBundleScriptAnalysisSchema.parse(analyze(script)); + result = webBundleScriptAnalysisSchema.parse( + analyze(script, request.semantic_precision), + ); } catch (cause) { throw new WebBundleWorkerOutputError( `Web bundle analyzer returned an invalid result: ${cause instanceof Error ? cause.message : String(cause)}`, diff --git a/src/browser/WebSourceMapFetcher.test.ts b/src/browser/WebSourceMapFetcher.test.ts index 0bc5cc1c26..114a5e7e23 100644 --- a/src/browser/WebSourceMapFetcher.test.ts +++ b/src/browser/WebSourceMapFetcher.test.ts @@ -169,3 +169,41 @@ it.each([ }); }, ); + +it("recovers a fetched map by raising its exact structural admission on the next call", async () => { + const text = JSON.stringify({ + version: 3, + sources: ["a.ts"], + sourcesContent: ["export const a = 1;"], + names: [], + mappings: "AAAA", + }); + const selected = { + ...input, + allowed_origins: [new URL(request.fetchUrl).origin], + }; + const host = { fetch: async () => new Response(text) }; + const refused = await fetchWebSourceMapsOk( + [request], + { ...selected, source_map_limits: { decoded_records: 4 } }, + undefined, + host, + ); + expect(refused.items[0]).toMatchObject({ + status: "fetch_failed", + limitation: expect.stringContaining( + "source_map_limits.decoded_records is 4", + ), + }); + const recovered = await fetchWebSourceMapsOk( + [request], + { ...selected, source_map_limits: { decoded_records: 5 } }, + undefined, + host, + ); + expect(recovered.items[0]).toMatchObject({ + status: "included", + original_sources: [{ source: "https://app.example.test/a.ts" }], + mappings: [{ generated_line: 1, generated_column: 0 }], + }); +}); diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index bade5de12f..264df32d63 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -43,11 +43,12 @@ import { SourceMapFormatFailure, type SourceMapLeaf, } from "../javascript/sourceMaps/SourceMapFormat.js"; +import { selectedBrowserDeadlineMs } from "../domain/browserDeadline.js"; import { - noRuntimeOverrideSentence, - selectedBrowserDeadlineMs, -} from "../domain/browserDeadline.js"; -import { WEB_SOURCE_MAP_LIMITS } from "../domain/webSourceLocation.js"; + selectedSourceMapLimits, + sourceMapLimitRecovery, + type SourceMapLimits, +} from "../domain/sourceMapLimits.js"; export interface WebSourceMapRequest { readonly scriptKey: string; @@ -70,9 +71,7 @@ interface SourceMapFetchHost { const SOURCE_MAP_FETCH_TIMEOUT_MS = DEFAULT_SOURCE_MAP_FETCH_TIMEOUT_MS; const sourceMapFetchDeadlineLimitation = (timeoutMs: number): string => - `Source-map fetch exceeded its ${String(timeoutMs)} ms source_map_fetch_timeout_ms deadline (fetch_failed). This is an item limitation, not a thrown analysis timeout, and not the source-map decode deadline. ${noRuntimeOverrideSentence(`The ${String(SOURCE_MAP_RESPONSE_BYTES)}-byte response budget stays fixed.`, "source_map_fetch_timeout_ms")}`; -const SOURCE_MAP_RESPONSE_BYTES = 64 * 1024 * 1024; -const SOURCE_MAP_OUTPUT_BYTES = WEB_SOURCE_MAP_LIMITS.outputBytes; + `Source-map fetch exceeded its ${String(timeoutMs)} ms source_map_fetch_timeout_ms deadline (fetch_failed). This is an item limitation, not a thrown analysis timeout, and not the source-map decode deadline. Response, representation and decoder budgets are independent source_map_limits fields.`; const SOURCE_MAP_FAILURE_LIMITATION_RESERVE = 512; const SOURCE_MAP_COLLECTION_ENVELOPE_RESERVE = 1_024; @@ -86,15 +85,17 @@ type ResponseIdentity = { readonly representation: "fetch-response-body-bytes"; }; type FailedSourceMapStatus = "fetch_failed" | "invalid" | "policy_filtered"; +interface SourceMapCollectionBudget { + records: number; + expandedBytes: number; + resolvedSourceBytes: number; + readonly limits: SourceMapLimits; +} interface SourceMapDecodeContext { readonly signal: AbortSignal | undefined; readonly deadlineAt: number; readonly deadlineLimitation: string; - readonly budget: { - records: number; - expandedBytes: number; - resolvedSourceBytes: number; - }; + readonly budget: SourceMapCollectionBudget; } /** Fetch and validate approved source maps without browser credentials. */ @@ -119,6 +120,7 @@ export const fetchWebSourceMaps = async ( items: [], }), ); + const limits = selectedSourceMapLimits(input.source_map_limits); const items: WebSourceMapItem[] = []; const operationController = new AbortController(); const fetchTimeoutMs = @@ -148,6 +150,7 @@ export const fetchWebSourceMaps = async ( records: 0, expandedBytes: SOURCE_MAP_COLLECTION_ENVELOPE_RESERVE, resolvedSourceBytes: 0, + limits, }, }; let limitation: string | undefined; @@ -184,9 +187,12 @@ export const fetchWebSourceMaps = async ( const reservation = sourceMapFailureReservation(request) + 1; if ( reservation > - SOURCE_MAP_OUTPUT_BYTES - budget.decodedRecords.expandedBytes + limits.outputBytes - budget.decodedRecords.expandedBytes ) { - limitation = sourceMapOutputLimitation(requests.length - items.length); + limitation = sourceMapOutputLimitation( + requests.length - items.length, + limits, + ); break; } const expandedBytesBeforeItem = budget.decodedRecords.expandedBytes; @@ -206,9 +212,10 @@ export const fetchWebSourceMaps = async ( ); if (item.status !== "included" && item.status !== "partial") { const failureBytes = jsonByteLength(item) + 1; - if (failureBytes > SOURCE_MAP_OUTPUT_BYTES - expandedBytesBeforeItem) { + if (failureBytes > limits.outputBytes - expandedBytesBeforeItem) { limitation = sourceMapOutputLimitation( requests.length - items.length, + limits, ); break; } @@ -236,7 +243,10 @@ export const fetchWebSourceMaps = async ( ), ); if (cause instanceof SourceMapFormatFailure && cause.reason === "limit") - limitation = sourceMapOutputLimitation(requests.length - items.length); + limitation = sourceMapOutputLimitation( + requests.length - items.length, + limits, + ); else throw cause; } finally { clearTimeout(timeout); @@ -261,8 +271,11 @@ export const fetchWebSourceMaps = async ( ); }; -const sourceMapOutputLimitation = (unprocessed: number): string => - `source-map output budget could not retain results for ${String(unprocessed)} requested map${unprocessed === 1 ? "" : "s"}. their results are unknown.`; +const sourceMapOutputLimitation = ( + unprocessed: number, + limits: SourceMapLimits, +): string => + `source-map output budget could not retain results for ${String(unprocessed)} requested map${unprocessed === 1 ? "" : "s"}. their results are unknown. ${sourceMapLimitRecovery("output_bytes", limits.outputBytes)}`; const sourceMapsStatus = ( items: readonly WebSourceMapItem[], @@ -287,9 +300,9 @@ class SourceMapDeadlineError extends Error { } class SourceMapSizeLimitError extends Error { - constructor() { + constructor(maxBytes: number) { super( - `Source-map response exceeded the ${String(SOURCE_MAP_RESPONSE_BYTES)}-byte retained budget. ${noRuntimeOverrideSentence("That response budget stays fixed.", "source_map_fetch_timeout_ms")}`, + `Source-map response exceeded its retained budget. ${sourceMapLimitRecovery("fetch_bytes", maxBytes)}`, ); this.name = "SourceMapSizeLimitError"; } @@ -320,11 +333,7 @@ const fetchOne = async ( retainedBytes: number; deadlineAt: number; deadlineLimitation: string; - decodedRecords: { - records: number; - expandedBytes: number; - resolvedSourceBytes: number; - }; + decodedRecords: SourceMapCollectionBudget; }, ): Promise => { if (!approvedUrl(request.fetchUrl, input.allowed_origins)) @@ -364,7 +373,7 @@ const fetchOne = async ( await response.body?.cancel(); } catch (cleanupCause: unknown) { if (cause instanceof SourceMapFormatFailure && cause.reason === "limit") - throw sourceMapOutputError(cause, { + throw sourceMapOutputError(cause, budget.decodedRecords.limits, { cause: new AggregateError( [cause, cleanupCause], "Source-map response cleanup failed after its output exceeded capacity.", @@ -397,7 +406,7 @@ const fetchOne = async ( const body = await readBoundedText( response, budget, - host.maxResponseBytes ?? SOURCE_MAP_RESPONSE_BYTES, + host.maxResponseBytes ?? budget.decodedRecords.limits.fetchBytes, signal, ); completedResponseIdentity = body.responseIdentity; @@ -531,8 +540,8 @@ const readBoundedText = async ( if (done) break; const nextBytes = value?.byteLength ?? 0; if (budget.retainedBytes + bytes + nextBytes > maxBytes) { - await reader.cancel(new SourceMapSizeLimitError()); - throw new SourceMapSizeLimitError(); + await reader.cancel(new SourceMapSizeLimitError(maxBytes)); + throw new SourceMapSizeLimitError(maxBytes); } if (value !== undefined) chunks.push(value); bytes += nextBytes; @@ -662,11 +671,11 @@ const normalizeSourceMap = ( SOURCE_MAP_FAILURE_ENVELOPE_BYTES + SOURCE_MAP_SUCCESS_ENVELOPE_BYTES + textBytes; - if (successBytes > SOURCE_MAP_OUTPUT_BYTES) + if (successBytes > context.budget.limits.outputBytes) return emptySourceMapItem( request, "fetch_failed", - "JSON-escaped response text and map metadata would exceed the remaining space in the shared 32 MiB source-map output budget; JSON parsing and source-map decoding were not attempted.", + `JSON-escaped response text and map metadata exceed the remaining source-map output budget; JSON parsing and decoding were not attempted. ${sourceMapLimitRecovery("output_bytes", context.budget.limits.outputBytes)}`, fetchedUrl, responseIdentity, ); @@ -691,6 +700,7 @@ const normalizeSourceMap = ( const inspected = inspectSourceMapValue(parsedJson.value, { profile: "browser-collection", budget: context.budget, + limits: context.budget.limits, check, }); const sourcesByRaw = new Map(); @@ -1043,7 +1053,7 @@ const sanitizeSource = (value: string): string => { const prepareResolvedSources = ( leaves: readonly SourceMapLeaf[], mapUrl: string, - budget: { readonly expandedBytes: number; resolvedSourceBytes: number }, + budget: SourceMapCollectionBudget, sanitizedByResolved: Map, check: () => void, ): ReadonlyMap> => { @@ -1071,14 +1081,14 @@ const prepareResolvedSources = ( resolved = candidate; } const bytes = Buffer.byteLength(resolved, "utf8"); - if (bytes > SOURCE_MAP_RESPONSE_BYTES - resolvedBytes) - throw resolvedSourceLimitFailure(); + if (bytes > budget.limits.fetchBytes - resolvedBytes) + throw resolvedSourceLimitFailure(budget.limits); resolvedBytes += bytes; sourcesForRoot.set(sourceKey, resolved); const sanitized = cachedSanitized(sanitizedByResolved, resolved); estimate += jsonByteLength(sanitized); - if (estimate > WEB_SOURCE_MAP_LIMITS.outputBytes) - throw expandedOutputFailure(); + if (estimate > budget.limits.outputBytes) + throw expandedOutputFailure(budget.limits); } } budget.resolvedSourceBytes = resolvedBytes; @@ -1086,36 +1096,41 @@ const prepareResolvedSources = ( }; const consumeExpandedBytes = ( - budget: { expandedBytes: number }, + budget: SourceMapCollectionBudget, bytes: number, ): void => { - if (bytes > WEB_SOURCE_MAP_LIMITS.outputBytes - budget.expandedBytes) - throw expandedOutputFailure(); + if (bytes > budget.limits.outputBytes - budget.expandedBytes) + throw expandedOutputFailure(budget.limits); budget.expandedBytes += bytes; }; -const expandedOutputFailure = (): SourceMapFormatFailure => +const expandedOutputFailure = ( + limits: SourceMapLimits, +): SourceMapFormatFailure => new SourceMapFormatFailure( "limit", - `Expanded source-map evidence exceeds the ${String(WEB_SOURCE_MAP_LIMITS.outputBytes)}-byte public representation budget. ${noRuntimeOverrideSentence("That representation budget stays fixed.", "source_map_fetch_timeout_ms")}`, + `Expanded source-map evidence exceeds public representation admission. ${sourceMapLimitRecovery("output_bytes", limits.outputBytes)}`, ); const sourceMapOutputError = ( cause: SourceMapFormatFailure, + limits: SourceMapLimits, options?: AnalysisErrorOptions, ): AnalysisResourceConstraintError => new AnalysisResourceConstraintError( "analyze_web_bundle", "transport", cause.message, - { maximum_source_map_output_bytes: SOURCE_MAP_OUTPUT_BYTES }, + { maximum_source_map_output_bytes: limits.outputBytes }, { cause, ...options }, ); -const resolvedSourceLimitFailure = (): SourceMapFormatFailure => +const resolvedSourceLimitFailure = ( + limits: SourceMapLimits, +): SourceMapFormatFailure => new SourceMapFormatFailure( "limit", - `Resolved source-map identities exceed the ${String(SOURCE_MAP_RESPONSE_BYTES / (1024 * 1024))} MiB decoder representation budget. ${noRuntimeOverrideSentence("That representation budget stays fixed.", "source_map_fetch_timeout_ms")}`, + `Resolved source-map identities exceed decoder representation admission. ${sourceMapLimitRecovery("fetch_bytes", limits.fetchBytes)}`, ); const artifactMetadataBytes = (mediaType: string, text: string): number => diff --git a/src/browser/assets/WebArtifactReads.ts b/src/browser/assets/WebArtifactReads.ts index a0b469160e..8e1c59dc6a 100644 --- a/src/browser/assets/WebArtifactReads.ts +++ b/src/browser/assets/WebArtifactReads.ts @@ -35,6 +35,8 @@ export interface WebArtifactReadContext { readonly targetPath: string; /** Whether the caller named this file, rather than a manifest deriving it. */ readonly callerSelected: boolean; + /** Exact resource recovery supplied by the selected artifact owner. */ + readonly limitRecovery?: string; } /** A malformed producer document, distinct from host I/O failures. */ @@ -199,8 +201,15 @@ const webArtifactPrimaryError = ( context.operation, cause.reason, cause.details, - cause.message, - { cause }, + cause.reason === "limit" && context.limitRecovery !== undefined + ? `${cause.message} ${context.limitRecovery}` + : cause.message, + { + cause, + ...(cause.reason === "limit" && context.limitRecovery !== undefined + ? { remediationAction: context.limitRecovery } + : {}), + }, ); if ( cause instanceof Error && diff --git a/src/browser/assets/WebSourceLocationArtifacts.ts b/src/browser/assets/WebSourceLocationArtifacts.ts index 78a14dd779..7a09025d6e 100644 --- a/src/browser/assets/WebSourceLocationArtifacts.ts +++ b/src/browser/assets/WebSourceLocationArtifacts.ts @@ -1,11 +1,12 @@ +import { + selectedSourceMapLimits, + sourceMapLimitRecovery, +} from "../../domain/sourceMapLimits.js"; import type { WebSourceLocationArtifactPort } from "../../application/WebSourceLocationPorts.js"; import type { WebScriptArtifactPort } from "../../application/WebScriptArtifactPort.js"; import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; -import { - WEB_SOURCE_MAP_LIMITS, - type WebSourceLocationInput, -} from "../../domain/webSourceLocation.js"; +import type { WebSourceLocationInput } from "../../domain/webSourceLocation.js"; import { err, ok } from "../../domain/result.js"; import { WebArtifactCleanup } from "../WebArtifactCleanup.js"; import { LocalWebScriptArtifacts } from "./LocalWebScriptArtifacts.js"; @@ -59,6 +60,10 @@ export class LocalWebSourceLocationArtifacts implements WebSourceLocationArtifac field: ["source_map", "path"], targetPath: input.source_map.path, callerSelected: true, + limitRecovery: sourceMapLimitRecovery( + "map_bytes", + selectedSourceMapLimits(input.source_map_limits).mapBytes, + ), }, options?.signal, ), @@ -83,7 +88,7 @@ export class LocalWebSourceLocationArtifacts implements WebSourceLocationArtifac const data = await acquisition.read( "source-map", input.source_map.path, - WEB_SOURCE_MAP_LIMITS.mapBytes, + selectedSourceMapLimits(input.source_map_limits).mapBytes, options?.signal, this.#resources, ); diff --git a/src/browser/history/CaptureBudgets.ts b/src/browser/history/CaptureBudgets.ts new file mode 100644 index 0000000000..2553582dc4 --- /dev/null +++ b/src/browser/history/CaptureBudgets.ts @@ -0,0 +1,18 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { + selectedWebNetworkCaptureBudgets, + type WebNetworkCaptureBudgets, +} from "../../domain/webNetworkCapture.js"; + +const accepted = new AsyncLocalStorage>(); +const defaults = Object.freeze(selectedWebNetworkCaptureBudgets({})); + +/** A decoder operation and its asynchronous consumers share immutable admitted facts. */ +export const withCaptureBudgets = ( + budgets: WebNetworkCaptureBudgets, + run: () => T, +): T => accepted.run(Object.freeze({ ...budgets }), run); + +/** Standalone format readers use the same defaults as the public operation. */ +export const captureBudgets = (): Readonly => + accepted.getStore() ?? defaults; diff --git a/src/browser/history/CaptureFailures.ts b/src/browser/history/CaptureFailures.ts index 5afbf75aeb..32e1166647 100644 --- a/src/browser/history/CaptureFailures.ts +++ b/src/browser/history/CaptureFailures.ts @@ -1,3 +1,4 @@ +import { captureBudgets } from "./CaptureBudgets.js"; import { analysisSettingLimit } from "../../domain/analysisSettingDefinitions.js"; import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; @@ -22,8 +23,9 @@ import { import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { - WEB_NETWORK_CAPTURE_LIMITS, selectedWebNetworkCaptureTimeoutMs, + webNetworkCaptureBudgetFacts, + webNetworkCaptureBudgetRecovery, webNetworkCaptureTimeoutRecovery, type InspectWebNetworkCaptureInput, } from "../../domain/webNetworkCapture.js"; @@ -162,6 +164,24 @@ export const historicalCaptureFailure = ( return new AnalysisCancelledError(OPERATION, { cause: artifactFileCancellationCause(cause), }); + if ( + phase === "decoder" && + origin === "owned-or-classified" && + cause instanceof ArtifactReaderFailure && + cause.reason === "limit" + ) + return new AnalysisOutputError(OPERATION, cause.message, { + cause, + remediationAction: webNetworkCaptureBudgetRecovery( + input, + "max_reply_bytes", + ), + diagnostics: { + boundary: "historical-capture-reply", + maximum_reply_bytes: captureBudgets().outputBytes, + budget: webNetworkCaptureBudgetFacts(input, "max_reply_bytes"), + }, + }); if ( phase !== "decoder" && origin === "owned-or-classified" && @@ -172,22 +192,34 @@ export const historicalCaptureFailure = ( ) return new AnalysisInputError( OPERATION, - { cause, ...stableArtifactInputRecovery(cause) }, + { + cause, + ...(cause.reason === "limit" + ? { + diagnostics: { + budget: webNetworkCaptureBudgetFacts(input, "max_input_bytes"), + }, + remediationAction: webNetworkCaptureBudgetRecovery( + input, + "max_input_bytes", + ), + } + : stableArtifactInputRecovery(cause)), + }, [ { path: inputPath, reason: cause.reason === "limit" ? "out_of_range" : "invalid_format", message: cause.reason === "limit" - ? `${cause.message}. Select a ${sourceLabel} within the ${String(WEB_NETWORK_CAPTURE_LIMITS.inputBytes)}-byte input budget. This release has no runtime override. capture_timeout_ms does not raise it.` + ? `${cause.message}. Increase max_input_bytes above ${String(captureBudgets().inputBytes)} on the next call without restarting REA, or select a smaller complete ${sourceLabel}. Reply, worker heap and capture_timeout_ms controls remain separate.` : cause.message, ...(cause.reason === "limit" ? { expected: { [descriptorRead ? "maximum_descriptor_set_bytes" - : "maximum_capture_bytes"]: - WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + : "maximum_capture_bytes"]: captureBudgets().inputBytes, }, } : {}), @@ -275,8 +307,8 @@ export const captureMemoryFailure = ( boundary: "historical-capture-decoder", capture_path: input.capture_path, format: input.format, - maximum_input_bytes: WEB_NETWORK_CAPTURE_LIMITS.inputBytes, - maximum_reply_bytes: WEB_NETWORK_CAPTURE_LIMITS.outputBytes, + maximum_input_bytes: captureBudgets().inputBytes, + maximum_reply_bytes: captureBudgets().outputBytes, ...(input.format !== "mitmproxy" && NODE_CAPTURE_HEAP_ENFORCED ? { old_generation_heap_mib: heap.oldGenerationMiB, diff --git a/src/browser/history/CaptureJson.ts b/src/browser/history/CaptureJson.ts index a73a934f45..c1b971301e 100644 --- a/src/browser/history/CaptureJson.ts +++ b/src/browser/history/CaptureJson.ts @@ -1,6 +1,6 @@ +import { captureBudgets } from "./CaptureBudgets.js"; import { visit } from "jsonc-parser"; import { LosslessNumber } from "lossless-json"; -import { WEB_NETWORK_CAPTURE_LIMITS } from "../../domain/webNetworkCapture.js"; import { CaptureFormatError } from "./CaptureFormatError.js"; import { TextDecoder } from "node:util"; @@ -65,10 +65,10 @@ export const parseCaptureJson = ( // container add no level, so empty and scalar-bearing containers agree. const admitContainer = (): void => { admitNode(); - if (frames.length >= WEB_NETWORK_CAPTURE_LIMITS.depth) + if (frames.length >= captureBudgets().depth) throw new CaptureFormatError( "input-limit", - `${format} exceeds the 64-level complete-evidence nesting budget. This release has no runtime override.`, + `${format} exceeds max_nesting_depth (${String(captureBudgets().depth)} levels).`, valuePointer(), ); }; diff --git a/src/browser/history/HarCapture.test.ts b/src/browser/history/HarCapture.test.ts index e1323bdefa..38a0c06097 100644 --- a/src/browser/history/HarCapture.test.ts +++ b/src/browser/history/HarCapture.test.ts @@ -963,7 +963,10 @@ it.each([ const enclosing = depth - 3; const text = `{"log":{"version":"1.2","creator":{"name":"fixture","version":"1"},"entries":[],"_extension":${"[".repeat(enclosing)}${innermost}${"]".repeat(enclosing)}}}`; if (accepted) expect(decodeHarCapture(text, []).total_records).toBe(0); - else expect(() => decodeHarCapture(text, [])).toThrow("nesting budget"); + else + expect(() => decodeHarCapture(text, [])).toThrow( + expect.objectContaining({ reason: "input-limit" }), + ); }, ); @@ -975,7 +978,7 @@ it("rejects excessive extension nesting instead of returning a partial capture", JSON.stringify({ ...historicalHar(), _extension: extension }), [], ), - ).toThrow("nesting budget"); + ).toThrow(expect.objectContaining({ reason: "input-limit" })); }); it.each([ diff --git a/src/browser/history/HarCapture.ts b/src/browser/history/HarCapture.ts index 5abc0c9d4b..ec12dceceb 100644 --- a/src/browser/history/HarCapture.ts +++ b/src/browser/history/HarCapture.ts @@ -1,12 +1,12 @@ +import { captureBudgets } from "./CaptureBudgets.js"; import { createHash } from "node:crypto"; import { LosslessNumber } from "lossless-json"; import { parseCaptureJson } from "./CaptureJson.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { isCanonicalBase64 } from "../../domain/stringPatterns.js"; -import { - WEB_NETWORK_CAPTURE_LIMITS, - type WebNetworkCapture, - type WebNetworkCaptureRecord, +import type { + WebNetworkCapture, + WebNetworkCaptureRecord, } from "../../domain/webNetworkCapture.js"; import { CaptureFormatError } from "./CaptureFormatError.js"; import { @@ -140,10 +140,10 @@ const projectHar = ( depth: number, parent?: Readonly>, ): JsonValue => { - if (depth > WEB_NETWORK_CAPTURE_LIMITS.depth) + if (depth > captureBudgets().depth) throw new CaptureFormatError( "input-limit", - "HAR exceeds the 64-level complete-evidence nesting budget. This release has no runtime override.", + `HAR exceeds max_nesting_depth (${String(captureBudgets().depth)} levels).`, pointer, ); if ( diff --git a/src/browser/history/HistoricalCaptureBudgets.test.ts b/src/browser/history/HistoricalCaptureBudgets.test.ts new file mode 100644 index 0000000000..b06e27ff6f --- /dev/null +++ b/src/browser/history/HistoricalCaptureBudgets.test.ts @@ -0,0 +1,194 @@ +import { createRequire } from "node:module"; +import { writeFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; +import { WebNetworkCaptureService } from "../../application/WebNetworkCaptureService.js"; +import { HistoricalCaptureDecoder } from "./HistoricalCaptureDecoder.js"; +import { HAR_CAPTURE_PROVIDER_IDENTITY } from "./CaptureProviderIdentities.js"; +import { historicalHar } from "../../../tests/fixtures/historicalHar.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { jsonValueValidationIssue } from "../../domain/jsonValue.js"; +import { MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH } from "../../domain/webNetworkCapture.js"; + +const require = createRequire(import.meta.url); + +/** Exercise the real private decoder protocol and parsers from the source checkout. */ +const service = () => + new WebNetworkCaptureService( + new HistoricalCaptureDecoder( + [ + { + format: "har", + identity: HAR_CAPTURE_PROVIDER_IDENTITY, + async command(requestPath) { + return { + command: process.execPath, + arguments: [ + "--import", + require.resolve("tsx"), + fileURLToPath( + new URL("./NodeCaptureProcess.ts", import.meta.url), + ), + requestPath, + "har", + ], + }; + }, + }, + ], + process.env, + ), + ); + +it.each(["max_input_bytes", "max_reply_bytes", "max_nesting_depth"] as const)( + "recovers the same complete capture by changing only %s", + async (control) => { + const root = await createTestTempDirectory("rea-capture-budgets-"); + const path = join(root, "capture.har"); + const fixture = historicalHar(); + const extended = { + ...fixture, + _extension: { sensitive: "exclude-me", payload: "x".repeat(4000) }, + }; + if (control === "max_nesting_depth") { + let nested: unknown = "retained-leaf"; + for (let index = 0; index < 70; index++) nested = { child: nested }; + Object.assign(extended._extension, { nested }); + } + const text = JSON.stringify(extended); + await writeFile(path, text); + const selected = + control === "max_input_bytes" + ? Buffer.byteLength(text) - 1 + : control === "max_reply_bytes" + ? 512 + : 64; + const raised = + control === "max_input_bytes" + ? Buffer.byteLength(text) + : control === "max_reply_bytes" + ? 16384 + : 128; + const capture = service(); + onTestFinished(() => capture.close()); + const request = { + capture_path: path, + format: "har", + sensitive_values: ["exclude-me"], + }; + const refused = await capture.inspect({ ...request, [control]: selected }); + if (refused.ok) + throw new Error("Selected budget unexpectedly admitted the capture"); + const error = projectAnalysisError(refused.error); + expect(error.remediation.action).toContain(control); + expect(error.details).toMatchObject({ + diagnostics: { + budget: { + parameter: control, + effective_value: selected, + source: "request", + rea_restart_required: false, + }, + }, + }); + const recovered = await capture.inspect({ ...request, [control]: raised }); + if (!recovered.ok) throw recovered.error; + expect(recovered.value.parameters[control]).toBe(raised); + expect(recovered.value.normalized_result).toMatchObject({ + artifact: { path, bytes: Buffer.byteLength(text) }, + total_records: 1, + records: [ + { + ordinal: 0, + location: { kind: "json-pointer", pointer: "/log/entries/0" }, + }, + ], + }); + expect(JSON.stringify(recovered.value)).not.toContain("exclude-me"); + // Admission changes cannot make malformed source valid. + await writeFile(path, '{"log":'); + const malformed = await capture.inspect({ ...request, [control]: raised }); + if (malformed.ok) + throw new Error("Malformed capture unexpectedly admitted"); + expect(malformed.error._tag).toBe("AnalysisInputError"); + }, +); + +it("keeps concurrent capture policies independent", async () => { + const root = await createTestTempDirectory("rea-capture-budget-isolation-"); + const path = join(root, "capture.har"); + let nested: unknown = "leaf"; + for (let index = 0; index < 70; index++) nested = { child: nested }; + await writeFile( + path, + JSON.stringify({ ...historicalHar(), _extension: nested }), + ); + const capture = service(); + onTestFinished(() => capture.close()); + const request = { capture_path: path, format: "har" }; + const [narrow, wide] = await Promise.all([ + capture.inspect({ ...request, max_nesting_depth: 64 }), + capture.inspect({ ...request, max_nesting_depth: 128 }), + ]); + if (narrow.ok) + throw new Error("Narrow operation borrowed the concurrent budget"); + expect(projectAnalysisError(narrow.error).details).toMatchObject({ + issues: [{ expected: { maximum_capture_nesting: 64 } }], + }); + if (!wide.ok) throw wide.error; + expect(wide.value.parameters.max_nesting_depth).toBe(128); + expect(wide.value.normalized_result).toMatchObject({ total_records: 1 }); +}); + +it("reserves reply and Evidence nesting before accepting a deep capture policy", async () => { + const root = await createTestTempDirectory("rea-capture-depth-envelope-"); + const path = join(root, "capture.har"); + const writeDepth = async (depth: number) => { + let nested: unknown = "retained-leaf"; + // The HAR document root contributes the first container level. + for (let index = 1; index < depth; index++) nested = { child: nested }; + await writeFile( + path, + JSON.stringify({ ...historicalHar(), _vendor: nested }), + ); + }; + const capture = service(); + onTestFinished(() => capture.close()); + const request = { capture_path: path, format: "har" }; + // Previously this accepted policy returned invalid full Evidence, or threw + // during normalized-result construction, after a successful decoder run. + await writeDepth(998); + const invalidPolicy = await capture.inspect({ + ...request, + max_nesting_depth: 1000, + }); + if (invalidPolicy.ok) + throw new Error("Unrepresentable capture policy was accepted"); + expect(projectAnalysisError(invalidPolicy.error)).toMatchObject({ + code: "invalid_request", + details: { issues: [{ path: ["max_nesting_depth"], maximum: 996 }] }, + }); + await writeDepth(MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH); + const accepted = await capture.inspect({ + ...request, + max_nesting_depth: MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH, + }); + if (!accepted.ok) throw accepted.error; + expect(jsonValueValidationIssue(accepted.value)).toBeUndefined(); + expect(accepted.value.normalized_result).toMatchObject({ total_records: 1 }); + await writeDepth(MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH + 1); + const exhausted = await capture.inspect({ + ...request, + max_nesting_depth: MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH, + }); + if (exhausted.ok) + throw new Error("Capture exceeded its selected nesting allowance"); + expect(projectAnalysisError(exhausted.error)).toMatchObject({ + details: { + diagnostics: { budget: { effective_value: 996, maximum: 996 } }, + }, + remediation: { action: expect.stringContaining("at its maximum") }, + }); +}); diff --git a/src/browser/history/HistoricalCaptureDecoder.ts b/src/browser/history/HistoricalCaptureDecoder.ts index 421ab8b21f..169230559e 100644 --- a/src/browser/history/HistoricalCaptureDecoder.ts +++ b/src/browser/history/HistoricalCaptureDecoder.ts @@ -1,3 +1,4 @@ +import { captureBudgets, withCaptureBudgets } from "./CaptureBudgets.js"; import type { FileSnapshotPartialObservation } from "../../domain/fileSnapshotObservation.js"; import { decodeCaptureProtobuf, @@ -49,6 +50,9 @@ import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorClean import { err, ok, type Result } from "../../domain/result.js"; import { WEB_NETWORK_CAPTURE_LIMITS, + selectedWebNetworkCaptureBudgets, + webNetworkCaptureBudgetFacts, + webNetworkCaptureBudgetRecovery, selectedWebNetworkCaptureTimeoutMs, type InspectWebNetworkCaptureInput, type WebNetworkCapture, @@ -104,25 +108,30 @@ export class HistoricalCaptureDecoder { input: InspectWebNetworkCaptureInput, options?: ExecutionOptions, ): Promise> { - try { - return await this.#resources.run(() => - this.inspectAdmitted(input, options), - ); - } catch (cause: unknown) { - const error = historicalCaptureFailure( - input, - cause, - "capture-read", - options, - this.#heap, - ); - return err( - redactExplicitFailure( - withCaptureReaderCleanup(error, cause), - input.sensitive_values, - ), - ); - } + return withCaptureBudgets( + selectedWebNetworkCaptureBudgets(input), + async () => { + try { + return await this.#resources.run(() => + this.inspectAdmitted(input, options), + ); + } catch (cause: unknown) { + const error = historicalCaptureFailure( + input, + cause, + "capture-read", + options, + this.#heap, + ); + return err( + redactExplicitFailure( + withCaptureReaderCleanup(error, cause), + input.sensitive_values, + ), + ); + } + }, + ); } private async inspectAdmitted( @@ -154,7 +163,7 @@ export class HistoricalCaptureDecoder { ); const snapshot = await readStableArtifact( input.capture_path, - WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + captureBudgets().inputBytes, options?.signal, undefined, this.#resources, @@ -181,6 +190,7 @@ export class HistoricalCaptureDecoder { snapshot_path: snapshotPath, reply_path: replyPath, sensitive_values: input.sensitive_values, + budgets: captureBudgets(), }), { mode: 0o600, flag: "wx" }, ); @@ -234,28 +244,54 @@ export class HistoricalCaptureDecoder { }, ]); if (reply.reason === "input-limit") - throw new AnalysisInputError(OPERATION, undefined, [ + throw new AnalysisInputError( + OPERATION, { - path: ["capture_path", reply.pointer], - reason: "out_of_range", - message: reply.message, - expected: { - maximum_capture_nesting: WEB_NETWORK_CAPTURE_LIMITS.depth, + diagnostics: { + budget: webNetworkCaptureBudgetFacts( + input, + "max_nesting_depth", + ), }, + remediationAction: webNetworkCaptureBudgetRecovery( + input, + "max_nesting_depth", + ), }, - ]); + [ + { + path: ["capture_path", reply.pointer], + reason: "out_of_range", + message: reply.message, + expected: { + maximum_capture_nesting: captureBudgets().depth, + }, + }, + ], + ); if (reply.reason === "expanded-limit") - throw new AnalysisInputError(OPERATION, undefined, [ + throw new AnalysisInputError( + OPERATION, { - path: ["capture_path", reply.pointer], - reason: "out_of_range", - message: reply.message, - expected: { - maximum_expanded_capture_bytes: - WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + diagnostics: { + budget: webNetworkCaptureBudgetFacts(input, "max_input_bytes"), }, + remediationAction: webNetworkCaptureBudgetRecovery( + input, + "max_input_bytes", + ), }, - ]); + [ + { + path: ["capture_path", reply.pointer], + reason: "out_of_range", + message: reply.message, + expected: { + maximum_expanded_capture_bytes: captureBudgets().inputBytes, + }, + }, + ], + ); if (reply.reason === "resource-limit") throw captureMemoryFailure( input, @@ -279,11 +315,15 @@ export class HistoricalCaptureDecoder { reply.message, ); throw new AnalysisOutputError(OPERATION, reply.message, { - userMessage: `The capture decoder refused a complete reply exceeding ${String(WEB_NETWORK_CAPTURE_LIMITS.outputBytes)} bytes: ${reply.message}`, - remediationAction: `The ${String(WEB_NETWORK_CAPTURE_LIMITS.outputBytes)}-byte complete-reply allowance has no runtime override. Select a smaller capture exported by its producer when a narrower observation is acceptable. Record selection, delivery mode and worker memory do not raise it; no completed capture Evidence is available from this refusal.`, + userMessage: `The capture decoder refused a complete reply exceeding ${String(captureBudgets().outputBytes)} bytes: ${reply.message}`, + remediationAction: webNetworkCaptureBudgetRecovery( + input, + "max_reply_bytes", + ), diagnostics: { - maximum_reply_bytes: WEB_NETWORK_CAPTURE_LIMITS.outputBytes, + maximum_reply_bytes: captureBudgets().outputBytes, boundary: "historical-capture-reply", + budget: webNetworkCaptureBudgetFacts(input, "max_reply_bytes"), }, }); } @@ -309,7 +349,7 @@ export class HistoricalCaptureDecoder { "This is retained producer evidence, without live browser transaction IDs, scenario provenance, execution attribution or deployment authenticity. Recorded URLs are never fetched.", "Every record is decoded before selection; no partial success is returned on malformed input or a resource limit.", "Known authentication headers/cookies and explicitly marked literal values are excluded. Unknown extension names do not establish sensitivity.", - `Complete evidence is bounded by a ${String(WEB_NETWORK_CAPTURE_LIMITS.inputBytes)}-byte input budget, a ${String(WEB_NETWORK_CAPTURE_LIMITS.outputBytes)}-byte reply budget, and a ${String(WEB_NETWORK_CAPTURE_LIMITS.depth)}-level nesting cap. Those have no runtime override. capture_timeout_ms is ${String(selectedWebNetworkCaptureTimeoutMs(input.capture_timeout_ms))} ms (omitted means ${String(WEB_NETWORK_CAPTURE_LIMITS.timeoutMs)}). ${nodeMemoryLimitation(reportedHeap)} Mitmproxy decoding uses a 768 MiB Linux address-space limit.`, + `Complete evidence is bounded by a ${String(captureBudgets().inputBytes)}-byte input budget, a ${String(captureBudgets().outputBytes)}-byte reply budget, and a ${String(captureBudgets().depth)}-level nesting cap. Set max_input_bytes, max_reply_bytes or max_nesting_depth explicitly for the next call to change those budgets without restarting REA. capture_timeout_ms is ${String(selectedWebNetworkCaptureTimeoutMs(input.capture_timeout_ms))} ms (omitted means ${String(WEB_NETWORK_CAPTURE_LIMITS.timeoutMs)}). ${nodeMemoryLimitation(reportedHeap)} Mitmproxy decoding uses a 768 MiB Linux address-space limit.`, ], }; if (input.format === "qlog") { @@ -352,13 +392,14 @@ export class HistoricalCaptureDecoder { "capacity", cause.message, { - maximum_encoded_capture_bytes: - WEB_NETWORK_CAPTURE_LIMITS.outputBytes, + maximum_encoded_capture_bytes: captureBudgets().outputBytes, }, { retryable: false, - remediationAction: - "Select smaller records, or inspect the retained original source bytes.", + remediationAction: webNetworkCaptureBudgetRecovery( + input, + "max_reply_bytes", + ), }, ) : cause instanceof CaptureFormatError diff --git a/src/browser/history/HistoricalCaptureReply.ts b/src/browser/history/HistoricalCaptureReply.ts index 9f9f035862..1adad77f90 100644 --- a/src/browser/history/HistoricalCaptureReply.ts +++ b/src/browser/history/HistoricalCaptureReply.ts @@ -1,3 +1,4 @@ +import { captureBudgets } from "./CaptureBudgets.js"; import { z } from "zod"; import type { JsonValue } from "../../domain/jsonValue.js"; import { CaptureRedaction, capturePointer } from "./CaptureRedaction.js"; @@ -16,10 +17,7 @@ import { } from "../../domain/analysisErrorCore.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { withCaptureReaderCleanup } from "./CaptureFailures.js"; -import { - WEB_NETWORK_CAPTURE_LIMITS, - webNetworkCapturePayloadSchema, -} from "../../domain/webNetworkCapture.js"; +import { webNetworkCapturePayloadSchema } from "../../domain/webNetworkCapture.js"; const replySchema = z.discriminatedUnion("ok", [ z.strictObject({ @@ -56,7 +54,7 @@ export const readHistoricalCaptureReply = async ( try { file = await readStableArtifact( path, - WEB_NETWORK_CAPTURE_LIMITS.outputBytes, + captureBudgets().outputBytes, signal, undefined, resources, diff --git a/src/browser/history/NodeCaptureProcess.ts b/src/browser/history/NodeCaptureProcess.ts index 1bcbda93ca..1887785dbe 100644 --- a/src/browser/history/NodeCaptureProcess.ts +++ b/src/browser/history/NodeCaptureProcess.ts @@ -1,13 +1,15 @@ import { readFile, writeFile } from "node:fs/promises"; import { z } from "zod"; import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; -import { WEB_NETWORK_CAPTURE_LIMITS } from "../../domain/webNetworkCapture.js"; +import { webNetworkCaptureBudgetsSchema } from "../../domain/webNetworkCapture.js"; +import { captureBudgets, withCaptureBudgets } from "./CaptureBudgets.js"; import { CaptureFormatError } from "./CaptureFormatError.js"; const requestSchema = z.strictObject({ snapshot_path: z.string(), reply_path: z.string(), sensitive_values: z.array(z.string().min(1)), + budgets: webNetworkCaptureBudgetsSchema, }); /** Parse untrusted captures in an owned heap and return one complete bounded reply. */ @@ -16,56 +18,58 @@ const main = async (): Promise => { const request = requestSchema.parse( JSON.parse(await readFile(process.argv[2] ?? "", "utf8")), ); - let reply: unknown; - try { - let value; - if (format !== "saz") { - const { decodeCaptureText } = await import("./CaptureJson.js"); - const decoder = - format === "har" - ? (await import("./HarCapture.js")).decodeHarCapture - : (await import("./QlogCapture.js")).decodeQlogCapture; - const snapshot = await readStableArtifact( - request.snapshot_path, - WEB_NETWORK_CAPTURE_LIMITS.inputBytes, - ); - value = decoder( - decodeCaptureText(snapshot.bytes, format === "har" ? "HAR" : "qlog"), - request.sensitive_values, - ); - } else { - const { decodeSazCapture } = await import("./SazCapture.js"); - value = await decodeSazCapture( - request.snapshot_path, - request.sensitive_values, - ); + await withCaptureBudgets(request.budgets, async () => { + let reply: unknown; + try { + let value; + if (format !== "saz") { + const { decodeCaptureText } = await import("./CaptureJson.js"); + const decoder = + format === "har" + ? (await import("./HarCapture.js")).decodeHarCapture + : (await import("./QlogCapture.js")).decodeQlogCapture; + const snapshot = await readStableArtifact( + request.snapshot_path, + captureBudgets().inputBytes, + ); + value = decoder( + decodeCaptureText(snapshot.bytes, format === "har" ? "HAR" : "qlog"), + request.sensitive_values, + ); + } else { + const { decodeSazCapture } = await import("./SazCapture.js"); + value = await decodeSazCapture( + request.snapshot_path, + request.sensitive_values, + ); + } + reply = { ok: true, value }; + } catch (cause: unknown) { + reply = + cause instanceof CaptureFormatError + ? { + ok: false, + reason: cause.reason, + message: cause.message, + pointer: cause.pointer, + } + : { + ok: false, + reason: "decoder", + message: `Capture decoder failed unexpectedly (${cause instanceof Error ? cause.name : "unknown exception"}).`, + pointer: "", + }; } - reply = { ok: true, value }; - } catch (cause: unknown) { - reply = - cause instanceof CaptureFormatError - ? { - ok: false, - reason: cause.reason, - message: cause.message, - pointer: cause.pointer, - } - : { - ok: false, - reason: "decoder", - message: `Capture decoder failed unexpectedly (${cause instanceof Error ? cause.name : "unknown exception"}).`, - pointer: "", - }; - } - let text = JSON.stringify(reply); - if (Buffer.byteLength(text) > WEB_NETWORK_CAPTURE_LIMITS.outputBytes) - text = JSON.stringify({ - ok: false, - reason: "limit", - message: - "Capture reply exceeds the complete-evidence output budget. This release has no runtime override.", - pointer: "", - }); - await writeFile(request.reply_path, text, { flag: "wx", mode: 0o600 }); + let text = JSON.stringify(reply); + if (Buffer.byteLength(text) > captureBudgets().outputBytes) + text = JSON.stringify({ + ok: false, + reason: "limit", + message: + "Capture reply exceeds max_reply_bytes; raise that per-call allowance and retry after cleanup.", + pointer: "", + }); + await writeFile(request.reply_path, text, { flag: "wx", mode: 0o600 }); + }); }; await main(); diff --git a/src/browser/history/QlogCapture.test.ts b/src/browser/history/QlogCapture.test.ts index 9224bfc3b7..cf15b976c0 100644 --- a/src/browser/history/QlogCapture.test.ts +++ b/src/browser/history/QlogCapture.test.ts @@ -441,7 +441,10 @@ it.each([ const enclosing = depth - 4; const text = `{"qlog_format":"JSON","qlog_version":"0.3","traces":[{"events":[],"ext":${"[".repeat(enclosing)}${innermost}${"]".repeat(enclosing)}}]}`; if (accepted) expect(facts(text).decoded.total_records).toBe(0); - else expect(() => facts(text)).toThrow("nesting budget"); + else + expect(() => facts(text)).toThrow( + expect.objectContaining({ reason: "input-limit" }), + ); }, ); @@ -454,7 +457,7 @@ it("rejects other qlog containers/versions and excessive nesting, accepts empty extension: JSON.parse("[".repeat(65) + "0" + "]".repeat(65)), }), ), - ).toThrow("nesting budget"); + ).toThrow(expect.objectContaining({ reason: "input-limit" })); expect( facts('{"qlog_format":"JSON","qlog_version":"0.3","traces":[]}').traces, ).toEqual([]); diff --git a/src/browser/history/SazCapture.ts b/src/browser/history/SazCapture.ts index a64ad3e2dc..4f2113f377 100644 --- a/src/browser/history/SazCapture.ts +++ b/src/browser/history/SazCapture.ts @@ -1,12 +1,10 @@ +import { captureBudgets } from "./CaptureBudgets.js"; import { ZipArtifactReader } from "../../artifacts/ZipArtifactReader.js"; import { ArtifactReaderFailure, type ArtifactEntry, } from "../../artifacts/ArtifactReader.js"; -import { - WEB_NETWORK_CAPTURE_LIMITS, - type WebNetworkCaptureRecord, -} from "../../domain/webNetworkCapture.js"; +import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js"; import { SAZ_CAPTURE_PROVIDER_IDENTITY } from "./CaptureProviderIdentities.js"; import { CaptureFormatError } from "./CaptureFormatError.js"; import { decodeSazHttp } from "./SazHttp.js"; @@ -35,7 +33,7 @@ export const decodeSazCapture = async ( const reader = new ZipArtifactReader( snapshotPath, "zip", - WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + captureBudgets().inputBytes, ); try { const groups = new Map(); @@ -64,10 +62,10 @@ export const decodeSazCapture = async ( throw new CaptureFormatError("format", "SAZ member size is invalid."); declaredBytes += entry.declaredSize; } - if (declaredBytes > WEB_NETWORK_CAPTURE_LIMITS.inputBytes) + if (declaredBytes > captureBudgets().inputBytes) throw new CaptureFormatError( "expanded-limit", - "SAZ expanded members exceed the 32 MiB complete-evidence budget. This release has no runtime override.", + `SAZ expanded members exceed max_input_bytes (${String(captureBudgets().inputBytes)} bytes); raise that per-call allowance and retry.`, ); inventory.push({ path: entry.path, diff --git a/src/browser/history/SazMetadata.ts b/src/browser/history/SazMetadata.ts index 1a74c6ab0d..7b1a9aa4a2 100644 --- a/src/browser/history/SazMetadata.ts +++ b/src/browser/history/SazMetadata.ts @@ -1,7 +1,7 @@ +import { captureBudgets } from "./CaptureBudgets.js"; import type { Node } from "@xmldom/xmldom"; import { parseXmlDocument } from "../../domain/xmlDocument.js"; import type { JsonValue } from "../../domain/jsonValue.js"; -import { WEB_NETWORK_CAPTURE_LIMITS } from "../../domain/webNetworkCapture.js"; import { decodeXmlPlistText, XmlPlistTextDecodeError, @@ -61,10 +61,10 @@ export const decodeSazMetadata = (bytes: Buffer): JsonValue => { "SAZ metadata must have a Session document element.", ); const visit = (node: Node, depth: number): JsonValue => { - if (depth > WEB_NETWORK_CAPTURE_LIMITS.depth) + if (depth > captureBudgets().depth) throw new CaptureFormatError( "input-limit", - "SAZ metadata exceeds the complete-evidence nesting budget. This release has no runtime override.", + `SAZ metadata exceeds max_nesting_depth (${String(captureBudgets().depth)} levels).`, ); const children: JsonValue[] = []; for (let child = node.firstChild; child !== null; child = child.nextSibling) diff --git a/src/browser/history/SazProjection.ts b/src/browser/history/SazProjection.ts index 8896b04511..63ebc7e15d 100644 --- a/src/browser/history/SazProjection.ts +++ b/src/browser/history/SazProjection.ts @@ -1,10 +1,8 @@ +import { captureBudgets } from "./CaptureBudgets.js"; import { createHash } from "node:crypto"; import { isUtf8 } from "node:buffer"; import type { JsonValue } from "../../domain/jsonValue.js"; -import { - WEB_NETWORK_CAPTURE_LIMITS, - type WebNetworkCaptureRecord, -} from "../../domain/webNetworkCapture.js"; +import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js"; import { CaptureRedaction, capturePointer } from "./CaptureRedaction.js"; import { CaptureFormatError } from "./CaptureFormatError.js"; @@ -27,10 +25,10 @@ export const projectSazRecord = ( depth: number, parent?: Readonly>, ): JsonValue => { - if (depth > WEB_NETWORK_CAPTURE_LIMITS.depth) + if (depth > captureBudgets().depth) throw new CaptureFormatError( "input-limit", - "SAZ projection exceeds the complete-evidence nesting budget. This release has no runtime override.", + `SAZ projection exceeds max_nesting_depth (${String(captureBudgets().depth)} levels).`, pointer, ); if (parent !== undefined && redactor.harCredential(pointer, parent)) diff --git a/src/browser/history/protobuf/ProtobufCapture.ts b/src/browser/history/protobuf/ProtobufCapture.ts index c6f9bd5ff4..86cb3c2574 100644 --- a/src/browser/history/protobuf/ProtobufCapture.ts +++ b/src/browser/history/protobuf/ProtobufCapture.ts @@ -1,3 +1,4 @@ +import { captureBudgets } from "../CaptureBudgets.js"; import { isAbsolute } from "node:path"; import { CaptureFormatError } from "../CaptureFormatError.js"; import { @@ -17,11 +18,10 @@ import { AnalysisCapabilityUnavailableError, } from "../../../domain/analysisErrorCore.js"; import { redactExplicitText } from "../../../domain/explicitSensitiveValues.js"; -import { - WEB_NETWORK_CAPTURE_LIMITS, - type InspectWebNetworkCaptureInput, - type WebNetworkCapture, - type WebNetworkCaptureRecord, +import type { + InspectWebNetworkCaptureInput, + WebNetworkCapture, + WebNetworkCaptureRecord, } from "../../../domain/webNetworkCapture.js"; import type { JsonValue } from "../../../domain/jsonValue.js"; import { err, ok, type Result } from "../../../domain/result.js"; @@ -418,13 +418,13 @@ export const prepareCaptureProtobuf = async ( continue; } if ( - field.bytes > WEB_NETWORK_CAPTURE_LIMITS.inputBytes || + field.bytes > captureBudgets().inputBytes || field.content_base64.length > - Math.ceil(WEB_NETWORK_CAPTURE_LIMITS.inputBytes / 3) * 4 + Math.ceil(captureBudgets().inputBytes / 3) * 4 ) throw new CaptureFormatError( "limit", - "Protobuf body exceeds the admitted 32 MiB source bound", + `Protobuf body exceeds max_input_bytes (${String(captureBudgets().inputBytes)} bytes)`, ); const body = Buffer.from(field.content_base64, "base64"); if ( @@ -542,7 +542,7 @@ export const decodeCaptureProtobuf = async ( ]); const snapshot = await readStableArtifact( path, - WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + captureBudgets().inputBytes, signal, undefined, resources, diff --git a/src/browser/history/protobuf/ProtobufFraming.ts b/src/browser/history/protobuf/ProtobufFraming.ts index 6c93feee8f..44a2f1679d 100644 --- a/src/browser/history/protobuf/ProtobufFraming.ts +++ b/src/browser/history/protobuf/ProtobufFraming.ts @@ -1,6 +1,6 @@ +import { captureBudgets } from "../CaptureBudgets.js"; import { err, ok } from "../../../domain/result.js"; import type { ProtobufDecoding } from "../../../domain/protobufWire.js"; -import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../domain/webNetworkCapture.js"; import { CaptureFormatError } from "../CaptureFormatError.js"; import { ProtobufBudget } from "./ProtobufWire.js"; @@ -92,10 +92,10 @@ export const prepareProtobufBody = async ( }, signal?: AbortSignal, ): Promise => { - if (body.length > WEB_NETWORK_CAPTURE_LIMITS.inputBytes) + if (body.length > captureBudgets().inputBytes) throw new CaptureFormatError( "limit", - "Protobuf body exceeds the admitted 32 MiB source bound", + `Protobuf body exceeds max_input_bytes (${String(captureBudgets().inputBytes)} bytes)`, ); const work = new ProtobufBudget(signal); await work.checkpoint(); diff --git a/src/browser/history/protobuf/ProtobufSchema.ts b/src/browser/history/protobuf/ProtobufSchema.ts index 6a18d6108a..cd22847c97 100644 --- a/src/browser/history/protobuf/ProtobufSchema.ts +++ b/src/browser/history/protobuf/ProtobufSchema.ts @@ -1,3 +1,4 @@ +import { captureBudgets } from "../CaptureBudgets.js"; import { err, ok, type Result } from "../../../domain/result.js"; import { createFileRegistry, @@ -18,7 +19,6 @@ import { } from "@bufbuild/protobuf/wkt"; import { BinaryReader } from "@bufbuild/protobuf/wire"; import type { ProtobufField } from "../../../domain/protobufWire.js"; -import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../domain/webNetworkCapture.js"; import { CaptureFormatError } from "../CaptureFormatError.js"; import { decodeProtobufWire, @@ -260,7 +260,7 @@ export class ProtobufSchema implements WireSchema { try { descriptor = fromBinary(FileDescriptorSetSchema, bytes, { readUnknownFields: true, - recursionLimit: WEB_NETWORK_CAPTURE_LIMITS.depth, + recursionLimit: captureBudgets().depth, }); } catch (cause) { return err({ diff --git a/src/browser/history/protobuf/ProtobufWire.ts b/src/browser/history/protobuf/ProtobufWire.ts index d6e2bdad7a..af341b3886 100644 --- a/src/browser/history/protobuf/ProtobufWire.ts +++ b/src/browser/history/protobuf/ProtobufWire.ts @@ -1,19 +1,19 @@ +import { captureBudgets } from "../CaptureBudgets.js"; import { err, ok, type Result } from "../../../domain/result.js"; import { setImmediate } from "node:timers/promises"; import type { ProtobufField } from "../../../domain/protobufWire.js"; import { CaptureFormatError } from "../CaptureFormatError.js"; -import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../domain/webNetworkCapture.js"; /** One charge covers all selected records, including temporary interpretation strings. */ export class ProtobufBudget { - #remaining = WEB_NETWORK_CAPTURE_LIMITS.outputBytes; + #remaining = captureBudgets().outputBytes; #work = 0; constructor(readonly signal?: AbortSignal) {} charge(bytes: number): void { if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.#remaining) throw new CaptureFormatError( "limit", - "Protobuf decoding exceeds the shared 96 MiB encoded-result and working-set budget.", + `Protobuf decoding exceeds max_reply_bytes (${String(captureBudgets().outputBytes)} bytes); raise that per-call allowance and retry.`, ); this.#remaining -= bytes; } @@ -115,7 +115,7 @@ const messageCandidate = async ( if (!parsed.ok) return false; const field = parsed.value; if (field.type === 3) { - if (groups.length >= WEB_NETWORK_CAPTURE_LIMITS.depth) return false; + if (groups.length >= captureBudgets().depth) return false; groups.push(field.number); } else if (field.type === 4 && groups.pop() !== field.number) return false; start = field.next; @@ -189,8 +189,10 @@ export const decodeProtobufWire = async ( return group === number ? { end: next, payloadEnd: at } : fail("Unexpected or mismatched end-group tag"); - if (type === 3 && depth >= WEB_NETWORK_CAPTURE_LIMITS.depth) - return fail("Group nesting exceeds the 64-level decoder budget"); + if (type === 3 && depth >= captureBudgets().depth) + return fail( + `Group nesting exceeds max_nesting_depth (${String(captureBudgets().depth)} levels)`, + ); let value = parsed.value.value?.toString(); if (type === 1 || type === 5) { let integer = 0n; @@ -265,11 +267,10 @@ export const decodeProtobufWire = async ( next = nested.end; } else if (type === 2 && schema?.nested(field)) { const error = - depth >= WEB_NETWORK_CAPTURE_LIMITS.depth + depth >= captureBudgets().depth ? { offset: payloadOffset, - reason: - "Schema message nesting exceeds the 64-level decoder budget", + reason: `Schema message nesting exceeds max_nesting_depth (${String(captureBudgets().depth)} levels)`, } : (await walk(payloadOffset, next, field, depth + 1)).error; // The outer length is an observed boundary. A bad interpretation of its diff --git a/src/captures/TsharkCaptureProvider.ts b/src/captures/TsharkCaptureProvider.ts index a66b2571c0..0532c62877 100644 --- a/src/captures/TsharkCaptureProvider.ts +++ b/src/captures/TsharkCaptureProvider.ts @@ -1,3 +1,7 @@ +import { + BOUNDED_PRODUCER_LIMITS as LIMITS, + resolveProducerLimits, +} from "../domain/producerResourceLimits.js"; import { TSHARK_PROVIDER_IDENTITY } from "./TsharkProviderIdentity.js"; import { artifactFileCancellationCause } from "../artifacts/ArtifactFileOperation.js"; import { @@ -10,7 +14,6 @@ import { FIELD_RECORD_BUDGET, } from "./parseFieldDiscovery.js"; import { - BOUNDED_PRODUCER_LIMITS as LIMITS, prlimitArgumentVector, readLinuxResourceLimits, type LinuxResourceLimits, @@ -131,8 +134,10 @@ export class TsharkCaptureProvider implements PacketCapturePort { ): Promise> { const operation = "path" in input ? OPERATION : "inspect_packet_fields"; // TShark 4.6.9 emits a 22.3 MB glossary; keep capture output capped at 4 MiB. - const limits = - "path" in input ? LIMITS : { ...LIMITS, output_bytes: 32 * 1024 * 1024 }; + const limits = resolveProducerLimits( + "path" in input ? LIMITS : { ...LIMITS, output_bytes: 32 * 1024 * 1024 }, + input.producer_limits, + ); let phase: "configuration" | "workspace" | "producer" | "integrity" = "configuration"; let capturedOutput: AnalysisCapturedOutput | undefined; @@ -308,15 +313,16 @@ export class TsharkCaptureProvider implements PacketCapturePort { limitations: [ "Protocol and field registrations describe the selected installation, not traffic observed in a capture. Duplicate abbreviations remain distinct producer records with source line numbers.", "Each request probes again without caching. The private empty personal profile does not fingerprint or isolate installed libraries, global configuration or plugins; they remain trusted.", - "No capture file, interface or device is opened. Resource ceilings do not constitute an OS sandbox. Glossary output is capped at 32 MiB and selected record representation at 8 MiB before expansion.", + "No capture file, interface or device is opened. Resource ceilings do not constitute an OS sandbox. Selected producer output budget is reported in limits.output_bytes and can be changed with producer_limits.output_bytes. Selected record representation remains independently capped at 8 MiB before expansion.", ], }), ); } else { const selected = await this.#resources.readSelectedArtifact({ path: input.path, - maximumBytes: LIMITS.input_bytes, - allowanceName: "the fixed packet-capture input allowance", + maximumBytes: limits.input_bytes, + allowanceName: "the packet-capture input allowance", + budgetParameter: "producer_limits.input_bytes", observedOutput: capturedOutput, signal: options?.signal, }); @@ -350,7 +356,7 @@ export class TsharkCaptureProvider implements PacketCapturePort { phase = "integrity"; resourcePath = snapshotPath; if ( - (await this.#resources.read(snapshotPath, LIMITS.input_bytes)) + (await this.#resources.read(snapshotPath, limits.input_bytes)) .sha256 !== selected.sha256 ) throw new AnalysisArtifactChangedError( @@ -402,7 +408,7 @@ export class TsharkCaptureProvider implements PacketCapturePort { "Packet limits count input packets before display filtering. Filtered results do not establish whole-capture or device traffic absence. Two-pass reassembly cannot recover bytes missing from the recording or beyond the read limit.", "Repeated field values remain ordered string arrays; raw packet bytes are included only when explicitly selected as a supported field. Export index/score metadata are omitted. All stderr, including warnings on exit zero, is retained.", "A private empty personal configuration is used. Installed libraries, global configuration and dissector plugins are trusted and not fully fingerprinted. File-only analysis and disabled name resolution do not constitute an OS sandbox.", - "Linux prlimit enforces soft and hard ceilings before execution, preserving stricter inherited limits: at most 1 GiB address space per process, 64 MiB per file, 30 CPU seconds and no core dumps. Effective limits are reported. Resident input is capped at 32 MiB, each command at 30 wall-clock seconds and combined output at 4 MiB. Output overflow fails. These are per-process/per-file limits, not aggregate memory or filesystem quotas; caller-supplied tools and the limiter are trusted. REA retains the tighter inherited limit. This release has no runtime override.", + "Linux prlimit enforces the reported per-process address-space, per-file and CPU limits before execution with no core dumps, preserving tighter inherited soft and hard limits. Input, command deadline and retained output budgets are reported in limits; producer_limits can change their requested defaults on the next call without restart. Output overflow fails. Parser and transport bounds remain independent. These are per-process/per-file limits, not aggregate memory or filesystem quotas; caller-supplied tools and the limiter are trusted.", ], }), ); @@ -504,6 +510,7 @@ export class TsharkCaptureProvider implements PacketCapturePort { limits, effectiveLimits, operation, + "producer_limits", ); if (phase === "workspace") return privateWorkspaceFailure( diff --git a/src/cli/apktoolCommands.ts b/src/cli/apktoolCommands.ts index d4689938a4..92b3eaea41 100644 --- a/src/cli/apktoolCommands.ts +++ b/src/cli/apktoolCommands.ts @@ -76,6 +76,23 @@ export const registerApktoolCommands = ( decodeTimeoutMs: cliDecimalField( apktoolInputSchemas.decode_android_resources.shape.decode_timeout_ms, ), + stringOffset: cliDecimalField( + apktoolInputSchemas.decode_android_resources.shape.string_offset, + ), + stringLimit: cliDecimalField( + apktoolInputSchemas.decode_android_resources.shape.string_limit, + ), + maxTotalStrings: cliDecimalField( + apktoolInputSchemas.decode_android_resources.shape.max_total_strings, + ), + maxStringTableBytes: cliDecimalField( + apktoolInputSchemas.decode_android_resources.shape + .max_string_table_bytes, + ), + maxTotalStringTableBytes: cliDecimalField( + apktoolInputSchemas.decode_android_resources.shape + .max_total_string_table_bytes, + ), strings: z .boolean() .default( @@ -104,6 +121,23 @@ export const registerApktoolCommands = ( ...(options.decodeTimeoutMs === undefined ? {} : { decode_timeout_ms: options.decodeTimeoutMs }), + ...(options.stringOffset === undefined + ? {} + : { string_offset: options.stringOffset }), + ...(options.stringLimit === undefined + ? {} + : { string_limit: options.stringLimit }), + ...(options.maxTotalStrings === undefined + ? {} + : { max_total_strings: options.maxTotalStrings }), + ...(options.maxStringTableBytes === undefined + ? {} + : { max_string_table_bytes: options.maxStringTableBytes }), + ...(options.maxTotalStringTableBytes === undefined + ? {} + : { + max_total_string_table_bytes: options.maxTotalStringTableBytes, + }), }, args.path, ), diff --git a/src/cli/artifactCommands.ts b/src/cli/artifactCommands.ts index 33ffbce6bb..ec98f5004d 100644 --- a/src/cli/artifactCommands.ts +++ b/src/cli/artifactCommands.ts @@ -16,6 +16,8 @@ import { keyedArchiveInputSchema } from "../domain/apple/keyedArchive.js"; import { localPathStringSchema } from "../domain/localPath.js"; import { z } from "incur"; import { cliDecimalField } from "./options.js"; +import { producerLimitsOption } from "./producerLimitsOption.js"; +import { artifactReaderLimitsInputSchema } from "../domain/artifactReaderLimits.js"; import { artifactExtractionInputSchema } from "../contracts/artifactToolContracts.js"; @@ -32,6 +34,13 @@ const keyedArchiveInputBinding = { fields: { path: "archive" }, } as const; +const artifactLimitsOption = producerLimitsOption( + artifactReaderLimitsInputSchema, + "artifact-limits", +).describe( + "JSON artifact producer admissions: bun, dotnet_bundle, node_sea, godot, and expanded_bytes. Omitted fields keep defaults; changes apply on the next call without restart. Independent of summary/paging and client delivery limits.", +); + export const registerArtifactCommands = ( cli: CliInstance, logger: Logger, @@ -254,6 +263,7 @@ const registerExtractionCommand = ( }), options: z.object({ // Incur needs an ordinary optional; keep the owning contract's metadata. + artifactLimits: artifactLimitsOption, outputRoot: artifactExtractionInputSchema.shape.output_root .unwrap() .optional() @@ -277,6 +287,9 @@ const registerExtractionCommand = ( "extract_artifact", { integrity_policy: options.integrityPolicy, + ...(options.artifactLimits === undefined + ? {} + : { artifact_limits: options.artifactLimits }), output_root: createArtifactExtractionDestination( options.outputRoot, ), @@ -306,6 +319,7 @@ const registerInspectionCommand = ( }), options: z.object({ ...analysisEvidenceDeliveryOptions, + artifactLimits: artifactLimitsOption, integrityPolicy: z .enum(["fail", "record-and-continue"]) .default("fail") @@ -327,7 +341,12 @@ const registerInspectionCommand = ( const complete = await runProviderAnalysis( args.path, "inspect_artifact", - { integrity_policy: options.integrityPolicy }, + { + integrity_policy: options.integrityPolicy, + ...(options.artifactLimits === undefined + ? {} + : { artifact_limits: options.artifactLimits }), + }, logger, signal, undefined, diff --git a/src/cli/binaryDiagnosticsCommands.ts b/src/cli/binaryDiagnosticsCommands.ts index 98ef19ae97..58c644c257 100644 --- a/src/cli/binaryDiagnosticsCommands.ts +++ b/src/cli/binaryDiagnosticsCommands.ts @@ -5,6 +5,8 @@ import { inspectBinaryHardeningInputSchema } from "../domain/native/binaryHarden import { inspectByteEntropyInputSchema } from "../domain/native/byteEntropy.js"; import { z } from "incur"; import { cliDecimalField } from "./options.js"; +import { producerLimitsOption } from "./producerLimitsOption.js"; +import { pwntoolsProducerLimitsSchema } from "../domain/native/pwntoolsResourceLimits.js"; import { createByteEntropyService, createRecordedCrashService, @@ -272,6 +274,12 @@ export const registerBinaryDiagnosticsCommands = ( maxEntries: cliDecimalField( inspectKernelModuleInputSchema.shape.max_entries, ), + maxModinfoKeyBytes: cliDecimalField( + inspectKernelModuleInputSchema.shape.max_modinfo_key_bytes, + ), + maxModinfoValueBytes: cliDecimalField( + inspectKernelModuleInputSchema.shape.max_modinfo_value_bytes, + ), }), run: ({ args, options }) => withCommandCancellation(async (signal) => { @@ -288,6 +296,8 @@ export const registerBinaryDiagnosticsCommands = ( path: absoluteCallerPath(args.path), max_file_bytes: options.maxFileBytes, max_entries: options.maxEntries, + max_modinfo_key_bytes: options.maxModinfoKeyBytes, + max_modinfo_value_bytes: options.maxModinfoValueBytes, }, { signal }, ), @@ -745,6 +755,7 @@ export const registerBinaryDiagnosticsCommands = ( ), }), options: z.object({ + producerLimits: producerLimitsOption(pwntoolsProducerLimitsSchema), pwntoolsTimeoutMs: cliDecimalField( inspectBinaryLayoutInputSchema.shape.pwntools_timeout_ms, ), @@ -755,6 +766,9 @@ export const registerBinaryDiagnosticsCommands = ( const result = await service.inspectAndClose( { path: absoluteCallerPath(args.path), + ...(options.producerLimits === undefined + ? {} + : { producer_limits: options.producerLimits }), ...(options.pwntoolsTimeoutMs === undefined ? {} : { pwntools_timeout_ms: options.pwntoolsTimeoutMs }), @@ -777,6 +791,7 @@ export const registerBinaryDiagnosticsCommands = ( ), }), options: z.object({ + producerLimits: producerLimitsOption(pwntoolsProducerLimitsSchema), debuggerContext: z .boolean() .default(false) @@ -792,6 +807,9 @@ export const registerBinaryDiagnosticsCommands = ( { path: absoluteCallerPath(args.path), include_debugger_context: options.debuggerContext, + ...(options.producerLimits === undefined + ? {} + : { producer_limits: options.producerLimits }), ...(options.pwntoolsTimeoutMs === undefined ? {} : { pwntools_timeout_ms: options.pwntoolsTimeoutMs }), diff --git a/src/cli/browserCommands.ts b/src/cli/browserCommands.ts index 93e5c966b9..4c8c7298ee 100644 --- a/src/cli/browserCommands.ts +++ b/src/cli/browserCommands.ts @@ -1,3 +1,8 @@ +import { parseCliJsonInput } from "../cliJsonInput.js"; +import { + fetchedSourceMapLimitCliOptions, + sourceMapLimitsFromCli, +} from "./sourceMapLimits.js"; import { projectCliAnalysisResult } from "./projectCliAnalysisResult.js"; import { webBundleDeliveryOptions } from "./javascriptRouteDelivery.js"; import { @@ -178,6 +183,12 @@ const registerBundleAnalysis = ( ...passiveBrowserScopeOptions, ...webBundleDeliveryOptions, observationMs: observationDuration(500), + semanticPrecision: z + .string() + .optional() + .describe( + "Inline JSON or JSON file of semantic_precision evaluator budgets for each inert script.", + ), fetchSourceMaps: z .boolean() .default(false) @@ -189,6 +200,7 @@ const registerBundleAnalysis = ( totalAnalysisTimeoutMs: cliDecimalField( analyzeWebBundleInputSchema.shape.total_analysis_timeout_ms, ), + ...fetchedSourceMapLimitCliOptions, sourceMapFetchTimeoutMs: cliDecimalField( analyzeWebBundleInputSchema.shape.source_map_fetch_timeout_ms, ), @@ -202,6 +214,15 @@ const registerBundleAnalysis = ( webRoutesDeliverySchema, ); if (!delivery.ok) return projectAnalysisError(delivery.error); + const precision = + options.semanticPrecision === undefined + ? undefined + : await parseCliJsonInput( + options.semanticPrecision, + "analyze_web_bundle", + signal, + ); + if (precision !== undefined && !precision.ok) return precision.error; const provider = createBrowserObservationProvider(); const input = { cdp_endpoint: args.endpoint, @@ -210,6 +231,10 @@ const registerBundleAnalysis = ( observation_ms: options.observationMs, include_script_sources: true, fetch_source_maps: options.fetchSourceMaps, + ...(precision === undefined + ? {} + : { semantic_precision: precision.value }), + ...sourceMapLimitsFromCli(options), ...(options.maxHeapMb === undefined ? {} : { max_heap_mb: options.maxHeapMb }), diff --git a/src/cli/captureCommands.ts b/src/cli/captureCommands.ts index a7a15c1d56..4a85077550 100644 --- a/src/cli/captureCommands.ts +++ b/src/cli/captureCommands.ts @@ -18,6 +18,7 @@ import { finishCliEvidenceCommand } from "./cliEvidenceCallerPath.js"; import { cliDecimalField, cliDecimalShape } from "./options.js"; import { localPathStringSchema } from "../domain/localPath.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; +import { producerLimitsOption } from "./producerLimitsOption.js"; export const registerCaptureCommands = ( cli: CliInstance, logger: Logger, @@ -27,11 +28,16 @@ export const registerCaptureCommands = ( "inspect" | "inspectFields" | "close" > = createPacketCaptureService(environment), ): void => { + const { producer_limits: discoveryLimits, ...discoveryShape } = + inspectPacketFieldsInputSchema.shape; + const { producer_limits: selectedLimits, ...selectedShape } = + packetCaptureOptionsSchema.shape; cli.command(CLI_COMMANDS.inspectPacketFields, { description: "Search installed TShark protocol and field definitions", options: z .object({ - ...cliDecimalShape(inspectPacketFieldsInputSchema.shape), + ...cliDecimalShape(discoveryShape), + producerLimits: producerLimitsOption(discoveryLimits.unwrap()), limit: cliDecimalField( z .number() @@ -54,13 +60,19 @@ export const registerCaptureCommands = ( message: "all and limit are mutually exclusive", }); }), - run: ({ options: { all, ...options } }) => + run: ({ options: { all, producerLimits, ...options } }) => withCommandCancellation((signal) => logCliCommand(logger, CLI_COMMANDS.inspectPacketFields, async () => finishAnalysisCommand( () => service.inspectFields( - { ...options, ...(all ? { limit: null } : {}) }, + { + ...options, + ...(producerLimits === undefined + ? {} + : { producer_limits: producerLimits }), + ...(all ? { limit: null } : {}), + }, { signal }, ), service, @@ -79,7 +91,8 @@ export const registerCaptureCommands = ( }), options: z .object({ - ...cliDecimalShape(packetCaptureOptionsSchema.shape), + ...cliDecimalShape(selectedShape), + producerLimits: producerLimitsOption(selectedLimits.unwrap()), match_limit: cliDecimalField( z .number() @@ -107,7 +120,7 @@ export const registerCaptureCommands = ( withinPacketCaptureSelectorBudget, packetCaptureSelectorBudgetIssue, ), - run: ({ args, options: { all_matches, ...options } }) => + run: ({ args, options: { all_matches, producerLimits, ...options } }) => withCommandCancellation((signal) => logCliCommand(logger, CLI_COMMANDS.inspectPacketCapture, async () => { return finishCliEvidenceCommand( @@ -116,6 +129,9 @@ export const registerCaptureCommands = ( service.inspect( { ...options, + ...(producerLimits === undefined + ? {} + : { producer_limits: producerLimits }), ...(all_matches ? { match_limit: null } : {}), path: absoluteCallerPath(args.path), }, diff --git a/src/cli/desktopCaptureCommands.ts b/src/cli/desktopCaptureCommands.ts index b833eaa392..7c74ced004 100644 --- a/src/cli/desktopCaptureCommands.ts +++ b/src/cli/desktopCaptureCommands.ts @@ -1,3 +1,6 @@ +import { desktopLibraryPolicyInputSchema } from "../domain/desktop/libraryPolicy.js"; +import { desktopViewPolicyInputSchema } from "../domain/desktop/desktopViewPolicy.js"; +import { safeParseJson } from "../domain/safeJson.js"; import { createInterface } from "node:readline"; import { z } from "incur"; @@ -27,11 +30,52 @@ import { SessionRequestQueue } from "../desktop/DesktopSessionRequests.js"; import { withCommandCancellation } from "./commandCancellation.js"; import type { CliInstance } from "./types.js"; +/** One coherent view policy; invalid JSON and unknown controls fail at admission. */ +const viewPolicyOption = z + .string() + .transform((text, context) => { + const parsed = safeParseJson(text); + if (!parsed.ok) { + context.addIssue({ + code: "custom", + message: "view-policy must be a JSON object", + }); + return z.NEVER; + } + return parsed.value; + }) + .pipe(desktopViewPolicyInputSchema) + .optional() + .describe( + "JSON budget overrides: file_bytes, sqlite_tables, sqlite_rows, sqlite_view_bytes, text_lines, xml_nodes, xml_depth, parsed_bytes, json_depth, zip_entries, zip_entry_bytes, zip_bytes, zip_ratio. Positive safe integers. Applies on this call and to exported readers; no restart. Does not recover bytes the capture never retained or raise Evidence/worker limits.", + ); + +/** Caller-selected inventory headroom; does not raise content-retention budgets. */ +const libraryPolicyOption = z + .string() + .transform((text, context) => { + const parsed = safeParseJson(text); + if (!parsed.ok) { + context.addIssue({ + code: "custom", + message: "library-policy must be a JSON object", + }); + return z.NEVER; + } + return parsed.value; + }) + .pipe(desktopLibraryPolicyInputSchema) + .optional() + .describe( + "JSON dependency inventory budgets: libraries (default 512, also per-image DT_NEEDED entries), search_path_entries (16 distinct offsets per DT_RUNPATH/DT_RPATH kind), string_bytes (4096 per string including NUL), table_bytes (1048576 per ELF program/dynamic table). Positive safe integers; applies to this capture without restart. Does not raise content or query-output budgets.", + ); + const positive = (fallback: number) => cliDecimalField(z.int().positive().default(fallback)); /** Options of `capture-desktop-application`. */ const captureOptions = z.object({ + libraryPolicy: libraryPolicyOption, desktopEntry: z .string() .min(1) @@ -150,8 +194,8 @@ const withStateReaders = async ( return finishAnalysisCommand( async () => { const result = await run({ - sqlite: (database, readSignal) => - readSqliteStateView(sqlite, database, readSignal), + sqlite: (database, readSignal, policy) => + readSqliteStateView(sqlite, database, readSignal, policy), dconf: (database, readSignal) => dumpDconfDatabase(database, environment, readSignal), }); @@ -271,6 +315,7 @@ const registerDesktopExportCommand = ( ), }), options: z.object({ + viewPolicy: viewPolicyOption, name: z .string() .optional() @@ -351,6 +396,7 @@ const registerDesktopExportCommand = ( ? undefined : absoluteCallerPath(options.screens), ...withheldRequestOf(options), + viewPolicy: options.viewPolicy, }, readers, signal, @@ -376,7 +422,7 @@ const registerDesktopTaskCommand = ( "Folder written by capture-desktop-application", ), }), - options: z.object(checkpointOptions), + options: z.object({ ...checkpointOptions, viewPolicy: viewPolicyOption }), run: ({ args, options }) => withCommandCancellation((signal) => logCliCommand( @@ -399,6 +445,7 @@ const registerDesktopTaskCommand = ( end: options.endCheckpoint, withhold: withholdOf(options), }, + options.viewPolicy, ), ); }, @@ -421,7 +468,16 @@ const registerDesktopStateCommand = ( "Folder written by capture-desktop-application", ), }), - run: ({ args }) => + options: z.object({ + viewPolicy: viewPolicyOption, + statePath: z + .array(z.string().min(1)) + .optional() + .describe( + "Exact original captured path to inspect (repeatable); omission examines every path", + ), + }), + run: ({ args, options }) => withCommandCancellation((signal) => logCliCommand( logger, @@ -438,6 +494,7 @@ const registerDesktopStateCommand = ( absoluteCallerPath(args.capture), readers, signal, + { viewPolicy: options.viewPolicy, paths: options.statePath }, ), ); }, @@ -477,6 +534,9 @@ export const registerDesktopCaptureCommands = ( output: options.output, statePaths: options.statePath ?? [], excludes: options.exclude ?? [], + ...(options.libraryPolicy === undefined + ? {} + : { libraryPolicy: options.libraryPolicy }), maxDurationSeconds: options.maxDurationSeconds, maxEntries: options.maxEntries, maxFileBytes: options.maxFileBytes, diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.ts index 3a86e6314b..73a4d4cd38 100644 --- a/src/cli/electronCommands.ts +++ b/src/cli/electronCommands.ts @@ -365,11 +365,24 @@ const registerJavaScriptApplicationCommand = ( signal, ); if (!sourcePaths.ok) return sourcePaths.error; + const precision = + options.semanticPrecision === undefined + ? undefined + : await parseCliJsonInput( + options.semanticPrecision, + "analyze_javascript_application", + signal, + ); + if (precision !== undefined && !precision.ok) + return precision.error; return runCliJavaScriptApplicationAnalysis( { // An empty path names no file; leave it for input validation. input_path: args.path === "" ? args.path : absoluteCallerPath(args.path), + ...(precision === undefined + ? {} + : { semantic_precision: precision.value }), format: options.artifactFormat, integrity_policy: options.integrityPolicy, ...(options.maxSemanticNodes === undefined diff --git a/src/cli/environmentCommands.ts b/src/cli/environmentCommands.ts index 9e4015b82b..68159b4e45 100644 --- a/src/cli/environmentCommands.ts +++ b/src/cli/environmentCommands.ts @@ -335,7 +335,7 @@ const conflictingRules = (): JsonValue => /** Inspect a HAR as inspect-web-network-capture does and serialize the Evidence as it prints it; a refusal is the command's output. */ const inspectBuildHar = async ( capturePath: string, - callerCapture: string, + callerInputPath: string, captureTimeoutMs: number | undefined, environment: Readonly, signal: AbortSignal, @@ -358,9 +358,7 @@ const inspectBuildHar = async ( : { capture_timeout_ms: captureTimeoutMs }), }, { signal }, - // Provenance keeps the caller's spelling. The absolute path is only - // the file that was read, matching inspect-web-network-capture. - { capture: callerCapture }, + { capture: callerInputPath }, ), service, "web-network-capture", diff --git a/src/cli/evmCommands.ts b/src/cli/evmCommands.ts index 836993d581..21e32b04e3 100644 --- a/src/cli/evmCommands.ts +++ b/src/cli/evmCommands.ts @@ -9,7 +9,11 @@ import { localPathStringSchema } from "../domain/localPath.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { finishCliEvidenceCommand } from "./cliEvidenceCallerPath.js"; import { cliDecimalField } from "./options.js"; -import { inspectEvmInterfaceInputSchema } from "../domain/evm/evmInterface.js"; +import { + inspectEvmInterfaceInputSchema, + evmProducerLimitsSchema, +} from "../domain/evm/evmInterface.js"; +import { producerLimitsOption } from "./producerLimitsOption.js"; /** Resolve operator paths and preserve explicit encoding through the shared workflow. */ export const registerEvmCommands = ( @@ -27,6 +31,7 @@ export const registerEvmCommands = ( .describe("Selected carrier representation"), }), options: z.object({ + producerLimits: producerLimitsOption(evmProducerLimitsSchema), timeoutMs: cliDecimalField( inspectEvmInterfaceInputSchema.shape.timeout_ms, ), @@ -41,6 +46,9 @@ export const registerEvmCommands = ( { path: absoluteCallerPath(args.path), encoding: args.encoding, + ...(options.producerLimits === undefined + ? {} + : { producer_limits: options.producerLimits }), ...(options.timeoutMs === undefined ? {} : { timeout_ms: options.timeoutMs }), diff --git a/src/cli/flutterCommands.ts b/src/cli/flutterCommands.ts index a5a6b51107..55869613f9 100644 --- a/src/cli/flutterCommands.ts +++ b/src/cli/flutterCommands.ts @@ -1,3 +1,4 @@ +import { safeParseJson } from "../domain/safeJson.js"; import { projectCliAnalysisResult } from "./projectCliAnalysisResult.js"; import { evidenceSchema, @@ -11,6 +12,7 @@ import { CLI_COMMANDS } from "../cliCommandNames.js"; import { logCliCommand } from "../cliLogging.js"; import { flutterInputSchemas, + dartPoolPolicySchema, type FlutterOperation, } from "../domain/flutter/flutterBuildAnalysis.js"; import type { Logger } from "pino"; @@ -58,11 +60,32 @@ export const registerFlutterCommands = ( }), optionSchema: z.object({ abi: flutterInputSchemas.inspect_dart_aot.shape.abi, + poolPolicy: z + .string() + .transform((text, context) => { + const parsed = safeParseJson(text); + if (!parsed.ok) { + context.addIssue({ + code: "custom", + message: "pool-policy must be a JSON object", + }); + return z.NEVER; + } + return parsed.value; + }) + .pipe(dartPoolPolicySchema) + .optional() + .describe( + "JSON printable-run selection: contains, uri_limit (500), token_limit (2000), run_bytes (1024), token_bytes (128). Filters apply before bounded collection; URI/token counts cover matches. Positive safe integers; next call, no restart. Does not raise APK/library/ELF/reply limits.", + ), }), operation: "inspect_dart_aot", toInput: (arguments_, options) => ({ path: arguments_.path, ...(options.abi === undefined ? {} : { abi: options.abi }), + ...(options.poolPolicy === undefined + ? {} + : { pool_policy: options.poolPolicy }), }), }, ]; diff --git a/src/cli/fridaCommands.ts b/src/cli/fridaCommands.ts index 098ccbd01f..f668c2554f 100644 --- a/src/cli/fridaCommands.ts +++ b/src/cli/fridaCommands.ts @@ -1,3 +1,4 @@ +import { fridaCapturePolicyInputSchema } from "../domain/fridaCapturePolicy.js"; import { projectCliAnalysisResult } from "./projectCliAnalysisResult.js"; import { z } from "incur"; @@ -176,6 +177,23 @@ export const registerFridaCommands = ( .array(z.string()) .optional() .describe("Arguments passed to a spawned target"), + captureSessionBytes: cliDecimalField( + fridaCapturePolicyInputSchema.shape.session_bytes, + ).describe( + "Session retention bytes including encoded payloads; select before starting the experiment", + ), + captureMessageTextBytes: cliDecimalField( + fridaCapturePolicyInputSchema.shape.message_text_bytes, + ).describe("Retained bytes per Frida error text field"), + captureMessageTypeBytes: cliDecimalField( + fridaCapturePolicyInputSchema.shape.message_type_bytes, + ).describe("Retained bytes per Frida message type"), + captureJsonNodes: cliDecimalField( + fridaCapturePolicyInputSchema.shape.json_nodes, + ).describe("Retained JSON nodes per Frida payload"), + captureJsonDepth: cliDecimalField( + fridaCapturePolicyInputSchema.shape.json_depth, + ).describe("Retained JSON nesting depth per Frida payload"), durationMs: cliDecimalField( z .number() @@ -274,6 +292,39 @@ export const registerFridaCommands = ( const result = await service.instrument( { ...targetInput, + ...([ + options.captureSessionBytes, + options.captureMessageTextBytes, + options.captureMessageTypeBytes, + options.captureJsonNodes, + options.captureJsonDepth, + ].every((value) => value === undefined) + ? {} + : { + captureLimits: { + ...(options.captureSessionBytes === undefined + ? {} + : { session_bytes: options.captureSessionBytes }), + ...(options.captureMessageTextBytes === undefined + ? {} + : { + message_text_bytes: + options.captureMessageTextBytes, + }), + ...(options.captureMessageTypeBytes === undefined + ? {} + : { + message_type_bytes: + options.captureMessageTypeBytes, + }), + ...(options.captureJsonNodes === undefined + ? {} + : { json_nodes: options.captureJsonNodes }), + ...(options.captureJsonDepth === undefined + ? {} + : { json_depth: options.captureJsonDepth }), + }, + }), source: options.scriptFile === undefined ? { sourceKind: "inline", source: options.script ?? "" } diff --git a/src/cli/goCommands.ts b/src/cli/goCommands.ts index e3ebc3daa4..dae1f82f7a 100644 --- a/src/cli/goCommands.ts +++ b/src/cli/goCommands.ts @@ -12,11 +12,7 @@ import { import { localPathStringSchema } from "../domain/localPath.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; import { machoSliceArchitectureSchema } from "../domain/apple/machoTargetSelection.js"; -import { - DEFAULT_MAX_GO_STRING_LITERALS, - MAX_GO_STRING_LITERALS_LIMIT, - goMaxStringLiteralsSchema, -} from "../domain/go/goBinary.js"; +import { goResourceBudgetFields } from "../domain/go/goBinary.js"; import { cliDecimalField } from "./options.js"; /** Resolve caller paths and inspect static build metadata through the shared workflow. */ @@ -42,10 +38,26 @@ export const registerGoCommands = (cli: CliInstance, logger: Logger): void => { .describe( "Also recover bounded inferred Go UTF-8 string literals from static (ptr, len) headers", ), + maxInputBytes: cliDecimalField(goResourceBudgetFields.max_input_bytes), + maxReportBytes: cliDecimalField(goResourceBudgetFields.max_report_bytes), + maxBuildInfoBytes: cliDecimalField( + goResourceBudgetFields.max_build_info_bytes, + ), + maxHeaderSearchBytes: cliDecimalField( + goResourceBudgetFields.max_header_search_bytes, + ), maxStringLiterals: cliDecimalField( - goMaxStringLiteralsSchema.optional(), - ).describe( - `Maximum distinct recovered string literals (1-${String(MAX_GO_STRING_LITERALS_LIMIT)}; default ${String(DEFAULT_MAX_GO_STRING_LITERALS)}; requires --include-string-literals)`, + goResourceBudgetFields.max_string_literals, + ), + maxStringLiteralBytes: cliDecimalField( + goResourceBudgetFields.max_string_literal_bytes, + ), + maxStringBytes: cliDecimalField(goResourceBudgetFields.max_string_bytes), + maxStringHeaderScanBytes: cliDecimalField( + goResourceBudgetFields.max_string_header_scan_bytes, + ), + maxStringCandidates: cliDecimalField( + goResourceBudgetFields.max_string_candidates, ), }), run: ({ args, options }) => @@ -70,9 +82,41 @@ export const registerGoCommands = (cli: CliInstance, logger: Logger): void => { include_string_literals: options.includeStringLiterals, }), + ...(options.maxInputBytes === undefined + ? {} + : { max_input_bytes: options.maxInputBytes }), + ...(options.maxReportBytes === undefined + ? {} + : { max_report_bytes: options.maxReportBytes }), + ...(options.maxBuildInfoBytes === undefined + ? {} + : { max_build_info_bytes: options.maxBuildInfoBytes }), + ...(options.maxHeaderSearchBytes === undefined + ? {} + : { + max_header_search_bytes: options.maxHeaderSearchBytes, + }), ...(options.maxStringLiterals === undefined ? {} : { max_string_literals: options.maxStringLiterals }), + ...(options.maxStringLiteralBytes === undefined + ? {} + : { + max_string_literal_bytes: + options.maxStringLiteralBytes, + }), + ...(options.maxStringBytes === undefined + ? {} + : { max_string_bytes: options.maxStringBytes }), + ...(options.maxStringHeaderScanBytes === undefined + ? {} + : { + max_string_header_scan_bytes: + options.maxStringHeaderScanBytes, + }), + ...(options.maxStringCandidates === undefined + ? {} + : { max_string_candidates: options.maxStringCandidates }), }, { signal }, ), diff --git a/src/cli/managedCommands.ts b/src/cli/managedCommands.ts index 06470529ff..a0835cfc4c 100644 --- a/src/cli/managedCommands.ts +++ b/src/cli/managedCommands.ts @@ -22,6 +22,32 @@ import { managedMembersInputSchema } from "../contracts/managed/managedToolContr import { localPathStringSchema } from "../domain/localPath.js"; import { withCommandCancellation } from "./commandCancellation.js"; import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; +import { managedInspectionBudgetShape } from "../domain/managed/managedInspectionLimits.js"; +import { cliDecimalField } from "./options.js"; + +const managedBudgetOptions = { + maxSnapshotBytes: cliDecimalField( + managedInspectionBudgetShape.max_snapshot_bytes, + ), + maxMetadataRepresentationBytes: cliDecimalField( + managedInspectionBudgetShape.max_metadata_representation_bytes, + ), +}; + +const managedBudgetParameters = (options: { + readonly maxSnapshotBytes?: number | undefined; + readonly maxMetadataRepresentationBytes?: number | undefined; +}) => ({ + ...(options.maxSnapshotBytes === undefined + ? {} + : { max_snapshot_bytes: options.maxSnapshotBytes }), + ...(options.maxMetadataRepresentationBytes === undefined + ? {} + : { + max_metadata_representation_bytes: + options.maxMetadataRepresentationBytes, + }), +}); export const registerManagedCommands = ( cli: CliInstance, @@ -49,12 +75,13 @@ const registerInspectManagedArtifact = ( "Managed PE executable or assembly path", ), }), - run: ({ args }) => + options: z.object(managedBudgetOptions), + run: ({ args, options }) => logCliCommand(logger, "inspect-managed-artifact", () => runProviderAnalysis( args.path, "inspect_managed_artifact", - {}, + managedBudgetParameters(options), logger, ).then(projectCliAnalysisResult), ), @@ -75,6 +102,7 @@ const registerInspectManagedMembers = ( ), }), options: z.object({ + ...managedBudgetOptions, type: managedMembersInputSchema.shape.type_names.describe( "Inspect only members declared by this exact type full name, and the string literals they load, such as System.Threading.Monitor; a nested type is named by its bare name, which selects every nested type of that name (repeatable)", ), @@ -93,7 +121,12 @@ const registerInspectManagedMembers = ( runProviderAnalysis( args.path, "inspect_managed_members", - options.type === undefined ? {} : { type_names: options.type }, + { + ...managedBudgetParameters(options), + ...(options.type === undefined + ? {} + : { type_names: options.type }), + }, logger, signal, { @@ -123,12 +156,13 @@ const registerInspectManagedNativeBoundaries = ( "Managed PE executable or assembly path", ), }), - run: ({ args }) => + options: z.object(managedBudgetOptions), + run: ({ args, options }) => logCliCommand(logger, "inspect-managed-native-boundaries", () => runProviderAnalysis( args.path, "inspect_managed_native_boundaries", - {}, + managedBudgetParameters(options), logger, ).then(projectCliAnalysisResult), ), @@ -153,6 +187,7 @@ const registerCompareManagedMembers = ( "Compare managed PE/CLI member inventories by exact declared type, name, and signature; names alone never form a match", args: compareManagedMemberArgs, options: z.object({ + ...managedBudgetOptions, type: managedMembersInputSchema.shape.type_names.describe( "Compare only members declared by this exact type full name on both sides, as selected by inspect-managed-members --type; use it when a whole-assembly comparison exceeds capacity (repeatable)", ), @@ -167,6 +202,15 @@ const registerCompareManagedMembers = ( { leftPath: args.leftPath, rightPath: args.rightPath, + ...(options.maxSnapshotBytes === undefined + ? {} + : { maxSnapshotBytes: options.maxSnapshotBytes }), + ...(options.maxMetadataRepresentationBytes === undefined + ? {} + : { + maxRepresentationBytes: + options.maxMetadataRepresentationBytes, + }), ...(options.type === undefined ? {} : { typeNames: options.type }), diff --git a/src/cli/nativeCallCommands.ts b/src/cli/nativeCallCommands.ts index dcd95e2a4b..35916d8a78 100644 --- a/src/cli/nativeCallCommands.ts +++ b/src/cli/nativeCallCommands.ts @@ -29,7 +29,7 @@ export const registerNativeCallCommands = ( inputJson: z .string() .describe( - "Inline JSON or a JSON file path: breakpoints, arguments, environment, duration_ms, max_events, argument_registers, backtrace_frames", + "Inline JSON or a JSON file path: breakpoints, arguments, environment, duration_ms, max_events, argument_registers, backtrace_frames, capture_limits", ), }), run: ({ args }) => diff --git a/src/cli/producerLimitsOption.ts b/src/cli/producerLimitsOption.ts new file mode 100644 index 0000000000..6fb3a3b47b --- /dev/null +++ b/src/cli/producerLimitsOption.ts @@ -0,0 +1,26 @@ +import { z } from "incur"; +import { safeParseJson } from "../domain/safeJson.js"; + +/** Incur passes object flags as text; validate the selected producer's exact policy. */ +export const producerLimitsOption = ( + schema: Schema, + flagName = "producer-limits", +) => + z + .string() + .transform((text, context) => { + const parsed = safeParseJson(text); + if (!parsed.ok) { + context.addIssue({ + code: "custom", + message: `${flagName} must be a JSON object`, + }); + return z.NEVER; + } + return parsed.value; + }) + .pipe(schema) + .optional() + .describe( + "JSON producer budget overrides. Keys and units follow this operation's producer_limits contract; changes apply on this call without restart and preserve tighter inherited OS limits.", + ); diff --git a/src/cli/reverseEngineeringCommands.ts b/src/cli/reverseEngineeringCommands.ts index 412cc3e648..496d83e5f9 100644 --- a/src/cli/reverseEngineeringCommands.ts +++ b/src/cli/reverseEngineeringCommands.ts @@ -17,6 +17,7 @@ import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { OBJDUMP_ARTIFACT_OPERATIONS, reverseCommandTimeoutSchema, + reverseCommandOutputBytesSchema, rizinInputSchema, } from "../domain/reverseEngineering.js"; import { logCliCommand } from "../cliLogging.js"; @@ -173,6 +174,7 @@ export const registerReverseEngineeringCommands = ( "Follow local DWARF debug links. External debug-file identity is not bound to Evidence.", ), commandTimeoutMs: cliDecimalField(reverseCommandTimeoutSchema), + maxOutputBytes: cliDecimalField(reverseCommandOutputBytesSchema), }), run: ({ args, options }) => withCommandCancellation((signal) => @@ -197,6 +199,9 @@ export const registerReverseEngineeringCommands = ( ...(options.symbol === undefined ? {} : { symbol: options.symbol }), + ...(options.maxOutputBytes === undefined + ? {} + : { max_output_bytes: options.maxOutputBytes }), ...(options.commandTimeoutMs === undefined ? {} : { command_timeout_ms: options.commandTimeoutMs }), @@ -220,6 +225,7 @@ export const registerReverseEngineeringCommands = ( options: z.object({ ...analysisEvidenceDeliveryOptions, commandTimeoutMs: cliDecimalField(reverseCommandTimeoutSchema), + maxOutputBytes: cliDecimalField(reverseCommandOutputBytesSchema), }), run: ({ options }) => withCommandCancellation((signal) => @@ -244,6 +250,9 @@ export const registerReverseEngineeringCommands = ( service.inspectWithObjdump( { operation: "architectures", + ...(options.maxOutputBytes === undefined + ? {} + : { max_output_bytes: options.maxOutputBytes }), ...(options.commandTimeoutMs === undefined ? {} : { command_timeout_ms: options.commandTimeoutMs }), @@ -277,6 +286,7 @@ export const registerReverseEngineeringCommands = ( asmCpu: rizinInputSchema.shape.asm_cpu, baseAddress: rizinInputSchema.shape.base_address, commandTimeoutMs: cliDecimalField(reverseCommandTimeoutSchema), + maxOutputBytes: cliDecimalField(reverseCommandOutputBytesSchema), }), run: ({ args, options }) => withCommandCancellation((signal) => @@ -300,6 +310,9 @@ export const registerReverseEngineeringCommands = ( ...(options.baseAddress === undefined ? {} : { base_address: options.baseAddress }), + ...(options.maxOutputBytes === undefined + ? {} + : { max_output_bytes: options.maxOutputBytes }), ...(options.commandTimeoutMs === undefined ? {} : { command_timeout_ms: options.commandTimeoutMs }), diff --git a/src/cli/rustCommands.ts b/src/cli/rustCommands.ts index c2baa01d4c..4d6c381393 100644 --- a/src/cli/rustCommands.ts +++ b/src/cli/rustCommands.ts @@ -1,4 +1,8 @@ import { z } from "incur"; +import { producerLimitsOption } from "./producerLimitsOption.js"; +import { rustProducerLimitsSchema } from "../domain/rust/rustResourceLimits.js"; +import { inspectRustBinaryInputSchema } from "../domain/rust/rustBinary.js"; +import { cliDecimalField } from "./options.js"; import { createRustBinaryService } from "../composition/rust.js"; import { CLI_COMMANDS } from "../cliCommandNames.js"; import { logCliCommand } from "../cliLogging.js"; @@ -33,6 +37,10 @@ export const registerRustCommands = ( }), options: z.object({ ...analysisEvidenceDeliveryOptions, + producerLimits: producerLimitsOption(rustProducerLimitsSchema), + maxRustcHashes: cliDecimalField( + inspectRustBinaryInputSchema.shape.max_rustc_hashes, + ), architecture: machoSliceArchitectureSchema .optional() .describe( @@ -76,6 +84,12 @@ export const registerRustCommands = ( const view = decoded.value; const input = inspectRustBinaryRequestSchema.safeParse({ path: absoluteCallerPath(args.path), + ...(options.producerLimits === undefined + ? {} + : { producer_limits: options.producerLimits }), + ...(options.maxRustcHashes === undefined + ? {} + : { max_rustc_hashes: options.maxRustcHashes }), ...(options.architecture === undefined ? {} : { architecture: options.architecture }), @@ -97,6 +111,12 @@ export const registerRustCommands = ( service.inspect( { path: input.data.path, + ...(input.data.producer_limits === undefined + ? {} + : { producer_limits: input.data.producer_limits }), + ...(input.data.max_rustc_hashes === undefined + ? {} + : { max_rustc_hashes: input.data.max_rustc_hashes }), ...(input.data.architecture === undefined ? {} : { architecture: input.data.architecture }), diff --git a/src/cli/signalCommands.ts b/src/cli/signalCommands.ts index fc25e02158..af69294c52 100644 --- a/src/cli/signalCommands.ts +++ b/src/cli/signalCommands.ts @@ -18,6 +18,7 @@ import { finishCliEvidenceCommand } from "./cliEvidenceCallerPath.js"; import { cliDecimalField, cliDecimalShape } from "./options.js"; import { localPathStringSchema } from "../domain/localPath.js"; import { absoluteCallerPath } from "../filesystem/callerPath.js"; +import { producerLimitsOption } from "./producerLimitsOption.js"; export const registerSignalCommands = ( cli: CliInstance, logger: Logger, @@ -27,11 +28,16 @@ export const registerSignalCommands = ( "decode" | "inspectDecoders" | "close" > = createSignalRecordingService(environment), ): void => { + const { producer_limits: discoveryLimits, ...discoveryShape } = + inspectSignalDecodersInputSchema.shape; + const { producer_limits: selectedLimits, ...selectedShape } = + signalRecordingOptionsSchema.shape; cli.command(CLI_COMMANDS.inspectSignalDecoders, { description: "Search installed sigrok decoders or selected decoder help", options: z .object({ - ...cliDecimalShape(inspectSignalDecodersInputSchema.shape), + ...cliDecimalShape(discoveryShape), + producerLimits: producerLimitsOption(discoveryLimits.unwrap()), limit: cliDecimalField( z .number() @@ -56,13 +62,19 @@ export const registerSignalCommands = ( message: "all and limit are mutually exclusive", }); }), - run: ({ options: { all, ...options } }) => + run: ({ options: { all, producerLimits, ...options } }) => withCommandCancellation((signal) => logCliCommand(logger, CLI_COMMANDS.inspectSignalDecoders, async () => finishAnalysisCommand( () => service.inspectDecoders( - { ...options, ...(all ? { limit: null } : {}) }, + { + ...options, + ...(producerLimits === undefined + ? {} + : { producer_limits: producerLimits }), + ...(all ? { limit: null } : {}), + }, { signal }, ), service, @@ -81,7 +93,8 @@ export const registerSignalCommands = ( }), options: z .object({ - ...cliDecimalShape(signalRecordingOptionsSchema.shape), + ...cliDecimalShape(selectedShape), + producerLimits: producerLimitsOption(selectedLimits.unwrap()), line_limit: cliDecimalField( z .number() @@ -109,7 +122,7 @@ export const registerSignalCommands = ( withinSignalRecordingSelectorBudget, signalRecordingSelectorBudgetIssue, ), - run: ({ args, options: { all_lines, ...options } }) => + run: ({ args, options: { all_lines, producerLimits, ...options } }) => withCommandCancellation((signal) => logCliCommand(logger, CLI_COMMANDS.decodeSignalRecording, async () => { return finishCliEvidenceCommand( @@ -118,6 +131,9 @@ export const registerSignalCommands = ( service.decode( { ...options, + ...(producerLimits === undefined + ? {} + : { producer_limits: producerLimits }), ...(all_lines ? { line_limit: null } : {}), path: absoluteCallerPath(args.path), }, diff --git a/src/cli/sourceMapLimits.ts b/src/cli/sourceMapLimits.ts new file mode 100644 index 0000000000..7f1434ef10 --- /dev/null +++ b/src/cli/sourceMapLimits.ts @@ -0,0 +1,64 @@ +import { cliDecimalField } from "./options.js"; +import { + sourceMapLimitsSchema, + type SourceMapLimitSelection, +} from "../domain/sourceMapLimits.js"; + +const shape = sourceMapLimitsSchema.shape; +const sharedOptions = { + sourceMapOutputBytes: cliDecimalField(shape.output_bytes), + sourceMapDecodedRows: cliDecimalField(shape.decoded_rows), + sourceMapDecodedSegments: cliDecimalField(shape.decoded_segments), + sourceMapDecodedRecords: cliDecimalField(shape.decoded_records), + sourceMapSectionDepth: cliDecimalField(shape.section_depth), +}; +export const localSourceMapLimitCliOptions = { + ...sharedOptions, + sourceMapBytes: cliDecimalField(shape.map_bytes), + sourceMapCodecHeapMib: cliDecimalField(shape.codec_heap_mib), +}; +export const fetchedSourceMapLimitCliOptions = { + ...sharedOptions, + sourceMapFetchBytes: cliDecimalField(shape.fetch_bytes), +}; +/** Keep omitted caller fields omitted in Evidence parameters. */ +export const sourceMapLimitsFromCli = (options: { + sourceMapBytes?: number | undefined; + sourceMapOutputBytes?: number | undefined; + sourceMapDecodedRows?: number | undefined; + sourceMapDecodedSegments?: number | undefined; + sourceMapDecodedRecords?: number | undefined; + sourceMapSectionDepth?: number | undefined; + sourceMapCodecHeapMib?: number | undefined; + sourceMapFetchBytes?: number | undefined; +}): { source_map_limits?: SourceMapLimitSelection } => { + const selected: SourceMapLimitSelection = { + ...(options.sourceMapBytes === undefined + ? {} + : { map_bytes: options.sourceMapBytes }), + ...(options.sourceMapOutputBytes === undefined + ? {} + : { output_bytes: options.sourceMapOutputBytes }), + ...(options.sourceMapDecodedRows === undefined + ? {} + : { decoded_rows: options.sourceMapDecodedRows }), + ...(options.sourceMapDecodedSegments === undefined + ? {} + : { decoded_segments: options.sourceMapDecodedSegments }), + ...(options.sourceMapDecodedRecords === undefined + ? {} + : { decoded_records: options.sourceMapDecodedRecords }), + ...(options.sourceMapSectionDepth === undefined + ? {} + : { section_depth: options.sourceMapSectionDepth }), + ...(options.sourceMapCodecHeapMib === undefined + ? {} + : { codec_heap_mib: options.sourceMapCodecHeapMib }), + ...(options.sourceMapFetchBytes === undefined + ? {} + : { fetch_bytes: options.sourceMapFetchBytes }), + }; + return Object.keys(selected).length === 0 + ? {} + : { source_map_limits: selected }; +}; diff --git a/src/cli/wasmCommands.ts b/src/cli/wasmCommands.ts index 14cad741b1..5773e2e737 100644 --- a/src/cli/wasmCommands.ts +++ b/src/cli/wasmCommands.ts @@ -75,6 +75,21 @@ export const registerWasmCommands = ( commandTimeoutMs: cliDecimalField( inspectWasmArtifactInputSchema.shape.command_timeout_ms, ), + maxInterfaceBytes: cliDecimalField( + inspectWasmArtifactInputSchema.shape.max_interface_bytes, + ), + maxArtifactPaths: cliDecimalField( + inspectWasmArtifactInputSchema.shape.max_artifact_paths, + ), + maxInputBytes: cliDecimalField( + inspectWasmArtifactInputSchema.shape.max_input_bytes, + ), + maxToolBytes: cliDecimalField( + inspectWasmArtifactInputSchema.shape.max_tool_bytes, + ), + maxOutputBytes: cliDecimalField( + inspectWasmArtifactInputSchema.shape.max_output_bytes, + ), glue_paths: wasmArtifactPathsSchema(localPathStringSchema) .optional() .describe("Selected local JavaScript glue paths"), @@ -94,6 +109,21 @@ export const registerWasmCommands = ( ...(options.commandTimeoutMs === undefined ? {} : { command_timeout_ms: options.commandTimeoutMs }), + ...(options.maxInterfaceBytes === undefined + ? {} + : { max_interface_bytes: options.maxInterfaceBytes }), + ...(options.maxArtifactPaths === undefined + ? {} + : { max_artifact_paths: options.maxArtifactPaths }), + ...(options.maxInputBytes === undefined + ? {} + : { max_input_bytes: options.maxInputBytes }), + ...(options.maxToolBytes === undefined + ? {} + : { max_tool_bytes: options.maxToolBytes }), + ...(options.maxOutputBytes === undefined + ? {} + : { max_output_bytes: options.maxOutputBytes }), glue_paths: options.glue_paths?.map((path) => absoluteCallerPath(path), ), diff --git a/src/cli/webNetworkCaptureCommands.ts b/src/cli/webNetworkCaptureCommands.ts index 44f5089654..5a33258feb 100644 --- a/src/cli/webNetworkCaptureCommands.ts +++ b/src/cli/webNetworkCaptureCommands.ts @@ -55,6 +55,15 @@ export const registerWebNetworkCaptureCommands = ( .optional() .describe("Zero-based producer ordinals; repeat to select several"), ...explicitSensitiveValueOptions, + maxInputBytes: cliDecimalField( + inspectWebNetworkCaptureInputSchema.shape.max_input_bytes, + ), + maxReplyBytes: cliDecimalField( + inspectWebNetworkCaptureInputSchema.shape.max_reply_bytes, + ), + maxNestingDepth: cliDecimalField( + inspectWebNetworkCaptureInputSchema.shape.max_nesting_depth, + ), // Decimal tokens only: numeric coercion would read "0x1" as 1 ms. captureTimeoutMs: cliDecimalField( inspectWebNetworkCaptureInputSchema.shape.capture_timeout_ms, @@ -74,6 +83,15 @@ export const registerWebNetworkCaptureCommands = ( capture_path: absoluteCallerPath(args.capturePath), format: args.format, sensitive_values: options.sensitiveValue, + ...(options.maxInputBytes === undefined + ? {} + : { max_input_bytes: options.maxInputBytes }), + ...(options.maxReplyBytes === undefined + ? {} + : { max_reply_bytes: options.maxReplyBytes }), + ...(options.maxNestingDepth === undefined + ? {} + : { max_nesting_depth: options.maxNestingDepth }), ...([ options.protobufFraming, options.protobufBodyPointer, diff --git a/src/cli/webSourceLocationCommands.ts b/src/cli/webSourceLocationCommands.ts index 64cb5baa93..3f894fccf7 100644 --- a/src/cli/webSourceLocationCommands.ts +++ b/src/cli/webSourceLocationCommands.ts @@ -1,4 +1,14 @@ -import { projectCliAnalysisResult } from "./projectCliAnalysisResult.js"; +import { + localSourceMapLimitCliOptions, + sourceMapLimitsFromCli, +} from "./sourceMapLimits.js"; +import { + analysisEvidenceDeliveryOptions, + parseCliAnalysisEvidenceDelivery, + deliverCliAnalysisEvidence, +} from "./analysisEvidenceDelivery.js"; +import { webSourceLocationDeliverySchema } from "../domain/analysisView/webSourceLocationView.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { z } from "incur"; import { createWebSourceLocationService } from "../composition/webSourceLocations.js"; import { webSourceLocationInputSchema } from "../domain/webSourceLocation.js"; @@ -8,7 +18,6 @@ import type { Logger } from "pino"; import type { CliInstance } from "./types.js"; import { cliDecimalField, cliIntegerArgument } from "./options.js"; import { withCommandCancellation } from "./commandCancellation.js"; -import { finishAnalysisCommand } from "./finishAnalysisCommand.js"; const sourceMapSelection = webSourceLocationInputSchema.shape.source_map; @@ -35,15 +44,29 @@ export const registerWebSourceLocationCommands = ( ), }), options: z.object({ + ...localSourceMapLimitCliOptions, + ...analysisEvidenceDeliveryOptions, + view: z + .string() + .optional() + .describe( + "JSON source-location matches/source-text view; requires --detail summary and --evidence_output", + ), decodeTimeoutMs: cliDecimalField( webSourceLocationInputSchema.shape.decode_timeout_ms, ), }), run: ({ args, options }) => withCommandCancellation((signal) => - logCliCommand(logger, CLI_COMMANDS.traceWebSourceLocation, () => - finishAnalysisCommand( - () => + logCliCommand(logger, CLI_COMMANDS.traceWebSourceLocation, async () => { + const delivery = parseCliAnalysisEvidenceDelivery( + "trace_web_source_location", + options, + webSourceLocationDeliverySchema, + ); + if (!delivery.ok) return projectAnalysisError(delivery.error); + return deliverCliAnalysisEvidence({ + run: () => service.trace( { manifest_path: args.manifestPath, @@ -53,6 +76,7 @@ export const registerWebSourceLocationCommands = ( url: args.sourceMapUrl, }, generated_position: { line: args.line, column: args.column }, + ...sourceMapLimitsFromCli(options), ...(options.decodeTimeoutMs === undefined ? {} : { decode_timeout_ms: options.decodeTimeoutMs }), @@ -60,11 +84,16 @@ export const registerWebSourceLocationCommands = ( { signal }, ), service, - "web-source-location", - "trace_web_source_location", + providerId: "web-source-location", + operation: "trace_web_source_location", signal, - ).then(projectCliAnalysisResult), - ), + detail: delivery.value.detail, + evidenceOutput: delivery.value.evidenceOutput, + ...(delivery.value.view === undefined + ? {} + : { view: delivery.value.view }), + }); + }), ), }); }; diff --git a/src/cliObservationOptions.ts b/src/cliObservationOptions.ts index f1ace24baa..417d11ec94 100644 --- a/src/cliObservationOptions.ts +++ b/src/cliObservationOptions.ts @@ -93,6 +93,12 @@ export const electronPageInspectionOptions = z.object({ }); export const javascriptApplicationOptions = z.object({ + semanticPrecision: z + .string() + .optional() + .describe( + "Inline JSON or JSON file with per-source semantic_precision evaluator budgets; independent of graph retention and worker memory.", + ), ...javascriptRouteDeliveryOptions, maxSemanticNodes: cliDecimalField( javaScriptAnalysisResourceControlsSchema.shape.max_semantic_nodes, diff --git a/src/composition/javascriptAnalysisWorkerProcess.ts b/src/composition/javascriptAnalysisWorkerProcess.ts index 8155d37eb1..855443ba37 100644 --- a/src/composition/javascriptAnalysisWorkerProcess.ts +++ b/src/composition/javascriptAnalysisWorkerProcess.ts @@ -22,6 +22,7 @@ const analyze: JavaScriptSourceWorkerAnalyzer = async ( callbacks, packageType, purpose, + precision, ) => { if (!file.text.included) throw new TypeError("JavaScript worker requires decoded source text"); @@ -33,7 +34,11 @@ const analyze: JavaScriptSourceWorkerAnalyzer = async ( } await callbacks.phase("static"); const facts = await completeJavaScriptAnalysisSteps( - analyzeParsedJavaScriptStaticSourceSteps(file.text.value, parsed), + analyzeParsedJavaScriptStaticSourceSteps( + file.text.value, + parsed, + precision, + ), ); if (purpose === "preload-discovery") { // Only declarations cross this preliminary pass. The normal pass owns @@ -70,6 +75,7 @@ const analyze: JavaScriptSourceWorkerAnalyzer = async ( parsed, moduleResolutionModeForJavaScriptReference(file.path, "static-import"), capacity.nodes, + precision, ), ); parsed = null; diff --git a/src/composition/webBundleAnalysisWorkerProcess.ts b/src/composition/webBundleAnalysisWorkerProcess.ts index 967bd623f5..c10b8203fd 100644 --- a/src/composition/webBundleAnalysisWorkerProcess.ts +++ b/src/composition/webBundleAnalysisWorkerProcess.ts @@ -4,8 +4,8 @@ import { type WebBundleScriptAnalyzer, } from "../browser/WebBundleAnalysisWorkerServer.js"; -const analyze: WebBundleScriptAnalyzer = (script) => - analyzeCapturedWebScript(script); +const analyze: WebBundleScriptAnalyzer = (script, precision) => + analyzeCapturedWebScript(script, precision); try { await serveWebBundleAnalysisWorker(analyze); diff --git a/src/config.test.ts b/src/config.test.ts index d75cff933d..662ce07a78 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -20,6 +20,23 @@ it("still rejects a NUL in an unresolved expression", () => { if (!result.ok) expect(JSON.stringify(result.error)).not.toContain("private"); }); +it.each([ + ["REA_ASAR_MAX_LISTING_BYTES", "9007199254740992"], + ["REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS", "0"], + ["REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION", "2147483648"], + ["REA_GHIDRA_VALUE_FLOW_MAX_EDGES", "1.5"], + ["REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB", "2048"], +])( + "rejects invalid %s at launch with its exact setting identity", + (setting, value) => { + const parsed = parseConfig({ [setting]: value }); + if (parsed.ok) throw new Error("Invalid resource policy accepted"); + expect(parsed.error.settings).toContainEqual( + expect.objectContaining({ setting }), + ); + }, +); + it("keeps the caller's opt-in Ghidra checkpoint directory and leaves it disabled by default", () => { const selected = parseConfig({ REA_GHIDRA_CHECKPOINT_DIR: "/tmp/checkpoints with spaces", diff --git a/src/config/analysisSettings.ts b/src/config/analysisSettings.ts index ad1e95e3e3..ad4e974218 100644 --- a/src/config/analysisSettings.ts +++ b/src/config/analysisSettings.ts @@ -4,6 +4,11 @@ import type { AnalysisSettingId } from "../domain/analysisSettings.js"; import { analysisSettingDefinition } from "../domain/analysisSettingDefinitions.js"; import { ConfigurationError } from "../domain/configurationErrors.js"; import { asarMaxHeaderBytesSchema } from "./asarHeaderAdmission.js"; +import { asarMaxListingBytesSchema } from "./asarListingAdmission.js"; +import { + ghidraValueFlowCountSchema, + ghidraDecompilePayloadMiBSchema, +} from "./ghidraValueFlowLimits.js"; import { providerStartupTimeoutSchema, ghidraDecompileTimeoutSchema, @@ -15,6 +20,11 @@ import { parseNativeMatchingAllowance } from "./nativeMatchingAllowance.js"; const environmentSchemas: Readonly>> = { REA_ASAR_MAX_HEADER_BYTES: asarMaxHeaderBytesSchema, + REA_ASAR_MAX_LISTING_BYTES: asarMaxListingBytesSchema, + REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS: ghidraValueFlowCountSchema, + REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION: ghidraValueFlowCountSchema, + REA_GHIDRA_VALUE_FLOW_MAX_EDGES: ghidraValueFlowCountSchema, + REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB: ghidraDecompilePayloadMiBSchema, REA_GHIDRA_STARTUP_TIMEOUT_MS: providerStartupTimeoutSchema, REA_HOPPER_STARTUP_TIMEOUT_MS: providerStartupTimeoutSchema, REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS: ghidraDecompileTimeoutSchema, diff --git a/src/config/asarHeaderAdmission.ts b/src/config/asarHeaderAdmission.ts index 52ba8bcedd..ae94a2b664 100644 --- a/src/config/asarHeaderAdmission.ts +++ b/src/config/asarHeaderAdmission.ts @@ -6,6 +6,7 @@ import { z } from "zod"; import { ConfigurationError } from "../domain/configurationErrors.js"; import { err, ok, type Result } from "../domain/result.js"; +import type { AsarListingAdmission } from "./asarListingAdmission.js"; export const ASAR_HEADER_ADMISSION_SETTING = "REA_ASAR_MAX_HEADER_BYTES"; @@ -13,6 +14,8 @@ export const ASAR_HEADER_ADMISSION_SETTING = "REA_ASAR_MAX_HEADER_BYTES"; export interface AsarHeaderAdmission { readonly maximumBytes: number; readonly source: "default" | "setting" | "runtime"; + /** Independently resolved expanded listing policy; omission uses its default. */ + readonly listing?: AsarListingAdmission; } export const DEFAULT_ASAR_HEADER_ADMISSION: AsarHeaderAdmission = { diff --git a/src/config/asarListingAdmission.ts b/src/config/asarListingAdmission.ts new file mode 100644 index 0000000000..61c35d2fdc --- /dev/null +++ b/src/config/asarListingAdmission.ts @@ -0,0 +1,42 @@ +import { z } from "zod"; +import { + DEFAULT_ASAR_MAX_LISTING_BYTES, + MAXIMUM_ASAR_MAX_LISTING_BYTES, +} from "../domain/analysisResourceLimits.js"; +import { ConfigurationError } from "../domain/configurationErrors.js"; +import { err, ok, type Result } from "../domain/result.js"; + +export const ASAR_LISTING_ADMISSION_SETTING = "REA_ASAR_MAX_LISTING_BYTES"; + +/** Expanded path and entry retention, independently of packed header admission. */ +export interface AsarListingAdmission { + readonly maximumBytes: number; + readonly source: "default" | "setting" | "runtime"; +} + +export const DEFAULT_ASAR_LISTING_ADMISSION: AsarListingAdmission = { + maximumBytes: DEFAULT_ASAR_MAX_LISTING_BYTES, + source: "default", +}; + +const constraint = `${ASAR_LISTING_ADMISSION_SETTING} must be a positive decimal safe integer in bytes, at most ${String(MAXIMUM_ASAR_MAX_LISTING_BYTES)}; the default is ${String(DEFAULT_ASAR_MAX_LISTING_BYTES)} bytes.`; + +export const asarMaxListingBytesSchema = z + .string() + .regex(/^[1-9]\d*$/u, { error: constraint }) + .transform(Number) + .pipe(z.number().int().min(1).max(MAXIMUM_ASAR_MAX_LISTING_BYTES)); + +export const parseAsarListingAdmission = ( + value: string | undefined, +): Result => { + if (value === undefined) return ok(DEFAULT_ASAR_LISTING_ADMISSION); + const parsed = asarMaxListingBytesSchema.safeParse(value); + return parsed.success + ? ok({ maximumBytes: parsed.data, source: "setting" }) + : err( + new ConfigurationError(constraint, { + settings: [{ setting: ASAR_LISTING_ADMISSION_SETTING, constraint }], + }), + ); +}; diff --git a/src/config/environment.ts b/src/config/environment.ts index b72eb280a2..212205a857 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -29,6 +29,11 @@ import { mcpOutputSchemaProfileSchema } from "./mcpOutputSchemaProfile.js"; import { mcpToolCatalogProfileSchema } from "./mcpToolCatalogProfile.js"; import { mcpResponseBudgetSchema } from "./mcpResponseBudget.js"; import { asarMaxHeaderBytesSchema } from "./asarHeaderAdmission.js"; +import { asarMaxListingBytesSchema } from "./asarListingAdmission.js"; +import { + ghidraValueFlowCountSchema, + ghidraDecompilePayloadMiBSchema, +} from "./ghidraValueFlowLimits.js"; import { bridgeMaxRequestBytesSchema } from "./bridgeMaxRequestBytes.js"; import { captureDecoderOldGenerationMiBSchema } from "./captureDecoderHeap.js"; import { debuggerSessionLimitEnvironmentShape } from "./debuggerSessionLimits.js"; @@ -57,6 +62,13 @@ const environmentSchema = z.object({ REA_MCP_MAX_RESPONSE_BYTES: mcpResponseBudgetSchema.optional(), REA_MCP_MAX_INPUT_ELEMENTS: mcpInputBudgetSchema.optional(), REA_ASAR_MAX_HEADER_BYTES: asarMaxHeaderBytesSchema.optional(), + REA_ASAR_MAX_LISTING_BYTES: asarMaxListingBytesSchema.optional(), + REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS: ghidraValueFlowCountSchema.optional(), + REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION: + ghidraValueFlowCountSchema.optional(), + REA_GHIDRA_VALUE_FLOW_MAX_EDGES: ghidraValueFlowCountSchema.optional(), + REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB: + ghidraDecompilePayloadMiBSchema.optional(), REA_CAPTURE_DECODER_OLD_GENERATION_MIB: captureDecoderOldGenerationMiBSchema.optional(), REA_REGEX_MATCHING_DEADLINE_MS: regexMatchingDeadlineMsSchema.optional(), diff --git a/src/config/ghidraValueFlowLimits.ts b/src/config/ghidraValueFlowLimits.ts new file mode 100644 index 0000000000..b70db61998 --- /dev/null +++ b/src/config/ghidraValueFlowLimits.ts @@ -0,0 +1,18 @@ +import { z } from "zod"; +import { + MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS, + MAXIMUM_GHIDRA_DECOMPILE_PAYLOAD_MIB, +} from "../domain/analysisResourceLimits.js"; + +/** Launch and runtime values use the same Java counter representability bound. */ +export const ghidraValueFlowCountSchema = z + .string() + .regex(/^[1-9]\d*$/u, "Expected a positive decimal integer") + .transform(Number) + .pipe(z.number().int().min(1).max(MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS)); + +export const ghidraDecompilePayloadMiBSchema = z + .string() + .regex(/^[1-9]\d*$/u, "Expected a positive decimal MiB value") + .transform(Number) + .pipe(z.number().int().min(1).max(MAXIMUM_GHIDRA_DECOMPILE_PAYLOAD_MIB)); diff --git a/src/config/parseConfig.ts b/src/config/parseConfig.ts index b0e1cb6aa6..59d0bd60cf 100644 --- a/src/config/parseConfig.ts +++ b/src/config/parseConfig.ts @@ -142,6 +142,27 @@ export const parseConfig = ( : { ghidraCheckpointDirectory: env.REA_GHIDRA_CHECKPOINT_DIR }), ghidraJavaHome: env.JAVA_HOME, ghidraStartupTimeoutMs: env.REA_GHIDRA_STARTUP_TIMEOUT_MS, + ...(env.REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB === undefined + ? {} + : { + ghidraDecompileMaxPayloadMiB: + env.REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB, + }), + ...(env.REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS === undefined + ? {} + : { + ghidraValueFlowMaxOperations: + env.REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS, + }), + ...(env.REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION === undefined + ? {} + : { + ghidraValueFlowMaxInputsPerOperation: + env.REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION, + }), + ...(env.REA_GHIDRA_VALUE_FLOW_MAX_EDGES === undefined + ? {} + : { ghidraValueFlowMaxEdges: env.REA_GHIDRA_VALUE_FLOW_MAX_EDGES }), ...(env.REA_GHIDRA_DECOMPILE_TIMEOUT_SECONDS === undefined ? {} : { diff --git a/src/config/runtimeAnalysisSettings.ts b/src/config/runtimeAnalysisSettings.ts index d4eb990a9b..437ac745ae 100644 --- a/src/config/runtimeAnalysisSettings.ts +++ b/src/config/runtimeAnalysisSettings.ts @@ -2,6 +2,12 @@ import type { AnalysisSettingsReader } from "../domain/analysisSettings.js"; import type { ConfigurationError } from "../domain/configurationErrors.js"; import { ok, type Result } from "../domain/result.js"; import type { AppConfig } from "./types.js"; +import { + DEFAULT_GHIDRA_VALUE_FLOW_MAX_OPERATIONS, + DEFAULT_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION, + DEFAULT_GHIDRA_VALUE_FLOW_MAX_EDGES, +} from "../domain/analysisResourceLimits.js"; +import { parseAsarListingAdmission } from "./asarListingAdmission.js"; import { parseAsarHeaderAdmission, type AsarHeaderAdmission, @@ -45,6 +51,35 @@ export const runtimeAnalysisConfig = ( ? undefined : value; }, + get ghidraDecompileMaxPayloadMiB() { + const value = read().overrides["ghidra.decompile_max_payload_mib"]; + return value === undefined + ? config.ghidraDecompileMaxPayloadMiB + : value === null + ? undefined + : value; + }, + get ghidraValueFlowMaxOperations() { + return ( + read().overrides["ghidra.value_flow_max_operations"] ?? + config.ghidraValueFlowMaxOperations ?? + DEFAULT_GHIDRA_VALUE_FLOW_MAX_OPERATIONS + ); + }, + get ghidraValueFlowMaxInputsPerOperation() { + return ( + read().overrides["ghidra.value_flow_max_inputs_per_operation"] ?? + config.ghidraValueFlowMaxInputsPerOperation ?? + DEFAULT_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION + ); + }, + get ghidraValueFlowMaxEdges() { + return ( + read().overrides["ghidra.value_flow_max_edges"] ?? + config.ghidraValueFlowMaxEdges ?? + DEFAULT_GHIDRA_VALUE_FLOW_MAX_EDGES + ); + }, regexMatchingDeadline: { get deadlineMs() { return ( @@ -79,10 +114,27 @@ export const runtimeAsarHeaderAdmission = ( environment.REA_ASAR_MAX_HEADER_BYTES, ); if (!parsed.ok) throw parsed.error; + const listing = parseAsarListingAdmission( + environment.REA_ASAR_MAX_LISTING_BYTES, + ); + if (!listing.ok) throw listing.error; const baseline = parsed.value; return read === undefined - ? baseline + ? { ...baseline, listing: listing.value } : { + listing: { + get maximumBytes() { + return ( + read().overrides["asar.max_listing_bytes"] ?? + listing.value.maximumBytes + ); + }, + get source() { + return read().overrides["asar.max_listing_bytes"] === undefined + ? listing.value.source + : "runtime"; + }, + }, get maximumBytes() { return ( read().overrides["asar.max_header_bytes"] ?? baseline.maximumBytes diff --git a/src/config/types.ts b/src/config/types.ts index 455d1160ca..154d1ead4d 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -27,6 +27,10 @@ export interface AppConfig { readonly ghidraStartupTimeoutMs: number; /** Per-function native decompiler limit; absent leaves decompiles unbounded. */ readonly ghidraDecompileTimeoutSeconds?: number | undefined; + readonly ghidraDecompileMaxPayloadMiB?: number | undefined; + readonly ghidraValueFlowMaxOperations?: number; + readonly ghidraValueFlowMaxInputsPerOperation?: number; + readonly ghidraValueFlowMaxEdges?: number; readonly ghidraLanguageOverride?: GhidraLanguageOverride; readonly ghidraDisabledAnalyzers?: readonly string[]; readonly ghidraSeedFile?: string; diff --git a/src/contracts/analysisSettingsContracts.ts b/src/contracts/analysisSettingsContracts.ts index e9716d562a..e9210e84ab 100644 --- a/src/contracts/analysisSettingsContracts.ts +++ b/src/contracts/analysisSettingsContracts.ts @@ -44,7 +44,7 @@ const descriptor = z.strictObject({ default_value: z.number().nullable(), source: z.enum(["default", "setting", "runtime", "invalid"]), configuration_error: analysisErrorProjectionSchema.optional(), - unit: z.enum(["bytes", "ms", "seconds", "MiB"]), + unit: z.enum(["bytes", "ms", "seconds", "MiB", "count"]), minimum: z.number(), maximum: z.number(), applies_at: z.enum(["next_operation", "next_worker", "next_startup"]), diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index d3a0ff1431..430c4d0d45 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -17,7 +17,7 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - 'Use native-procedure-references to page retained procedure reference or unresolved-call rows. Endpoint kind selected-procedure refers to the summary procedure, null preserves the original null, and shared carries an index into items.endpoints, where each other procedure is stored once per page. Exact source_address and target_address filters apply to reference rows. For inspect_api_contract parents, graphql-contract pages retained SDL types/fields, operation definitions, field references, fragment-spreads or diagnostics with exact filters and selected columns. Use javascript-routes collections to query route patterns, source-linked occurrences, source status, unknowns or summary counts from retained JavaScript application or web-bundle Evidence. Use web-bundle collections endpoints or inferences to page endpoint observations and inferences from retained analyze_web_bundle Evidence; summary returns target URL, origin, and counts. For retained trace_feature Evidence, kind native-feature-trace pages matches, references or unknowns; matches filter exact address, references filter exact source_address and target_address, source_index identifies each original row, and list_unknowns pages durable question heads. Inspect selected views of supported completed Evidence without repeating analysis. Use recorded-array with an exact raw_result object-member path to page a provider-recorded array when the producer supplies that path; rows keep their raw meaning and the parent retains coverage. For procedure_pseudo_code Evidence, summary reports text availability and the original selection; kind native with facet pseudocode pages the original UTF-16 text. Other native facets require analyze_function. Choose a view compatible with the parent operation described below. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Native export tables use kind native-exports, collection exports/issues, exact names or original indices, selected fields and matching-row pagination; parent Evidence plus source_index identifies each row. Source coverage stays separate from page coverage. BEAM views use kind beam, collections atoms/exports/imports/lambdas/chunks, exact names/indices/fields and matching-row pages; import modules intersects function names. source_index identifies original parent-and-collection rows, preserving every arity and raw atom index. beam.counts preserves null missing tables versus decoded empty tables; runtime absence remains unknown. PYC views use kind python-pyc, collection code_objects, names/indices/fields and paging; source_index scopes every same-name alternative to the parent, nested_code keeps code-constant references, and pyc.coverage preserves source coverage. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. Rust metadata uses kind rust, collection packages/edges, exact names/versions or original indices, selected fields, edge from_indices/to_indices and matching-row pagination. Embedded-record coverage is separate from unknown runtime dependencies. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent also supports summary and browser-scenario steps/events/transactions/text pages, preserving step_indices, event_kinds, transaction_ids and selected artifacts. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state (a list also learns the filters, search, sort and paging its recordings verify, and answers new values of them from the state), and every other read tool is a recorded view with a reason. It also reports a write plan per state-changing tool (create, update, delete or document, built from its first successful call) and write_clock; a plan is served only when replaying the capture shows a later read confirming its effect and none contradicting it. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. The environment-coverage view, view {kind: "environment-coverage", template_rules: [...]}, summarizes that state without its entities: reads.state and reads.recorded count read tools answered from the state and replayed (recorded_reasons per view reason), writes.served lists the served write tools, and writes.withheld lists each other write tool with its reason, plan record, contradicting read and a hint about what to record next; it refuses what the environment-state view refuses. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. The file may also be an Evidence bundle written by CLI --evidence_output or export_evidence_bundle; select evidence_id when it contains more than one record. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; exported symbol bindings are not module units and stay out of them. use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings. Exact names and declaring_types filters intersect before paging where supported; fields selects columns (distinct from collection: "fields"). Identity/source_index, module MVID, source coverage and unresolved filter counts remain; omit body to avoid IL anchors. Omitted fields preserves full rows. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. APK signing views use kind apk-signature and collection signers/digests/lineage, with ids, signer_ids, schemes, fields and paging. Identity links remain when fields is empty. Declared identities and digest matches do not establish signature verification or authenticity. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types/generic_method_instances/generic_method_issues. Select indices, fields and pages; sections/images/types/methods/fields/properties/events/parameters/return_types/field_types/parameter_types/generic_method_instances accept exact names (native type names match retained rendered values, preserving same-name rows; null names never match), types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses and generic_method_instances accept addresses/states (generic_method_instances also method_indices), preserving every folded alternative; generic_method_issues accept rows. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native load-image Evidence uses kind native-load-image with collection source_files/mappings/relocations/entry_points/entry_context/checks/sections, original indices, collection-specific fields and offset/limit paging. Null collections remain unrecorded or inapplicable; pages preserve unsupported verification. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence accepts kind summary for identity, availability and facet counts, or kind native for selected facts: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native_api and value-flow observations. For parameter type paths when value flow is recorded, use facet: "value_flow_parameters"; its decompiler-observed path may be passed to inspect_native_data_type, but exact database lookup can be unavailable and does not by itself establish type identity. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Retained batch_decompile Evidence accepts kind summary for counts and sizes, or view: {kind: "batch-decompile", collection: "items", where: {statuses: ["unavailable", "error"]}} for body-free rows with exact indices, addresses, procedure_addresses or statuses (include_pseudocode adds bodies), or collection "pseudocode" with an index to page one body. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', + 'For trace_web_source_location parents, summary returns body-free source/map identities, lookup and original-position metadata with total/next_offset for the first match page. Use source-location collection matches to page every original match_index without embedded text; collection source-text selects an exact match_index and UTF-16 offset/limit, returning content identity and only that text page. Both accept max_bytes (default 1048576); increase it or reduce limit after a byte refusal. Complete map text and diagnostics remain in the parent, even after local artifacts are removed. Use native-procedure-references to page retained procedure reference or unresolved-call rows. Endpoint kind selected-procedure refers to the summary procedure, null preserves the original null, and shared carries an index into items.endpoints, where each other procedure is stored once per page. Exact source_address and target_address filters apply to reference rows. For inspect_api_contract parents, graphql-contract pages retained SDL types/fields, operation definitions, field references, fragment-spreads or diagnostics with exact filters and selected columns. Use javascript-routes collections to query route patterns, source-linked occurrences, source status, unknowns or summary counts from retained JavaScript application or web-bundle Evidence. Use web-bundle collections endpoints or inferences to page endpoint observations and inferences from retained analyze_web_bundle Evidence; summary returns target URL, origin, and counts. For retained trace_feature Evidence, kind native-feature-trace pages matches, references or unknowns; matches filter exact address, references filter exact source_address and target_address, source_index identifies each original row, and list_unknowns pages durable question heads. Inspect selected views of supported completed Evidence without repeating analysis. Use recorded-array with an exact raw_result object-member path to page a provider-recorded array when the producer supplies that path; rows keep their raw meaning and the parent retains coverage. For procedure_pseudo_code Evidence, summary reports text availability and the original selection; kind native with facet pseudocode pages the original UTF-16 text. Other native facets require analyze_function. Choose a view compatible with the parent operation described below. Command summary views preserve ordered GDB records or opaque objdump stdout/stderr once, source parameters, completion facts and parent identity. They do not establish current target state. Historical capture summaries return identity and counts without bodies or archive inventories. Capture views page retained records: view {kind: "capture", detail: "preview" | "complete", offset: 0, limit: 20}; preview selects recorded headline fields and binary descriptors, complete returns exact records with all sidecars. Offsets follow the selected parent record order, preserving original producer ordinals. Paging never reopens the source or reruns the decoder. Native export tables use kind native-exports, collection exports/issues, exact names or original indices, selected fields and matching-row pagination; parent Evidence plus source_index identifies each row. Source coverage stays separate from page coverage. BEAM views use kind beam, collections atoms/exports/imports/lambdas/chunks, exact names/indices/fields and matching-row pages; import modules intersects function names. source_index identifies original parent-and-collection rows, preserving every arity and raw atom index. beam.counts preserves null missing tables versus decoded empty tables; runtime absence remains unknown. PYC views use kind python-pyc, collection code_objects, names/indices/fields and paging; source_index scopes every same-name alternative to the parent, nested_code keeps code-constant references, and pyc.coverage preserves source coverage. Go metadata uses kind go, collection dependencies/settings, exact paths/keys or indices, fields and pagination. Row index refers to the original collection; empty selections return no rows, and dependency coverage remains unknown. Rust metadata uses kind rust, collection packages/edges, exact names/versions or original indices, selected fields, edge from_indices/to_indices and matching-row pagination. Embedded-record coverage is separate from unknown runtime dependencies. For inspect_pcap_capture parents, kind pcap pages endpoints, dns-queries, tls-sni, protocols, http-uris or unattributed-addresses. Select an exact query for dns-addresses, or a tshark/capinfos stream for UTF-16 diagnostics text. Page totals count retained facts; source coverage remains explicit. Qlog summaries keep profile/scope and counts; kind qlog pages traces, streams, loss-declarations, incomplete-records, contributions, ranges or final-sizes. Except traces, select trace_index; stream-specific facets also require exact stream_id and direction. Facts retain all-recorded-events-before-selection scope, and interval overlap or endpoint loss declarations do not prove retransmission or network loss. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. The environment-manifest view projects the same HAR requests on the primary origin (most JSON-like records) into a replayable tool surface: view {kind: "environment-manifest", template_rules: ["after-uid-literal", "opaque-token", "sibling-variance"], include_query_parameter_names: true}. Each tool has a deterministic name, a path template and parameters from the four format rules plus the selected opt-in rules, observed query parameter names, a value-free body shape for POST/PUT/PATCH, status and content-type sets and record ordinals; records are pointers into the parent, excluded groups carry a reason and three fixed unknowns are always stated. Templates, parameters and names are inferences; the view copies no header, cookie or body values and refuses parents inspected with sensitive values. A capture_browser_scenario parent also supports summary and browser-scenario steps/events/transactions/text pages, preserving step_indices, event_kinds, transaction_ids and selected artifacts. A capture_browser_scenario parent that selected response bodies or header values projects the same way, one record per network transaction in request order; it relies on capture-time redaction of declared secrets instead of that refusal, and a transaction whose URL was redacted is left unrepresented. The environment-state view extracts, from the same parents, the initial state the recorded reads observed: view {kind: "environment-state", template_rules: [...]} with the rules of the manifest it accompanies. It reads response bodies recorded before the first related write and reports entity types with an identity field (proved by a path parameter or inferred from a field name), entities in collection order with per-view values where views differ, documents for non-parameterized reads that are not entities and have one initial request line (item-shaped paths stay recorded), and a render plan per read tool: item and list views reproduce their latest recordings from the state (a list also learns the filters, search, sort and paging its recordings verify, and answers new values of them from the state), and every other read tool is a recorded view with a reason. It also reports a write plan per state-changing tool (create, update, delete or document, built from its first successful call) and write_clock; a plan is served only when replaying the capture shows a later read confirming its effect and none contradicting it. Types and identity are inferences; values are recorded data, so it refuses parents inspected with sensitive values, and a scenario parent must have selected response bodies; its values keep the capture redaction markers, and a write whose URL it redacted ends the state of every type. The environment-coverage view, view {kind: "environment-coverage", template_rules: [...]}, summarizes that state without its entities: reads.state and reads.recorded count read tools answered from the state and replayed (recorded_reasons per view reason), writes.served lists the served write tools, and writes.withheld lists each other write tool with its reason, plan record, contradicting read and a hint about what to record next; it refuses what the environment-state view refuses. For a retained record use source: {kind: "retained-evidence", evidence_id: ""}. Use normalized_result.parent_evidence_id from a summary receipt. Source kind is required. Portable source uses {kind: "inline", evidence: }. A completed Evidence JSON file on this host uses {kind: "evidence-file", path: ""}: REA reads and authenticates it, then projects it like retained Evidence, so a large capture is bounded by the file reader rather than by inline-input accounting. The file may also be an Evidence bundle written by CLI --evidence_output or export_evidence_bundle; select evidence_id when it contains more than one record. For layout, JavaScript, inventory, WASM, Mach-O, Interface Builder, EVM, managed artifact, Android package and Dart AOT sources, request a summary with view: {kind: "summary"}; summary is a view kind and has no facet. The WASM summary gives validation, counts, WAT digest/length, tool profile and execution/network states without reading WAT text. Select a layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; exported symbol bindings are not module units and stay out of them. use node_id for ambiguous or unavailable paths. Typed module references use view {kind: "javascript", collection: "modules", projection: "references", where: {path: "main.js", kind: "javascript-module"}, offset: 0, limit: 64}; where matches exact retained strings (null selects unavailable paths). Rows include canonical identity, retained source-range count and observation support states. Source coverage and selection remain explicit; exhausting matching rows does not prove runtime loading or complete analysis. For exact file/native facts and processing gaps, use view {kind: "javascript-facts", where: {paths: ["package.json"]}, property_keys: ["name", "operation", "analysis_status"]}. Rows page original observations with canonical identities and Evidence; missing properties remain omitted. Paths match recorded artifact-path/source-range locations; optional kinds filter nodes. To discover field names without values, opt into include_available_property_keys: true with property_keys: []. Canonical identity and Evidence are separate from properties; keep each node_id/observation_id pair together when composing results. Inventory uses view: {kind: "inventory", collection: "occurrences", logical_paths: ["package.json"], offset: 0, limit: 64}. Inventory views page nodes, occurrences, edges or integrity contradictions. Occurrences accept exact logical_paths and occurrence_ids; nodes accept artifact_ids. Batch filters intersect (omitted means all available, [] means none); fields selects columns while retaining identity links. source_complete describes retained graph membership; source_coverage describes producer coverage, including unknown encrypted PCK directories. Page exhaustion does not prove absence in a partial source. WASM views use kind wasm and facet sections/imports/exports/candidates/glue/wat/headers/details/tool_profile; text facets use UTF-16 code units and source forms remain exact. Mach-O views use kind macho and facet entrypoints/architectures/build_metadata/load_commands/dependencies/imports/exports/segments/provenance; source_exhaustive false means unreturned rows may exist. Interface Builder views use kind interface-builder and collection documents/nodes/edges/coverage; nodes and edges accept an exact document relative_path. EVM summary returns all inferred interface candidates, carrier/decoded identities and explicit unknowns without hex or duplicate stdout. EVM views use kind evm and facet functions/bytecode_hex/stdout/stderr; function pages preserve the original candidate rows, and text offsets/limits count UTF-16 code units (hex has two characters per decoded byte). Exhausting candidate pages does not establish global interface completeness. Managed artifact summaries preserve complete identity/classification, metadata status, module/MVID, frameworks, coverage and limitations without unrelated inventories. Managed pages use kind managed and collection references/resources/attributes, retaining source coverage and build-local token scope; page exhaustion does not establish complete metadata. Managed member summaries keep identity, selection with unmatched names, coverage and per-collection counts; managed-members pages use collection types/exported_types/fields/methods/member_refs/call_edges/field_accesses/user_strings. Exact names and declaring_types filters intersect before paging where supported; fields selects columns (distinct from collection: "fields"). Identity/source_index, module MVID, source coverage and unresolved filter counts remain; omit body to avoid IL anchors. Omitted fields preserves full rows. Android package summaries retain all declarations, engine identity and counts without decoded manifest text. Android manifest pages use kind android, facet manifest, offset and limit in UTF-16 code units; source_status preserves partial decoded coverage even when the retained text page is exhausted. Dart AOT summary preserves target/ABI/library identity, snapshot headers and pool lower bounds. String pages use kind dart-aot, collection package_uris/dart_uris/class_like_tokens; source coverage remains partial when the producer says partial. Protobuf views use kind protobuf, collection bodies/frames/fields, with record_ordinals, body_pointers and fields; field pages also select frame_indices, field_ids, field_numbers or schema_names. Byte identity, offset basis and body status remain present. Text and schema_value are opt-in page columns. APK signing views use kind apk-signature and collection signers/digests/lineage, with ids, signer_ids, schemes, fields and paging. Identity links remain when fields is empty. Declared identities and digest matches do not establish signature verification or authenticity. Model views use kind model-artifact and collection metadata (keys), tensors (names/types) or types (derived totals), with indices, fields and paging. Header SHA-256 scopes declarations; full-file identity remains unknown unless full hashing was selected. Arrays retain typed eight-element previews with explicit omissions. IL2CPP metadata views use kind il2cpp-metadata and collection sections/images/types/methods/fields/properties/events/parameters/string_literals/code_gen_modules/method_addresses/return_types/field_types/parameter_types/generic_method_instances/generic_method_issues. Select indices, fields and pages; sections/images/types/methods/fields/properties/events/parameters/return_types/field_types/parameter_types/generic_method_instances accept exact names (native type names match retained rendered values, preserving same-name rows; null names never match), types also accept image_indices/namespaces/declaring_types, and methods/members accept declaring_types. Parameters and parameter_types accept method_indices; code_gen_modules accept image_indices; method_addresses and generic_method_instances accept addresses/states (generic_method_instances also method_indices), preserving every folded alternative; generic_method_issues accept rows. Row index is scoped to the metadata digest; native pages add binary_source. Uncollected native tables have null counts and remain unavailable. Desktop directory views use kind desktop-application and collection files/omissions/candidates/components/next_probes, with fields and pagination; files and omissions accept exact paths or their stable IDs, and candidates accept families. Source coverage and stable row IDs remain available. Native load-image Evidence uses kind native-load-image with collection source_files/mappings/relocations/entry_points/entry_context/checks/sections, original indices, collection-specific fields and offset/limit paging. Null collections remain unrecorded or inapplicable; pages preserve unsupported verification. Native inventory Evidence from list_names, list/search_procedures or list/search_strings uses view: {kind: "native-inventory", offset: 0, limit: 64, fields: ["value"]}. Rows keep source_index and address even with selected fields. Summary counts describe retained rows; provider discovery coverage remains unknown. Native function Evidence accepts kind summary for identity, availability and facet counts, or kind native for selected facts: view: {kind: "native", facet: "procedure"}. Facets select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native_api and value-flow observations. For parameter type paths when value flow is recorded, use facet: "value_flow_parameters"; its decompiler-observed path may be passed to inspect_native_data_type, but exact database lookup can be unavailable and does not by itself establish type identity. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Retained batch_decompile Evidence accepts kind summary for counts and sizes, or view: {kind: "batch-decompile", collection: "items", where: {statuses: ["unavailable", "error"]}} for body-free rows with exact indices, addresses, procedure_addresses or statuses (include_pseudocode adds bodies), or collection "pseudocode" with an index to page one body. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.', inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ diff --git a/src/contracts/apktool/apktoolToolContracts.ts b/src/contracts/apktool/apktoolToolContracts.ts index 0848ca4ae8..690a2daa1e 100644 --- a/src/contracts/apktool/apktoolToolContracts.ts +++ b/src/contracts/apktool/apktoolToolContracts.ts @@ -22,7 +22,7 @@ export const APKTOOL_TOOL_CONTRACTS = [ name: "decode_android_resources", ...toolContractMetadata("decode_android_resources"), description: - "Decode one local APK's resources with `apktool d --no-src` in a provider-owned workspace that is removed after projection: the readable AndroidManifest.xml, apktool.yml version and SDK facts, the app's string resources (default or selected locales, bounded to 2000 entries per table), selectable string locales, and the decoded file count. The target APK is digest with SHA-256 before decoding; smali is skipped because Java-source recovery belongs to the JADX family. Optional decode_timeout_ms is the decode wall clock in milliseconds (default 600000, range 1..2147483647); omitted means 600000 and the value applies on the next decode. It does not raise the fixed output capture allowance or the version probe.", + "Decode one local APK's resources with `apktool d --no-src` in a provider-owned workspace that is removed after projection: the readable AndroidManifest.xml, apktool.yml version and SDK facts, selected string resources, selectable locales, and decoded file count. Strings default to the first 2000 valid entries per table; string_offset/string_limit retrieve later pages, with table.page.next_offset and full source counts. Verify target.sha256 across calls. Per-call max_total_strings, max_string_table_bytes and max_total_string_table_bytes adjust output/read budgets without a REA restart. Smali is skipped because Java-source recovery belongs to JADX. decode_timeout_ms is the decode wall clock in milliseconds (default 600000, range 1..2147483647); it does not raise output capture or version-probe limits.", kind: "apktool-provider", inputSchema: apktoolInputSchemas.decode_android_resources, outputSchema: evidenceResultOf( @@ -45,6 +45,15 @@ export const APKTOOL_TOOL_CONTRACTS = [ string_tables_only: true, }, }, + { + title: "Retrieve strings after the default first page", + input: { + path: "/targets/Example.apk", + string_offset: 2000, + string_limit: 100, + string_tables_only: true, + }, + }, ], }, ] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/artifactToolContracts.ts b/src/contracts/artifactToolContracts.ts index 1dd20e3a66..a937bd735b 100644 --- a/src/contracts/artifactToolContracts.ts +++ b/src/contracts/artifactToolContracts.ts @@ -13,9 +13,15 @@ import { dylibResolutionInputSchema } from "../domain/apple/dylibResolution.js"; import { keyedArchiveInputSchema } from "../domain/apple/keyedArchive.js"; import { absoluteLocalPathStringSchema } from "../domain/localPath.js"; import { analysisDetailSchema } from "./analysisDetail.js"; +import { artifactReaderLimitsInputSchema } from "../domain/artifactReaderLimits.js"; /** Exact caller boundary for deterministic artifact inventory. */ export const artifactInventoryInputSchema = z.strictObject({ integrity_policy: artifactIntegrityPolicySchema, + artifact_limits: artifactReaderLimitsInputSchema + .exactOptional() + .describe( + "Per-call Bun, .NET bundle, Node SEA and Godot producer admission, plus optional aggregate expanded plaintext work. Omitted fields retain existing defaults; changes apply on the next call without restart. Independent of summary/paging and MCP response limits.", + ), }); /** Delivery selection belongs to the adapter, not the inventory producer. */ @@ -31,6 +37,7 @@ export const artifactInspectionInputSchema = /** Omitted selection preserves complete extraction of the active reader. */ export const artifactExtractionInputSchema = z.strictObject({ + artifact_limits: artifactInventoryInputSchema.shape.artifact_limits, integrity_policy: artifactIntegrityPolicySchema, output_root: absoluteLocalPathStringSchema .exactOptional() diff --git a/src/contracts/evm/evmToolContracts.ts b/src/contracts/evm/evmToolContracts.ts index 824811b73a..6578a2363f 100644 --- a/src/contracts/evm/evmToolContracts.ts +++ b/src/contracts/evm/evmToolContracts.ts @@ -24,7 +24,7 @@ export const INSPECT_EVM_INTERFACE_CONTRACT = { kind: "artifact-provider", requiresActiveTarget: true, description: - 'Inspect explicit local flat EVM bytecode without target execution or chain/RPC access. Select raw or hex encoding explicitly. Omitted or complete detail returns original carrier and decoded identities, full bytecode and all inferred interface candidates inline. For all inferred selectors/offsets/arguments/mutability and identities without bytecode hex or duplicated stdout, select detail: "summary" on this first call: it retains unchanged complete Evidence and returns a distinct linked inspect_analysis_view summary. Read original function rows or bytecode_hex/stdout/stderr text pages using view kind evm and normalized_result.parent_evidence_id; original complete Evidence remains accessible. Bytecode kind, hardfork, deployed authenticity and discovery completeness stay unknown; summary coverage covers only producer-returned candidates. Unchanged bundled EVMole 0.9.3/WASM in an owned bounded worker is initially real-verified on Linux x64; EOF-style EF00 containers are unsupported. Adjust timeout_ms (default 30000 ms, integer 1..2147483647) for the owned worker command on the next call; omitted timeout_ms keeps 30000 ms. Fixed byte, address-space, and CPU allowances stay unchanged. No signature lookup, wallet action or transaction broadcast.', + 'Inspect explicit local flat EVM bytecode without target execution or chain/RPC access. Select raw or hex encoding explicitly. Omitted or complete detail returns original carrier and decoded identities, full bytecode and all inferred interface candidates inline. For all inferred selectors/offsets/arguments/mutability and identities without bytecode hex or duplicated stdout, select detail: "summary" on this first call: it retains unchanged complete Evidence and returns a distinct linked inspect_analysis_view summary. Read original function rows or bytecode_hex/stdout/stderr text pages using view kind evm and normalized_result.parent_evidence_id; original complete Evidence remains accessible. Bytecode kind, hardfork, deployed authenticity and discovery completeness stay unknown; summary coverage covers only producer-returned candidates. Unchanged bundled EVMole 0.9.3/WASM in an owned bounded worker is initially real-verified on Linux x64; EOF-style EF00 containers are unsupported. Adjust timeout_ms (default 30000 ms, integer 1..2147483647) for the owned worker command on the next call; omitted timeout_ms keeps 30000 ms. Per-call producer_limits adjusts carrier/reply/diagnostic bytes, address-space/file bytes and CPU seconds; omitted file_bytes follows selected output_bytes. Tighter inherited Linux limits stay enforced; timeout_ms and the 256 MiB worker heap remain independent. No signature lookup, wallet action or transaction broadcast.', inputSchema: evmInterfaceInspectionInputSchema, outputSchema: analysisDetailOutputSchemaOf( evmInterfaceSchema, diff --git a/src/contracts/flutter/flutterToolContracts.ts b/src/contracts/flutter/flutterToolContracts.ts index 8976971118..5767f090d0 100644 --- a/src/contracts/flutter/flutterToolContracts.ts +++ b/src/contracts/flutter/flutterToolContracts.ts @@ -30,7 +30,7 @@ export const FLUTTER_TOOL_CONTRACTS = [ name: "inspect_dart_aot", ...toolContractMetadata("inspect_dart_aot"), description: - 'Inspect one ABI\'s legacy split VM/isolate Dart AOT snapshot inside an APK (complete structural coverage: Dart 2.14 FullAOT kind 3). Unified _kDartSnapshotData/_kDartSnapshotText profiles are unsupported. Returns the four snapshot exports through the ELF dynamic symbol table with offsets, sizes and headers (magic, kind, length), snapshot hash, and a bounded string-pool projection from isolate data (package: source URIs, dart: SDK URIs, identifier-like tokens). String-pool entries are byte-pattern observations, not deserialized cluster semantics; typed class and function recovery needs an SDK-specific parser. Omitted or complete detail returns the full observation. Select detail: "summary" for target/ABI/library identity, all snapshot headers, coverage and original lower-bound counts without string lists. The complete source stays retained; read returned strings via inspect_analysis_view with parent_evidence_id and kind dart-aot, collection package_uris/dart_uris/class_like_tokens, offset and limit. Exhausting pages does not imply complete pool coverage. No engine, no execution.', + 'Inspect one ABI\'s legacy split VM/isolate Dart AOT snapshot inside an APK (complete structural coverage: Dart 2.14 FullAOT kind 3). Unified _kDartSnapshotData/_kDartSnapshotText profiles are unsupported. Returns the four snapshot exports through the ELF dynamic symbol table with offsets, sizes and headers (magic, kind, length), snapshot hash, and a bounded string-pool projection from isolate data (package: source URIs, dart: SDK URIs, identifier-like tokens). Use pool_policy.contains to select omitted strings before collection, or raise uri_limit/token_limit/run_bytes/token_bytes for the next call without restart. pool_policy does not raise APK, library, ELF or response limits. String-pool entries are byte-pattern observations, not deserialized cluster semantics; typed class and function recovery needs an SDK-specific parser. Omitted or complete detail returns the full observation. Select detail: "summary" for target/ABI/library identity, all snapshot headers, coverage and original lower-bound counts without string lists. The complete source stays retained; read returned strings via inspect_analysis_view with parent_evidence_id and kind dart-aot, collection package_uris/dart_uris/class_like_tokens, offset and limit. Exhausting pages does not imply complete pool coverage. No engine, no execution.', kind: "flutter-provider", inputSchema: dartAotInspectionInputSchema, outputSchema: evidenceResultOf( diff --git a/src/contracts/fridaToolContracts.ts b/src/contracts/fridaToolContracts.ts index 46244a3f73..8c20c5edb3 100644 --- a/src/contracts/fridaToolContracts.ts +++ b/src/contracts/fridaToolContracts.ts @@ -1,3 +1,7 @@ +import { + fridaCapturePolicySchema, + fridaCapturePolicyInputSchema, +} from "../domain/fridaCapturePolicy.js"; import { z } from "zod"; import { localPathStringSchema } from "../domain/localPath.js"; @@ -56,6 +60,7 @@ const exclusiveDeviceSelector = (schema: Schema) => const sessionId = z.string().uuid(); const scriptOutput = z.strictObject({ + capture_limits: fridaCapturePolicySchema.optional(), script_id: z.string().uuid(), source_kind: z.enum(["inline", "file"]), source_path: z.string().nullable(), @@ -107,6 +112,7 @@ export const FRIDA_TOOL_CONTRACTS = [ inputSchema: z .strictObject({ mode: z.enum(["attach", "spawn"]), + capture_limits: fridaCapturePolicyInputSchema.optional(), ...deviceSelector.shape, pid: z.int().positive().optional(), program: localPathStringSchema.optional(), @@ -245,6 +251,7 @@ export const FRIDA_TOOL_CONTRACTS = [ inputSchema: z .strictObject({ mode: z.enum(["attach", "spawn"]), + capture_limits: fridaCapturePolicyInputSchema.optional(), ...deviceSelector.shape, pid: z.int().positive().optional(), program: localPathStringSchema.optional(), diff --git a/src/contracts/go/goToolContracts.ts b/src/contracts/go/goToolContracts.ts index d24d051638..ae80281b8b 100644 --- a/src/contracts/go/goToolContracts.ts +++ b/src/contracts/go/goToolContracts.ts @@ -15,7 +15,7 @@ export const GO_TOOL_CONTRACTS = [ kind: "artifact-provider", requiresActiveTarget: true, description: - "Inspect embedded Go compiler version, main module, dependency/replacement versions, build settings and optional inferred static (ptr, len) UTF-8 string literals in an explicit local ELF, PE or thin Mach-O image, or the slice of a universal Mach-O named by architecture (read without lipo; artifact stays the universal file, selected_slice records the slice and source ranges stay file offsets). Default detail complete returns original artifact SHA-256, exact build-info source ranges, raw framed module bytes and unparsed module-text lines inline as observed Evidence; include_string_literals: true adds bounded deduplicated string_literals (default max_string_literals 2,000, max 10,000, <=4,096 bytes each, <=512 KiB aggregate, 64 MiB header-scan budget) recovered from word-aligned (ptr, len) headers in non-executable mapped data pointing within single read-only spans. Explicit detail summary retains the same complete parent and returns compiler/main-module identity, counts, source ranges and parent_evidence_id. Query retained rows through inspect_analysis_view kind go, collection dependencies, settings or string_literals (exact values or substring contains when collected); select indices, fields and pagination. Rows always retain their original index scoped to the parent Evidence; empty filters select none. Runtime dependency completeness and string reachability remain unknown. Supports inline and historical pointer encodings without installing Go, opening a disassembler, executing the target, writing temporary files or accessing the network. Missing metadata is reported as unknown, not proof that an image is not Go. Does not recover functions, source mappings, types or runtime behavior. Resource guards bound files to 256 MiB and charge each distinct build-info search span in full against a cumulative 256 MiB header-search budget, with partial overlaps charged separately; they also bound aggregate embedded build-info strings to 1 MiB, structural table decoding to 16 MiB and complete inline metadata to 8 MiB; malformed, unsupported and oversized inputs receive typed failures rather than partial success.", + "Inspect embedded Go compiler version, main module, dependency/replacement versions, build settings and optional inferred static (ptr, len) UTF-8 string literals in an explicit local ELF, PE or thin Mach-O image, or the slice of a universal Mach-O named by architecture (read without lipo; artifact stays the universal file, selected_slice records the slice and source ranges stay file offsets). Default detail complete returns original artifact SHA-256, exact build-info source ranges, raw framed module bytes and unparsed module-text lines inline as observed Evidence; include_string_literals: true adds bounded deduplicated string_literals (default max_string_literals 2,000, max_string_literal_bytes 4,096, max_string_bytes 512 KiB, max_string_header_scan_bytes 64 MiB and max_string_candidates 100,000; each budget is selectable per call) recovered from word-aligned (ptr, len) headers in non-executable mapped data pointing within single read-only spans. Explicit detail summary retains the same complete parent and returns compiler/main-module identity, counts, source ranges and parent_evidence_id. Query retained rows through inspect_analysis_view kind go, collection dependencies, settings or string_literals (exact values or substring contains when collected); select indices, fields and pagination. Rows always retain their original index scoped to the parent Evidence; empty filters select none. Runtime dependency completeness and string reachability remain unknown. Supports inline and historical pointer encodings without installing Go, opening a disassembler, executing the target, writing temporary files or accessing the network. Missing metadata is reported as unknown, not proof that an image is not Go. Does not recover functions, source mappings, types or runtime behavior. Per-call max_input_bytes defaults to 256 MiB, max_header_search_bytes to 256 MiB of cumulative distinct search-span work (partial overlaps charged separately), max_build_info_bytes to 1 MiB and max_report_bytes to 8 MiB. Increase the exact reported request budget and retry the same complete artifact without restarting; these controls do not raise MCP delivery limits or the separate 16 MiB structural decoding boundary. Malformed, unsupported and oversized inputs receive typed failures rather than partial success.", inputSchema: inspectGoBinaryInputSchema, outputSchema: analysisDetailOutputSchemaOf( goBinarySchema, diff --git a/src/contracts/managed/managedToolContracts.ts b/src/contracts/managed/managedToolContracts.ts index a0aaa50563..cae4c0f38c 100644 --- a/src/contracts/managed/managedToolContracts.ts +++ b/src/contracts/managed/managedToolContracts.ts @@ -6,9 +6,11 @@ import { managedOutputSchemas } from "../toolOutputSchemaGroups.js"; import { toolContractMetadata } from "../toolEffects.js"; import { absoluteLocalPathStringSchema } from "../../domain/localPath.js"; import { analysisDetailSchema } from "../analysisDetail.js"; +import { managedInspectionBudgetShape } from "../../domain/managed/managedInspectionLimits.js"; /** Exact caller boundary for execution-free PE/CLI triage and identity. */ export const managedTargetInputSchema = z.strictObject({ + ...managedInspectionBudgetShape, path: absoluteLocalPathStringSchema .describe( "Explicit absolute managed PE/CLI path; omit to reuse the previously selected managed target", diff --git a/src/contracts/native/binaryDiagnosticsToolContracts.ts b/src/contracts/native/binaryDiagnosticsToolContracts.ts index ac29afe750..53f504080b 100644 --- a/src/contracts/native/binaryDiagnosticsToolContracts.ts +++ b/src/contracts/native/binaryDiagnosticsToolContracts.ts @@ -355,7 +355,7 @@ export const BINARY_DIAGNOSTICS_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_binary_layout"), kind: "native-provider", description: - "Inspect an explicit local binary without launching it as a host process. Returns file-backed section/segment ranges, linked addresses, original symbol/REL/RELA table identities, complete encoded RELR tables and upstream-derived offsets with unknown per-word locations/addends, raw name bytes, dependency/interpreter names and static mitigation inferences inline with artifact SHA-256 Evidence. Profile: EXEC/DYN/REL ELF64 x86-64, ELF64 little-endian AArch64, ELF32 i386, ELF32 little-endian ARM and ELF32 little- or big-endian MIPS on Linux x64 via caller-supplied Python with unchanged pwntools 4.15.0, pyelftools 0.33 and Unicorn 2.1.2. Runtime addresses/library paths, overall relocation inventory completeness and complete derived GOT/PLT coverage remain unknown. Zero executable entry values mean absence; relocatable entries are not applicable. Upstream can emulate PLT instructions with Unicorn for derived static maps. Uses bounded complete output and owned temporary files/processes; malformed, unsupported or oversized objects return no partial success. Select detail summary to retain the complete layout Evidence in this session and receive only its summary view, then inspect sections, symbols, mitigations or linkage with inspect_analysis_view. pwntools_timeout_ms is each owned pwntools command's deadline (omitted means 30000 ms, range 1..2147483647, next call). cpuSeconds 30 and the output, diagnostic, and address-space byte allowances have no runtime override.", + "Inspect an explicit local binary without launching it as a host process. Returns file-backed section/segment ranges, linked addresses, original symbol/REL/RELA table identities, complete encoded RELR tables and upstream-derived offsets with unknown per-word locations/addends, raw name bytes, dependency/interpreter names and static mitigation inferences inline with artifact SHA-256 Evidence. Profile: EXEC/DYN/REL ELF64 x86-64, ELF64 little-endian AArch64, ELF32 i386, ELF32 little-endian ARM and ELF32 little- or big-endian MIPS on Linux x64 via caller-supplied Python with unchanged pwntools 4.15.0, pyelftools 0.33 and Unicorn 2.1.2. Runtime addresses/library paths, overall relocation inventory completeness and complete derived GOT/PLT coverage remain unknown. Zero executable entry values mean absence; relocatable entries are not applicable. Upstream can emulate PLT instructions with Unicorn for derived static maps. Uses bounded complete output and owned temporary files/processes; malformed, unsupported or oversized objects return no partial success. Select detail summary to retain the complete layout Evidence in this session and receive only its summary view, then inspect sections, symbols, mitigations or linkage with inspect_analysis_view. pwntools_timeout_ms is each owned pwntools command's deadline (omitted means 30000 ms, range 1..2147483647, next call). producer_limits selects input/output/diagnostic/file/address-space bytes and CPU seconds for the next call; defaults remain unchanged and inherited tighter OS limits are retained. pwntools_timeout_ms does not raise those independent limits.", inputSchema: inspectBinaryLayoutRequestSchema, outputSchema: analysisDetailOutputSchemaOf( binaryLayoutSchema, @@ -377,7 +377,7 @@ export const BINARY_DIAGNOSTICS_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_recorded_crash"), kind: "native-provider", description: - "Inspect a supplied Linux ELF64 amd64 or AArch64 little-endian core without launching its target or attaching to any live PID. Returns all recorded segments, raw note owners/descriptors and source ranges, every Linux CORE PRSTATUS thread/register and SIGINFO fields inline with SHA-256 Evidence. Historical PID is recorded metadata; current process/file identity, note coverage and signal-thread associations stay unknown. Optionally add derived core-only GDB/pwndbg mapping candidates through explicit BYO configuration; zero map flags mean unknown permissions. Uses the same stable snapshot for both stages, bounded complete output and owned cleanup. Linux x64 host profile: unchanged pwntools 4.15.0/pyelftools 0.33/Unicorn 2.1.2; optional unchanged pwndbg 2026.09.15. No implicit executable/library pairing, GDB command language, host configuration changes or dependency installation. pwntools_timeout_ms is each owned pwntools command's deadline (omitted means 30000 ms, range 1..2147483647, next call). cpuSeconds 30 and the output, diagnostic, and address-space byte allowances have no runtime override.", + "Inspect a supplied Linux ELF64 amd64 or AArch64 little-endian core without launching its target or attaching to any live PID. Returns all recorded segments, raw note owners/descriptors and source ranges, every Linux CORE PRSTATUS thread/register and SIGINFO fields inline with SHA-256 Evidence. Historical PID is recorded metadata; current process/file identity, note coverage and signal-thread associations stay unknown. Optionally add derived core-only GDB/pwndbg mapping candidates through explicit BYO configuration; zero map flags mean unknown permissions. Uses the same stable snapshot for both stages, bounded complete output and owned cleanup. Linux x64 host profile: unchanged pwntools 4.15.0/pyelftools 0.33/Unicorn 2.1.2; optional unchanged pwndbg 2026.09.15. No implicit executable/library pairing, GDB command language, host configuration changes or dependency installation. pwntools_timeout_ms is each owned pwntools command's deadline (omitted means 30000 ms, range 1..2147483647, next call). producer_limits selects input/output/diagnostic/file/address-space bytes and CPU seconds for the next call; defaults remain unchanged and inherited tighter OS limits are retained. pwntools_timeout_ms does not raise those independent limits.", inputSchema: inspectRecordedCrashInputSchema, outputSchema: evidenceResultOf(recordedCrashSchema), examples: [ diff --git a/src/contracts/native/nativeToolContracts.ts b/src/contracts/native/nativeToolContracts.ts index 112f365101..5c789531ca 100644 --- a/src/contracts/native/nativeToolContracts.ts +++ b/src/contracts/native/nativeToolContracts.ts @@ -96,7 +96,7 @@ export const NATIVE_TOOL_CONTRACTS = [ ), native( "observe_native_calls", - "Launch the active Mach-O as an owned process under LLDB and record each entry into caller-selected functions or Objective-C methods: thread, module, symbol, load and file address, raw argument registers, selector, receiver class and optional caller frames. Every hit auto-continues. No expression is evaluated and no other process is attached. The process is killed when max_events or duration_ms is reached; exit status, captured stdout/stderr, unresolved breakpoints and limitations are returned inline. The process runs with the current user's permissions and may change files, show UI or use the network. A hardened-runtime target needs the get-task-allow entitlement.", + "Launch the active Mach-O as an owned process under LLDB and record each entry into caller-selected functions or Objective-C methods: thread, module, symbol, load and file address, raw argument registers, selector, receiver class and optional caller frames. Every hit auto-continues. No expression is evaluated and no other process is attached. The process is killed when max_events or duration_ms is reached; exit status, captured stdout/stderr, unresolved breakpoints and limitations are returned inline. The process runs with the current user's permissions and may change files, show UI or use the network. A hardened-runtime target needs the get-task-allow entitlement. Select capture_limits before launch to adjust trace bytes/frames, breakpoint metadata, repeated stops, selector reads and retained output; omitted fields keep defaults, and changing them cannot recover an earlier observation.", nativeCallObservationInputSchema, ), native( diff --git a/src/contracts/reverseEngineeringToolContracts.ts b/src/contracts/reverseEngineeringToolContracts.ts index b0098803e1..5a3299ac0a 100644 --- a/src/contracts/reverseEngineeringToolContracts.ts +++ b/src/contracts/reverseEngineeringToolContracts.ts @@ -17,7 +17,7 @@ export const REVERSE_ENGINEERING_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_with_objdump"), kind: "native-provider", description: - "Run a caller-selected objdump command using GNU-style arguments against a local artifact, or request operation=architectures without a path to capture its response to -i. The executable implementation and version are not verified; other implementations may differ in options and output. For operation=disassemble, optional symbol selects native symbol-scoped disassembly before output collection; omission preserves all code sections. Matching and symbol boundaries follow the selected executable. Preserves decoded stdout and stderr inline with SHA-256 Evidence for artifact inspections, up to a shared 16 MiB combined limit; invalid UTF-8 bytes are represented by replacement characters and flagged in Evidence limitations. output_truncated reports output beyond that limit and completion_status is unknown if buffer enforcement terminated the process. The raw output is authoritative and REA does not claim a stable normalized format. DWARF does not follow local debug links by default. Set follow_debug_links to true to follow them; identities of external debug files are not bound to Evidence. REA clears DEBUGINFOD_URLS for objdump so DWARF processing does not make debuginfod network requests. Default detail complete returns original Evidence. Explicit detail summary retains the complete parent and returns its command observation once, without the raw mirror; parent_evidence_id supports later inspect_analysis_view summary queries and complete Evidence export. Architecture output is subjectless and does not establish support for any artifact. Optional command_timeout_ms is the per-phase wall clock in milliseconds (default 120000, range 1..2147483647); omitted means 120000 and the value applies on the next call. It does not raise the 16 MiB output cap.", + "Run a caller-selected objdump command using GNU-style arguments against a local artifact, or request operation=architectures without a path to capture its response to -i. The executable implementation and version are not verified; other implementations may differ in options and output. For operation=disassemble, optional symbol selects native symbol-scoped disassembly before output collection; omission preserves all code sections. Matching and symbol boundaries follow the selected executable. Preserves decoded stdout and stderr inline with SHA-256 Evidence for artifact inspections, up to max_output_bytes (a shared 16 MiB combined limit by default); invalid UTF-8 bytes are represented by replacement characters and flagged in Evidence limitations. output_truncated reports output beyond that limit and completion_status is unknown if buffer enforcement terminated the process. The raw output is authoritative and REA does not claim a stable normalized format. DWARF does not follow local debug links by default. Set follow_debug_links to true to follow them; identities of external debug files are not bound to Evidence. REA clears DEBUGINFOD_URLS for objdump so DWARF processing does not make debuginfod network requests. Default detail complete returns original Evidence. Explicit detail summary retains the complete parent and returns its command observation once, without the raw mirror; parent_evidence_id supports later inspect_analysis_view summary queries and complete Evidence export. Architecture output is subjectless and does not establish support for any artifact. Optional command_timeout_ms is the per-phase wall clock in milliseconds (default 120000, range 1..2147483647); omitted means 120000 and the value applies on the next call. Output admission is selected independently with max_output_bytes; raising it affects the next call without restarting REA and does not raise client receive limits.", inputSchema: objdumpInputSchema, outputSchema: analysisDetailOutputSchemaOf( commandObservationSchema, @@ -39,7 +39,7 @@ export const REVERSE_ENGINEERING_TOOL_CONTRACTS = [ ...toolContractMetadata("execute_rizin_command"), kind: "native-provider", description: - "Execute one caller-selected Rizin command against a local artifact using the Rizin CLI profile that suppresses user settings and scripts while retaining plugins. Preserves decoded stdout and stderr inline with SHA-256 Evidence, up to a shared 16 MiB combined limit; invalid UTF-8 bytes are represented by replacement characters and flagged in Evidence limitations. output_truncated reports output beyond that limit and completion_status is unknown if buffer enforcement terminated the process. Rizin commands and retained plugins can execute foreign code, mutate files, access the network, and read files outside the selected artifact; identities and contents of those external inputs are not bound to Evidence. Command effects depend on the Rizin command and loaded plugins; JSON is command-specific and raw output remains authoritative. For a raw image, asm_arch, asm_bits and asm_cpu select the decoder and base_address maps file offset 0 there; base_address also passes -n, which skips format loaders even for an ELF, and REA confirms neither. In a shared MCP server, this admitted command closes any active binary-analysis session; after its effects have settled, call open_binary before further analysis. Close an active Ghidra session explicitly first. Optional command_timeout_ms is the per-phase wall clock in milliseconds (default 120000, range 1..2147483647); omitted means 120000 and the value applies on the next call. It does not raise the 16 MiB output cap or a Rizin debugger session deadline.", + "Execute one caller-selected Rizin command against a local artifact using the Rizin CLI profile that suppresses user settings and scripts while retaining plugins. Preserves decoded stdout and stderr inline with SHA-256 Evidence, up to max_output_bytes (a shared 16 MiB combined limit by default); invalid UTF-8 bytes are represented by replacement characters and flagged in Evidence limitations. output_truncated reports output beyond that limit and completion_status is unknown if buffer enforcement terminated the process. Rizin commands and retained plugins can execute foreign code, mutate files, access the network, and read files outside the selected artifact; identities and contents of those external inputs are not bound to Evidence. Command effects depend on the Rizin command and loaded plugins; JSON is command-specific and raw output remains authoritative. For a raw image, asm_arch, asm_bits and asm_cpu select the decoder and base_address maps file offset 0 there; base_address also passes -n, which skips format loaders even for an ELF, and REA confirms neither. In a shared MCP server, this admitted command closes any active binary-analysis session; after its effects have settled, call open_binary before further analysis. Close an active Ghidra session explicitly first. Optional command_timeout_ms is the per-phase wall clock in milliseconds (default 120000, range 1..2147483647); omitted means 120000 and the value applies on the next call. Output admission is selected independently with max_output_bytes; raising it affects the next call without restarting REA and does not raise client receive limits or a Rizin debugger session deadline.", inputSchema: rizinInputSchema, outputSchema: evidenceResultOf(commandObservationSchema), examples: [ diff --git a/src/contracts/wasm/wasmComparisonContracts.ts b/src/contracts/wasm/wasmComparisonContracts.ts index a54a2cd3ae..fca6e863a9 100644 --- a/src/contracts/wasm/wasmComparisonContracts.ts +++ b/src/contracts/wasm/wasmComparisonContracts.ts @@ -1,7 +1,10 @@ import { evidenceResultOf } from "../toolOutputSchemaPrimitives.js"; import { z } from "zod"; import { evidenceInputSchema } from "../../domain/evidenceReference.js"; -import { wasmInterfaceSchema } from "../../domain/wasm/wasmInterface.js"; +import { + wasmInterfaceSchema, + wasmInterfaceByteBudgetSchema, +} from "../../domain/wasm/wasmInterface.js"; import { toolContractMetadata } from "../toolEffects.js"; import type { ToolContract } from "../toolContractTypes.js"; @@ -12,6 +15,7 @@ export const compareWasmInterfacesInputSchema = z.strictObject({ right: evidenceInputSchema.describe( "Exact complete inspect_wasm_artifact Evidence or retained session reference", ), + max_interface_bytes: wasmInterfaceByteBudgetSchema, scope: z .enum(["imports", "exports", "both"]) .describe( @@ -56,7 +60,7 @@ export const COMPARE_WASM_INTERFACES_CONTRACT = { ...toolContractMetadata("compare_wasm_interfaces"), kind: "application", description: - "Compare typed core WASM import/export names, kinds and exact signatures from two authenticated inspect_wasm_artifact Evidence records. Select imports, exports or both explicitly. MCP references resolve in the current session; CLI references use an explicitly selected --evidence-bundle. Duplicate names, unsupported types and missing historical interface facets remain unknown. Preserve original module digests and every occurrence; no source acquisition, module execution or runtime-compatibility inference.", + "Compare typed core WASM import/export names, kinds and exact signatures from two authenticated inspect_wasm_artifact Evidence records. Select imports, exports or both explicitly. MCP references resolve in the current session; CLI references use an explicitly selected --evidence-bundle. max_interface_bytes selects this comparison's independent representation allowance (default 32MiB); raising it cannot restore omitted producer facts. Duplicate names, unsupported types and missing historical interface facets remain unknown. Preserve original module digests and every occurrence; no source acquisition, module execution or runtime-compatibility inference.", inputSchema: compareWasmInterfacesInputSchema, outputSchema: evidenceResultOf(wasmInterfaceComparisonSchema), examples: [], diff --git a/src/contracts/wasm/wasmToolContracts.ts b/src/contracts/wasm/wasmToolContracts.ts index 3a62582982..9657f0b7b9 100644 --- a/src/contracts/wasm/wasmToolContracts.ts +++ b/src/contracts/wasm/wasmToolContracts.ts @@ -23,7 +23,7 @@ export const INSPECT_WASM_ARTIFACT_CONTRACT = { kind: "artifact-provider", requiresActiveTarget: true, description: - 'Validate an explicit local WASM artifact with caller-supplied WABT 1.0.42. Optional custom-section interpretation errors do not invalidate the core module; warnings and name-section interpretation status remain in the result. Omitted or complete detail returns exact byte SHA-256, sections, upstream import/export WAT forms and decoded WAT inline as Evidence. For counts, validation and WAT identity without WAT text, pass detail: "summary" on this first call; it retains the complete original Evidence and returns the same typed summary as inspect_analysis_view, avoiding oversized complete delivery. Read exact imports/exports or WAT pages with inspect_analysis_view using normalized_result.parent_evidence_id. Delivery selection does not change analysis, provider inputs or source identity. Adjust command_timeout_ms for each WABT command (default 30000 ms), separately from the client deadline; input/output safety limits stay fixed. Optionally associate selected JavaScript glue literals with explicit local candidates while preserving ambiguous URL/basename matches. Configure absolute REA_WABT_BIN_DIRECTORY. No installation, implicit fetch, WASM execution or original-source claim.', + 'Validate an explicit local WASM artifact with caller-supplied WABT 1.0.42. Optional custom-section interpretation errors do not invalidate the core module; warnings and name-section interpretation status remain in the result. Omitted or complete detail returns exact byte SHA-256, sections, upstream import/export WAT forms and decoded WAT inline as Evidence. For counts, validation and WAT identity without WAT text, pass detail: "summary" on this first call; it retains the complete original Evidence and returns the same typed summary as inspect_analysis_view, avoiding oversized complete delivery. Read exact imports/exports or WAT pages with inspect_analysis_view using normalized_result.parent_evidence_id. Delivery selection does not change analysis, provider inputs or source identity. Adjust command_timeout_ms for each WABT command (default 30000 ms), separately from the client deadline. max_artifact_paths admits auxiliary candidate/glue selections (default 16 paths per list). max_interface_bytes selects typed interface precision admission (default 32MiB), independently from producer text retention. max_input_bytes, max_tool_bytes and max_output_bytes select independent per-call file/tool/combined-command-output allowances with unchanged defaults of 8/64/32 MiB; limit errors name the exhausted control and retry action. Optionally associate selected JavaScript glue literals with explicit local candidates while preserving ambiguous URL/basename matches. Configure absolute REA_WABT_BIN_DIRECTORY. No installation, implicit fetch, WASM execution or original-source claim.', inputSchema: wasmArtifactInspectionInputSchema, outputSchema: analysisDetailOutputSchemaOf( wasmArtifactSchema, diff --git a/src/contracts/webScriptToolContracts.ts b/src/contracts/webScriptToolContracts.ts index 3947cfcf5a..0724df6149 100644 --- a/src/contracts/webScriptToolContracts.ts +++ b/src/contracts/webScriptToolContracts.ts @@ -1,3 +1,14 @@ +import { z } from "zod"; +import { analysisDetailOutputSchemaOf } from "./analysisDetail.js"; +import { + webSourceLocationCompleteDeliveryShape, + webSourceLocationSummaryDeliveryShape, +} from "../domain/analysisView/webSourceLocationView.js"; +import { + webSourceLocationSummaryViewResultSchema, + webSourceLocationMatchesViewResultSchema, + webSourceLocationTextViewResultSchema, +} from "../domain/analysisView/analysisView.js"; import { exportWebScriptsInputSchema, webScriptExportResultSchema, @@ -10,25 +21,40 @@ import { import { toolContractMetadata } from "./toolEffects.js"; import { evidenceResultOf } from "./toolOutputSchemaPrimitives.js"; import { + WEB_SOURCE_MAP_LIMITS, webSourceLocationInputSchema, webSourceLocationResultSchema, } from "../domain/webSourceLocation.js"; +export const webSourceLocationRequestSchema = z + .discriminatedUnion("detail", [ + webSourceLocationInputSchema.extend(webSourceLocationCompleteDeliveryShape), + webSourceLocationInputSchema.extend(webSourceLocationSummaryDeliveryShape), + ]) + .meta({ "x-rea-root-discriminator": "detail" }); + /** Local captured-script publication contracts shared by CLI and MCP. */ export const WEB_SCRIPT_TOOL_CONTRACTS = [ { name: "trace_web_source_location", ...toolContractMetadata("trace_web_source_location"), - description: - "Trace one generated position in an exported website script through an explicitly paired local source map. Verifies manifest, selected script and map SHA-256/byte identities; returns every equal-position mapping, reported source/root, resolved source URL, embedded sourcesContent and original position inline. One-based lines and zero-based UTF-16 columns; greatest generated position <= requested across lines. Duplicate sources and generated-only mappings remain distinct. Caller-selected pairing does not establish deployment authenticity or execution. Regular maps and inline indexed sections only; external section URLs are unsupported. No network access or application execution; uses the pinned upstream codec in an owned Node process with private temporary files, independent 192 MiB old-generation heap and 20-second deadline. 4 MiB map, 32 MiB reply; complete evidence or actionable resource failure. Cleanup may extend the deadline.", + description: `Trace one generated position in an exported website script through an explicitly paired local source map. Verifies manifest, selected script and map SHA-256/byte identities; returns every equal-position mapping, reported source/root, resolved source URL, embedded sourcesContent and original position inline. One-based lines and zero-based UTF-16 columns; greatest generated position <= requested across lines. Duplicate sources and generated-only mappings remain distinct. Caller-selected pairing does not establish deployment authenticity or execution. Regular maps and inline indexed sections only; external section URLs are unsupported. No network access or application execution; uses the pinned upstream codec in an owned Node process with private temporary files, independent default ${WEB_SOURCE_MAP_LIMITS.codecHeapMiB} MiB old-generation heap and ${WEB_SOURCE_MAP_LIMITS.decodeTimeoutMs / 1000}-second deadline. Defaults: ${WEB_SOURCE_MAP_LIMITS.mapBytes / (1024 * 1024)} MiB map, ${WEB_SOURCE_MAP_LIMITS.outputBytes / (1024 * 1024)} MiB reply, ${WEB_SOURCE_MAP_LIMITS.decodedRecords} decoder records. Select source_map_limits to change byte, decoder or heap admission for the next call without restart; failures name the exhausted field and recovery. Complete delivery remains the default. Select detail summary to retain complete Evidence and return source/lookup identities and body-free match metadata with explicit page coverage. An optional source-location matches view selects offset/limit/max_bytes; source-text selects an exact match_index and UTF-16 offset/limit, returning content identity and only the requested text. inspect_analysis_view queries the same parent without rereading artifacts; export_evidence_bundle preserves the complete record. Summary requires session retention. Complete evidence or actionable resource failure. Cleanup may extend the deadline.`, kind: "application", - inputSchema: webSourceLocationInputSchema, - outputSchema: evidenceResultOf(webSourceLocationResultSchema), + inputSchema: webSourceLocationRequestSchema, + outputSchema: analysisDetailOutputSchemaOf( + webSourceLocationResultSchema, + z.union([ + webSourceLocationSummaryViewResultSchema, + webSourceLocationMatchesViewResultSchema, + webSourceLocationTextViewResultSchema, + ]), + ), examples: [ { title: - "Trace a selected captured JavaScript point to embedded original source", + "Trace a selected captured JavaScript point with retained compact delivery", input: { + detail: "summary", manifest_path: "/analysis/scripts/manifest.json", script_index: 0, source_map: { diff --git a/src/desktop/DebianPackageInventory.ts b/src/desktop/DebianPackageInventory.ts index 9fd7b1ca5f..204d40996a 100644 --- a/src/desktop/DebianPackageInventory.ts +++ b/src/desktop/DebianPackageInventory.ts @@ -1,3 +1,4 @@ +import type { DesktopLibraryPolicy } from "../domain/desktop/libraryPolicy.js"; import { readdir } from "node:fs/promises"; import { @@ -41,6 +42,7 @@ export type DebianPackageHost = AptRepositoryHost; /** What package identification reads with and adds to. */ export interface InventoryContext { + readonly libraryPolicy?: DesktopLibraryPolicy; readonly host: DebianPackageHost; readonly signal: AbortSignal | undefined; readonly limitations: string[]; @@ -138,6 +140,7 @@ const installFacts = async ( installed.name ?? "", runQuery, context.signal, + context.libraryPolicy, ), ); context.limitations.push(...vendor.limitations); diff --git a/src/desktop/LibraryPackageInventory.test.ts b/src/desktop/LibraryPackageInventory.test.ts index c4d7a8016c..0e87a60e37 100644 --- a/src/desktop/LibraryPackageInventory.test.ts +++ b/src/desktop/LibraryPackageInventory.test.ts @@ -1,3 +1,4 @@ +import { desktopLibraryPolicy } from "../domain/desktop/libraryPolicy.js"; import { existsSync } from "node:fs"; import { mkdir, realpath, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -139,3 +140,60 @@ it.skipIf(process.platform === "win32")( expect(result).not.toHaveProperty("library_packages_reason"); }, ); + +it.skipIf(process.platform === "win32")( + "recovers a 513-library closure with the caller-selected capture policy", + async () => { + const root = await createTestTempDirectory("rea-library-policy-"); + const bin = join(root, "bin"); + const lib = join(root, "lib"); + await Promise.all([mkdir(bin), mkdir(lib)]); + const paths = Array.from({ length: 513 }, (_, index) => + join(lib, `lib${index}.so`), + ); + const executable = join(bin, "app"); + await Promise.all([ + writeFile(executable, elfImage({ needed: [paths[0] ?? ""] })), + ...paths.map((path, index) => + writeFile( + path, + elfImage({ + needed: + paths[index + 1] === undefined ? [] : [paths[index + 1] ?? ""], + }), + ), + ), + ]); + const query = async (command: string, args: readonly string[]) => ({ + stdout: + command === "ldconfig" + ? "" + : args + .slice(1) + .map((path) => `owner-${paths.indexOf(path)}: ${path}`) + .join("\n"), + exitCode: 0, + }); + const limited = await inspectLibraryPackages( + executable, + "app", + query, + undefined, + ); + expect(limited.library_packages).toHaveLength(512); + expect(limited.library_packages_reason).toContain( + "library-policy.libraries=512", + ); + expect(limited.library_packages_reason).toContain("--library-policy"); + const recovered = await inspectLibraryPackages( + executable, + "app", + query, + undefined, + desktopLibraryPolicy({ libraries: 513 }), + ); + expect(recovered.library_packages).toHaveLength(513); + expect(recovered.library_packages).toContain("owner-512"); + expect(recovered.library_packages_reason).toBeUndefined(); + }, +); diff --git a/src/desktop/LibraryPackageInventory.ts b/src/desktop/LibraryPackageInventory.ts index 257e670204..cd25f32f7e 100644 --- a/src/desktop/LibraryPackageInventory.ts +++ b/src/desktop/LibraryPackageInventory.ts @@ -1,3 +1,8 @@ +import { + desktopLibraryLimit, + desktopLibraryPolicy, + type DesktopLibraryPolicy, +} from "../domain/desktop/libraryPolicy.js"; import { realpath } from "node:fs/promises"; import type { DesktopPackage } from "../domain/desktop/desktopCapture.js"; @@ -35,13 +40,12 @@ export type LibraryPackages = { "library_packages_reason" | "uninstallable_library_packages" >; -/** Libraries resolved per executable; an Electron app loads about a hundred. */ -const MAX_LIBRARIES = 512; const QUERY_BATCH = 200; const elfDependencies = async ( path: string, signal: AbortSignal | undefined, + policy: DesktopLibraryPolicy, ): Promise> => { const options = { symlinks: "follow" as const, @@ -52,6 +56,7 @@ const elfDependencies = async ( readElfDependencies( (offset, length) => readFileHandleRange(handle, offset, length), initial.size, + policy, ), ); } catch (cause: unknown) { @@ -116,11 +121,12 @@ const packageOwners = async ( const incompleteReason = ( closure: Awaited>, + policy: DesktopLibraryPolicy, ): string | undefined => { const [first] = closure.unreadable; const reasons = [ ...(closure.truncated - ? [`REA stopped after ${String(MAX_LIBRARIES)} libraries`] + ? [desktopLibraryLimit(policy, "libraries", "REA dependency closure")] : []), ...(first === undefined ? [] @@ -141,6 +147,7 @@ export const inspectLibraryPackages = async ( ownPackage: string, runQuery: QueryRunner, signal: AbortSignal | undefined, + policy: DesktopLibraryPolicy = desktopLibraryPolicy(), ): Promise => { const none = (reason: string): LibraryPackages => ({ library_packages: [], @@ -148,7 +155,7 @@ export const inspectLibraryPackages = async ( unpackaged_libraries: [], library_packages_reason: reason, }); - const root = await elfDependencies(executable, signal); + const root = await elfDependencies(executable, signal, policy); if (!root.ok) return none(`the libraries of ${executable} were not read: ${root.error}`); const cache = await linkerCache(runQuery, signal); @@ -158,10 +165,10 @@ export const inspectLibraryPackages = async ( { path: executable, dependencies: root.value }, parseLinkerCache(cache), { - dependencies: (path) => elfDependencies(path, signal), + dependencies: (path) => elfDependencies(path, signal, policy), isFile: isHostFile, }, - MAX_LIBRARIES, + policy.libraries, ); const located = await Promise.all( closure.libraries.map(async ({ path }) => ({ @@ -196,7 +203,7 @@ export const inspectLibraryPackages = async ( else ownershipUnresolved = true; } const reason = [ - incompleteReason(closure), + incompleteReason(closure, policy), owners.ok || !ownershipUnresolved ? undefined : owners.error.reason, ] .filter((value): value is string => value !== undefined) diff --git a/src/desktop/LinuxApplicationInventory.ts b/src/desktop/LinuxApplicationInventory.ts index d6f2f20cbe..dd388294de 100644 --- a/src/desktop/LinuxApplicationInventory.ts +++ b/src/desktop/LinuxApplicationInventory.ts @@ -1,3 +1,7 @@ +import { + desktopLibraryPolicy, + type DesktopLibraryPolicyInput, +} from "../domain/desktop/libraryPolicy.js"; import { lstat, realpath, stat } from "node:fs/promises"; import { basename, dirname, isAbsolute, join, resolve } from "node:path"; @@ -37,6 +41,7 @@ const DESKTOP_ENTRY_MAX_BYTES = 1024 * 1024; /** Which application to launch, as the caller named it. */ export interface LinuxApplicationRequest { + readonly libraryPolicy?: DesktopLibraryPolicyInput; readonly desktopEntry: string | undefined; readonly executable: string | undefined; readonly arguments: readonly string[]; @@ -428,6 +433,7 @@ export const inspectLinuxApplication = async ( const { path } = found; const resolved = await realpath(path).catch(() => null); const context: InventoryContext = { + libraryPolicy: desktopLibraryPolicy(request.libraryPolicy), host, signal, limitations: [], diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index 7e2a80b66c..440e7e0a28 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -143,6 +143,16 @@ export const analysisErrorRemediationAction = ( ) return error.remediationAction; if (error instanceof AnalysisTimeoutError) return timeoutRemediation(error); + if ( + error instanceof ArtifactOperationError && + error.remediationAction !== undefined + ) + return error.remediationAction; + if ( + error instanceof ArtifactOperationError && + error.readerLimit !== undefined + ) + return error.userMessage ?? error.detail ?? error.message; if ( error instanceof ArtifactOperationError && error.reason === "limit" && diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 976963ff89..c4c536f254 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -524,6 +524,9 @@ const projectError = ( RETRYABLE_CODES.has(code) || (error instanceof AnalysisCapabilityUnavailableError && error.retryable) || + (error instanceof ArtifactOperationError && + error.readerLimit?.can_increase === true && + !error.cleanupIncomplete) || (code === "resource_constraint" && error instanceof AnalysisResourceConstraintError && (error.retryable ?? error.resource === "capacity")), @@ -795,13 +798,24 @@ const artifactStateErrorDetails = ( ...(error.reason === "limit" ? { truncated: true } : {}), ...(error.detail === undefined ? {} : { detail: error.detail }), ...(error.boundary === undefined ? {} : { boundary: error.boundary }), + ...(error.readerLimit === undefined + ? {} + : { reader_limit: { ...error.readerLimit } }), ...(error.admission === undefined ? {} : { admission: error.admission.kind, - maximum_header_bytes: error.admission.maximumBytes, - maximum_header_bytes_source: error.admission.source, - setting: "REA_ASAR_MAX_HEADER_BYTES", + ...(error.admission.kind === "asar-header" + ? { + maximum_header_bytes: error.admission.maximumBytes, + maximum_header_bytes_source: error.admission.source, + setting: "REA_ASAR_MAX_HEADER_BYTES", + } + : { + maximum_listing_bytes: error.admission.maximumBytes, + maximum_listing_bytes_source: error.admission.source, + setting: "REA_ASAR_MAX_LISTING_BYTES", + }), }), }; if (error instanceof UnknownRegistryError) diff --git a/src/domain/analysisResourceLimits.ts b/src/domain/analysisResourceLimits.ts index 806e48ef34..54a692b8d2 100644 --- a/src/domain/analysisResourceLimits.ts +++ b/src/domain/analysisResourceLimits.ts @@ -14,6 +14,19 @@ export const DEFAULT_ASAR_MAX_HEADER_BYTES = 16 * 1024 * 1024; */ export const MAXIMUM_ASAR_MAX_HEADER_BYTES = 256 * 1024 * 1024; +/** Expanded ASAR path/entry retention is independent of packed header bytes. */ +export const DEFAULT_ASAR_MAX_LISTING_BYTES = 64 * 1024 * 1024; +/** Accounting must remain exactly representable; actual heap admission is separate. */ +export const MAXIMUM_ASAR_MAX_LISTING_BYTES = Number.MAX_SAFE_INTEGER; + +export const DEFAULT_GHIDRA_VALUE_FLOW_MAX_OPERATIONS = 3_000; +export const DEFAULT_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION = 64; +export const DEFAULT_GHIDRA_VALUE_FLOW_MAX_EDGES = 12_000; +/** Bridge counters and p-code input indices are Java signed integers. */ +export const MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS = 2_147_483_647; +/** Ghidra 12.1.4 shifts MiB by 20 into a signed Java int response capacity. */ +export const MAXIMUM_GHIDRA_DECOMPILE_PAYLOAD_MIB = 2_047; + /** Native function-matching representation admitted when the setting is unset. */ export const DEFAULT_NATIVE_MATCHING_REPRESENTATION_BYTES = 96 * 1024 * 1024; diff --git a/src/domain/analysisSettingDefinitions.ts b/src/domain/analysisSettingDefinitions.ts index 1dd1feef6c..f6f2b030df 100644 --- a/src/domain/analysisSettingDefinitions.ts +++ b/src/domain/analysisSettingDefinitions.ts @@ -7,6 +7,13 @@ import type { import { DEFAULT_ASAR_MAX_HEADER_BYTES, MAXIMUM_ASAR_MAX_HEADER_BYTES, + DEFAULT_ASAR_MAX_LISTING_BYTES, + MAXIMUM_ASAR_MAX_LISTING_BYTES, + DEFAULT_GHIDRA_VALUE_FLOW_MAX_OPERATIONS, + DEFAULT_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION, + DEFAULT_GHIDRA_VALUE_FLOW_MAX_EDGES, + MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS, + MAXIMUM_GHIDRA_DECOMPILE_PAYLOAD_MIB, DEFAULT_GHIDRA_STARTUP_TIMEOUT_MS, DEFAULT_HOPPER_STARTUP_TIMEOUT_MS, MAX_NODE_TIMER_DELAY_MS, @@ -37,7 +44,7 @@ interface SettingDefinition { readonly defaultValue: number | null; readonly minimum: number; readonly maximum: number; - readonly unit: "bytes" | "ms" | "seconds" | "MiB"; + readonly unit: "bytes" | "ms" | "seconds" | "MiB" | "count"; readonly appliesAt: "next_operation" | "next_worker" | "next_startup"; readonly provider: string; readonly relatedProviders?: readonly string[]; @@ -52,6 +59,88 @@ const timer = { } as const; /** Metadata aggregates existing owner definitions; no environment mutation or engine discovery. */ export const ANALYSIS_SETTINGS = { + "ghidra.decompile_max_payload_mib": { + environment: "REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB", + defaultValue: null, + minimum: 1, + maximum: MAXIMUM_GHIDRA_DECOMPILE_PAYLOAD_MIB, + unit: "MiB", + appliesAt: "next_operation", + provider: "ghidra", + operations: [ + "procedure_pseudo_code", + "analyze_function", + "inspect_native_api", + "decompile_all", + "trace_native_values", + "trace_native_ui_action", + ], + }, + "asar.max_listing_bytes": { + environment: "REA_ASAR_MAX_LISTING_BYTES", + defaultValue: DEFAULT_ASAR_MAX_LISTING_BYTES, + minimum: 1, + maximum: MAXIMUM_ASAR_MAX_LISTING_BYTES, + unit: "bytes", + appliesAt: "next_operation", + provider: "rea-artifact-graph", + relatedProviders: [ + "rea-javascript-application", + "rea-cdp-electron", + "rea-v8-inspector", + ], + operations: [ + "open_binary", + "inspect_artifact", + "inventory_artifact", + "extract_artifact", + "analyze_javascript_application", + "inspect_electron_page", + "observe_javascript_runtime", + ], + }, + "ghidra.value_flow_max_operations": { + environment: "REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS", + defaultValue: DEFAULT_GHIDRA_VALUE_FLOW_MAX_OPERATIONS, + minimum: 1, + maximum: MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS, + unit: "count", + appliesAt: "next_operation", + provider: "ghidra", + operations: [ + "analyze_function", + "inspect_native_api", + "trace_native_values", + ], + }, + "ghidra.value_flow_max_inputs_per_operation": { + environment: "REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION", + defaultValue: DEFAULT_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION, + minimum: 1, + maximum: MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS, + unit: "count", + appliesAt: "next_operation", + provider: "ghidra", + operations: [ + "analyze_function", + "inspect_native_api", + "trace_native_values", + ], + }, + "ghidra.value_flow_max_edges": { + environment: "REA_GHIDRA_VALUE_FLOW_MAX_EDGES", + defaultValue: DEFAULT_GHIDRA_VALUE_FLOW_MAX_EDGES, + minimum: 1, + maximum: MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS, + unit: "count", + appliesAt: "next_operation", + provider: "ghidra", + operations: [ + "analyze_function", + "inspect_native_api", + "trace_native_values", + ], + }, "asar.max_header_bytes": { environment: "REA_ASAR_MAX_HEADER_BYTES", defaultValue: DEFAULT_ASAR_MAX_HEADER_BYTES, @@ -223,7 +312,10 @@ export const analysisSettingValueSchema = (id: AnalysisSettingId) => { .int() .min(definition.minimum) .max(definition.maximum); - return id === "ghidra.decompile_timeout_seconds" ? number.nullable() : number; + return id === "ghidra.decompile_timeout_seconds" || + id === "ghidra.decompile_max_payload_mib" + ? number.nullable() + : number; }; /** Project one owner's accepted budget, including the explicit per-call precedence. */ export const analysisSettingLimit = ( diff --git a/src/domain/analysisSettings.ts b/src/domain/analysisSettings.ts index 03476d1b46..b9251232ee 100644 --- a/src/domain/analysisSettings.ts +++ b/src/domain/analysisSettings.ts @@ -1,9 +1,14 @@ /** Runtime policy overrides, separate from target state and launch environment. */ export interface AnalysisSettingValues { readonly "asar.max_header_bytes": number; + readonly "asar.max_listing_bytes": number; + readonly "ghidra.value_flow_max_operations": number; + readonly "ghidra.value_flow_max_inputs_per_operation": number; + readonly "ghidra.value_flow_max_edges": number; readonly "ghidra.startup_timeout_ms": number; readonly "hopper.startup_timeout_ms": number; readonly "ghidra.decompile_timeout_seconds": number | null; + readonly "ghidra.decompile_max_payload_mib": number | null; readonly "jadx.operation_timeout_ms": number; readonly "regex.matching_deadline_ms": number; readonly "regex.worker_heap_request_mib": number; @@ -25,7 +30,7 @@ export type AnalysisSettingsReader = () => AnalysisSettingsSnapshot; export interface AnalysisSettingLimit { readonly setting: AnalysisSettingId; readonly effective_value: number | null; - readonly unit: "bytes" | "ms" | "seconds" | "MiB"; + readonly unit: "bytes" | "ms" | "seconds" | "MiB" | "count"; readonly minimum: number; readonly maximum: number; readonly source?: "default" | "setting" | "runtime" | "request"; diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 0c5215984c..353ff7f420 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -1,3 +1,4 @@ +import { projectWebSourceLocationView } from "./webSourceLocationView.js"; import { projectNativeProcedureReferencesView } from "./nativeProcedureReferencesView.js"; import { projectPlaywrightTraceView } from "./playwrightTraceView.js"; import { projectApiContractView } from "./apiContractView.js"; @@ -106,6 +107,9 @@ import { type InspectAnalysisViewInput, } from "./analysisViewRequest.js"; import { + webSourceLocationSummaryViewResultSchema, + webSourceLocationMatchesViewResultSchema, + webSourceLocationTextViewResultSchema, procedurePseudocodeSummaryViewResultSchema, nativeObservationSummaryViewResultSchemas, byteEntropySummaryViewResultSchema, @@ -183,6 +187,9 @@ export { type AnalysisViewParent, }; export { + webSourceLocationSummaryViewResultSchema, + webSourceLocationMatchesViewResultSchema, + webSourceLocationTextViewResultSchema, procedurePseudocodeSummaryViewResultSchema, nativeObservationSummaryViewResultSchemas, nativeFeatureTraceSummaryViewResultSchema, @@ -384,6 +391,7 @@ const retainedViewProjectors = new Map([ retain("inspect_artifact", projectArtifactInventoryView), retain("inventory_artifact", projectArtifactInventoryView), retain("analyze_web_bundle", projectWebBundleView), + retain("trace_web_source_location", projectWebSourceLocationView), ]); /** Budgets retained across source admission and the selected grouping projection. */ diff --git a/src/domain/analysisView/analysisViewRequest.ts b/src/domain/analysisView/analysisViewRequest.ts index b77dac46a7..a041f33f45 100644 --- a/src/domain/analysisView/analysisViewRequest.ts +++ b/src/domain/analysisView/analysisViewRequest.ts @@ -1,3 +1,4 @@ +import { webSourceLocationViewRequestSchema } from "./webSourceLocationView.js"; import { nativeProcedureReferencesViewRequestSchema } from "./nativeProcedureReferencesView.js"; import { graphqlContractViewRequestSchema } from "../apiContract/graphqlContractQuery.js"; import { playwrightTraceViewRequestSchema } from "../playwrightTrace/playwrightTraceQuery.js"; @@ -97,6 +98,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ nativeFeatureTraceViewRequestSchema, batchDecompileViewRequestSchema, summaryViewSchema, + webSourceLocationViewRequestSchema, goBinaryViewRequestSchema, rustBinaryViewRequestSchema, desktopViewRequestSchema, diff --git a/src/domain/analysisView/analysisViewResult.ts b/src/domain/analysisView/analysisViewResult.ts index dce7b88e67..ee22e35ad8 100644 --- a/src/domain/analysisView/analysisViewResult.ts +++ b/src/domain/analysisView/analysisViewResult.ts @@ -1,3 +1,10 @@ +import { + webSourceLocationViewRequestSchema, + webSourceLocationContextSchema, + webSourceLocationSummarySchema, + webSourceLocationMatchViewSchema, +} from "./webSourceLocationView.js"; +import { analysisTextPageItemSchema } from "./analysisViewText.js"; import { graphqlContractViewRequestSchema } from "../apiContract/graphqlContractQuery.js"; import { graphqlContractViewContextSchema, @@ -357,6 +364,7 @@ const javascriptSummarySchema = z.strictObject({ const viewResultBase = { parent_evidence_id: prefixedDigestSchema("ev"), parent_operation: z.enum([ + "trace_web_source_location", "inspect_python_pyc", "inspect_erlang_beam", "inspect_devicetree", @@ -410,6 +418,30 @@ const summaryViewResultBase = { ...viewResultBase, } as const; +export const webSourceLocationSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultBase, + parent_operation: z.literal("trace_web_source_location"), + summary: webSourceLocationSummarySchema, +}); +export const webSourceLocationMatchesViewResultSchema = z.strictObject({ + ...viewResultBase, + kind: z.literal("source-location"), + parent_operation: z.literal("trace_web_source_location"), + view: webSourceLocationViewRequestSchema.options[0], + source_location: webSourceLocationContextSchema, + items: z.array(webSourceLocationMatchViewSchema), +}); +export const webSourceLocationTextViewResultSchema = z.strictObject({ + ...viewResultBase, + kind: z.literal("source-location-text"), + coverage: nullableCollectionViewCoverageSchema, + parent_operation: z.literal("trace_web_source_location"), + view: webSourceLocationViewRequestSchema.options[1], + source_location: webSourceLocationContextSchema, + match: webSourceLocationMatchViewSchema, + item: analysisTextPageItemSchema.nullable(), +}); + /** Original selector and text availability without the complete decompiled text. */ export const procedurePseudocodeSummaryViewResultSchema = z.strictObject({ ...summaryViewResultBase, @@ -1332,6 +1364,7 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ view: environmentCoverageViewRequestSchema, }), z.discriminatedUnion("parent_operation", [ + webSourceLocationSummaryViewResultSchema, procedurePseudocodeSummaryViewResultSchema, nativeObservationSummaryViewResultSchema, nativeValueTraceSummaryViewResultSchema, @@ -1389,6 +1422,8 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ devicetree: devicetreePageContextSchema, items: devicetreeViewItemsSchema, }), + webSourceLocationMatchesViewResultSchema, + webSourceLocationTextViewResultSchema, erlangBeamTableViewResultSchema, pythonPycCodeObjectsViewResultSchema, z.strictObject({ diff --git a/src/domain/analysisView/artifactInventoryView.ts b/src/domain/analysisView/artifactInventoryView.ts index cd368e68a3..fab64fc8fe 100644 --- a/src/domain/analysisView/artifactInventoryView.ts +++ b/src/domain/analysisView/artifactInventoryView.ts @@ -15,6 +15,7 @@ import { prefixedDigestSchema } from "../digests.js"; import type { ArtifactOccurrence } from "../artifactGraph.js"; import { isImmutableJsonSnapshot } from "../immutableJson.js"; import { z } from "zod"; +import { artifactReaderLimitsSchema } from "../artifactReaderLimits.js"; import { AnalysisInputError } from "../analysisErrorCore.js"; import type { AnalysisError } from "../analysisErrorBase.js"; import { uniqueSorted } from "../canonicalOrdering.js"; @@ -93,6 +94,7 @@ export const artifactInventoryItemsSchema = z.array( ); export const artifactInventorySummarySchema = z.strictObject({ + effective_limits: artifactReaderLimitsSchema.exactOptional(), source_coverage: artifactInventoryCoverageSchema.exactOptional(), available_hash_status_counts: z .strictObject({ @@ -333,6 +335,9 @@ export const projectArtifactInventoryView = ( kind: "summary" as const, view, summary: { + ...(inventory.effectiveLimits === undefined + ? {} + : { effective_limits: inventory.effectiveLimits }), manifest: inventory.manifest, available_counts: { nodes: inventory.nodes.length, diff --git a/src/domain/analysisView/goBinaryView.ts b/src/domain/analysisView/goBinaryView.ts index 131fb5c5e1..d04545b894 100644 --- a/src/domain/analysisView/goBinaryView.ts +++ b/src/domain/analysisView/goBinaryView.ts @@ -4,10 +4,10 @@ import { z } from "zod"; import { goBinarySchema, goBuildInfoSchema, - goMaxStringLiteralsSchema, goModuleMetadataSchema, goModuleSchema, goStringLiteralSchema, + goStringLiteralsSchema, } from "../go/goBinary.js"; import { err, ok } from "../result.js"; import type { AnalysisViewParent } from "./analysisViewParent.js"; @@ -116,15 +116,9 @@ export const goBinarySummarySchema = z.strictObject({ }), }) .nullable(), - string_literals: z - .strictObject({ - evidence_kind: z.literal("inferred"), - available_count: count, - discovered_count: count, - aggregate_bytes: count, - max_items: goMaxStringLiteralsSchema, - truncated: z.boolean(), - }) + string_literals: goStringLiteralsSchema + .omit({ items: true }) + .extend({ available_count: count }) .exactOptional(), }); @@ -227,7 +221,14 @@ export const projectGoBinaryView = ( runtimeUnknown, ]), }; - if (view.kind === "summary") + if (view.kind === "summary") { + let stringSummary: z.output< + typeof goBinarySummarySchema + >["string_literals"]; + if (stringLiterals !== undefined) { + const { items, ...facts } = stringLiterals; + stringSummary = { ...facts, available_count: items.length }; + } return ok({ kind: "summary" as const, view, @@ -263,21 +264,15 @@ export const projectGoBinaryView = ( }, }, }, - ...(stringLiterals === undefined + ...(stringSummary === undefined ? {} : { - string_literals: { - evidence_kind: stringLiterals.evidence_kind, - available_count: stringLiterals.items.length, - discovered_count: stringLiterals.discovered_count, - aggregate_bytes: stringLiterals.aggregate_bytes, - max_items: stringLiterals.max_items, - truncated: stringLiterals.truncated, - }, + string_literals: stringSummary, }), }, coverage: completeWithinViewCoverage(1, 1), }); + } const indices = view.indices === undefined ? undefined : new Set(view.indices); const fields = diff --git a/src/domain/analysisView/rustBinaryView.ts b/src/domain/analysisView/rustBinaryView.ts index 027d17eeda..ed63ecf06b 100644 --- a/src/domain/analysisView/rustBinaryView.ts +++ b/src/domain/analysisView/rustBinaryView.ts @@ -88,6 +88,13 @@ const summarize = ( cyclic_packages: metadata.cyclic_dependency_indices.length, }, }; + let compilerSummary: z.output< + typeof rustBinarySummarySchema + >["rustc_commit"] = null; + if (compiler !== null) { + const { hashes, ...facts } = compiler; + compilerSummary = { ...facts, hashes_count: hashes.length }; + } return { format: report.format, architecture: report.architecture, @@ -98,13 +105,7 @@ const summarize = ( : { selected_slice: report.selected_slice }), bytes: artifact.bytes, auditable_metadata: section, - rustc_commit: - compiler === null - ? null - : { - evidence_kind: compiler.evidence_kind, - hashes_count: compiler.hashes.length, - }, + rustc_commit: compilerSummary, }; }; diff --git a/src/domain/analysisView/webSourceLocationView.ts b/src/domain/analysisView/webSourceLocationView.ts new file mode 100644 index 0000000000..9166418638 --- /dev/null +++ b/src/domain/analysisView/webSourceLocationView.ts @@ -0,0 +1,317 @@ +import { z } from "zod"; +import { + webSourceLocationResultSchema, + webSourceMapMatchSchema, +} from "../webSourceLocation.js"; +import { + MAX_SOURCE_MAP_ENCODED_BYTES, + sourceMapLimitsSchema, +} from "../sourceMapLimits.js"; +import { jsonValueByteLength } from "../jsonValueByteLength.js"; +import type { JsonValue } from "../jsonValue.js"; +import { AnalysisResourceConstraintError } from "../analysisErrorCore.js"; +import { err, ok } from "../result.js"; +import { + createAnalysisViewResultReader, + type AnalysisViewParent, +} from "./analysisViewParent.js"; +import { + ownedView, + type AdmittedView, + type SummaryAnalysisView, +} from "./analysisViewShape.js"; +import { analysisViewInputError } from "./analysisViewErrors.js"; +import { + boundedViewMessages, + pageViewCoverage, + unavailableViewCoverage, +} from "./analysisViewCoverage.js"; +import { analysisTextPage } from "./analysisViewText.js"; + +const DEFAULT_PAGE_BYTES = 1024 * 1024; +const DEFAULT_MATCH_LIMIT = 64; +const DEFAULT_TEXT_LIMIT = 1024; +const viewByteSchema = sourceMapLimitsSchema.shape.output_bytes.unwrap(); +const page = { + kind: z.literal("source-location"), + offset: z.int().nonnegative().default(0), + max_bytes: viewByteSchema + .default(DEFAULT_PAGE_BYTES) + .describe( + `Serialized selected-view byte allowance; default ${String(DEFAULT_PAGE_BYTES)}. Independent of complete Evidence and MCP delivery limits.`, + ), +}; +export const webSourceLocationViewRequestSchema = z.discriminatedUnion( + "collection", + [ + z.strictObject({ + ...page, + collection: z.literal("matches"), + limit: z.int().positive().default(DEFAULT_MATCH_LIMIT), + }), + z.strictObject({ + ...page, + collection: z.literal("source-text"), + match_index: z.int().nonnegative(), + limit: z.int().positive().default(DEFAULT_TEXT_LIMIT), + }), + ], +); +const generated = webSourceMapMatchSchema.options[0]; +const mapped = webSourceMapMatchSchema.options[1]; +const content = z.discriminatedUnion("state", [ + mapped.shape.content.options[0] + .omit({ text: true }) + .extend({ code_units: z.int().nonnegative() }), + mapped.shape.content.options[1], +]); +export const webSourceLocationMatchViewSchema = z.discriminatedUnion("state", [ + generated.extend({ match_index: z.int().nonnegative() }), + mapped + .omit({ content: true }) + .extend({ match_index: z.int().nonnegative(), content }), +]); +export const webSourceLocationContextSchema = webSourceLocationResultSchema + .omit({ matches: true, limitations: true, diagnostics: true }) + .extend({ + match_count: z.int().nonnegative(), + diagnostic_count: z.int().nonnegative(), + }); +export const webSourceLocationSummarySchema = + webSourceLocationContextSchema.extend({ + matches: z.array(webSourceLocationMatchViewSchema), + }); +export const webSourceLocationCompleteDeliveryShape = { + detail: z.literal("complete").optional().default("complete"), + view: z.never().exactOptional(), +}; +export const webSourceLocationSummaryDeliveryShape = { + detail: z.literal("summary"), + view: webSourceLocationViewRequestSchema.exactOptional(), +}; +export const webSourceLocationDeliverySchema = z.discriminatedUnion("detail", [ + z.strictObject(webSourceLocationCompleteDeliveryShape), + z.strictObject(webSourceLocationSummaryDeliveryShape), +]); +const readReport = createAnalysisViewResultReader( + webSourceLocationResultSchema, +); +type Match = z.output; +const metadata = ( + match: Match, + match_index: number, +): z.output => { + if (match.state === "generated-only") return { ...match, match_index }; + const { content: sourceContent, ...mappedMatch } = match; + if (sourceContent.state === "unavailable") + return { ...mappedMatch, match_index, content: sourceContent }; + const { text, ...identity } = sourceContent; + return { + ...mappedMatch, + match_index, + content: { ...identity, code_units: text.length }, + }; +}; +const tooLarge = (maxBytes: number, parentEvidenceId: string) => + err( + new AnalysisResourceConstraintError( + "inspect_analysis_view", + "memory", + "The selected source-location view exceeds its byte allowance.", + { + max_bytes: maxBytes, + evidence_reference: { + kind: "retained-evidence", + evidence_id: parentEvidenceId, + }, + }, + { + remediationAction: + maxBytes < MAX_SOURCE_MAP_ENCODED_BYTES + ? `Select a smaller source-location limit or increase view.max_bytes (maximum ${String(MAX_SOURCE_MAP_ENCODED_BYTES)}) and retry this retained parent. A single metadata row may require a larger allowance; complete Evidence remains available for export.` + : "view.max_bytes is already at its supported maximum. Select a smaller source-location limit or export complete parent Evidence; a single metadata row that cannot fit requires export.", + }, + ), + ); +const withinBudget = (value: unknown, maxBytes: number) => + jsonValueByteLength( + { ...(value as Record), view_digest: "0".repeat(64) }, + maxBytes, + ) <= maxBytes; + +type Report = z.output; +type LocationView = z.output; +const viewContext = (parent: AnalysisViewParent, report: Report) => { + const { matches, diagnostics, limitations, ...identity } = report; + const context = { + ...identity, + match_count: matches.length, + diagnostic_count: diagnostics.length, + }; + const shared = { + parent_evidence_id: parent.evidenceId, + parent_operation: "trace_web_source_location" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: { path: identity.source.path, sha256: identity.source.sha256 }, + limitations: boundedViewMessages( + [ + ...parent.limitations, + ...limitations, + "Match pages preserve retained lookup alternatives; exhausting a page does not establish execution or deployment authenticity. Complete map text and diagnostics remain in parent Evidence.", + ], + parent.evidenceId, + "limitations", + ), + unknowns: [] as string[], + }; + return { shared, context }; +}; +type Context = ReturnType; +const projectSourceText = ( + { shared, context }: Context, + matches: readonly Match[], + view: Extract, +) => { + const match = matches[view.match_index]; + if (match === undefined) + return err( + analysisViewInputError([ + { + path: ["view", "match_index"], + reason: "invalid_value", + message: "No match exists at this original match index.", + }, + ]), + ); + const selected = metadata(match, view.match_index); + if (match.state === "generated-only" || match.content.state !== "embedded") { + const result = { + ...shared, + kind: "source-location-text" as const, + view, + source_location: context, + match: selected, + item: null, + coverage: unavailableViewCoverage(), + }; + return withinBudget(result, view.max_bytes) + ? ok(result) + : tooLarge(view.max_bytes, shared.parent_evidence_id); + } + const selectedUnits = Math.min( + view.limit, + Math.max(0, match.content.text.length - view.offset), + ); + // Each retained UTF-16 unit needs at least one JSON byte. Refuse before copying a huge selected range; exact escaping is checked below. + if (selectedUnits + 2 > view.max_bytes) + return tooLarge(view.max_bytes, shared.parent_evidence_id); + // Text is already bounded and retained by the parent admission. Check serialized expansion before Evidence cloning/encoding. + const text = analysisTextPage( + match.content.text, + view.offset, + view.limit, + "Source text", + ); + if (!text.ok) return text; + const result = { + ...shared, + kind: "source-location-text" as const, + view, + source_location: context, + match: selected, + ...text.value, + }; + return withinBudget(result, view.max_bytes) + ? ok(result) + : tooLarge(view.max_bytes, shared.parent_evidence_id); +}; +const projectMatches = ( + { shared, context }: Context, + matches: readonly Match[], + view: SummaryAnalysisView | Extract, +) => { + const offset = view.kind === "summary" ? 0 : view.offset; + const limit = view.kind === "summary" ? DEFAULT_MATCH_LIMIT : view.limit; + const maxBytes = + view.kind === "summary" ? DEFAULT_PAGE_BYTES : view.max_bytes; + const items: z.output[] = []; + const common = { + ...shared, + coverage: pageViewCoverage(offset, 0, matches.length), + }; + const build = () => + view.kind === "summary" + ? { + ...common, + view, + kind: "summary" as const, + summary: { ...context, matches: items }, + } + : { + ...common, + view, + kind: "source-location" as const, + source_location: context, + items, + }; + const base = build(); + // Reserve maximum numeric coverage widths once; row admission remains linear in returned rows. + let bytes = jsonValueByteLength( + { + ...base, + view_digest: "0".repeat(64), + coverage: { + ...common.coverage, + examined: Number.MAX_SAFE_INTEGER, + next_offset: Number.MAX_SAFE_INTEGER, + }, + } as JsonValue, + maxBytes, + ); + if (bytes > maxBytes) return tooLarge(maxBytes, shared.parent_evidence_id); + for ( + let index = offset; + index < matches.length && items.length < limit; + index++ + ) { + const item = metadata(matches[index]!, index); + const added = + jsonValueByteLength(item as JsonValue, maxBytes - bytes) + + (items.length === 0 ? 0 : 1); + if (bytes + added > maxBytes) { + if (items.length === 0) + return tooLarge(maxBytes, shared.parent_evidence_id); + break; + } + items.push(item); + bytes += added; + } + common.coverage = pageViewCoverage(offset, items.length, matches.length); + return ok(build()); +}; +/** Query authenticated source-location facts without reopening any selected artifact. */ +export const projectWebSourceLocationView = ( + parent: AnalysisViewParent, + admitted: AdmittedView, +) => { + const view = ownedView< + SummaryAnalysisView | z.output + >(admitted, "summary", "source-location"); + if (view === undefined) + return err( + analysisViewInputError([ + { + path: ["view"], + reason: "invalid_value", + message: + "Source-location Evidence supports summary and source-location views.", + }, + ]), + ); + const parsed = readReport(parent); + if (!parsed.ok) return parsed; + const context = viewContext(parent, parsed.value); + return view.kind !== "summary" && view.collection === "source-text" + ? projectSourceText(context, parsed.value.matches, view) + : projectMatches(context, parsed.value.matches, view); +}; diff --git a/src/domain/apktool/apktoolResourceAnalysis.ts b/src/domain/apktool/apktoolResourceAnalysis.ts index a10a6819d7..1454c36252 100644 --- a/src/domain/apktool/apktoolResourceAnalysis.ts +++ b/src/domain/apktool/apktoolResourceAnalysis.ts @@ -2,10 +2,11 @@ import { z } from "zod"; import { localPathStringSchema } from "../localPath.js"; import { digestSchema } from "../digests.js"; -export const MAX_STRINGS = 2_000; +export const DEFAULT_APKTOOL_STRING_LIMIT = 2_000; export const MAX_SELECTED_LOCALES = 20; -export const MAX_AGGREGATE_STRINGS = 10_000; -export const AGGREGATE_STRINGS_XML_MAX_BYTES = 32 * 1024 * 1024; +export const DEFAULT_APKTOOL_TOTAL_STRINGS = 10_000; +export const DEFAULT_APKTOOL_STRING_TABLE_BYTES = 8 * 1024 * 1024; +export const DEFAULT_APKTOOL_TOTAL_STRING_TABLE_BYTES = 32 * 1024 * 1024; /** Decode wall clock used when decode_timeout_ms is omitted. */ export const DEFAULT_APKTOOL_DECODE_TIMEOUT_MS = 600_000; /** Largest delay supported without overflowing Node timers. */ @@ -72,7 +73,42 @@ export const apktoolInputSchemas = { .boolean() .default(true) .describe( - `Project res/values/strings.xml entries (bounded to ${String(MAX_STRINGS)} entries)`, + "Project res/values/strings.xml entries using the selected string page and read budgets", + ), + string_offset: z + .int() + .nonnegative() + .optional() + .describe( + "Zero-based offset into valid parsed strings in each selected table, default 0. Retry the same APK and locale with the returned page.next_offset to retrieve later strings; verify target.sha256 across calls.", + ), + string_limit: z + .int() + .positive() + .optional() + .describe( + `Maximum strings returned per selected table, default ${String(DEFAULT_APKTOOL_STRING_LIMIT)}. Applies on this decode; no restart. Independent of XML read and aggregate output budgets.`, + ), + max_total_strings: z + .int() + .positive() + .optional() + .describe( + `Aggregate returned string copies, default ${String(DEFAULT_APKTOOL_TOTAL_STRINGS)}. The legacy multi-locale shape charges the primary strings twice; string_tables_only avoids that copy. Raise this or select fewer locales when exhausted.`, + ), + max_string_table_bytes: z + .int() + .positive() + .optional() + .describe( + `UTF-8 source byte admission for each strings.xml, default ${String(DEFAULT_APKTOOL_STRING_TABLE_BYTES)}. Raise this and, when necessary, max_total_string_table_bytes to retry an oversized table. Applies on the next decode; no restart.`, + ), + max_total_string_table_bytes: z + .int() + .positive() + .optional() + .describe( + `Aggregate XML bytes read across distinct physical string tables, default ${String(DEFAULT_APKTOOL_TOTAL_STRING_TABLE_BYTES)}. Raise this or select fewer locales. Does not raise per-table admission or string output budgets.`, ), string_tables_only: z .boolean() @@ -91,7 +127,7 @@ export const apktoolInputSchemas = { .max(MAX_SELECTED_LOCALES) .optional() .describe( - `Project multiple localized string tables from res/values-/strings.xml alongside the default table (bounded to ${String(MAX_SELECTED_LOCALES)} locales)`, + `Project multiple localized string tables from res/values-/strings.xml alongside the default table (up to ${String(MAX_SELECTED_LOCALES)} locales per call; split larger selections into separate calls and verify target.sha256)`, ), decode_timeout_ms: z .number() @@ -115,6 +151,27 @@ export type ApktoolRequest = { }; }[ApktoolOperation]; +/** Explicit page/budget callers need the examined table's counts and continuation. */ +export const apktoolStringTablesRequested = ( + input: Pick< + Extract["input"], + | "locales" + | "string_tables_only" + | "string_offset" + | "string_limit" + | "max_total_strings" + | "max_string_table_bytes" + | "max_total_string_table_bytes" + >, +): boolean => + input.locales !== undefined || + input.string_tables_only === true || + input.string_offset !== undefined || + input.string_limit !== undefined || + input.max_total_strings !== undefined || + input.max_string_table_bytes !== undefined || + input.max_total_string_table_bytes !== undefined; + /** Validate operation and input together so their types remain correlated. */ export const apktoolRequestSchema = z.discriminatedUnion("operation", [ z.strictObject({ @@ -166,6 +223,17 @@ export const stringTableSchema = z.strictObject({ .describe( "Entries skipped due to nested markup or missing attributes, or null if the table was not examined or was malformed", ), + page: z + .strictObject({ + offset: z.int().nonnegative(), + limit: z.int().positive(), + returned_count: z.int().nonnegative(), + next_offset: z.int().nonnegative().nullable(), + }) + .optional() + .describe( + "Selected valid-entry page, present for explicit string selection/budgets when the table was examined. next_offset is null at the end; otherwise retry that offset against the same APK digest and locale. If next_offset equals offset, raise max_total_strings or use string_tables_only before retrying. Table coverage still describes the entire table.", + ), coverage: z.enum(["complete", "partial"]), }); diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 29bab3725d..f9b9728aad 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -6,6 +6,7 @@ import { } from "./artifactIntegrity.js"; import { artifactFormatMetadataSchema } from "./artifactFormatMetadata.js"; import { z } from "zod"; +import { artifactReaderLimitsSchema } from "./artifactReaderLimits.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; import { inesInventorySchema } from "./inesContainer.js"; @@ -253,6 +254,11 @@ export type ArtifactInventoryCoverage = z.infer< /** Current inventory observation containing the stable artifact graph. */ export const artifactInventoryResultSchema = z.strictObject({ + effective_limits: artifactReaderLimitsSchema + .exactOptional() + .describe( + "Accepted resource policy when selected; each format group applies only if that format is encountered. Not measured process memory or confirmation every reader ran.", + ), manifest: artifactGraphManifestSchema, nodes: z.array(artifactNodeSchema), occurrences: z.array(artifactOccurrenceSchema), @@ -284,6 +290,7 @@ const artifactExtractionManifestSchema = z.object({ /** Safe extraction observation with containment and cleanup verification. */ export const artifactExtractionResultSchema = z.object({ + effective_limits: artifactReaderLimitsSchema.exactOptional(), manifest: artifactGraphManifestSchema, extraction_manifest: artifactExtractionManifestSchema, output_root: z.string().min(1), diff --git a/src/domain/artifactInventoryEvidence.ts b/src/domain/artifactInventoryEvidence.ts index 2cf462a03f..497f43679f 100644 --- a/src/domain/artifactInventoryEvidence.ts +++ b/src/domain/artifactInventoryEvidence.ts @@ -33,6 +33,7 @@ import { /** Complete or partially assembled inventory used by comparison workflows. */ export interface InventorySet { + readonly effectiveLimits?: ArtifactInventoryResult["effective_limits"]; readonly manifest: ArtifactInventoryResult["manifest"]; readonly nodes: readonly ArtifactNode[]; readonly occurrences: readonly ArtifactOccurrence[]; @@ -383,6 +384,13 @@ const assembleInventoryResults = ( const occurrences = new Map(); const edges = new Map(); for (const page of pages) { + if ( + canonicalJson(page.effective_limits ?? null, "Artifact reader policy") !== + canonicalJson(first.effective_limits ?? null, "Artifact reader policy") + ) + throw new ArtifactInventoryCompositionError( + "Artifact inventory pages do not share one accepted reader policy", + ); if (canonicalJson(page.manifest, "Artifact inventory") !== manifestJson) throw new ArtifactInventoryCompositionError( "Artifact inventory pages do not share one manifest", @@ -477,6 +485,9 @@ const orderedInventory = ( ].sort(compareUnicodeCodePoints); return { manifest: first.manifest, + ...(first.effective_limits === undefined + ? {} + : { effectiveLimits: first.effective_limits }), nodes, occurrences, edges, diff --git a/src/domain/artifactOperationError.ts b/src/domain/artifactOperationError.ts index 6e656d7ccd..46c4beb832 100644 --- a/src/domain/artifactOperationError.ts +++ b/src/domain/artifactOperationError.ts @@ -6,13 +6,14 @@ import { AnalysisError, type AnalysisErrorOptions, } from "./analysisErrorBase.js"; +import type { ArtifactReaderLimitFact } from "./artifactReaderLimits.js"; /** The resource class that refused an artifact operation. */ export type ArtifactLimitBoundary = "file-size" | "capacity" | "memory"; /** Admission named by the producing reader, with its effective byte ceiling and its source. */ export interface ArtifactAdmissionLimit { - readonly kind: "asar-header"; + readonly kind: "asar-header" | "asar-listing"; readonly requiredBytes?: number; readonly maximumBytes: number; readonly source: "default" | "setting" | "runtime"; @@ -22,8 +23,10 @@ export interface ArtifactAdmissionLimit { export class ArtifactOperationError extends AnalysisError { readonly _tag = "ArtifactOperationError"; override readonly userMessage: string | undefined; + readonly remediationAction: string | undefined; readonly admission: ArtifactAdmissionLimit | undefined; readonly boundary: ArtifactLimitBoundary | undefined; + readonly readerLimit: ArtifactReaderLimitFact | undefined; constructor( readonly operation: @@ -63,8 +66,10 @@ export class ArtifactOperationError extends AnalysisError { readonly detail?: string, options?: AnalysisErrorOptions & { readonly userMessage?: string; + readonly remediationAction?: string; readonly admission?: ArtifactAdmissionLimit; readonly boundary?: ArtifactLimitBoundary; + readonly readerLimit?: ArtifactReaderLimitFact; }, ) { super( @@ -75,6 +80,8 @@ export class ArtifactOperationError extends AnalysisError { ); this.admission = options?.admission; this.boundary = options?.boundary; + this.readerLimit = options?.readerLimit; + this.remediationAction = options?.remediationAction; this.userMessage = options?.userMessage ?? (options?.admission === undefined ? undefined : detail); diff --git a/src/domain/artifactReaderLimits.ts b/src/domain/artifactReaderLimits.ts new file mode 100644 index 0000000000..5241400f99 --- /dev/null +++ b/src/domain/artifactReaderLimits.ts @@ -0,0 +1,210 @@ +import { z } from "zod"; +import { constants as bufferConstants } from "node:buffer"; + +const bytes = z.int().min(1).max(Number.MAX_SAFE_INTEGER); +const count = z.int().min(1).max(0x7fffffff); +const representedBytes = bytes.max(bufferConstants.MAX_STRING_LENGTH); + +export const DEFAULT_BUN_READER_LIMITS = { + payload_bytes: 256 * 1024 * 1024, + module_count: 100_000, + representation_bytes: 256 * 1024 * 1024, + metadata_read_bytes: 256 * 1024 * 1024, + recognition_table_bytes: 1024 * 1024, + recognition_load_command_bytes: 16 * 1024 * 1024, + path_parts: 256, + path_nodes: 250_000, +}; +export const bunReaderLimitsSchema = z.strictObject({ + payload_bytes: bytes, + module_count: count, + representation_bytes: representedBytes, + metadata_read_bytes: bytes, + recognition_table_bytes: bytes, + recognition_load_command_bytes: bytes, + path_parts: count, + path_nodes: count, +}); +export type BunReaderLimits = z.output; + +export const DEFAULT_DOTNET_BUNDLE_READER_LIMITS = { + metadata_bytes: 16 * 1024 * 1024, + entry_count: 100_000, + string_bytes: 64 * 1024, + path_parts: 256, + path_nodes: 250_000, + member_bytes: 512 * 1024 * 1024, + expanded_bytes: 2 * 1024 * 1024 * 1024, +}; +export const dotnetBundleReaderLimitsSchema = z.strictObject({ + metadata_bytes: bytes, + entry_count: count, + string_bytes: representedBytes, + path_parts: count, + path_nodes: count, + member_bytes: bytes, + expanded_bytes: bytes, +}); +export type DotnetBundleReaderLimits = z.output< + typeof dotnetBundleReaderLimitsSchema +>; + +export const DEFAULT_NODE_SEA_READER_LIMITS = { + blob_bytes: 256 * 1024 * 1024, + asset_count: 100_000, + asset_name_bytes: 8 * 1024 * 1024, + argument_count: 4096, + argument_bytes: 1024 * 1024, + recognition_header_count: 4096, + recognition_load_command_bytes: 8 * 1024 * 1024, +}; +export const nodeSeaReaderLimitsSchema = z.strictObject({ + blob_bytes: bytes, + asset_count: count, + asset_name_bytes: bytes, + argument_count: count, + argument_bytes: bytes, + recognition_header_count: count, + recognition_load_command_bytes: bytes, +}); +export type NodeSeaReaderLimits = z.output; + +export const DEFAULT_GODOT_READER_LIMITS = { + directory_bytes: 64 * 1024 * 1024, +}; +export const godotReaderLimitsSchema = z.strictObject({ + directory_bytes: representedBytes, +}); +export type GodotReaderLimits = z.output; + +/** Input selections preserve omission; only the producing operation resolves defaults. */ +export const artifactReaderLimitsInputSchema = z.strictObject({ + bun: z + .strictObject({ + payload_bytes: bunReaderLimitsSchema.shape.payload_bytes.exactOptional(), + module_count: bunReaderLimitsSchema.shape.module_count.exactOptional(), + representation_bytes: + bunReaderLimitsSchema.shape.representation_bytes.exactOptional(), + metadata_read_bytes: + bunReaderLimitsSchema.shape.metadata_read_bytes.exactOptional(), + recognition_table_bytes: + bunReaderLimitsSchema.shape.recognition_table_bytes.exactOptional(), + recognition_load_command_bytes: + bunReaderLimitsSchema.shape.recognition_load_command_bytes.exactOptional(), + path_parts: bunReaderLimitsSchema.shape.path_parts.exactOptional(), + path_nodes: bunReaderLimitsSchema.shape.path_nodes.exactOptional(), + }) + .exactOptional() + .describe( + `Bun admission; omitted fields use ${JSON.stringify(DEFAULT_BUN_READER_LIMITS)}. Recognition limits precede graph decoding; payload, retained representation and physical metadata-read work are independent.`, + ), + dotnet_bundle: z + .strictObject({ + metadata_bytes: + dotnetBundleReaderLimitsSchema.shape.metadata_bytes.exactOptional(), + entry_count: + dotnetBundleReaderLimitsSchema.shape.entry_count.exactOptional(), + string_bytes: + dotnetBundleReaderLimitsSchema.shape.string_bytes.exactOptional(), + path_parts: + dotnetBundleReaderLimitsSchema.shape.path_parts.exactOptional(), + path_nodes: + dotnetBundleReaderLimitsSchema.shape.path_nodes.exactOptional(), + member_bytes: + dotnetBundleReaderLimitsSchema.shape.member_bytes.exactOptional(), + expanded_bytes: + dotnetBundleReaderLimitsSchema.shape.expanded_bytes.exactOptional(), + }) + .exactOptional() + .describe( + `.NET bundle admission; omitted fields use ${JSON.stringify(DEFAULT_DOTNET_BUNDLE_READER_LIMITS)}. String bytes apply to paths and bundle IDs; member bytes bound stored and expanded members; expanded bytes bound aggregate work.`, + ), + node_sea: z + .strictObject({ + blob_bytes: nodeSeaReaderLimitsSchema.shape.blob_bytes.exactOptional(), + asset_count: nodeSeaReaderLimitsSchema.shape.asset_count.exactOptional(), + asset_name_bytes: + nodeSeaReaderLimitsSchema.shape.asset_name_bytes.exactOptional(), + argument_count: + nodeSeaReaderLimitsSchema.shape.argument_count.exactOptional(), + argument_bytes: + nodeSeaReaderLimitsSchema.shape.argument_bytes.exactOptional(), + recognition_header_count: + nodeSeaReaderLimitsSchema.shape.recognition_header_count.exactOptional(), + recognition_load_command_bytes: + nodeSeaReaderLimitsSchema.shape.recognition_load_command_bytes.exactOptional(), + }) + .exactOptional() + .describe( + `Node SEA admission; omitted fields use ${JSON.stringify(DEFAULT_NODE_SEA_READER_LIMITS)}. Asset-name and argument bytes are aggregate decoded text budgets. Recognition limits apply before blob decoding.`, + ), + godot: z + .strictObject({ + directory_bytes: + godotReaderLimitsSchema.shape.directory_bytes.exactOptional(), + }) + .exactOptional() + .describe( + `Godot directory representation admission; omitted fields use ${JSON.stringify(DEFAULT_GODOT_READER_LIMITS)}. Applies before path allocation and again to retained metadata.`, + ), + expanded_bytes: bytes + .exactOptional() + .describe( + "Aggregate plaintext work for archive-backed readers; omitted retains the encoded-size-derived allowance. Includes repeated readable logical occurrences.", + ), +}); +export type ArtifactReaderLimitsInput = z.output< + typeof artifactReaderLimitsInputSchema +>; +export const artifactReaderLimitsSchema = z.strictObject({ + bun: bunReaderLimitsSchema, + dotnet_bundle: dotnetBundleReaderLimitsSchema, + node_sea: nodeSeaReaderLimitsSchema, + godot: godotReaderLimitsSchema, + expanded_bytes: bytes.exactOptional(), +}); +export type ArtifactReaderLimits = z.output; + +export const resolveArtifactReaderLimits = ( + selected?: ArtifactReaderLimitsInput, +): ArtifactReaderLimits => ({ + bun: { ...DEFAULT_BUN_READER_LIMITS, ...selected?.bun }, + dotnet_bundle: { + ...DEFAULT_DOTNET_BUNDLE_READER_LIMITS, + ...selected?.dotnet_bundle, + }, + node_sea: { ...DEFAULT_NODE_SEA_READER_LIMITS, ...selected?.node_sea }, + godot: { ...DEFAULT_GODOT_READER_LIMITS, ...selected?.godot }, + ...(selected?.expanded_bytes === undefined + ? {} + : { expanded_bytes: selected.expanded_bytes }), +}); + +/** A per-call reader admission, independent of runtime-setting registry membership. */ +export interface ArtifactReaderLimitFact { + readonly parameter: string; + readonly effective_value: number; + readonly unit: "bytes" | "count"; + readonly minimum: 1; + readonly maximum: number; + readonly required_value?: number | string; + readonly applies_at: "next_operation"; + readonly agent_editable: true; + readonly can_increase: boolean; + readonly rea_restart_required: false; +} + +/** Runtime representation ceilings differ from aggregate plaintext-work budgets. */ +export const artifactReaderLimitMaximum = ( + parameter: string, + unit: ArtifactReaderLimitFact["unit"], +): number => + unit === "count" + ? 0x7fffffff + : [ + "artifact_limits.bun.representation_bytes", + "artifact_limits.dotnet_bundle.string_bytes", + "artifact_limits.godot.directory_bytes", + ].includes(parameter) + ? bufferConstants.MAX_STRING_LENGTH + : Number.MAX_SAFE_INTEGER; diff --git a/src/domain/captures/packetCapture.ts b/src/domain/captures/packetCapture.ts index bafec50bbf..e64b35d88a 100644 --- a/src/domain/captures/packetCapture.ts +++ b/src/domain/captures/packetCapture.ts @@ -1,5 +1,9 @@ import { z } from "zod"; import { fixedTupleSchema } from "../fixedTupleSchema.js"; +import { + boundedProducerLimitsSchema, + boundedProducerLimitsResultSchema, +} from "../producerResourceLimits.js"; import { absoluteLocalPathStringSchema, localPathStringSchema, @@ -11,6 +15,7 @@ const argument = z .max(8192) .refine((value) => !value.includes("\0"), "NUL is not an argument character"); const captureInputShape = z.strictObject({ + producer_limits: boundedProducerLimitsSchema, path: localPathStringSchema.describe("Absolute local capture path"), fields: z .array(argument) @@ -148,14 +153,7 @@ export const packetCaptureSchema = z sha256: z.string().regex(/^[a-f0-9]{64}$/), arguments: z.array(z.string()), }), - limits: z.strictObject({ - input_bytes: z.number().int().positive(), - output_bytes: z.number().int().positive(), - command_timeout_ms: z.number().int().positive(), - address_space_bytes: z.number().int().nonnegative(), - file_bytes: z.number().int().nonnegative(), - cpu_seconds: z.number().int().nonnegative(), - }), + limits: boundedProducerLimitsResultSchema, hardware_acquisition: z.literal("not-performed"), }) .superRefine((report, context) => { diff --git a/src/domain/captures/packetFields.ts b/src/domain/captures/packetFields.ts index 1cd75c6e36..b889eb8a05 100644 --- a/src/domain/captures/packetFields.ts +++ b/src/domain/captures/packetFields.ts @@ -1,7 +1,15 @@ import { z } from "zod"; import { packetCaptureSchema } from "./packetCapture.js"; +import { boundedProducerLimitsSchema } from "../producerResourceLimits.js"; export const inspectPacketFieldsInputSchema = z.strictObject({ + producer_limits: boundedProducerLimitsSchema + .unwrap() + .omit({ input_bytes: true }) + .optional() + .describe( + "Per-call producer budgets; defaults match packet capture except output_bytes defaults to 32 MiB for the complete field glossary. No capture input is read. Tighter inherited OS limits still win; no restart. Does not raise the separate parsed-record budget.", + ), kind: z .enum(["protocols", "fields"]) .default("protocols") diff --git a/src/domain/commandObservation.ts b/src/domain/commandObservation.ts index 152296054b..ff383e64be 100644 --- a/src/domain/commandObservation.ts +++ b/src/domain/commandObservation.ts @@ -6,6 +6,8 @@ export const commandObservationSchema = z.strictObject({ stderr: z.string(), exit_code: z.union([z.string(), z.number(), z.null()]), signal: z.string().nullable(), + output_budget_bytes: z.int().positive().exactOptional(), + output_budget_source: z.enum(["default", "request"]).exactOptional(), output_truncated: z.boolean(), completion_status: z.enum(["complete", "unknown"]), artifact_sha256_after: z @@ -15,6 +17,13 @@ export const commandObservationSchema = z.strictObject({ artifact_changed: z.boolean().nullable(), }); +/** Exact producer admission that omitted command output; kinds define the unit. */ +export const gdbOutputLimitSchema = z.strictObject({ + kind: z.enum(["command_bytes", "command_records", "mi_line_bytes"]), + maximum: z.int().positive(), +}); +export type GdbOutputLimit = z.infer; + /** The GDB owner retains ordered records and derives its stream groupings. */ export const gdbCommandResultSchema = z.strictObject({ command: z.string(), @@ -31,6 +40,7 @@ export const gdbCommandResultSchema = z.strictObject({ process_exit_code: z.number().int().nullable().exactOptional(), process_exit_signal: z.string().nullable().exactOptional(), output_truncated: z.boolean(), + output_limits: z.array(gdbOutputLimitSchema).min(1).exactOptional(), output_invalid_utf8: z.literal(true).exactOptional(), completion_status: z.enum(["complete", "unknown"]), }); diff --git a/src/domain/desktop/appExport.test.ts b/src/domain/desktop/appExport.test.ts index 0209b577ba..676e78964f 100644 --- a/src/domain/desktop/appExport.test.ts +++ b/src/domain/desktop/appExport.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from "vitest"; import type { DesktopApplicationCapture } from "./desktopCapture.js"; +import type { DesktopApplicationState } from "./desktopState.js"; import { capturedHomePath, + exportedStateFiles, appModuleFile, className, environmentName, @@ -428,6 +430,39 @@ describe("README and class wording", () => { }); }); +it.each(["directory", "file", undefined] as const)( + "preserves a selected state alias's recorded kind (%s)", + (kind) => { + const home = "/home/ada"; + const target = `${home}/.config/notes`; + const selection = { + path: `${home}/settings`, + physical: target, + ...(kind === undefined ? {} : { kind }), + }; + const files = exportedStateFiles( + { + host: { home }, + application: { state_directories: [{ path: target }] }, + session: { windows: [], opened_files: [] }, + state: { + state_directories: [target], + content_roots: [], + selected_paths: [selection], + }, + } as unknown as DesktopApplicationCapture, + { paths: [] } as unknown as DesktopApplicationState, + ); + expect(files.stateAliases).toEqual([ + { + alias: "settings", + target: ".config/notes", + ...(kind === undefined ? {} : { kind }), + }, + ]); + }, +); + it("keeps a nested selection under the state path of a selected alias", () => { const home = "/home/ada"; const state = `${home}/.config/geany`; diff --git a/src/domain/desktop/appExport.ts b/src/domain/desktop/appExport.ts index bf58dee5f4..7f0b4a5b79 100644 --- a/src/domain/desktop/appExport.ts +++ b/src/domain/desktop/appExport.ts @@ -1,4 +1,8 @@ import type { DownloadInstall } from "./appDownloadInstall.js"; +import { + desktopViewPolicy, + type DesktopViewPolicyInput, +} from "./desktopViewPolicy.js"; import { INSTALL_SCRIPT, type ExportAptSource, @@ -13,7 +17,7 @@ import { } from "./appExportInstallText.js"; import { PYTHON_DATA_HOOKS, - PYTHON_HELPERS, + renderPythonHelpers, PYTHON_IMPORTS, PYTHON_MODULE_NAMES, PYTHON_READY_CHECK, @@ -47,6 +51,7 @@ export const DESKTOP_EXPORT_FRAMEWORK_VERSION = "0.9.1464"; /** Everything the export renders, decided from a capture and its state view. */ export interface DesktopAppExportPlan { + readonly viewPolicy?: DesktopViewPolicyInput | undefined; /** Environment and folder name. */ readonly name: string; readonly className: string; @@ -641,7 +646,7 @@ SEED_MODES = {${Object.entries(plan.seedModes) STATE_ALIASES = [${(plan.stateAliases ?? []).map(({ alias, target, kind }) => `(${py(alias)}, ${py(target)}, ${py(kind)})`).join(", ")}] WINDOW_CLASS = ${py(plan.windowClass)} WITHHELD = {${(plan.withheld ?? []).map(({ setting, file }) => `${py(setting)}: ${py(file)}`).join(", ")}} -${PYTHON_HELPERS} +${renderPythonHelpers(desktopViewPolicy(plan.viewPolicy))} @computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ${pyList(plan.architectures)}}]) @linux( diff --git a/src/domain/desktop/appExportPython.ts b/src/domain/desktop/appExportPython.ts index 4f9c04d812..fe5d134a8e 100644 --- a/src/domain/desktop/appExportPython.ts +++ b/src/domain/desktop/appExportPython.ts @@ -1,4 +1,9 @@ import { XML_ELEMENT_OPENING } from "../xmlDocument.js"; +import { + DESKTOP_VIEW_DEFAULTS, + pythonDesktopViewPolicy, + type DesktopViewPolicy, +} from "./desktopViewPolicy.js"; import { PYTHON_JSON_READER } from "./appExportPythonJson.js"; import { PYTHON_XML_READER } from "./appExportPythonXml.js"; import { PYTHON_ZIP_HELPERS } from "./appExportPythonZip.js"; @@ -48,7 +53,7 @@ const PYTHON_XML_ELEMENT_OPENING = XML_ELEMENT_OPENING.replaceAll( ); /** Python helpers the class uses to seed the home, load dconf keys and read typed views. */ -export const PYTHON_HELPERS = String.raw` +const PYTHON_HELPER_BODY = String.raw` XML_SUFFIXES = (".xml", ".xbel", ".plist", ".svg", ".ui") XML_ELEMENT_OPENING = re.compile(r"${PYTHON_XML_ELEMENT_OPENING}") def _opens_element(text: str) -> bool: @@ -79,17 +84,13 @@ GVDB_HEADER = b"GVariant" ZIP_HEADER = b"PK\x03\x04" GZIP_HEADER = b"\x1f\x8b\x08" XML_SPACE = "xml:space" -MAX_VIEW_BYTES = 64 * 1024 * 1024 MAX_SAFE_INTEGER = 2**53 - 1 -MAX_TABLES = 64 -MAX_ROWS = 500 -MAX_LINES = 5000 -MAX_XML_NODES = 5000 -MAX_XML_DEPTH = 64 SQLITE_BUSY_S = 0.25 -MAX_PARSED_BYTES = 4 * 1024 * 1024 -PARSED_TOO_LARGE = "larger than the 4 MiB limit for JSON and key file views" -MAX_JSON_VIEW_DEPTH = 500 +def _view_byte_limit(value: int) -> str: + return f"{value // (1024 * 1024)} MiB" if value % (1024 * 1024) == 0 else f"{value} bytes" + + +PARSED_TOO_LARGE = f"larger than the {_view_byte_limit(MAX_PARSED_BYTES)} limit for JSON and key file views" JSON_TOO_DEEP = f"nested deeper than the {MAX_JSON_VIEW_DEPTH} levels a JSON view keeps" @@ -223,6 +224,40 @@ def _cell(value, encoding: str): return value +def _view_json_bytes(value, budget: int) -> int: + """Size the table/row JSON shapes without quoting a whole large cell. + + Above budget the count establishes overflow only. Numeric spelling follows + Python's compact JSON encoder, rather than assuming JavaScript's spelling. + """ + if isinstance(value, str): + total = 2 + for offset in range(0, len(value), 8192): + if total > budget: + break + # Public JSON quoting preserves escapes; remove each chunk's two + # quotes. A control-filled native cell never becomes one huge part. + total += len(json.dumps(value[offset:offset + 8192], ensure_ascii=False).encode("utf-8")) - 2 + return total + if isinstance(value, list): + total = 2 + max(0, len(value) - 1) + for member in value: + if total > budget: + break + total += _view_json_bytes(member, budget - total) + return total + if isinstance(value, dict): + total = 2 + max(0, len(value) - 1) + for key, member in value.items(): + if total > budget: + break + total += _view_json_bytes(key, budget - total) + 1 + if total <= budget: + total += _view_json_bytes(member, budget - total) + return total + return len(json.dumps(value, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + + def _table(db, name: str, primary_key: list, encoding: str) -> dict: quoted = '"' + name.replace('"', '""') + '"' columns = [column[0] for column in db.execute(f"select * from main.{quoted} limit 0").description] @@ -233,14 +268,31 @@ def _table(db, name: str, primary_key: list, encoding: str) -> dict: projected.append(f"CASE typeof({column}) WHEN 'text' THEN 't' || hex(CAST({column} AS BLOB)) " f"WHEN 'blob' THEN 'b' || hex({column}) ELSE {column} END") cursor = db.execute(f"select {', '.join(projected)} from main.{quoted} not indexed limit {MAX_ROWS + 1}") - rows = [[_cell(value, encoding) for value in row] for row in cursor] - return {"columns": columns, "primary_key": primary_key, - "rows": rows[:MAX_ROWS], "truncated": len(rows) > MAX_ROWS} + view = {"columns": columns, "primary_key": primary_key, "rows": [], "truncated": True} + spent = _view_json_bytes(view, MAX_SQLITE_VIEW_BYTES) + if spent > MAX_SQLITE_VIEW_BYTES: + return {"error": SQLITE_VIEW_TOO_LARGE} + for index, row in enumerate(cursor): + if index == MAX_ROWS: + break + projected = [_cell(value, encoding) for value in row] + spent += (1 if view["rows"] else 0) + _view_json_bytes(projected, MAX_SQLITE_VIEW_BYTES - spent) + if spent > MAX_SQLITE_VIEW_BYTES: + return {"error": SQLITE_VIEW_TOO_LARGE} + view["rows"].append(projected) + else: + view["truncated"] = False + # JSON false takes one byte more than true, reserved only when known. + spent += 1 + return {"error": SQLITE_VIEW_TOO_LARGE} if spent > MAX_SQLITE_VIEW_BYTES else view def _sqlite(path: Path) -> dict: """The database's ordinary tables, in schema order, up to REA's table and row limits.""" with contextlib.closing(sqlite3.connect(path.resolve().as_uri() + "?mode=ro", uri=True, timeout=SQLITE_BUSY_S)) as db: + # Match the independent native value/record boundary before SQL hex + # expansion; the selected projected-view allowance is separate. + db.setlimit(sqlite3.SQLITE_LIMIT_LENGTH, SQLITE_VALUE_BYTES) kinds = dict(db.execute("select name, type from temp.pragma_table_list where schema = 'main'")) names = [name for (name,) in db.execute("select name from sqlite_schema where type = 'table'") if kinds.get(name) == "table"] @@ -249,10 +301,21 @@ def _sqlite(path: Path) -> dict: for name in names[:MAX_TABLES]} encoding = {"UTF-8": "utf-8", "UTF-16le": "utf-16-le", "UTF-16be": "utf-16-be"}[ db.execute("PRAGMA encoding").fetchone()[0]] - tables = {name: _table(db, name, keys[name], encoding) for name in names[:MAX_TABLES]} + tables = {} + for name in names[:MAX_TABLES]: + try: + tables[name] = _table(db, name, keys[name], encoding) + except sqlite3.Error as error: + reason = str(error) + if getattr(error, "sqlite_errorcode", None) == sqlite3.SQLITE_TOOBIG: + reason += (f". SQLite value/record exceeds the fixed {SQLITE_VALUE_BYTES}-byte native limit, including SQL hex expansion. " + "Increasing sqlite_view_bytes does not raise this boundary. Keep the complete database; " + "use a narrower REA SQLite table/column projection only if it preserves the required facts, " + "or use a format-specific reader.") + tables[name] = {"error": reason} cut = len(names) > MAX_TABLES return {"format": "sqlite", "view": {"tables": tables, "tables_truncated": cut}, - "truncated": cut or any(table["truncated"] for table in tables.values())} + "truncated": cut or any(table.get("truncated", "error" in table) for table in tables.values())} def _text(text: str) -> dict: @@ -347,20 +410,24 @@ def _bytes_view(name: str, data: bytes) -> dict: def _view(path: Path) -> dict: """A typed JSON view of one file in the shapes of REA's state view; truncated says the view is partial.""" - too_large = {"format": None, "view": None, "error": "larger than 64 MiB, the most the view reads"} + too_large = {"format": None, "view": None, "error": VIEW_TOO_LARGE} if path.stat().st_size > MAX_VIEW_BYTES: return too_large with path.open("rb") as file: - data = file.read(8192) + data = file.read(min(8192, MAX_VIEW_BYTES + 1)) + if len(data) > MAX_VIEW_BYTES: + return too_large whole = data.startswith(GZIP_HEADER) or (not data.startswith(SQLITE_HEADER) and b"\x00" not in data) - data += file.read(MAX_VIEW_BYTES + 1 - len(data)) if whole else b"" + remaining = MAX_VIEW_BYTES + 1 - len(data) + if whole and remaining > 0: + data += file.read(remaining) + if len(data) > MAX_VIEW_BYTES: + return too_large if data.startswith(SQLITE_HEADER): return _sqlite(path) if data.startswith(ZIP_HEADER): with path.open("rb") as file: return _zip(file, os.fstat(file.fileno()).st_size) - if len(data) > MAX_VIEW_BYTES: - return too_large if data.startswith(GZIP_HEADER): return _gzip(path.name, data) return _bytes_view(path.name, data) @@ -393,6 +460,13 @@ def _views(home: Path, paths: list, leaving_out=()) -> dict: return files `.concat(PYTHON_ZIP_HELPERS); +/** Render the accepted policy before helpers which consume it. */ +export const renderPythonHelpers = (policy: DesktopViewPolicy): string => + `\n${pythonDesktopViewPolicy(policy)}\n${PYTHON_HELPER_BODY}`; + +/** Default helpers for existing generated-reader consumers. */ +export const PYTHON_HELPERS = renderPythonHelpers(DESKTOP_VIEW_DEFAULTS); + /** Constants the class module assigns before the helpers. */ const PYTHON_CONSTANTS = [ "HERE", diff --git a/src/domain/desktop/appExportPythonZip.ts b/src/domain/desktop/appExportPythonZip.ts index 9da8204e33..3197fdd8b1 100644 --- a/src/domain/desktop/appExportPythonZip.ts +++ b/src/domain/desktop/appExportPythonZip.ts @@ -5,10 +5,6 @@ * with REA's limits and wording. Part of the module's helpers. */ export const PYTHON_ZIP_HELPERS = String.raw` -MAX_ZIP_ENTRIES = 1000 -MAX_ZIP_ENTRY_BYTES = 16 * 1024 * 1024 -MAX_ZIP_BYTES = 64 * 1024 * 1024 -MAX_ZIP_RATIO = 100 ZIP_READ_CHUNK = 64 * 1024 #: Small, since the end of each gzip member copies what is left of the piece it was read from. GZIP_READ_CHUNK = 4 * 1024 @@ -37,11 +33,11 @@ def _zip_refusal(info, spent: int): if info.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED): return f"compressed with method {info.compress_type}, which the view does not read" if info.file_size > MAX_ZIP_ENTRY_BYTES: - return "larger than the 16 MiB an entry's view reads" + return f"larger than the {_view_byte_limit(MAX_ZIP_ENTRY_BYTES)} an entry's view reads" if info.file_size > MAX_ZIP_RATIO * info.compress_size: return f"expands more than {MAX_ZIP_RATIO} times, so it is not read" if spent + info.file_size > MAX_ZIP_BYTES: - return "past the 64 MiB read from one archive" + return f"past the {_view_byte_limit(MAX_ZIP_BYTES)} read from one archive" return None @@ -271,7 +267,7 @@ def _gzip(name: str, data: bytes) -> dict: if document is None: return {**compressed, "error": f"expands more than {MAX_ZIP_RATIO} times, so it is not read" if MAX_ZIP_RATIO * len(data) < MAX_ZIP_ENTRY_BYTES - else "larger than the 16 MiB a compressed document's view reads"} + else f"larger than the {_view_byte_limit(MAX_ZIP_ENTRY_BYTES)} a compressed document's view reads"} for header, form, inside in NOT_READ_INSIDE: if form != "zip" and document.startswith(header): return {**compressed, "format": form, "error": f"{inside} inside a gzip file is not read"} diff --git a/src/domain/desktop/desktopState.ts b/src/domain/desktop/desktopState.ts index 929a50a3f5..64f772751c 100644 --- a/src/domain/desktop/desktopState.ts +++ b/src/domain/desktop/desktopState.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { jsonValueSchema } from "../jsonValue.js"; import { desktopApplicationCaptureSchema } from "./desktopCapture.js"; +import { desktopViewPolicyInputSchema } from "./desktopViewPolicy.js"; const sha256 = z.string().regex(/^[0-9a-f]{64}$/u); const format = z.enum([ @@ -105,6 +106,14 @@ export const versionAt = ( /** The normalized result of `inspect_desktop_application_state`. */ export const desktopApplicationStateSchema = z.strictObject({ + /** Policy actually used to produce these views; older Evidence may omit it. */ + effective_view_policy: desktopViewPolicyInputSchema + .required() + // Historical policy records predate the additive table-view byte control. + .extend({ + sqlite_view_bytes: desktopViewPolicyInputSchema.shape.sqlite_view_bytes, + }) + .optional(), capture: z.strictObject({ evidence_id: z.string().min(1), home: z.string().min(1), diff --git a/src/domain/desktop/desktopViewPolicy.ts b/src/domain/desktop/desktopViewPolicy.ts new file mode 100644 index 0000000000..f364da3207 --- /dev/null +++ b/src/domain/desktop/desktopViewPolicy.ts @@ -0,0 +1,126 @@ +import { z } from "zod"; +import { MAX_JSON_DEPTH } from "../jsonValue.js"; +import { SQLITE_DATABASE_OUTPUT_BYTES } from "../sqlite/sqliteDatabase.js"; + +// The deepest supported view is a ZIP member in a captured version. Nested +// archives are not interpreted. Count every enclosing property/array member +// from the complete Evidence record to that member's view. +const VIEW_ENVELOPE_DEPTH = [ + "normalized_result", + "paths", + 0, + "versions", + 0, + "view", + "entries", + 0, + "view", +].length; +export const MAX_DESKTOP_JSON_VIEW_DEPTH = MAX_JSON_DEPTH - VIEW_ENVELOPE_DEPTH; +// XML projects an element to {root:{attributes:{key:value},children:[...]}}: +// root + attribute dictionary/value add three levels; each element child adds two. +export const MAX_DESKTOP_XML_VIEW_DEPTH = Math.floor( + (MAX_JSON_DEPTH - VIEW_ENVELOPE_DEPTH - 3) / 2, +); + +/** Admission and retention for one desktop state view, shared with exported readers. */ +export const DESKTOP_VIEW_DEFAULTS = { + file_bytes: 64 * 1024 * 1024, + sqlite_tables: 64, + sqlite_rows: 500, + sqlite_view_bytes: SQLITE_DATABASE_OUTPUT_BYTES, + text_lines: 5000, + xml_nodes: 5000, + xml_depth: 64, + parsed_bytes: 4 * 1024 * 1024, + json_depth: MAX_JSON_DEPTH / 2, + zip_entries: 1000, + zip_entry_bytes: 16 * 1024 * 1024, + zip_bytes: 64 * 1024 * 1024, + zip_ratio: 100, +} as const; + +const positive = z.int().positive(); +export const desktopViewPolicyInputSchema = z.strictObject({ + file_bytes: positive.optional(), + sqlite_tables: positive.optional(), + sqlite_rows: positive.optional(), + sqlite_view_bytes: positive + .optional() + .describe( + `Compact UTF-8 JSON bytes retained per projected SQLite table, default ${String(SQLITE_DATABASE_OUTPUT_BYTES)}. Applies on this call and in exported readers. Each reader counts its native numeric spelling, so float thresholds can differ. Independent native value/record and worker reply limits remain fixed and may refuse earlier.`, + ), + text_lines: positive.optional(), + xml_nodes: positive.optional(), + xml_depth: positive.max(MAX_DESKTOP_XML_VIEW_DEPTH).optional(), + parsed_bytes: positive.optional(), + json_depth: positive.max(MAX_DESKTOP_JSON_VIEW_DEPTH).optional(), + zip_entries: positive.optional(), + zip_entry_bytes: positive.optional(), + zip_bytes: positive.optional(), + zip_ratio: positive.optional(), +}); +export type DesktopViewPolicyInput = z.output< + typeof desktopViewPolicyInputSchema +>; +export type DesktopViewPolicy = { + readonly [Key in keyof typeof DESKTOP_VIEW_DEFAULTS]: number; +}; + +export const desktopViewPolicy = ( + input: DesktopViewPolicyInput = {}, +): DesktopViewPolicy => { + const policy: { -readonly [Key in keyof DesktopViewPolicy]: number } = { + ...DESKTOP_VIEW_DEFAULTS, + }; + for (const key of Object.keys(policy) as (keyof DesktopViewPolicy)[]) { + const value = input[key]; + if (value !== undefined) policy[key] = value; + } + return policy; +}; + +/** Keep refusals exact even when callers select a sub-MiB byte budget. */ +export const desktopViewByteDisplay = (bytes: number): string => + bytes % (1024 * 1024) === 0 + ? `${String(bytes / (1024 * 1024))} MiB` + : `${String(bytes)} bytes`; + +/** Refusal shared by shipped seed checks and the generated file reader. */ +export const desktopViewFileLimitMessage = ( + policy: DesktopViewPolicy, +): string => + `larger than ${desktopViewByteDisplay(policy.file_bytes)}, the most the view reads`; + +/** Projected-table admission, independent of the SQLite worker's transport. */ +export const desktopSqliteViewLimitMessage = ( + policy: DesktopViewPolicy, +): string => + `SQLite table view exceeds sqlite_view_bytes=${String(policy.sqlite_view_bytes)} UTF-8 JSON bytes. Increase sqlite_view_bytes in --view-policy on the next call; no restart is needed. The separate SQLite value/record and worker reply limits remain fixed at ${String(SQLITE_DATABASE_OUTPUT_BYTES)} bytes and may refuse earlier.`; + +/** Assign the same effective policy to the exported Python reader. */ +export const pythonDesktopViewPolicy = (policy: DesktopViewPolicy): string => { + const names: Record = { + file_bytes: "MAX_VIEW_BYTES", + sqlite_tables: "MAX_TABLES", + sqlite_rows: "MAX_ROWS", + sqlite_view_bytes: "MAX_SQLITE_VIEW_BYTES", + text_lines: "MAX_LINES", + xml_nodes: "MAX_XML_NODES", + xml_depth: "MAX_XML_DEPTH", + parsed_bytes: "MAX_PARSED_BYTES", + json_depth: "MAX_JSON_VIEW_DEPTH", + zip_entries: "MAX_ZIP_ENTRIES", + zip_entry_bytes: "MAX_ZIP_ENTRY_BYTES", + zip_bytes: "MAX_ZIP_BYTES", + zip_ratio: "MAX_ZIP_RATIO", + }; + return [ + ...(Object.keys(names) as (keyof DesktopViewPolicy)[]).map( + (key) => `${names[key]} = ${String(policy[key])}`, + ), + `VIEW_TOO_LARGE = ${JSON.stringify(desktopViewFileLimitMessage(policy))}`, + `SQLITE_VIEW_TOO_LARGE = ${JSON.stringify(desktopSqliteViewLimitMessage(policy))}`, + `SQLITE_VALUE_BYTES = ${String(SQLITE_DATABASE_OUTPUT_BYTES)}`, + ].join("\n"); +}; diff --git a/src/domain/desktop/elfDependencies.test.ts b/src/domain/desktop/elfDependencies.test.ts index 64fed028e2..c68c02b498 100644 --- a/src/domain/desktop/elfDependencies.test.ts +++ b/src/domain/desktop/elfDependencies.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { desktopLibraryPolicy } from "./libraryPolicy.js"; import { readElfDependencies } from "./elfDependencies.js"; import { elfImage, readImage } from "./elfImage.fixture.js"; @@ -61,10 +62,34 @@ describe("readElfDependencies", () => { { length: 513 }, (_, index) => `lib${String(index)}.so`, ); - expect(await dependencies(elfImage({ needed: names }))).toEqual({ + const image = elfImage({ needed: names }); + expect(await dependencies(image)).toMatchObject({ ok: false, - error: "ELF dynamic table names more than 512 distinct needed libraries", + error: expect.stringContaining("library-policy.libraries=512"), }); + expect( + await readElfDependencies( + readImage(image), + image.length, + desktopLibraryPolicy({ libraries: 513 }), + ), + ).toMatchObject({ ok: true, value: { needed: names } }); + }); + + it("distinguishes a bounded string read from truncation and recovers with its named budget", async () => { + const needed = "l".repeat(4096); + const image = elfImage({ needed: [needed] }); + expect(await dependencies(image)).toMatchObject({ + ok: false, + error: expect.stringContaining("library-policy.string_bytes=4096 bytes"), + }); + expect( + await readElfDependencies( + readImage(image), + image.length, + desktopLibraryPolicy({ string_bytes: 4097 }), + ), + ).toMatchObject({ ok: true, value: { needed: [needed] } }); }); it("reports no dependencies for a static image", async () => { diff --git a/src/domain/desktop/elfDependencies.ts b/src/domain/desktop/elfDependencies.ts index f2dbf3afcf..83968d3f85 100644 --- a/src/domain/desktop/elfDependencies.ts +++ b/src/domain/desktop/elfDependencies.ts @@ -1,3 +1,8 @@ +import { + desktopLibraryLimit, + desktopLibraryPolicy, + type DesktopLibraryPolicy, +} from "./libraryPolicy.js"; import { readElfDynamicEntry } from "../elfDynamicEntry.js"; import { readElfFixedHeader } from "../elfFixedHeader.js"; import { @@ -32,12 +37,6 @@ const DT_STRTAB = 5n; const DT_STRSZ = 10n; const DT_RPATH = 15n; const DT_RUNPATH = 29n; -const MAX_PROGRAM_HEADER_BYTES = 1024 * 1024; -const MAX_DYNAMIC_BYTES = 1024 * 1024; -const MAX_STRING_BYTES = 4096; -/** Distinct strings decoded per image: the library walk's own limit, and a few search path entries. */ -const MAX_NEEDED = 512; -const MAX_SEARCH_PATHS = 16; const readExactly = async ( read: ReadAt, @@ -70,6 +69,7 @@ const programHeaders = async ( header: Buffer, size: number, bits: 32 | 64, + policy: DesktopLibraryPolicy, ): Promise> => { const little = header[5] === 1; const view = new DataView(header.buffer, header.byteOffset, header.length); @@ -81,8 +81,10 @@ const programHeaders = async ( const count = view.getUint16(bits === 64 ? 56 : 44, little); if (entrySize < (bits === 64 ? 56 : 32) || count === 0xffff) return err("unsupported ELF program header table"); - if (entrySize * count > MAX_PROGRAM_HEADER_BYTES) - return err("ELF program header table is too large"); + if (entrySize * count > policy.table_bytes) + return err( + desktopLibraryLimit(policy, "table_bytes", "ELF program header table"), + ); const table = await readExactly( read, offset, @@ -108,6 +110,7 @@ const dynamicTags = ( table: Buffer, bits: 32 | 64, little: boolean, + policy: DesktopLibraryPolicy, ): Result< { readonly needed: ReadonlySet; @@ -127,9 +130,30 @@ const dynamicTags = ( strsz: undefined as bigint | undefined, }; const lists = new Map([ - [DT_NEEDED, { offsets: found.needed, limit: MAX_NEEDED }], - [DT_RUNPATH, { offsets: found.runpath, limit: MAX_SEARCH_PATHS }], - [DT_RPATH, { offsets: found.rpath, limit: MAX_SEARCH_PATHS }], + [ + DT_NEEDED, + { + offsets: found.needed, + limit: policy.libraries, + key: "libraries" as const, + }, + ], + [ + DT_RUNPATH, + { + offsets: found.runpath, + limit: policy.search_path_entries, + key: "search_path_entries" as const, + }, + ], + [ + DT_RPATH, + { + offsets: found.rpath, + limit: policy.search_path_entries, + key: "search_path_entries" as const, + }, + ], ]); for (let at = 0; at + width <= table.length; at += width) { const entry = readElfDynamicEntry( @@ -145,7 +169,11 @@ const dynamicTags = ( list.offsets.add(value); if (list.offsets.size > list.limit) return err( - `ELF dynamic table names more than ${String(list.limit)} distinct ${tag === DT_NEEDED ? "needed libraries" : "search paths"}`, + desktopLibraryLimit( + policy, + list.key, + `ELF dynamic table ${tag === DT_NEEDED ? "needed libraries" : "search path entries"}`, + ), ); } else if (tag === DT_STRTAB) found.strtab = value; else if (tag === DT_STRSZ) found.strsz = value; @@ -158,23 +186,31 @@ const dynamicStrings = async ( read: ReadAt, file: { readonly size: number; readonly at: bigint; readonly bytes: bigint }, offsets: ReadonlySet, + policy: DesktopLibraryPolicy, ): Promise> => { const values: string[] = []; for (const offset of offsets) { if (offset >= file.bytes) return err("ELF dynamic string lies outside its table"); const length = - BigInt(MAX_STRING_BYTES) < file.bytes - offset - ? BigInt(MAX_STRING_BYTES) + BigInt(policy.string_bytes) < file.bytes - offset + ? BigInt(policy.string_bytes) : file.bytes - offset; const window = await readExactly(read, file.at + offset, length, file.size); - const value = - window === undefined - ? null - : elfStringTableText( - observeElfStringTableEntry(window, 0, MAX_STRING_BYTES), - ); - if (value === null) return err("ELF dynamic string is truncated"); + if (window === undefined) return err("ELF dynamic string is truncated"); + const value = elfStringTableText( + observeElfStringTableEntry(window, 0, policy.string_bytes), + ); + if (value === null) + return err( + length < file.bytes - offset + ? desktopLibraryLimit( + policy, + "string_bytes", + "ELF dynamic string read", + ) + : "ELF dynamic string is truncated", + ); values.push(value); } return ok(values); @@ -194,22 +230,23 @@ const directories = (paths: readonly string[]): readonly string[] => [ export const readElfDependencies = async ( read: ReadAt, size: number, + policy: DesktopLibraryPolicy = desktopLibraryPolicy(), ): Promise> => { const header = Buffer.from(await read(0, 64)); const fixed = readElfFixedHeader(header, size); if (!fixed.ok) return fixed; const { elfClass: bits, machine } = fixed.value; const little = fixed.value.byteOrder === "little"; - const headers = await programHeaders(read, header, size, bits); + const headers = await programHeaders(read, header, size, bits, policy); if (!headers.ok) return headers; const none = { machine, bits, needed: [], runpath: undefined, rpath: [] }; const dynamic = headers.value.find((program) => program.type === PT_DYNAMIC); if (dynamic === undefined) return ok(none); - if (dynamic.fileSize > BigInt(MAX_DYNAMIC_BYTES)) - return err("ELF dynamic table is too large"); + if (dynamic.fileSize > BigInt(policy.table_bytes)) + return err(desktopLibraryLimit(policy, "table_bytes", "ELF dynamic table")); const table = await readExactly(read, dynamic.offset, dynamic.fileSize, size); if (table === undefined) return err("ELF dynamic table is truncated"); - const tags = dynamicTags(table, bits, little); + const tags = dynamicTags(table, bits, little, policy); if (!tags.ok) return tags; const { strtab, strsz, needed, runpath, rpath } = tags.value; const stringsAt = @@ -224,12 +261,13 @@ export const readElfDependencies = async ( ? ok(none) : err("ELF dynamic string table is not mapped from the file"); const strings = { size, at: stringsAt, bytes: strsz }; - const names = await dynamicStrings(read, strings, needed); + const names = await dynamicStrings(read, strings, needed, policy); if (!names.ok) return names; const search = await dynamicStrings( read, strings, runpath.size > 0 ? runpath : rpath, + policy, ); if (!search.ok) return search; return ok({ diff --git a/src/domain/desktop/gzipDocument.test.ts b/src/domain/desktop/gzipDocument.test.ts index f58bb2926b..d0b1630240 100644 --- a/src/domain/desktop/gzipDocument.test.ts +++ b/src/domain/desktop/gzipDocument.test.ts @@ -5,6 +5,11 @@ import { describe, expect, it } from "vitest"; import { openStateDocument, UNREADABLE_GZIP } from "./gzipDocument.js"; import { textualView } from "./stateFormats.js"; +import { + DESKTOP_VIEW_DEFAULTS, + desktopViewPolicy, + type DesktopViewPolicy, +} from "./desktopViewPolicy.js"; import { zipArchive } from "./zipArchive.fixture.js"; const MIB = 1024 * 1024; @@ -18,16 +23,53 @@ const GNUMERIC = ` `; /** What the state view reads from a file's bytes: its format, compression, and view or error. */ -const read = (name: string, bytes: Uint8Array) => { - const document = openStateDocument(name, bytes); +const read = ( + name: string, + bytes: Uint8Array, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, +) => { + const document = openStateDocument(name, bytes, policy); if ("error" in document) return document; const { format, compression } = document; return format === "text" || format === "xml" - ? { format, compression, view: textualView(format, document.bytes) } + ? { format, compression, view: textualView(format, document.bytes, policy) } : { format, compression }; }; describe("openStateDocument", () => { + it("recovers the selected gzip document size and inner text view budgets", () => { + const bytes = gzipSync("one\ntwo\nthree"); + expect( + read("notes", bytes, desktopViewPolicy({ zip_entry_bytes: 12 })), + ).toMatchObject({ + compression: "gzip", + error: "larger than the 12 bytes a compressed document's view reads", + }); + expect( + read( + "notes", + bytes, + desktopViewPolicy({ zip_entry_bytes: 13, text_lines: 2 }), + ), + ).toMatchObject({ + compression: "gzip", + view: { ok: true, value: { lines: ["one", "two"], truncated: true } }, + }); + expect( + read( + "notes", + bytes, + desktopViewPolicy({ zip_entry_bytes: 13, text_lines: 3 }), + ), + ).toMatchObject({ + compression: "gzip", + view: { + ok: true, + value: { lines: ["one", "two", "three"], truncated: false }, + }, + }); + }); + it("reads a gzip file's XML document as the same view the document has uncompressed", () => { expect(read("Budget.gnumeric", gzipSync(GNUMERIC))).toEqual({ ...read("Budget.gnumeric", Buffer.from(GNUMERIC)), diff --git a/src/domain/desktop/gzipDocument.ts b/src/domain/desktop/gzipDocument.ts index a513ef3827..29abe53e34 100644 --- a/src/domain/desktop/gzipDocument.ts +++ b/src/domain/desktop/gzipDocument.ts @@ -2,9 +2,12 @@ import { gunzipSync, gzipSync, inflateRawSync, type Zlib } from "node:zlib"; import { err, ok, type Result } from "../result.js"; import { detectStateFormat, type DesktopStateFormat } from "./stateFormats.js"; -import { MAX_ZIP_ENTRY_BYTES, MAX_ZIP_RATIO } from "./zipView.js"; +import { + DESKTOP_VIEW_DEFAULTS, + desktopViewByteDisplay, + type DesktopViewPolicy, +} from "./desktopViewPolicy.js"; -const MIB = 1024 * 1024; /** RFC 1952's operating system byte for an unknown one, so a recompressed file is the same whichever host made it. */ const UNKNOWN_OS = 0xff; /** RFC 1952's header flags for a header CRC, an extra field, a file name and a comment. */ @@ -72,8 +75,10 @@ export const hasExtraField = (bytes: Uint8Array): boolean => { }; /** The most a gzip file of `size` bytes may expand to for its view to read it: an archive entry's size and ratio limits. */ -const readLimit = (size: number): number => - Math.min(MAX_ZIP_ENTRY_BYTES, MAX_ZIP_RATIO * size); +const readLimit = ( + size: number, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, +): number => Math.min(policy.zip_entry_bytes, policy.zip_ratio * size); /** Bytes gzip-compressed at `level` alike on every host: no file name, comment or extra field, modification time 0 and the operating system unknown. */ const gzipAt = (bytes: Uint8Array, level: number): Buffer => { @@ -90,7 +95,7 @@ const gzipAt = (bytes: Uint8Array, level: number): Buffer => { export const gzipDeterministic = (document: Uint8Array): Buffer => { const deflated = gzipAt(document, 1); return document.length > readLimit(deflated.length) && - document.length <= MAX_ZIP_ENTRY_BYTES + document.length <= DESKTOP_VIEW_DEFAULTS.zip_entry_bytes ? gzipAt(document, 0) : deflated; }; @@ -109,10 +114,10 @@ export type StateDocument = }; /** Why a gzip file whose document passes an archive entry's limits is not read. */ -const tooLarge = (size: number): string => - MAX_ZIP_RATIO * size < MAX_ZIP_ENTRY_BYTES - ? `expands more than ${String(MAX_ZIP_RATIO)} times, so it is not read` - : `larger than the ${String(MAX_ZIP_ENTRY_BYTES / MIB)} MiB a compressed document's view reads`; +const tooLarge = (size: number, policy: DesktopViewPolicy): string => + policy.zip_ratio * size < policy.zip_entry_bytes + ? `expands more than ${String(policy.zip_ratio)} times, so it is not read` + : `larger than the ${desktopViewByteDisplay(policy.zip_entry_bytes)} a compressed document's view reads`; /** * The document a file's view reads: its own bytes, or a gzip file's @@ -122,17 +127,21 @@ const tooLarge = (size: number): string => export const openStateDocument = ( path: string, bytes: Uint8Array, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, ): StateDocument => { - if (!isGzip(bytes)) return { format: detectStateFormat(path, bytes), bytes }; - const inner = gunzip(bytes, readLimit(bytes.length)); + if (!isGzip(bytes)) + return { format: detectStateFormat(path, bytes, policy), bytes }; + const inner = gunzip(bytes, readLimit(bytes.length, policy)); if (!inner.ok) return { format: null, compression: "gzip", error: - inner.error === "too_large" ? tooLarge(bytes.length) : UNREADABLE_GZIP, + inner.error === "too_large" + ? tooLarge(bytes.length, policy) + : UNREADABLE_GZIP, }; - const format = detectStateFormat(path, inner.value); + const format = detectStateFormat(path, inner.value, policy); const inside = NOT_READ_INSIDE[format]; return inside === undefined ? { format, bytes: inner.value, compression: "gzip" } diff --git a/src/domain/desktop/libraryPolicy.ts b/src/domain/desktop/libraryPolicy.ts new file mode 100644 index 0000000000..c5662808bb --- /dev/null +++ b/src/domain/desktop/libraryPolicy.ts @@ -0,0 +1,45 @@ +import { z } from "zod"; + +/** Resource admission for one desktop dependency inventory, independent of capture content. */ +const DESKTOP_LIBRARY_DEFAULTS = { + libraries: 512, + search_path_entries: 16, + string_bytes: 4096, + table_bytes: 1024 * 1024, +} as const; + +export const desktopLibraryPolicyInputSchema = z.strictObject({ + libraries: z.int().positive().optional(), + search_path_entries: z.int().positive().optional(), + string_bytes: z.int().positive().optional(), + table_bytes: z.int().positive().optional(), +}); +export type DesktopLibraryPolicyInput = z.infer< + typeof desktopLibraryPolicyInputSchema +>; +export type DesktopLibraryPolicy = { + readonly [Key in keyof typeof DESKTOP_LIBRARY_DEFAULTS]: number; +}; + +export const desktopLibraryPolicy = ( + input?: DesktopLibraryPolicyInput, +): DesktopLibraryPolicy => { + const selected = desktopLibraryPolicyInputSchema.parse(input ?? {}); + return { + libraries: selected.libraries ?? DESKTOP_LIBRARY_DEFAULTS.libraries, + search_path_entries: + selected.search_path_entries ?? + DESKTOP_LIBRARY_DEFAULTS.search_path_entries, + string_bytes: + selected.string_bytes ?? DESKTOP_LIBRARY_DEFAULTS.string_bytes, + table_bytes: selected.table_bytes ?? DESKTOP_LIBRARY_DEFAULTS.table_bytes, + }; +}; + +/** A later capture needs a new explicit budget; changing content or heap controls cannot raise it. */ +export const desktopLibraryLimit = ( + policy: DesktopLibraryPolicy, + key: keyof DesktopLibraryPolicy, + boundary: string, +): string => + `${boundary} reached library-policy.${key}=${String(policy[key])}${key.endsWith("bytes") ? " bytes" : " entries"}. For a new capture, increase ${key} in --library-policy JSON; no REA restart is required. Capture content limits and worker heaps do not raise this allowance.`; diff --git a/src/domain/desktop/stateFormats.test.ts b/src/domain/desktop/stateFormats.test.ts index 5300c93860..ab93c6c333 100644 --- a/src/domain/desktop/stateFormats.test.ts +++ b/src/domain/desktop/stateFormats.test.ts @@ -1,3 +1,8 @@ +import { + desktopViewPolicy, + desktopViewPolicyInputSchema, +} from "./desktopViewPolicy.js"; +import { jsonValueValidationIssue } from "../jsonValue.js"; import { describe, expect, it } from "vitest"; import { @@ -319,3 +324,25 @@ describe("captured data safety and fidelity", () => { expect(xmlView(deep)).toMatchObject({ truncated: true }); }); }); + +it("keeps maximum-depth views encodable inside captured ZIP-member Evidence", () => { + const policy = desktopViewPolicy({ xml_depth: 494, json_depth: 991 }); + expect(desktopViewPolicyInputSchema.safeParse(policy).success).toBe(true); + const xml = xmlView( + `${''.repeat(495)}${"".repeat(495)}`, + policy, + ); + const json = jsonView(`${"[".repeat(991)}0${"]".repeat(991)}`, policy); + if (!json.ok) throw new Error(json.error); + expect(xml).toMatchObject({ truncated: false }); + for (const view of [xml, json.value]) { + // The real enclosing shape: complete Evidence -> captured path/version -> ZIP entry -> member view. + const wrapped = { + normalized_result: { + paths: [{ versions: [{ view: { entries: [{ view }] } }] }], + }, + }; + expect(jsonValueValidationIssue(wrapped)).toBeUndefined(); + expect(() => JSON.stringify(wrapped)).not.toThrow(); + } +}); diff --git a/src/domain/desktop/stateFormats.ts b/src/domain/desktop/stateFormats.ts index 58291b89f5..e3d16b2ef2 100644 --- a/src/domain/desktop/stateFormats.ts +++ b/src/domain/desktop/stateFormats.ts @@ -1,7 +1,12 @@ import type { Element, Node } from "@xmldom/xmldom"; import { isExactJsonNumber } from "../exactJsonNumber.js"; -import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js"; +import type { JsonValue } from "../jsonValue.js"; +import { + DESKTOP_VIEW_DEFAULTS, + desktopViewByteDisplay, + type DesktopViewPolicy, +} from "./desktopViewPolicy.js"; import { err, ok, type Result } from "../result.js"; import { safeParseJson } from "../safeJson.js"; import { @@ -39,14 +44,6 @@ export const XML_POLICY = { lineEndings: "normalize", nameSupported: expatXmlName, } as const; -const MAX_TEXT_LINES = 5000; -const MAX_XML_NODES = 5000; -const MAX_XML_DEPTH = 64; -/** Half of Evidence's nesting limit, leaving room for the result around a view. */ -const MAX_JSON_VIEW_DEPTH = MAX_JSON_DEPTH / 2; -const MIB = 1024 * 1024; -/** Largest JSON or key file read into a view, whose whole value enters Evidence. */ -const MAX_PARSED_VIEW_BYTES = 4 * MIB; type KeyfileLine = | { readonly kind: "skip" } @@ -337,6 +334,7 @@ export const textSyntaxFormat = ( export const detectStateFormat = ( path: string, bytes: Uint8Array, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, ): DesktopStateFormat => { const header = new TextDecoder("latin1").decode(bytes.subarray(0, 16)); if (header === SQLITE_HEADER) return "sqlite"; @@ -348,7 +346,7 @@ export const detectStateFormat = ( if ( namedJson(path, trimmed) && // JSON too large for a view is told apart without building its value. - (bytes.length > MAX_PARSED_VIEW_BYTES + (bytes.length > policy.parsed_bytes ? isJsonText(text) : safeParseJson(text).ok) ) @@ -384,7 +382,10 @@ export const deeperThan = (root: JsonValue, limit: number): boolean => { }; /** A JSON file's value, refused when a number would be rounded or it nests too deeply. */ -export const jsonView = (text: string): Result => { +export const jsonView = ( + text: string, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, +): Result => { let rounded = false; const parsed = safeParseJson(text, (_key, value, context) => { if (typeof value === "number" && !isExactJsonNumber(value, context?.source)) @@ -397,21 +398,24 @@ export const jsonView = (text: string): Result => { "a number in it cannot be read exactly, such as an integer above 2^53", ); const value = parsed.value as JsonValue; - return deeperThan(value, MAX_JSON_VIEW_DEPTH) + return deeperThan(value, policy.json_depth) ? err( - `nested deeper than the ${String(MAX_JSON_VIEW_DEPTH)} levels a JSON view keeps`, + `nested deeper than the ${String(policy.json_depth)} levels a JSON view keeps`, ) : ok(value); }; /** A text file as lines, bounded. */ -export const textView = (text: string): JsonValue => { +export const textView = ( + text: string, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, +): JsonValue => { const lines = text.split(/\r?\n/u); if (lines.at(-1) === "") lines.pop(); return { - lines: lines.slice(0, MAX_TEXT_LINES), + lines: lines.slice(0, policy.text_lines), line_count: lines.length, - truncated: lines.length > MAX_TEXT_LINES, + truncated: lines.length > policy.text_lines, }; }; @@ -480,8 +484,9 @@ const xmlNode = ( depth: number, budget: XmlBudget, scope: XmlScope, + policy: DesktopViewPolicy, ): JsonValue | undefined => { - if (budget.remaining <= 0 || depth > MAX_XML_DEPTH) { + if (budget.remaining <= 0 || depth > policy.xml_depth) { budget.truncated = true; return undefined; } @@ -511,7 +516,7 @@ const xmlNode = ( child !== null; child = child.nextSibling ) { - const value = xmlNode(child, depth + 1, budget, inner); + const value = xmlNode(child, depth + 1, budget, inner, policy); if (value !== undefined) children.push(value); } return { name: element.tagName, attributes, children }; @@ -522,15 +527,24 @@ const xmlNode = ( * is recorded as `doctype`, its DTD never read; one with an internal subset is * refused, as is a reference to any entity but XML's five. */ -export const xmlView = (text: string): JsonValue | undefined => { +export const xmlView = ( + text: string, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, +): JsonValue | undefined => { const parsed = parseXmlDocument(text, XML_POLICY); const root = parsed.ok ? parsed.value.documentElement : null; if (!parsed.ok || root === null) return undefined; - const budget: XmlBudget = { remaining: MAX_XML_NODES, truncated: false }; - const tree = xmlNode(root, 0, budget, { - preserve: false, - elementContent: false, - }); + const budget: XmlBudget = { remaining: policy.xml_nodes, truncated: false }; + const tree = xmlNode( + root, + 0, + budget, + { + preserve: false, + elementContent: false, + }, + policy, + ); if (tree === undefined) return undefined; const doctype = declaredDocumentType(parsed.value); return { @@ -564,24 +578,25 @@ export const isTextualFormat = ( export const textualView = ( format: TextualStateFormat, bytes: Uint8Array, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, ): Result => { if ( (format === "json" || format === "keyfile") && - bytes.length > MAX_PARSED_VIEW_BYTES + bytes.length > policy.parsed_bytes ) return err( - `larger than the ${String(MAX_PARSED_VIEW_BYTES / MIB)} MiB limit for JSON and key file views`, + `larger than the ${desktopViewByteDisplay(policy.parsed_bytes)} limit for JSON and key file views`, ); const text = decodeStateText(bytes) ?? ""; switch (format) { case "text": - return ok(textView(text)); + return ok(textView(text, policy)); case "json": - return jsonView(text); + return jsonView(text, policy); case "keyfile": return ok({ groups: keyfileView(text) }); case "xml": { - const tree = xmlView(text); + const tree = xmlView(text, policy); return tree === undefined ? err("not well-formed XML, or its DOCTYPE has an internal subset") : ok(tree); diff --git a/src/domain/desktop/zipView.ts b/src/domain/desktop/zipView.ts index b9ac74d95b..cfadd58f48 100644 --- a/src/domain/desktop/zipView.ts +++ b/src/domain/desktop/zipView.ts @@ -9,15 +9,11 @@ import { } from "./stateFormats.js"; import { locatePointer } from "./viewPointer.js"; -const MIB = 1024 * 1024; -/** Entries a zip view lists; past them it counts entries and is truncated. */ -export const MAX_ZIP_ENTRIES = 1000; -/** Largest entry read, uncompressed. */ -export const MAX_ZIP_ENTRY_BYTES = 16 * MIB; -/** Uncompressed bytes read from one archive, the same as the largest file the state view reads. */ -const MAX_ZIP_BYTES = 64 * MIB; -/** Most an entry may expand over its compressed size; a zip bomb expands about a thousand times. */ -export const MAX_ZIP_RATIO = 100; +import { + DESKTOP_VIEW_DEFAULTS, + desktopViewByteDisplay, + type DesktopViewPolicy, +} from "./desktopViewPolicy.js"; /** Why an archive has no view; the exported reader words it the same. */ export const UNREADABLE_ZIP = "not a readable zip archive"; @@ -37,16 +33,17 @@ export interface ZipEntryHeader { export const zipEntryRefusal = ( header: ZipEntryHeader, spent: number, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, ): string | undefined => { if (header.encrypted) return "encrypted, so it is not read"; if (header.method !== 0 && header.method !== 8) return `compressed with method ${String(header.method)}, which the view does not read`; - if (header.size > MAX_ZIP_ENTRY_BYTES) - return `larger than the ${String(MAX_ZIP_ENTRY_BYTES / MIB)} MiB an entry's view reads`; - if (header.size > MAX_ZIP_RATIO * header.compressedSize) - return `expands more than ${String(MAX_ZIP_RATIO)} times, so it is not read`; - if (spent + header.size > MAX_ZIP_BYTES) - return `past the ${String(MAX_ZIP_BYTES / MIB)} MiB read from one archive`; + if (header.size > policy.zip_entry_bytes) + return `larger than the ${desktopViewByteDisplay(policy.zip_entry_bytes)} an entry's view reads`; + if (header.size > policy.zip_ratio * header.compressedSize) + return `expands more than ${String(policy.zip_ratio)} times, so it is not read`; + if (spent + header.size > policy.zip_bytes) + return `past the ${desktopViewByteDisplay(policy.zip_bytes)} read from one archive`; return undefined; }; @@ -67,13 +64,14 @@ export const unreadZipEntry = ( const entryView = ( format: DesktopStateFormat, bytes: Uint8Array, + policy: DesktopViewPolicy, ): Result => { switch (format) { case "text": case "json": case "keyfile": case "xml": - return textualView(format, bytes); + return textualView(format, bytes, policy); case "sqlite": return err("a SQLite database inside an archive is not read"); case "dconf": @@ -89,9 +87,10 @@ const entryView = ( export const zipEntry = ( header: ZipEntryHeader, bytes: Uint8Array, + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, ): JsonValue => { - const format = detectStateFormat(header.name, bytes); - const read = entryView(format, bytes); + const format = detectStateFormat(header.name, bytes, policy); + const read = entryView(format, bytes, policy); return { name: header.name, size: header.size, diff --git a/src/domain/evm/evmInterface.ts b/src/domain/evm/evmInterface.ts index 165f3cc6bc..9612c5f6ed 100644 --- a/src/domain/evm/evmInterface.ts +++ b/src/domain/evm/evmInterface.ts @@ -2,6 +2,42 @@ import { z } from "zod"; import { absoluteLocalPathStringSchema } from "../localPath.js"; import { solidityMetadataSchema } from "./solidityMetadata.js"; +import { + producerResourceLimitsSchema, + resolveProducerLimits, +} from "../producerResourceLimits.js"; + +/** EVM advertises only producer controls it actually enforces. */ +export const evmProducerLimitsSchema = producerResourceLimitsSchema.omit({ + command_timeout_ms: true, +}); +export const EVM_PRODUCER_LIMIT_DEFAULTS = { + input_bytes: 4 * 1024 * 1024, + output_bytes: 16 * 1024 * 1024, + diagnostic_bytes: 1024 * 1024, + address_space_bytes: 3 * 1024 * 1024 * 1024, + file_bytes: 16 * 1024 * 1024, + cpu_seconds: 30, +} as const; +export const selectedEvmProducerLimits = ( + selected: z.output | undefined, +) => + resolveProducerLimits( + { + ...EVM_PRODUCER_LIMIT_DEFAULTS, + file_bytes: + selected?.output_bytes ?? EVM_PRODUCER_LIMIT_DEFAULTS.file_bytes, + }, + selected, + ); +export type EvmProducerLimits = ReturnType; + +/** Recover the exact producer budget; inherited OS restrictions remain external. */ +export const evmProducerLimitRecovery = ( + limits: EvmProducerLimits, + field: keyof EvmProducerLimits, +): string => + `Increase producer_limits.${field} above ${String(limits[field])} on the next inspect_evm_interface call; no REA restart is needed. Defaults are ${String(EVM_PRODUCER_LIMIT_DEFAULTS[field])}; the field accepts positive safe integers. Tighter inherited Linux soft/hard limits remain enforced and cannot be raised by this request. timeout_ms, the 256 MiB worker heap and client transport limits are separate. Resolve incomplete cleanup before retrying; do not repeat the unchanged failed request.`; const byteOffset = z.int().nonnegative(); const sha256 = z.string().regex(/^[a-f0-9]{64}$/); @@ -20,6 +56,11 @@ export const inspectEvmInterfaceInputSchema = z.strictObject({ .describe( "Raw bytes or UTF-8 hex with optional 0x prefix and outer ASCII whitespace", ), + producer_limits: evmProducerLimitsSchema + .optional() + .describe( + "Per-call carrier/reply/diagnostic bytes, address-space/file bytes and CPU seconds. Omission preserves documented defaults; omitted file_bytes follows selected output_bytes. Tighter inherited Linux limits remain enforced. timeout_ms and the 256 MiB worker heap are independent.", + ), timeout_ms: z .number() .int() @@ -27,7 +68,7 @@ export const inspectEvmInterfaceInputSchema = z.strictObject({ .max(MAX_EVM_TIMEOUT_MS) .optional() .describe( - `Deadline in milliseconds for the owned EVMole worker command (default ${String(DEFAULT_EVM_TIMEOUT_MS)}), not carrier acquisition or final reply reads. An integer from 1 to ${String(MAX_EVM_TIMEOUT_MS)} applies on the next call. Fixed input, reply, diagnostic, address-space, and CPU allowances stay unchanged.`, + `Deadline in milliseconds for the owned EVMole worker command (default ${String(DEFAULT_EVM_TIMEOUT_MS)}), not carrier acquisition or final reply reads. An integer from 1 to ${String(MAX_EVM_TIMEOUT_MS)} applies on the next call. Does not raise the separately selected producer_limits or the 256 MiB worker heap.`, ), }); diff --git a/src/domain/evm/evmInterfaceEvidence.ts b/src/domain/evm/evmInterfaceEvidence.ts index 6be8b2466f..e9064b2f47 100644 --- a/src/domain/evm/evmInterfaceEvidence.ts +++ b/src/domain/evm/evmInterfaceEvidence.ts @@ -26,6 +26,9 @@ export const evmInterfaceEvidence = ( parameters: { path: input.path, encoding: input.encoding, + ...(input.producer_limits === undefined + ? {} + : { producer_limits: input.producer_limits }), ...(input.timeout_ms === undefined ? {} : { timeout_ms: input.timeout_ms }), diff --git a/src/domain/explicitSensitiveFailure.ts b/src/domain/explicitSensitiveFailure.ts index c16559dc4d..4c7cd25114 100644 --- a/src/domain/explicitSensitiveFailure.ts +++ b/src/domain/explicitSensitiveFailure.ts @@ -264,11 +264,16 @@ const redactFailure = ( error.issues.map(issueProjection), ); if (error instanceof AnalysisOutputError) - return new AnalysisOutputError( - error.operation, - text(error.reason), - context, - ); + return new AnalysisOutputError(error.operation, text(error.reason), { + ...context, + ...(error.userMessage === undefined + ? {} + : { userMessage: text(error.userMessage) }), + ...(error.remediationAction === undefined + ? {} + : { remediationAction: text(error.remediationAction) }), + cleanupOnly: error.cleanupOnly, + }); if (error instanceof AnalysisCapabilityUnavailableError) return new AnalysisCapabilityUnavailableError( error.providerId, diff --git a/src/domain/flutter/flutterBuildAnalysis.ts b/src/domain/flutter/flutterBuildAnalysis.ts index b7d93af1cb..487327bb1c 100644 --- a/src/domain/flutter/flutterBuildAnalysis.ts +++ b/src/domain/flutter/flutterBuildAnalysis.ts @@ -44,6 +44,43 @@ const abiSelectorSchema = z .regex(/^[^\x2f]*$/u, ABI_SELECTOR_MESSAGE) .meta({ maxLength: ABI_SELECTOR_MAX_CHARACTERS }); +/** Bounded printable-run selection, independent of snapshot structural coverage. */ +const DART_POOL_DEFAULTS = { + uri_limit: 500, + token_limit: 2000, + run_bytes: 1024, + token_bytes: 128, +} as const; +export const dartPoolPolicySchema = z.strictObject({ + uri_limit: z.int().positive().optional(), + token_limit: z.int().positive().optional(), + run_bytes: z.int().positive().optional(), + token_bytes: z.int().positive().optional(), + contains: z + .string() + .optional() + .describe( + "Literal case-sensitive substring applied before collection; URI/token counts describe matches, printable_run_count is unfiltered", + ), +}); +type DartPoolPolicyInput = z.infer; +export type DartPoolPolicy = { + readonly uri_limit: number; + readonly token_limit: number; + readonly run_bytes: number; + readonly token_bytes: number; + readonly contains?: string | undefined; +}; +export const dartPoolPolicy = ( + selected?: DartPoolPolicyInput, +): DartPoolPolicy => ({ + uri_limit: selected?.uri_limit ?? DART_POOL_DEFAULTS.uri_limit, + token_limit: selected?.token_limit ?? DART_POOL_DEFAULTS.token_limit, + run_bytes: selected?.run_bytes ?? DART_POOL_DEFAULTS.run_bytes, + token_bytes: selected?.token_bytes ?? DART_POOL_DEFAULTS.token_bytes, + ...(selected?.contains === undefined ? {} : { contains: selected.contains }), +}); + /** * Caller intent for Flutter build identification. This family is pure * parsing: it reads an APK's `lib//libapp.so` and `libflutter.so` @@ -54,6 +91,11 @@ export const flutterInputSchemas = { path: localPathStringSchema.describe("Local APK file to inspect"), }), inspect_dart_aot: z.strictObject({ + pool_policy: dartPoolPolicySchema + .optional() + .describe( + "Printable-run selection and budgets for this call: uri_limit=500 per URI category, token_limit=2000, run_bytes=1024, token_bytes=128; optional contains filters before bounded collection. Positive safe integers. Applies on the next call without restart. Does not raise APK, library, ELF or response limits. Omitted facts can be recovered with a narrower contains or larger relevant budget.", + ), path: localPathStringSchema.describe( "Local APK file carrying a Flutter AOT payload", ), @@ -191,6 +233,11 @@ export const flutterResultSchemas = { ), }), string_pool: z.strictObject({ + policy: dartPoolPolicySchema + .optional() + .describe( + "Effective selection and budgets; absent from older Evidence", + ), coverage: z .enum(["complete", "partial"]) .describe( diff --git a/src/domain/fridaCapturePolicy.ts b/src/domain/fridaCapturePolicy.ts new file mode 100644 index 0000000000..00800913a4 --- /dev/null +++ b/src/domain/fridaCapturePolicy.ts @@ -0,0 +1,57 @@ +import { z } from "zod"; +import { MAX_JSON_DEPTH } from "./jsonValue.js"; + +/** Payload nesting reserves the enclosing message, status and Evidence containers. */ +export const FRIDA_CAPTURE_MAX_JSON_DEPTH = MAX_JSON_DEPTH - 8; + +const CAPTURE_POLICY_DESCRIPTION = + "Retention budgets selected before attaching or spawning, fixed for this session. Defaults: 16 MiB session, 4 KiB type, 32 KiB error text, 250000 JSON nodes and depth64. Session bytes include JSON/base64 expansion and capture metadata; a budget too small for one event retains no event. Raising these cannot recover already discarded observations or raise MCP/client delivery limits. Select larger budgets before a new explicitly chosen experiment; no REA restart is required."; + +/** Capture policy is accepted before attachment and remains fixed for that session. */ +export const fridaCapturePolicySchema = z + .strictObject({ + session_bytes: z + .int() + .positive() + .default(16 * 1024 * 1024), + message_type_bytes: z + .int() + .positive() + .default(4 * 1024), + message_text_bytes: z + .int() + .positive() + .default(32 * 1024), + json_nodes: z.int().positive().default(250_000), + json_depth: z.int().min(1).max(FRIDA_CAPTURE_MAX_JSON_DEPTH).default(64), + }) + .describe(CAPTURE_POLICY_DESCRIPTION); + +/** Request fields preserve omission; defaults are resolved only by the session owner. */ +export const fridaCapturePolicyInputSchema = z + .strictObject({ + session_bytes: fridaCapturePolicySchema.shape.session_bytes + .removeDefault() + .optional(), + message_type_bytes: fridaCapturePolicySchema.shape.message_type_bytes + .removeDefault() + .optional(), + message_text_bytes: fridaCapturePolicySchema.shape.message_text_bytes + .removeDefault() + .optional(), + json_nodes: fridaCapturePolicySchema.shape.json_nodes + .removeDefault() + .optional(), + json_depth: fridaCapturePolicySchema.shape.json_depth + .removeDefault() + .optional(), + }) + .describe(CAPTURE_POLICY_DESCRIPTION); + +export type FridaCapturePolicy = z.output; +export type FridaCapturePolicyInput = z.input< + typeof fridaCapturePolicyInputSchema +>; +export const DEFAULT_FRIDA_CAPTURE_POLICY = Object.freeze( + fridaCapturePolicySchema.parse({}), +); diff --git a/src/domain/fridaSessionStatus.ts b/src/domain/fridaSessionStatus.ts index fbff44043e..80551d21ba 100644 --- a/src/domain/fridaSessionStatus.ts +++ b/src/domain/fridaSessionStatus.ts @@ -1,3 +1,4 @@ +import { fridaCapturePolicySchema } from "./fridaCapturePolicy.js"; import { z } from "zod"; /** Delivery selection is separate from the provider's live session lookup. */ @@ -8,6 +9,7 @@ export const fridaSessionStatusInputSchema = z.strictObject({ /** Session identity and state shared by start, status and one-shot observations. */ export const fridaSessionObservationSchema = z.strictObject({ + capture_limits: fridaCapturePolicySchema.optional(), session_id: z.string().uuid(), device_id: z.string(), state: z.enum(["paused", "running", "detached"]), diff --git a/src/domain/go/goBinary.ts b/src/domain/go/goBinary.ts index d3af5413a6..8c580b35f2 100644 --- a/src/domain/go/goBinary.ts +++ b/src/domain/go/goBinary.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { constants as bufferConstants } from "node:buffer"; import { binaryMetadataArchitecture } from "../binaryMetadataArchitecture.js"; import { machoSelectionContainsRange, @@ -18,24 +19,126 @@ const sourceRangeSchema = z.strictObject({ bytes: z.int().nonnegative(), }); -export const DEFAULT_MAX_GO_STRING_LITERALS = 2000; -export const MAX_GO_STRING_LITERALS_LIMIT = 10_000; -export const MAX_GO_STRING_LITERAL_BYTES = 4096; -export const MAX_GO_STRING_AGGREGATE_BYTES = 512 * 1024; -export const MAX_GO_STRING_HEADER_SCAN_BYTES = 64 * 1024 * 1024; +/** Defaults applied at the owning reader; omitted request fields stay omitted in Evidence. */ +export const GO_ANALYSIS_BUDGET_DEFAULTS = { + max_input_bytes: 256 * 1024 * 1024, + max_report_bytes: 8 * 1024 * 1024, + max_build_info_bytes: 1024 * 1024, + max_header_search_bytes: 256 * 1024 * 1024, + max_string_literals: 2000, + max_string_literal_bytes: 4096, + max_string_bytes: 512 * 1024, + max_string_header_scan_bytes: 64 * 1024 * 1024, + max_string_candidates: 100_000, +} as const; -export const goMaxStringLiteralsSchema = z - .number() - .int() - .min(1) - .max(MAX_GO_STRING_LITERALS_LIMIT); +/** Actual single-buffer/string representability; work and row counts remain safe integers. */ +export const GO_ANALYSIS_BUDGET_MAXIMA = { + max_input_bytes: bufferConstants.MAX_LENGTH, + max_report_bytes: bufferConstants.MAX_STRING_LENGTH, + max_build_info_bytes: Math.floor(bufferConstants.MAX_STRING_LENGTH / 4) * 3, + max_header_search_bytes: Number.MAX_SAFE_INTEGER, + max_string_literals: Number.MAX_SAFE_INTEGER, + max_string_literal_bytes: bufferConstants.MAX_STRING_LENGTH, + max_string_bytes: Number.MAX_SAFE_INTEGER, + max_string_header_scan_bytes: Number.MAX_SAFE_INTEGER, + max_string_candidates: Number.MAX_SAFE_INTEGER, +} as const; + +export const goMaxStringLiteralsSchema = z.int().positive(); +export type GoAnalysisBudgets = { + readonly [Key in keyof typeof GO_ANALYSIS_BUDGET_DEFAULTS]: number; +}; +const budget = (parameter: keyof GoAnalysisBudgets, description: string) => + z + .int() + .positive() + .max(GO_ANALYSIS_BUDGET_MAXIMA[parameter]) + .exactOptional() + .describe( + `${description} Default ${String(GO_ANALYSIS_BUDGET_DEFAULTS[parameter])}. Positive safe integer, maximum ${String(GO_ANALYSIS_BUDGET_MAXIMA[parameter])}; byte maxima reflect runtime Buffer/string representability including base64 expansion. Applies on the next inspection without restart.`, + ); +export const goResourceBudgetFields = { + max_input_bytes: budget( + "max_input_bytes", + "Stable input byte admission; does not raise report or parser budgets.", + ), + max_report_bytes: budget( + "max_report_bytes", + "Complete normalized Go report byte admission; independent of MCP response delivery.", + ), + max_build_info_bytes: budget( + "max_build_info_bytes", + "Aggregate embedded Go build-info string bytes, including compiler version and framed module text.", + ), + max_header_search_bytes: budget( + "max_header_search_bytes", + "Cumulative Go build-info discovery bytes; overlapping search ranges consume work.", + ), + max_string_literals: budget( + "max_string_literals", + "Maximum returned inferred strings; requires include_string_literals: true.", + ), + max_string_literal_bytes: budget( + "max_string_literal_bytes", + "Largest inferred string candidate in bytes; requires include_string_literals: true.", + ), + max_string_bytes: budget( + "max_string_bytes", + "Aggregate returned inferred string bytes; requires include_string_literals: true.", + ), + max_string_header_scan_bytes: budget( + "max_string_header_scan_bytes", + "Static string-header and candidate-byte scan work; requires include_string_literals: true.", + ), + max_string_candidates: budget( + "max_string_candidates", + "Distinct static string candidates tracked before projection; requires include_string_literals: true.", + ), +} as const; +export const goAnalysisBudgets = ( + selection: Partial = {}, +): GoAnalysisBudgets => ({ + max_input_bytes: + selection.max_input_bytes ?? GO_ANALYSIS_BUDGET_DEFAULTS.max_input_bytes, + max_report_bytes: + selection.max_report_bytes ?? GO_ANALYSIS_BUDGET_DEFAULTS.max_report_bytes, + max_build_info_bytes: + selection.max_build_info_bytes ?? + GO_ANALYSIS_BUDGET_DEFAULTS.max_build_info_bytes, + max_header_search_bytes: + selection.max_header_search_bytes ?? + GO_ANALYSIS_BUDGET_DEFAULTS.max_header_search_bytes, + max_string_literals: + selection.max_string_literals ?? + GO_ANALYSIS_BUDGET_DEFAULTS.max_string_literals, + max_string_literal_bytes: + selection.max_string_literal_bytes ?? + GO_ANALYSIS_BUDGET_DEFAULTS.max_string_literal_bytes, + max_string_bytes: + selection.max_string_bytes ?? GO_ANALYSIS_BUDGET_DEFAULTS.max_string_bytes, + max_string_header_scan_bytes: + selection.max_string_header_scan_bytes ?? + GO_ANALYSIS_BUDGET_DEFAULTS.max_string_header_scan_bytes, + max_string_candidates: + selection.max_string_candidates ?? + GO_ANALYSIS_BUDGET_DEFAULTS.max_string_candidates, +}); +export const GO_STRING_BUDGET_KEYS = [ + "max_string_literals", + "max_string_literal_bytes", + "max_string_bytes", + "max_string_header_scan_bytes", + "max_string_candidates", +] as const; +export type GoStringBudgetKey = (typeof GO_STRING_BUDGET_KEYS)[number]; /** One distinct printable UTF-8 literal inferred from static (ptr, len) headers. */ export const goStringLiteralSchema = z.strictObject({ value: z.string().min(1), location: z.strictObject({ offset: z.int().nonnegative(), - bytes: z.int().min(1).max(MAX_GO_STRING_LITERAL_BYTES), + bytes: z.int().min(1), }), first_header_offset: z.int().nonnegative(), reference_count: z.int().positive(), @@ -44,11 +147,12 @@ export const goStringLiteralSchema = z.strictObject({ /** Bounded static (ptr, len) string-header inventory, separate from observed build_info. */ export const goStringLiteralsSchema = z.strictObject({ evidence_kind: z.literal("inferred"), - items: z.array(goStringLiteralSchema).max(MAX_GO_STRING_LITERALS_LIMIT), + items: z.array(goStringLiteralSchema), discovered_count: z.int().nonnegative(), - aggregate_bytes: z.int().nonnegative().max(MAX_GO_STRING_AGGREGATE_BYTES), + aggregate_bytes: z.int().nonnegative(), max_items: goMaxStringLiteralsSchema, truncated: z.boolean(), + limits_reached: z.array(z.enum(GO_STRING_BUDGET_KEYS)).exactOptional(), }); const moduleIdentitySchema = z.strictObject({ @@ -86,11 +190,7 @@ export const inspectGoBinaryInputSchema = z .describe( "Also recover bounded inferred Go UTF-8 string literals referenced by static (ptr, len) headers in non-executable mapped data", ), - max_string_literals: goMaxStringLiteralsSchema - .describe( - `Maximum distinct recovered string literals (1-${String(MAX_GO_STRING_LITERALS_LIMIT)}; default ${String(DEFAULT_MAX_GO_STRING_LITERALS)}; requires include_string_literals: true)`, - ) - .exactOptional(), + ...goResourceBudgetFields, detail: z .enum(["complete", "summary"]) .default("complete") @@ -99,16 +199,17 @@ export const inspectGoBinaryInputSchema = z ), }) .superRefine((input, context) => { - if ( - input.max_string_literals !== undefined && - input.include_string_literals !== true - ) - context.addIssue({ - code: "custom", - path: ["max_string_literals"], - message: - "max_string_literals requires include_string_literals to be true.", - }); + for (const parameter of GO_STRING_BUDGET_KEYS) { + if ( + input[parameter] !== undefined && + input.include_string_literals !== true + ) + context.addIssue({ + code: "custom", + path: [parameter], + message: `${parameter} requires include_string_literals to be true.`, + }); + } }); /** Compiler-written build information and its original file-backed source ranges. */ diff --git a/src/domain/javascript/electronWrappedCallPaths.test.ts b/src/domain/javascript/electronWrappedCallPaths.test.ts index e6e081bd6e..d593826fc7 100644 --- a/src/domain/javascript/electronWrappedCallPaths.test.ts +++ b/src/domain/javascript/electronWrappedCallPaths.test.ts @@ -1284,7 +1284,9 @@ it("reports nested bridge depth truncation while retaining shallow paths", () => expect(result.electron.call_paths).toContainEqual( expect.objectContaining({ entry: "desktop.shallow", channel: "shallow" }), ); - expect(result.limitations.join(" ")).toContain("depth 128"); + expect(result.limitations.join(" ")).toContain( + "semantic_precision.wrapper_depth=128", + ); }); it("preserves conditions when a wrapper dispatches through conditional external methods", () => { const result = analyzeJavaScriptStaticSource(` diff --git a/src/domain/javascript/electronWrappedCallPaths.ts b/src/domain/javascript/electronWrappedCallPaths.ts index 17303e2d6d..81c8d2287e 100644 --- a/src/domain/javascript/electronWrappedCallPaths.ts +++ b/src/domain/javascript/electronWrappedCallPaths.ts @@ -263,7 +263,7 @@ export function* collectElectronWrappedCallPathsSteps( ): Generator { // Bounds repeated interprocedural expansion independently of bundle bytes. // Exhaustion is reported as unknown coverage, never as a negative finding. - let remainingVisits = 1_000_000; + let remainingVisits = state.precision.wrapper_visits; const prefixes = new ElectronCallPathBlockPrefixes( () => --remainingVisits >= 0, state.parentsByNode, @@ -820,9 +820,9 @@ export function* collectElectronWrappedCallPathsSteps( readonly fn: SymbolicFunction; }> { if (active.has(value)) return; - if (active.size >= 128) { + if (active.size >= state.precision.wrapper_depth) { accumulator.unknownFindings += 1; - accumulator.electronCallPathLimitReached = true; + (accumulator.electronCallPathLimits ??= new Set()).add("wrapper_depth"); return; } const nested = new Set(active); @@ -1580,7 +1580,9 @@ export function* collectElectronWrappedCallPathsSteps( recordBridgeExposures(); if (remainingVisits < 0 || resolver.truncated) { accumulator.unknownFindings += 1; - accumulator.electronCallPathLimitReached = true; + const limits = (accumulator.electronCallPathLimits ??= new Set()); + if (remainingVisits < 0) limits.add("wrapper_visits"); + for (const limit of resolver.resourceLimits) limits.add(limit); } else // A dynamic site's channels are enumerated only when the complete trace // reached it with literal channels, no other invocation outcome, and no diff --git a/src/domain/javascript/javascriptAnalysisResourceControls.ts b/src/domain/javascript/javascriptAnalysisResourceControls.ts index bd5dd36786..7a6c7c4401 100644 --- a/src/domain/javascript/javascriptAnalysisResourceControls.ts +++ b/src/domain/javascript/javascriptAnalysisResourceControls.ts @@ -1,3 +1,7 @@ +import { + JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, + javaScriptSemanticPrecisionSchema, +} from "./javascriptSemanticPrecision.js"; import { z } from "zod"; /** Complete static checkpoint / per-script result transport ceiling. */ @@ -200,6 +204,15 @@ export const javaScriptAnalysisCoverageLimits = ( /** Optional operational choices; omission keeps them out of Evidence parameters. */ export const javaScriptAnalysisResourceControlsSchema = z.strictObject({ + semantic_precision: javaScriptSemanticPrecisionSchema.optional().describe( + `Per-source evaluator budgets. Omitted fields preserve defaults: ${Object.entries( + JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, + ) + .map(([name, value]) => `${name} ${String(value)}`) + .join( + ", ", + )}. Raising these does not raise graph retention, worker memory, transfer or parent result budgets; cyclic and unsupported values remain unknown.`, + ), max_semantic_nodes: z .number() .int() diff --git a/src/domain/javascript/javascriptAstValues.test.ts b/src/domain/javascript/javascriptAstValues.test.ts index 31f6a206ed..8ff1052439 100644 --- a/src/domain/javascript/javascriptAstValues.test.ts +++ b/src/domain/javascript/javascriptAstValues.test.ts @@ -9,7 +9,6 @@ import { staticObjectCopyPropertyReader, staticObjectPropertyNames, } from "./javascriptAstValues.js"; -import { SEMANTIC_EXPRESSION_DEPTH_LIMIT } from "./javascriptSemanticResourceLimits.js"; it("keeps string spread indices separate from named descriptor fields", () => { for (const text of ["", "x", "😀", "x".repeat(1_000_000)]) { @@ -145,17 +144,17 @@ it("limits unary-plus keys to transparent numeric literals", () => { ).toBeNull(); }); -it("bounds nested inline object spreads without losing a shallow final property", () => { +it("reads nested inline object spreads beyond the former evaluator depth cap", () => { let object = t.objectExpression([ t.objectProperty(t.identifier("stdio"), t.stringLiteral("inherit")), ]); - for (let depth = 0; depth < SEMANTIC_EXPRESSION_DEPTH_LIMIT; depth++) + for (let depth = 0; depth < 1_024; depth++) object = t.objectExpression([t.spreadElement(object)]); expect(staticObjectProperty(object, "stdio").status).toBe("explicit"); expect( staticObjectProperty(t.objectExpression([t.spreadElement(object)]), "stdio") .status, - ).toBe("dynamic"); + ).toBe("explicit"); }); it("collects complete own-key sets through bounded literal spreads", () => { @@ -222,17 +221,17 @@ it("retains known keys alongside unresolved object membership", () => { ).toEqual({ status: "partial", names: ["direct"] }); }); -it("does not claim a complete key set after exceeding spread depth", () => { +it("keeps complete key sets across deeply nested finite inline spreads", () => { let object = t.objectExpression([ t.objectProperty(t.identifier("query"), t.objectExpression([])), ]); - for (let depth = 0; depth < SEMANTIC_EXPRESSION_DEPTH_LIMIT; depth++) + for (let depth = 0; depth < 1_024; depth++) object = t.objectExpression([t.spreadElement(object)]); expect(staticObjectPropertyNames(object).status).toBe("complete"); expect( staticObjectPropertyNames(t.objectExpression([t.spreadElement(object)])) .status, - ).toBe("partial"); + ).toBe("complete"); }); it("enumerates a wide inline spread without a variadic call overflow", () => { diff --git a/src/domain/javascript/javascriptAstValues.ts b/src/domain/javascript/javascriptAstValues.ts index 1684724c73..2f26a7ecba 100644 --- a/src/domain/javascript/javascriptAstValues.ts +++ b/src/domain/javascript/javascriptAstValues.ts @@ -1,7 +1,6 @@ import * as t from "@babel/types"; import type { JavaScriptSemanticPrimitive } from "./javascriptSemanticValueTypes.js"; -import { SEMANTIC_EXPRESSION_DEPTH_LIMIT } from "./javascriptSemanticResourceLimits.js"; import { semanticArrayIndex } from "./javascriptSemanticPropertyPaths.js"; /** Computed definition keys use the surrounding evaluation environment. */ @@ -304,14 +303,22 @@ export const isObjectPrototypeSetter = ( !property.shorthand && key === "__proto__"; -/** Expand only bounded literal spreads, preserving unknown writes in order. */ +/** Iterate literal syntax once, preserving unknown writes and prototype order. */ const appendStaticObjectMembers = ( members: StaticObjectMember[], object: t.ObjectExpression, depth: number, isNullish?: (node: t.Node) => boolean, ): void => { - for (const property of object.properties) { + const pending = [{ object, depth, next: 0 }]; + while (pending.length > 0) { + const frame = pending[pending.length - 1]!; + const property = frame.object.properties[frame.next++]; + if (property === undefined) { + pending.pop(); + continue; + } + const depth = frame.depth; if (!t.isSpreadElement(property)) { const key = semanticStaticPropertyKey(property.key, property.computed); if ( @@ -337,34 +344,36 @@ const appendStaticObjectMembers = ( members.push({ kind: "property", property, key }); continue; } - const { node: spread, depth: wrappers } = unwrapJavaScriptExpression( + const nested = appendEnumerableSourceMembers( + members, property.argument, + depth, + isNullish, ); - appendEnumerableSourceMembers(members, spread, depth + wrappers, isNullish); + if (nested !== undefined) pending.push({ ...nested, next: 0 }); } }; -/** Copy own enumerable facts; inherited properties and prototype syntax are not writes. */ +/** Record atomic own-key facts or return literal syntax for iterative copying. */ const appendEnumerableSourceMembers = ( members: StaticObjectMember[], source: t.Node, depth: number, isNullish?: (node: t.Node) => boolean, -): void => { +): + | { readonly object: t.ObjectExpression; readonly depth: number } + | undefined => { const { node: spread, depth: wrappers } = unwrapJavaScriptExpression(source); if (t.isObjectExpression(spread)) { - const nextDepth = depth + wrappers + 1; if ( - nextDepth > SEMANTIC_EXPRESSION_DEPTH_LIMIT || spread.properties.some((entry) => t.isObjectMethod(entry, { kind: "get" }), ) ) { members.push(UNKNOWN_STATIC_OBJECT_MEMBER); - return; + return undefined; } - appendStaticObjectMembers(members, spread, nextDepth, isNullish); - return; + return { object: spread, depth: depth + wrappers + 1 }; } if (t.isArrayExpression(spread)) { let firstSpread = -1; @@ -384,7 +393,7 @@ const appendEnumerableSourceMembers = ( elements: spread.elements, firstSpread, }); - return; + return undefined; } const primitive = primitiveOwnKeys(spread, isNullish); if (primitive === "indices") { @@ -394,7 +403,7 @@ const appendEnumerableSourceMembers = ( ? literal.value.length : null; if (length !== 0) members.push({ kind: "primitive-indices", length }); - return; + return undefined; } if ( primitive === "none" || @@ -402,8 +411,9 @@ const appendEnumerableSourceMembers = ( t.isArrowFunctionExpression(spread) || t.isRegExpLiteral(spread) ) - return; + return undefined; members.push(UNKNOWN_STATIC_OBJECT_MEMBER); + return undefined; }; const staticObjectMembers = ( @@ -485,7 +495,9 @@ export const staticObjectCopyPropertyReader = ( ): ((name: string) => StaticObjectProperty) => { if (source === undefined) return indexStaticObjectProperties("missing"); const members: StaticObjectMember[] = []; - appendEnumerableSourceMembers(members, source, 0, isNullish); + const nested = appendEnumerableSourceMembers(members, source, 0, isNullish); + if (nested !== undefined) + appendStaticObjectMembers(members, nested.object, nested.depth, isNullish); return indexStaticObjectProperties(members); }; diff --git a/src/domain/javascript/javascriptBundledRequireOrigin.ts b/src/domain/javascript/javascriptBundledRequireOrigin.ts index 9a137d0eb5..e61c4e596b 100644 --- a/src/domain/javascript/javascriptBundledRequireOrigin.ts +++ b/src/domain/javascript/javascriptBundledRequireOrigin.ts @@ -11,7 +11,6 @@ import { } from "./javascriptAstValues.js"; import { collectJavaScriptIdentityWritesSteps } from "./javascriptElectronMemberWrites.js"; import { semanticBindingAtRead } from "./javascriptSemanticMutationInitializers.js"; -import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; import { createSemanticModuleInterop } from "./javascriptModuleInterop.js"; import { semanticRequireOriginCandidate, @@ -187,7 +186,7 @@ const canonicalWebpackInteropBindings = ( const node = pending.pop(); if (node === undefined || seen.has(node)) continue; seen.add(node); - if (seen.size > SEMANTIC_ALIAS_SELECTION_LIMIT) { + if (seen.size > state.precision.alias_selections) { state.valueResourceLimits.add("alias-selections"); return true; } diff --git a/src/domain/javascript/javascriptCommonJsExportReceivers.ts b/src/domain/javascript/javascriptCommonJsExportReceivers.ts index 0fb297c2aa..49d8146c8d 100644 --- a/src/domain/javascript/javascriptCommonJsExportReceivers.ts +++ b/src/domain/javascript/javascriptCommonJsExportReceivers.ts @@ -24,7 +24,6 @@ import { semanticEffectTiming, } from "./javascriptSemanticMutationInitializers.js"; import { javaScriptRuntimeContext } from "./javascriptSourceParser.js"; -import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; const exportInvocations = new WeakMap< JavaScriptSemanticAnalysisState, @@ -92,7 +91,7 @@ const copiedOwnFields = ( while (pending.length > 0) { const node = unwrapJavaScriptExpression(pending.pop()!).node; if (seen.has(node)) continue; - if (seen.size >= SEMANTIC_ALIAS_SELECTION_LIMIT) { + if (seen.size >= state.precision.alias_selections) { state.valueResourceLimits.add("alias-selections"); return unknown; } @@ -132,7 +131,7 @@ const copiedOwnFields = ( return unknown; if ( seen.size + pending.length + selected.length > - SEMANTIC_ALIAS_SELECTION_LIMIT + state.precision.alias_selections ) { state.valueResourceLimits.add("alias-selections"); return unknown; diff --git a/src/domain/javascript/javascriptElectronAppPathChanges.ts b/src/domain/javascript/javascriptElectronAppPathChanges.ts index 99e32759ce..db56a6d084 100644 --- a/src/domain/javascript/javascriptElectronAppPathChanges.ts +++ b/src/domain/javascript/javascriptElectronAppPathChanges.ts @@ -20,7 +20,6 @@ import { type SemanticCapturedSelection, } from "./javascriptSemanticProjectedSources.js"; import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js"; -import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; import { resolveSemanticBindingState, type JavaScriptSemanticAnalysisState, @@ -96,7 +95,7 @@ export const createSemanticElectronAppPathChangeQuery = ( const node = unwrapJavaScriptExpression(candidate.node).node; // Close the finite source graph after the precise context budget is // exhausted. Widening alone is never evidence of an Electron origin. - if (!widened && scheduled >= SEMANTIC_ALIAS_SELECTION_LIMIT) { + if (!widened && scheduled >= state.precision.alias_selections) { state.valueResourceLimits.add("alias-selections"); widened = true; } diff --git a/src/domain/javascript/javascriptSemanticAliasSelections.ts b/src/domain/javascript/javascriptSemanticAliasSelections.ts index 6a0b409433..427bf621e8 100644 --- a/src/domain/javascript/javascriptSemanticAliasSelections.ts +++ b/src/domain/javascript/javascriptSemanticAliasSelections.ts @@ -1,5 +1,5 @@ import type { JavaScriptSemanticPropertyPath } from "./javascriptSemanticPropertyPaths.js"; -import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; +import { JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS } from "./javascriptSemanticPrecision.js"; type PropertyKey = JavaScriptSemanticPropertyPath[number]; @@ -76,7 +76,10 @@ export class SemanticAliasSelections { #calls = 0; #limited = false; - constructor(readonly onLimit: () => void) {} + constructor( + readonly onLimit: () => void, + readonly maxSelections: number = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS.alias_selections, + ) {} get limited(): boolean { return this.#limited; @@ -95,10 +98,7 @@ export class SemanticAliasSelections { ); const covered = paths.find((prior) => pathCovers(prior, candidate)); if (covered !== undefined) return covered; - if ( - existing !== undefined && - ++this.#calls > SEMANTIC_ALIAS_SELECTION_LIMIT - ) { + if (existing !== undefined && ++this.#calls > this.maxSelections) { this.#limited = true; this.#paths = new WeakMap(); this.onLimit(); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 8d4015e845..f9abd62983 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -1,3 +1,7 @@ +import { + resolveJavaScriptSemanticPrecision, + type JavaScriptSemanticPrecisionInput, +} from "./javascriptSemanticPrecision.js"; import { semanticPatternPropertyReadSources } from "./javascriptSemanticIterationSources.js"; import { createSemanticOpaqueGlobalObserver, @@ -182,11 +186,12 @@ interface AddBindingInput { /** Recover lexical bindings, aliases, constants, and module links. */ export const analyzeJavaScriptSemantics = ( source: string, + precision?: JavaScriptSemanticPrecisionInput, ): JavaScriptSemanticIr => { const file = parseJavaScriptSource(source); return file === null ? failedJavaScriptSemanticIr() - : analyzeParsedJavaScriptSemantics(file); + : analyzeParsedJavaScriptSemantics(file, "import", precision); }; /** Recover lexical references, optionally for one name, without evaluating values or provenance. */ @@ -406,8 +411,9 @@ const semanticPathsOverlap = ( /** Share the lexical index with bounded, demand-driven boundary analyses. */ export function* indexJavaScriptBindingsSteps( program: t.Program, + precision?: JavaScriptSemanticPrecisionInput, ): Generator { - const state = createState(program); + const state = createState(program, "import", precision); yield* collectDefinitionsSteps(program, state); yield* indexSemanticAmbientWritesSteps(state); return state; @@ -417,9 +423,15 @@ export function* indexJavaScriptBindingsSteps( export const analyzeParsedJavaScriptSemantics = ( file: ParsedJavaScriptSource, staticImportResolutionMode: JavaScriptModuleResolutionMode = "import", + precision?: JavaScriptSemanticPrecisionInput, ): JavaScriptSemanticIr => completeSemanticSteps( - analyzeParsedJavaScriptSemanticsSteps(file, staticImportResolutionMode), + analyzeParsedJavaScriptSemanticsSteps( + file, + staticImportResolutionMode, + undefined, + precision, + ), ); /** @@ -432,8 +444,13 @@ export function* analyzeParsedJavaScriptSemanticsSteps( file: ParsedJavaScriptSource, staticImportResolutionMode: JavaScriptModuleResolutionMode = "import", expressionNodeBudget?: number, + precision?: JavaScriptSemanticPrecisionInput, ): Generator { - const state = createState(file.program, staticImportResolutionMode); + const state = createState( + file.program, + staticImportResolutionMode, + precision, + ); yield* collectDefinitionsSteps(file.program, state); yield* indexSemanticAmbientWritesSteps(state); yield* indexSemanticMemberMutationFactsSteps(state); @@ -467,6 +484,7 @@ export function* analyzeParsedJavaScriptSemanticsSteps( ...new Set([...state.valueResourceLimits, ...state.getterResourceLimits]), ].sort(), [...state.returnedReferenceWidenings].map(range), + state.precision, ); return { schema: "JavaScriptSemanticIR", @@ -486,14 +504,14 @@ export function* analyzeParsedJavaScriptSemanticsSteps( ] : []), ...(state.valueResourceLimits.has("alias-selections") - ? [semanticResourceLimitReason("alias-selections")] + ? [semanticResourceLimitReason("alias-selections", state.precision)] : []), ...(state.returnedReferenceWidenings.size > 0 ? [RETURNED_REFERENCE_WIDENING_LIMITATION] : []), ...[...state.getterResourceLimits] .sort() - .map(semanticResourceLimitReason), + .map((limit) => semanticResourceLimitReason(limit, state.precision)), ...(parserPartial ? [ "The parser recovered from syntax errors; affected bindings are partial.", @@ -537,8 +555,9 @@ export interface JavaScriptReceiverIndex extends JavaScriptBindingIndex { export function* indexJavaScriptBindingsWithReceiversSteps( program: t.Program, hostContext?: "commonjs-wrapper", + precision?: JavaScriptSemanticPrecisionInput, ): Generator { - const state = createState(program); + const state = createState(program, "import", precision); if (hostContext === "commonjs-wrapper") { const scope = currentSemanticScope(state.scopes); const binding: JavaScriptSemanticBindingState = { @@ -1154,6 +1173,7 @@ const bindingOrigin = ( const createState = ( program: t.Program, staticImportResolutionMode: JavaScriptModuleResolutionMode = "import", + precision?: JavaScriptSemanticPrecisionInput, ): JavaScriptSemanticAnalysisState => { const root: JavaScriptSemanticScopeState = { scopeId: semanticScopeId("program", program), @@ -1164,6 +1184,7 @@ const createState = ( bindings: new Map(), }; return { + precision: resolveJavaScriptSemanticPrecision(precision), arrayIterationUnknown: false, memberMutationFactsComplete: false, program, diff --git a/src/domain/javascript/javascriptSemanticCalls.ts b/src/domain/javascript/javascriptSemanticCalls.ts index a6a116dfeb..1442a921fc 100644 --- a/src/domain/javascript/javascriptSemanticCalls.ts +++ b/src/domain/javascript/javascriptSemanticCalls.ts @@ -269,7 +269,7 @@ const directLiteral = ( } => { const literal = readExactJavaScriptLiteral(node); if (!literal.found) return {}; - const parsed = semanticPrimitiveSet([literal.value]); + const parsed = semanticPrimitiveSet([literal.value], state.precision); if (parsed.status === "literal") return { literal: parsed.value }; if ( parsed.status === "unknown" && diff --git a/src/domain/javascript/javascriptSemanticCoverage.ts b/src/domain/javascript/javascriptSemanticCoverage.ts index 6d73d3c5f3..81b4bee3e4 100644 --- a/src/domain/javascript/javascriptSemanticCoverage.ts +++ b/src/domain/javascript/javascriptSemanticCoverage.ts @@ -1,13 +1,11 @@ +import { semanticResourceLimitFacts } from "./javascriptSemanticResourceLimits.js"; import type { JavaScriptSourceRange } from "./javascriptStaticAnalysisTypes.js"; import type { JavaScriptSemanticResourceLimit } from "./javascriptSemanticValueTypes.js"; import type { ApplicationCoverage } from "./javascriptApplicationEvidenceSchemas.js"; import { - SEMANTIC_EXPRESSION_DEPTH_LIMIT, - SEMANTIC_GETTER_EXPANSION_LIMIT, - SEMANTIC_ALIAS_SELECTION_LIMIT, - SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT, - SEMANTIC_PRIMITIVE_JSON_BYTES_LIMIT, -} from "./javascriptSemanticResourceLimits.js"; + JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, + type JavaScriptSemanticPrecision, +} from "./javascriptSemanticPrecision.js"; /** Exact retention state for one callable's direct returns. */ export type JavaScriptSemanticReturnCoverage = { @@ -23,18 +21,21 @@ export type JavaScriptSemanticCoverage = readonly status: "complete"; readonly omittedCount: 0; readonly resourceLimits?: readonly JavaScriptSemanticResourceLimit[]; + readonly precision?: JavaScriptSemanticPrecision; readonly widenedReturnedReferences?: readonly JavaScriptSourceRange[]; } | { readonly status: "partial"; readonly omittedCount: 0 | null; readonly resourceLimits?: readonly JavaScriptSemanticResourceLimit[]; + readonly precision?: JavaScriptSemanticPrecision; readonly widenedReturnedReferences?: readonly JavaScriptSourceRange[]; } | { readonly status: "failed"; readonly omittedCount: null; readonly resourceLimits?: readonly JavaScriptSemanticResourceLimit[]; + readonly precision?: JavaScriptSemanticPrecision; readonly widenedReturnedReferences?: readonly JavaScriptSourceRange[]; }; @@ -47,15 +48,22 @@ export const semanticCoverage = ( parserPartial: boolean, resourceLimits: readonly JavaScriptSemanticResourceLimit[] = [], widenedReturnedReferences: readonly JavaScriptSourceRange[] = [], + precision: JavaScriptSemanticPrecision = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, ): JavaScriptSemanticCoverage => { - const widened = - widenedReturnedReferences.length > 0 ? { widenedReturnedReferences } : {}; + const details = { + ...(widenedReturnedReferences.length > 0 + ? { widenedReturnedReferences } + : {}), + ...(precision === JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS + ? {} + : { precision }), + }; if (resourceLimits.length > 0) return { status: "partial", omittedCount: null, resourceLimits, - ...widened, + ...details, }; return { status: @@ -63,7 +71,7 @@ export const semanticCoverage = ( ? "partial" : "complete", omittedCount: 0, - ...widened, + ...details, }; }; @@ -72,40 +80,22 @@ export const semanticCoverageResourceLimits = ( coverage: JavaScriptSemanticCoverage, ): readonly JavaScriptSemanticResourceLimit[] => coverage.resourceLimits ?? []; -const semanticResourceBounds: Record< - JavaScriptSemanticResourceLimit, - Pick -> = { - "primitive-candidates": { - value: SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT, - unit: "items", - }, - "primitive-bytes": { - value: SEMANTIC_PRIMITIVE_JSON_BYTES_LIMIT, - unit: "bytes", - }, - "expression-depth": { - value: SEMANTIC_EXPRESSION_DEPTH_LIMIT, - unit: "depth", - }, - "getter-expansion": { - value: SEMANTIC_GETTER_EXPANSION_LIMIT, - unit: "items", - }, - "alias-selections": { - value: SEMANTIC_ALIAS_SELECTION_LIMIT, - unit: "items", - }, -}; - -/** Map semantic evaluator bounds to their caller-visible application coverage. */ +/** Effective evaluator facts, rather than reconstructed default-only bounds. */ export const semanticResourceLimitCoverage = ( resourceLimits: readonly JavaScriptSemanticResourceLimit[], + precision: JavaScriptSemanticPrecision = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, ): ApplicationCoverage["limits"] => - resourceLimits.map((resourceLimit) => ({ - name: `javascript_semantic_${resourceLimit.replaceAll("-", "_")}`, - ...semanticResourceBounds[resourceLimit], - })); + resourceLimits.map((resourceLimit) => { + const { value, unit } = semanticResourceLimitFacts( + resourceLimit, + precision, + ); + return { + name: `javascript_semantic_${resourceLimit.replaceAll("-", "_")}`, + value, + unit, + }; + }); /** Classify one callable's direct-return coverage. */ export const semanticReturnCoverage = ( diff --git a/src/domain/javascript/javascriptSemanticDataEffects.ts b/src/domain/javascript/javascriptSemanticDataEffects.ts index 0267344c86..f63782365a 100644 --- a/src/domain/javascript/javascriptSemanticDataEffects.ts +++ b/src/domain/javascript/javascriptSemanticDataEffects.ts @@ -220,7 +220,7 @@ const collectDefaults = ( if (candidate === undefined) return; const literal = readExactJavaScriptLiteral(node.right); const value: JavaScriptSemanticValue = literal.found - ? semanticPrimitiveSet([literal.value]) + ? semanticPrimitiveSet([literal.value], context.state.precision) : { status: "unknown", reason: "Default is not an exact primitive." }; recordSemanticValueLimit(context.state, value); output.push({ diff --git a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts index 3752482346..d5580e4298 100644 --- a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts @@ -188,7 +188,7 @@ function* collectExpressionFlowsSteps( const expression = unwrapJavaScriptExpression(returnValue).node; const exact = readExactJavaScriptLiteral(expression); if (exact.found) { - const value = semanticPrimitiveSet([exact.value]); + const value = semanticPrimitiveSet([exact.value], state.precision); if (value.status === "literal" && literalReturns.length < budget) literalReturns.push({ location: range(expression), @@ -270,7 +270,9 @@ function* collectExpressionFlowsSteps( (t.isSequenceExpression(node) && index !== values.length - 1); const operand = unwrapJavaScriptExpression(value).node; const exact = readExactJavaScriptLiteral(operand); - const literal = exact.found ? semanticPrimitiveSet([exact.value]) : null; + const literal = exact.found + ? semanticPrimitiveSet([exact.value], state.precision) + : null; if ( literal?.status === "unknown" && literal.resourceLimit === "primitive-bytes" diff --git a/src/domain/javascript/javascriptSemanticExpressionShapes.ts b/src/domain/javascript/javascriptSemanticExpressionShapes.ts index 395a5af1d6..cde0cda017 100644 --- a/src/domain/javascript/javascriptSemanticExpressionShapes.ts +++ b/src/domain/javascript/javascriptSemanticExpressionShapes.ts @@ -16,7 +16,6 @@ import { isUnshadowedGlobal, semanticBindingAtRead, } from "./javascriptSemanticMutationInitializers.js"; -import { SEMANTIC_EXPRESSION_DEPTH_LIMIT } from "./javascriptSemanticResourceLimits.js"; import { resolveSemanticBindingState, semanticStableInitializer, @@ -66,7 +65,7 @@ export const createSemanticExpressionShapeResolver = ( const unwrapped = unwrapJavaScriptExpression(node); node = unwrapped.node; depth += unwrapped.depth; - if (depth > SEMANTIC_EXPRESSION_DEPTH_LIMIT) { + if (depth > state.precision.expression_depth) { state.valueResourceLimits.add("expression-depth"); return "unknown"; } diff --git a/src/domain/javascript/javascriptSemanticGetterQueries.ts b/src/domain/javascript/javascriptSemanticGetterQueries.ts index b87eac3e8a..d857d9b7ce 100644 --- a/src/domain/javascript/javascriptSemanticGetterQueries.ts +++ b/src/domain/javascript/javascriptSemanticGetterQueries.ts @@ -2,7 +2,6 @@ import type * as t from "@babel/types"; import type { JavaScriptSemanticObjectReference as Reference } from "./javascriptSemanticObjectReference.js"; import type { JavaScriptSemanticPropertyPath as PropertyPath } from "./javascriptSemanticPropertyPaths.js"; import type { JavaScriptSemanticAnalysisState } from "./javascriptSemanticState.js"; -import { SEMANTIC_GETTER_EXPANSION_LIMIT } from "./javascriptSemanticResourceLimits.js"; export interface GetterInvocation { readonly getter: t.ObjectMethod | t.ClassMethod; @@ -71,7 +70,10 @@ export class SemanticGetterQuery { /** Charge retained local work and getter jobs to the same precise query. */ reserveResult(): boolean { if (this.#limited) return false; - if (!this.#fallback && this.#visits >= SEMANTIC_GETTER_EXPANSION_LIMIT) { + if ( + !this.#fallback && + this.#visits >= this.state.precision.getter_expansions + ) { this.#limited = true; return false; } diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index e028b4e54f..451597cc81 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -67,7 +67,6 @@ import { semanticScopeExitEffect, } from "./javascriptSemanticMutationInitializers.js"; import { semanticCoercionOperands } from "./javascriptSemanticLanguageInvocations.js"; -import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; import { semanticKnownArrayIteration, semanticReferenceSourcesInactive, @@ -148,7 +147,7 @@ const localAccessorMethods = ( if (paths.has(key)) continue; paths.add(key); examined.set(node, paths); - if (++selections > SEMANTIC_ALIAS_SELECTION_LIMIT) { + if (++selections > state.precision.alias_selections) { state.valueResourceLimits.add("alias-selections"); return undefined; } @@ -407,7 +406,7 @@ export function* collectSemanticMemberMutationsSteps( ]); if (examined.has(identity)) continue; examined.add(identity); - if (examined.size > SEMANTIC_ALIAS_SELECTION_LIMIT) { + if (examined.size > state.precision.alias_selections) { state.valueResourceLimits.add("alias-selections"); return [{ node: root, path: [] }]; } @@ -810,7 +809,7 @@ export function* collectSemanticMemberMutationsSteps( if (exact.has(selection)) return; if ( exact.size > 0 && - ++additionalAlternatives > SEMANTIC_ALIAS_SELECTION_LIMIT + ++additionalAlternatives > state.precision.alias_selections ) { projectionsWidened = true; state.valueResourceLimits.add("alias-selections"); diff --git a/src/domain/javascript/javascriptSemanticPrecision.test.ts b/src/domain/javascript/javascriptSemanticPrecision.test.ts new file mode 100644 index 0000000000..93d5fe0a07 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticPrecision.test.ts @@ -0,0 +1,106 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; +import { onlyBinding } from "./javascriptSemanticAnalysis.fixture.js"; +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +import { semanticResourceLimitCoverage } from "./javascriptSemanticCoverage.js"; +import { javaScriptSemanticPrecisionSchema } from "./javascriptSemanticPrecision.js"; + +const alternatives = (start: number, count: number): string => + count === 1 + ? JSON.stringify(`value_${String(start)}`) + : `(flag ? ${alternatives(start, count / 2)} : ${alternatives(start + count / 2, count / 2)})`; + +it("recovers the same finite candidate set with a source-owned precision budget", () => { + const source = `export const selected = ${alternatives(0, 512)};`; + const bounded = analyzeJavaScriptSemantics(source); + expect(onlyBinding(bounded, "selected").value).toMatchObject({ + status: "unknown", + resourceLimit: "primitive-candidates", + reason: expect.stringContaining( + "semantic_precision.primitive_candidates=256", + ), + }); + const recovered = analyzeJavaScriptSemantics(source, { + primitive_candidates: 512, + }); + const value = onlyBinding(recovered, "selected").value; + expect(value.status).toBe("union"); + if (value.status !== "union") + throw new Error("Expected finite primitive alternatives"); + expect(value.values).toHaveLength(512); + expect(value.values).toContain("value_511"); + expect(recovered.coverage.resourceLimits).toBeUndefined(); + // A later source does not inherit a previous source's caller choice. + expect( + onlyBinding(analyzeJavaScriptSemantics(source), "selected").value, + ).toMatchObject({ + status: "unknown", + resourceLimit: "primitive-candidates", + }); +}); + +it("preserves exact effective precision when an increased budget still exhausts", () => { + const source = `const selected = ${alternatives(0, 512)};`; + const result = analyzeJavaScriptSemantics(source, { + primitive_candidates: 300, + }); + expect(onlyBinding(result, "selected").value).toMatchObject({ + reason: expect.stringContaining( + "semantic_precision.primitive_candidates=300", + ), + }); + expect( + semanticResourceLimitCoverage( + result.coverage.resourceLimits ?? [], + result.coverage.precision, + ), + ).toContainEqual({ + name: "javascript_semantic_primitive_candidates", + value: 300, + unit: "items", + }); +}); + +it("distinguishes wrapper work recovery from unchanged unsupported or cyclic syntax", () => { + const source = `const e = require("electron"); e.ipcMain.on("first", event => event.sender.copy());`; + const bounded = analyzeJavaScriptStaticSource(source, { wrapper_visits: 1 }); + expect( + bounded.analysis_coverage?.examination.electron_call_paths_limited, + ).toBe(true); + expect(bounded.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("semantic_precision.wrapper_visits=1"), + ]), + ); + const recovered = analyzeJavaScriptStaticSource(source, { + wrapper_visits: 10_000, + }); + expect( + recovered.analysis_coverage?.examination.electron_call_paths_limited, + ).toBe(false); + expect((recovered.electron.call_paths ?? []).length).toBeGreaterThan(0); + const cycle = analyzeJavaScriptSemantics( + "const a = b; const b = a; const selected = a;", + { primitive_candidates: 512 }, + ); + expect(onlyBinding(cycle, "selected").value).toMatchObject({ + status: "unknown", + }); + expect(onlyBinding(cycle, "selected").value).not.toHaveProperty( + "resourceLimit", + ); +}); + +it.each([ + { primitive_candidates: 0 }, + { primitive_bytes: -1 }, + { wrapper_visits: 1.5 }, + { expression_depth: Number.POSITIVE_INFINITY }, +])( + "rejects invalid precision without silently replacing the caller choice: %o", + (input) => + expect(javaScriptSemanticPrecisionSchema.safeParse(input).success).toBe( + false, + ), +); diff --git a/src/domain/javascript/javascriptSemanticPrecision.ts b/src/domain/javascript/javascriptSemanticPrecision.ts new file mode 100644 index 0000000000..e45120162b --- /dev/null +++ b/src/domain/javascript/javascriptSemanticPrecision.ts @@ -0,0 +1,41 @@ +import { z } from "zod"; + +/** Per-source evaluator work, distinct from retained graph size and worker memory. */ +export const JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS = { + primitive_candidates: 256, + primitive_bytes: 1_048_576, + expression_depth: 256, + getter_expansions: 512, + alias_selections: 512, + wrapper_visits: 1_000_000, + wrapper_depth: 128, +} as const; + +const work = z.number().int().min(1).max(Number.MAX_SAFE_INTEGER); +export const javaScriptSemanticPrecisionSchema = z.strictObject({ + primitive_candidates: work.exactOptional(), + primitive_bytes: work + .exactOptional() + .describe( + `Conservative JSON string-payload allowance per evaluated candidate set; default ${String(JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS.primitive_bytes)}. Admission charges six bytes per UTF-16 code unit plus quotes before creating concatenated strings, so ordinary ASCII can exhaust this estimate before its actual JSON encoding reaches the selected byte count. Independent of graph, checkpoint and worker-transfer budgets.`, + ), + expression_depth: work.exactOptional(), + getter_expansions: work.exactOptional(), + alias_selections: work.exactOptional(), + wrapper_visits: work.exactOptional(), + wrapper_depth: work.exactOptional(), +}); +export type JavaScriptSemanticPrecisionInput = z.output< + typeof javaScriptSemanticPrecisionSchema +>; +export type JavaScriptSemanticPrecision = { + readonly [Key in keyof typeof JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS]: number; +}; + +/** Resolve once at source admission; consumers share this exact policy. */ +export const resolveJavaScriptSemanticPrecision = ( + input?: JavaScriptSemanticPrecisionInput, +): JavaScriptSemanticPrecision => + input === undefined + ? JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS + : { ...JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, ...input }; diff --git a/src/domain/javascript/javascriptSemanticPrimitives.ts b/src/domain/javascript/javascriptSemanticPrimitives.ts index b66451cddb..eddf7484a5 100644 --- a/src/domain/javascript/javascriptSemanticPrimitives.ts +++ b/src/domain/javascript/javascriptSemanticPrimitives.ts @@ -1,3 +1,7 @@ +import { + JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, + type JavaScriptSemanticPrecision, +} from "./javascriptSemanticPrecision.js"; import type { JavaScriptSemanticPrimitive, JavaScriptSemanticValue, @@ -5,7 +9,6 @@ import type { import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { semanticPrimitiveKey } from "./javascriptSemanticProvenance.js"; import { - SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT, exceedsSemanticPrimitiveStringByteBudget, semanticResourceLimitUnknown, } from "./javascriptSemanticResourceLimits.js"; @@ -13,6 +16,7 @@ import { /** Normalize one bounded collection of possible primitive values. */ export const semanticPrimitiveSet = ( values: readonly JavaScriptSemanticPrimitive[], + precision: JavaScriptSemanticPrecision = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, ): JavaScriptSemanticValue => { let uniqueValues: Set | undefined; for (const value of values) { @@ -30,13 +34,13 @@ export const semanticPrimitiveSet = ( // or canonical-key allocation. Check strings before retaining the value. if (values.length === 1) return typeof value === "string" && - exceedsSemanticPrimitiveStringByteBudget([value]) - ? semanticResourceLimitUnknown("primitive-bytes") + exceedsSemanticPrimitiveStringByteBudget([value], precision) + ? semanticResourceLimitUnknown("primitive-bytes", precision) : { status: "literal", value }; uniqueValues ??= new Set(); uniqueValues.add(value); - if (uniqueValues.size > SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT) - return semanticResourceLimitUnknown("primitive-candidates"); + if (uniqueValues.size > precision.primitive_candidates) + return semanticResourceLimitUnknown("primitive-candidates", precision); } if (uniqueValues === undefined) return { status: "union", values: [] }; if ( @@ -44,9 +48,10 @@ export const semanticPrimitiveSet = ( [...uniqueValues].flatMap((value) => typeof value === "string" ? [value] : [], ), + precision, ) ) - return semanticResourceLimitUnknown("primitive-bytes"); + return semanticResourceLimitUnknown("primitive-bytes", precision); const unique = [...uniqueValues] .map((value) => ({ key: semanticPrimitiveKey(value), value })) .sort((left, right) => compareUnicodeCodePoints(left.key, right.key)) diff --git a/src/domain/javascript/javascriptSemanticProjectedSources.ts b/src/domain/javascript/javascriptSemanticProjectedSources.ts index 963d0d8903..2f0a6f5458 100644 --- a/src/domain/javascript/javascriptSemanticProjectedSources.ts +++ b/src/domain/javascript/javascriptSemanticProjectedSources.ts @@ -10,7 +10,6 @@ import { semanticArrayPropertyReferences, type JavaScriptSemanticObjectReference, } from "./javascriptSemanticObjectReferences.js"; -import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; import { semanticBindingAtRead, semanticDefiniteEffectPrecedesReference, @@ -115,7 +114,7 @@ const hasFreshProjectedReceiver = ( if (paths.has(key)) return false; paths.add(key); examined.set(node, paths); - if (++selections > SEMANTIC_ALIAS_SELECTION_LIMIT) { + if (++selections > state.precision.alias_selections) { state.valueResourceLimits.add("alias-selections"); return false; } @@ -138,7 +137,7 @@ const hasFreshProjectedReceiver = ( if (sources.length === 0) return false; if ( selections + pending.length + sources.length > - SEMANTIC_ALIAS_SELECTION_LIMIT + state.precision.alias_selections ) { state.valueResourceLimits.add("alias-selections"); return false; diff --git a/src/domain/javascript/javascriptSemanticRequireOrigin.ts b/src/domain/javascript/javascriptSemanticRequireOrigin.ts index 33f82ff467..94d95f004b 100644 --- a/src/domain/javascript/javascriptSemanticRequireOrigin.ts +++ b/src/domain/javascript/javascriptSemanticRequireOrigin.ts @@ -53,7 +53,6 @@ import { semanticAliasSelectionChoices, type SemanticAliasSelectionPath, } from "./javascriptSemanticAliasSelections.js"; -import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; import { semanticRequireOriginReaders, type JavaScriptRequireLoaderFact, @@ -413,7 +412,7 @@ const selectedRequireSourcesAt = ( if (paths.has(key)) continue; paths.add(key); seen.set(node, paths); - if (++count > SEMANTIC_ALIAS_SELECTION_LIMIT) { + if (++count > state.precision.alias_selections) { state.valueResourceLimits.add("alias-selections"); return { loader: "unknown", sourceRelative: false }; } @@ -1179,7 +1178,7 @@ const hasCommonJsModuleSource = ( // Keep depths separate and join finite key alternatives at each source read. const selections = new SemanticAliasSelections(() => { state.valueResourceLimits.add("alias-selections"); - }); + }, state.precision.alias_selections); const contexts = new WeakMap< t.Node, Map< @@ -1228,10 +1227,10 @@ const hasCommonJsModuleSource = ( // Consumption contexts must not provide an unbounded second path to // expansion beyond the shared selector budget. if (exhausted) { - if (++drained > SEMANTIC_ALIAS_SELECTION_LIMIT) return "incomplete"; + if (++drained > state.precision.alias_selections) return "incomplete"; } else if ( (byMode.size > 1 || context.identities.size > 0) && - ++additionalStates > SEMANTIC_ALIAS_SELECTION_LIMIT + ++additionalStates > state.precision.alias_selections ) { state.valueResourceLimits.add("alias-selections"); if (includeUnknown) return "possible"; diff --git a/src/domain/javascript/javascriptSemanticResourceLimits.ts b/src/domain/javascript/javascriptSemanticResourceLimits.ts index 5a94e90e80..31a5fbc96a 100644 --- a/src/domain/javascript/javascriptSemanticResourceLimits.ts +++ b/src/domain/javascript/javascriptSemanticResourceLimits.ts @@ -3,28 +3,53 @@ import type { JavaScriptSemanticValue, } from "./javascriptSemanticValueTypes.js"; -/** Maximum distinct alternatives retained in one normalized primitive value. */ -export const SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT = 256; -/** Maximum active expression nesting during inert semantic evaluation. */ -export const SEMANTIC_EXPRESSION_DEPTH_LIMIT = 256; -/** Getter jobs and local result work retained in one precise query. */ -export const SEMANTIC_GETTER_EXPANSION_LIMIT = 512; -/** Additional selector states retained by one precise alias query. */ -export const SEMANTIC_ALIAS_SELECTION_LIMIT = 512; -/** Bound normalized strings and transient keys independently of transport limits. */ -export const SEMANTIC_PRIMITIVE_JSON_BYTES_LIMIT = 1_048_576; +import { + JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, + type JavaScriptSemanticPrecision, +} from "./javascriptSemanticPrecision.js"; -const RESOURCE_LIMIT_REASON: Record = { - "primitive-candidates": - "Primitive candidate budget exceeded (maximum 256 alternatives).", - "primitive-bytes": - "Primitive string-byte budget exceeded (maximum 1048576 estimated JSON bytes).", - "expression-depth": - "Semantic expression depth budget exceeded (maximum 256 nested expressions).", - "getter-expansion": - "Getter expansion was conservatively bounded (maximum 512 scheduled states or a growing receiver cycle).", - "alias-selections": - "Alias selection expansion was conservatively bounded (maximum 512 additional selector states); selected paths remain unknown.", +const resourceControls = { + "primitive-candidates": ["primitive_candidates", "items", "alternatives"], + "primitive-bytes": ["primitive_bytes", "bytes", "estimated JSON bytes"], + "expression-depth": ["expression_depth", "depth", "nested expressions"], + "getter-expansion": ["getter_expansions", "items", "scheduled states"], + "alias-selections": [ + "alias_selections", + "items", + "additional selector states", + ], +} as const; + +/** One owner maps exhaustion classes to controls and effective coverage facts. */ +export const semanticResourceLimitFacts = ( + resourceLimit: JavaScriptSemanticResourceLimit, + precision: JavaScriptSemanticPrecision = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, +) => { + const [control, unit, displayUnit] = resourceControls[resourceLimit]; + return { + control, + unit, + displayUnit, + value: precision[control], + defaultValue: JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS[control], + }; +}; + +/** Exact caller control and effective value at the evaluator boundary. */ +export const semanticResourceLimitReason = ( + resourceLimit: JavaScriptSemanticResourceLimit, + precision: JavaScriptSemanticPrecision = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, +): string => { + const { control, displayUnit, value, defaultValue } = + semanticResourceLimitFacts(resourceLimit, precision); + const prefixes = { + "primitive-candidates": `Primitive candidate budget exceeded (maximum ${String(precision.primitive_candidates)} alternatives).`, + "primitive-bytes": `Primitive string-byte budget exceeded (maximum ${String(precision.primitive_bytes)} estimated JSON bytes).`, + "expression-depth": `Semantic expression depth budget exceeded (maximum ${String(precision.expression_depth)} nested expressions).`, + "getter-expansion": `Getter expansion was conservatively bounded (maximum ${String(precision.getter_expansions)} scheduled states or a growing receiver cycle).`, + "alias-selections": `Alias selection expansion was conservatively bounded (maximum ${String(precision.alias_selections)} additional selector states); selected paths remain unknown.`, + }; + return `${prefixes[resourceLimit]} Semantic ${resourceLimit} budget reached: semantic_precision.${control}=${String(value)} ${displayUnit} (default ${String(defaultValue)}). Increase semantic_precision.${control} on the next analysis call (analyze_javascript_application or analyze_web_bundle), or pass --semantic-precision JSON on the CLI; no REA restart is required. Worker memory, transfer and graph-retention limits remain independent. Cyclic or unsupported evaluations remain unknown regardless of budget.`; }; /** Check UTF-16 length against worst-case JSON escaping without flattening. */ @@ -34,11 +59,12 @@ const jsonStringBytesFromLength = (codeUnits: number): number => /** Bound string payload before JSON key serialization or candidate creation. */ export const exceedsSemanticPrimitiveStringByteBudget = ( strings: Iterable, + precision: JavaScriptSemanticPrecision = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, ): boolean => { let total = 0; for (const value of strings) { total += jsonStringBytesFromLength(value.length); - if (total > SEMANTIC_PRIMITIVE_JSON_BYTES_LIMIT) return true; + if (total > precision.primitive_bytes) return true; } return false; }; @@ -47,6 +73,7 @@ export const exceedsSemanticPrimitiveStringByteBudget = ( export const exceedsSemanticPrimitiveAdditionByteBudget = ( left: readonly (string | number | boolean | null)[], right: readonly (string | number | boolean | null)[], + precision: JavaScriptSemanticPrecision = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, ): boolean => { let total = 0; for (const leftValue of left) { @@ -55,7 +82,7 @@ export const exceedsSemanticPrimitiveAdditionByteBudget = ( continue; const length = String(leftValue).length + String(rightValue).length; total += jsonStringBytesFromLength(length); - if (total > SEMANTIC_PRIMITIVE_JSON_BYTES_LIMIT) return true; + if (total > precision.primitive_bytes) return true; } } return false; @@ -65,27 +92,24 @@ export const exceedsSemanticPrimitiveAdditionByteBudget = ( export const exceedsSemanticTemplateByteBudget = ( prefixes: readonly string[], suffixes: readonly string[], + precision: JavaScriptSemanticPrecision = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, ): boolean => { let total = 0; for (const prefix of prefixes) { for (const suffix of suffixes) { total += jsonStringBytesFromLength(prefix.length + suffix.length); - if (total > SEMANTIC_PRIMITIVE_JSON_BYTES_LIMIT) return true; + if (total > precision.primitive_bytes) return true; } } return false; }; -/** Make an explicit unknown value tagged with the semantic resource bound. */ +/** Make an explicit unknown value tagged with its effective semantic bound. */ export const semanticResourceLimitUnknown = ( resourceLimit: JavaScriptSemanticResourceLimit, + precision: JavaScriptSemanticPrecision = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, ): JavaScriptSemanticValue => ({ status: "unknown", - reason: RESOURCE_LIMIT_REASON[resourceLimit], + reason: semanticResourceLimitReason(resourceLimit, precision), resourceLimit, }); - -/** Display label for one stable resource classification. */ -export const semanticResourceLimitReason = ( - resourceLimit: JavaScriptSemanticResourceLimit, -): string => RESOURCE_LIMIT_REASON[resourceLimit]; diff --git a/src/domain/javascript/javascriptSemanticReturnedReferences.ts b/src/domain/javascript/javascriptSemanticReturnedReferences.ts index 4a3e93cb00..0cc8e377b9 100644 --- a/src/domain/javascript/javascriptSemanticReturnedReferences.ts +++ b/src/domain/javascript/javascriptSemanticReturnedReferences.ts @@ -31,7 +31,6 @@ import { semanticShiftCapturedSelections, type SemanticCapturedSelection, } from "./javascriptSemanticProjectedSources.js"; -import { SEMANTIC_ALIAS_SELECTION_LIMIT } from "./javascriptSemanticResourceLimits.js"; import { semanticPropertyPathKeyMatches, type JavaScriptSemanticPropertyPath, @@ -184,7 +183,7 @@ export const createSemanticReturnedReferences = ( const selections = new SemanticAliasSelections(() => { state.valueResourceLimits.add("alias-selections"); result.precision = "widened"; - }); + }, state.precision.alias_selections); const enqueue = ( node: t.Node, path: Candidate["path"], @@ -255,7 +254,7 @@ export const createSemanticReturnedReferences = ( !contextsWidened && (capturedSelections !== undefined || invocationContext !== undefined) && paths.size > 0 && - ++additionalContexts > SEMANTIC_ALIAS_SELECTION_LIMIT + ++additionalContexts > state.precision.alias_selections ) { contextsWidened = true; state.valueResourceLimits.add("alias-selections"); diff --git a/src/domain/javascript/javascriptSemanticState.ts b/src/domain/javascript/javascriptSemanticState.ts index 408ccc0e21..35c8b98008 100644 --- a/src/domain/javascript/javascriptSemanticState.ts +++ b/src/domain/javascript/javascriptSemanticState.ts @@ -1,3 +1,4 @@ +import type { JavaScriptSemanticPrecision } from "./javascriptSemanticPrecision.js"; import type { SemanticPropertyReadSource } from "./javascriptSemanticIterationSources.js"; import * as t from "@babel/types"; import { @@ -149,6 +150,7 @@ export type SemanticPrototypeOwner = (typeof semanticPrototypeOwners)[number]; /** Shared state for semantic collection and evaluation. */ export interface JavaScriptSemanticAnalysisState { + readonly precision: JavaScriptSemanticPrecision; arrayIterationUnknown: boolean; /** Receiver isolation requires the completed member-effect inventory. */ memberMutationFactsComplete: boolean; diff --git a/src/domain/javascript/javascriptSemanticValues.memo.test.ts b/src/domain/javascript/javascriptSemanticValues.memo.test.ts index 5833da822f..7911c40953 100644 --- a/src/domain/javascript/javascriptSemanticValues.memo.test.ts +++ b/src/domain/javascript/javascriptSemanticValues.memo.test.ts @@ -5,7 +5,7 @@ import { indexJavaScriptBindingsSteps, } from "./javascriptSemanticAnalysis.js"; import { SemanticDependencyProofs } from "./javascriptSemanticDependencies.js"; -import { SEMANTIC_EXPRESSION_DEPTH_LIMIT } from "./javascriptSemanticResourceLimits.js"; +import { JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS } from "./javascriptSemanticPrecision.js"; import type { JavaScriptSemanticValue } from "./javascriptSemanticValueTypes.js"; import { clearSemanticValueCaches, @@ -62,7 +62,8 @@ it("reuses dependency-aware read snapshots across a reconvergent value chain", ( it("preserves the remaining depth budget when reusing a read snapshot", () => { const depth = 12; - const nesting = SEMANTIC_EXPRESSION_DEPTH_LIMIT - depth; + const nesting = + JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS.expression_depth - depth; const source = `let a0=1;${Array.from({ length: depth }, (_, index) => `let a${index + 1}=a${index}+a${index};`).join("")} const root={shallow:a${depth},deep:${"[".repeat(nesting)}a${depth}${"]".repeat(nesting)},after:a${depth}}; ${Array.from({ length: depth + 1 }, (_, index) => `a${index}=0;`).join("")}`; @@ -95,7 +96,7 @@ ${Array.from({ length: depth + 1 }, (_, index) => `a${index}=0;`).join("")}`; }); it("bounds reconvergent read work when the expression depth budget is exhausted", () => { - const depth = SEMANTIC_EXPRESSION_DEPTH_LIMIT; + const depth = JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS.expression_depth; const source = `let a0=1;${Array.from({ length: depth }, (_, index) => `let a${index + 1}=a${index}+a${index};`).join("")} const saved=a${depth};${Array.from({ length: depth + 1 }, (_, index) => `a${index}=0;`).join("")}`; const parsed = parseJavaScriptSource(source, "snapshot-limit.js"); diff --git a/src/domain/javascript/javascriptSemanticValues.ts b/src/domain/javascript/javascriptSemanticValues.ts index 8754f5be33..363c9cb3a6 100644 --- a/src/domain/javascript/javascriptSemanticValues.ts +++ b/src/domain/javascript/javascriptSemanticValues.ts @@ -1,3 +1,4 @@ +import type { JavaScriptSemanticPrecision } from "./javascriptSemanticPrecision.js"; import { semanticBindingMayHaveOpaqueEffect, semanticBindingCanBeRebound, @@ -61,8 +62,6 @@ import { semanticPrimitiveSet as primitiveSet, } from "./javascriptSemanticPrimitives.js"; import { - SEMANTIC_EXPRESSION_DEPTH_LIMIT, - SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT, exceedsSemanticPrimitiveAdditionByteBudget, exceedsSemanticTemplateByteBudget, semanticResourceLimitUnknown, @@ -650,18 +649,24 @@ const evaluateExpressionValue = ( node: t.Node, context: EvaluationContext, ): JavaScriptSemanticValue => { - if (context.expressionDepth > SEMANTIC_EXPRESSION_DEPTH_LIMIT) { + if (context.expressionDepth > context.state.precision.expression_depth) { context.dependence.lowest = EVERY_CALLER; - return semanticResourceLimitUnknown("expression-depth"); + return semanticResourceLimitUnknown( + "expression-depth", + context.state.precision, + ); } const unwrapped = unwrapJavaScriptExpression(node); if (unwrapped.depth > 0) { if ( context.expressionDepth + unwrapped.depth > - SEMANTIC_EXPRESSION_DEPTH_LIMIT + context.state.precision.expression_depth ) { context.dependence.lowest = EVERY_CALLER; - return semanticResourceLimitUnknown("expression-depth"); + return semanticResourceLimitUnknown( + "expression-depth", + context.state.precision, + ); } return evaluateExpression(unwrapped.node, { ...context, @@ -669,7 +674,8 @@ const evaluateExpressionValue = ( }); } const literal = readExactJavaScriptLiteral(node); - if (literal.found) return primitiveSet([literal.value]); + if (literal.found) + return primitiveSet([literal.value], context.state.precision); if ( t.isFunctionExpression(node) || t.isFunctionDeclaration(node) || @@ -722,6 +728,7 @@ const evaluateExpressionValue = ( branch.value ?? evaluateExpression(branch.node, nestedContext(context)), ), + context.state.precision, ); if (t.isObjectExpression(node)) return evaluateObject(node, context); if (t.isArrayExpression(node)) return evaluateArray(node, context); @@ -746,8 +753,17 @@ const evaluateTemplate = ( status: "unknown", reason: "Template text has no cooked representation.", }; - if (exceedsSemanticTemplateByteBudget(candidates, [text])) - return semanticResourceLimitUnknown("primitive-bytes"); + if ( + exceedsSemanticTemplateByteBudget( + candidates, + [text], + context.state.precision, + ) + ) + return semanticResourceLimitUnknown( + "primitive-bytes", + context.state.precision, + ); candidates = candidates.map((prefix) => `${prefix}${text}`); const expression = node.expressions[index]; if (expression === undefined) continue; @@ -760,16 +776,31 @@ const evaluateTemplate = ( reason: "Template expression is not a bounded primitive.", }; } - if (candidates.length > SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT / values.length) - return semanticResourceLimitUnknown("primitive-candidates"); + if ( + candidates.length > + context.state.precision.primitive_candidates / values.length + ) + return semanticResourceLimitUnknown( + "primitive-candidates", + context.state.precision, + ); const suffixes = values.map((value) => String(value)); - if (exceedsSemanticTemplateByteBudget(candidates, suffixes)) - return semanticResourceLimitUnknown("primitive-bytes"); + if ( + exceedsSemanticTemplateByteBudget( + candidates, + suffixes, + context.state.precision, + ) + ) + return semanticResourceLimitUnknown( + "primitive-bytes", + context.state.precision, + ); candidates = candidates.flatMap((prefix) => suffixes.map((suffix) => `${prefix}${suffix}`), ); } - return primitiveSet(candidates); + return primitiveSet(candidates, context.state.precision); }; /** @@ -1032,7 +1063,7 @@ const evaluateAddition = ( current = parent.node.right; break; } - value = addPrimitiveValues(parent.left, value); + value = addPrimitiveValues(parent.left, value, context.state.precision); pending.pop(); } } @@ -1041,6 +1072,7 @@ const evaluateAddition = ( const addPrimitiveValues = ( leftValue: JavaScriptSemanticValue, rightValue: JavaScriptSemanticValue, + precision: JavaScriptSemanticPrecision, ): JavaScriptSemanticValue => { const left = primitiveCandidates(leftValue); const right = primitiveCandidates(rightValue); @@ -1049,10 +1081,10 @@ const addPrimitiveValues = ( if (isSemanticResourceLimit(rightValue)) return rightValue; return { status: "unknown", reason: "Non-primitive addition." }; } - if (left.length > SEMANTIC_PRIMITIVE_CANDIDATE_LIMIT / right.length) - return semanticResourceLimitUnknown("primitive-candidates"); - if (exceedsSemanticPrimitiveAdditionByteBudget(left, right)) - return semanticResourceLimitUnknown("primitive-bytes"); + if (left.length > precision.primitive_candidates / right.length) + return semanticResourceLimitUnknown("primitive-candidates", precision); + if (exceedsSemanticPrimitiveAdditionByteBudget(left, right, precision)) + return semanticResourceLimitUnknown("primitive-bytes", precision); const values = left.flatMap((leftValue) => right.map((rightValue) => typeof leftValue === "string" || typeof rightValue === "string" @@ -1060,7 +1092,7 @@ const addPrimitiveValues = ( : Number(leftValue) + Number(rightValue), ), ); - return primitiveSet(values); + return primitiveSet(values, precision); }; const evaluateUnary = ( @@ -1074,11 +1106,20 @@ const evaluateUnary = ( return { status: "unknown", reason: "Non-primitive unary operand." }; } if (node.operator === "!") - return primitiveSet(argument.map((value) => !value)); + return primitiveSet( + argument.map((value) => !value), + context.state.precision, + ); if (node.operator === "+") - return primitiveSet(argument.map((value) => Number(value))); + return primitiveSet( + argument.map((value) => Number(value)), + context.state.precision, + ); if (node.operator === "-") - return primitiveSet(argument.map((value) => -Number(value))); + return primitiveSet( + argument.map((value) => -Number(value)), + context.state.precision, + ); return { status: "unknown", reason: `Unsupported unary ${node.operator}.` }; }; @@ -1248,25 +1289,28 @@ const provenanceForExpression = ( node: t.Node, context: EvaluationContext, ): JavaScriptBindingProvenance => { - if (context.expressionDepth > SEMANTIC_EXPRESSION_DEPTH_LIMIT) { + if (context.expressionDepth > context.state.precision.expression_depth) { context.dependence.lowest = EVERY_CALLER; context.state.valueResourceLimits.add("expression-depth"); return semanticUnresolvedProvenance( "unknown", - semanticResourceLimitReason("expression-depth"), + semanticResourceLimitReason("expression-depth", context.state.precision), ); } const unwrapped = unwrapJavaScriptExpression(node); if (unwrapped.depth > 0) { if ( context.expressionDepth + unwrapped.depth > - SEMANTIC_EXPRESSION_DEPTH_LIMIT + context.state.precision.expression_depth ) { context.dependence.lowest = EVERY_CALLER; context.state.valueResourceLimits.add("expression-depth"); return semanticUnresolvedProvenance( "unknown", - semanticResourceLimitReason("expression-depth"), + semanticResourceLimitReason( + "expression-depth", + context.state.precision, + ), ); } return provenanceForExpression(unwrapped.node, { @@ -1406,6 +1450,7 @@ const isLocallyCreatedValue = (node: t.Node): boolean => const mergeValues = ( values: readonly JavaScriptSemanticValue[], + precision: JavaScriptSemanticPrecision, ): JavaScriptSemanticValue => { const resourceLimit = values.find(isSemanticResourceLimit); if (resourceLimit !== undefined) return resourceLimit; @@ -1415,7 +1460,7 @@ const mergeValues = ( const merged: JavaScriptSemanticValue = values.every( (value) => value.status === "union" || value.status === "literal", ) - ? primitiveSet(primitives) + ? primitiveSet(primitives, precision) : { status: "ambiguous", reason: "Branches have incompatible values." }; if (values.length > 0 && values.every(semanticValueDefinitelyDefined)) definitelyDefinedValues.add(merged); diff --git a/src/domain/javascript/javascriptSourceAnalysis.ts b/src/domain/javascript/javascriptSourceAnalysis.ts index f7e02df35b..77eaa9c290 100644 --- a/src/domain/javascript/javascriptSourceAnalysis.ts +++ b/src/domain/javascript/javascriptSourceAnalysis.ts @@ -1,3 +1,4 @@ +import type { JavaScriptSemanticPrecisionInput } from "./javascriptSemanticPrecision.js"; import type { JavaScriptSemanticProjectionCapacity, JavaScriptSemanticFileProjection, @@ -38,6 +39,7 @@ export interface JavaScriptSourceAnalysisFailure { /** Operational choices for source analysis, independent of transport framing. */ export interface JavaScriptSourceAnalysisLimits { + readonly semanticPrecision?: JavaScriptSemanticPrecisionInput; readonly heapMb: number; readonly timeoutMs: number; } diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index b7f611af88..27082ded33 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -1,3 +1,7 @@ +import { + JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS, + type JavaScriptSemanticPrecisionInput, +} from "./javascriptSemanticPrecision.js"; import { createXhrOpenRequestQuery } from "./javascriptNetworkRequests.js"; import { uniqueSortedBy } from "../canonicalOrdering.js"; import * as t from "@babel/types"; @@ -86,28 +90,37 @@ import { /** Parse one JavaScript artifact and recover static structure only. */ export const analyzeJavaScriptStaticSource = ( source: string, + precision?: JavaScriptSemanticPrecisionInput, ): JavaScriptStaticAnalysis => { const file = parseJavaScriptSource(source); return file === null ? failedJavaScriptStaticAnalysis() - : analyzeParsedJavaScriptStaticSource(source, file); + : analyzeParsedJavaScriptStaticSource(source, file, precision); }; /** Recover static structure from an already parsed JavaScript artifact. */ export const analyzeParsedJavaScriptStaticSource = ( source: string, file: ParsedJavaScriptSource, + precision?: JavaScriptSemanticPrecisionInput, ): JavaScriptStaticAnalysis => - completeSemanticSteps(analyzeParsedJavaScriptStaticSourceSteps(source, file)); + completeSemanticSteps( + analyzeParsedJavaScriptStaticSourceSteps(source, file, precision), + ); /** Static analysis that yields during its source traversal; results are unchanged. */ export function* analyzeParsedJavaScriptStaticSourceSteps( source: string, file: ParsedJavaScriptSource, + precision?: JavaScriptSemanticPrecisionInput, ): Generator { const accumulator = createJavaScriptAnalysisAccumulator(source.length); // One definitions pass serves receiver classification and binding lookup. - const index = yield* indexJavaScriptBindingsWithReceiversSteps(file.program); + const index = yield* indexJavaScriptBindingsWithReceiversSteps( + file.program, + undefined, + precision, + ); const { bindings, receivers } = index; yield* index.indexAmbientWritesSteps(); if (hasPossibleCommonJsDelegateWrite(bindings)) @@ -254,18 +267,21 @@ const finalizeStaticAnalysis = ( examination: { status: resourceLimits.length > 0 || - accumulator.electronCallPathLimitReached === true + (accumulator.electronCallPathLimits?.size ?? 0) > 0 ? "limited" : "complete", semantic_resource_limits: resourceLimits, + ...(bindings.precision === JAVASCRIPT_SEMANTIC_PRECISION_DEFAULTS + ? {} + : { semantic_precision: bindings.precision }), electron_call_paths_limited: - accumulator.electronCallPathLimitReached === true, + (accumulator.electronCallPathLimits?.size ?? 0) > 0, }, applicability: { electron: electronObserved ? "observed" : "unknown" }, }; const limitations = [ ...(bindings.valueResourceLimits.has("alias-selections") - ? [semanticResourceLimitReason("alias-selections")] + ? [semanticResourceLimitReason("alias-selections", bindings.precision)] : []), ...(bindings.returnedReferenceWidenings.size > 0 ? [RETURNED_REFERENCE_WIDENING_LIMITATION] @@ -275,11 +291,12 @@ const finalizeStaticAnalysis = ( "Bundled Electron call paths use static argument and closure substitutions. Conditions, ambient helper overrides, runtime reachability and arbitrary side effects remain unproven.", ] : []), - ...(accumulator.electronCallPathLimitReached - ? [ - "Electron wrapper expansion reached its resource budget (1000000 visits/substitutions or depth 128); additional paths remain unknown.", - ] - : []), + ...[...(accumulator.electronCallPathLimits ?? [])] + .sort() + .map( + (control) => + `Electron wrapper expansion reached semantic_precision.${control}=${String(bindings.precision[control])}. Increase semantic_precision.${control} on the next analyze_javascript_application call (--semantic-precision JSON); no REA restart is required. Worker memory and transfer admission remain independent; cyclic or unsupported paths remain unknown.`, + ), ...(parserErrors === 0 ? [] : [ diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts index 1ef15fcbf7..04c179262b 100644 --- a/src/domain/javascript/javascriptStaticAnalysisState.ts +++ b/src/domain/javascript/javascriptStaticAnalysisState.ts @@ -59,7 +59,7 @@ export interface JavaScriptAnalysisAccumulator { )[]; readonly appPathChanges: LocatedJavaScriptFinding[]; electronCallPaths?: readonly ElectronCallPath[]; - electronCallPathLimitReached?: boolean; + electronCallPathLimits?: Set<"wrapper_visits" | "wrapper_depth">; readonly references: LocatedJavaScriptFinding[]; readonly endpoints: LocatedJavaScriptFinding[]; /** Target and receiver identities awaiting the shared mutation index. */ diff --git a/src/domain/javascript/javascriptStaticCoverage.ts b/src/domain/javascript/javascriptStaticCoverage.ts index b216b647e1..61e6c63310 100644 --- a/src/domain/javascript/javascriptStaticCoverage.ts +++ b/src/domain/javascript/javascriptStaticCoverage.ts @@ -1,3 +1,4 @@ +import { javaScriptSemanticPrecisionSchema } from "./javascriptSemanticPrecision.js"; import { z } from "zod"; import { javaScriptSemanticResourceLimitSchema } from "./javascriptSemanticValueTypes.js"; @@ -31,6 +32,7 @@ export const javaScriptStaticCoverageSchema = z.strictObject({ examination: z.strictObject({ status: z.enum(["complete", "limited", "not-examined"]), semantic_resource_limits: z.array(javaScriptSemanticResourceLimitSchema), + semantic_precision: javaScriptSemanticPrecisionSchema.optional(), electron_call_paths_limited: z.boolean(), }), applicability: z.strictObject({ electron: z.enum(["observed", "unknown"]) }), diff --git a/src/domain/javascript/javascriptSymbolicBindings.ts b/src/domain/javascript/javascriptSymbolicBindings.ts index 18dc9122c9..18a706a616 100644 --- a/src/domain/javascript/javascriptSymbolicBindings.ts +++ b/src/domain/javascript/javascriptSymbolicBindings.ts @@ -67,8 +67,6 @@ interface SymbolicReadBoundary { } type SymbolicNamespace = Extract; -const SYMBOLIC_DEPTH_LIMIT = 128; - const emptyEnvironment: SymbolicEnvironment = { values: new Map() }; const containsOwnedReturn = (root: t.Node): boolean => { @@ -120,9 +118,13 @@ export class JavaScriptSymbolicBindings { readonly tableAliases: ReadonlySet; readonly modules: Map; }> = []; - #remainingEvaluations = 1_000_000; + #remainingEvaluations: number; #depth = 0; - truncated = false; + readonly resourceLimits = new Set<"wrapper_visits" | "wrapper_depth">(); + + get truncated(): boolean { + return this.resourceLimits.size > 0; + } unresolved = false; constructor( @@ -132,6 +134,7 @@ export class JavaScriptSymbolicBindings { indexJavaScriptElectronIdentitySteps(program, state), ), ) { + this.#remainingEvaluations = state.precision.wrapper_visits; this.#electronIdentity = electronIdentity; const shapes = createSemanticExpressionShapeResolver(state); const defineProperty = createGlobalReferenceReader(state, [ @@ -218,9 +221,11 @@ export class JavaScriptSymbolicBindings { #admitEvaluation(): boolean { if ( --this.#remainingEvaluations < 0 || - this.#depth >= SYMBOLIC_DEPTH_LIMIT + this.#depth >= this.state.precision.wrapper_depth ) { - this.truncated = true; + this.resourceLimits.add( + this.#remainingEvaluations < 0 ? "wrapper_visits" : "wrapper_depth", + ); return false; } return true; @@ -269,7 +274,7 @@ export class JavaScriptSymbolicBindings { : 0); if (cost > this.#remainingEvaluations) { this.#remainingEvaluations = 0; - this.truncated = true; + this.resourceLimits.add("wrapper_visits"); return symbolicUnknown("symbolic condition resource limit"); } this.#remainingEvaluations -= cost; @@ -319,7 +324,7 @@ export class JavaScriptSymbolicBindings { first === undefined ? undefined : this.#completionGroups.get(first); if (group === undefined || group.members.size !== values.length) return []; if (values.length > this.#remainingEvaluations) { - this.truncated = true; + this.resourceLimits.add("wrapper_visits"); this.#remainingEvaluations = 0; return []; } @@ -981,7 +986,7 @@ export class JavaScriptSymbolicBindings { current = current.parent ) { if (--this.#remainingEvaluations < 0) { - this.truncated = true; + this.resourceLimits.add("wrapper_visits"); return false; } if (current.activation === activation) return true; @@ -1000,7 +1005,7 @@ export class JavaScriptSymbolicBindings { current = current.caller ) { if (--this.#remainingEvaluations < 0) { - this.truncated = true; + this.resourceLimits.add("wrapper_visits"); return undefined; } const point = current.point; @@ -1031,7 +1036,7 @@ export class JavaScriptSymbolicBindings { const boundary = read === undefined ? undefined : this.#namespaceReadBoundary(value, read); if (this.#remainingEvaluations <= 0) { - this.truncated = true; + this.resourceLimits.add("wrapper_visits"); return undefined; } const sourcePoint = boundary?.node[boundary.edge]; @@ -1049,7 +1054,7 @@ export class JavaScriptSymbolicBindings { if (commonThrough != null) { if (writes.length > this.#remainingEvaluations) { this.#remainingEvaluations = 0; - this.truncated = true; + this.resourceLimits.add("wrapper_visits"); return undefined; } this.#remainingEvaluations -= writes.length; @@ -1147,7 +1152,7 @@ export class JavaScriptSymbolicBindings { ) { if (variants.length > this.#remainingEvaluations) { this.#remainingEvaluations = 0; - this.truncated = true; + this.resourceLimits.add("wrapper_visits"); } else { this.#remainingEvaluations -= variants.length; const exits = variants as readonly SymbolicNamespace[]; @@ -1277,8 +1282,13 @@ export class JavaScriptSymbolicBindings { ...(unresolvedFallthrough ? { conditionUnresolved: true } : {}), }); for (const statement of statements) { - if (--this.#remainingEvaluations < 0 || depth >= 128) { - this.truncated = true; + if ( + --this.#remainingEvaluations < 0 || + depth >= this.state.precision.wrapper_depth + ) { + this.resourceLimits.add( + this.#remainingEvaluations < 0 ? "wrapper_visits" : "wrapper_depth", + ); return { values: [...possible, symbolicUnknown("symbolic resource limit")], fallsThrough: false, diff --git a/src/domain/managed/managedInspectionLimits.ts b/src/domain/managed/managedInspectionLimits.ts new file mode 100644 index 0000000000..37b16f0f4e --- /dev/null +++ b/src/domain/managed/managedInspectionLimits.ts @@ -0,0 +1,22 @@ +import { z } from "zod"; + +/** Defaults for one execution-free managed inspection, not PE format limits. */ +export const DEFAULT_MANAGED_SNAPSHOT_BYTES = 128 * 1024 * 1024; +export const DEFAULT_MANAGED_REPRESENTATION_BYTES = 64 * 1024 * 1024; + +export const managedInspectionBudgetShape = { + max_snapshot_bytes: z + .int() + .positive() + .optional() + .describe( + `Maximum eagerly retained PE snapshot bytes; omitted means ${String(DEFAULT_MANAGED_SNAPSHOT_BYTES)}. Raise for the next call to admit the same larger complete file; no restart. This does not raise metadata representation or host memory.`, + ), + max_metadata_representation_bytes: z + .int() + .positive() + .optional() + .describe( + `Maximum accounted metadata, CIL and projected JSON bytes; omitted means ${String(DEFAULT_MANAGED_REPRESENTATION_BYTES)}. Raise for the next call after resource-limit coverage, or select fewer type_names for member inspection. No restart; snapshot, transport and host memory limits remain independent.`, + ), +}; diff --git a/src/domain/native/binaryLayout.ts b/src/domain/native/binaryLayout.ts index d80c9ca4ca..a129168d05 100644 --- a/src/domain/native/binaryLayout.ts +++ b/src/domain/native/binaryLayout.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { canBorrowParsedJsonSnapshot } from "../immutableJson.js"; import { pwntoolsTimeoutField } from "./nativeDeadlines.js"; +import { pwntoolsProducerLimitsField } from "./pwntoolsResourceLimits.js"; import { absoluteLocalPathStringSchema } from "../localPath.js"; import { canonicalBase64Schema } from "../stringPatterns.js"; @@ -23,6 +24,7 @@ export const inspectBinaryLayoutInputSchema = z.strictObject({ "Absolute filesystem path to the selected binary", ), pwntools_timeout_ms: pwntoolsTimeoutField, + producer_limits: pwntoolsProducerLimitsField, }); /** File and linked-image evidence; numeric ELF values never pass through unsafe JSON numbers. */ diff --git a/src/domain/native/kernelModule.ts b/src/domain/native/kernelModule.ts index 01116721ed..9d2c40e752 100644 --- a/src/domain/native/kernelModule.ts +++ b/src/domain/native/kernelModule.ts @@ -17,16 +17,9 @@ export const MAX_INSPECT_KERNEL_MODULE_FILE_BYTES = 536870912; * file-backed __ksymtab* sections combined. */ export const MAX_INSPECT_KERNEL_MODULE_ENTRIES = 65536; -/** - * Retained .modinfo key length. Longer keys stay malformed per entry and are - * not copied into the result. - */ -export const MAX_KERNEL_MODULE_KEY_BYTES = 256; -/** - * Retained .modinfo value length. Longer values stay malformed per entry and - * are not copied into the result. - */ -export const MAX_KERNEL_MODULE_VALUE_BYTES = 4096; +/** Default per-field retention budgets; omission does not establish malformed syntax. */ +export const DEFAULT_KERNEL_MODULE_KEY_BYTES = 256; +export const DEFAULT_KERNEL_MODULE_VALUE_BYTES = 4096; /** * Retained __ksymtab* section-name length. Longer names stay listed with a * null name and are not copied into the result. @@ -58,6 +51,7 @@ export const KERNEL_MODULE_MODINFO_ENTRY_REASONS = [ "unterminated_modinfo_entry", "modinfo_missing_separator", "modinfo_empty_key", + // Accepted when decoding older Evidence; new observations separate omission from validity. "modinfo_field_exceeds_limit", ] as const; @@ -111,6 +105,13 @@ const modinfoEntrySchema = z.strictObject({ raw_hex: hex.nullable(), status: entryStatus, reason: z.enum(KERNEL_MODULE_MODINFO_ENTRY_REASONS).nullable(), + omitted_fields: z + .array(z.enum(["key", "value"])) + .max(2) + .default([]) + .describe( + "Fields omitted by retention policy, independently of structural status/reason", + ), }); const modinfoSchema = z.strictObject({ @@ -216,6 +217,22 @@ export const inspectKernelModuleInputSchema = z.strictObject({ .describe( "Maximum .modinfo and __versions entries plus file-backed __ksymtab* sections to examine; exceeding the budget fails without partial success", ), + max_modinfo_key_bytes: z + .int() + .min(1) + .max(MAX_INSPECT_KERNEL_MODULE_FILE_BYTES) + .default(DEFAULT_KERNEL_MODULE_KEY_BYTES) + .describe( + "Maximum bytes retained per .modinfo key (default 256). Larger keys stay valid when their syntax is valid, with key null and omitted_fields naming key. Raise on the next call to retain them; no restart. Does not raise input, entry or raw-hex budgets.", + ), + max_modinfo_value_bytes: z + .int() + .min(1) + .max(MAX_INSPECT_KERNEL_MODULE_FILE_BYTES) + .default(DEFAULT_KERNEL_MODULE_VALUE_BYTES) + .describe( + "Maximum bytes retained per .modinfo value (default 4096). Larger values stay valid when their syntax is valid, with value null and omitted_fields naming value. Raise on the next call to retain them; no restart. Does not raise input, entry or raw-hex budgets.", + ), }); /** diff --git a/src/domain/native/nativeCallCaptureLimits.ts b/src/domain/native/nativeCallCaptureLimits.ts new file mode 100644 index 0000000000..c642ba5ec7 --- /dev/null +++ b/src/domain/native/nativeCallCaptureLimits.ts @@ -0,0 +1,64 @@ +import { z } from "zod"; +import { constants as bufferConstants } from "node:buffer"; + +/** Defaults are shared with bridge configuration and partial-read admission. */ +export const NATIVE_CALL_CAPTURE_DEFAULTS = { + trace_bytes: 8 * 1024 * 1024, + trace_frames: 65_536, + breakpoint_location_bytes: 8 * 1024 * 1024, + other_stops: 256, + selector_bytes: 1024, + output_bytes: 1024 * 1024, +} as const; + +/** Captures are read as bounded Buffers and decoded to a single string. */ +export const MAX_NATIVE_CALL_CAPTURE_BYTES = Math.min( + bufferConstants.MAX_LENGTH, + bufferConstants.MAX_STRING_LENGTH, +); +const bytes = z.int().min(1).max(MAX_NATIVE_CALL_CAPTURE_BYTES); +const count = z.int().min(1); + +/** Retention choices apply before the owned observation starts. */ +export const nativeCallCaptureLimitsSchema = z.strictObject({ + trace_bytes: bytes.default(NATIVE_CALL_CAPTURE_DEFAULTS.trace_bytes), + trace_frames: count.default(NATIVE_CALL_CAPTURE_DEFAULTS.trace_frames), + breakpoint_location_bytes: bytes.default( + NATIVE_CALL_CAPTURE_DEFAULTS.breakpoint_location_bytes, + ), + other_stops: count.default(NATIVE_CALL_CAPTURE_DEFAULTS.other_stops), + selector_bytes: bytes.default(NATIVE_CALL_CAPTURE_DEFAULTS.selector_bytes), + output_bytes: bytes.default(NATIVE_CALL_CAPTURE_DEFAULTS.output_bytes), +}); + +/** Preserve caller omission separately from effective capture policy. */ +export const nativeCallCaptureLimitsInputSchema = z.strictObject({ + trace_bytes: nativeCallCaptureLimitsSchema.shape.trace_bytes + .removeDefault() + .optional(), + trace_frames: nativeCallCaptureLimitsSchema.shape.trace_frames + .removeDefault() + .optional(), + breakpoint_location_bytes: + nativeCallCaptureLimitsSchema.shape.breakpoint_location_bytes + .removeDefault() + .optional(), + other_stops: nativeCallCaptureLimitsSchema.shape.other_stops + .removeDefault() + .optional(), + selector_bytes: nativeCallCaptureLimitsSchema.shape.selector_bytes + .removeDefault() + .optional(), + output_bytes: nativeCallCaptureLimitsSchema.shape.output_bytes + .removeDefault() + .optional(), +}); + +export type NativeCallCaptureLimits = z.output< + typeof nativeCallCaptureLimitsSchema +>; + +export const nativeCallCaptureLimits = ( + selected?: z.input, +): NativeCallCaptureLimits => + nativeCallCaptureLimitsSchema.parse(selected ?? {}); diff --git a/src/domain/native/nativeCallObservation.ts b/src/domain/native/nativeCallObservation.ts index 00fee67331..e2aeebf9ed 100644 --- a/src/domain/native/nativeCallObservation.ts +++ b/src/domain/native/nativeCallObservation.ts @@ -1,4 +1,8 @@ import { z } from "zod"; +import { + nativeCallCaptureLimitsInputSchema, + nativeCallCaptureLimitsSchema, +} from "./nativeCallCaptureLimits.js"; import { jsonRecordSchema } from "../jsonRecord.js"; import { localPathStringSchema } from "../localPath.js"; @@ -76,6 +80,11 @@ const nativeCallEnvironmentSchema = jsonRecordSchema( * Duration and event bounds keep runtime observation finite. */ export const nativeCallObservationInputSchema = z.strictObject({ + capture_limits: nativeCallCaptureLimitsInputSchema + .optional() + .describe( + "Retention budgets for this observation. Omitted fields keep documented defaults. Select before launch; increasing them cannot recover an earlier observation. Limits do not extend duration_ms or max_events.", + ), breakpoints: z.array(nativeCallBreakpointSchema).min(1), arguments: z.array(nativeString).default([]), environment: nativeCallEnvironmentSchema @@ -165,6 +174,7 @@ export const nativeCapturedOutputSchema = z.strictObject({ /** Observed calls, the process lifecycle, and breakpoint resolution. */ export const nativeCallObservationResultSchema = z.strictObject({ + capture_limits: nativeCallCaptureLimitsSchema.optional(), target: z.strictObject({ path: z.string(), sha256: z.string().regex(/^[0-9a-f]{64}$/u), @@ -227,6 +237,7 @@ export const nativeCallObservationResultSchema = z.strictObject({ status: z.enum(["complete", "partial"]), event_limit_reached: z.boolean(), resource_limit_reached: z.boolean(), + capture_limit: z.enum(["trace_bytes", "trace_frames"]).optional(), /** Aggregate breakpoint-location metadata exhausted its retained-byte budget. */ breakpoint_locations_truncated: z.boolean().optional(), /** Requests that matched no code in any image loaded while observing. */ diff --git a/src/domain/native/nativeCallPartialObservation.ts b/src/domain/native/nativeCallPartialObservation.ts index 28a3e49fba..003ad31017 100644 --- a/src/domain/native/nativeCallPartialObservation.ts +++ b/src/domain/native/nativeCallPartialObservation.ts @@ -10,6 +10,7 @@ const completed = nativeCallObservationResultSchema.shape; /** Native facts retained on failure, including completed facts before finalization. */ export const nativeCallPartialObservationSchema = z.strictObject({ kind: z.literal("native-call-observation"), + capture_limits: completed.capture_limits, target: completed.target.omit({ launch_identity: true }).extend({ launch_identity: completed.target.shape.launch_identity.optional(), }), diff --git a/src/domain/native/nativeDeadlines.ts b/src/domain/native/nativeDeadlines.ts index 4bc09b271f..856b524b51 100644 --- a/src/domain/native/nativeDeadlines.ts +++ b/src/domain/native/nativeDeadlines.ts @@ -48,7 +48,7 @@ export const nativeUiDeadlineField = deadlineField( /** Per-call pwntools command deadline. Omitted means 30000 ms. */ export const pwntoolsTimeoutField = deadlineField( - "Deadline in milliseconds for each owned pwntools command in this call. Omitted means 30000. Integer from 1 to 2147483647. Applies on the next call. Does not raise cpuSeconds 30 or the output (67108864 bytes), diagnostic (1048576 bytes), or address-space (3221225472 bytes) allowances. Artifact admission stays REA_PWNTOOLS_MAX_INPUT_BYTES (default 33554432 bytes).", + "Deadline in milliseconds for each owned pwntools command in this call. Omitted means 30000, range 1..2147483647; applies on the next call. Independent CPU, output, diagnostic, file and address-space allowances are selected in producer_limits. input_bytes overrides REA_PWNTOOLS_MAX_INPUT_BYTES; otherwise that launch default applies.", ); export type NativeDeadlineRead = diff --git a/src/domain/native/nativeValueFlow.ts b/src/domain/native/nativeValueFlow.ts index 1e1b1c47c4..ff6cd88b56 100644 --- a/src/domain/native/nativeValueFlow.ts +++ b/src/domain/native/nativeValueFlow.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS } from "../analysisResourceLimits.js"; /** A decompiler p-code varnode location or literal value. */ export const nativeVarnodeSchema = z.strictObject({ @@ -45,10 +46,25 @@ export const nativePcodeEffectSchema = z.strictObject({ value_input: z.int().nonnegative().nullable(), }); +/** Accepted projection budgets; historical observations may omit these facts. */ +export const nativeValueFlowLimitsSchema = z.strictObject({ + max_operations: z.int().min(1).max(MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS), + max_inputs_per_operation: z + .int() + .min(1) + .max(MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS), + max_edges: z.int().min(1).max(MAXIMUM_GHIDRA_VALUE_FLOW_RECORDS), +}); +export type NativeValueFlowLimits = z.infer; + /** Bounded data-flow observations; STORE is not necessarily a persistent/global write. */ export const availableNativeValueFlowSchema = z.strictObject({ available: z.literal(true), provenance: z.literal("ghidra-high-pcode"), + limits: nativeValueFlowLimitsSchema.exactOptional(), + limits_reached: z + .array(z.enum(["max_operations", "max_inputs_per_operation", "max_edges"])) + .exactOptional(), operations: z.array(nativePcodeOperationSchema), def_use: z.array(nativePcodeDefUseSchema), effects: z diff --git a/src/domain/native/pwntoolsResourceLimits.ts b/src/domain/native/pwntoolsResourceLimits.ts new file mode 100644 index 0000000000..79d1038b69 --- /dev/null +++ b/src/domain/native/pwntoolsResourceLimits.ts @@ -0,0 +1,43 @@ +import { + BOUNDED_PRODUCER_LIMITS, + producerResourceLimitsSchema, + resolveProducerLimits, +} from "../producerResourceLimits.js"; +import type { z } from "zod"; +import { DEFAULT_PWNTOOLS_TIMEOUT_MS } from "./nativeDeadlines.js"; + +/** Owned Python defaults; the bridge consumes these accepted values from its request. */ +export const PWNTOOLS_LIMITS = { + input_bytes: BOUNDED_PRODUCER_LIMITS.input_bytes, + output_bytes: 64 * 1024 * 1024, + diagnostic_bytes: 1024 * 1024, + command_timeout_ms: DEFAULT_PWNTOOLS_TIMEOUT_MS, + address_space_bytes: 3 * 1024 * 1024 * 1024, + cpu_seconds: 30, +} as const; + +export const pwntoolsProducerLimitsSchema = producerResourceLimitsSchema.omit({ + command_timeout_ms: true, +}); +export type PwntoolsProducerLimitsInput = z.output< + typeof pwntoolsProducerLimitsSchema +>; + +export const resolvePwntoolsLimits = ( + selected?: PwntoolsProducerLimitsInput, + inputBytes: number = PWNTOOLS_LIMITS.input_bytes, +) => + resolveProducerLimits( + { + ...PWNTOOLS_LIMITS, + input_bytes: inputBytes, + file_bytes: selected?.output_bytes ?? PWNTOOLS_LIMITS.output_bytes, + }, + selected, + ); +export type PwntoolsProducerLimits = ReturnType; +export const pwntoolsProducerLimitsField = pwntoolsProducerLimitsSchema + .optional() + .describe( + `Per-call owned Python input, output, diagnostic, address-space and file bytes, and CPU seconds. Defaults: input ${PWNTOOLS_LIMITS.input_bytes}, output ${PWNTOOLS_LIMITS.output_bytes}, diagnostics ${PWNTOOLS_LIMITS.diagnostic_bytes}, address space ${PWNTOOLS_LIMITS.address_space_bytes} bytes, CPU ${PWNTOOLS_LIMITS.cpu_seconds} seconds. File allowance follows selected output_bytes unless file_bytes is explicit. input_bytes overrides REA_PWNTOOLS_MAX_INPUT_BYTES. Positive safe integers; applies on the next call without restart. Tighter inherited OS limits are retained. pwntools_timeout_ms remains the independent wall deadline.`, + ); diff --git a/src/domain/native/recordedCrash.ts b/src/domain/native/recordedCrash.ts index f28a00a3b9..f1fa165e80 100644 --- a/src/domain/native/recordedCrash.ts +++ b/src/domain/native/recordedCrash.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { pwntoolsProducerLimitsField } from "./pwntoolsResourceLimits.js"; import type { BinaryArchitecture } from "../binaryTargetTypes.js"; import { pwntoolsTimeoutField } from "./nativeDeadlines.js"; @@ -160,6 +161,7 @@ export const inspectRecordedCrashInputSchema = z.strictObject({ "Add core-only mapping candidates through caller-supplied GDB/pwndbg", ), pwntools_timeout_ms: pwntoolsTimeoutField, + producer_limits: pwntoolsProducerLimitsField, }); /** Debugger map candidates preserve reported flags; zero does not establish absent permissions. */ diff --git a/src/domain/producerResourceLimits.ts b/src/domain/producerResourceLimits.ts new file mode 100644 index 0000000000..fc44bbb33c --- /dev/null +++ b/src/domain/producerResourceLimits.ts @@ -0,0 +1,59 @@ +import { z } from "zod"; +import { MAX_NODE_TIMER_DELAY_MS } from "./analysisResourceLimits.js"; + +/** Per-call owned producer budgets. Each adapter advertises only consumed fields. */ +export const producerResourceLimitsSchema = z.strictObject({ + input_bytes: z.int().positive().optional(), + output_bytes: z.int().positive().optional(), + diagnostic_bytes: z.int().positive().optional(), + command_timeout_ms: z.int().min(1).max(MAX_NODE_TIMER_DELAY_MS).optional(), + address_space_bytes: z.int().positive().optional(), + file_bytes: z.int().positive().optional(), + cpu_seconds: z.int().positive().optional(), +}); + +/** Bound the resident input copy and retained text before allocation/collection. */ +export const BOUNDED_PRODUCER_LIMITS = { + input_bytes: 32 * 1024 * 1024, + output_bytes: 4 * 1024 * 1024, + command_timeout_ms: 30_000, + address_space_bytes: 1024 * 1024 * 1024, + file_bytes: 64 * 1024 * 1024, + cpu_seconds: 30, +} as const; + +export type ProducerResourceLimits = z.output< + typeof producerResourceLimitsSchema +>; + +/** Explicit per-call fields override defaults; tighter inherited OS limits still win. */ +export const resolveProducerLimits = < + Defaults extends Partial>, +>( + defaults: Defaults, + selected: ProducerResourceLimits | undefined, +): { readonly [Key in keyof Defaults]: number } => + Object.fromEntries( + Object.entries(defaults).map(([key, value]) => [ + key, + selected?.[key as keyof ProducerResourceLimits] ?? value, + ]), + ) as { readonly [Key in keyof Defaults]: number }; + +/** Combined-output producers have no separate diagnostic capture control. */ +export const boundedProducerLimitsSchema = producerResourceLimitsSchema + .omit({ diagnostic_bytes: true }) + .optional() + .describe( + "Per-call producer limits in bytes, CPU seconds and wall-clock milliseconds. Omitted fields preserve defaults (32 MiB input, 4 MiB combined output, 30000 ms command, 1 GiB address space, 64 MiB per file, 30 CPU seconds); changes apply on the next call without restart. Tighter inherited OS limits remain enforced. These controls do not raise parser, protocol or transport limits.", + ); + +/** Accepted collection policy plus effective native ceilings; inherited limits may be zero. */ +export const boundedProducerLimitsResultSchema = producerResourceLimitsSchema + .omit({ diagnostic_bytes: true }) + .required() + .extend({ + address_space_bytes: z.int().nonnegative(), + file_bytes: z.int().nonnegative(), + cpu_seconds: z.int().nonnegative(), + }); diff --git a/src/domain/reverseEngineering.ts b/src/domain/reverseEngineering.ts index 55926af60b..db0420d4a3 100644 --- a/src/domain/reverseEngineering.ts +++ b/src/domain/reverseEngineering.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { constants as bufferConstants } from "node:buffer"; import { commandEvidenceDetailSchema } from "./commandObservation.js"; import { localPathStringSchema } from "./localPath.js"; import { RAW_BINARY_ENTRY_POINT } from "./rawBinary.js"; @@ -8,6 +9,22 @@ export const DEFAULT_REVERSE_COMMAND_TIMEOUT_MS = 120_000; /** Node timer ceiling. Larger deadlines would be clamped instead of honored. */ export const MAX_REVERSE_COMMAND_TIMEOUT_MS = 2_147_483_647; +/** Combined stdout/stderr admission, before UTF-8 decoding and Evidence expansion. */ +export const DEFAULT_REVERSE_COMMAND_OUTPUT_BYTES = 16 * 1024 * 1024; +/** Both raw and normalized output are JSON-escaped in one Evidence string. */ +export const MAX_REVERSE_COMMAND_OUTPUT_BYTES = Math.floor( + (bufferConstants.MAX_STRING_LENGTH - 1024 * 1024) / (2 * 6), +); +export const reverseCommandOutputBytesSchema = z + .number() + .int() + .min(1) + .max(MAX_REVERSE_COMMAND_OUTPUT_BYTES) + .exactOptional() + .describe( + `Combined stdout/stderr byte budget before decoding (default ${String(DEFAULT_REVERSE_COMMAND_OUTPUT_BYTES)}, range 1..${String(MAX_REVERSE_COMMAND_OUTPUT_BYTES)}). Applies on the next call without restarting REA. The maximum reserves JSON-escaping space for both Evidence mirrors under Node's single-string ceiling. Reaching it stops collection and requests provider termination; completion remains unknown. Confirm effects and cleanup before repeating a command. Does not raise the MCP client's receive buffer.`, + ); + /** Per-call deadline for one-shot objdump and rizin commands. */ export const reverseCommandTimeoutSchema = z .number() @@ -16,11 +33,12 @@ export const reverseCommandTimeoutSchema = z .max(MAX_REVERSE_COMMAND_TIMEOUT_MS) .exactOptional() .describe( - "Wall-clock budget in milliseconds. Target preflight shares one budget across stat and digest, and the one-shot command receives that budget again (default 120000 each). Omitted means 120000. Applies on the next call. Does not raise the 16 MiB combined output cap, Ghidra or Hopper startup deadlines, or debugger session deadlines.", + "Wall-clock budget in milliseconds. Target preflight shares one budget across stat and digest, and the one-shot command receives that budget again (default 120000 each). Omitted means 120000. Applies on the next call. Output admission is separately controlled by max_output_bytes; does not raise Ghidra or Hopper startup deadlines or debugger session deadlines.", ); -const commandTimeoutMsField = { +const commandExecutionFields = { command_timeout_ms: reverseCommandTimeoutSchema, + max_output_bytes: reverseCommandOutputBytesSchema, }; export const OBJDUMP_ARTIFACT_OPERATIONS = [ @@ -38,7 +56,7 @@ const objdumpArtifactInputSchema = z path: localPathStringSchema, operation: z.enum(OBJDUMP_ARTIFACT_OPERATIONS).exclude(["disassemble"]), follow_debug_links: z.boolean().default(false), - ...commandTimeoutMsField, + ...commandExecutionFields, }) .strict(); @@ -61,7 +79,7 @@ const objdumpDisassemblyInputSchema = objdumpArtifactInputSchema.extend({ const objdumpArchitecturesInputSchema = z.strictObject({ operation: z.literal("architectures"), - ...commandTimeoutMsField, + ...commandExecutionFields, }); /** Caller-selected native selection and independent Evidence delivery. */ @@ -122,7 +140,7 @@ export const rizinInputSchema = z .describe( "Address of file offset 0. REA passes -n and -m, so Rizin maps the file there as a raw image, reads no format headers, and keeps its default decoder unless asm_arch and asm_bits select one.", ), - ...commandTimeoutMsField, + ...commandExecutionFields, }) .strict(); diff --git a/src/domain/rust/rustBinary.ts b/src/domain/rust/rustBinary.ts index 89f2e1db44..3b6d6e84da 100644 --- a/src/domain/rust/rustBinary.ts +++ b/src/domain/rust/rustBinary.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { rustProducerLimitsField } from "./rustResourceLimits.js"; import { digestSchema } from "../digests.js"; import { absoluteLocalPathStringSchema, @@ -14,6 +15,8 @@ import { /** Known revisions used to interpret compatible cargo-auditable documents. */ export const RUST_AUDITABLE_FORMAT_VERSIONS = [0, 1, 8] as const; +/** Default compiler-path retention; callers can select a larger bounded prefix. */ +export const DEFAULT_RUSTC_HASH_LIMIT = 64; const index = z.int().nonnegative(); const rustcHash = z.string().regex(/^[0-9a-f]{40}$/u); @@ -55,6 +58,13 @@ export const rustAuditableMetadataSchema = z.strictObject({ /** Inferred compiler id from read-only `/rustc/<40-hex>/` paths. */ export const rustcCommitSchema = z.strictObject({ evidence_kind: z.literal("inferred"), + coverage: z + .strictObject({ + status: z.literal("partial"), + reason: z.enum(["max_rustc_hashes", "report_bytes"]), + effective_limit: z.int().positive(), + }) + .exactOptional(), hashes: z .array( z.strictObject({ @@ -71,6 +81,14 @@ export const inspectRustBinaryInputSchema = z.strictObject({ "Absolute filesystem path to the selected executable or library", ), architecture: machoSliceArchitectureSchema.exactOptional(), + producer_limits: rustProducerLimitsField, + max_rustc_hashes: z + .int() + .positive() + .exactOptional() + .describe( + `Maximum distinct inferred compiler paths to retain (default ${String(DEFAULT_RUSTC_HASH_LIMIT)}). Omitted paths leave compiler coverage partial without discarding dependency facts. Raise this value on the next call to recover a larger prefix; complete-report byte admission still applies.`, + ), }); /** Bounded cargo-auditable observation. A missing section is null, not an empty dependency list. */ diff --git a/src/domain/rust/rustResourceLimits.ts b/src/domain/rust/rustResourceLimits.ts new file mode 100644 index 0000000000..6136f6da8e --- /dev/null +++ b/src/domain/rust/rustResourceLimits.ts @@ -0,0 +1,24 @@ +import type { z } from "zod"; +import { + producerResourceLimitsSchema, + resolveProducerLimits, +} from "../producerResourceLimits.js"; + +/** Defaults for complete offline Rust input and report admission. */ +export const RUST_PRODUCER_DEFAULTS = { + input_bytes: 256 * 1024 * 1024, + output_bytes: 8 * 1024 * 1024, +} as const; +export const rustProducerLimitsSchema = producerResourceLimitsSchema.pick({ + input_bytes: true, + output_bytes: true, +}); +export const rustProducerLimitsField = rustProducerLimitsSchema + .optional() + .describe( + "Per-call input_bytes and output_bytes admission for the same complete Rust artifact/report, default 268435456 and 8388608 bytes. Positive safe integers; applies on the next call without restart. Compressed dependency section 1 MiB, inflated JSON 8 MiB/64x, parser nesting and transport limits remain independent.", + ); +export type RustProducerLimitsInput = z.output; +export const resolveRustProducerLimits = (selected?: RustProducerLimitsInput) => + resolveProducerLimits(RUST_PRODUCER_DEFAULTS, selected); +export type RustProducerLimits = ReturnType; diff --git a/src/domain/signals/signalDecoders.ts b/src/domain/signals/signalDecoders.ts index 4d86b36ea6..76ca9432c8 100644 --- a/src/domain/signals/signalDecoders.ts +++ b/src/domain/signals/signalDecoders.ts @@ -1,7 +1,12 @@ import { z } from "zod"; import { signalRecordingSchema } from "./signalRecording.js"; +import { boundedProducerLimitsSchema } from "../producerResourceLimits.js"; export const inspectSignalDecodersInputSchema = z.strictObject({ + producer_limits: boundedProducerLimitsSchema + .unwrap() + .omit({ input_bytes: true }) + .optional(), decoder: z .string() .regex(/^[a-zA-Z0-9_+\x2d]+$/u) diff --git a/src/domain/signals/signalRecording.ts b/src/domain/signals/signalRecording.ts index ebce5bd09a..314cda1d90 100644 --- a/src/domain/signals/signalRecording.ts +++ b/src/domain/signals/signalRecording.ts @@ -4,6 +4,10 @@ import { localPathStringSchema, } from "../localPath.js"; import { windowAnnotationText } from "./annotationTextWindow.js"; +import { + boundedProducerLimitsSchema, + boundedProducerLimitsResultSchema, +} from "../producerResourceLimits.js"; // These strings are single sigrok arguments, never shell fragments. const selection = z @@ -12,6 +16,7 @@ const selection = z .max(8192) .refine((value) => !value.includes("\0"), "NUL is not an argument character"); const signalInputShape = z.strictObject({ + producer_limits: boundedProducerLimitsSchema, path: localPathStringSchema.describe("Absolute local recording path"), decoders: selection.describe( "sigrok decoder stack, options and channel assignments, e.g. uart:rx=RX:baudrate=9600", @@ -123,14 +128,7 @@ export const signalRecordingSchema = z sha256: z.string().regex(/^[a-f0-9]{64}$/), arguments: z.array(z.string()), }), - limits: z.strictObject({ - input_bytes: z.number().int().positive(), - output_bytes: z.number().int().positive(), - command_timeout_ms: z.number().int().positive(), - address_space_bytes: z.number().int().nonnegative(), - file_bytes: z.number().int().nonnegative(), - cpu_seconds: z.number().int().nonnegative(), - }), + limits: boundedProducerLimitsResultSchema, hardware_acquisition: z.literal("not-performed"), }) .superRefine((report, context) => { diff --git a/src/domain/sourceMapLimits.ts b/src/domain/sourceMapLimits.ts new file mode 100644 index 0000000000..3b171837be --- /dev/null +++ b/src/domain/sourceMapLimits.ts @@ -0,0 +1,130 @@ +import { z } from "zod"; + +/** Local map admission calibrated against public Actual Budget maps; other policies remain conservative. */ +export const WEB_SOURCE_MAP_LIMITS = { + mapBytes: 8 * 1024 * 1024, + outputBytes: 32 * 1024 * 1024, + decodedRows: 262144, + decodedSegments: 262144, + decodedRecords: 262144, + sectionDepth: 64, + decodeTimeoutMs: 20000, + codecHeapMiB: 192, + fetchBytes: 64 * 1024 * 1024, +}; + +// A complete encoded request/reply must remain below Node's single-string range. +export const MAX_SOURCE_MAP_ENCODED_BYTES = 536870887; +const MAX_RECORD_COUNT = 0x7fffffff; +const MAX_CODEC_HEAP_MIB = 16384; +const bytes = z.int().min(1).max(MAX_SOURCE_MAP_ENCODED_BYTES); +const count = z.int().min(1).max(MAX_RECORD_COUNT); +export const sourceMapLimitsSchema = z.strictObject({ + map_bytes: bytes + .exactOptional() + .describe( + `Local source-map byte admission; default ${String(WEB_SOURCE_MAP_LIMITS.mapBytes)}. Applies before reading and again in the codec.`, + ), + output_bytes: bytes + .exactOptional() + .describe( + `Complete encoded request/reply or fetched-map representation budget; default ${String(WEB_SOURCE_MAP_LIMITS.outputBytes)}.`, + ), + decoded_rows: count + .exactOptional() + .describe( + `Generated-row admission before codec allocation; default ${String(WEB_SOURCE_MAP_LIMITS.decodedRows)}.`, + ), + decoded_segments: count + .exactOptional() + .describe( + `Mapping-segment admission before codec allocation; default ${String(WEB_SOURCE_MAP_LIMITS.decodedSegments)}.`, + ), + decoded_records: count + .exactOptional() + .describe( + `Shared structural record admission, including declarations, rows, segments and sections; default ${String(WEB_SOURCE_MAP_LIMITS.decodedRecords)}.`, + ), + section_depth: count + .exactOptional() + .describe( + `Indexed-section nesting admission; default ${String(WEB_SOURCE_MAP_LIMITS.sectionDepth)}. Each retained section also consumes the shared structural-record budget.`, + ), + codec_heap_mib: z + .int() + .min(128) + .max(MAX_CODEC_HEAP_MIB) + .exactOptional() + .describe( + `Local codec old-generation heap request in MiB; default ${String(WEB_SOURCE_MAP_LIMITS.codecHeapMiB)}. Independent of byte/record budgets and client receive limits.`, + ), + fetch_bytes: bytes + .exactOptional() + .describe( + `Aggregate fetched response bytes and, independently, retained resolved-identity bytes; each defaults to ${String(WEB_SOURCE_MAP_LIMITS.fetchBytes)}.`, + ), +}); +export type SourceMapLimitSelection = z.output; +export type SourceMapLimits = typeof WEB_SOURCE_MAP_LIMITS; +export const selectedSourceMapLimits = ( + selected?: SourceMapLimitSelection, +): SourceMapLimits => ({ + ...WEB_SOURCE_MAP_LIMITS, + mapBytes: selected?.map_bytes ?? WEB_SOURCE_MAP_LIMITS.mapBytes, + outputBytes: selected?.output_bytes ?? WEB_SOURCE_MAP_LIMITS.outputBytes, + decodedRows: selected?.decoded_rows ?? WEB_SOURCE_MAP_LIMITS.decodedRows, + decodedSegments: + selected?.decoded_segments ?? WEB_SOURCE_MAP_LIMITS.decodedSegments, + decodedRecords: + selected?.decoded_records ?? WEB_SOURCE_MAP_LIMITS.decodedRecords, + sectionDepth: selected?.section_depth ?? WEB_SOURCE_MAP_LIMITS.sectionDepth, + codecHeapMiB: selected?.codec_heap_mib ?? WEB_SOURCE_MAP_LIMITS.codecHeapMiB, + fetchBytes: selected?.fetch_bytes ?? WEB_SOURCE_MAP_LIMITS.fetchBytes, +}); +export const sourceMapLimitRecovery = ( + field: keyof SourceMapLimitSelection, + value: number, +): string => { + const maximum = + field === "codec_heap_mib" + ? MAX_CODEC_HEAP_MIB + : field.endsWith("bytes") + ? MAX_SOURCE_MAP_ENCODED_BYTES + : MAX_RECORD_COUNT; + const minimum = field === "codec_heap_mib" ? 128 : 1; + const unit = + field === "codec_heap_mib" + ? "MiB" + : field.endsWith("bytes") + ? "bytes" + : field === "section_depth" + ? "levels" + : "records"; + const action = + value < maximum + ? `Increase that per-call field (range ${String(minimum)}..${String(maximum)}) and retry the same complete map after cleanup` + : "This is the maximum supported value; use a smaller complete map or an independent source-map reader for this workload"; + return `source_map_limits.${field} is ${String(value)} ${unit}. ${action}; no REA restart is required. Other byte, record, heap and client limits remain independent.`; +}; + +/** Carry the effective policy to the codec before it reads the private request. */ +export const sourceMapLimitSelection = ( + limits: SourceMapLimits, +): SourceMapLimitSelection => ({ + map_bytes: limits.mapBytes, + output_bytes: limits.outputBytes, + decoded_rows: limits.decodedRows, + decoded_segments: limits.decodedSegments, + decoded_records: limits.decodedRecords, + section_depth: limits.sectionDepth, + codec_heap_mib: limits.codecHeapMiB, + fetch_bytes: limits.fetchBytes, +}); + +export const localSourceMapLimitsSchema = sourceMapLimitsSchema.omit({ + fetch_bytes: true, +}); +export const fetchedSourceMapLimitsSchema = sourceMapLimitsSchema.omit({ + map_bytes: true, + codec_heap_mib: true, +}); diff --git a/src/domain/sqlite/sqliteDatabase.ts b/src/domain/sqlite/sqliteDatabase.ts index 4d3fa929e5..abeca92461 100644 --- a/src/domain/sqlite/sqliteDatabase.ts +++ b/src/domain/sqlite/sqliteDatabase.ts @@ -2,6 +2,9 @@ import { z } from "zod"; import { absoluteLocalPathStringSchema } from "../localPath.js"; import type { JsonValue } from "../jsonValue.js"; +/** Fixed SQLite value/record and complete-worker-reply allowance. */ +export const SQLITE_DATABASE_OUTPUT_BYTES = 16 * 1024 * 1024; + /** SQLite exposes this table as a row source but omits its own catalog entry. */ export const SQLITE_SCHEMA_TABLE_NAME = "sqlite_schema"; export const sqliteSchemaHasObjectDeclaration = (name: string): boolean => diff --git a/src/domain/wasm/wasmArtifact.ts b/src/domain/wasm/wasmArtifact.ts index d53278eebf..b1bcc39a7e 100644 --- a/src/domain/wasm/wasmArtifact.ts +++ b/src/domain/wasm/wasmArtifact.ts @@ -1,6 +1,10 @@ -import { wasmInterfaceSchema } from "./wasmInterface.js"; +import { + wasmInterfaceSchema, + wasmInterfaceByteBudgetSchema, +} from "./wasmInterface.js"; import { z } from "zod"; import { createHash } from "node:crypto"; +import { constants as bufferConstants } from "node:buffer"; import type { EvidenceSubjectTarget } from "../evidence.js"; import { absoluteLocalPathStringSchema } from "../localPath.js"; @@ -23,10 +27,42 @@ const artifact = z.strictObject({ /** Default execution budget; upper bound is the runtime's signed 32-bit timer range. */ export const DEFAULT_WASM_COMMAND_TIMEOUT_MS = 30_000; export const MAX_WASM_COMMAND_TIMEOUT_MS = 2_147_483_647; -/** Selected artifact lists share the same resource bound across path transports. */ +export const DEFAULT_WASM_ARTIFACT_PATHS = 16; +export const MAX_WASM_ARTIFACT_PATHS = 0xffff_ffff; +/** Defaults preserve the original WABT admission and capture envelope. */ +export const WASM_RESOURCE_DEFAULTS = { + inputBytes: 8 * 1024 * 1024, + toolBytes: 64 * 1024 * 1024, + outputBytes: 32 * 1024 * 1024, +} as const; +/** Inputs may be decoded as glue text; output is retained as single strings. */ +export const WASM_RESOURCE_MAXIMUMS = { + inputBytes: Math.min( + bufferConstants.MAX_LENGTH, + bufferConstants.MAX_STRING_LENGTH, + ), + toolBytes: bufferConstants.MAX_LENGTH, + outputBytes: bufferConstants.MAX_STRING_LENGTH, +} as const; +export interface WasmResourceLimits { + readonly inputBytes: number; + readonly toolBytes: number; + readonly outputBytes: number; +} +const byteBudget = (name: string, defaultValue: number, maximum: number) => + z + .number() + .int() + .min(1) + .max(maximum) + .exactOptional() + .describe( + `${name} in bytes (default ${String(defaultValue)}; maximum ${String(maximum)}, the runtime representation limit). Applies to the next inspection; no restart. Does not raise the other byte budgets, command deadline, client receive budget or available host memory.`, + ); +/** Caller selections are admitted against max_artifact_paths by the shared workflow. */ export const wasmArtifactPathsSchema = ( pathSchema: z.ZodString = absoluteLocalPathStringSchema, -) => z.array(pathSchema).max(16); +) => z.array(pathSchema); /** All artifacts are explicitly selected local files; references never trigger acquisition. */ export const inspectWasmArtifactInputSchema = z.strictObject({ path: absoluteLocalPathStringSchema.describe( @@ -34,6 +70,31 @@ export const inspectWasmArtifactInputSchema = z.strictObject({ ), glue_paths: wasmArtifactPathsSchema().default([]), candidate_paths: wasmArtifactPathsSchema().default([]), + max_artifact_paths: z + .number() + .int() + .min(1) + .max(MAX_WASM_ARTIFACT_PATHS) + .exactOptional() + .describe( + `Maximum paths in each auxiliary glue/candidate selection (default ${String(DEFAULT_WASM_ARTIFACT_PATHS)}, range 1..${String(MAX_WASM_ARTIFACT_PATHS)}, the runtime Array-length ceiling). Admission occurs before reading files or starting WABT. Applies to the next inspection without restart; byte budgets remain independent.`, + ), + max_interface_bytes: wasmInterfaceByteBudgetSchema, + max_input_bytes: byteBudget( + "Maximum bytes per selected module, candidate or glue file", + WASM_RESOURCE_DEFAULTS.inputBytes, + WASM_RESOURCE_MAXIMUMS.inputBytes, + ), + max_tool_bytes: byteBudget( + "Maximum bytes per caller-supplied WABT executable", + WASM_RESOURCE_DEFAULTS.toolBytes, + WASM_RESOURCE_MAXIMUMS.toolBytes, + ), + max_output_bytes: byteBudget( + "Combined retained stdout/stderr per WABT command", + WASM_RESOURCE_DEFAULTS.outputBytes, + WASM_RESOURCE_MAXIMUMS.outputBytes, + ), command_timeout_ms: z .number() .int() @@ -41,9 +102,20 @@ export const inspectWasmArtifactInputSchema = z.strictObject({ .max(MAX_WASM_COMMAND_TIMEOUT_MS) .exactOptional() .describe( - `Deadline in milliseconds for each WABT version/analysis command (default ${String(DEFAULT_WASM_COMMAND_TIMEOUT_MS)}), not the whole inspection. Allow additional client time for all commands, file checks and cleanup. Input/output safety limits remain fixed.`, + `Deadline in milliseconds for each WABT version/analysis command (default ${String(DEFAULT_WASM_COMMAND_TIMEOUT_MS)}), not the whole inspection. Allow additional client time for all commands, file checks and cleanup. Does not raise max_input_bytes, max_tool_bytes or max_output_bytes.`, ), }); +/** Resolve once at the accepting owner; omission stays omitted in Evidence. */ +export const wasmResourceLimits = ( + input: Pick< + z.output, + "max_input_bytes" | "max_tool_bytes" | "max_output_bytes" + >, +): WasmResourceLimits => ({ + inputBytes: input.max_input_bytes ?? WASM_RESOURCE_DEFAULTS.inputBytes, + toolBytes: input.max_tool_bytes ?? WASM_RESOURCE_DEFAULTS.toolBytes, + outputBytes: input.max_output_bytes ?? WASM_RESOURCE_DEFAULTS.outputBytes, +}); /** Upstream output stays intact; import/export names are never split on ambiguous punctuation. */ export const wasmArtifactSchema = z .strictObject({ diff --git a/src/domain/wasm/wasmInterface.ts b/src/domain/wasm/wasmInterface.ts index 6f4145e18b..050d94a849 100644 --- a/src/domain/wasm/wasmInterface.ts +++ b/src/domain/wasm/wasmInterface.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { constants as bufferConstants } from "node:buffer"; const kind = z.enum(["func", "table", "memory", "global", "tag", "unknown"]); const valueType = z.enum([ @@ -87,7 +88,18 @@ export type WasmInterfaceSignature = NonNullable< export type WasmInterfaceValueType = z.infer; /** Bound eager typed output and repeated signature expansion independently of WAT bytes. */ -export const WASM_INTERFACE_REPRESENTATION_BYTES = 32 * 1024 * 1024; +export const DEFAULT_WASM_INTERFACE_REPRESENTATION_BYTES = 32 * 1024 * 1024; +export const MAX_WASM_INTERFACE_REPRESENTATION_BYTES = + bufferConstants.MAX_STRING_LENGTH; +export const wasmInterfaceByteBudgetSchema = z + .number() + .int() + .min(1) + .max(MAX_WASM_INTERFACE_REPRESENTATION_BYTES) + .exactOptional() + .describe( + `Typed interface representation admission in bytes (default ${String(DEFAULT_WASM_INTERFACE_REPRESENTATION_BYTES)}, range 1..${String(MAX_WASM_INTERFACE_REPRESENTATION_BYTES)}; maximum follows the runtime string ceiling). Applies on the next inspection or comparison without restart. Raising this does not restore facts omitted from previously produced partial Evidence; inspect again with a larger producer budget first.`, + ); /** Conservative JSON/retention charge; shared arrays still expand at every occurrence. */ export const wasmInterfaceEntryRepresentationBytes = ( diff --git a/src/domain/wasm/wasmInterfaceBudget.ts b/src/domain/wasm/wasmInterfaceBudget.ts new file mode 100644 index 0000000000..9f89e34db0 --- /dev/null +++ b/src/domain/wasm/wasmInterfaceBudget.ts @@ -0,0 +1,41 @@ +import { AnalysisResourceConstraintError } from "../analysisErrorCore.js"; +import { + DEFAULT_WASM_INTERFACE_REPRESENTATION_BYTES, + MAX_WASM_INTERFACE_REPRESENTATION_BYTES, +} from "./wasmInterface.js"; + +/** One selected interface allowance, shared by nested producer reads and comparison. */ +export class WasmInterfaceBudget { + #remaining: number; + readonly maximumBytes: number; + readonly source: "default" | "request"; + constructor( + maximumBytes?: number, + readonly operation = "inspect_wasm_artifact", + ) { + this.maximumBytes = + maximumBytes ?? DEFAULT_WASM_INTERFACE_REPRESENTATION_BYTES; + this.source = maximumBytes === undefined ? "default" : "request"; + this.#remaining = this.maximumBytes; + } + reserve(bytes: number): void { + if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.#remaining) + throw new AnalysisResourceConstraintError( + this.operation, + "capacity", + `Typed WASM interface representation exceeds max_interface_bytes (${String(this.maximumBytes)} bytes)`, + { + maximum_bytes: this.maximumBytes, + default_bytes: DEFAULT_WASM_INTERFACE_REPRESENTATION_BYTES, + maximum_supported_bytes: MAX_WASM_INTERFACE_REPRESENTATION_BYTES, + source: this.source, + unit: "bytes", + }, + { + retryable: false, + remediationAction: `${this.maximumBytes < MAX_WASM_INTERFACE_REPRESENTATION_BYTES ? `Raise max_interface_bytes above ${String(this.maximumBytes)} bytes (maximum ${String(MAX_WASM_INTERFACE_REPRESENTATION_BYTES)}) on the next ${this.operation} call.` : `max_interface_bytes is already at the runtime representation maximum of ${String(MAX_WASM_INTERFACE_REPRESENTATION_BYTES)} bytes; no higher interface allowance is supported. A comparison may select only imports or exports; otherwise report the unsupported complete representation.`} No restart is required. Other output, heap and deadline controls do not raise this allowance. A comparison cannot restore facts omitted from partial producer Evidence; inspect the original module with a larger producer budget first when that budget is below its maximum.`, + }, + ); + this.#remaining -= bytes; + } +} diff --git a/src/domain/webBundleAnalysis.ts b/src/domain/webBundleAnalysis.ts index 1b370857ad..07d83330b5 100644 --- a/src/domain/webBundleAnalysis.ts +++ b/src/domain/webBundleAnalysis.ts @@ -1,3 +1,5 @@ +import { javaScriptSemanticPrecisionSchema } from "./javascript/javascriptSemanticPrecision.js"; +import { fetchedSourceMapLimitsSchema } from "./sourceMapLimits.js"; import { z } from "zod"; import { browserExclusionSchema } from "./browserCompleteness.js"; import { webPageInspectionSchema } from "./browserObservationSchemas.js"; @@ -41,6 +43,16 @@ const webBundleTotalTimeoutDescription = `Total analysis deadline phase: maximum export const analyzeWebBundleInputSchema = inspectWebPageWithSourceInputSchema .safeExtend({ fetch_source_maps: z.boolean().default(false), + semantic_precision: javaScriptSemanticPrecisionSchema + .exactOptional() + .describe( + "Per-script semantic evaluator budgets; omitted fields preserve defaults. Independent of worker memory, deadlines and reply budgets.", + ), + source_map_limits: fetchedSourceMapLimitsSchema + .exactOptional() + .describe( + "Source-map fetch/representation and decoder admission for this call. Omitted fields keep existing defaults; no restart required.", + ), max_heap_mb: z .number() .int() @@ -55,10 +67,17 @@ export const analyzeWebBundleInputSchema = inspectWebPageWithSourceInputSchema webBundleTotalTimeoutDescription, ), source_map_fetch_timeout_ms: optionalBrowserDeadlineMs( - `Source-map fetch deadline in milliseconds (default ${String(DEFAULT_SOURCE_MAP_FETCH_TIMEOUT_MS)}). Omitted means ${String(DEFAULT_SOURCE_MAP_FETCH_TIMEOUT_MS)} and applies on the next fetch. A fetch overrun is an item limitation (fetch_failed), not a thrown analysis timeout. Does not raise the fixed 64 MiB response budget or the source-map decode deadline. The response budget has no runtime override.`, + `Source-map fetch deadline in milliseconds (default ${String(DEFAULT_SOURCE_MAP_FETCH_TIMEOUT_MS)}). Omitted means ${String(DEFAULT_SOURCE_MAP_FETCH_TIMEOUT_MS)} and applies on the next fetch. A fetch overrun is an item limitation (fetch_failed), not a thrown analysis timeout. Response/representation and decoder budgets are independent source_map_limits fields.`, ), }) .superRefine((input, context) => { + if (input.source_map_limits !== undefined && !input.fetch_source_maps) + context.addIssue({ + code: "custom", + path: ["source_map_limits"], + message: + "source_map_limits requires fetch_source_maps: true; inspection does not fetch maps unless explicitly selected.", + }); const perScript = input.analysis_timeout_ms ?? DEFAULT_WEB_BUNDLE_SCRIPT_TIMEOUT_MS; const total = diff --git a/src/domain/webBundleAnalyzer.ts b/src/domain/webBundleAnalyzer.ts index 8e2b8b06ce..07c8b4931e 100644 --- a/src/domain/webBundleAnalyzer.ts +++ b/src/domain/webBundleAnalyzer.ts @@ -1,3 +1,4 @@ +import type { JavaScriptSemanticPrecisionInput } from "./javascript/javascriptSemanticPrecision.js"; import { analyzeScript } from "./webBundleAnalyzerInspection.js"; import { createWebBundleAnalysisAccumulator, @@ -11,9 +12,10 @@ import { /** Analyze one selected captured script without executing its source. */ export const analyzeCapturedWebScript = ( script: IncludedScript, + precision?: JavaScriptSemanticPrecisionInput, ): WebBundleScriptAnalysis => { const accumulator = createWebBundleAnalysisAccumulator(); - const routeFacts = analyzeScript(script, accumulator); + const routeFacts = analyzeScript(script, accumulator, precision); const failed = accumulator.failedScripts.length > 0; const recovered = accumulator.recoveredScripts.length > 0; return webBundleScriptAnalysisSchema.parse({ diff --git a/src/domain/webBundleAnalyzerInspection.ts b/src/domain/webBundleAnalyzerInspection.ts index 68ea3cb9af..af373a5cbc 100644 --- a/src/domain/webBundleAnalyzerInspection.ts +++ b/src/domain/webBundleAnalyzerInspection.ts @@ -1,3 +1,4 @@ +import type { JavaScriptSemanticPrecisionInput } from "./javascript/javascriptSemanticPrecision.js"; import { isBrowserNetworkRequest, createXhrOpenRequestQuery, @@ -54,6 +55,7 @@ type Inference = WebBundleAnalysis["inferences"][number]; export const analyzeScript = ( script: IncludedScript, accumulator: AnalysisAccumulator, + precision?: JavaScriptSemanticPrecisionInput, ): JavaScriptRouteSourceFacts | null => { const file = parseJavaScriptSource(script.source.artifact.text); if (file === null) { @@ -65,7 +67,11 @@ export const analyzeScript = ( accumulator.recoveredScripts.push(script.script_key); detectVendorFingerprints(script, accumulator); const index = completeSemanticSteps( - indexJavaScriptBindingsWithReceiversSteps(file.program), + indexJavaScriptBindingsWithReceiversSteps( + file.program, + undefined, + precision, + ), ); completeSemanticSteps(index.indexAmbientWritesSteps()); const { receivers } = index; diff --git a/src/domain/webDeadlineControls.test.ts b/src/domain/webDeadlineControls.test.ts index 54c0632df7..38465a1460 100644 --- a/src/domain/webDeadlineControls.test.ts +++ b/src/domain/webDeadlineControls.test.ts @@ -99,13 +99,21 @@ describe("web deadline controls", () => { ).toBe(true); }); - it("names each deadline and the fixed budgets it does not raise", () => { + it("names each deadline and the independent budgets it does not raise", () => { expect(webBundleScriptTimeoutRecovery(30_000).userMessage).toContain( "analysis_timeout_ms", ); - expect( - webNetworkCaptureTimeoutRecovery(30_000).remediationAction, - ).toContain("no runtime override"); + const captureRecovery = webNetworkCaptureTimeoutRecovery(30_000); + for (const control of [ + "max_input_bytes", + "max_reply_bytes", + "max_nesting_depth", + ]) { + expect(captureRecovery.remediationAction).toContain(control); + } + expect(captureRecovery.remediationAction).toContain( + "capture_timeout_ms does not raise them", + ); expect(webSourceMapDecodeTimeoutRecovery(20_000).userMessage).toContain( "decode_timeout_ms", ); diff --git a/src/domain/webNetworkCapture.ts b/src/domain/webNetworkCapture.ts index dd90bd3bf3..a2239d6a51 100644 --- a/src/domain/webNetworkCapture.ts +++ b/src/domain/webNetworkCapture.ts @@ -2,11 +2,10 @@ import { z } from "zod"; import { browserDeadlineIncrease, browserDeadlineRetry, - noRuntimeOverrideSentence, optionalBrowserDeadlineMs, selectedBrowserDeadlineMs, } from "./browserDeadline.js"; -import { jsonValueSchema } from "./jsonValue.js"; +import { jsonValueSchema, MAX_JSON_DEPTH } from "./jsonValue.js"; import { absoluteLocalPathStringSchema } from "./localPath.js"; import { protobufDecodingSchema, @@ -15,7 +14,7 @@ import { import { qlogCaptureFactsSchema } from "./qlogCapture.js"; import { canBorrowParsedJsonSnapshot } from "./immutableJson.js"; -/** Complete historical evidence budgets; exceeding a budget returns no partial capture. */ +/** Default historical evidence budgets; exceeding a budget returns no partial capture. */ export const WEB_NETWORK_CAPTURE_LIMITS = { inputBytes: 32 * 1024 * 1024, outputBytes: 96 * 1024 * 1024, @@ -23,6 +22,80 @@ export const WEB_NETWORK_CAPTURE_LIMITS = { depth: 64, } as const; +// Native/SAZ projections start at the reported record root. The complete +// reply (value.records[0].reported) and Evidence envelope have four enclosing +// edges; count them before admitting parser/projection nesting. +const CAPTURE_REPORTED_RECORD_DEPTH = [ + "normalized_result", + "records", + 0, + "reported", +].length; +export const MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH = + MAX_JSON_DEPTH - CAPTURE_REPORTED_RECORD_DEPTH; + +/** One admitted capture's complete-input, complete-reply and parser policy. */ +export const webNetworkCaptureBudgetsSchema = z.strictObject({ + inputBytes: z.int().min(1), + outputBytes: z.int().min(1), + depth: z.int().min(1).max(MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH), +}); +export type WebNetworkCaptureBudgets = z.infer< + typeof webNetworkCaptureBudgetsSchema +>; + +export const selectedWebNetworkCaptureBudgets = (input: { + readonly max_input_bytes?: number | undefined; + readonly max_reply_bytes?: number | undefined; + readonly max_nesting_depth?: number | undefined; +}): WebNetworkCaptureBudgets => ({ + inputBytes: input.max_input_bytes ?? WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + outputBytes: input.max_reply_bytes ?? WEB_NETWORK_CAPTURE_LIMITS.outputBytes, + depth: input.max_nesting_depth ?? WEB_NETWORK_CAPTURE_LIMITS.depth, +}); + +/** Explicit per-call budget facts; these controls do not mutate runtime defaults. */ +export const webNetworkCaptureBudgetFacts = ( + input: Parameters[0], + parameter: "max_input_bytes" | "max_reply_bytes" | "max_nesting_depth", +) => { + const budgets = selectedWebNetworkCaptureBudgets(input); + const field = + parameter === "max_input_bytes" + ? "inputBytes" + : parameter === "max_reply_bytes" + ? "outputBytes" + : "depth"; + const effective = budgets[field]; + return { + parameter, + effective_value: effective, + default_value: WEB_NETWORK_CAPTURE_LIMITS[field], + source: input[parameter] === undefined ? "default" : "request", + unit: parameter === "max_nesting_depth" ? "levels" : "bytes", + minimum: 1, + maximum: + parameter === "max_nesting_depth" + ? MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH + : Number.MAX_SAFE_INTEGER, + applies_at: "next_operation", + rea_restart_required: false, + }; +}; + +/** Recovery follows the owning budget, rather than a different deadline or heap. */ +export const webNetworkCaptureBudgetRecovery = ( + input: Parameters[0], + parameter: Parameters[1], +): string => { + const facts = webNetworkCaptureBudgetFacts(input, parameter); + const next = + facts.effective_value < facts.maximum + ? `Increase ${parameter} above ${String(facts.effective_value)} on the next inspect_web_network_capture call` + : `${parameter} is at its maximum; select a smaller complete producer export when that preserves the required observation`; + return `${next}. Effective ${parameter}: ${String(facts.effective_value)} ${facts.unit} (source ${facts.source}, default ${String(facts.default_value)}, floor ${String(facts.minimum)}, maximum ${String(facts.maximum)}). No REA restart is required. Other capture budgets, worker limits and client limits remain separate. Selection and delivery do not recreate a completed report from a refused decode.`; +}; + /** Capture deadline used when capture_timeout_ms is omitted. */ export const DEFAULT_WEB_NETWORK_CAPTURE_TIMEOUT_MS = WEB_NETWORK_CAPTURE_LIMITS.timeoutMs; @@ -32,12 +105,12 @@ export const selectedWebNetworkCaptureTimeoutMs = ( ): number => selectedBrowserDeadlineMs(selected, DEFAULT_WEB_NETWORK_CAPTURE_TIMEOUT_MS); -/** Caller-visible capture deadline. Byte and depth caps stay fixed. */ +/** Caller-visible capture deadline; byte/depth budgets have separate per-call controls. */ export const webNetworkCaptureTimeoutRecovery = ( timeoutMs: number, ): { readonly userMessage: string; readonly remediationAction: string } => ({ userMessage: `inspect_web_network_capture reached its ${String(timeoutMs)} ms capture deadline (deadlineExpired) for capture_timeout_ms. No partial capture is returned. This is not the MCP client deadline.`, - remediationAction: `${browserDeadlineIncrease("capture_timeout_ms", timeoutMs, DEFAULT_WEB_NETWORK_CAPTURE_TIMEOUT_MS, "--capture-timeout-ms")} ${browserDeadlineRetry} ${noRuntimeOverrideSentence(`The ${String(WEB_NETWORK_CAPTURE_LIMITS.inputBytes)}-byte input budget, ${String(WEB_NETWORK_CAPTURE_LIMITS.outputBytes)}-byte reply budget, and ${String(WEB_NETWORK_CAPTURE_LIMITS.depth)}-level nesting cap stay fixed.`, "capture_timeout_ms")}`, + remediationAction: `${browserDeadlineIncrease("capture_timeout_ms", timeoutMs, DEFAULT_WEB_NETWORK_CAPTURE_TIMEOUT_MS, "--capture-timeout-ms")} ${browserDeadlineRetry} The separate max_input_bytes, max_reply_bytes and max_nesting_depth controls apply to the next call without restarting REA; capture_timeout_ms does not raise them.`, }); export const WEB_NETWORK_CAPTURE_FORMATS = [ @@ -60,6 +133,21 @@ export const inspectWebNetworkCaptureInputSchema = z.strictObject({ .describe( "Zero-based producer record ordinals, in caller-selected order. Omit to inspect every record.", ), + max_input_bytes: webNetworkCaptureBudgetsSchema.shape.inputBytes + .optional() + .describe( + "Complete capture, expanded SAZ member and protobuf descriptor admission in bytes (default 33554432); applies to the next call without restart. Does not raise worker heap, reply or client limits.", + ), + max_reply_bytes: webNetworkCaptureBudgetsSchema.shape.outputBytes + .optional() + .describe( + "Complete decoder reply and protobuf working representation allowance in bytes (default 100663296); applies to the next call without restart. Record selection occurs after complete decoding. Does not raise worker heap or client receive limits.", + ), + max_nesting_depth: webNetworkCaptureBudgetsSchema.shape.depth + .optional() + .describe( + `Capture/parser nesting allowance (default 64, maximum ${String(MAX_WEB_NETWORK_CAPTURE_NESTING_DEPTH)}, reserving the enclosing reply/Evidence levels within REA JSON representation); applies on the next call without restart. Does not make malformed or unsupported structures valid.`, + ), protobuf: protobufOptionsSchema.optional(), sensitive_values: z .array(z.string().min(1)) @@ -68,7 +156,7 @@ export const inspectWebNetworkCaptureInputSchema = z.strictObject({ "Literal text excluded from capture payloads, byte/numeric projections, producer coordinates, artifact paths/digests and diagnostic text. Declaration strings are never persisted in Evidence parameters. Canonical contract keys, enums and provider identities remain intact.", ), capture_timeout_ms: optionalBrowserDeadlineMs( - `Capture wall-clock deadline in milliseconds (default ${String(DEFAULT_WEB_NETWORK_CAPTURE_TIMEOUT_MS)}), from decoder start through process exit. Omitted means ${String(DEFAULT_WEB_NETWORK_CAPTURE_TIMEOUT_MS)} and applies on the next capture. deadlineExpired returns no partial capture. Does not raise the fixed ${String(WEB_NETWORK_CAPTURE_LIMITS.inputBytes)}-byte input budget, ${String(WEB_NETWORK_CAPTURE_LIMITS.outputBytes)}-byte reply budget, or ${String(WEB_NETWORK_CAPTURE_LIMITS.depth)}-level nesting cap. Those have no runtime override.`, + `Capture wall-clock deadline in milliseconds (default ${String(DEFAULT_WEB_NETWORK_CAPTURE_TIMEOUT_MS)}), from decoder start through process exit. Omitted means ${String(DEFAULT_WEB_NETWORK_CAPTURE_TIMEOUT_MS)} and applies on the next capture. deadlineExpired returns no partial capture. Does not raise the separately selected max_input_bytes, max_reply_bytes or max_nesting_depth budgets.`, ), }); diff --git a/src/domain/webSourceLocation.ts b/src/domain/webSourceLocation.ts index 049ce28247..f22e9b64b2 100644 --- a/src/domain/webSourceLocation.ts +++ b/src/domain/webSourceLocation.ts @@ -1,8 +1,12 @@ +import { + localSourceMapLimitsSchema, + WEB_SOURCE_MAP_LIMITS, +} from "./sourceMapLimits.js"; +export { WEB_SOURCE_MAP_LIMITS } from "./sourceMapLimits.js"; import { z } from "zod"; import { browserDeadlineIncrease, browserDeadlineRetry, - noRuntimeOverrideSentence, optionalBrowserDeadlineMs, } from "./browserDeadline.js"; import { digestSchema } from "./digests.js"; @@ -35,8 +39,13 @@ export const webSourceLocationInputSchema = selectedWebScriptInputSchema.extend( ), }), generated_position: webSourcePositionSchema, + source_map_limits: localSourceMapLimitsSchema + .exactOptional() + .describe( + "Per-call source-map admission, decoder and representation budgets. Omitted fields preserve existing defaults; changes apply on the next call without restart.", + ), decode_timeout_ms: optionalBrowserDeadlineMs( - "Source-map decode deadline in milliseconds (default 20000), for the owned codec process. Omitted means 20000 and applies on the next trace_web_source_location call. deadlineExpired returns no partial mapping. Does not raise the fixed 4 MiB map budget, 32 MiB reply budget, or 192 MiB codec heap. Those have no runtime override.", + "Source-map decode deadline in milliseconds (default 20000), for the owned codec process. Omitted means 20000 and applies on the next trace_web_source_location call. deadlineExpired returns no partial mapping. Byte, record and heap budgets are independent; select source_map_limits to change them.", ), }, ); @@ -118,19 +127,6 @@ export const webSourceMapReportSchema = z.strictObject({ diagnostics: z.array(z.string()), }); -/** Bounds for complete source-map evidence and upstream decoded layouts. */ -export const WEB_SOURCE_MAP_LIMITS = { - mapBytes: 4 * 1024 * 1024, - outputBytes: 32 * 1024 * 1024, - decodedRows: 262144, - decodedSegments: 262144, - decodedRecords: 262144, - sectionDepth: 64, - decodeTimeoutMs: 20000, - /** V8 old-generation heap for the codec, and its Bun process budget. */ - codecHeapMiB: 192, -} as const; - /** Decode deadline used when decode_timeout_ms is omitted. */ export const DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS = WEB_SOURCE_MAP_LIMITS.decodeTimeoutMs; @@ -140,7 +136,7 @@ export const webSourceMapDecodeTimeoutRecovery = ( timeoutMs: number, ): { readonly userMessage: string; readonly remediationAction: string } => ({ userMessage: `trace_web_source_location reached its ${String(timeoutMs)} ms source-map decode deadline (deadlineExpired) for decode_timeout_ms. No partial mapping is returned. This is not the MCP client deadline.`, - remediationAction: `${browserDeadlineIncrease("decode_timeout_ms", timeoutMs, DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS, "--decode-timeout-ms")} ${browserDeadlineRetry} ${noRuntimeOverrideSentence(`The ${String(WEB_SOURCE_MAP_LIMITS.mapBytes)}-byte map budget, ${String(WEB_SOURCE_MAP_LIMITS.outputBytes)}-byte reply budget, and ${String(WEB_SOURCE_MAP_LIMITS.codecHeapMiB)} MiB codec heap stay fixed.`, "decode_timeout_ms")}`, + remediationAction: `${browserDeadlineIncrease("decode_timeout_ms", timeoutMs, DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS, "--decode-timeout-ms")} ${browserDeadlineRetry} Byte, record and heap admission are separate per-call source_map_limits fields; changing the deadline does not raise them.`, }); export type WebSourceMapReport = z.output; diff --git a/src/dotnet/ManagedArtifactInspector.ts b/src/dotnet/ManagedArtifactInspector.ts index d72f083bdd..7314fdd8ae 100644 --- a/src/dotnet/ManagedArtifactInspector.ts +++ b/src/dotnet/ManagedArtifactInspector.ts @@ -371,6 +371,7 @@ const partialMetadataResult = ({ export const inspectManagedArtifactBytes = ( bytes: Buffer, target: BinaryTarget, + maxRepresentationBytes?: number, ): ManagedArtifactInspection => { const layout = readManagedPeLayout(bytes); if (layout.cli === null) return unavailableResult(target, bytes, layout); @@ -386,6 +387,7 @@ export const inspectManagedArtifactBytes = ( bytes, metadataOffset, cli.metadata.size, + maxRepresentationBytes, ); } catch (cause: unknown) { if (!(cause instanceof ManagedReaderFailure)) throw cause; diff --git a/src/dotnet/ManagedDecodeBudget.ts b/src/dotnet/ManagedDecodeBudget.ts index cc1911a485..fbd8e05414 100644 --- a/src/dotnet/ManagedDecodeBudget.ts +++ b/src/dotnet/ManagedDecodeBudget.ts @@ -4,17 +4,15 @@ import { jsonStringByteLength } from "../domain/jsonStringByteLength.js"; import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js"; import { ManagedReaderFailure } from "./ManagedReaderFailure.js"; - -/** Per-inspection capacity for eagerly retained metadata and its JSON projection. */ -const MAX_METADATA_REPRESENTATION_BYTES = 64 * 1024 * 1024; +import { DEFAULT_MANAGED_REPRESENTATION_BYTES } from "../domain/managed/managedInspectionLimits.js"; /** The capacity owner's diagnostic, also used to admit a recovery envelope. */ export class ManagedRepresentationFailure extends ManagedReaderFailure { - constructor(offset: number | null) { + constructor(offset: number | null, maximumBytes: number) { super({ code: "resource-limit", scope: "metadata.representation", - detail: `Managed metadata exceeds the ${String(MAX_METADATA_REPRESENTATION_BYTES)} byte representation budget`, + detail: `Managed metadata exceeds max_metadata_representation_bytes=${String(maximumBytes)} bytes. Increase max_metadata_representation_bytes on the next inspection (CLI --max-metadata-representation-bytes), or select fewer type_names for member inspection. No REA restart is required; snapshot, transport and host memory limits are independent.`, offset, }); } @@ -27,13 +25,21 @@ interface CachedString { /** The layout owns decoding capacity and heap identity for all of its consumers. */ export class ManagedDecodeBudget { - #remaining = MAX_METADATA_REPRESENTATION_BYTES; + #remaining: number; #transient = 0; readonly #rows = new Map>(); readonly #strings = new Map(); readonly #digests = new WeakMap>(); readonly #prepaid = new WeakSet(); + constructor(readonly maximumBytes = DEFAULT_MANAGED_REPRESENTATION_BYTES) { + if (!Number.isSafeInteger(maximumBytes) || maximumBytes < 1) + throw new RangeError( + "max_metadata_representation_bytes must be a positive safe integer", + ); + this.#remaining = maximumBytes; + } + reserve(bytes: number, offset: number | null): void { this.#admit(bytes, offset); this.#remaining -= bytes; @@ -68,7 +74,7 @@ export class ManagedDecodeBudget { bytes < 0 || bytes > this.#remaining - this.#transient ) - throw new ManagedRepresentationFailure(offset); + throw new ManagedRepresentationFailure(offset, this.maximumBytes); } /** UTF-8 cannot decode to more UTF-16 units than bytes; JSON uses at most six bytes per unit. */ diff --git a/src/dotnet/ManagedMemberInspector.ts b/src/dotnet/ManagedMemberInspector.ts index 49926d1bcd..56850f8cf1 100644 --- a/src/dotnet/ManagedMemberInspector.ts +++ b/src/dotnet/ManagedMemberInspector.ts @@ -111,7 +111,11 @@ const unavailable = ( }), ); -const readMemberLayout = (bytes: Buffer, pe: ManagedPeLayout) => { +const readMemberLayout = ( + bytes: Buffer, + pe: ManagedPeLayout, + maxRepresentationBytes?: number, +) => { const cli = pe.cli; if (cli === null) throw new TypeError("Managed inventory requires CLI metadata"); @@ -124,6 +128,7 @@ const readMemberLayout = (bytes: Buffer, pe: ManagedPeLayout) => { bytes, rootOffset, cli.metadata.size, + maxRepresentationBytes, ); const memberLists = readTypeMemberListFacts(bytes, layout); return { layout, memberLists }; @@ -172,12 +177,14 @@ export const inspectManagedMembersBytes = ( methodOffset = 0, methodLimit = Number.MAX_SAFE_INTEGER, userStrings: readUserStrings = true, + maxRepresentationBytes, }: { readonly typeNames?: readonly string[]; /** Zero-based offset into MethodDef rows. */ readonly methodOffset?: number; readonly methodLimit?: number; readonly userStrings?: boolean; + readonly maxRepresentationBytes?: number; } = {}, ): ManagedMemberInspection => { if ( @@ -195,7 +202,11 @@ export const inspectManagedMembersBytes = ( const issues: ManagedParseIssue[] = []; let recover: ReturnType | undefined; try { - const { layout, memberLists } = readMemberLayout(bytes, pe); + const { layout, memberLists } = readMemberLayout( + bytes, + pe, + maxRepresentationBytes, + ); const methodTotal = layout.table(6)?.rowCount ?? 0; const methodRows = { offset: Math.min(methodOffset, methodTotal), diff --git a/src/dotnet/ManagedMemberRecovery.ts b/src/dotnet/ManagedMemberRecovery.ts index 155d007040..580b334c6c 100644 --- a/src/dotnet/ManagedMemberRecovery.ts +++ b/src/dotnet/ManagedMemberRecovery.ts @@ -34,6 +34,7 @@ export const prepareManagedMemberRecovery = ( ): ((offset: number | null) => ManagedMemberInspection) => { const capacityIssue = new ManagedRepresentationFailure( Number.MAX_SAFE_INTEGER, + budget.maximumBytes, ).issue; const report: ManagedMemberInspection = { ...inventory, diff --git a/src/dotnet/ManagedMetadataLayout.ts b/src/dotnet/ManagedMetadataLayout.ts index d114e6cd77..34917dac15 100644 --- a/src/dotnet/ManagedMetadataLayout.ts +++ b/src/dotnet/ManagedMetadataLayout.ts @@ -449,6 +449,7 @@ export const readManagedMetadataLayout = ( artifact: Buffer, rootOffset: number, size: number, + maxRepresentationBytes?: number, ): ManagedMetadataLayout => { if (size < 20 || rootOffset < 0 || rootOffset > artifact.length - size) throw managedFailure( @@ -458,7 +459,7 @@ export const readManagedMetadataLayout = ( rootOffset, ); const bytes = artifact.subarray(rootOffset, rootOffset + size); - const budget = new ManagedDecodeBudget(); + const budget = new ManagedDecodeBudget(maxRepresentationBytes); const { streams, version, minimalDeltaOffset } = readMetadataStreams( bytes, rootOffset, diff --git a/src/dotnet/ManagedNativeBoundaryInspector.ts b/src/dotnet/ManagedNativeBoundaryInspector.ts index 90b59c78c9..f6ff62e0e7 100644 --- a/src/dotnet/ManagedNativeBoundaryInspector.ts +++ b/src/dotnet/ManagedNativeBoundaryInspector.ts @@ -105,6 +105,7 @@ const readBoundaryInventory = ( pe: ManagedPeLayout, cli: NonNullable, issues: ManagedParseIssue[], + maxRepresentationBytes?: number, ): { readonly layout: ManagedMetadataLayout; readonly inventory: Inventory; @@ -118,6 +119,7 @@ const readBoundaryInventory = ( bytes, metadataOffset, cli.metadata.size, + maxRepresentationBytes, ); const memberLists = readTypeMemberListFacts(bytes, layout); const inventory = readManagedMetadataInventory( @@ -133,6 +135,7 @@ const readBoundaryInventory = ( export const inspectManagedNativeBoundariesBytes = ( bytes: Buffer, target: BinaryTarget, + maxRepresentationBytes?: number, ): ManagedNativeBoundaryInspection => { // An unmappable PE layout is invalid input for every managed operation. const pe = readManagedPeLayout(bytes); @@ -147,7 +150,13 @@ export const inspectManagedNativeBoundariesBytes = ( let layout: ManagedMetadataLayout; let inventory: Inventory; try { - ({ layout, inventory } = readBoundaryInventory(bytes, pe, pe.cli, issues)); + ({ layout, inventory } = readBoundaryInventory( + bytes, + pe, + pe.cli, + issues, + maxRepresentationBytes, + )); } catch (cause: unknown) { if (!(cause instanceof ManagedReaderFailure)) throw cause; return emptyInspection(target, bytes, "malformed", { diff --git a/src/dotnet/ManagedSnapshot.ts b/src/dotnet/ManagedSnapshot.ts index 5f8247d919..ac2f660d4c 100644 --- a/src/dotnet/ManagedSnapshot.ts +++ b/src/dotnet/ManagedSnapshot.ts @@ -22,14 +22,11 @@ import { AnalysisResourceConstraintError, } from "../domain/analysisErrorCore.js"; import { err, ok, type Result } from "../domain/result.js"; -import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; +import { DEFAULT_MANAGED_SNAPSHOT_BYTES } from "../domain/managed/managedInspectionLimits.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { RegularFileChangedError } from "../filesystem/RegularFile.js"; import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; -// Managed static inspection retains the complete PE snapshot in memory. -export const MAX_MANAGED_SNAPSHOT_BYTES = 128 * 1024 * 1024; - interface ManagedSourceObservation { readonly byteLength: number; readonly sha256: string; @@ -42,6 +39,7 @@ interface ManagedSnapshot extends ManagedSourceObservation { interface ManagedSnapshotReadOptions extends RegularFileReadOptions { readonly expectedSha256?: string; + readonly maxBytes?: number; } /** Admit one observed source digest before any parser or cache consumes its bytes. */ @@ -69,13 +67,17 @@ export const verifyManagedSourceIdentity = ( const managedSnapshotCapacityError = ( operation: string, + maxBytes: number, ): AnalysisResourceConstraintError => new AnalysisResourceConstraintError( operation, "memory", "Managed static inspection requires an eagerly retained PE snapshot within its input byte budget.", - { max_snapshot_bytes: MAX_MANAGED_SNAPSHOT_BYTES }, - fixedAllowanceOptions("The managed PE snapshot allowance", "parser"), + { max_snapshot_bytes: maxBytes }, + { + retryable: false, + remediationAction: `Increase max_snapshot_bytes above ${String(maxBytes)} bytes on the next inspection (CLI --max-snapshot-bytes) to admit the complete selected file. No REA restart is required. Metadata representation, transport and host memory limits remain independent; repeating the unchanged request cannot raise this budget.`, + }, ); export const readManagedSnapshot = async ( @@ -83,19 +85,17 @@ export const readManagedSnapshot = async ( operation: string, options: ManagedSnapshotReadOptions = {}, ): Promise => { + const maxBytes = snapshotBudget(options); return readManagedFile( path, operation, async (handle, metadata) => { const signal = options.signal; - if (metadata.size > MAX_MANAGED_SNAPSHOT_BYTES) - throw managedSnapshotCapacityError(operation); - const bytes = await readBoundedFileBytes( - handle, - MAX_MANAGED_SNAPSHOT_BYTES, - { signal }, - ); - if (bytes === undefined) throw managedSnapshotCapacityError(operation); + if (metadata.size > maxBytes) + throw managedSnapshotCapacityError(operation, maxBytes); + const bytes = await readBoundedFileBytes(handle, maxBytes, { signal }); + if (bytes === undefined) + throw managedSnapshotCapacityError(operation, maxBytes); signal?.throwIfAborted(); return { byteLength: bytes.length, @@ -111,14 +111,15 @@ export const hashManagedSource = async ( path: string, operation: string, options: ManagedSnapshotReadOptions = {}, -): Promise => - readManagedFile( +): Promise => { + const maxBytes = snapshotBudget(options); + return readManagedFile( path, operation, async (handle, metadata) => { const signal = options.signal; - if (metadata.size > MAX_MANAGED_SNAPSHOT_BYTES) - throw managedSnapshotCapacityError(operation); + if (metadata.size > maxBytes) + throw managedSnapshotCapacityError(operation, maxBytes); const digest = createHash("sha256"); let byteLength = 0; const chunk = Buffer.allocUnsafe(64 * 1024); @@ -127,13 +128,13 @@ export const hashManagedSource = async ( const { bytesRead } = await handle.read( chunk, 0, - Math.min(chunk.length, MAX_MANAGED_SNAPSHOT_BYTES - byteLength + 1), + Math.min(chunk.length, maxBytes - byteLength + 1), null, ); signal?.throwIfAborted(); if (bytesRead === 0) break; - if (bytesRead > MAX_MANAGED_SNAPSHOT_BYTES - byteLength) - throw managedSnapshotCapacityError(operation); + if (bytesRead > maxBytes - byteLength) + throw managedSnapshotCapacityError(operation, maxBytes); byteLength += bytesRead; digest.update(chunk.subarray(0, bytesRead)); } @@ -141,6 +142,14 @@ export const hashManagedSource = async ( }, options, ); +}; + +const snapshotBudget = (options: ManagedSnapshotReadOptions): number => { + const maxBytes = options.maxBytes ?? DEFAULT_MANAGED_SNAPSHOT_BYTES; + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) + throw new RangeError("max_snapshot_bytes must be a positive safe integer"); + return maxBytes; +}; const readManagedFile = ( path: string, diff --git a/src/dotnet/ManagedStaticProvider.ts b/src/dotnet/ManagedStaticProvider.ts index d36294f577..3797f6ce50 100644 --- a/src/dotnet/ManagedStaticProvider.ts +++ b/src/dotnet/ManagedStaticProvider.ts @@ -39,6 +39,8 @@ import { inspectManagedMembersBytes } from "./ManagedMemberInspector.js"; import { inspectManagedNativeBoundariesBytes } from "./ManagedNativeBoundaryInspector.js"; import { RegularFileCleanupFailure } from "../filesystem/OwnedRegularFile.js"; import { managedInspectionInputFailure } from "./ManagedReaderFailure.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import type { z } from "zod"; import { hashManagedSource, managedCleanupError, @@ -96,6 +98,26 @@ class ManagedStaticClient implements AnalysisClient { ), ); try { + const input = + operation === "inspect_managed_members" + ? managedMembersInputSchema.safeParse(parameters) + : managedTargetInputSchema.safeParse(parameters); + if (!input.success) + return err( + analysisInputErrorFromIssues( + operation, + input.error.issues, + parameters, + ), + ); + const snapshotOptions = { + expectedSha256: this.target.sha256, + signal: options?.signal, + resources: this.#resources, + ...(input.data.max_snapshot_bytes === undefined + ? {} + : { maxBytes: input.data.max_snapshot_bytes }), + }; if (options?.signal?.aborted === true) return err( new AnalysisCancelledError(operation, { @@ -105,16 +127,16 @@ class ManagedStaticClient implements AnalysisClient { const snapshot = this.#snapshotBytes; const observed = snapshot === undefined - ? await readManagedSnapshot(this.target.path, operation, { - expectedSha256: this.target.sha256, - signal: options?.signal, - resources: this.#resources, - }) - : await hashManagedSource(this.target.path, operation, { - expectedSha256: this.target.sha256, - signal: options?.signal, - resources: this.#resources, - }); + ? await readManagedSnapshot( + this.target.path, + operation, + snapshotOptions, + ) + : await hashManagedSource( + this.target.path, + operation, + snapshotOptions, + ); const admitted = verifyManagedSourceIdentity( this.target, observed, @@ -127,7 +149,7 @@ class ManagedStaticClient implements AnalysisClient { this.#snapshotBytes = bytes; const result = inspectManagedOperation( operation, - parameters, + input.data, bytes, this.target, ); @@ -202,7 +224,7 @@ const isManagedOperation = ( const inspectManagedOperation = ( operation: ManagedToolName, - parameters: Readonly>, + parameters: Omit, "detail">, bytes: Buffer, target: BinaryTarget, ): @@ -210,19 +232,29 @@ const inspectManagedOperation = ( | ManagedMemberInspection | ManagedNativeBoundaryInspection => { if (operation === "inspect_managed_members") { - const { type_names: typeNames } = - managedMembersInputSchema.parse(parameters); - return inspectManagedMembersBytes( + const { type_names: typeNames } = parameters; + return inspectManagedMembersBytes(bytes, target, { + ...(typeNames === undefined ? {} : { typeNames }), + ...(parameters.max_metadata_representation_bytes === undefined + ? {} + : { + maxRepresentationBytes: + parameters.max_metadata_representation_bytes, + }), + }); + } + if (operation === "inspect_managed_artifact") { + return inspectManagedArtifactBytes( bytes, target, - typeNames === undefined ? {} : { typeNames }, + parameters.max_metadata_representation_bytes, ); } - managedTargetInputSchema.parse(parameters); - if (operation === "inspect_managed_artifact") { - return inspectManagedArtifactBytes(bytes, target); - } - return inspectManagedNativeBoundariesBytes(bytes, target); + return inspectManagedNativeBoundariesBytes( + bytes, + target, + parameters.max_metadata_representation_bytes, + ); }; const managedLocations = ( diff --git a/src/evm/EvmProducerLimits.test.ts b/src/evm/EvmProducerLimits.test.ts new file mode 100644 index 0000000000..1abefecfa1 --- /dev/null +++ b/src/evm/EvmProducerLimits.test.ts @@ -0,0 +1,63 @@ +import { createRequire } from "node:module"; +import { readFile, writeFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { selectedEvmProducerLimits } from "../domain/evm/evmInterface.js"; +import { runOwnedCommand } from "../process/OwnedCommand.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; + +const require = createRequire(import.meta.url); + +it("recovers complete EVMole worker evidence for the same carrier after raising reply admission", async () => { + const root = await createTestTempDirectory("rea-evm-producer-limits-"); + const snapshot = join(root, "carrier.snapshot"); + await writeFile(snapshot, "0x60006000f3\n"); + const inspect = async (outputBytes: number, index: number) => { + const requestPath = join(root, `request-${index}.json`); + const reply = join(root, `reply-${index}.json`); + await writeFile( + requestPath, + JSON.stringify({ + snapshot_path: snapshot, + reply_path: reply, + failure_marker_path: join(root, `failure-${index}`), + encoding: "hex", + producer_limits: selectedEvmProducerLimits({ + output_bytes: outputBytes, + }), + }), + ); + await runOwnedCommand( + { + command: process.execPath, + arguments: [ + "--import", + require.resolve("tsx"), + "--max-old-space-size=256", + "--disable-wasm-trap-handler", + fileURLToPath(new URL("./EvmoleInterfaceWorker.ts", import.meta.url)), + requestPath, + ], + cwd: root, + runId: `evm-producer-limits-${index}`, + }, + { timeoutMs: 10000, diagnosticBytes: 1024 * 1024 }, + ); + return JSON.parse(await readFile(reply, "utf8")); + }; + const refused = await inspect(256, 0); + expect(refused).toMatchObject({ ok: false, reason: "output-limit" }); + const recovered = await inspect(4096, 1); + expect(recovered).toMatchObject({ + ok: true, + version: "0.9.3", + bytecode: { + bytes: 5, + hex: "60006000f3", + }, + }); + // A larger resource budget does not change the explicitly selected encoding contract. + await writeFile(snapshot, "0xzz"); + expect(await inspect(4096, 2)).toMatchObject({ ok: false, reason: "format" }); +}); diff --git a/src/evm/EvmWorkerLimits.test.ts b/src/evm/EvmWorkerLimits.test.ts index a7631689e5..3efc69e3ff 100644 --- a/src/evm/EvmWorkerLimits.test.ts +++ b/src/evm/EvmWorkerLimits.test.ts @@ -1,5 +1,6 @@ import { expect, it } from "vitest"; import { selectEvmWorkerLimits } from "./EvmWorkerLimits.js"; +import { selectedEvmProducerLimits } from "../domain/evm/evmInterface.js"; const limits = ( soft = "unlimited", @@ -84,3 +85,33 @@ it.each([ expect(selectEvmWorkerLimits(input)).toEqual({ ok: false, error }); }, ); + +it("applies requested producer budgets and retains narrower inherited limits", () => { + const requested = selectedEvmProducerLimits({ + output_bytes: 64, + address_space_bytes: 4096, + cpu_seconds: 2, + }); + expect(selectEvmWorkerLimits(limits(), requested)).toEqual({ + ok: true, + value: { + addressSpaceBytes: 4096, + cpuSeconds: 2, + fileSizeBytes: 64, + }, + }); + const tighter = limits("1024", "2048") + .replace("time unlimited unlimited", "time 1 1") + .replace("size unlimited unlimited", "size 32 32"); + expect(selectEvmWorkerLimits(tighter, requested)).toEqual({ + ok: true, + value: { + addressSpaceBytes: 1024, + cpuSeconds: 1, + fileSizeBytes: 32, + }, + }); + expect( + selectedEvmProducerLimits({ output_bytes: 64, file_bytes: 128 }).file_bytes, + ).toBe(128); +}); diff --git a/src/evm/EvmWorkerLimits.ts b/src/evm/EvmWorkerLimits.ts index 235c0a651c..d75494b693 100644 --- a/src/evm/EvmWorkerLimits.ts +++ b/src/evm/EvmWorkerLimits.ts @@ -1,5 +1,8 @@ import { err, ok, type Result } from "../domain/result.js"; -import { EVM_CPU_SECONDS, EVM_INTERFACE_LIMITS } from "./EvmoleRelease.js"; +import { + selectedEvmProducerLimits, + type EvmProducerLimits, +} from "../domain/evm/evmInterface.js"; /** Selected worker soft limits to configure; inherited hard limits are left unchanged. */ export interface EvmWorkerLimits { @@ -11,6 +14,7 @@ export interface EvmWorkerLimits { /** Read only the three selected Linux limits, retaining tighter soft/hard boundaries. */ export const selectEvmWorkerLimits = ( raw: unknown, + requested: EvmProducerLimits = selectedEvmProducerLimits(undefined), ): Result => { if (typeof raw !== "string" || Buffer.byteLength(raw) > 64 * 1024) return err( @@ -18,9 +22,9 @@ export const selectEvmWorkerLimits = ( ); const selected: number[] = []; for (const [name, units, maximum] of [ - ["Max address space", "bytes", EVM_INTERFACE_LIMITS.addressSpaceBytes], - ["Max cpu time", "seconds", EVM_CPU_SECONDS], - ["Max file size", "bytes", EVM_INTERFACE_LIMITS.outputBytes], + ["Max address space", "bytes", requested.address_space_bytes], + ["Max cpu time", "seconds", requested.cpu_seconds], + ["Max file size", "bytes", requested.file_bytes], ] as const) { const entries = raw .split("\n") diff --git a/src/evm/EvmoleFailures.test.ts b/src/evm/EvmoleFailures.test.ts index 74c2108a44..67f6c7fd7a 100644 --- a/src/evm/EvmoleFailures.test.ts +++ b/src/evm/EvmoleFailures.test.ts @@ -42,7 +42,9 @@ it("projects a complete EVM reply allowance refusal as a deterministic resource reported_limits: { maximum_reply_bytes: 16777216 }, captured_output: capturedOutput, }, - remediation: { action: expect.stringContaining("fixed report allowance") }, + remediation: { + action: expect.stringContaining("producer_limits.output_bytes"), + }, }); expect(projected.remediation.action).not.toContain("doctor"); }); @@ -67,7 +69,10 @@ it.each([ cancellation: { reason }, }, ); - const result = evmInterfaceFailure(source, "worker", "/selected.hex"); + const result = evmInterfaceFailure(source, { + phase: "worker", + path: "/selected.hex", + }); expect(result).toBeInstanceOf(AnalysisCancelledError); expect(result.cause).toBe(reason); expect(result.capturedOutput).toEqual({ @@ -87,15 +92,11 @@ it("names timeout_ms, the omitted default, and the caps it does not raise", () = null, ); const projected = projectAnalysisError( - evmInterfaceFailure( - source, - "worker", - "/selected.hex", - undefined, - undefined, - undefined, - 45_000, - ), + evmInterfaceFailure(source, { + phase: "worker", + path: "/selected.hex", + timeoutMs: 45_000, + }), ); expect(projected).toMatchObject({ code: "provider_timeout", @@ -109,7 +110,7 @@ it("names timeout_ms, the omitted default, and the caps it does not raise", () = expect(projected.remediation.action).toContain("no server restart"); }); -it("says a CPU signal has no runtime override and timeout_ms does not raise it", () => { +it("names the selected CPU budget separately from tighter inherited limits", () => { const source = new OwnedCommandFailure( "process", "signaled", @@ -121,30 +122,35 @@ it("says a CPU signal has no runtime override and timeout_ms does not raise it", null, ); const projected = projectAnalysisError( - evmInterfaceFailure(source, "worker", "/selected.hex", { - addressSpaceBytes: 1024, - cpuSeconds: 30, - fileSizeBytes: 1024, + evmInterfaceFailure(source, { + phase: "worker", + path: "/selected.hex", + workerLimits: { + addressSpaceBytes: 1024, + cpuSeconds: 30, + fileSizeBytes: 1024, + }, }), ); expect(projected).toMatchObject({ code: "resource_constraint", details: { resource: "cpu" }, }); - expect(projected.remediation.action).toContain("no runtime override"); - expect(projected.remediation.action).toContain( - "timeout_ms does not raise it", - ); + expect(projected.remediation.action).toContain("producer_limits.cpu_seconds"); + expect(projected.remediation.action).toContain("inherited"); }); -it("reports the fixed carrier input allowance as a resource constraint", () => { +it("reports the selected carrier input allowance as a resource constraint", () => { const source = new ArtifactReaderFailure( "limit", "Selected artifact exceeds 4194304 bytes: /selected.bin", { boundary: "file-size" }, ); const projected = projectAnalysisError( - evmInterfaceFailure(source, "artifact-read", "/selected.bin"), + evmInterfaceFailure(source, { + phase: "artifact-read", + path: "/selected.bin", + }), ); expect(projected).toMatchObject({ code: "resource_constraint", @@ -156,15 +162,8 @@ it("reports the fixed carrier input allowance as a resource constraint", () => { reported_limits: { maximum_input_bytes: 4194304 }, }, }); - expect(projected.message).toContain("4194304"); - expect(projected.message).toContain("no runtime override"); - expect(projected.message).toContain("timeout_ms does not raise it"); + expect(projected.remediation.action).toContain("producer_limits.input_bytes"); expect(projected.remediation.action).toContain("4194304"); - expect(projected.remediation.action).toContain("no runtime override"); - expect(projected.remediation.action).toContain("Do not truncate"); - expect(projected.remediation.action).toContain( - "repeating the unchanged request fails the same way", - ); expect(projected.recovery).toBeUndefined(); }); @@ -176,11 +175,10 @@ it.each([ { reason: "format" as const, message: "Unsupported carrier format." }, ])("keeps reader $reason refusals as invalid input", ({ reason, message }) => { const projected = projectAnalysisError( - evmInterfaceFailure( - new ArtifactReaderFailure(reason, message), - "artifact-read", - "/selected.bin", - ), + evmInterfaceFailure(new ArtifactReaderFailure(reason, message), { + phase: "artifact-read", + path: "/selected.bin", + }), ); expect(projected).toMatchObject({ code: "invalid_request", @@ -210,7 +208,10 @@ it.each([ }, ["exact-owner"], ); - const result = evmInterfaceFailure(source, "worker", "/selected.hex"); + const result = evmInterfaceFailure(source, { + phase: "worker", + path: "/selected.hex", + }); expect(result._tag).toBe(tag); if (cleanup !== null) { expect(result.cleanupIncomplete).toBe(true); @@ -234,7 +235,10 @@ it.skipIf(process.platform === "win32")( } catch (cause) { source = cause; } - const failure = evmInterfaceFailure(source, "artifact-read", path); + const failure = evmInterfaceFailure(source, { + phase: "artifact-read", + path: path, + }); expect(failure.cause).toBe(source); expect(projectAnalysisError(failure)).toMatchObject({ code: "invalid_request", @@ -248,9 +252,9 @@ it.skipIf(process.platform === "win32")( ], }, }); - expect(evmInterfaceFailure(source, "worker", path)._tag).toBe( - "ProviderAdapterError", - ); + expect( + evmInterfaceFailure(source, { phase: "worker", path: path })._tag, + ).toBe("ProviderAdapterError"); } }, ); diff --git a/src/evm/EvmoleFailures.ts b/src/evm/EvmoleFailures.ts index bfa320f067..fb92d36bd9 100644 --- a/src/evm/EvmoleFailures.ts +++ b/src/evm/EvmoleFailures.ts @@ -27,43 +27,40 @@ import { OwnedCommandFailure } from "../process/OwnedCommand.js"; import { DEFAULT_EVM_TIMEOUT_MS, MAX_EVM_TIMEOUT_MS, + selectedEvmProducerLimits, + evmProducerLimitRecovery, + type EvmProducerLimits, } from "../domain/evm/evmInterface.js"; -import { - EVM_CPU_SECONDS, - EVM_INTERFACE_LIMITS, - EVM_FILE_SIZE_FAILURE_EXIT, -} from "./EvmoleRelease.js"; +import { EVM_FILE_SIZE_FAILURE_EXIT } from "./EvmoleRelease.js"; import { EVMOLE_PROVIDER_IDENTITY } from "./EvmoleProviderIdentity.js"; -import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; /** Both a structured worker refusal and reply-file admission name this allowance. */ export const evmReplyBudgetFailure = ( reason: string, options: AnalysisErrorOptions, + limits: EvmProducerLimits = selectedEvmProducerLimits(undefined), ) => new AnalysisResourceConstraintError( "inspect_evm_interface", "file-size", reason, - { maximum_reply_bytes: EVM_INTERFACE_LIMITS.outputBytes }, + { + maximum_reply_bytes: limits.output_bytes, + parameter: "producer_limits.output_bytes", + }, { ...options, - ...fixedAllowanceOptions( - `The complete EVMole worker reply allowance of ${String(EVM_INTERFACE_LIMITS.outputBytes)} bytes (this release has no runtime override)`, - "report", - ), + retryable: false, + remediationAction: evmProducerLimitRecovery(limits, "output_bytes"), }, ); -const fixedEvmRecovery = (allowance: string): string => - `${allowance}. This release has no runtime override. timeout_ms does not raise it. Repeating the unchanged request fails the same way.`; - const evmTimeoutRecovery = (timeoutMs: number): string => `${ timeoutMs < MAX_EVM_TIMEOUT_MS ? `Increase timeout_ms above ${String(timeoutMs)} (integer milliseconds, maximum ${String(MAX_EVM_TIMEOUT_MS)}), or use rea inspect-evm-interface --timeout-ms on the CLI.` : `timeout_ms is already at the maximum supported ${String(MAX_EVM_TIMEOUT_MS)} ms; no higher worker deadline is supported. Report this blocker or select another suitable complete carrier with caller approval.` - } This is a per-call setting; no server restart is needed. It covers the owned EVMole worker command, not carrier acquisition or final reply reads. Allow the MCP client deadline to cover acquisition, the worker, reply reads, cleanup and delivery; increasing only that client deadline does not change this budget. Fixed input (${String(EVM_INTERFACE_LIMITS.inputBytes)} bytes), reply (${String(EVM_INTERFACE_LIMITS.outputBytes)} bytes), diagnostic (${String(EVM_INTERFACE_LIMITS.diagnosticBytes)} bytes), address-space (${String(EVM_INTERFACE_LIMITS.addressSpaceBytes)} bytes), and CPU (${String(EVM_CPU_SECONDS)} seconds) allowances stay unchanged, and timeout_ms does not raise them. Resolve any incomplete cleanup before retrying; do not repeat the unchanged failed call.`; + } This is a per-call setting; no server restart is needed. It covers the owned EVMole worker command, not carrier acquisition or final reply reads. Allow the MCP client deadline to cover acquisition, the worker, reply reads, cleanup and delivery; increasing only that client deadline does not change this budget. Separate producer_limits control carrier/reply/diagnostic bytes, address-space/file bytes and CPU seconds; timeout_ms does not raise them. The 256 MiB worker heap stays independent. Resolve any incomplete cleanup before retrying; do not repeat the unchanged failed call.`; /** Correlate a reserved exit with the owned worker's actual failure branch. */ export interface EvmFileSizeFailureEvidence { @@ -72,14 +69,27 @@ export interface EvmFileSizeFailureEvidence { } /** Keep carrier read, engine and owned lifecycle failures distinct. */ +export interface EvmFailureContext { + readonly phase: "configuration" | "artifact-read" | "worker"; + readonly path: string; + readonly workerLimits?: EvmWorkerLimits | undefined; + readonly limiter?: string | undefined; + readonly fileFailure?: EvmFileSizeFailureEvidence | undefined; + readonly timeoutMs?: number | undefined; + readonly producerLimits?: EvmProducerLimits | undefined; +} + export const evmInterfaceFailure = ( cause: unknown, - phase: "configuration" | "artifact-read" | "worker", - path: string, - workerLimits?: EvmWorkerLimits, - limiter = "/usr/bin/prlimit", - fileFailure?: EvmFileSizeFailureEvidence, - timeoutMs = DEFAULT_EVM_TIMEOUT_MS, + { + phase, + path, + workerLimits, + limiter = "/usr/bin/prlimit", + fileFailure, + timeoutMs = DEFAULT_EVM_TIMEOUT_MS, + producerLimits = selectedEvmProducerLimits(undefined), + }: EvmFailureContext, ): AnalysisError => { const cleanup = cause instanceof ArtifactReaderFailure ? cause.cleanup : undefined; @@ -124,9 +134,14 @@ export const evmInterfaceFailure = ( if (cause.reason === "output-limit") return new AnalysisOutputError("inspect_evm_interface", cause.message, { ...outputOptions, - userMessage: `EVMole worker diagnostics exceeded the ${String(EVM_INTERFACE_LIMITS.diagnosticBytes)}-byte capture allowance. This release has no runtime override.`, - remediationAction: fixedEvmRecovery( - `The EVMole diagnostic capture allowance is ${String(EVM_INTERFACE_LIMITS.diagnosticBytes)} bytes`, + userMessage: `EVMole worker diagnostics exceeded producer_limits.diagnostic_bytes (${String(producerLimits.diagnostic_bytes)} bytes).`, + diagnostics: { + parameter: "producer_limits.diagnostic_bytes", + effective_value: producerLimits.diagnostic_bytes, + }, + remediationAction: evmProducerLimitRecovery( + producerLimits, + "diagnostic_bytes", ), }); if ( @@ -159,12 +174,8 @@ export const evmInterfaceFailure = ( retryable: false, remediationAction: cause.snapshot.signal === "SIGXCPU" - ? fixedEvmRecovery( - `The EVMole CPU ceiling is ${String(EVM_CPU_SECONDS)} seconds, or a tighter inherited limit when one applies`, - ) - : fixedEvmRecovery( - `The EVMole file-size ceiling is ${String(EVM_INTERFACE_LIMITS.outputBytes)} bytes, or a tighter inherited limit when one applies`, - ), + ? evmProducerLimitRecovery(producerLimits, "cpu_seconds") + : evmProducerLimitRecovery(producerLimits, "file_bytes"), }, ); } @@ -204,13 +215,19 @@ export const evmInterfaceFailure = ( return new AnalysisResourceConstraintError( "inspect_evm_interface", "file-size", - `${cause.message} The input allowance is ${String(EVM_INTERFACE_LIMITS.inputBytes)} bytes. This release has no runtime override. timeout_ms does not raise it.`, - { maximum_input_bytes: EVM_INTERFACE_LIMITS.inputBytes }, + `${cause.message} producer_limits.input_bytes is ${String(producerLimits.input_bytes)} bytes; timeout_ms does not raise it.`, + { + maximum_input_bytes: producerLimits.input_bytes, + parameter: "producer_limits.input_bytes", + }, { cause, ...cleanupOptions, retryable: false, - remediationAction: `Select a suitable complete carrier within the ${String(EVM_INTERFACE_LIMITS.inputBytes)}-byte input allowance. This release has no runtime override. timeout_ms does not raise it, and reply or diagnostic allowances do not admit a larger carrier. Do not truncate required bytes; repeating the unchanged request fails the same way.`, + remediationAction: evmProducerLimitRecovery( + producerLimits, + "input_bytes", + ), }, ); if (phase === "artifact-read" && ["path", "format"].includes(cause.reason)) diff --git a/src/evm/EvmoleInterfaceProvider.ts b/src/evm/EvmoleInterfaceProvider.ts index 4dfa48ba8e..fcc6249f09 100644 --- a/src/evm/EvmoleInterfaceProvider.ts +++ b/src/evm/EvmoleInterfaceProvider.ts @@ -57,6 +57,7 @@ import { import { evmInterfaceEvidence } from "../domain/evm/evmInterfaceEvidence.js"; import { DEFAULT_EVM_TIMEOUT_MS, + selectedEvmProducerLimits, type InspectEvmInterfaceInput, evmInterfaceSchema, } from "../domain/evm/evmInterface.js"; @@ -77,8 +78,7 @@ import { projectEvmoleFunctions, } from "./EvmoleInterfaceAdapter.js"; import { - EVM_CPU_SECONDS, - EVM_INTERFACE_LIMITS, + EVM_WORKER_HEAP_MIB, EVM_FILE_SIZE_FAILURE_EXIT, } from "./EvmoleRelease.js"; import { EVMOLE_PROVIDER_IDENTITY } from "./EvmoleProviderIdentity.js"; @@ -438,15 +438,14 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { this.#inspect(input, environment, options), ); } catch (cause: unknown) { - const failure = evmInterfaceFailure( - cause, - "artifact-read", - input.path, - undefined, - environment.REA_EVM_PRLIMIT_COMMAND ?? EVM_PRLIMIT_COMMAND_DEFAULT, - undefined, - input.timeout_ms ?? DEFAULT_EVM_TIMEOUT_MS, - ); + const failure = evmInterfaceFailure(cause, { + phase: "artifact-read", + path: input.path, + limiter: + environment.REA_EVM_PRLIMIT_COMMAND ?? EVM_PRLIMIT_COMMAND_DEFAULT, + timeoutMs: input.timeout_ms ?? DEFAULT_EVM_TIMEOUT_MS, + producerLimits: selectedEvmProducerLimits(input.producer_limits), + }); if (cause instanceof ArtifactReaderFailure) bindArtifactFileCleanup(cause, failure); return err(failure); @@ -513,6 +512,7 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { return rootLease; }; const timeoutMs = input.timeout_ms ?? DEFAULT_EVM_TIMEOUT_MS; + const producerLimits = selectedEvmProducerLimits(input.producer_limits); let retainedOutput: AnalysisCapturedOutput | undefined; let replyFinalization: AnalysisError | undefined; let phase: "configuration" | "artifact-read" | "worker" = "configuration"; @@ -570,7 +570,7 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { { cause, userMessage: reason }, ); } - const limits = selectEvmWorkerLimits(inheritedLimits); + const limits = selectEvmWorkerLimits(inheritedLimits, producerLimits); if (!limits.ok) throw new AnalysisCapabilityUnavailableError( EVMOLE_PROVIDER_IDENTITY.id, @@ -582,7 +582,7 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { phase = "artifact-read"; const carrier = await readStableArtifact( input.path, - EVM_INTERFACE_LIMITS.inputBytes, + producerLimits.input_bytes, options?.signal, undefined, this.#resources, @@ -605,6 +605,7 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { reply_path: replyPath, failure_marker_path: join(root.path, "resource.failure"), encoding: input.encoding, + producer_limits: producerLimits, }), { flag: "wx", mode: 0o600 }, ); @@ -620,7 +621,7 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { `--cpu=${String(workerLimits.cpuSeconds)}:`, "--", process.execPath, - "--max-old-space-size=256", + `--max-old-space-size=${String(EVM_WORKER_HEAP_MIB)}`, "--v8-pool-size=1", "--disable-wasm-trap-handler", fileURLToPath( @@ -635,7 +636,7 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { }, { timeoutMs, - diagnosticBytes: EVM_INTERFACE_LIMITS.diagnosticBytes, + diagnosticBytes: producerLimits.diagnostic_bytes, }, { ...(options?.signal === undefined ? {} : { signal: options.signal }), @@ -661,7 +662,7 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { try { file = await readStableArtifact( replyPath, - EVM_INTERFACE_LIMITS.outputBytes, + producerLimits.output_bytes, options?.signal, undefined, this.#resources, @@ -699,11 +700,15 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { cause instanceof ArtifactReaderFailure && cause.reason === "limit" && cause.boundary === "file-size" - ? evmReplyBudgetFailure(cause.message, { - cause, - capturedOutput, - ...(cleanup === undefined ? {} : { cleanup }), - }) + ? evmReplyBudgetFailure( + cause.message, + { + cause, + capturedOutput, + ...(cleanup === undefined ? {} : { cleanup }), + }, + producerLimits, + ) : new AnalysisOutputError( OPERATION, `Owned interface worker reply failed for ${input.path}: ${cause instanceof Error ? cause.message : String(cause)}`, @@ -734,9 +739,13 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { }, ]); if (reply.reason === "output-limit") - throw evmReplyBudgetFailure(reply.message, { - capturedOutput, - }); + throw evmReplyBudgetFailure( + reply.message, + { + capturedOutput, + }, + producerLimits, + ); throw new ProviderAdapterError(EVMOLE_PROVIDER_IDENTITY.id, OPERATION, { diagnostics: { phase, @@ -773,7 +782,7 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { "Bytecode kind, hardfork and deployed-contract authenticity are unknown. This inspection performs no target runtime execution, chain/RPC lookup or network request.", "Original carrier and decoded bytecode SHA-256 are distinct identities; SHA-256 is not the Ethereum Keccak code hash.", "Terminal CBOR metadata is an independently decoded candidate, not proof of Solidity origin, source identity or deployment. Nested or unsupported values retain explicit unavailable interpretation and original byte ranges.", - `The owned worker has a 256 MiB JavaScript heap. The profile maxima are ${String(EVM_INTERFACE_LIMITS.addressSpaceBytes)} address-space bytes and ${String(EVM_CPU_SECONDS)} CPU seconds; the configured soft limits above retain tighter inherited limits. The command deadline is timeout_ms at ${String(timeoutMs)} ms (default ${String(DEFAULT_EVM_TIMEOUT_MS)}). Those byte, address-space, and CPU maxima have no runtime override, and timeout_ms does not raise them. Virtual address space is distinct from resident memory; limits fail without partial success.`, + `The owned worker has a 256 MiB JavaScript heap. The profile maxima are ${String(producerLimits.address_space_bytes)} address-space bytes and ${String(producerLimits.cpu_seconds)} CPU seconds; the configured soft limits above retain tighter inherited limits. The command deadline is timeout_ms at ${String(timeoutMs)} ms (default ${String(DEFAULT_EVM_TIMEOUT_MS)}). producer_limits can change those requested byte, address-space and CPU allowances on the next call without restart; timeout_ms does not raise them. The 256 MiB worker heap remains independent. Virtual address space is distinct from resident memory; limits fail without partial success.`, ]; const validated = evmInterfaceSchema.safeParse({ artifact: { @@ -881,15 +890,15 @@ export class EvmoleInterfaceProvider implements EvmInterfacePort { ? { cleanup: cause.cleanup } : {}), }) - : evmInterfaceFailure( - cause, + : evmInterfaceFailure(cause, { phase, - input.path, + path: input.path, workerLimits, limiter, fileFailure, timeoutMs, - ); + producerLimits, + }); if (markerCleanupFailure !== undefined) { const cleanup = retainedCleanupFailure(markerCleanupFailure); failure = analysisErrorWithCleanupFailure( diff --git a/src/evm/EvmoleInterfaceWorker.ts b/src/evm/EvmoleInterfaceWorker.ts index 96e795749b..05ae09bd17 100644 --- a/src/evm/EvmoleInterfaceWorker.ts +++ b/src/evm/EvmoleInterfaceWorker.ts @@ -8,16 +8,19 @@ import { } from "./EvmBytecodeCarrier.js"; import { evmoleInterfaceReplySchema } from "./EvmoleInterfaceAdapter.js"; import { inspectSolidityMetadata } from "./SolidityMetadata.js"; +import { EVM_FILE_SIZE_FAILURE_EXIT } from "./EvmoleRelease.js"; import { - EVM_INTERFACE_LIMITS, - EVM_FILE_SIZE_FAILURE_EXIT, -} from "./EvmoleRelease.js"; + evmProducerLimitsSchema, + selectedEvmProducerLimits, +} from "../domain/evm/evmInterface.js"; +import { readStableArtifact } from "../artifacts/readStableArtifact.js"; const requestSchema = z.strictObject({ snapshot_path: z.string(), reply_path: z.string(), failure_marker_path: z.string(), encoding: z.enum(["raw", "hex"]), + producer_limits: evmProducerLimitsSchema, }); class UnsupportedContainer extends Error {} @@ -28,10 +31,12 @@ const main = async (): Promise => { const request = requestSchema.parse( JSON.parse(await readFile(requestPath, "utf8")), ); + const limits = selectedEvmProducerLimits(request.producer_limits); let reply: unknown; try { const bytes = decodeEvmBytecodeCarrier( - await readFile(request.snapshot_path), + (await readStableArtifact(request.snapshot_path, limits.input_bytes)) + .bytes, request.encoding, ); if (bytes[0] === 0xef && bytes[1] === 0x00) @@ -83,11 +88,11 @@ const main = async (): Promise => { }; } let encoded = JSON.stringify(reply); - if (Buffer.byteLength(encoded) > EVM_INTERFACE_LIMITS.outputBytes) + if (Buffer.byteLength(encoded) > limits.output_bytes) encoded = JSON.stringify({ ok: false, reason: "output-limit", - message: "Complete interface evidence exceeds the 16 MiB reply budget.", + message: `Complete interface evidence exceeds producer_limits.output_bytes (${String(limits.output_bytes)} bytes).`, }); try { await writeFile(request.reply_path, encoded, { flag: "wx", mode: 0o600 }); diff --git a/src/evm/EvmoleRelease.ts b/src/evm/EvmoleRelease.ts index 4432ce366a..c98133e6ec 100644 --- a/src/evm/EvmoleRelease.ts +++ b/src/evm/EvmoleRelease.ts @@ -1,14 +1,4 @@ -import { DEFAULT_EVM_TIMEOUT_MS } from "../domain/evm/evmInterface.js"; - /** Worker-reserved status for an observed EFBIG write without a complete reply. */ export const EVM_FILE_SIZE_FAILURE_EXIT = 76; -/** Complete offline inspection budgets; worker JavaScript heap is bounded separately. */ -export const EVM_INTERFACE_LIMITS = { - inputBytes: 4 * 1024 * 1024, - outputBytes: 16 * 1024 * 1024, - timeoutMs: DEFAULT_EVM_TIMEOUT_MS, - diagnosticBytes: 1024 * 1024, - addressSpaceBytes: 3 * 1024 * 1024 * 1024, -} as const; -/** Process CPU ceiling. A longer timeout_ms does not raise it. */ -export const EVM_CPU_SECONDS = 30; +/** Independent old-generation heap request; not an aggregate memory measurement. */ +export const EVM_WORKER_HEAP_MIB = 256; diff --git a/src/firmware/FirmwareRelease.ts b/src/firmware/FirmwareRelease.ts index 653cdea3b9..68cd23058f 100644 --- a/src/firmware/FirmwareRelease.ts +++ b/src/firmware/FirmwareRelease.ts @@ -1,4 +1,4 @@ -import { BOUNDED_PRODUCER_LIMITS } from "../process/LinuxResourceLimits.js"; +import { BOUNDED_PRODUCER_LIMITS } from "../domain/producerResourceLimits.js"; /** Resource policy for this Linux integration, not a claim of aggregate containment. */ export const FIRMWARE_LIMITS = { diff --git a/src/flutter/FlutterBuildProvider.ts b/src/flutter/FlutterBuildProvider.ts index 6af0a5a35a..6669f6cd59 100644 --- a/src/flutter/FlutterBuildProvider.ts +++ b/src/flutter/FlutterBuildProvider.ts @@ -22,6 +22,8 @@ import { import { createAnalysisExecutionEvidence } from "../domain/analysisObservation.js"; import { MAX_FLUTTER_LIBRARY_PATH_BYTES, + dartPoolPolicy, + type DartPoolPolicy, type FlutterRequest, } from "../domain/flutter/flutterBuildAnalysis.js"; import { @@ -79,8 +81,6 @@ const APK_MAX_BYTES = 2 * 1024 * 1024 * 1024; const MAX_FLUTTER_ABIS = 32; const flutterBudget = (allowance: string, kind: "parser" | "input") => fixedAllowanceOptions(allowance, kind); -const MAX_POOL_URIS = 500; -const MAX_POOL_TOKENS = 2_000; /** A complete AOT image carries exactly one VM and one isolate section. */ const SYMBOL_PAIR_COUNT = 2; const IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/u; @@ -731,6 +731,7 @@ export class FlutterBuildProvider { snapshot, reader, libraries, + dartPoolPolicy(request.input.pool_policy), ) : await this.identifyFlutterBuild( path, @@ -1085,6 +1086,7 @@ export class FlutterBuildProvider { snapshot: OwnedSnapshot, reader: ZipArtifactReader, libraries: readonly FlutterLibrary[], + poolPolicy: DartPoolPolicy, ): Promise> { const candidates = libraries.filter((library) => library.kind === "libapp"); if (candidates.length === 0) @@ -1296,7 +1298,7 @@ export class FlutterBuildProvider { ), ); } - const pool = await projectStringPool(isolateData, signal); + const pool = await projectStringPool(isolateData, signal, poolPolicy); const snapshotPartial = inventory.failures.length > 0 || aliasedDataSymbols || @@ -1309,9 +1311,9 @@ export class FlutterBuildProvider { const poolPartial = isolateData === null || pool.collectionCapped || - pool.packageUriCount > MAX_POOL_URIS || - pool.dartUriCount > MAX_POOL_URIS || - pool.classLikeTokenCount > MAX_POOL_TOKENS; + pool.packageUriCount > poolPolicy.uri_limit || + pool.dartUriCount > poolPolicy.uri_limit || + pool.classLikeTokenCount > poolPolicy.token_limit; return ok( createAnalysisExecution( { @@ -1328,12 +1330,16 @@ export class FlutterBuildProvider { sections, }, string_pool: { + policy: { ...poolPolicy }, coverage: poolPartial ? "partial" : "complete", - package_uris: pool.packageUris.slice(0, MAX_POOL_URIS), + package_uris: pool.packageUris.slice(0, poolPolicy.uri_limit), package_uri_count_at_least: pool.packageUriCount, - dart_uris: pool.dartUris.slice(0, MAX_POOL_URIS), + dart_uris: pool.dartUris.slice(0, poolPolicy.uri_limit), dart_uri_count_at_least: pool.dartUriCount, - class_like_tokens: pool.classLikeTokens.slice(0, MAX_POOL_TOKENS), + class_like_tokens: pool.classLikeTokens.slice( + 0, + poolPolicy.token_limit, + ), class_like_token_count_at_least: pool.classLikeTokenCount, printable_run_count: pool.printableRunCount, }, @@ -1349,7 +1355,7 @@ export class FlutterBuildProvider { "Snapshot hash sources are the headers at the start of the VM and isolate data symbols; complete decoding supports kind 3 (the Dart 2.14 FullAOT layout), a valid hash and a declared length that fits its symbol, including the four magic bytes; other kind values remain observed with partial coverage", "String-pool entries are byte-pattern observations from the isolate data section, not deserialized cluster semantics; typed class and function recovery needs an SDK-specific parser", "class_like_tokens are identifier-shaped runs and include incidental matches", - "string-pool counts are lower bounds: collection stops at twice each projection bound, and any value beyond that is counted only through the capped flag", + "URI/token counts are lower bounds for runs matching pool_policy.contains: distinct collection stops at twice each selected projection bound. printable_run_count counts complete printable runs before filtering; overflowed runs are omitted, never fabricated as prefixes", ...inventory.failures.map((failure) => failure.reason), ...(sections.length < symbolNames.length ? [ @@ -1372,24 +1378,23 @@ export class FlutterBuildProvider { (section) => `${section.name} has an incomplete or invalid snapshot header: kind ${String(section.header.kind)} (AOT app requires 3), declared length ${String(section.header.headerLength)} excludes the four magic bytes, symbol size ${String(section.size)}, valid hash ${String(section.header.hash !== null)}`, ), - ...(pool.collectionCapped + ...pool.limitsReached.map( + (limit) => + `String-pool admission reached pool_policy.${limit}=${String(poolPolicy[limit])}. Retry with a narrower pool_policy.contains or increase ${limit} (CLI --pool-policy JSON); no REA restart is required. Counts remain lower bounds and overflowed runs are omitted. APK, library and response limits are unchanged`, + ), + ...(pool.dartUriCount > poolPolicy.uri_limit ? [ - "String-pool collection or printable-run capacity was exceeded; counts are lower bounds and overflowed runs are omitted", + `dart_uris reports ${String(poolPolicy.uri_limit)} of at least ${String(pool.dartUriCount)} distinct URIs; narrow pool_policy.contains or raise pool_policy.uri_limit (CLI --pool-policy JSON) on a new call, no restart required`, ] : []), - ...(pool.dartUriCount > MAX_POOL_URIS + ...(pool.packageUriCount > poolPolicy.uri_limit ? [ - `dart_uris reports ${String(MAX_POOL_URIS)} of at least ${String(pool.dartUriCount)} distinct URIs`, + `package_uris reports ${String(poolPolicy.uri_limit)} of at least ${String(pool.packageUriCount)} distinct URIs; narrow pool_policy.contains or raise pool_policy.uri_limit (CLI --pool-policy JSON) on a new call, no restart required`, ] : []), - ...(pool.packageUriCount > MAX_POOL_URIS + ...(pool.classLikeTokenCount > poolPolicy.token_limit ? [ - `package_uris reports ${String(MAX_POOL_URIS)} of at least ${String(pool.packageUriCount)} distinct URIs`, - ] - : []), - ...(pool.classLikeTokenCount > MAX_POOL_TOKENS - ? [ - `class_like_tokens reports ${String(MAX_POOL_TOKENS)} of at least ${String(pool.classLikeTokenCount)} distinct tokens`, + `class_like_tokens reports ${String(poolPolicy.token_limit)} of at least ${String(pool.classLikeTokenCount)} distinct tokens; narrow pool_policy.contains or raise pool_policy.token_limit (CLI --pool-policy JSON) on a new call, no restart required`, ] : []), ], @@ -1409,6 +1414,7 @@ export class FlutterBuildProvider { const projectStringPool = async ( section: Buffer | null, signal: AbortSignal | undefined, + poolPolicy: DartPoolPolicy, ): Promise<{ packageUris: string[]; packageUriCount: number; @@ -1418,38 +1424,44 @@ const projectStringPool = async ( classLikeTokenCount: number; printableRunCount: number; collectionCapped: boolean; + limitsReached: readonly Exclude[]; }> => { const packages = new Set(); const darts = new Set(); const tokens = new Set(); let runCount = 0; - let collectionCapped = false; + const limitsReached = new Set>(); const COLLECTION_SLACK = 2; - /** One printable run never materializes beyond this length. */ - const MAX_RUN_BYTES = 1024; - const packageBudget = MAX_POOL_URIS * COLLECTION_SLACK; - const dartBudget = MAX_POOL_URIS * COLLECTION_SLACK; - const tokenBudget = MAX_POOL_TOKENS * COLLECTION_SLACK; + // Input bytes bound useful capacity before slack multiplication; large safe + // integer requests never overflow the collection-budget arithmetic. + const packageBudget = + Math.min(poolPolicy.uri_limit, section?.length ?? 0) * COLLECTION_SLACK; + const dartBudget = packageBudget; + const tokenBudget = + Math.min(poolPolicy.token_limit, section?.length ?? 0) * COLLECTION_SLACK; if (section !== null) { let run = ""; const consider = (text: string): void => { if (text.length < 5) return; runCount += 1; + if ( + poolPolicy.contains !== undefined && + !text.includes(poolPolicy.contains) + ) + return; if (text.startsWith("package:")) { if (!packages.has(text) && packages.size >= packageBudget) - collectionCapped = true; + limitsReached.add("uri_limit"); else packages.add(text); } else if (text.startsWith("dart:")) { if (!darts.has(text) && darts.size >= dartBudget) - collectionCapped = true; + limitsReached.add("uri_limit"); else darts.add(text); - } else if ( - text.length <= 128 && - text.length >= 5 && - IDENTIFIER_PATTERN.test(text) - ) { - if (!tokens.has(text) && tokens.size >= tokenBudget) - collectionCapped = true; + } else if (IDENTIFIER_PATTERN.test(text)) { + if (text.length > poolPolicy.token_bytes) + limitsReached.add("token_bytes"); + else if (!tokens.has(text) && tokens.size >= tokenBudget) + limitsReached.add("token_limit"); else tokens.add(text); } }; @@ -1463,10 +1475,11 @@ const projectStringPool = async ( // materializes: growth stops at the cap and the entire run is // discarded at its delimiter, since a truncated prefix would be // fabricated evidence rather than an observation. - if (run.length < MAX_RUN_BYTES) run += String.fromCharCode(byte); + if (run.length < poolPolicy.run_bytes) + run += String.fromCharCode(byte); else { overflowed = true; - collectionCapped = true; + limitsReached.add("run_bytes"); } } else { if (!overflowed) consider(run); @@ -1486,6 +1499,7 @@ const projectStringPool = async ( classLikeTokens: [...tokens].sort(), classLikeTokenCount: tokens.size, printableRunCount: runCount, - collectionCapped, + collectionCapped: limitsReached.size > 0, + limitsReached: [...limitsReached], }; }; diff --git a/src/frida/FridaInstrumentationManager.test.ts b/src/frida/FridaInstrumentationManager.test.ts index 5666c6ee61..7c8443a0c6 100644 --- a/src/frida/FridaInstrumentationManager.test.ts +++ b/src/frida/FridaInstrumentationManager.test.ts @@ -1453,32 +1453,152 @@ it("redacts remote credentials echoed by a target script", async () => { await manager.closeAll(); }); -it("bounds oversized Frida message metadata before retaining it", async () => { +it.each([ + { captureLimits: undefined, retained: 32_768, truncated: true }, + { + captureLimits: { message_text_bytes: 65_536 }, + retained: 65_536, + truncated: true, + }, + { + captureLimits: { message_text_bytes: 1_048_576 }, + retained: 1_000_000, + truncated: false, + }, +])( + "honors session text capture policy $captureLimits before observing messages", + async ({ captureLimits, retained, truncated }) => { + const bindings = new FakeDeviceManager(); + bindings.device.session.nextDescription = "d".repeat(1_000_000); + bindings.device.session.nextStack = "s".repeat(1_000_000); + const manager = new FridaInstrumentationManager({ + deviceManager: bindings, + }); + onTestFinished(() => manager.closeAll()); + const started = await manager.startSession({ + mode: "attach", + deviceId: "local", + pid: 512, + ...(captureLimits === undefined ? {} : { captureLimits }), + }); + if (!started.ok) throw started.error; + const service = new FridaInstrumentationService(manager); + const loaded = await service.loadScript(started.value.sessionId, { + sourceKind: "inline", + source: "send('metadata');", + }); + if (!loaded.ok) throw loaded.error; + const result = loaded.value.normalized_result as { + messages: { + description: string; + stack: string; + description_truncated?: boolean; + stack_truncated?: boolean; + }[]; + }; + const message = result.messages[0]; + expect(message?.description).toHaveLength(retained); + expect(message?.stack).toHaveLength(retained); + expect(message?.description_truncated ?? false).toBe(truncated); + expect(message?.stack_truncated ?? false).toBe(truncated); + if (truncated) { + expect(loaded.value.limitations.join("\n")).toContain( + `capture_limits.message_text_bytes (${retained} bytes)`, + ); + expect(loaded.value.limitations.join("\n")).toContain( + "Already discarded observations cannot be recovered", + ); + } + }, +); + +it.each([ + { nodes: 2, truncated: true }, + { nodes: 4, truncated: false }, +])( + "captures a selected JSON payload with json_nodes=$nodes", + async ({ nodes, truncated }) => { + const bindings = new FakeDeviceManager(); + bindings.device.session.nextPayload = [1, 2, 3]; + const manager = new FridaInstrumentationManager({ + deviceManager: bindings, + }); + onTestFinished(() => manager.closeAll()); + const started = await manager.startSession({ + mode: "attach", + pid: 512, + captureLimits: { json_nodes: nodes }, + }); + if (!started.ok) throw started.error; + const service = new FridaInstrumentationService(manager); + const loaded = await service.loadScript(started.value.sessionId, { + sourceKind: "inline", + source: "send([1,2,3])", + }); + if (!loaded.ok) throw loaded.error; + const message = ( + loaded.value.normalized_result as { messages: Record[] } + ).messages[0]; + expect(message?.payload_truncated ?? false).toBe(truncated); + if (truncated) { + expect(message?.payload_capture_limit).toBe("json_nodes"); + expect(loaded.value.limitations.join("\n")).toContain( + "capture_limits.json_nodes (2 nodes)", + ); + } else expect(message?.payload).toEqual([1, 2, 3]); + }, +); + +it.each([ + { sessionBytes: 200_000, truncated: true }, + { sessionBytes: 300_000, truncated: false }, +])( + "accounts base64 against selected session bytes=$sessionBytes", + async ({ sessionBytes, truncated }) => { + const bindings = new FakeDeviceManager(); + const manager = new FridaInstrumentationManager({ + deviceManager: bindings, + }); + onTestFinished(() => manager.closeAll()); + const started = await manager.startSession({ + mode: "attach", + pid: 512, + captureLimits: { session_bytes: sessionBytes }, + }); + if (!started.ok) throw started.error; + const loaded = await manager.loadScript(started.value.sessionId, { + sourceKind: "inline", + source: "send(null, new ArrayBuffer(65536))", + }); + if (!loaded.ok) throw loaded.error; + bindings.device.session.scripts[0]?.message.emit( + { type: "send", payload: null }, + Buffer.alloc(65_536), + ); + const observed = manager.status(started.value.sessionId); + const message = observed?.messages.at(-1) as Record; + expect(message.data_truncated ?? false).toBe(truncated); + if (truncated) { + expect(message.data_capture_limit).toBe("session_bytes"); + expect(message.data_bytes).toBe(65_536); + } else + expect(Buffer.from(message.data_base64 as string, "base64")).toHaveLength( + 65_536, + ); + }, +); + +it("rejects invalid capture policy before attaching or spawning", async () => { const bindings = new FakeDeviceManager(); - bindings.device.session.nextDescription = "d".repeat(1_000_000); - bindings.device.session.nextStack = "s".repeat(1_000_000); const manager = new FridaInstrumentationManager({ deviceManager: bindings }); const started = await manager.startSession({ mode: "attach", - deviceId: "local", pid: 512, + captureLimits: { json_nodes: 0 }, }); - expect(started.ok).toBe(true); - if (!started.ok) return; - const loaded = await manager.loadScript(started.value.sessionId, { - sourceKind: "inline", - source: "send('metadata');", - }); - expect(loaded.ok).toBe(true); - if (loaded.ok) { - expect(loaded.value.messagesTruncated).toBe(false); - const message = loaded.value.messages[0]; - expect(message).toMatchObject({ - description_truncated: true, - stack_truncated: true, - }); - expect(JSON.stringify(message).length).toBeLessThan(100_000); - } + expect(started.ok).toBe(false); + if (!started.ok) expect(started.error).toBeInstanceOf(AnalysisInputError); + expect(bindings.device.attachedTargets).toEqual([]); await manager.closeAll(); }); diff --git a/src/frida/FridaInstrumentationManager.ts b/src/frida/FridaInstrumentationManager.ts index a7ae3fd67b..e263962f33 100644 --- a/src/frida/FridaInstrumentationManager.ts +++ b/src/frida/FridaInstrumentationManager.ts @@ -45,15 +45,14 @@ import { unknownFridaSession, } from "../application/frida/fridaInputErrors.js"; import { - MAX_BINARY_MESSAGE_BYTES, - MAX_CAPTURE_JSON_NODES, - MAX_JSON_DEPTH, MAX_MESSAGE_METADATA_BYTES, - MAX_MESSAGE_TEXT_BYTES, - MAX_MESSAGE_TYPE_BYTES, - MAX_SESSION_CAPTURE_BYTES, type FridaOwnedCaptureLimit, } from "./fridaCaptureLimits.js"; +import { + fridaCapturePolicySchema, + type FridaCapturePolicy, +} from "../domain/fridaCapturePolicy.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; const DEVICE_LOOKUP_TIMEOUT_MS = 1_000; @@ -96,6 +95,7 @@ interface PendingStartupCleanup extends ManagedRemoteConnection { } interface ManagedSession extends ManagedRemoteConnection { + readonly captureLimits: FridaCapturePolicy; readonly id: string; readonly device: Device; readonly session: Session; @@ -305,6 +305,20 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { async #startSession( input: StartFridaSessionInput, ): Promise> { + const policy = fridaCapturePolicySchema.safeParse( + input.captureLimits ?? {}, + ); + if (!policy.success) + return err( + analysisInputErrorFromIssues( + "start_frida_session", + policy.error.issues.map((issue) => ({ + ...issue, + path: ["capture_limits", ...issue.path], + })), + { capture_limits: input.captureLimits }, + ), + ); let manager: DeviceManager | undefined; let device: Device | undefined; let remoteHeld = false; @@ -339,6 +353,7 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { const id = randomUUID(); const managed: ManagedSession = { id, + captureLimits: policy.data, device, deviceManager: manager, remote: input.remote, @@ -527,6 +542,7 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { } const observation: FridaScriptObservation = { scriptId, + captureLimits: managed.captureLimits, sourceKind: source.sourceKind, sourcePath: source.sourceKind === "file" ? source.path : null, sourceSha256, @@ -1079,6 +1095,7 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { message: FridaMessage, data: Buffer | null, ): void { + const limits = managed.captureLimits; const payload = message.payload; const secrets = [managed.remote?.token, managed.remote?.certificate].filter( (value): value is string => value !== undefined && value.length > 0, @@ -1095,8 +1112,9 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { try { payloadResult = boundedJsonValue( payload, - MAX_SESSION_CAPTURE_BYTES - MAX_MESSAGE_METADATA_BYTES, + Math.max(0, limits.session_bytes - MAX_MESSAGE_METADATA_BYTES), secrets, + limits, ); } catch { payloadResult = { @@ -1112,35 +1130,25 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { const payloadBytes = payloadResult?.bytes ?? 0; const remainingMessageBytes = Math.max( 0, - MAX_SESSION_CAPTURE_BYTES - - MAX_MESSAGE_METADATA_BYTES - - payloadBytes - - 4096, - ); - const messageBinaryLimit = Math.min( - MAX_BINARY_MESSAGE_BYTES, - Math.floor((remainingMessageBytes * 3) / 4), + limits.session_bytes - MAX_MESSAGE_METADATA_BYTES - payloadBytes - 4096, ); + const messageBinaryLimit = Math.floor((remainingMessageBytes * 3) / 4); if (secrets.some((secret) => data.includes(secret, 0, "utf8"))) { binaryResult = { data_redacted: true }; } else if (data.byteLength > messageBinaryLimit) { - const sessionConstrained = - messageBinaryLimit < MAX_BINARY_MESSAGE_BYTES; binaryResult = { data_truncated: true, data_bytes: data.byteLength, - data_capture_limit: sessionConstrained - ? "session_bytes" - : "binary_bytes", + data_capture_limit: "session_bytes", }; - if (sessionConstrained) markMessageTruncated(managed, scriptId); + markMessageTruncated(managed, scriptId); } else { binaryResult = { data_base64: data.toString("base64") }; } } const type = boundedRedactedString( message.type, - MAX_MESSAGE_TYPE_BYTES, + limits.message_type_bytes, secrets, ); const description = @@ -1148,19 +1156,23 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { ? undefined : boundedRedactedString( message.description, - MAX_MESSAGE_TEXT_BYTES, + limits.message_text_bytes, secrets, ); const stack = message.stack === undefined ? undefined - : boundedRedactedString(message.stack, MAX_MESSAGE_TEXT_BYTES, secrets); + : boundedRedactedString( + message.stack, + limits.message_text_bytes, + secrets, + ); const fileName = message.fileName === undefined ? undefined : boundedRedactedString( message.fileName, - MAX_MESSAGE_TEXT_BYTES, + limits.message_text_bytes, secrets, ); const observation: JsonValue = { @@ -1211,13 +1223,13 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort { ...binaryResult, }; const charge = Buffer.byteLength(JSON.stringify(observation)); - if (charge > MAX_SESSION_CAPTURE_BYTES) { + if (charge > limits.session_bytes) { markMessageTruncated(managed, scriptId); return; } while ( managed.messages.size > 0 && - managed.capturedBytes + charge > MAX_SESSION_CAPTURE_BYTES + managed.capturedBytes + charge > limits.session_bytes ) { const removed = managed.messages.shift(); if (removed !== undefined) { @@ -1451,9 +1463,10 @@ const boundedJsonValue = ( input: unknown, maximumBytes: number, secrets: readonly string[], + limits: FridaCapturePolicy, ): BoundedJson & { readonly bytes: number } => { const seen = new WeakSet(); - let remainingNodes = MAX_CAPTURE_JSON_NODES; + let remainingNodes = limits.json_nodes; const visit = ( value: unknown, budget: number, @@ -1461,7 +1474,7 @@ const boundedJsonValue = ( ): BoundedJson => { remainingNodes -= 1; if (remainingNodes < 0) return clipped("json_nodes"); - if (depth > MAX_JSON_DEPTH) return clipped("json_depth"); + if (depth > limits.json_depth) return clipped("json_depth"); if (budget < 8) return clipped("payload_bytes"); if (value === null) return { value: null, truncated: false }; if (typeof value === "boolean" || typeof value === "number") { @@ -1596,6 +1609,7 @@ const sessionObservation = ( managed: ManagedSession, ): FridaSessionObservation => ({ sessionId: managed.id, + captureLimits: managed.captureLimits, deviceId: managed.device.id, target: managed.target, pid: managed.pid, diff --git a/src/frida/fridaCaptureLimits.ts b/src/frida/fridaCaptureLimits.ts index a8cbc31d5d..5a6c4de78a 100644 --- a/src/frida/fridaCaptureLimits.ts +++ b/src/frida/fridaCaptureLimits.ts @@ -1,14 +1,12 @@ -/** Fixed Frida capture caps. This release has no runtime override. */ -export const MAX_SESSION_CAPTURE_BYTES = 16 * 1024 * 1024; +import { + DEFAULT_FRIDA_CAPTURE_POLICY, + fridaCapturePolicySchema, + FRIDA_CAPTURE_MAX_JSON_DEPTH, + type FridaCapturePolicy, +} from "../domain/fridaCapturePolicy.js"; + +/** Reserved envelope bytes, independent of the caller's payload allowance. */ export const MAX_MESSAGE_METADATA_BYTES = 128 * 1024; -export const MAX_MESSAGE_TYPE_BYTES = 4 * 1024; -export const MAX_MESSAGE_TEXT_BYTES = 32 * 1024; -/** Bound the in-memory object graph as well as its serialized JSON byte size. */ -export const MAX_CAPTURE_JSON_NODES = 250_000; -export const MAX_JSON_DEPTH = 64; -export const MAX_BINARY_MESSAGE_BYTES = Math.floor( - ((MAX_SESSION_CAPTURE_BYTES - 1024 * 1024) * 3) / 4, -); /** Why a payload or binary field was shortened. Caller payloads cannot set these. */ export const FRIDA_OWNED_CAPTURE_LIMITS = [ @@ -32,26 +30,34 @@ const namedLimit = (value: unknown): FridaOwnedCaptureLimit | undefined => ? (value as FridaOwnedCaptureLimit) : undefined; -const noOverride = "This release has no runtime override."; - -const limitationFor = (limit: FridaOwnedCaptureLimit): string => { +const limitationFor = ( + limit: FridaOwnedCaptureLimit, + policy: FridaCapturePolicy, +): string => { + const recover = (field: string, value: number, unit: string): string => + `Frida capture reached capture_limits.${field} (${String(value)} ${unit}). Select a larger capture_limits.${field} before starting a new explicitly chosen session or instrument_with_frida experiment; no REA restart is required. Already discarded observations cannot be recovered. Session bytes and MCP/client delivery budgets remain independent.`; switch (limit) { case "session_bytes": - return `Frida session message capture reached its ${String(MAX_SESSION_CAPTURE_BYTES)} byte budget. ${noOverride}`; - case "message_type": - return `A Frida message type reached its ${String(MAX_MESSAGE_TYPE_BYTES)} byte cap. ${noOverride}`; - case "message_text": - return `A Frida message text field reached its ${String(MAX_MESSAGE_TEXT_BYTES)} byte cap. ${noOverride}`; case "payload_bytes": - return `A Frida message payload reached its ${String(MAX_SESSION_CAPTURE_BYTES - MAX_MESSAGE_METADATA_BYTES)} byte cap. ${noOverride}`; - case "json_nodes": - return `A Frida message payload reached its ${String(MAX_CAPTURE_JSON_NODES)} JSON node cap. ${noOverride}`; - case "json_depth": - return `A Frida message payload reached its ${String(MAX_JSON_DEPTH)} level nesting cap. ${noOverride}`; case "binary_bytes": - return `A Frida binary message reached its ${String(MAX_BINARY_MESSAGE_BYTES)} byte cap. ${noOverride}`; + return recover( + "session_bytes", + policy.session_bytes, + "bytes including JSON/base64 expansion and metadata", + ); + case "message_type": + return recover("message_type_bytes", policy.message_type_bytes, "bytes"); + case "message_text": + return recover("message_text_bytes", policy.message_text_bytes, "bytes"); + case "json_nodes": + return recover("json_nodes", policy.json_nodes, "nodes"); + case "json_depth": + return policy.json_depth < FRIDA_CAPTURE_MAX_JSON_DEPTH + ? recover("json_depth", policy.json_depth, "levels") + + ` The maximum is ${String(FRIDA_CAPTURE_MAX_JSON_DEPTH)} levels, reserving the surrounding Evidence envelope.` + : `Frida capture reached capture_limits.json_depth (${String(policy.json_depth)} levels), the shared JSON representation allowance after reserving the Evidence envelope. No higher depth is supported. Select a script that sends shallower payloads for a new explicitly chosen experiment. Already discarded observations cannot be recovered.`; case "unrepresentable": - return `A Frida message value could not be retained in full. ${noOverride}`; + return "A Frida value could not be represented as JSON. Increasing capture budgets cannot recover cyclic, non-finite or unsupported values. Already discarded observations cannot be recovered."; } }; @@ -90,8 +96,14 @@ const collectOwnedCaps = ( /** Name only the caps that REA itself recorded while dropping data. */ export const fridaCaptureLimitations = (result: unknown): readonly string[] => { const caps = new Set(); + const selected = + typeof result === "object" && result !== null && "capture_limits" in result + ? fridaCapturePolicySchema.safeParse(result.capture_limits) + : undefined; + const policy = + selected?.success === true ? selected.data : DEFAULT_FRIDA_CAPTURE_POLICY; collectOwnedCaps(result, caps); return FRIDA_OWNED_CAPTURE_LIMITS.flatMap((limit) => - caps.has(limit) ? [limitationFor(limit)] : [], + caps.has(limit) ? [limitationFor(limit, policy)] : [], ); }; diff --git a/src/gdb/GdbSessionManager.ts b/src/gdb/GdbSessionManager.ts index cdeff4be34..5e20b1d339 100644 --- a/src/gdb/GdbSessionManager.ts +++ b/src/gdb/GdbSessionManager.ts @@ -18,7 +18,10 @@ import { type AnalysisErrorOptions, } from "../domain/analysisErrorBase.js"; import { createEvidence, type Evidence } from "../domain/evidence.js"; -import type { GdbCommandResult } from "../domain/commandObservation.js"; +import type { + GdbCommandResult, + GdbOutputLimit, +} from "../domain/commandObservation.js"; import { DEBUGGER_CONTROL_REPLY_BYTES } from "../domain/analysisResourceLimits.js"; import { err, ok, type Result } from "../domain/result.js"; import { @@ -88,8 +91,20 @@ const MAX_RECENT_MI_LINE_BYTES = 64 * 1024; const MAX_DIAGNOSTIC_BYTES = 64 * 1024; const RECENT_RECORD_TRUNCATION_MARKER = "[recent record truncated]"; -const gdbRetainedOutputLimitation = (output: DebuggerAdmissionLimit): string => - `Retained MI records reached the ${String(output.value)} byte command-output cap or the ${String(MAX_COMMAND_OUTPUT_RECORDS)} record cap. ${debuggerAdmissionRecovery(output)} The record cap has no separate setting. Output completeness is unknown.`; +const gdbRetainedOutputLimitations = ( + result: GdbCommandResult, + output: DebuggerAdmissionLimit, +): string[] => + (result.output_limits ?? []).map((limit) => { + switch (limit.kind) { + case "command_bytes": + return `Retained MI records reached the ${String(limit.maximum)} byte command-output cap. ${debuggerAdmissionRecovery(output)} Retained output is partial.`; + case "command_records": + return `Retained MI records reached the ${String(limit.maximum)} record cap. This release has no runtime override for that count; REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES does not raise it. Narrow the command's output on the next call. Retained output is partial.`; + case "mi_line_bytes": + return `GDB emitted a stream record exceeding the ${String(limit.maximum)} byte MI line limit; that record was dropped. This release has no runtime override for that protocol limit; REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES does not raise it. Select a command that emits smaller records on the next call. Retained output is partial.`; + } + }); const cancelledWithCallerReason = ( operation: string, @@ -123,7 +138,7 @@ interface PendingCommand { outputBytes: number; /** Caller output uses the setting. Startup and detach replies use the control budget. */ readonly outputBudgetBytes: number; - outputTruncated: boolean; + readonly outputLimits: Set; outputInvalidUtf8: boolean; terminalRecord: string | undefined; completionToken: number | undefined; @@ -133,6 +148,26 @@ interface PendingCommand { timer: ReturnType; } +/** Preserve the producer's exact exhaustion facts through every completion path. */ +const gdbOutputCapture = ( + pending: PendingCommand, +): Pick => ({ + output_truncated: pending.outputLimits.size > 0, + ...(pending.outputLimits.size === 0 + ? {} + : { + output_limits: [...pending.outputLimits].map((kind) => ({ + kind, + maximum: + kind === "command_bytes" + ? pending.outputBudgetBytes + : kind === "command_records" + ? MAX_COMMAND_OUTPUT_RECORDS + : MAX_MI_LINE_BYTES, + })), + }), +}); + class GdbCommandTimeoutError extends AnalysisTimeoutError { constructor( readonly partial: GdbCommandResult, @@ -1258,6 +1293,9 @@ export class GdbSessionManager { target: result.target, log: result.log, output_truncated: result.output_truncated, + ...(result.output_limits === undefined + ? {} + : { output_limits: result.output_limits }), ...(result.output_invalid_utf8 ? { output_invalid_utf8: true as const } : {}), @@ -1271,6 +1309,9 @@ export class GdbSessionManager { target: result.target, log: result.log, output_truncated: result.output_truncated, + ...(result.output_limits === undefined + ? {} + : { output_limits: result.output_limits }), ...(result.output_invalid_utf8 ? { output_invalid_utf8: true as const } : {}), @@ -1285,9 +1326,7 @@ export class GdbSessionManager { ] : []), "MI records are returned as decoded text; their semantics depend on GDB version and target.", - ...(result.output_truncated - ? [gdbRetainedOutputLimitation(this.#commandOutput)] - : []), + ...gdbRetainedOutputLimitations(result, this.#commandOutput), ...(result.output_invalid_utf8 ? [ "MI stream bytes were not valid UTF-8; returned text contains replacement characters.", @@ -1327,9 +1366,10 @@ export class GdbSessionManager { "The active GDB inferior identity is not established by the debugger session identifier.", limitations: [ "GDB exited before returning a correlated MI result record; only records retained for the active command are returned raw.", - ...(result.output_truncated - ? [gdbRetainedOutputLimitation(this.#commandOutput)] - : []), + ...gdbRetainedOutputLimitations( + result, + this.#commandOutput, + ), ], }, ); @@ -1343,7 +1383,9 @@ export class GdbSessionManager { { cause, partialObservation: evidence, - ...(result.output_truncated + ...(result.output_limits?.some( + (limit) => limit.kind === "command_bytes", + ) ? { diagnostics: debuggerAdmissionDiagnostics( this.#commandOutput, @@ -1670,7 +1712,7 @@ export class GdbSessionManager { session.bufferTailBytes = 0; session.bufferBytes = 0; for (const pending of session.pending.values()) - pending.outputTruncated = true; + pending.outputLimits.add("mi_line_bytes"); session.recentRecordsTruncated = true; if (newline < 0) { session.discardingOversizedStreamRecord = true; @@ -1728,7 +1770,12 @@ export class GdbSessionManager { pending.lines.push(line); pending.outputBytes += lineBytes; pending.outputInvalidUtf8 ||= invalidUtf8; - } else pending.outputTruncated = true; + } else { + if (pending.outputBytes + lineBytes > pending.outputBudgetBytes) + pending.outputLimits.add("command_bytes"); + if (pending.lines.length >= MAX_COMMAND_OUTPUT_RECORDS) + pending.outputLimits.add("command_records"); + } } if (line === "(gdb)" || line === "(gdb) ") { // Shell stdout can precede its result token without an LF. Never scan @@ -1793,7 +1840,7 @@ export class GdbSessionManager { pending.terminalRecord ?? (token === pending.completionToken ? "" : line), ...snapshotGdbCommandRecords(pending.lines), - output_truncated: pending.outputTruncated, + ...gdbOutputCapture(pending), ...(pending.outputInvalidUtf8 ? { output_invalid_utf8: true as const } : {}), @@ -1837,7 +1884,7 @@ export class GdbSessionManager { command: pending.evidenceCommand, mi: "", ...snapshotGdbCommandRecords(pending.lines), - output_truncated: pending.outputTruncated, + ...gdbOutputCapture(pending), ...(pending.outputInvalidUtf8 ? { output_invalid_utf8: true as const } : {}), @@ -2030,7 +2077,7 @@ export class GdbSessionManager { process_exit_observed: true, process_exit_code: code, process_exit_signal: signalCode, - output_truncated: pending.outputTruncated, + ...gdbOutputCapture(pending), ...(pending.outputInvalidUtf8 ? { output_invalid_utf8: true as const } : {}), @@ -2096,7 +2143,7 @@ export class GdbSessionManager { command: pending.evidenceCommand, mi: "", ...snapshotGdbCommandRecords(pending.lines), - output_truncated: pending.outputTruncated, + ...gdbOutputCapture(pending), ...(pending.outputInvalidUtf8 ? { output_invalid_utf8: true as const } : {}), @@ -2129,6 +2176,7 @@ export class GdbSessionManager { "The active GDB inferior identity is not established by the debugger session identifier.", limitations: [ `The GDB command exceeded its ${String(this.#commandDeadline.value)} ms deadline (deadlineExpired). ${debuggerAdmissionRecovery(this.#commandDeadline)} Retained MI and stream records are partial and command completion is unknown.`, + ...gdbRetainedOutputLimitations(partial, this.#commandOutput), "Cleanup of the owned GDB process was requested because output correlation could no longer be guaranteed.", "The unrestricted GDB console can access local shell, filesystem, network, and target controls.", ], @@ -2192,7 +2240,7 @@ export class GdbSessionManager { outputBudgetBytes: command.startsWith("-interpreter-exec console ") ? session.maxCommandOutputBytes : session.controlReplyBytes, - outputTruncated: false, + outputLimits: new Set(), outputInvalidUtf8: false, terminalRecord: undefined, completionToken: undefined, @@ -2204,7 +2252,7 @@ export class GdbSessionManager { command: pending.evidenceCommand, mi: "", ...snapshotGdbCommandRecords(pending.lines), - output_truncated: pending.outputTruncated, + ...gdbOutputCapture(pending), ...(pending.outputInvalidUtf8 ? { output_invalid_utf8: true as const } : {}), @@ -2223,6 +2271,7 @@ export class GdbSessionManager { "The active GDB inferior identity is not established by the debugger session identifier.", limitations: [ "The GDB command was cancelled; retained MI records are partial and command completion is unknown.", + ...gdbRetainedOutputLimitations(partial, this.#commandOutput), "Shutdown of the owned GDB process was requested because output correlation could no longer be guaranteed.", "The unrestricted GDB console can access local shell, filesystem, network, and target controls.", ], diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index c129ed4f43..83fa0320f2 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -61,6 +61,7 @@ const sharedAnalysisSemantics = { function_references: "complete-body-and-entry-reference-manager-v3", call_relationship_coverage: "observed-call-inventory-and-discovery-v7", function_dossier_coverage: "observed-facets-and-call-relationships-v1", + native_value_flow_projection: "accepted-runtime-budgets-v1", value_flow_effect_scope: "pcode-load-store-explicit-v1", string_inventory_evidence: "defined-data-coverage-v1", inventory_search_projection: "provider-inventory-facts-v1", diff --git a/src/ghidra/GhidraClient.ts b/src/ghidra/GhidraClient.ts index 4447ec0b26..2973392ac3 100644 --- a/src/ghidra/GhidraClient.ts +++ b/src/ghidra/GhidraClient.ts @@ -15,6 +15,7 @@ import type { ProgressReporter } from "../application/ProgressReporter.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import type { NativeValueFlowLimits } from "../domain/native/nativeValueFlow.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { providerRetryAction, @@ -400,6 +401,8 @@ export class GhidraClient { readonly signal?: AbortSignal; readonly progress?: ProgressReporter; readonly decompileTimeoutSeconds?: number | null; + readonly decompileMaxPayloadMiB?: number | null; + readonly valueFlowLimits?: NativeValueFlowLimits; } = {}, ): Promise> { const started = await this.start(options.signal, options.progress); @@ -423,6 +426,8 @@ export class GhidraClient { options: { readonly signal?: AbortSignal; readonly decompileTimeoutSeconds?: number | null; + readonly decompileMaxPayloadMiB?: number | null; + readonly valueFlowLimits?: NativeValueFlowLimits; }, ): Promise> { const pattern = @@ -445,6 +450,12 @@ export class GhidraClient { ...(options.decompileTimeoutSeconds === undefined ? {} : { decompileTimeoutSeconds: options.decompileTimeoutSeconds }), + ...(options.valueFlowLimits === undefined + ? {} + : { valueFlowLimits: options.valueFlowLimits }), + ...(options.decompileMaxPayloadMiB === undefined + ? {} + : { decompileMaxPayloadMiB: options.decompileMaxPayloadMiB }), }, ); } diff --git a/src/ghidra/GhidraClientTypes.ts b/src/ghidra/GhidraClientTypes.ts index 79b814b117..404382864b 100644 --- a/src/ghidra/GhidraClientTypes.ts +++ b/src/ghidra/GhidraClientTypes.ts @@ -7,6 +7,7 @@ import type { GhidraLauncher } from "./GhidraLauncher.js"; import type { GhidraSessionError } from "./GhidraSessionError.js"; import type { GhidraSessionInfo } from "./GhidraSessionValues.js"; import type { GhidraTransportKind } from "./GhidraTransport.js"; +import type { NativeValueFlowLimits } from "../domain/native/nativeValueFlow.js"; /** Result of opening and authenticating one headless Ghidra session. */ export type GhidraStartResult = Result; @@ -14,6 +15,8 @@ export type GhidraStartResult = Result; /** Cancellation and deadline controls shared by authenticated requests. */ export interface GhidraRequestOptions { readonly decompileTimeoutSeconds?: number | null; + readonly decompileMaxPayloadMiB?: number | null; + readonly valueFlowLimits?: NativeValueFlowLimits; readonly signal?: AbortSignal; /** Internal startup/shutdown deadline; analysis calls have no fixed deadline. */ readonly timeoutMs?: number; diff --git a/src/ghidra/GhidraClientWire.test.ts b/src/ghidra/GhidraClientWire.test.ts index c1929bbe4e..c4a2a566bc 100644 --- a/src/ghidra/GhidraClientWire.test.ts +++ b/src/ghidra/GhidraClientWire.test.ts @@ -141,16 +141,30 @@ describe("GhidraWire request ceiling", () => { }); it.each([null, 7])( - "sends the accepted decompile budget %s in the request envelope", + "sends accepted decompile and value-flow budgets with deadline %s in the request envelope", async (budget) => { const { wire, writes, pending } = harness(limit(8192)); const result = wire.request( "procedure_pseudo_code", { procedure: "0x1000" }, - { decompileTimeoutSeconds: budget }, + { + decompileTimeoutSeconds: budget, + decompileMaxPayloadMiB: budget, + valueFlowLimits: { + max_operations: 6000, + max_inputs_per_operation: 128, + max_edges: 24000, + }, + }, ); expect(JSON.parse(writes[0]!)).toMatchObject({ decompile_timeout_seconds: budget, + decompile_max_payload_mib: budget, + value_flow_limits: { + max_operations: 6000, + max_inputs_per_operation: 128, + max_edges: 24000, + }, }); pending.settle(1, ok("pseudocode")); expect(await result).toEqual(ok("pseudocode")); diff --git a/src/ghidra/GhidraClientWire.ts b/src/ghidra/GhidraClientWire.ts index 5e255dc67a..7981cda32a 100644 --- a/src/ghidra/GhidraClientWire.ts +++ b/src/ghidra/GhidraClientWire.ts @@ -71,6 +71,12 @@ export class GhidraWire { ...(options.decompileTimeoutSeconds === undefined ? {} : { decompile_timeout_seconds: options.decompileTimeoutSeconds }), + ...(options.valueFlowLimits === undefined + ? {} + : { value_flow_limits: options.valueFlowLimits }), + ...(options.decompileMaxPayloadMiB === undefined + ? {} + : { decompile_max_payload_mib: options.decompileMaxPayloadMiB }), }; const payloadBytes = jsonValueByteLength(payloadValue); // Refuse before registering the operation and before any socket write, so diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 07679732ff..b3bdcdbbac 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -64,6 +64,8 @@ import { } from "../domain/ghidraValues.fixture.js"; import { parseConfig } from "../config/parseConfig.js"; +import { runtimeAnalysisConfig } from "../config/runtimeAnalysisSettings.js"; +import { AnalysisSettings } from "../application/AnalysisSettings.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { parseExecutableHeader } from "../domain/binaryTarget.js"; import { elf, thinMach } from "../domain/binaryTarget.fixture.js"; @@ -1174,6 +1176,89 @@ describe("Ghidra startup configuration", () => { }); }); +it("reobserves value-flow budgets on the same session while ordinary address lookups stay cached", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-ghidra-flow-budgets-")); + onTestFinished(() => rm(root, { recursive: true, force: true })); + const path = join(root, "fixture.elf"); + await writeFile(path, elf(2, 1, 62)); + const parsed = parseConfig({ GHIDRA_INSTALL_DIR: INSTALL }); + if (!parsed.ok) throw parsed.error; + const settings = new AnalysisSettings({}); + const received: number[] = []; + let ordinaryReads = 0; + let clients = 0; + const ghidra = new GhidraProvider( + runtimeAnalysisConfig(parsed.value, settings.read), + silentLogger, + {}, + installationHost(), + (options) => { + clients += 1; + return { + start: async () => { + const info = sessionInfo(); + return ok({ + ...info, + profile_digest: options.profileDigest, + target: { ...info.target, sha256: options.targetSha256 }, + }); + }, + callTool: async (operation, _parameters, controls) => { + if (operation === "analyze_function") { + received.push(controls?.valueFlowLimits?.max_operations ?? -1); + return ok({ value: ghidraFunctionDossier(), limitations: [] }); + } + if (operation === "address_name") { + ordinaryReads += 1; + return ok("fixture_main"); + } + throw new Error(`Unexpected operation: ${operation}`); + }, + close: async () => ok(null), + }; + }, + ); + const session = composeBinarySession(new AnalysisProviderRegistry([ghidra])); + try { + const opened = await session.open(path, { providerId: "ghidra" }); + if (!opened.ok) throw opened.error; + for (const value of [1, 6000, 2]) { + const updated = settings.update({ + changes: [ + { + setting: "ghidra.value_flow_max_operations", + action: "set", + value, + }, + ], + }); + if (!updated.ok) throw updated.error; + const result = await settings.run(() => + session.execute("analyze_function", { procedure: "0x401000" }), + ); + if (!result.ok) throw result.error; + expect(result.value.cachePolicy).toBe("live"); + } + expect(received).toEqual([1, 6000, 2]); + for (let attempt = 0; attempt < 2; attempt++) { + const result = await session.execute("address_name", { + document: "fixture", + address: "0x401000", + }); + if (!result.ok) throw result.error; + } + expect(ordinaryReads).toBe(1); + expect(clients).toBe(1); + expect( + session + .exportEvidenceBundle() + .records.some((record) => record.operation === "analyze_function"), + ).toBe(true); + } finally { + await expect(session.close()).resolves.toMatchObject({ ok: true }); + } +}); + describe("Ghidra client projection", () => { it("commits exact provider, isolation, and resource semantics", async () => { const toolCalls: Array<{ @@ -1235,6 +1320,7 @@ describe("Ghidra client projection", () => { compiler_spec_id: "auto-default", call_relationship_coverage: "observed-call-inventory-and-discovery-v7", function_dossier_coverage: "observed-facets-and-call-relationships-v1", + native_value_flow_projection: "accepted-runtime-budgets-v1", }); const result = await ghidra @@ -1257,7 +1343,15 @@ describe("Ghidra client projection", () => { { operation: "list_procedures", input: { document: null }, - options: { decompileTimeoutSeconds: null }, + options: { + decompileTimeoutSeconds: null, + decompileMaxPayloadMiB: null, + valueFlowLimits: { + max_operations: 3000, + max_inputs_per_operation: 64, + max_edges: 12000, + }, + }, }, ]); expect(result).toMatchObject({ diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index 41d4e99af2..f122004859 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -239,12 +239,18 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( : {}), available: true, reason: null, - // Project writes are private import machinery. Analysis reads are bound - // to the artifact/profile; session annotations disable snapshot replay. + // Dossier projections consume runtime budgets. Reobserve even an imported + // snapshot so a raised or lowered budget cannot replay stale flow coverage. + // Ghidra retains its own native decompiler cache. ...(GHIDRA_MUTATING_OPERATIONS.has(operation) || operation === "list_documents" ? {} - : { cachePolicy: "snapshot" as const }), + : { + cachePolicy: + operation === "analyze_function" + ? ("live" as const) + : ("snapshot" as const), + }), effects: Object.freeze({ mutatesArtifact: GHIDRA_MUTATING_OPERATIONS.has(operation), launchesProcess: true, diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index a99294735f..0e2e1e34fa 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -7,7 +7,12 @@ import { ghidraProjectHistoryEvidence } from "./GhidraProjectHistory.js"; import type { Evidence, EvidenceLocation } from "../domain/evidence.js"; import { ghidraProjectCommitmentSchema } from "./GhidraProjectSnapshot.js"; import { analysisSettingLimit } from "../domain/analysisSettingDefinitions.js"; -import { MAX_GHIDRA_DECOMPILE_TIMEOUT_SECONDS } from "../domain/analysisResourceLimits.js"; +import { + MAX_GHIDRA_DECOMPILE_TIMEOUT_SECONDS, + DEFAULT_GHIDRA_VALUE_FLOW_MAX_OPERATIONS, + DEFAULT_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION, + DEFAULT_GHIDRA_VALUE_FLOW_MAX_EDGES, +} from "../domain/analysisResourceLimits.js"; import { z } from "zod"; import { analysisClientFailureForOperation } from "../providers/AnalysisClientFailure.js"; import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; @@ -163,8 +168,6 @@ export type GhidraProviderClientFactory = ( > >; -type GhidraProviderClientInstance = ReturnType; - interface GhidraExecutionMetadata { readonly sourceEvidence?: readonly Evidence[]; readonly cachePolicy?: "live"; @@ -860,6 +863,12 @@ export const createGhidraProviderClient = (input: { ) => { return createAnalysisExecution(normalized, committedProfile.provider, { ...observation, + ...(operation === "analyze_function" + ? { + requiresSourceEvidence: true as const, + cachePolicy: "live" as const, + } + : {}), ...(projectHistory === undefined ? {} : { @@ -983,6 +992,18 @@ export const createGhidraProviderClient = (input: { if (!started.ok) return started; const called = await client.callTool(operation, input.value, { decompileTimeoutSeconds: config.ghidraDecompileTimeoutSeconds ?? null, + decompileMaxPayloadMiB: config.ghidraDecompileMaxPayloadMiB ?? null, + valueFlowLimits: { + max_operations: + config.ghidraValueFlowMaxOperations ?? + DEFAULT_GHIDRA_VALUE_FLOW_MAX_OPERATIONS, + max_inputs_per_operation: + config.ghidraValueFlowMaxInputsPerOperation ?? + DEFAULT_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION, + max_edges: + config.ghidraValueFlowMaxEdges ?? + DEFAULT_GHIDRA_VALUE_FLOW_MAX_EDGES, + }, ...(options?.signal === undefined ? {} : { signal: options.signal }), ...(options?.progress === undefined ? {} diff --git a/src/ghidra/GhidraRawProfileDigest.test.ts b/src/ghidra/GhidraRawProfileDigest.test.ts index f4da6c1ce1..682576a8ab 100644 --- a/src/ghidra/GhidraRawProfileDigest.test.ts +++ b/src/ghidra/GhidraRawProfileDigest.test.ts @@ -8,12 +8,12 @@ import { GhidraProvider } from "./GhidraProvider.js"; // Digests commit to the current shared policy, including the instruction // status, load-image relocation-symbol, procedure search, byte-search and -// bit-field base-type and inventory-search revisions that prevent reuse of snapshots with earlier +// bit-field base-type, runtime value-flow budgets and inventory-search revisions that prevent reuse of snapshots with earlier // results. const RAW_DIGESTS = { - "6502": "540037ef007380e737a606cd14c402a9bdf055132cf44fb73d29ab1ec9f84af6", + "6502": "51160ff6e94cb32eaeb87ac248915afb3bc17e33b11a1a101222344dd3907c6f", "x86-disabled-analyzer": - "8498e58d5a1147cdfe7dad2b486b4b4087efc76944ee1902c9af9d0c0da56237", + "94e93961a699cf9250844d53fe95802c72063c08656fc308b3fed523dd86ea1c", } as const; const configFor = (environment: NodeJS.ProcessEnv) => { @@ -51,6 +51,7 @@ describe("raw-binary analysis profile digest", () => { instruction_status: "unlisted-and-containing-code-data-v3", inventory_search_projection: "provider-inventory-facts-v1", value_flow_effect_scope: "pcode-load-store-explicit-v1", + native_value_flow_projection: "accepted-runtime-budgets-v1", byte_search_boundary: "contiguous-eligible-initialized-blocks-v1", byte_search_coverage: "separate-loaded-overlays-v1", data_type_observations: "bit-field-base-and-lookup-v1", @@ -77,6 +78,7 @@ describe("raw-binary analysis profile digest", () => { expect(resolved.value.profile?.parameters).toMatchObject({ inventory_search_projection: "provider-inventory-facts-v1", value_flow_effect_scope: "pcode-load-store-explicit-v1", + native_value_flow_projection: "accepted-runtime-budgets-v1", byte_search_coverage: "separate-loaded-overlays-v1", analysis_options: { disabled_analyzers: ["Aggressive Instruction Finder"], diff --git a/src/ghidra/GhidraSnapshotCache.test.ts b/src/ghidra/GhidraSnapshotCache.test.ts index b8d848a081..d396e4e504 100644 --- a/src/ghidra/GhidraSnapshotCache.test.ts +++ b/src/ghidra/GhidraSnapshotCache.test.ts @@ -224,7 +224,11 @@ describe("Ghidra relationship coverage replay", () => { expect( (await initial.session.execute("analyze_function", parameters)).ok, ).toBe(true); - expect(initial.calls).toEqual(["analyze_function", "analyze_function"]); + expect(initial.calls).toEqual([ + "analyze_function", + "analyze_function", + "analyze_function", + ]); await initial.session.close(); const replay = fixture(); @@ -254,7 +258,7 @@ describe("Ghidra relationship coverage replay", () => { limited_directions: ["callers"], }, ] satisfies (RelationshipCoverage | undefined)[])( - "preserves relationship coverage %j through live execution, cache and snapshot replay", + "preserves relationship coverage %j as historical Evidence while reobserving new calls", async (coverage) => { const initial = fixture("12.1.4", coverage); await open(initial.session); @@ -267,13 +271,17 @@ describe("Ghidra relationship coverage replay", () => { if (!execution.ok) throw execution.error; expect(execution.value.relationshipCoverage).toEqual(coverage); } - expect(initial.calls).toEqual(["analyze_function"]); + expect(initial.calls).toEqual(["analyze_function", "analyze_function"]); const snapshot = initial.session.exportAnalysisSnapshot(); if (!snapshot.ok) throw snapshot.error; + expect(snapshot.value.entries).toEqual([]); + expect( + snapshot.value.evidence_bundle.records[0]?.relationship_coverage, + ).toEqual(coverage ?? null); await initial.session.close(); const liveCoverage: RelationshipCoverage = { status: "unknown", - reason: "live fixture must not be replayed", + reason: "Current observation after import", }; const replay = fixture("12.1.4", liveCoverage); expect(replay.session.importAnalysisSnapshot(snapshot.value).ok).toBe( @@ -290,16 +298,15 @@ describe("Ghidra relationship coverage replay", () => { ); if (!execution.ok) throw execution.error; expect(execution.value.relationshipCoverage ?? undefined).toEqual( - coverage, + liveCoverage, ); - expect(execution.value.limitations).toEqual( - expect.arrayContaining([ - expect.stringContaining("local REA analysis snapshot"), - ]), - ); - expect(replay.calls).toEqual( - admittedOperation === undefined ? [] : [admittedOperation], + expect(execution.value.limitations.join(" ")).not.toContain( + "local REA analysis snapshot", ); + expect(replay.calls).toEqual([ + ...(admittedOperation === undefined ? [] : [admittedOperation]), + "analyze_function", + ]); await replay.session.close(); }, ); @@ -367,6 +374,11 @@ describe("Ghidra snapshot profile admission", () => { current: "complete-body-and-entry-reference-manager-v3", legacy: "complete-body-and-entry-reference-manager-v2", }, + { + parameter: "native_value_flow_projection", + current: "accepted-runtime-budgets-v1", + legacy: undefined, + }, { parameter: "call_relationship_coverage", current: "observed-call-inventory-and-discovery-v7", @@ -437,7 +449,6 @@ describe("Ghidra snapshot profile admission", () => { describe("Ghidra snapshot cache", () => { it.each([ { operation: "read_binary_inventory_summary", parameters: {} }, - { operation: "analyze_function", parameters: { procedure: "0x1000" } }, { operation: "procedure_pseudo_code", parameters: { procedure: "0x1000", document: "fixture" }, diff --git a/src/ghidra/GhidraTransport.ts b/src/ghidra/GhidraTransport.ts index 0b3b0c73d1..0d515f73df 100644 --- a/src/ghidra/GhidraTransport.ts +++ b/src/ghidra/GhidraTransport.ts @@ -196,6 +196,14 @@ const observationFailure = ( cause: unknown, ): GhidraSessionError => { const { cause: primary, origin } = artifactFilePrimaryFacts(cause); + // The native reader rejects expanded TCP records before JSON parsing. + // Keep that protocol refusal distinct from an unrelated observation failure. + const observationMessage = + endpoint.transport === "authenticated-loopback-tcp" && + cause instanceof ArtifactReaderFailure && + cause.reason === "limit" + ? "Ghidra TCP endpoint is invalid" + : "Ghidra endpoint observation failed"; let failure = origin === "owned-or-classified" && primary instanceof GhidraSessionError ? primary @@ -206,11 +214,7 @@ const observationFailure = ( }))("cancelled", "Ghidra endpoint observation was cancelled", cause, { cancellation: { reason: artifactFileCancellationCause(cause) }, }) - : endpointFailure( - endpoint, - "Ghidra endpoint observation failed", - cause, - ); + : endpointFailure(endpoint, observationMessage, cause); if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) { const cleanup = new ProviderCleanupError( "ghidra", diff --git a/src/go/GoBinaryImage.ts b/src/go/GoBinaryImage.ts index 57fc62254f..3ecc5a5615 100644 --- a/src/go/GoBinaryImage.ts +++ b/src/go/GoBinaryImage.ts @@ -10,9 +10,11 @@ import { type ZeroFillMapping, } from "../native/metadata/BinaryContainer.js"; import { readBinaryContainer } from "../native/metadata/readBinaryContainer.js"; -import type { - GoModuleMetadata, - GoStringLiterals, +import { + goAnalysisBudgets, + type GoAnalysisBudgets, + type GoBinary, + type GoStringLiterals, } from "../domain/go/goBinary.js"; import { parseGoModuleBytes } from "./GoModuleText.js"; import { @@ -24,27 +26,10 @@ import { recoverGoStringLiterals } from "./GoStringLiterals.js"; const MAGIC = Buffer.from("ff20476f206275696c64696e663a", "hex"); const MODULE_START = Buffer.from("3077af0c9274080241e1c107e6d618e6", "hex"); const MODULE_END = Buffer.from("f932433186182072008242104116d8f2", "hex"); -const MAX_BUILD_INFO_BYTES = 1024 * 1024; -const MAX_HEADER_SEARCH_BYTES = 256 * 1024 * 1024; -/** Exact file span of one embedded string's contents, excluding its length/header. */ -export interface GoStringLocation { - readonly offset: number; - readonly bytes: number; -} - -/** Observed compiler string and validated module body with its original framing bytes. */ -export interface GoBuildInfo { - readonly header_offset: number; - readonly encoding: "inline" | "pointer"; - readonly go_version: string | null; - readonly go_version_bytes_base64: string; - readonly module_text: string | null; - readonly module_bytes_base64: string; - readonly version_location: GoStringLocation; - readonly module_location: GoStringLocation; - readonly module: GoModuleMetadata; -} +/** Producer facts use the domain contract's source-value representation. */ +export type GoBuildInfo = NonNullable; +export type GoStringLocation = GoBuildInfo["version_location"]; /** Native image identity and optional Go build information; absence is not a language verdict. */ export interface GoBinaryImage { @@ -61,8 +46,8 @@ export interface GoBinaryImage { export interface ReadGoBinaryImageOptions { readonly includeStringLiterals?: boolean; - readonly maxStringLiterals?: number | undefined; readonly checkActive?: (() => void) | undefined; + readonly budgets?: Partial; } /** Inspect bounded executable bytes using native mappings, without loading or executing code. */ @@ -70,16 +55,12 @@ export const readGoBinaryImage = ( bytes: Buffer, options?: ReadGoBinaryImageOptions, ): GoBinaryImage => { + const budgets = goAnalysisBudgets(options?.budgets); const container = readBinaryContainer(bytes); const { format, architecture, bits, byte_order } = container; const stringLiterals = options?.includeStringLiterals === true - ? recoverGoStringLiterals( - bytes, - container, - options.maxStringLiterals, - options.checkActive, - ) + ? recoverGoStringLiterals(bytes, container, budgets, options.checkActive) : undefined; const stringMetadata = stringLiterals === undefined @@ -93,7 +74,7 @@ export const readGoBinaryImage = ( let selected: { search: GoBuildInfoSearchRange; header: number } | null = null; const searched = new Set(); - let remainingSearchBytes = MAX_HEADER_SEARCH_BYTES; + let remainingSearchBytes = budgets.max_header_search_bytes; const sectionNamesLimitation = container.section_names_limitation; for (const search of goBuildInfoSearch(bytes, container)) { // Identical file spans and alignment inspect the same candidate bytes. @@ -104,8 +85,8 @@ export const readGoBinaryImage = ( if (search.size > remainingSearchBytes) throw new BinaryResourceFailure( "search", - MAX_HEADER_SEARCH_BYTES, - "Go build-info discovery exceeds the cumulative 256 MiB header-search budget", + budgets.max_header_search_bytes, + `Go build-info discovery exceeds max_header_search_bytes=${String(budgets.max_header_search_bytes)}`, ); remainingSearchBytes -= search.size; const header = findHeader(bytes, search); @@ -161,14 +142,20 @@ export const readGoBinaryImage = ( reader, header + 32, sourceEnd - header - 32, + budgets.max_build_info_bytes, ); const moduleExtent = readInlineExtent( reader, versionExtent.next, sourceEnd - versionExtent.next, + budgets.max_build_info_bytes, ); verifyMapping(moduleExtent.next - header); - assertAggregateContentLimit(versionExtent.location, moduleExtent.location); + assertAggregateContentLimit( + versionExtent.location, + moduleExtent.location, + budgets.max_build_info_bytes, + ); version = { location: versionExtent.location, bytes: reader.range( @@ -192,23 +179,29 @@ export const readGoBinaryImage = ( "Go build-info pointer width must be 4 or 8", ); const pointerBits = ptrSize === 4 ? 32 : 64; - version = readPointerString( + const pointerStrings: PointerStringContext = { reader, mappings, - container.zero_fills, + zeroFills: container.zero_fills, + maximumBytes: budgets.max_build_info_bytes, + }; + version = readPointerString( + pointerStrings, reader.word(header + 16, pointerBits), pointerBits, ); const moduleField = header + 16 + ptrSize; module = readOptionalModuleString( - reader, - mappings, - container.zero_fills, + pointerStrings, reader.word(moduleField, pointerBits), moduleField, pointerBits, ); - assertAggregateContentLimit(version.location, module.location); + assertAggregateContentLimit( + version.location, + module.location, + budgets.max_build_info_bytes, + ); } if (version.bytes.length === 0) throw new BinaryFormatFailure( @@ -310,12 +303,13 @@ interface LocatedString { const assertAggregateContentLimit = ( version: GoStringLocation, module: GoStringLocation, + maximumBytes: number, ): void => { - if (version.bytes + module.bytes > MAX_BUILD_INFO_BYTES) + if (version.bytes + module.bytes > maximumBytes) throw new BinaryResourceFailure( "build-info", - MAX_BUILD_INFO_BYTES, - "Go build-info strings exceed the aggregate 1 MiB limit", + maximumBytes, + `Go build-info strings exceed max_build_info_bytes=${String(maximumBytes)}`, ); }; @@ -346,6 +340,7 @@ const readInlineExtent = ( reader: BinaryReader, offset: number, remaining: number, + maximumBytes: number, ): { readonly location: GoStringLocation; readonly next: number } => { let length = 0n; for (let index = 0; index < 10; index++) { @@ -362,11 +357,11 @@ const readInlineExtent = ( ); length |= BigInt(byte & 127) << BigInt(index * 7); if ((byte & 128) === 0) { - if (length > BigInt(MAX_BUILD_INFO_BYTES)) + if (length > BigInt(maximumBytes)) throw new BinaryResourceFailure( "build-info", - MAX_BUILD_INFO_BYTES, - "Go build-info string length exceeds the 1 MiB limit", + maximumBytes, + `Go build-info string length exceeds max_build_info_bytes=${String(maximumBytes)}`, ); const size = Number(length); if (size > remaining - index - 1) @@ -387,10 +382,15 @@ const readInlineExtent = ( ); }; +interface PointerStringContext { + readonly reader: BinaryReader; + readonly mappings: readonly FileMapping[]; + readonly zeroFills: readonly ZeroFillMapping[]; + readonly maximumBytes: number; +} + const readPointerString = ( - reader: BinaryReader, - mappings: readonly FileMapping[], - zeroFills: readonly ZeroFillMapping[], + { reader, mappings, zeroFills, maximumBytes }: PointerStringContext, address: bigint, pointerBits: 32 | 64, ): LocatedString => { @@ -398,11 +398,11 @@ const readPointerString = ( const header = mappedFileOffset(mappings, address, width * 2, zeroFills); const valueAddress = reader.word(header, pointerBits); const length = reader.word(header + width, pointerBits); - if (length > BigInt(MAX_BUILD_INFO_BYTES)) + if (length > BigInt(maximumBytes)) throw new BinaryResourceFailure( "build-info", - MAX_BUILD_INFO_BYTES, - "Go build-info pointer string length exceeds the 1 MiB limit", + maximumBytes, + `Go build-info pointer string length exceeds max_build_info_bytes=${String(maximumBytes)}`, ); const size = Number(length); // Zero-length Go strings may carry a zero data pointer. @@ -417,13 +417,16 @@ const readPointerString = ( }; const readOptionalModuleString = ( - reader: BinaryReader, - mappings: readonly FileMapping[], - zeroFills: readonly ZeroFillMapping[], + context: PointerStringContext, address: bigint, pointerField: number, pointerBits: 32 | 64, ): LocatedString => - isZeroFilledSpan(mappings, zeroFills, address, (pointerBits / 8) * 2) + isZeroFilledSpan( + context.mappings, + context.zeroFills, + address, + (pointerBits / 8) * 2, + ) ? { bytes: Buffer.alloc(0), location: { offset: pointerField, bytes: 0 } } - : readPointerString(reader, mappings, zeroFills, address, pointerBits); + : readPointerString(context, address, pointerBits); diff --git a/src/go/GoBinaryProvider.ts b/src/go/GoBinaryProvider.ts index 9404a04a21..1b3e32f42f 100644 --- a/src/go/GoBinaryProvider.ts +++ b/src/go/GoBinaryProvider.ts @@ -1,3 +1,4 @@ +import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; import { GO_BINARY_PROVIDER_IDENTITY } from "./GoBinaryProviderIdentity.js"; import { projectStableArtifactFailure, @@ -31,6 +32,9 @@ import { } from "../domain/analysisErrorCore.js"; import { goBinarySchema, + goAnalysisBudgets, + GO_ANALYSIS_BUDGET_MAXIMA, + type GoAnalysisBudgets, type GoBinary, type GoStringLiterals, type InspectGoBinaryInput, @@ -46,12 +50,19 @@ import { } from "../native/metadata/BinaryContainer.js"; const OPERATION = "inspect_go_binary"; -const INPUT_BYTES = 256 * 1024 * 1024; -const OUTPUT_BYTES = 8 * 1024 * 1024; + +const budgetRecovery = ( + parameter: keyof GoAnalysisBudgets, + value: number, +): string => + value < GO_ANALYSIS_BUDGET_MAXIMA[parameter] + ? `Increase ${parameter} above ${String(value)} (maximum ${String(GO_ANALYSIS_BUDGET_MAXIMA[parameter])}) on the next inspect_go_binary request (CLI --${parameter.replaceAll("_", "-")}); no restart is required.` + : `${parameter} is at its runtime representation maximum ${String(value)}; no larger value is supported. Preserve the complete selected artifact and report this boundary rather than repeating the unchanged request.`; const failureForPrimary = ( facts: ArtifactFilePrimaryFacts, path: string, + budgets: GoAnalysisBudgets, ): AnalysisError => { const { cause, origin } = facts; if (origin === "owned-or-classified" && cause instanceof AnalysisError) @@ -72,24 +83,36 @@ const failureForPrimary = ( return new AnalysisCancelledError(OPERATION, { cause: artifactFileCancellationCause(cause), }); - if (cause instanceof BinaryResourceFailure) + if (cause instanceof BinaryResourceFailure) { + const parameter = + cause.boundary === "search" + ? "max_header_search_bytes" + : cause.boundary === "build-info" + ? "max_build_info_bytes" + : undefined; return new AnalysisResourceConstraintError( OPERATION, cause.boundary === "search" ? "cpu" : "memory", `Selected artifact ${path}: ${cause.message}`, - { boundary: cause.boundary, maximum_bytes: cause.maximum_bytes }, + { + boundary: cause.boundary, + maximum_bytes: cause.maximum_bytes, + ...(parameter === undefined ? {} : { parameter, unit: "bytes" }), + }, { cause, - ...fixedAllowanceOptions( - cause.boundary === "search" - ? "The Go build-info header-search allowance" - : cause.boundary === "build-info" - ? "The Go build-info string allowance" - : "The Go structural decoding allowance", - cause.boundary === "search" ? "work" : "parser", - ), + ...(parameter === undefined + ? fixedAllowanceOptions( + "The Go structural decoding allowance", + "parser", + ) + : { + retryable: false, + remediationAction: `${cause.boundary === "search" ? "The Go build-info header-search allowance" : "The Go build-info string allowance"} is ${String(budgets[parameter])} bytes. ${budgetRecovery(parameter, budgets[parameter])} Other byte and report budgets are independent. Repeating the unchanged request cannot raise this allowance.`, + }), }, ); + } if (cause instanceof BinaryFormatFailure) return cause.kind === "unsupported" ? new AnalysisUnsupportedTargetError(OPERATION, path, cause.message, { @@ -111,10 +134,15 @@ const failureForPrimary = ( OPERATION, "file-size", cause.message, - { maximum_input_bytes: INPUT_BYTES }, + { + maximum_input_bytes: budgets.max_input_bytes, + parameter: "max_input_bytes", + unit: "bytes", + }, { cause, - remediationAction: `Select a suitable stable artifact within the reported ${String(INPUT_BYTES)}-byte input byte allowance. This release has no runtime override. Do not truncate required bytes; retrying the unchanged oversized input will fail again.`, + retryable: false, + remediationAction: `${budgetRecovery("max_input_bytes", budgets.max_input_bytes)} Report, build-info and structural allowances remain independent. Preserve the complete artifact; do not truncate required bytes.`, }, ); if (cause.reason === "path") @@ -150,9 +178,14 @@ const failureForPrimary = ( const failureFor = ( cause: unknown, path: string, + budgets: GoAnalysisBudgets, snapshot?: StableArtifactSnapshot, ): AnalysisError => { - const primary = failureForPrimary(artifactFilePrimaryFacts(cause), path); + const primary = failureForPrimary( + artifactFilePrimaryFacts(cause), + path, + budgets, + ); return projectStableArtifactFailure( cause, primary, @@ -163,66 +196,77 @@ const failureFor = ( ); }; -const outputLimit = (path: string) => +const outputLimit = (path: string, maximumBytes: number) => new AnalysisResourceConstraintError( OPERATION, "transport", `Complete Go metadata for ${path} exceeds the inline output budget; no partial success is returned.`, - { maximum_output_bytes: OUTPUT_BYTES }, - fixedAllowanceOptions("The Go complete-report allowance", "report"), + { + maximum_output_bytes: maximumBytes, + parameter: "max_report_bytes", + unit: "bytes", + }, + { + retryable: false, + remediationAction: `The Go complete-report allowance is ${String(maximumBytes)} bytes. ${budgetRecovery("max_report_bytes", maximumBytes)} This did not produce complete Evidence to page. MCP delivery limits remain independent.`, + }, ); /** Fixed reader boundaries plus unknowns left by the observed metadata. */ const limitationsFor = ( - info: GoBinaryImage["build_info"], - sectionNamesLimitation: string | undefined, + image: GoBinaryImage, stringLiterals: GoStringLiterals | undefined, - machoChainedFixups: boolean | undefined, -): string[] => [ - "Embedded build metadata is observed file content, not proof of compiler identity, dependency authenticity or runtime behavior.", - stringLiterals === undefined - ? "Only Go build information is inspected; function names, source mappings, type metadata and decompiled code are not recovered." - : "Only Go build information and inferred static (ptr, len) string-header candidates are inspected; function names, source mappings, type metadata and decompiled code are not recovered.", - stringLiterals === undefined - ? "Resource guards bound stable artifact reads and cumulative header searches to 256 MiB each, aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and complete metadata payloads to 8 MiB; these are not operating-system memory or CPU limits." - : "Resource guards bound stable artifact reads and cumulative header searches to 256 MiB each, aggregate embedded build-info strings to 1 MiB, structural table decoding to 16 MiB, static string-header scanning to 64 MiB, recovered string literals to 512 KiB (up to 4,096 bytes each) and complete metadata payloads to 8 MiB; these are not operating-system memory or CPU limits.", - ...(info !== null && (info.go_version === null || info.module_text === null) - ? [ - "Some embedded Go strings contain non-UTF8 bytes; undecodable text is null and exact source bytes are preserved as base64.", - ] - : []), - ...(info === null - ? [ - "No recognized Go build-info record was found in the inspected image; absent metadata does not establish that the binary is not Go.", - ] - : []), - ...(sectionNamesLimitation === undefined ? [] : [sectionNamesLimitation]), - ...(info !== null && info.module.main === null - ? [ - "Dependency coverage is unknown because no main-module record was decoded from the embedded module information, as in GOPATH-mode builds. Decoded dependency records remain visible; an empty list does not establish the absence of dependencies.", - ] - : []), - ...(info?.module.complete === false - ? [ - "Some embedded module-text records could not be parsed completely; their original lines are preserved and additional metadata remains unknown.", - ] - : []), - ...(stringLiterals === undefined - ? [] - : [ - "Recovered string_literals are inferred from pointer-aligned (ptr, len) pairs in non-executable mapped data pointing to printable UTF-8 slices within single read-only spans; compiler-inlined or stack-constructed strings without static headers, unapplied relocation addends and runtime reachability are not established.", - ...(machoChainedFixups === true - ? [ - "This Mach-O declares LC_DYLD_CHAINED_FIXUPS; rebase-encoded pointer slots in writable data may not hold raw virtual addresses statically, so (ptr, len) recovery can miss writable-section string headers.", - ] - : []), - ...(stringLiterals.truncated - ? [ - "Recovered string_literals were truncated at the configured item, 512 KiB aggregate-byte or 64 MiB scan budget; additional static (ptr, len) candidates may exist.", - ] - : []), - ]), -]; + budgets: GoAnalysisBudgets, +): string[] => { + const { + build_info: info, + section_names_limitation: sectionNamesLimitation, + macho_chained_fixups: machoChainedFixups, + } = image; + return [ + "Embedded build metadata is observed file content, not proof of compiler identity, dependency authenticity or runtime behavior.", + stringLiterals === undefined + ? "Only Go build information is inspected; function names, source mappings, type metadata and decompiled code are not recovered." + : "Only Go build information and inferred static (ptr, len) string-header candidates are inspected; function names, source mappings, type metadata and decompiled code are not recovered.", + `Resource guards use max_input_bytes=${String(budgets.max_input_bytes)}, max_header_search_bytes=${String(budgets.max_header_search_bytes)}, max_build_info_bytes=${String(budgets.max_build_info_bytes)} and max_report_bytes=${String(budgets.max_report_bytes)}; increase the named per-call budget for the next inspection when needed. Native structural table decoding retains its separate 16 MiB bound. These are not operating-system memory or CPU limits.`, + ...(info !== null && (info.go_version === null || info.module_text === null) + ? [ + "Some embedded Go strings contain non-UTF8 bytes; undecodable text is null and exact source bytes are preserved as base64.", + ] + : []), + ...(info === null + ? [ + "No recognized Go build-info record was found in the inspected image; absent metadata does not establish that the binary is not Go.", + ] + : []), + ...(sectionNamesLimitation === undefined ? [] : [sectionNamesLimitation]), + ...(info !== null && info.module.main === null + ? [ + "Dependency coverage is unknown because no main-module record was decoded from the embedded module information, as in GOPATH-mode builds. Decoded dependency records remain visible; an empty list does not establish the absence of dependencies.", + ] + : []), + ...(info?.module.complete === false + ? [ + "Some embedded module-text records could not be parsed completely; their original lines are preserved and additional metadata remains unknown.", + ] + : []), + ...(stringLiterals === undefined + ? [] + : [ + "Recovered string_literals are inferred from pointer-aligned (ptr, len) pairs in non-executable mapped data pointing to printable UTF-8 slices within single read-only spans; compiler-inlined or stack-constructed strings without static headers, unapplied relocation addends and runtime reachability are not established.", + ...(machoChainedFixups === true + ? [ + "This Mach-O declares LC_DYLD_CHAINED_FIXUPS; rebase-encoded pointer slots in writable data may not hold raw virtual addresses statically, so (ptr, len) recovery can miss writable-section string headers.", + ] + : []), + ...(stringLiterals.truncated + ? [ + `Recovered string_literals were truncated at ${stringLiterals.limits_reached?.map((parameter) => `${parameter}=${String(budgets[parameter])}`).join(", ") ?? "a configured recovery budget"}; increase the named per-call budget(s), then inspect the same complete artifact again. Additional static (ptr, len) candidates may exist. No restart is required.`, + ] + : []), + ]), + ]; +}; /** Locate image-relative build-info ranges in the selected file. */ const inFile = ( @@ -285,11 +329,12 @@ export class GoBinaryProvider implements GoBinaryPort { options?: ExecutionOptions, ): Promise> { let snapshot: StableArtifactSnapshot | undefined; + const budgets = goAnalysisBudgets(input); try { return await this.#resources.run(async () => { snapshot = await readStableArtifact( input.path, - INPUT_BYTES, + budgets.max_input_bytes, options?.signal, undefined, this.#resources, @@ -303,12 +348,12 @@ export class GoBinaryProvider implements GoBinaryPort { ); const image = readGoBinaryImage(selected.bytes, { includeStringLiterals: input.include_string_literals === true, - maxStringLiterals: input.max_string_literals, + budgets, checkActive: () => assertArtifactReadActive(options?.signal), }); const { - section_names_limitation: sectionNamesLimitation, - macho_chained_fixups: machoChainedFixups, + section_names_limitation: _sectionNamesLimitation, + macho_chained_fixups: _machoChainedFixups, string_literals: rawStringLiterals, ...publicImage } = image; @@ -331,15 +376,13 @@ export class GoBinaryProvider implements GoBinaryPort { ...(stringLiterals === undefined ? {} : { string_literals: stringLiterals }), - limitations: limitationsFor( - info, - sectionNamesLimitation, - stringLiterals, - machoChainedFixups, - ), + limitations: limitationsFor(image, stringLiterals, budgets), }; - if (Buffer.byteLength(JSON.stringify(report), "utf8") > OUTPUT_BYTES) - throw outputLimit(input.path); + if ( + jsonValueByteLength(report, budgets.max_report_bytes) > + budgets.max_report_bytes + ) + throw outputLimit(input.path, budgets.max_report_bytes); const parsed = goBinarySchema.safeParse(report); if (!parsed.success) throw new AnalysisOutputError( @@ -351,7 +394,7 @@ export class GoBinaryProvider implements GoBinaryPort { return ok(parsed.data); }); } catch (cause: unknown) { - return err(failureFor(cause, input.path, snapshot)); + return err(failureFor(cause, input.path, budgets, snapshot)); } } } diff --git a/src/go/GoStringLiterals.test.ts b/src/go/GoStringLiterals.test.ts index f3bc0ee15f..4daecbc989 100644 --- a/src/go/GoStringLiterals.test.ts +++ b/src/go/GoStringLiterals.test.ts @@ -1,8 +1,5 @@ import { expect, it } from "vitest"; -import { - MAX_GO_STRING_AGGREGATE_BYTES, - MAX_GO_STRING_LITERAL_BYTES, -} from "../domain/go/goBinary.js"; +import { GO_ANALYSIS_BUDGET_DEFAULTS } from "../domain/go/goBinary.js"; import type { BinaryContainer, FileMapping, @@ -13,6 +10,9 @@ import { createGoRodataStringFixture } from "../../tests/fixtures/go/rodataStrin import { readGoBinaryImage } from "./GoBinaryImage.js"; import { recoverGoStringLiterals } from "./GoStringLiterals.js"; +const DEFAULT_AGGREGATE_BYTES = 512 * 1024; +const DEFAULT_LITERAL_BYTES = 4096; + const LITERALS = [ "ListResourceProviders", "genai", @@ -151,14 +151,14 @@ it.each([ ]); }); -it("rejects unaligned, code-section, cross-section, out-of-bounds and non-printable UTF-8 candidates", () => { +it("rejects invalid candidates and reports valid ranges omitted by the string-length budget", () => { const validUnicode = Buffer.from("café\t日本語\n", "utf8"); const invalidUtf8 = Buffer.from([0xff, 0xfe, 0x41]); const embeddedNul = Buffer.from("bad\0str", "utf8"); const c1Control = Buffer.from([0xc2, 0x85]); const noncharacter = Buffer.from([0xef, 0xbf, 0xbf]); - const maxAllowed = Buffer.alloc(MAX_GO_STRING_LITERAL_BYTES, 0x61); - const overMax = Buffer.alloc(MAX_GO_STRING_LITERAL_BYTES + 1, 0x62); + const maxAllowed = Buffer.alloc(DEFAULT_LITERAL_BYTES, 0x61); + const overMax = Buffer.alloc(DEFAULT_LITERAL_BYTES + 1, 0x62); const rodata = Buffer.concat([ validUnicode, invalidUtf8, @@ -194,13 +194,13 @@ it("rejects unaligned, code-section, cross-section, out-of-bounds and non-printa section: "data", offsetInSection: 16, ptr: baseRo + BigInt(badPrefixLen), - len: BigInt(MAX_GO_STRING_LITERAL_BYTES), + len: BigInt(DEFAULT_LITERAL_BYTES), }, { section: "data", offsetInSection: 32, ptr: baseRo + BigInt(badPrefixLen + maxAllowed.length), - len: BigInt(MAX_GO_STRING_LITERAL_BYTES + 1), + len: BigInt(DEFAULT_LITERAL_BYTES + 1), }, { section: "data", offsetInSection: 48, ptr: baseRo, len: 0n }, { @@ -263,15 +263,16 @@ it("rejects unaligned, code-section, cross-section, out-of-bounds and non-printa }).string_literals; expect(recovered?.items.map((item) => item.value)).toEqual([ "café\t日本語\n", - "a".repeat(MAX_GO_STRING_LITERAL_BYTES), + "a".repeat(DEFAULT_LITERAL_BYTES), ]); - expect(recovered?.truncated).toBe(false); + expect(recovered?.truncated).toBe(true); + expect(recovered?.limits_reached).toEqual(["max_string_literal_bytes"]); }); it("enforces maxStringLiterals and the 512 KiB aggregate byte cap deterministically", () => { const chunks = Array.from({ length: 130 }, (_, index) => Buffer.from( - `${String(index).padStart(4, "0")}:${"x".repeat(MAX_GO_STRING_LITERAL_BYTES - 5)}`, + `${String(index).padStart(4, "0")}:${"x".repeat(DEFAULT_LITERAL_BYTES - 5)}`, "utf8", ), ); @@ -291,14 +292,14 @@ it("enforces maxStringLiterals and the 512 KiB aggregate byte cap deterministica const cappedByCount = readGoBinaryImage(fixture.bytes, { includeStringLiterals: true, - maxStringLiterals: 2, + budgets: { max_string_literals: 2 }, }); expect(cappedByCount.macho_chained_fixups).toBe(true); expect(cappedByCount.string_literals).toMatchObject({ discovered_count: 130, max_items: 2, truncated: true, - aggregate_bytes: MAX_GO_STRING_LITERAL_BYTES * 2, + aggregate_bytes: DEFAULT_LITERAL_BYTES * 2, }); expect(cappedByCount.string_literals?.items).toHaveLength(2); expect( @@ -310,16 +311,16 @@ it("enforces maxStringLiterals and the 512 KiB aggregate byte cap deterministica const cappedByBytes = readGoBinaryImage(fixture.bytes, { includeStringLiterals: true, - maxStringLiterals: 500, + budgets: { max_string_literals: 500 }, }); expect(cappedByBytes.string_literals).toMatchObject({ discovered_count: 130, max_items: 500, - aggregate_bytes: MAX_GO_STRING_AGGREGATE_BYTES, + aggregate_bytes: DEFAULT_AGGREGATE_BYTES, truncated: true, }); expect(cappedByBytes.string_literals?.items).toHaveLength( - MAX_GO_STRING_AGGREGATE_BYTES / MAX_GO_STRING_LITERAL_BYTES, + DEFAULT_AGGREGATE_BYTES / DEFAULT_LITERAL_BYTES, ); }); @@ -593,7 +594,7 @@ it("indexes 25,000 Mach-O load regions and sections without quadratic stalls and load_regions: trackedLoadRegions, sections: trackedSections, }, - 2000, + GO_ANALYSIS_BUDGET_DEFAULTS, () => { if (inRegionLoop && !inSectionLoop) { regionCancelChecks += 1; @@ -613,7 +614,7 @@ it("indexes 25,000 Mach-O load regions and sections without quadratic stalls and recoverGoStringLiterals( bytes, { ...container, sections: trackedSections }, - 2000, + GO_ANALYSIS_BUDGET_DEFAULTS, () => { if (inSectionLoop) throw new Error("cancelled during section indexing"); }, @@ -742,3 +743,38 @@ it.each([ expect(recovered.items.map((item) => item.value)).toEqual(expected); }, ); + +it("recovers longer literals with a selected budget without conflating offset/length identities", () => { + const fixture = createGoRodataStringFixture({ + format: "elf", + rodata: Buffer.alloc(9000, 0x61), + headers: [ + { section: "data", offsetInSection: 0, ptr: 0x102000n, len: 9000n }, + { section: "data", offsetInSection: 16, ptr: 0x102001n, len: 808n }, + ], + }); + const limited = readGoBinaryImage(fixture.bytes, { + includeStringLiterals: true, + }); + expect(limited.string_literals).toMatchObject({ + truncated: true, + limits_reached: ["max_string_literal_bytes"], + }); + const recovered = readGoBinaryImage(fixture.bytes, { + includeStringLiterals: true, + budgets: { max_string_literal_bytes: 9000 }, + }); + expect(recovered.string_literals).toMatchObject({ + truncated: false, + discovered_count: 2, + }); + expect( + recovered.string_literals?.items.map((item) => [ + item.location.bytes, + item.reference_count, + ]), + ).toEqual([ + [9000, 1], + [808, 1], + ]); +}); diff --git a/src/go/GoStringLiterals.ts b/src/go/GoStringLiterals.ts index 75f33a6e2d..91e2afb4b1 100644 --- a/src/go/GoStringLiterals.ts +++ b/src/go/GoStringLiterals.ts @@ -1,9 +1,8 @@ import { isUtf8 } from "node:buffer"; import { - DEFAULT_MAX_GO_STRING_LITERALS, - MAX_GO_STRING_AGGREGATE_BYTES, - MAX_GO_STRING_HEADER_SCAN_BYTES, - MAX_GO_STRING_LITERAL_BYTES, + GO_ANALYSIS_BUDGET_DEFAULTS, + type GoAnalysisBudgets, + type GoStringBudgetKey, type GoStringLiteral, type GoStringLiterals, } from "../domain/go/goBinary.js"; @@ -17,7 +16,6 @@ import { type LoadRegion, } from "../native/metadata/BinaryContainer.js"; -const MAX_TRACKED_CANDIDATES = 100_000; const MAX_REJECTED_SLICE_CACHE = 4096; const PE_MEM_READ = 0x40000000; const PE_MEM_EXECUTE = 0x20000000; @@ -48,9 +46,11 @@ interface HeaderScanSpan { export const recoverGoStringLiterals = ( bytes: Buffer, container: BinaryContainer, - maxItems: number = DEFAULT_MAX_GO_STRING_LITERALS, + budgets: GoAnalysisBudgets = GO_ANALYSIS_BUDGET_DEFAULTS, checkActive?: () => void, ): GoStringLiterals => { + const limitsReached = new Set(); + const maxItems = budgets.max_string_literals; checkActive?.(); const index = createFileMappingIndex( imageFileMappings(container), @@ -93,16 +93,17 @@ export const recoverGoStringLiterals = ( const minTargetAddr = firstTarget.address; const maxTargetEnd = lastTarget.address + BigInt(lastTarget.size); - const accepted = new Map(); - const rejected = new Set(); + const accepted = new Map(); + const rejected = new Set(); const candidates: CandidateLiteral[] = []; - let remainingBudget = MAX_GO_STRING_HEADER_SCAN_BYTES; + let remainingBudget = budgets.max_string_header_scan_bytes; let budgetTruncated = false; let steps = 0; outer: for (const { startOffset, endOffset } of headerSpans) { if (remainingBudget < wordBytes * 2) { budgetTruncated = true; + limitsReached.add("max_string_header_scan_bytes"); break; } remainingBudget -= wordBytes; @@ -116,6 +117,7 @@ export const recoverGoStringLiterals = ( if ((++steps & 0xffff) === 0) checkActive?.(); if (remainingBudget < wordBytes) { budgetTruncated = true; + limitsReached.add("max_string_header_scan_bytes"); break outer; } remainingBudget -= wordBytes; @@ -124,13 +126,18 @@ export const recoverGoStringLiterals = ( const rawLen = word1; word0 = word1; - if (rawLen < 1n || rawLen > BigInt(MAX_GO_STRING_LITERAL_BYTES)) continue; + if (rawLen < 1n) continue; if (ptr < minTargetAddr || ptr >= maxTargetEnd) continue; const targetIndex = lastSpanAtOrBelow(targets, ptr); if (targetIndex < 0) continue; const target = targets[targetIndex] as FileMapping; if (ptr + rawLen > target.address + BigInt(target.size)) continue; + if (rawLen > BigInt(budgets.max_string_literal_bytes)) { + limitsReached.add("max_string_literal_bytes"); + budgetTruncated = true; + continue; + } const byteLen = Number(rawLen); const sliceOffset = target.offset + Number(ptr - target.address); if ( @@ -139,7 +146,7 @@ export const recoverGoStringLiterals = ( ) continue; - const key = sliceOffset * 8192 + byteLen; + const key = `${String(sliceOffset)}:${String(byteLen)}`; const existing = accepted.get(key); if (existing !== undefined) { existing.referenceCount += 1; @@ -149,11 +156,13 @@ export const recoverGoStringLiterals = ( } if (rejected.has(key)) continue; - if (candidates.length >= MAX_TRACKED_CANDIDATES) { + if (candidates.length >= budgets.max_string_candidates) { + limitsReached.add("max_string_candidates"); budgetTruncated = true; break outer; } if (remainingBudget < byteLen) { + limitsReached.add("max_string_header_scan_bytes"); budgetTruncated = true; break outer; } @@ -187,8 +196,11 @@ export const recoverGoStringLiterals = ( for (const candidate of candidates) { if ( items.length >= maxItems || - aggregateBytes + candidate.bytes > MAX_GO_STRING_AGGREGATE_BYTES + aggregateBytes + candidate.bytes > budgets.max_string_bytes ) { + limitsReached.add( + items.length >= maxItems ? "max_string_literals" : "max_string_bytes", + ); outputTruncated = true; break; } @@ -210,6 +222,7 @@ export const recoverGoStringLiterals = ( aggregate_bytes: aggregateBytes, max_items: maxItems, truncated: budgetTruncated || outputTruncated, + ...(limitsReached.size === 0 ? {} : { limits_reached: [...limitsReached] }), }; }; diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.ts index ca7f2daea4..e01df48ad2 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorker.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.ts @@ -348,6 +348,9 @@ export class JavaScriptAnalysisWorker { id, node_budget: capacity.nodes, unknown_budget: capacity.unknowns, + ...(this.limits.semanticPrecision === undefined + ? {} + : { semantic_precision: this.limits.semanticPrecision }), text_sha256: createHash("sha256").update(file.text.value).digest("hex"), file: metadata, package_type: packageType, diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts index 4af0dacfce..70945ea7da 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts @@ -1,3 +1,4 @@ +import { javaScriptSemanticPrecisionSchema } from "../../domain/javascript/javascriptSemanticPrecision.js"; import { z } from "zod"; import { ANALYSIS_RESOURCE_KINDS } from "../../domain/analysisErrorCore.js"; import { jsonValueSchema } from "../../domain/jsonValue.js"; @@ -25,6 +26,7 @@ export const javaScriptCheckpointCeilingRemediation = ( /** Caller-selected artifact identity, separate from the decoded source-text digest. */ export const javaScriptWorkerRequestSchema = z.strictObject({ id: z.uuid(), + semantic_precision: javaScriptSemanticPrecisionSchema.optional(), unknown_budget: z.int().min(0).max(JAVASCRIPT_SEMANTIC_ITEM_MAXIMUM), node_budget: z.int().min(0).max(JAVASCRIPT_SEMANTIC_ITEM_MAXIMUM), text_sha256: digestSchema, diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts index 17b7e7ca14..d8e2bb6671 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -1,3 +1,7 @@ +import { + resolveJavaScriptSemanticPrecision, + javaScriptSemanticPrecisionSchema, +} from "../../domain/javascript/javascriptSemanticPrecision.js"; import { javaScriptSemanticUnknownOmissionsSchema, javaScriptSemanticNodeSchema, @@ -609,18 +613,27 @@ const coverage = z status: z.literal("complete"), omittedCount: z.literal(0), resourceLimits: z.array(javaScriptSemanticResourceLimitSchema).optional(), + precision: javaScriptSemanticPrecisionSchema + .transform((value) => resolveJavaScriptSemanticPrecision(value)) + .exactOptional(), widenedReturnedReferences: z.array(range).optional(), }), z.strictObject({ status: z.literal("partial"), omittedCount: z.union([z.literal(0), z.null()]), resourceLimits: z.array(javaScriptSemanticResourceLimitSchema).optional(), + precision: javaScriptSemanticPrecisionSchema + .transform((value) => resolveJavaScriptSemanticPrecision(value)) + .exactOptional(), widenedReturnedReferences: z.array(range).optional(), }), z.strictObject({ status: z.literal("failed"), omittedCount: z.null(), resourceLimits: z.array(javaScriptSemanticResourceLimitSchema).optional(), + precision: javaScriptSemanticPrecisionSchema + .transform((value) => resolveJavaScriptSemanticPrecision(value)) + .exactOptional(), widenedReturnedReferences: z.array(range).optional(), }), ]) diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts index ae46284020..a49dade03a 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts @@ -1,3 +1,4 @@ +import type { JavaScriptSemanticPrecisionInput } from "../../domain/javascript/javascriptSemanticPrecision.js"; import { fitJavaScriptRouteSourceFacts } from "../../domain/javascript/javascriptRouteResultBudget.js"; import { jsonValueByteLength } from "../../domain/jsonValueByteLength.js"; import type { JsonValue } from "../../domain/jsonValue.js"; @@ -46,6 +47,7 @@ export interface JavaScriptSourceWorkerAnalyzer { }, packageType: JavaScriptPackageType, purpose: "analysis" | "preload-discovery", + precision?: JavaScriptSemanticPrecisionInput, ): Promise<{ readonly module: JavaScriptModuleSemanticIr; readonly projection: JavaScriptSemanticFileProjection; @@ -191,6 +193,7 @@ const analyzeJob = async ( }, request.package_type, request.purpose, + request.semantic_precision, ); await send({ kind: "phase", id: request.id, phase: "transfer" }); const descriptor = diff --git a/src/javascript/recovery/WakaruRelease.ts b/src/javascript/recovery/WakaruRelease.ts index 9153b2d9a3..7c1ed1d653 100644 --- a/src/javascript/recovery/WakaruRelease.ts +++ b/src/javascript/recovery/WakaruRelease.ts @@ -1,4 +1,4 @@ -import { BOUNDED_PRODUCER_LIMITS } from "../../process/LinuxResourceLimits.js"; +import { BOUNDED_PRODUCER_LIMITS } from "../../domain/producerResourceLimits.js"; /** * Releases the report parser accepts. Wakaru changes `--version`, `--json` diff --git a/src/javascript/sourceMaps/SourceMapAdmission.test.ts b/src/javascript/sourceMaps/SourceMapAdmission.test.ts index f7ade1288b..9ba401ba62 100644 --- a/src/javascript/sourceMaps/SourceMapAdmission.test.ts +++ b/src/javascript/sourceMaps/SourceMapAdmission.test.ts @@ -1,3 +1,7 @@ +import { webSourceLocationInputSchema } from "../../domain/webSourceLocation.js"; +import { analyzeWebBundleInputSchema } from "../../domain/webBundleAnalysis.js"; +import { webSourceLocationArgs } from "../../../tests/fixtures/webSourceLocation.js"; +import { selectedSourceMapLimits } from "../../domain/sourceMapLimits.js"; import { expect, it } from "vitest"; import { inspectSourceMapEnvelope } from "../../domain/sourceMapEnvelope.js"; @@ -49,3 +53,76 @@ it("admits numeric version 3 through the same consumers", () => { }).matches, ).toMatchObject([{ section_path: [0, 0], reported_source: "original.ts" }]); }); + +it("recovers the same finite map by raising the exact decoder record admission", () => { + const text = JSON.stringify({ + version: 3, + sources: ["a.ts"], + sourcesContent: ["original"], + names: [], + mappings: "AAAA", + }); + const position = { line: 1, column: 0 }; + expect(() => + traceSourceMap( + text, + "https://example.test/app.map", + position, + selectedSourceMapLimits({ decoded_records: 4 }), + ), + ).toThrow( + expect.objectContaining({ + reason: "limit", + message: expect.stringContaining( + "source_map_limits.decoded_records is 4", + ), + }), + ); + expect( + traceSourceMap( + text, + "https://example.test/app.map", + position, + selectedSourceMapLimits({ decoded_records: 5 }), + ).matches, + ).toMatchObject([{ reported_source: "a.ts", content: { text: "original" } }]); +}); + +it("rejects invalid and inapplicable budget selections at the public owning contracts", () => { + for (const selection of [ + { map_bytes: 0 }, + { output_bytes: 536870888 }, + { decoded_records: 0 }, + { decoded_records: undefined }, + { codec_heap_mib: 127 }, + { fetch_bytes: 1024 }, + ]) { + expect( + webSourceLocationInputSchema.safeParse({ + ...webSourceLocationArgs, + source_map_limits: selection, + }).success, + ).toBe(false); + } + const bundle = { + cdp_endpoint: "http://127.0.0.1:9222", + allowed_origins: ["https://app.test"], + target_id: "page", + fetch_source_maps: true, + }; + expect( + analyzeWebBundleInputSchema.safeParse({ + ...bundle, + fetch_source_maps: false, + source_map_limits: { decoded_records: 5 }, + }).success, + ).toBe(false); + for (const selection of [{ map_bytes: 1024 }, { codec_heap_mib: 256 }]) { + expect( + analyzeWebBundleInputSchema.safeParse({ + ...bundle, + source_map_limits: selection, + }).success, + ).toBe(false); + } +}); diff --git a/src/javascript/sourceMaps/SourceMapCodecProcess.ts b/src/javascript/sourceMaps/SourceMapCodecProcess.ts index 8cedb0a508..952a1b6c7f 100644 --- a/src/javascript/sourceMaps/SourceMapCodecProcess.ts +++ b/src/javascript/sourceMaps/SourceMapCodecProcess.ts @@ -1,5 +1,9 @@ import { readStableArtifact } from "../../artifacts/readStableArtifact.js"; -import { WEB_SOURCE_MAP_LIMITS } from "../../domain/webSourceLocation.js"; +import { + selectedSourceMapLimits, + sourceMapLimitRecovery, + sourceMapLimitsSchema, +} from "../../domain/sourceMapLimits.js"; import { traceSourceMap } from "./TraceSourceMap.js"; import { SourceMapFormatFailure } from "./SourceMapFormat.js"; import { @@ -17,26 +21,32 @@ const execute = async (): Promise => { "format", "Codec request path is missing.", ); - const data = await readStableArtifact( - path, - WEB_SOURCE_MAP_LIMITS.outputBytes, + const selected = sourceMapLimitsSchema.parse( + JSON.parse(process.argv[3] ?? "{}"), ); + const limits = selectedSourceMapLimits(selected); + const data = await readStableArtifact(path, limits.outputBytes); const text = new TextDecoder("utf-8", { fatal: true }).decode(data.bytes); const value: unknown = JSON.parse(text); const input = sourceMapCodecInputSchema.parse(value); - if (Buffer.byteLength(input.text) > WEB_SOURCE_MAP_LIMITS.mapBytes) + if (Buffer.byteLength(input.text) > limits.mapBytes) throw new SourceMapFormatFailure( "limit", - "Source map exceeds the 4 MiB codec input budget.", + `Source map exceeds codec input admission. ${sourceMapLimitRecovery("map_bytes", limits.mapBytes)}`, ); - const report = traceSourceMap(input.text, input.url, input.position); + const report = traceSourceMap( + input.text, + input.url, + input.position, + limits, + ); const reply = JSON.stringify( sourceMapCodecReplySchema.parse({ state: "success", report }), ); - if (Buffer.byteLength(reply) > WEB_SOURCE_MAP_LIMITS.outputBytes) + if (Buffer.byteLength(reply) > limits.outputBytes) throw new SourceMapFormatFailure( "limit", - "Complete source-map point evidence exceeds the 32 MiB output budget; no partial evidence was returned.", + `Complete source-map point evidence exceeds output admission; no partial evidence was returned. ${sourceMapLimitRecovery("output_bytes", limits.outputBytes)}`, ); return reply; } catch (cause: unknown) { diff --git a/src/javascript/sourceMaps/SourceMapCommand.test.ts b/src/javascript/sourceMaps/SourceMapCommand.test.ts index fd5c188115..4171464e40 100644 --- a/src/javascript/sourceMaps/SourceMapCommand.test.ts +++ b/src/javascript/sourceMaps/SourceMapCommand.test.ts @@ -21,10 +21,14 @@ import { import { traceSourceMap } from "./TraceSourceMap.js"; import { runSourceMapCommand, - SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, type SourceMapCleanupRetention, } from "./SourceMapCommand.js"; +const expectedProcessMemoryLimitBytes = + process.platform === "win32" && process.versions.bun !== undefined + ? 704 * 1024 * 1024 + : null; + const report = traceSourceMap( JSON.stringify({ version: 3, sources: [], names: [], mappings: "" }), "https://app.test/app.map", @@ -37,7 +41,7 @@ const completedReport: WebSourceMapReport = { ? { ...report.runtime, v8_heap_limit_bytes: 200 * 1024 * 1024 } : { ...report.runtime, - process_memory_limit_bytes: SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, + process_memory_limit_bytes: expectedProcessMemoryLimitBytes, }, }; const completedCodec = `process.stdout.write(${JSON.stringify(JSON.stringify({ state: "success", report: completedReport }))})`; @@ -361,7 +365,7 @@ it("preserves an observed process-memory limit over the codec exit status", asyn expect(result.error).toMatchObject({ resource: "memory", reportedLimits: { - codec_process_memory_limit_bytes: SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, + codec_process_memory_limit_bytes: expectedProcessMemoryLimitBytes, }, capturedOutput: { stderr: "memory limit reached" }, }); diff --git a/src/javascript/sourceMaps/SourceMapCommand.ts b/src/javascript/sourceMaps/SourceMapCommand.ts index 2fb2c25ff3..3a538086be 100644 --- a/src/javascript/sourceMaps/SourceMapCommand.ts +++ b/src/javascript/sourceMaps/SourceMapCommand.ts @@ -1,3 +1,8 @@ +import { + sourceMapLimitSelection, + sourceMapLimitRecovery, + type SourceMapLimits, +} from "../../domain/sourceMapLimits.js"; import { randomUUID } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -15,7 +20,6 @@ import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { err, type Result } from "../../domain/result.js"; -import { noRuntimeOverrideSentence } from "../../domain/browserDeadline.js"; import { DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS, WEB_SOURCE_MAP_LIMITS, @@ -43,9 +47,6 @@ import { prepareProcessOwnershipInspection } from "../../process/ProcessOwnershi const OPERATION = "trace_web_source_location"; const PROVIDER = "source-map-decoder"; -/** Process commit limit enforced on the codec; null where V8 heap flags apply. */ -export const SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES = - javaScriptWorkerMemoryLimitBytes(WEB_SOURCE_MAP_LIMITS.codecHeapMiB); /** Launcher seam retains the actual codec protocol and ownership supervisor. */ export type SourceMapCodecLauncher = ( options: OwnedProviderProcessSpawnOptions, @@ -83,7 +84,7 @@ type CodecPhysicalObservation = readonly message: string; readonly cause: Error; } - | { readonly kind: "output-limit" }; + | { readonly kind: "output-limit"; readonly stream: "stdout" | "stderr" }; type CodecOutcome = | { readonly kind: "physical"; @@ -109,6 +110,7 @@ export const runSourceMapCommand = async ( dependencies: SourceMapDecoderDependencies, retainedCleanup: SourceMapCleanupRetention, decodeTimeoutMs: number = DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS, + limits: SourceMapLimits = WEB_SOURCE_MAP_LIMITS, ): Promise> => { try { await (dependencies.prepareOwnership ?? prepareProcessOwnershipInspection)( @@ -136,14 +138,21 @@ export const runSourceMapCommand = async ( decodeTimeoutMs, options?.signal, ); - const command = new OwnedCodec(path, dependencies, deadline, decodeTimeoutMs); + const command = new OwnedCodec( + path, + dependencies, + deadline, + decodeTimeoutMs, + limits, + ); let result: Result; try { await command.start(request); result = parseSourceMapReply( await command.collect(), path, - SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, + javaScriptWorkerMemoryLimitBytes(limits.codecHeapMiB), + limits.codecHeapMiB, ); } catch (cause: unknown) { result = err(command.failure(cause)); @@ -189,6 +198,7 @@ class OwnedCodec implements SourceMapCleanupOwner { readonly dependencies: SourceMapDecoderDependencies, readonly deadline: ProviderStartupDeadline, readonly decodeTimeoutMs: number, + readonly limits: SourceMapLimits, ) { this.#onInterruption = () => this.#observeInterruption(); if (deadline.signal.aborted) this.#observeInterruption(); @@ -227,20 +237,25 @@ class OwnedCodec implements SourceMapCleanupOwner { )({ command: process.execPath, arguments: [ - `--max-old-space-size=${String(WEB_SOURCE_MAP_LIMITS.codecHeapMiB)}`, + `--max-old-space-size=${String(this.limits.codecHeapMiB)}`, "--max-semi-space-size=8", "--v8-pool-size=1", "--single-threaded", "--stack_size=2048", fileURLToPath(new URL("./SourceMapCodecProcess.js", import.meta.url)), requestPath, + JSON.stringify(sourceMapLimitSelection(this.limits)), ], runId: this.#runId, cwd: this.#runtime.path, hostEnvironment: environment, - ...(SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES === null + ...(javaScriptWorkerMemoryLimitBytes(this.limits.codecHeapMiB) === null ? {} - : { memoryLimitBytes: SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES }), + : { + memoryLimitBytes: javaScriptWorkerMemoryLimitBytes( + this.limits.codecHeapMiB, + )!, + }), signal: this.deadline.signal, }); this.#supervisor = new ProviderProcessSupervisor( @@ -252,7 +267,7 @@ class OwnedCodec implements SourceMapCleanupOwner { (() => cleanupOwnedProcessGroup(spawned.ownership)), }, { - maxDiagnosticBytes: WEB_SOURCE_MAP_LIMITS.outputBytes + 1024 * 1024, + maxDiagnosticBytes: this.limits.outputBytes + 1024 * 1024, onDiagnostic: (event) => { if ( event.type === "exit" && @@ -274,10 +289,13 @@ class OwnedCodec implements SourceMapCleanupOwner { (event.truncated === true || event.totalBytes > (event.stream === "stdout" - ? WEB_SOURCE_MAP_LIMITS.outputBytes + ? this.limits.outputBytes : 1024 * 1024)) ) { - this.#recordPhysicalFailure({ kind: "output-limit" }); + this.#recordPhysicalFailure({ + kind: "output-limit", + stream: event.stream, + }); } }, }, @@ -375,7 +393,7 @@ class OwnedCodec implements SourceMapCleanupOwner { } if (observation.kind === "process-error") return this.#processErrorFailure(observation, snapshot); - return this.#outputLimitFailure(snapshot); + return this.#outputLimitFailure(snapshot, observation.stream); } #exitFailure( @@ -425,6 +443,7 @@ class OwnedCodec implements SourceMapCleanupOwner { #outputLimitFailure( snapshot: ProviderProcessSnapshot | undefined, + stream: "stdout" | "stderr", ): AnalysisError { const outputComplete = this.#supervisor?.observedOutputClose() ?? false; const capturedOutput = this.#capturedOutput( @@ -434,8 +453,14 @@ class OwnedCodec implements SourceMapCleanupOwner { ); return new AnalysisOutputError( OPERATION, - "Codec output exceeds its complete-evidence or diagnostic budget.", + stream === "stdout" + ? "Codec output exceeds its complete-evidence budget." + : "Codec diagnostic output exceeds its 1048576-byte allowance.", { + remediationAction: + stream === "stdout" + ? sourceMapLimitRecovery("output_bytes", this.limits.outputBytes) + : "The codec stderr diagnostic allowance is fixed at 1048576 bytes; no source_map_limits field raises it. Inspect retained diagnostics before another call.", capturedOutput: { stdout: capturedOutput?.stdout ?? "", stderr: capturedOutput?.stderr ?? "", @@ -469,9 +494,11 @@ class OwnedCodec implements SourceMapCleanupOwner { return new AnalysisResourceConstraintError( OPERATION, "memory", - `Source-map codec reached its ${SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES === null ? "" : `${String(SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES)}-byte `}process memory limit for ${this.path}; no partial evidence was returned. ${noRuntimeOverrideSentence(`The ${String(WEB_SOURCE_MAP_LIMITS.codecHeapMiB)} MiB codec heap stays fixed.`, "decode_timeout_ms")}`, + `Source-map codec reached its ${javaScriptWorkerMemoryLimitBytes(this.limits.codecHeapMiB) === null ? "" : `${String(javaScriptWorkerMemoryLimitBytes(this.limits.codecHeapMiB))}-byte `}process memory limit for ${this.path}; no partial evidence was returned. ${sourceMapLimitRecovery("codec_heap_mib", this.limits.codecHeapMiB)}`, { - codec_process_memory_limit_bytes: SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, + codec_process_memory_limit_bytes: javaScriptWorkerMemoryLimitBytes( + this.limits.codecHeapMiB, + ), }, { capturedOutput: { @@ -481,9 +508,9 @@ class OwnedCodec implements SourceMapCleanupOwner { snapshot.diagnosticTruncated === true || !(this.#supervisor?.observedOutputClose() ?? false), }, - remediationAction: noRuntimeOverrideSentence( - `The ${String(WEB_SOURCE_MAP_LIMITS.codecHeapMiB)} MiB codec heap, ${String(WEB_SOURCE_MAP_LIMITS.mapBytes)}-byte map budget, and ${String(WEB_SOURCE_MAP_LIMITS.outputBytes)}-byte reply budget stay fixed.`, - "decode_timeout_ms", + remediationAction: sourceMapLimitRecovery( + "codec_heap_mib", + this.limits.codecHeapMiB, ), }, ); diff --git a/src/javascript/sourceMaps/SourceMapDecoder.ts b/src/javascript/sourceMaps/SourceMapDecoder.ts index 9b359e08b3..d193ee4498 100644 --- a/src/javascript/sourceMaps/SourceMapDecoder.ts +++ b/src/javascript/sourceMaps/SourceMapDecoder.ts @@ -1,3 +1,8 @@ +import { + selectedSourceMapLimits, + localSourceMapLimitsSchema, + sourceMapLimitRecovery, +} from "../../domain/sourceMapLimits.js"; import type { ExecutionOptions } from "../../application/AnalysisProvider.js"; import type { WebSourceMapPort } from "../../application/WebSourceLocationPorts.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; @@ -7,10 +12,8 @@ import { } from "../../domain/analysisErrorCore.js"; import { ArtifactOperationError } from "../../domain/artifactOperationError.js"; import { err, type Result } from "../../domain/result.js"; -import { noRuntimeOverrideSentence } from "../../domain/browserDeadline.js"; import { DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS, - WEB_SOURCE_MAP_LIMITS, type WebSourceMapReport, } from "../../domain/webSourceLocation.js"; import { sourceMapCodecInputSchema } from "./SourceMapCodecProtocol.js"; @@ -84,21 +87,31 @@ export class SourceMapDecoder implements WebSourceMapPort { }); if (!request.success) return err(new AnalysisInputError(OPERATION, { cause: request.error })); - const capacityFailure = () => - new ArtifactOperationError( + const selected = localSourceMapLimitsSchema.safeParse(input.limits ?? {}); + if (!selected.success) + return err(new AnalysisInputError(OPERATION, { cause: selected.error })); + const limits = selectedSourceMapLimits(selected.data); + const capacityFailure = ( + field: "map_bytes" | "output_bytes", + value: number, + ) => { + const recovery = sourceMapLimitRecovery(field, value); + return new ArtifactOperationError( OPERATION, "limit", undefined, - `${input.path}: Source-map input exceeds the ${String(WEB_SOURCE_MAP_LIMITS.mapBytes)}-byte map or ${String(WEB_SOURCE_MAP_LIMITS.outputBytes)}-byte encoded request budget. ${noRuntimeOverrideSentence("Those budgets stay fixed.", "decode_timeout_ms")}`, + `${input.path}: Source-map admission exhausted. ${recovery}`, + { remediationAction: recovery }, ); - if (Buffer.byteLength(input.text) > WEB_SOURCE_MAP_LIMITS.mapBytes) - return err(capacityFailure()); + }; + if (Buffer.byteLength(input.text) > limits.mapBytes) + return err(capacityFailure("map_bytes", limits.mapBytes)); const encoded: string[] = []; let encodedBytes = 0; for (const part of jsonParts(request.data)) { const bytes = Buffer.byteLength(part); - if (bytes > WEB_SOURCE_MAP_LIMITS.outputBytes - encodedBytes) - return err(capacityFailure()); + if (bytes > limits.outputBytes - encodedBytes) + return err(capacityFailure("output_bytes", limits.outputBytes)); encoded.push(part); encodedBytes += bytes; } @@ -115,6 +128,7 @@ export class SourceMapDecoder implements WebSourceMapPort { this.dependencies, retainedCleanup, input.decodeTimeoutMs ?? DEFAULT_WEB_SOURCE_MAP_DECODE_TIMEOUT_MS, + limits, ); if (retainedCleanup.owner !== undefined) this.#pendingCleanup = retainedCleanup.owner; diff --git a/src/javascript/sourceMaps/SourceMapFormat.ts b/src/javascript/sourceMaps/SourceMapFormat.ts index fc5e6eeb8b..4fe7a051d2 100644 --- a/src/javascript/sourceMaps/SourceMapFormat.ts +++ b/src/javascript/sourceMaps/SourceMapFormat.ts @@ -1,5 +1,9 @@ import { z } from "zod"; -import { WEB_SOURCE_MAP_LIMITS } from "../../domain/webSourceLocation.js"; +import { + WEB_SOURCE_MAP_LIMITS, + sourceMapLimitRecovery, + type SourceMapLimits, +} from "../../domain/sourceMapLimits.js"; import { classifySourceMapSection, classifySourceMapSections, @@ -41,13 +45,14 @@ interface PendingMap { readonly stop: Offset | null; } -/** Shared structural constraints configurable only for stricter consumers. */ +/** Aggregate structural work shared by maps in one selected inspection. */ export interface SourceMapStructureBudget { records: number; } export interface SourceMapStructureOptions { readonly profile?: "browser-collection"; + readonly limits?: SourceMapLimits; readonly budget?: SourceMapStructureBudget; readonly check?: () => void; } @@ -98,7 +103,11 @@ const codecLeaf = ( const compare = (a: Offset, b: Offset): number => a.line - b.line || a.column - b.column; -const addOffset = (base: Offset, offset: Offset): Offset => { +const addOffset = ( + base: Offset, + offset: Offset, + limits: SourceMapLimits, +): Offset => { const result = { line: base.line + offset.line, column: offset.column + (offset.line === 0 ? base.column : 0), @@ -106,20 +115,24 @@ const addOffset = (base: Offset, offset: Offset): Offset => { if ( !Number.isSafeInteger(result.line) || !Number.isSafeInteger(result.column) || - result.line >= WEB_SOURCE_MAP_LIMITS.decodedRows + result.line >= limits.decodedRows ) throw new SourceMapFormatFailure( "limit", - "Indexed offset exceeds the 262144 generated-row decoder budget.", + `Indexed offset exceeds the generated-row decoder budget. ${sourceMapLimitRecovery("decoded_rows", limits.decodedRows)}`, ); return result; }; -const consumeRecords = (budget: { records: number }, count: number): void => { - if (count > WEB_SOURCE_MAP_LIMITS.decodedRecords - budget.records) +const consumeRecords = ( + budget: { records: number }, + limits: SourceMapLimits, + count: number, +): void => { + if (count > limits.decodedRecords - budget.records) throw new SourceMapFormatFailure( "limit", - `Source-map structure exceeds the ${WEB_SOURCE_MAP_LIMITS.decodedRecords}-record decoder budget.`, + `Source-map structure exceeds the decoder budget. ${sourceMapLimitRecovery("decoded_records", limits.decodedRecords)}`, ); budget.records += count; }; @@ -137,8 +150,9 @@ const addSections = ({ readonly budget: { records: number }; readonly options: SourceMapStructureOptions; }): void => { + const limits = options.limits ?? WEB_SOURCE_MAP_LIMITS; options.check?.(); - consumeRecords(budget, rawSections.length); + consumeRecords(budget, limits, rawSections.length); const sections = z.array(sectionSchema).parse(rawSections); const embeddedMaps: Extract< SourceMapSection, @@ -176,14 +190,14 @@ const addSections = ({ ); const next = sections[index + 1]; const nextOffset = - next === undefined ? null : addOffset(node.offset, next.offset); + next === undefined ? null : addOffset(node.offset, next.offset, limits); const stop = nextOffset === null ? node.stop : node.stop === null || compare(nextOffset, node.stop) < 0 ? nextOffset : node.stop; - const offset = addOffset(node.offset, section.offset); + const offset = addOffset(node.offset, section.offset, limits); pending.push({ value: map, path: [...node.path, index], @@ -195,6 +209,7 @@ const addSections = ({ const inspectMappingLayout = ( mappings: string, + limits: SourceMapLimits, check?: () => void, ): { lines: number; segments: number } => { let lines = 1; @@ -241,13 +256,10 @@ const inspectMappingLayout = ( digits = 0; } } - if ( - segments > WEB_SOURCE_MAP_LIMITS.decodedSegments || - lines > WEB_SOURCE_MAP_LIMITS.decodedRows - ) + if (segments > limits.decodedSegments || lines > limits.decodedRows) throw new SourceMapFormatFailure( "limit", - "Source-map layout exceeds the 262144 segment/generated-row decoder budget.", + `Source-map layout exceeds decoder admission. ${sourceMapLimitRecovery(segments > limits.decodedSegments ? "decoded_segments" : "decoded_rows", segments > limits.decodedSegments ? limits.decodedSegments : limits.decodedRows)}`, ); } finish(); @@ -259,8 +271,9 @@ const inspectLeaf = ( budget: { records: number }, options: SourceMapStructureOptions, ): z.output => { + const limits = options.limits ?? WEB_SOURCE_MAP_LIMITS; options.check?.(); - consumeRecords(budget, 1); + consumeRecords(budget, limits, 1); if ( typeof node.value !== "object" || node.value === null || @@ -271,20 +284,28 @@ const inspectLeaf = ( const rawSources = rawMap.sources; const rawNames = rawMap.names; const rawContents = rawMap.sourcesContent; - consumeRecords(budget, Array.isArray(rawSources) ? rawSources.length : 0); - consumeRecords(budget, Array.isArray(rawNames) ? rawNames.length : 0); - consumeRecords(budget, Array.isArray(rawContents) ? rawContents.length : 0); + consumeRecords( + budget, + limits, + Array.isArray(rawSources) ? rawSources.length : 0, + ); + consumeRecords(budget, limits, Array.isArray(rawNames) ? rawNames.length : 0); + consumeRecords( + budget, + limits, + Array.isArray(rawContents) ? rawContents.length : 0, + ); options.check?.(); const leaf = leafSchema.parse(node.value); - const layout = inspectMappingLayout(leaf.mappings, options.check); - consumeRecords(budget, layout.segments + layout.lines); + const layout = inspectMappingLayout(leaf.mappings, limits, options.check); + consumeRecords(budget, limits, layout.segments + layout.lines); if ( - layout.segments > WEB_SOURCE_MAP_LIMITS.decodedSegments || - node.offset.line + layout.lines > WEB_SOURCE_MAP_LIMITS.decodedRows + layout.segments > limits.decodedSegments || + node.offset.line + layout.lines > limits.decodedRows ) throw new SourceMapFormatFailure( "limit", - "Source-map layout exceeds the 262144 segment/generated-row decoder budget.", + `Source-map layout exceeds decoder admission. ${sourceMapLimitRecovery(layout.segments > limits.decodedSegments ? "decoded_segments" : "decoded_rows", layout.segments > limits.decodedSegments ? limits.decodedSegments : limits.decodedRows)}`, ); return leaf; }; @@ -292,10 +313,11 @@ const inspectLeaf = ( /** Validate structure before allocation, retaining raw declarations and section boundaries. */ export const inspectSourceMap = ( text: string, + options: SourceMapStructureOptions = {}, ): ReturnType => { const jsonText = sourceMapJsonText(text); const value: unknown = JSON.parse(jsonText); - return inspectSourceMapValue(value); + return inspectSourceMapValue(value, options); }; /** Return validated leaves when a consumer needs decoded rows after preflight. */ @@ -308,6 +330,7 @@ export const inspectSourceMapValue = ( readonly declarations: readonly SourceMapSourceDeclaration[]; readonly leaves: readonly SourceMapLeaf[]; } => { + const limits = options.limits ?? WEB_SOURCE_MAP_LIMITS; options.check?.(); const root = mapHeaderSchema.parse(value); const declarations: SourceMapSourceDeclaration[] = []; @@ -320,10 +343,10 @@ export const inspectSourceMapValue = ( options.check?.(); const node = pending.pop(); if (node === undefined) break; - if (node.path.length > WEB_SOURCE_MAP_LIMITS.sectionDepth) + if (node.path.length > limits.sectionDepth) throw new SourceMapFormatFailure( "limit", - "Indexed source-map nesting exceeds the 64-level decoder stack budget.", + `Indexed source-map nesting exceeds decoder admission. ${sourceMapLimitRecovery("section_depth", limits.sectionDepth)}`, ); const sections = classifySourceMapSections(node.value); if (sections.kind === "invalid") diff --git a/src/javascript/sourceMaps/SourceMapReply.ts b/src/javascript/sourceMaps/SourceMapReply.ts index 702738a41d..6910ced041 100644 --- a/src/javascript/sourceMaps/SourceMapReply.ts +++ b/src/javascript/sourceMaps/SourceMapReply.ts @@ -1,3 +1,4 @@ +import { WEB_SOURCE_MAP_LIMITS } from "../../domain/sourceMapLimits.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { AnalysisCapabilityUnavailableError, @@ -17,6 +18,7 @@ export const parseSourceMapReply = ( text: string, path: string, processMemoryLimitBytes: number | null, + heapMiB: number = WEB_SOURCE_MAP_LIMITS.codecHeapMiB, ): Result => { let value: unknown; try { @@ -42,6 +44,7 @@ export const parseSourceMapReply = ( const runtimeFailure = codecRuntimeFailure( reply.data.report.runtime, processMemoryLimitBytes, + heapMiB, ); return runtimeFailure === undefined ? ok(reply.data.report) @@ -52,6 +55,7 @@ export const parseSourceMapReply = ( const codecRuntimeFailure = ( runtime: WebSourceMapReport["runtime"], processMemoryLimitBytes: number | null, + heapMiB: number, ): string | undefined => { if ( runtime.id !== @@ -60,7 +64,7 @@ const codecRuntimeFailure = ( return `Codec reported runtime ${runtime.id}, not the runtime that launched it.`; if ( runtime.id === "node-v8" && - runtime.v8_heap_limit_bytes > 256 * 1024 * 1024 + runtime.v8_heap_limit_bytes > (heapMiB + 64) * 1024 * 1024 ) return "Codec reported a V8 heap limit exceeding its independent process budget."; if ( @@ -83,6 +87,7 @@ const codecFailure = ( "limit", undefined, `${path}: ${detail}`, + { remediationAction: detail }, ) : reason === "unsupported" ? new AnalysisCapabilityUnavailableError( diff --git a/src/javascript/sourceMaps/TraceSourceMap.ts b/src/javascript/sourceMaps/TraceSourceMap.ts index f71279cb78..e4aafac160 100644 --- a/src/javascript/sourceMaps/TraceSourceMap.ts +++ b/src/javascript/sourceMaps/TraceSourceMap.ts @@ -1,3 +1,8 @@ +import { + WEB_SOURCE_MAP_LIMITS, + sourceMapLimitRecovery, + type SourceMapLimits, +} from "../../domain/sourceMapLimits.js"; import { createHash } from "node:crypto"; import { createRequire } from "node:module"; import { getHeapStatistics } from "node:v8"; @@ -15,7 +20,6 @@ import { resolveJavaScriptSourceMapUrl } from "../../domain/javascript/javascrip import { webSourceMapReportSchema, type WebSourceMapReport, - WEB_SOURCE_MAP_LIMITS, } from "../../domain/webSourceLocation.js"; import type { WebSourcePosition } from "../../domain/webSourcePosition.js"; import { @@ -30,8 +34,9 @@ export const traceSourceMap = ( text: string, url: string, position: WebSourcePosition, + limits: SourceMapLimits = WEB_SOURCE_MAP_LIMITS, ): WebSourceMapReport => { - const inspected = inspectSourceMap(text); + const inspected = inspectSourceMap(text, { limits }); const decodedLeaves = decodeValidatedSourceMapLeaves(inspected.leaves, url); let map: TraceMap; if (inspected.format === "regular") { @@ -71,10 +76,10 @@ export const traceSourceMap = ( for (const segment of segments) { const match = originalMatch(segment, map, url, inspected.declarations); bytes += Buffer.byteLength(JSON.stringify(match)) + 1; - if (bytes > WEB_SOURCE_MAP_LIMITS.outputBytes) + if (bytes > limits.outputBytes) throw new SourceMapFormatFailure( "limit", - "Complete source-map point evidence exceeds the 32 MiB output budget; no partial evidence was returned.", + `Complete source-map point evidence exceeds output admission; no partial evidence was returned. ${sourceMapLimitRecovery("output_bytes", limits.outputBytes)}`, ); matches.push(match); } @@ -104,7 +109,7 @@ export const traceSourceMap = ( node_compatibility_version: runtime.nodeCompatibilityVersion, v8_heap_limit_bytes: null, process_memory_limit_bytes: javaScriptWorkerMemoryLimitBytes( - WEB_SOURCE_MAP_LIMITS.codecHeapMiB, + limits.codecHeapMiB, ), }, format: inspected.format, diff --git a/src/native/LldbCallTracer.ts b/src/native/LldbCallTracer.ts index 6955a627c1..92aebdfb79 100644 --- a/src/native/LldbCallTracer.ts +++ b/src/native/LldbCallTracer.ts @@ -7,10 +7,15 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { z } from "zod"; +import { + MAX_NATIVE_CALL_CAPTURE_BYTES, + nativeCallCaptureLimits, +} from "../domain/native/nativeCallCaptureLimits.js"; import { AnalysisCancelledError, AnalysisCapabilityUnavailableError, + AnalysisResourceConstraintError, AnalysisTimeoutError, } from "../domain/analysisErrorCore.js"; import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; @@ -67,8 +72,6 @@ const OPERATION = "observe_native_calls"; const PROVIDER = "native-macos"; const LLDB_REQUIREMENT = "LLDB from Xcode or the Command Line Tools is not available"; -/** Captured target output kept per stream; the rest is counted, not stored. */ -const MAX_OUTPUT_BYTES = 1024 * 1024; /** * LLDB startup, symbol loading and teardown beyond the observation window. * This grace has no runtime override; duration_ms is the caller-selected window. @@ -82,7 +85,7 @@ const lldbGraceRecovery = ( const total = durationMs + DEADLINE_GRACE_MS; return { userMessage: `observe_native_calls reached its ${String(total)} ms server/provider deadline: duration_ms ${String(durationMs)} plus a fixed ${String(DEADLINE_GRACE_MS)} ms LLDB startup, symbol loading, and teardown grace. This is not the MCP client deadline.`, - remediationAction: `This release has no runtime override for the ${String(DEADLINE_GRACE_MS)} ms grace. Increase duration_ms on the next call when the observation window was too short; the grace stays added to that window. Captured output beyond ${String(MAX_OUTPUT_BYTES)} bytes per stream is counted and not kept, and that byte allowance has no runtime override. Failure diagnostics keep at most ${String(DIAGNOSTIC_BYTES)} bytes and that excerpt has no runtime override. Allow execution, cleanup, and delivery headroom in the outer MCP client deadline. Confirm the owned process was killed and cleanup finished before retrying.`, + remediationAction: `This release has no runtime override for the ${String(DEADLINE_GRACE_MS)} ms grace. Increase duration_ms on the next call when the observation window was too short; the grace stays added to that window. Captured output retention is selected with capture_limits.output_bytes before launch; changing it cannot recover an earlier capture. Failure diagnostics keep at most ${String(DIAGNOSTIC_BYTES)} bytes and that excerpt has no runtime override. Allow execution, cleanup, and delivery headroom in the outer MCP client deadline. Confirm the owned process was killed and cleanup finished before retrying.`, }; }; @@ -113,6 +116,7 @@ const tracedSchema = z.strictObject({ stable: z.boolean(), }), resource_limit_reached: z.boolean(), + resource_limit: z.enum(["trace_bytes", "trace_frames"]).nullable().optional(), breakpoint_locations_truncated: z.boolean().optional(), target_output: z.strictObject({ stdout_bytes: z.int().nonnegative(), @@ -607,6 +611,7 @@ export class LldbCallTracer implements NativeCallTracer { const directory = retained.root.path; const paths = lldbRuntimePaths(directory); const { input } = request; + const captureLimits = nativeCallCaptureLimits(input.capture_limits); await writeFile( paths.config, JSON.stringify({ @@ -625,7 +630,7 @@ export class LldbCallTracer implements NativeCallTracer { max_events: input.max_events, argument_registers: input.argument_registers, backtrace_frames: input.backtrace_frames, - max_output_bytes: MAX_OUTPUT_BYTES, + capture_limits: captureLimits, stdout_path: paths.stdout, stderr_path: paths.stderr, stdout_capture_path: paths.stdoutCapture, @@ -725,6 +730,7 @@ export class LldbCallTracer implements NativeCallTracer { run.target_output.stdout_truncated, run.target_output.stdout_complete, retained.files, + captureLimits.output_bytes, ), stderr: await capturedOutput( paths.stderrCapture, @@ -732,6 +738,7 @@ export class LldbCallTracer implements NativeCallTracer { run.target_output.stderr_truncated, run.target_output.stderr_complete, retained.files, + captureLimits.output_bytes, ), }); const getPartialObservation = async ( @@ -855,6 +862,16 @@ export class LldbCallTracer implements NativeCallTracer { }, ), ); + const readbackFailure = + readback !== undefined && !readback.ok ? readback.error : undefined; + if ( + output === undefined && + readbackFailure instanceof AnalysisResourceConstraintError + ) + return failureWithPartial((partialObservation) => { + readbackFailure.partialObservation = partialObservation; + return readbackFailure; + }); if (output === undefined) return failureWithPartial( (partialObservation) => @@ -1056,13 +1073,29 @@ const partialNativeObservation = async (options: { readonly version: string | null; readonly reason: NativeCallPartialObservation["coverage"]["reason"]; }): Promise => { + const captureLimits = nativeCallCaptureLimits( + options.request.input.capture_limits, + ); const [journal, pid, stdout, stderr] = await Promise.all([ - readLldbObservationJournal(options.observationPath, options.files), + readLldbObservationJournal( + options.observationPath, + options.files, + captureLimits.trace_bytes, + ), options.files.readTargetPid(options.pidPath), - readPartialCapture(options.stdoutPath, options.files), - readPartialCapture(options.stderrPath, options.files), + readPartialCapture( + options.stdoutPath, + options.files, + captureLimits.output_bytes, + ), + readPartialCapture( + options.stderrPath, + options.files, + captureLimits.output_bytes, + ), ]); const observation = projectLldbPartialObservation({ + captureLimits, target: { path: options.request.executable, sha256: options.request.expectedSha256, @@ -1140,12 +1173,39 @@ const readTracerOutput = async ( path: string, files: LldbReadbackFiles, ): Promise, unknown>> => { - const bytes = await files.read(path, (file) => file.readFile()); + const bytes = await files.read(path, async (file) => { + const { size } = await file.stat(); + if (size > MAX_NATIVE_CALL_CAPTURE_BYTES) + return err( + new AnalysisResourceConstraintError( + OPERATION, + "file-size", + "LLDB tracer result exceeds the runtime Buffer/single-string representation ceiling", + { + maximum_result_bytes: MAX_NATIVE_CALL_CAPTURE_BYTES, + observed_result_bytes: size, + }, + { + retryable: false, + remediationAction: + "The combined LLDB result cannot fit the running Node Buffer/single-string representation. Select smaller capture_limits.trace_bytes or capture_limits.breakpoint_location_bytes, or fewer breakpoints before a new authorized observation. Increasing the client receive buffer cannot raise this ceiling. Retained journal/output observations remain partial; earlier omitted facts cannot be recovered.", + }, + ), + ); + const read = await readBoundedPrefix( + size, + async (buffer, offset, length, position) => + (await file.read(buffer, offset, length, position)).bytesRead, + ); + if (read.status === "read-failure") return err(read.cause); + return ok(read.buffer.subarray(0, read.bytesRead)); + }); if (!bytes.ok) return bytes; + if (!bytes.value.ok) return bytes.value; let text: string; try { text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - bytes.value, + bytes.value.value, ); } catch (cause: unknown) { return err(cause); @@ -1176,11 +1236,12 @@ const capturedOutput = async ( truncated: boolean, complete: boolean, files: LldbReadbackFiles, + maximumBytes: number, ): Promise => { const readback = await files.read(path, async (handle) => { const { size } = await handle.stat(); const read = await readBoundedPrefix( - Math.min(size, MAX_OUTPUT_BYTES), + Math.min(size, maximumBytes), async (target, offset, length, position) => (await handle.read(target, offset, length, position)).bytesRead, ); @@ -1192,13 +1253,11 @@ const capturedOutput = async ( text: buffer.subarray(0, bytesRead).toString("utf8"), bytes: observedBytes, truncated: - truncated || - size > MAX_OUTPUT_BYTES || - read.status === "read-failure", + truncated || size > maximumBytes || read.status === "read-failure", complete: read.status !== "read-failure" && complete && - bytesRead >= Math.min(observedBytes, MAX_OUTPUT_BYTES), + bytesRead >= Math.min(observedBytes, maximumBytes), }, ...(read.status === "read-failure" ? { readFailure: read.cause } : {}), }; diff --git a/src/native/LldbRetainedObservations.ts b/src/native/LldbRetainedObservations.ts index 5f2492f7f8..3fa72d86b4 100644 --- a/src/native/LldbRetainedObservations.ts +++ b/src/native/LldbRetainedObservations.ts @@ -1,3 +1,7 @@ +import { + NATIVE_CALL_CAPTURE_DEFAULTS, + type NativeCallCaptureLimits, +} from "../domain/native/nativeCallCaptureLimits.js"; import type { FileHandle } from "node:fs/promises"; import type { LldbReadbackFiles } from "./LldbReadbackFiles.js"; import type { Result } from "../domain/result.js"; @@ -5,10 +9,6 @@ import type { Result } from "../domain/result.js"; import { nativeCallEventSchema } from "../domain/native/nativeCallObservation.js"; import type { NativeCallPartialObservation } from "../domain/native/nativeCallPartialObservation.js"; -/** Maximum journal size enforced by the LLDB bridge. */ -export const LLDB_OBSERVATION_JOURNAL_MAX_BYTES = 8 * 1024 * 1024; -const MAX_CAPTURE_BYTES = 1024 * 1024; - export interface ParsedLldbJournal { readonly events: ReturnType[]; readonly otherStops: string[]; @@ -58,6 +58,7 @@ export type BoundedPrefixRead = /** Join safely retained journal and process evidence into its failure contract. */ export const projectLldbPartialObservation = (input: { + readonly captureLimits?: NativeCallCaptureLimits; readonly target: NativeCallPartialObservation["target"]; readonly pid: number | undefined; readonly stdout: NativeCallPartialObservation["process"]["stdout"]; @@ -69,6 +70,9 @@ export const projectLldbPartialObservation = (input: { readonly reason: NativeCallPartialObservation["coverage"]["reason"]; }): NativeCallPartialObservation => ({ kind: "native-call-observation", + ...(input.captureLimits === undefined + ? {} + : { capture_limits: input.captureLimits }), target: input.target, process: { pid: input.pid ?? null, @@ -209,19 +213,20 @@ const readPrefix = (handle: FileHandle, requestedBytes: number) => export const readLldbObservationJournal = ( path: string, files: LldbReadbackFiles, + maximumBytes: number = NATIVE_CALL_CAPTURE_DEFAULTS.trace_bytes, ): Promise> => files.read(path, async (handle) => { const { size } = await handle.stat(); - const length = Math.min(size, LLDB_OBSERVATION_JOURNAL_MAX_BYTES); + const length = Math.min(size, maximumBytes); const read = await readPrefix(handle, length); const { buffer, bytesRead } = read; if (read.status === "read-failure" && bytesRead === 0) throw read.cause; const value: LldbObservationJournalValue = { parsed: parseLldbObservationJournal(buffer.subarray(0, bytesRead)), limitations: [ - ...(size > LLDB_OBSERVATION_JOURNAL_MAX_BYTES + ...(size > maximumBytes ? [ - "The observation journal exceeded 8388608 bytes (8 MiB); only its bounded prefix was read. This release has no runtime override.", + `The observation journal exceeded capture_limits.trace_bytes (${String(maximumBytes)} bytes); only its bounded prefix was read. Select a larger trace_bytes budget before a new authorized observation; earlier omitted facts cannot be recovered.`, ] : []), ...(read.status === "eof" @@ -245,10 +250,11 @@ export const readLldbObservationJournal = ( export const readPartialCapture = ( path: string, files: LldbReadbackFiles, + maximumBytes: number = NATIVE_CALL_CAPTURE_DEFAULTS.output_bytes, ): Promise> => files.read(path, async (handle) => { const { size } = await handle.stat(); - const length = Math.min(size, MAX_CAPTURE_BYTES); + const length = Math.min(size, maximumBytes); const read = await readPrefix(handle, length); const { buffer, bytesRead } = read; if (read.status === "read-failure" && bytesRead === 0) throw read.cause; diff --git a/src/native/NativeCallObservation.ts b/src/native/NativeCallObservation.ts index d8ab0c683a..9ac33a2a1b 100644 --- a/src/native/NativeCallObservation.ts +++ b/src/native/NativeCallObservation.ts @@ -1,3 +1,4 @@ +import { nativeCallCaptureLimits } from "../domain/native/nativeCallCaptureLimits.js"; import { AnalysisCapabilityUnavailableError, AnalysisCancelledError, @@ -43,8 +44,8 @@ export const NATIVE_CALL_OBSERVATION_LIMITATIONS = [ export const NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION = "The breakpoint location list is incomplete; location_count gives the full matching count."; -export const NATIVE_CALL_BREAKPOINT_LOCATION_BUDGET_LIMITATION = - "Resolved breakpoint locations exceeded the aggregate 8388608-byte (8 MiB) metadata budget; omitted locations are counted in location_count. This metadata limit does not change the process outcome. This release has no runtime override."; +const breakpointLocationBudgetLimitation = (bytes: number): string => + `Resolved breakpoint locations exceeded capture_limits.breakpoint_location_bytes (${String(bytes)} bytes); omitted locations are counted in location_count. This metadata limit does not change the process outcome. Select a larger budget before a new authorized observation; earlier omitted facts cannot be recovered.`; /** Launch the active Mach-O under LLDB, record requested call entries, then stop it. */ export const observeNativeCalls = async ( @@ -122,6 +123,7 @@ const nativeCallFacts = ( trace: NativeCallReadback, ) => { const { run } = trace; + const limits = nativeCallCaptureLimits(input.capture_limits); const terminated = run.terminated && trace.terminated; const terminationSummary = terminated ? "the process was killed" @@ -131,6 +133,14 @@ const nativeCallFacts = ( .map(({ index }) => index); const eventLimit = run.outcome === "event-limit"; const resourceLimit = run.outcome === "resource-limit"; + const traceByteBudget = `capture_limits.trace_bytes (${String(limits.trace_bytes)} bytes)`; + const traceFrameBudget = `capture_limits.trace_frames (${String(limits.trace_frames)} frames)`; + const exhaustedTraceBudget = + run.resource_limit === "trace_bytes" + ? traceByteBudget + : run.resource_limit === "trace_frames" + ? traceFrameBudget + : `${traceByteBudget} or ${traceFrameBudget}`; const breakpointLocationLimit = run.breakpoint_locations_truncated === true; const partial = eventLimit || @@ -140,6 +150,7 @@ const nativeCallFacts = ( run.outcome === "stop-limit" || unresolved.length > 0; return { + capture_limits: limits, target: { path: target.path, sha256: target.sha256, @@ -183,6 +194,9 @@ const nativeCallFacts = ( status: partial ? "partial" : "complete", event_limit_reached: eventLimit, resource_limit_reached: resourceLimit, + ...(run.resource_limit == null + ? {} + : { capture_limit: run.resource_limit }), ...(breakpointLocationLimit ? { breakpoint_locations_truncated: true } : {}), @@ -198,7 +212,7 @@ const nativeCallFacts = ( : []), ...(resourceLimit ? [ - `Observation stopped before retaining the next complete event because the estimated 8388608-byte (8 MiB) trace-payload or 65536-frame aggregate budget was reached; ${terminationSummary}. This release has no runtime override for those budgets.`, + `Observation stopped before retaining the next complete event because ${exhaustedTraceBudget} was reached; ${terminationSummary}. Select a larger corresponding budget before a new authorized observation; earlier omitted calls cannot be recovered.`, ] : []), ...(run.outcome === "duration-elapsed" @@ -213,7 +227,7 @@ const nativeCallFacts = ( ]), ...(run.outcome === "stop-limit" ? [ - `Observation stopped after repeated signal or exception stops; ${terminationSummary}.`, + `Observation stopped at capture_limits.other_stops (${String(limits.other_stops)} signal or exception stops); ${terminationSummary}. Select a larger budget before a new authorized observation; earlier omitted stops cannot be recovered.`, ] : []), ...(unresolved.length === 0 @@ -228,14 +242,14 @@ const nativeCallFacts = ( ? [NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION] : []), ...(breakpointLocationLimit - ? [NATIVE_CALL_BREAKPOINT_LOCATION_BUDGET_LIMITATION] + ? [breakpointLocationBudgetLimitation(limits.breakpoint_location_bytes)] : []), ...(trace.stdout === null || trace.stderr === null ? ["One or more target output captures were unavailable."] : []), ...(trace.stdout?.truncated === true || trace.stderr?.truncated === true ? [ - "Target output beyond 1048576 bytes (1 MiB) per stream is counted but not kept. This release has no runtime override.", + `Target output beyond capture_limits.output_bytes (${String(limits.output_bytes)} bytes per stream) is counted but not kept. Select a larger budget before a new authorized observation; earlier omitted output cannot be recovered.`, ] : []), ...(trace.stdout?.complete === false || trace.stderr?.complete === false diff --git a/src/native/elf/KernelModuleParser.test.ts b/src/native/elf/KernelModuleParser.test.ts index 797f388b8c..d5013bed3f 100644 --- a/src/native/elf/KernelModuleParser.test.ts +++ b/src/native/elf/KernelModuleParser.test.ts @@ -171,32 +171,24 @@ it("keeps earlier .modinfo entries when the last span is unterminated", () => { }); }); -it("reports a missing separator, an empty key, and an over-long field per entry", () => { +it("reports a missing separator and an empty key as malformed entries", () => { const report = parse( kernelModuleFixture({ - modinfo: Buffer.concat([ - Buffer.from("license\0=GPL\0", "latin1"), - Buffer.from(`${"k".repeat(257)}=x\0`, "latin1"), - Buffer.from(`name=${"v".repeat(4097)}\0`, "latin1"), - ]), + modinfo: Buffer.from("license\0=GPL\0", "latin1"), }), ); expect(report.modinfo?.entries.map((entry) => entry.reason)).toEqual([ "modinfo_missing_separator", "modinfo_empty_key", - "modinfo_field_exceeds_limit", - "modinfo_field_exceeds_limit", ]); expect(report.coverage.status).toBe("partial"); expect(report.modinfo?.entries[1]).toMatchObject({ key: null, value: "GPL", }); - expect(report.modinfo?.entries[2]).toMatchObject({ key: null, value: "x" }); - expect(report.modinfo?.entries[3]).toMatchObject({ - key: "name", - value: null, - }); + expect( + report.modinfo?.entries.every((entry) => entry.status === "malformed"), + ).toBe(true); }); it("closes __versions when the section size is not a multiple of 64", () => { diff --git a/src/native/elf/KernelModuleParser.ts b/src/native/elf/KernelModuleParser.ts index 9d41244303..f02f2f0cf0 100644 --- a/src/native/elf/KernelModuleParser.ts +++ b/src/native/elf/KernelModuleParser.ts @@ -4,10 +4,11 @@ import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; import { elfObjectTypeName } from "../../domain/native/elfNotes.js"; import { KERNEL_MODULE_VERSION_ENTRY_BYTES, - MAX_KERNEL_MODULE_KEY_BYTES, + DEFAULT_KERNEL_MODULE_KEY_BYTES, MAX_KERNEL_MODULE_RAW_BYTES, MAX_KERNEL_MODULE_SECTION_NAME_BYTES, - MAX_KERNEL_MODULE_VALUE_BYTES, + DEFAULT_KERNEL_MODULE_VALUE_BYTES, + MAX_INSPECT_KERNEL_MODULE_FILE_BYTES, MODULE_SIGNATURE_INFO_BYTES, MODULE_SIGNATURE_MAGIC, MODULE_SIGNATURE_PKCS7_ID_TYPE, @@ -164,7 +165,7 @@ const parseModinfo = ( section: ElfSection | undefined, use: SectionUse, examined: { count: number }, - maxEntries: number, + input: InspectKernelModuleInput, ): NonNullable => { if (section === undefined) return { @@ -196,6 +197,8 @@ const parseModinfo = ( reason: "modinfo_section_compressed", }; const entries: NonNullable["entries"] = []; + let largestKeyBytes = 0; + let largestValueBytes = 0; let at = 0; while (at < stored.length) { if (stored[at] === 0) { @@ -208,25 +211,27 @@ const parseModinfo = ( const record = stored.subarray(at, stop); const raw = stored.subarray(at, terminated ? end + 1 : stored.length); examined.count += 1; - exceeds(examined.count, maxEntries); + exceeds(examined.count, input.max_entries); const separator = record.indexOf(0x3d); let key: string | null = null; let value: string | null = null; + const omittedFields: Array<"key" | "value"> = []; let reason: NonNullable["entries"][number]["reason"] = terminated ? null : "unterminated_modinfo_entry"; if (separator < 0) reason ??= "modinfo_missing_separator"; else { const keyBytes = record.subarray(0, separator); const valueBytes = record.subarray(separator + 1); + largestKeyBytes = Math.max(largestKeyBytes, keyBytes.length); + largestValueBytes = Math.max(largestValueBytes, valueBytes.length); if (keyBytes.length === 0) reason ??= "modinfo_empty_key"; - if ( - keyBytes.length > MAX_KERNEL_MODULE_KEY_BYTES || - valueBytes.length > MAX_KERNEL_MODULE_VALUE_BYTES - ) - reason ??= "modinfo_field_exceeds_limit"; - if (keyBytes.length > 0 && keyBytes.length <= MAX_KERNEL_MODULE_KEY_BYTES) + if (keyBytes.length > input.max_modinfo_key_bytes) + omittedFields.push("key"); + if (valueBytes.length > input.max_modinfo_value_bytes) + omittedFields.push("value"); + if (keyBytes.length > 0 && keyBytes.length <= input.max_modinfo_key_bytes) key = keyBytes.toString("latin1"); - if (valueBytes.length <= MAX_KERNEL_MODULE_VALUE_BYTES) + if (valueBytes.length <= input.max_modinfo_value_bytes) value = valueBytes.toString("latin1"); } entries.push({ @@ -241,10 +246,30 @@ const parseModinfo = ( raw_hex: rawHex(raw), status: reason === null ? "observed" : "malformed", reason, + omitted_fields: omittedFields, }); if (!terminated) break; at = end + 1; } + for (const [field, selected, required, defaultBytes] of [ + [ + "key", + input.max_modinfo_key_bytes, + largestKeyBytes, + DEFAULT_KERNEL_MODULE_KEY_BYTES, + ], + [ + "value", + input.max_modinfo_value_bytes, + largestValueBytes, + DEFAULT_KERNEL_MODULE_VALUE_BYTES, + ], + ] as const) { + if (required > selected) + use.limitations.push( + `.modinfo ${field} text exceeds max_modinfo_${field}_bytes=${String(selected)} bytes (request budget; default ${String(defaultBytes)}, minimum 1, maximum ${String(MAX_INSPECT_KERNEL_MODULE_FILE_BYTES)}). Omitted fields keep their source range and digest; omission does not establish malformed syntax. Retry inspect_kernel_module with max_modinfo_${field}_bytes at least ${String(required)}, or --max-modinfo-${field}-bytes on the CLI. Applies on the next call without a REA restart; input, entry and raw-hex budgets do not change.`, + ); + } return { status: "present", location, @@ -532,7 +557,7 @@ export const parseKernelModule = ( firstNamedSection(".modinfo"), use, examined, - input.max_entries, + input, ); const versions = parseVersions( bytes, @@ -611,6 +636,7 @@ export const parseKernelModule = ( unbacked > 0 || ksymtab.some((section) => section.reason !== null) || sectionIncomplete(modinfo) || + modinfo.entries.some((entry) => entry.omitted_fields.length > 0) || sectionIncomplete(versions) || modulePresence.status === "unsupported" || (modulePresence.status === "present" && modulePresence.location === null) || diff --git a/src/native/pwndbg/PwndbgCoreContext.ts b/src/native/pwndbg/PwndbgCoreContext.ts index 1d3c5eecb6..cc04055542 100644 --- a/src/native/pwndbg/PwndbgCoreContext.ts +++ b/src/native/pwndbg/PwndbgCoreContext.ts @@ -33,8 +33,12 @@ import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { capturedPwntoolsOutput, pwntoolsDecoderFailure, + pwntoolsBudgetRecovery, } from "../pwntools/PwntoolsFailures.js"; -import { PWNTOOLS_LIMITS } from "../pwntools/PwntoolsRelease.js"; +import { + resolvePwntoolsLimits, + type PwntoolsProducerLimits, +} from "../../domain/native/pwntoolsResourceLimits.js"; import { readPwntoolsFailureEvidence } from "../pwntools/PwntoolsResourceLimits.js"; import type { PwntoolsCommand } from "../pwntools/PwntoolsDecoder.js"; @@ -71,6 +75,7 @@ export const inspectPwndbgCore = async ({ runCommand, timeoutMs, probePath, + producerLimits = resolvePwntoolsLimits(), }: { readonly environment: Readonly; readonly architecture: RecordedCrash["architecture"]; @@ -81,6 +86,7 @@ export const inspectPwndbgCore = async ({ readonly runCommand: PwntoolsCommand; /** Effective `pwntools_timeout_ms` for this debugger command. */ readonly timeoutMs: number; + readonly producerLimits?: PwntoolsProducerLimits; readonly probePath?: ( path: string, kind: "file" | "directory", @@ -149,6 +155,7 @@ export const inspectPwndbgCore = async ({ architecture, gdbinit: entry, venv, + producer_limits: producerLimits, }), { flag: "wx", mode: 0o600 }, ); @@ -158,6 +165,7 @@ export const inspectPwndbgCore = async ({ providerId: "pwndbg-core", precedingOutput: diagnostics, timeoutMs, + producerLimits, }; const failure = (cause: unknown) => pwntoolsDecoderFailure( @@ -207,7 +215,7 @@ export const inspectPwndbgCore = async ({ try { raw = await readStableArtifact( replyPath, - PWNTOOLS_LIMITS.outputBytes, + producerLimits.output_bytes, options?.signal, undefined, resources, @@ -249,6 +257,9 @@ export const inspectPwndbgCore = async ({ `Core-only debugger reply failed: ${cause instanceof Error ? cause.message : String(cause)}`, { cause, + ...(cause instanceof ArtifactReaderFailure && cause.reason === "limit" + ? { remediationAction: pwntoolsBudgetRecovery(producerLimits) } + : {}), ...(cause instanceof ArtifactReaderFailure ? { cleanup: cause.cleanup } : {}), @@ -276,6 +287,9 @@ export const inspectPwndbgCore = async ({ throw pwndbgContextFinalizationFailure( new AnalysisOutputError(OPERATION, reply.message, { capturedOutput: recordedCrashStageOutput(diagnostics, output), + ...(reply.reason === "output-limit" + ? { remediationAction: pwntoolsBudgetRecovery(producerLimits) } + : {}), }), finalization, ); @@ -309,7 +323,12 @@ export const inspectPwndbgCore = async ({ "debugger", gdb, environment.REA_PWNTOOLS_PYTHON ?? "", - await readPwntoolsFailureEvidence(cause, stage, resources), + await readPwntoolsFailureEvidence( + cause, + stage, + resources, + producerLimits, + ), failureContext, ); throw primary; diff --git a/src/native/pwntools/PwntoolsDecoder.ts b/src/native/pwntools/PwntoolsDecoder.ts index 5bfbc1ea1f..6e609530b4 100644 --- a/src/native/pwntools/PwntoolsDecoder.ts +++ b/src/native/pwntools/PwntoolsDecoder.ts @@ -12,6 +12,7 @@ import { pwntoolsDecoderFailure, pwntoolsUnavailable, capturedPwntoolsOutput, + pwntoolsBudgetRecovery, } from "./PwntoolsFailures.js"; import { randomUUID } from "node:crypto"; import { access, stat, writeFile } from "node:fs/promises"; @@ -61,11 +62,18 @@ import type { } from "../../process/ProviderProcess.js"; import { PendingOperations } from "../../process/PendingOperations.js"; import { - pwntoolsResourceLimitsSchema, + pwntoolsResourceLimitsSchemaFor, readPwntoolsFailureEvidence, } from "./PwntoolsResourceLimits.js"; import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js"; -import { PWNTOOLS_LIMITS } from "./PwntoolsRelease.js"; +import { + PWNTOOLS_LIMITS, + pwntoolsProducerLimitsSchema, + resolvePwntoolsLimits, + type PwntoolsProducerLimitsInput, + type PwntoolsProducerLimits, +} from "../../domain/native/pwntoolsResourceLimits.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; import { PWNTOOLS_PROVIDER_IDENTITY } from "./PwntoolsProviderIdentity.js"; import { pwntoolsInputBytes } from "./PwntoolsInputLimit.js"; @@ -100,6 +108,7 @@ export interface PwntoolsDecoderProfile< Input extends { readonly path: string; readonly pwntools_timeout_ms?: number | undefined; + readonly producer_limits?: PwntoolsProducerLimitsInput | undefined; }, Payload, Report extends DecoderReport, @@ -130,6 +139,7 @@ export interface PwntoolsDecoderProfile< readonly options: ExecutionOptions | undefined; readonly runCommand: PwntoolsCommand; readonly timeoutMs: number; + readonly producerLimits: PwntoolsProducerLimits; }) => Promise; } @@ -138,6 +148,7 @@ export class PwntoolsDecoder< Input extends { readonly path: string; readonly pwntools_timeout_ms?: number | undefined; + readonly producer_limits?: PwntoolsProducerLimitsInput | undefined; }, Payload, Report extends DecoderReport, @@ -244,14 +255,36 @@ export class PwntoolsDecoder< options?: ExecutionOptions, ): Promise> { const environment = snapshotEnvironment(this.environment); + const selectedLimits = pwntoolsProducerLimitsSchema.safeParse( + input.producer_limits === undefined ? {} : input.producer_limits, + ); + if (!selectedLimits.success) + return err( + analysisInputErrorFromIssues( + this.profile.operation, + selectedLimits.error.issues.map((issue) => ({ + ...issue, + path: ["producer_limits", ...issue.path], + })), + input, + ), + ); const pending = await this.#retire(); if (pending !== undefined) return err(pending); const OPERATION = this.profile.operation; - const timeoutMs = input.pwntools_timeout_ms ?? PWNTOOLS_LIMITS.timeoutMs; + const timeoutMs = + input.pwntools_timeout_ms ?? PWNTOOLS_LIMITS.command_timeout_ms; + let producerLimits = { + ...resolvePwntoolsLimits(selectedLimits.data), + command_timeout_ms: timeoutMs, + }; const context = { operation: OPERATION, providerId: this.identity.id, timeoutMs, + get producerLimits() { + return producerLimits; + }, }; const replySchema = z.discriminatedUnion("ok", [ z.strictObject({ @@ -271,7 +304,9 @@ export class PwntoolsDecoder< "decoder", ]), message: z.string(), - reported_limits: pwntoolsResourceLimitsSchema.nullable().optional(), + reported_limits: pwntoolsResourceLimitsSchemaFor(producerLimits) + .nullable() + .optional(), }), ]); let root: Pick | undefined; @@ -324,7 +359,13 @@ export class PwntoolsDecoder< undefined, context, ); - const inputLimit = pwntoolsInputBytes(environment); + const inputLimit = + selectedLimits.data.input_bytes === undefined + ? pwntoolsInputBytes(environment) + : { + configured: true as const, + bytes: selectedLimits.data.input_bytes, + }; if (!inputLimit.configured) throw pwntoolsUnavailable( inputLimit.error, @@ -337,6 +378,10 @@ export class PwntoolsDecoder< }, ); phase = "artifact-read"; + producerLimits = { + ...resolvePwntoolsLimits(selectedLimits.data, inputLimit.bytes), + command_timeout_ms: timeoutMs, + }; selectedPath = input.path; const snapshot = await readStableArtifact( input.path, @@ -380,11 +425,15 @@ export class PwntoolsDecoder< } await writeFile( requestPath, - JSON.stringify({ snapshot_path: snapshotPath, reply_path: replyPath }), + JSON.stringify({ + snapshot_path: snapshotPath, + reply_path: replyPath, + producer_limits: producerLimits, + }), { flag: "wx", mode: 0o600 }, ); const execution = await this.#runCommand( - timeoutMs, + producerLimits, { command: environment.REA_PWNTOOLS_PYTHON ?? "", arguments: [ @@ -412,7 +461,7 @@ export class PwntoolsDecoder< try { file = await readStableArtifact( replyPath, - PWNTOOLS_LIMITS.outputBytes, + producerLimits.output_bytes, options?.signal, undefined, this.#resources, @@ -483,6 +532,12 @@ export class PwntoolsDecoder< { cause, capturedOutput, + ...(cause instanceof ArtifactReaderFailure && + cause.reason === "limit" + ? { + remediationAction: `Increase producer_limits.output_bytes above ${String(producerLimits.output_bytes)} bytes on the next call (CLI --producer-limits JSON); no restart is required. File, worker memory and transport limits remain independent.`, + } + : {}), ...(cause instanceof ArtifactReaderFailure ? { cleanup: cause.cleanup } : {}), @@ -503,11 +558,11 @@ export class PwntoolsDecoder< throw new AnalysisResourceConstraintError( OPERATION, "memory", - `${reply.message} cpuSeconds is ${String(PWNTOOLS_LIMITS.cpuSeconds)}. Output is ${String(PWNTOOLS_LIMITS.outputBytes)} bytes, diagnostics are ${String(PWNTOOLS_LIMITS.diagnosticBytes)} bytes, and address space is ${String(PWNTOOLS_LIMITS.addressSpaceBytes)} bytes. This release has no runtime override.`, + `${reply.message} ${pwntoolsBudgetRecovery(producerLimits)}`, reply.reported_limits ?? null, { capturedOutput, - remediationAction: `Review the reported worker memory limits and available host memory. cpuSeconds ${String(PWNTOOLS_LIMITS.cpuSeconds)}, the ${String(PWNTOOLS_LIMITS.outputBytes)}-byte output allowance, the ${String(PWNTOOLS_LIMITS.diagnosticBytes)}-byte diagnostic allowance, and the ${String(PWNTOOLS_LIMITS.addressSpaceBytes)}-byte address-space allowance have no runtime override. pwntools_timeout_ms does not raise them. Repeating the unchanged request fails the same way.`, + remediationAction: `Review reported limits and available host memory; allocation failure alone does not prove exhaustion. ${pwntoolsBudgetRecovery(producerLimits)}`, }, ); if (reply.reason === "format") @@ -536,6 +591,7 @@ export class PwntoolsDecoder< if (reply.reason === "output-limit") throw new AnalysisOutputError(OPERATION, reply.message, { capturedOutput, + remediationAction: `Increase producer_limits.output_bytes above ${String(producerLimits.output_bytes)} bytes on the next call (CLI --producer-limits JSON); no restart is required. File output follows the selected output allowance unless file_bytes is explicit. Tighter inherited limits and transport remain independent.`, }); throw new ProviderAdapterError(this.identity.id, OPERATION, { diagnostics: { @@ -573,8 +629,14 @@ export class PwntoolsDecoder< resources: this.#resources, options, runCommand: (spawn, classifyFailure) => - this.#runCommand(timeoutMs, spawn, options, classifyFailure), + this.#runCommand( + producerLimits, + spawn, + options, + classifyFailure, + ), timeoutMs, + producerLimits, }), ); } @@ -605,6 +667,7 @@ export class PwntoolsDecoder< cause, root?.path, this.#resources, + producerLimits, ); const primary = pwntoolsDecoderFailure( cause, @@ -650,7 +713,7 @@ export class PwntoolsDecoder< } async #runCommand( - timeoutMs: number, + limits: PwntoolsProducerLimits, spawn: OwnedProviderProcessSpawnOptions, options: ExecutionOptions | undefined, failure: (cause: unknown) => AnalysisError, @@ -671,8 +734,8 @@ export class PwntoolsDecoder< expectedCommand: spawn.expectedCommand ?? null, }, { - timeoutMs, - diagnosticBytes: PWNTOOLS_LIMITS.diagnosticBytes, + timeoutMs: limits.command_timeout_ms, + diagnosticBytes: limits.diagnostic_bytes, }, { ...(options?.signal === undefined diff --git a/src/native/pwntools/PwntoolsFailures.test.ts b/src/native/pwntools/PwntoolsFailures.test.ts index fb3840b066..c11f5ff80d 100644 --- a/src/native/pwntools/PwntoolsFailures.test.ts +++ b/src/native/pwntools/PwntoolsFailures.test.ts @@ -29,7 +29,7 @@ it.each([undefined, null, false, 0, "stop", { stop: true }, new Error("stop")])( }, ); -it("names pwntools_timeout_ms and leaves cpuSeconds and byte allowances fixed", () => { +it("distinguishes the wall deadline from caller-selected producer resource budgets", () => { const projected = projectAnalysisError( pwntoolsDecoderFailure( new OwnedCommandFailure("timeout", "deadline expired"), @@ -48,8 +48,9 @@ it("names pwntools_timeout_ms and leaves cpuSeconds and byte allowances fixed", expect(projected.message).toContain("4500 ms"); expect(projected.message).toContain("pwntools_timeout_ms"); expect(projected.remediation.action).toContain("--pwntools-timeout-ms"); - expect(projected.remediation.action).toContain("cpuSeconds is 30"); - expect(projected.remediation.action).toContain("no runtime override"); + expect(projected.remediation.action).toContain("cpu_seconds=30"); + expect(projected.remediation.action).toContain("producer_limits"); + expect(projected.remediation.action).toContain("no REA restart"); expect(projected.remediation.action).toContain("67108864"); }); @@ -80,12 +81,12 @@ it("names the reported file-size allowance instead of address space", () => { expect(projected.code).toBe("resource_constraint"); expect(projected.message).toContain("file_size_bytes"); expect(projected.message).toContain("4096"); - expect(projected.message).toContain("outputBytes"); + expect(projected.message).toContain("producer_limits.file_bytes"); expect(projected.message).toContain("67108864"); expect(projected.remediation.action).toContain("4096"); expect(projected.remediation.action).toContain("67108864"); expect(projected.remediation.action).toContain("tighter inherited limit"); - expect(projected.remediation.action).not.toContain("address-space"); + expect(projected.remediation.action).not.toContain("address_space_bytes"); expect(projected.remediation.action).not.toContain("1111"); expect(projected.remediation.action).not.toContain("3221225472"); }); @@ -105,12 +106,12 @@ it("names the file-size cap when SIGXFSZ has no reported limit", () => { { marker: { resource: "file-size", failure: null } }, ), ); - expect(projected.remediation.action).toContain("outputBytes"); + expect(projected.remediation.action).toContain("producer_limits.file_bytes"); expect(projected.remediation.action).toContain("67108864"); expect(projected.remediation.action).toContain("file_size_bytes"); expect(projected.remediation.action).toContain("not reported"); expect(projected.remediation.action).toContain("tighter inherited limit"); - expect(projected.remediation.action).not.toContain("address-space"); + expect(projected.remediation.action).not.toContain("address_space_bytes"); expect(projected.remediation.action).not.toContain("3221225472"); }); @@ -132,7 +133,7 @@ it("names memory in the marker recovery for an allocation failure", () => { expect(projected.code).toBe("resource_constraint"); expect(projected.details).toMatchObject({ resource: "memory" }); expect(projected.remediation.action).toContain("memory"); - expect(projected.remediation.action).toContain("address-space"); + expect(projected.remediation.action).toContain("address_space_bytes"); expect(projected.remediation.action).toContain("3221225472"); }); diff --git a/src/native/pwntools/PwntoolsFailures.ts b/src/native/pwntools/PwntoolsFailures.ts index 721a4869a3..d692a92a69 100644 --- a/src/native/pwntools/PwntoolsFailures.ts +++ b/src/native/pwntools/PwntoolsFailures.ts @@ -1,3 +1,8 @@ +import { + PWNTOOLS_LIMITS, + resolvePwntoolsLimits, + type PwntoolsProducerLimits, +} from "../../domain/native/pwntoolsResourceLimits.js"; import { artifactFileCancellationCause, bindArtifactFileCleanup, @@ -26,7 +31,6 @@ import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; import type { PwntoolsFailureEvidence } from "./PwntoolsResourceLimits.js"; import { nativeDeadlineRecovery } from "../../domain/native/nativeDeadlines.js"; import { - PWNTOOLS_LIMITS, PWNTOOLS_MEMORY_FAILURE_EXIT, PWNTOOLS_FILE_SIZE_FAILURE_EXIT, } from "./PwntoolsRelease.js"; @@ -39,18 +43,26 @@ export interface PwntoolsFailureContext { readonly precedingOutput?: AnalysisCapturedOutput; readonly configurationKey?: string; readonly timeoutMs?: number; + readonly producerLimits?: PwntoolsProducerLimits; } -const fixedPwntoolsBudgets = `cpuSeconds is ${String(PWNTOOLS_LIMITS.cpuSeconds)}. The output allowance is ${String(PWNTOOLS_LIMITS.outputBytes)} bytes, the diagnostic allowance is ${String(PWNTOOLS_LIMITS.diagnosticBytes)} bytes, and the address-space allowance is ${String(PWNTOOLS_LIMITS.addressSpaceBytes)} bytes. This release has no runtime override for those budgets. pwntools_timeout_ms does not raise them. Artifact admission stays REA_PWNTOOLS_MAX_INPUT_BYTES (default ${String(PWNTOOLS_LIMITS.inputBytes)} bytes).`; +/** Recovery identifies the selected controls rather than unrelated wall deadlines. */ +export const pwntoolsBudgetRecovery = ( + limits: PwntoolsProducerLimits = resolvePwntoolsLimits(), +): string => + `Selected producer_limits: cpu_seconds=${String(limits.cpu_seconds)} seconds, output_bytes=${String(limits.output_bytes)}, diagnostic_bytes=${String(limits.diagnostic_bytes)}, address_space_bytes=${String(limits.address_space_bytes)}, file_bytes=${String(limits.file_bytes)}, input_bytes=${String(limits.input_bytes)} bytes. Increase the exhausted producer_limits field on the next call (CLI --producer-limits JSON); no REA restart is required. Tighter inherited OS limits remain enforced and are not raised by this request. pwntools_timeout_ms changes only the wall deadline.`; /** Name the file-size cap and any reported effective limit, not address space. */ -const fileSizeAllowance = (reportedBytes: number | undefined): string => { - const cap = String(PWNTOOLS_LIMITS.outputBytes); +const fileSizeAllowance = ( + reportedBytes: number | undefined, + selectedBytes: number, +): string => { + const cap = String(selectedBytes); const inherited = - "A tighter inherited limit is retained. This release has no runtime override. pwntools_timeout_ms does not raise the file-size allowance."; + "A tighter inherited limit is retained. Increase producer_limits.file_bytes on the next call (CLI --producer-limits JSON); no restart is required. pwntools_timeout_ms does not raise the file-size allowance."; return reportedBytes === undefined - ? `The file-size cap is ${cap} bytes (outputBytes). The effective file_size_bytes limit was not reported. ${inherited}` - : `The effective file-size allowance is ${String(reportedBytes)} bytes (file_size_bytes). The file-size cap is ${cap} bytes (outputBytes). ${inherited}`; + ? `The file-size cap is ${cap} bytes (producer_limits.file_bytes). The effective file_size_bytes limit was not reported. ${inherited}` + : `The effective file-size allowance is ${String(reportedBytes)} bytes (file_size_bytes). The file-size cap is ${cap} bytes (producer_limits.file_bytes). ${inherited}`; }; const DEFAULT_CONTEXT = { @@ -91,6 +103,8 @@ const classifyPwntoolsDecoderFailure = ( context: PwntoolsFailureContext, ): AnalysisError => { const limitReport = failureEvidence.limits; + const limits = context.producerLimits ?? resolvePwntoolsLimits(); + const budgetRecovery = pwntoolsBudgetRecovery(limits); if (cause instanceof AnalysisError) return cause; if (cause instanceof OwnedCommandFailure) { const currentOutput = @@ -152,24 +166,24 @@ const classifyPwntoolsDecoderFailure = ( : cause.cancellation.reason, }); if (cause.reason === "timeout") { - const timeoutMs = context.timeoutMs ?? PWNTOOLS_LIMITS.timeoutMs; + const timeoutMs = context.timeoutMs ?? PWNTOOLS_LIMITS.command_timeout_ms; return new AnalysisTimeoutError(context.operation, timeoutMs, { ...outputOptions, ...nativeDeadlineRecovery({ timeoutMs, setting: "pwntools_timeout_ms", cliFlag: "--pwntools-timeout-ms", - defaultMs: PWNTOOLS_LIMITS.timeoutMs, + defaultMs: PWNTOOLS_LIMITS.command_timeout_ms, phase: "pwntools command deadline", - unchanged: fixedPwntoolsBudgets, + unchanged: budgetRecovery, }), }); } if (cause.reason === "output-limit") return new AnalysisOutputError(context.operation, cause.message, { ...outputOptions, - userMessage: `${cause.message} ${fixedPwntoolsBudgets}`, - remediationAction: fixedPwntoolsBudgets, + userMessage: `${cause.message} ${budgetRecovery}`, + remediationAction: budgetRecovery, }); if ( cause.reason === "process" && @@ -178,7 +192,10 @@ const classifyPwntoolsDecoderFailure = ( cause.snapshot?.exitCode === PWNTOOLS_FILE_SIZE_FAILURE_EXIT && cause.snapshot.signal === null)) ) { - const allowance = fileSizeAllowance(limitReport?.limits?.file_size_bytes); + const allowance = fileSizeAllowance( + limitReport?.limits?.file_size_bytes, + limits.file_bytes, + ); return new AnalysisResourceConstraintError( context.operation, "file-size", @@ -204,11 +221,11 @@ const classifyPwntoolsDecoderFailure = ( (limitReport?.failure === null || limitReport === undefined ? "" : ` Effective limit report unavailable: ${limitReport.failure}`) + - ` cpuSeconds is ${String(PWNTOOLS_LIMITS.cpuSeconds)}. This release has no runtime override.`, + ` Selected producer_limits.cpu_seconds is ${String(limits.cpu_seconds)} seconds; inherited tighter limits remain enforced.`, limitReport?.limits ?? null, { ...outputOptions, - remediationAction: `Review the reported worker CPU limits and observed signal. cpuSeconds is ${String(PWNTOOLS_LIMITS.cpuSeconds)}. This release has no runtime override. pwntools_timeout_ms does not raise it. Repeating the unchanged request fails the same way.`, + remediationAction: `Review the reported worker CPU limit and observed signal; the signal alone does not establish exhaustion. Increase producer_limits.cpu_seconds above ${String(limits.cpu_seconds)} on the next call (CLI --producer-limits JSON) when that policy was limiting. No restart is required; a tighter inherited limit remains enforced. pwntools_timeout_ms does not raise CPU admission.`, }, ); if ( @@ -224,11 +241,11 @@ const classifyPwntoolsDecoderFailure = ( (limitReport?.failure === null || limitReport === undefined ? "" : ` Effective limit report unavailable: ${limitReport.failure}`) + - ` The address-space allowance is ${String(PWNTOOLS_LIMITS.addressSpaceBytes)} bytes. This release has no runtime override.`, + ` The selected producer_limits.address_space_bytes allowance is ${String(limits.address_space_bytes)} bytes; tighter inherited limits remain enforced.`, limitReport?.limits ?? null, { ...outputOptions, - remediationAction: `Review the reported worker memory limits and available host memory. ${fixedPwntoolsBudgets}`, + remediationAction: `Review the reported worker memory limits and available host memory. ${budgetRecovery}`, }, ); } @@ -278,7 +295,8 @@ const classifyPwntoolsDecoderFailure = ( ...stableArtifactInputRecovery(cause), ...(cause.reason === "limit" ? { - userMessage: `${cause.message} Admitted artifact bytes use REA_PWNTOOLS_MAX_INPUT_BYTES (default ${String(PWNTOOLS_LIMITS.inputBytes)} bytes, maximum 1073741824). The 1073741824-byte maximum has no runtime override. Restart REA after changing the environment value. pwntools_timeout_ms does not raise this allowance. ${fixedPwntoolsBudgets}`, + userMessage: `${cause.message} Selected producer_limits.input_bytes is ${String(limits.input_bytes)} bytes. ${budgetRecovery}`, + remediationAction: `Increase producer_limits.input_bytes above ${String(limits.input_bytes)} on the next call (CLI --producer-limits JSON). No restart is required. Changing REA_PWNTOOLS_MAX_INPUT_BYTES affects the launch default instead. CPU, memory, output and transport allowances remain independent.`, } : {}), }, @@ -289,7 +307,7 @@ const classifyPwntoolsDecoderFailure = ( cause.reason === "limit" ? "out_of_range" : "invalid_format", message: cause.reason === "limit" - ? `${cause.message} REA_PWNTOOLS_MAX_INPUT_BYTES defaults to ${String(PWNTOOLS_LIMITS.inputBytes)} bytes.` + ? `${cause.message} REA_PWNTOOLS_MAX_INPUT_BYTES defaults to ${String(PWNTOOLS_LIMITS.input_bytes)} bytes.` : cause.message, }, ], diff --git a/src/native/pwntools/PwntoolsInputLimit.test.ts b/src/native/pwntools/PwntoolsInputLimit.test.ts index 2595c03c50..8539a815c4 100644 --- a/src/native/pwntools/PwntoolsInputLimit.test.ts +++ b/src/native/pwntools/PwntoolsInputLimit.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from "vitest"; import { pwntoolsInputBytes } from "./PwntoolsInputLimit.js"; import { pwntoolsUnavailable } from "./PwntoolsFailures.js"; -import { PWNTOOLS_LIMITS } from "./PwntoolsRelease.js"; +import { PWNTOOLS_LIMITS } from "../../domain/native/pwntoolsResourceLimits.js"; import { PWNTOOLS_PROVIDER_IDENTITY } from "./PwntoolsProviderIdentity.js"; const CONSTRAINT = @@ -13,9 +13,9 @@ describe("pwntools input bound", () => { it("keeps the shipped default when the override is absent", () => { expect(pwntoolsInputBytes({})).toEqual({ configured: true, - bytes: PWNTOOLS_LIMITS.inputBytes, + bytes: PWNTOOLS_LIMITS.input_bytes, }); - expect(PWNTOOLS_LIMITS.inputBytes).toBe(32 * 1024 * 1024); + expect(PWNTOOLS_LIMITS.input_bytes).toBe(32 * 1024 * 1024); }); it.each([ @@ -62,7 +62,7 @@ describe("pwntools input bound", () => { }); it("never narrows the bound below the shipped default implicitly", () => { - const accepted = [1, 1024, PWNTOOLS_LIMITS.inputBytes].map((bytes) => + const accepted = [1, 1024, PWNTOOLS_LIMITS.input_bytes].map((bytes) => pwntoolsInputBytes({ REA_PWNTOOLS_MAX_INPUT_BYTES: String(bytes) }), ); expect(accepted.every((limit) => limit.configured)).toBe(true); diff --git a/src/native/pwntools/PwntoolsInputLimit.ts b/src/native/pwntools/PwntoolsInputLimit.ts index 283b2d0f11..8a903e2b3c 100644 --- a/src/native/pwntools/PwntoolsInputLimit.ts +++ b/src/native/pwntools/PwntoolsInputLimit.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { PWNTOOLS_LIMITS } from "./PwntoolsRelease.js"; +import { PWNTOOLS_LIMITS } from "../../domain/native/pwntoolsResourceLimits.js"; // The artifact is read into one in-memory buffer and staged again as a private // snapshot before the owned bridge parses it under a 3 GiB address-space @@ -32,7 +32,7 @@ export const pwntoolsInputBytes = ( ): PwntoolsInputBytes => { const value = environment.REA_PWNTOOLS_MAX_INPUT_BYTES; if (value === undefined) - return { configured: true, bytes: PWNTOOLS_LIMITS.inputBytes }; + return { configured: true, bytes: PWNTOOLS_LIMITS.input_bytes }; const parsed = pwntoolsInputBytesSchema.safeParse(value); return parsed.success ? { configured: true, bytes: parsed.data } diff --git a/src/native/pwntools/PwntoolsRecordedCrashProvider.ts b/src/native/pwntools/PwntoolsRecordedCrashProvider.ts index 92fd9c1bc4..c11eaea4b3 100644 --- a/src/native/pwntools/PwntoolsRecordedCrashProvider.ts +++ b/src/native/pwntools/PwntoolsRecordedCrashProvider.ts @@ -127,6 +127,7 @@ export class PwntoolsRecordedCrashProvider options, runCommand, timeoutMs, + producerLimits, }) => { if (!input.include_debugger_context) return value; const diagnostics = value.decoder_diagnostics; @@ -162,6 +163,7 @@ export class PwntoolsRecordedCrashProvider options, runCommand, timeoutMs, + producerLimits, }); const context = completed.context; const enriched = validate(context); diff --git a/src/native/pwntools/PwntoolsRelease.ts b/src/native/pwntools/PwntoolsRelease.ts index b52baeb367..f4b3f35625 100644 --- a/src/native/pwntools/PwntoolsRelease.ts +++ b/src/native/pwntools/PwntoolsRelease.ts @@ -1,20 +1,4 @@ -import { DEFAULT_PWNTOOLS_TIMEOUT_MS } from "../../domain/native/nativeDeadlines.js"; - /** Bridge-reserved status for MemoryError when no structured reply can be written. */ export const PWNTOOLS_MEMORY_FAILURE_EXIT = 75; /** Bridge-reserved status for an observed EFBIG failure without a complete reply. */ export const PWNTOOLS_FILE_SIZE_FAILURE_EXIT = 76; - -/** - * Complete evidence budgets; address-space is separate from resident memory. - * `timeoutMs` is the omitted `pwntools_timeout_ms`. cpuSeconds and the byte - * allowances stay fixed. - */ -export const PWNTOOLS_LIMITS = { - inputBytes: 32 * 1024 * 1024, - outputBytes: 64 * 1024 * 1024, - diagnosticBytes: 1024 * 1024, - timeoutMs: DEFAULT_PWNTOOLS_TIMEOUT_MS, - addressSpaceBytes: 3 * 1024 * 1024 * 1024, - cpuSeconds: 30, -} as const; diff --git a/src/native/pwntools/PwntoolsResourceLimits.ts b/src/native/pwntools/PwntoolsResourceLimits.ts index 47a52563d2..fad02005a4 100644 --- a/src/native/pwntools/PwntoolsResourceLimits.ts +++ b/src/native/pwntools/PwntoolsResourceLimits.ts @@ -1,3 +1,7 @@ +import { + resolvePwntoolsLimits, + type PwntoolsProducerLimits, +} from "../../domain/native/pwntoolsResourceLimits.js"; import { safeParseUtf8Json } from "../../domain/safeJson.js"; import { join } from "node:path"; import { @@ -8,21 +12,27 @@ import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; import { PWNTOOLS_MEMORY_FAILURE_EXIT, PWNTOOLS_FILE_SIZE_FAILURE_EXIT, - PWNTOOLS_LIMITS, } from "./PwntoolsRelease.js"; import { z } from "zod"; import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; /** Actual limits reported by the owned bridge after lowering inherited soft limits. */ export const pwntoolsResourceLimitsSchema = z.strictObject({ - address_space_bytes: z - .int() - .nonnegative() - .max(PWNTOOLS_LIMITS.addressSpaceBytes), - cpu_seconds: z.int().nonnegative().max(PWNTOOLS_LIMITS.cpuSeconds), - file_size_bytes: z.int().nonnegative().max(PWNTOOLS_LIMITS.outputBytes), + address_space_bytes: z.int().nonnegative(), + cpu_seconds: z.int().nonnegative(), + file_size_bytes: z.int().nonnegative(), }); +/** A report must fit this operation's selected ceilings, including raised budgets. */ +export const pwntoolsResourceLimitsSchemaFor = ( + limits: PwntoolsProducerLimits, +) => + pwntoolsResourceLimitsSchema.extend({ + address_space_bytes: z.int().nonnegative().max(limits.address_space_bytes), + cpu_seconds: z.int().nonnegative().max(limits.cpu_seconds), + file_size_bytes: z.int().nonnegative().max(limits.file_bytes), + }); + /** A missing or invalid report leaves limits unknown and retains its read failure. */ export interface PwntoolsLimitReport { readonly limits: z.output | null; @@ -44,6 +54,7 @@ export const readPwntoolsFailureEvidence = async ( cause: unknown, rootPath?: string, resources?: ArtifactResourceScope, + selectedLimits: PwntoolsProducerLimits = resolvePwntoolsLimits(), ): Promise => { const finalizationFailures: Record = {}; let limitReport: PwntoolsLimitReport | undefined; @@ -115,7 +126,9 @@ export const readPwntoolsFailureEvidence = async ( const decoded = safeParseUtf8Json(reported.bytes); if (!decoded.ok) throw decoded.cause; limitReport = { - limits: pwntoolsResourceLimitsSchema.parse(decoded.value), + limits: pwntoolsResourceLimitsSchemaFor(selectedLimits).parse( + decoded.value, + ), failure: null, }; } catch (reportFailure: unknown) { diff --git a/src/pcap/PcapNativeBound.test.ts b/src/pcap/PcapNativeBound.test.ts index 87b66a6229..5e5572cd64 100644 --- a/src/pcap/PcapNativeBound.test.ts +++ b/src/pcap/PcapNativeBound.test.ts @@ -1,3 +1,4 @@ +import { BOUNDED_PRODUCER_LIMITS } from "../domain/producerResourceLimits.js"; import { chmod, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; @@ -9,7 +10,6 @@ import { AnalysisOutputError, } from "../domain/analysisErrorCore.js"; import { - BOUNDED_PRODUCER_LIMITS, prlimitCeilingArguments, type LinuxResourceLimits, } from "../process/LinuxResourceLimits.js"; diff --git a/src/pcap/PcapNativeBound.ts b/src/pcap/PcapNativeBound.ts index d14e0885b8..ac90039d56 100644 --- a/src/pcap/PcapNativeBound.ts +++ b/src/pcap/PcapNativeBound.ts @@ -1,3 +1,4 @@ +import { BOUNDED_PRODUCER_LIMITS } from "../domain/producerResourceLimits.js"; import { selectArtifactCommandPath } from "../process/ArtifactCommandPath.js"; import { pcapCaptureHostSupported, @@ -16,10 +17,7 @@ import { AnalysisOutputError, } from "../domain/analysisErrorCore.js"; import type { ArtifactAnalysisResources } from "../process/ArtifactAnalysisResources.js"; -import { - BOUNDED_PRODUCER_LIMITS, - prlimitCeilingArguments, -} from "../process/LinuxResourceLimits.js"; +import { prlimitCeilingArguments } from "../process/LinuxResourceLimits.js"; import type { runPcapCommand } from "./PcapCommand.js"; import type { ArtifactCommandPath } from "../process/ArtifactCommandPath.js"; diff --git a/src/process/ArtifactAnalysisResources.ts b/src/process/ArtifactAnalysisResources.ts index 30dc6ae23a..587c24b96f 100644 --- a/src/process/ArtifactAnalysisResources.ts +++ b/src/process/ArtifactAnalysisResources.ts @@ -45,6 +45,7 @@ import type { ProviderProcessSupervisor, } from "./ProviderProcess.js"; import { OwnedCommandFailure, ownedCommandResources } from "./OwnedCommand.js"; +import { MAX_NODE_TIMER_DELAY_MS } from "../domain/analysisResourceLimits.js"; export type ArtifactAnalysisRuntime = Pick< PrivateRuntimeRoot, @@ -57,6 +58,7 @@ interface SelectedArtifactRead { readonly allowanceName: string; readonly observedOutput: AnalysisCapturedOutput | undefined; readonly signal?: AbortSignal | undefined; + readonly budgetParameter?: "producer_limits.input_bytes"; } export const artifactCommandOutput = ( @@ -207,7 +209,12 @@ export class ArtifactAnalysisResources { { input_bytes: maximumBytes }, { cause, - ...fixedProducerInputAllowance(maximumBytes, allowanceName), + ...(input.budgetParameter === undefined + ? fixedProducerInputAllowance(maximumBytes, allowanceName) + : { + retryable: false, + remediationAction: `Increase ${input.budgetParameter} above ${String(maximumBytes)} bytes on the next call to admit the same complete selected artifact; no REA restart is required. Do not truncate required input bytes. Output, parser, transport and inherited OS limits remain independent.`, + }), }, ); if (cause instanceof ArtifactReaderFailure && cause.reason === "integrity") @@ -501,6 +508,7 @@ export class ArtifactAnalysisResources { }, effectiveLimits: LinuxResourceLimits | undefined, operation = this.operation, + budgetParameter?: "producer_limits", ): AnalysisError { const capturedOutput = cause.snapshot === null @@ -519,8 +527,11 @@ export class ArtifactAnalysisResources { if (cause.reason === "timeout") return new AnalysisTimeoutError(operation, limits.command_timeout_ms, { ...detail, - userMessage: `The owned producer command reached its fixed ${String(limits.command_timeout_ms)} ms deadline. This is not a client-side timeout; extending only the MCP client's deadline does not change this budget.`, - remediationAction: `REA's fixed producer command deadline is ${String(limits.command_timeout_ms)} ms and this release has no runtime override. Inspect any partial output and cleanup before retrying. Repeating the unchanged request reaches the same deadline.`, + userMessage: `The owned producer command reached its ${String(limits.command_timeout_ms)} ms deadline. This is not a client-side timeout; extending only the MCP client's deadline does not change this budget.`, + remediationAction: + budgetParameter === undefined + ? `REA's fixed producer command deadline is ${String(limits.command_timeout_ms)} ms and this release has no runtime override. Inspect any partial output and cleanup before retrying. Repeating the unchanged request reaches the same deadline.` + : `${limits.command_timeout_ms < MAX_NODE_TIMER_DELAY_MS ? `Increase ${budgetParameter}.command_timeout_ms above ${String(limits.command_timeout_ms)} milliseconds on the next call (range 1..${String(MAX_NODE_TIMER_DELAY_MS)}); no restart is required.` : `${budgetParameter}.command_timeout_ms is already at the runtime timer maximum ${String(MAX_NODE_TIMER_DELAY_MS)} milliseconds; no higher deadline is supported. Narrow the selected work if that preserves the required observation.`} Inspect retained observations and confirm cleanup before retrying. This does not raise CPU, memory, output, parser or external client limits.`, }); if (cause.reason === "output-limit") return new AnalysisResourceConstraintError( @@ -530,10 +541,15 @@ export class ArtifactAnalysisResources { { output_bytes: limits.output_bytes }, { ...detail, - ...fixedAllowanceOptions( - `The retained command-output allowance of ${String(limits.output_bytes)} bytes (this release has no runtime override)`, - "capture", - ), + ...(budgetParameter === undefined + ? fixedAllowanceOptions( + `The retained command-output allowance of ${String(limits.output_bytes)} bytes (this release has no runtime override)`, + "capture", + ) + : { + retryable: false, + remediationAction: `Increase ${budgetParameter}.output_bytes above ${String(limits.output_bytes)} bytes on the next call; no restart is required. Inspect captured output and confirm cleanup before retrying. This controls collection, not parser, transport, host memory or inherited OS limits. Repeating the unchanged request cannot raise it.`, + }), }, ); if ( @@ -556,7 +572,7 @@ export class ArtifactAnalysisResources { { ...detail, retryable: true, - remediationAction: `${reported} REA retains the tighter inherited limit. This release has no runtime override for that ceiling. A smaller artifact may finish inside the retained limit. The signal does not prove that ceiling was exhausted, so an unchanged retry may succeed or may fail the same way.`, + remediationAction: `${reported} REA retains the tighter inherited limit. ${budgetParameter === undefined ? "This release has no runtime override for that ceiling." : `Review ${budgetParameter}.${signal === "SIGXFSZ" ? "file_bytes" : "cpu_seconds"} for a new call; a higher requested policy cannot raise a tighter inherited soft or hard limit. No REA restart is needed for a per-call update.`} A smaller artifact may finish inside the retained limit. The signal does not prove that ceiling was exhausted, so an unchanged retry may succeed or may fail the same way.`, }, ); } diff --git a/src/process/LinuxResourceLimits.test.ts b/src/process/LinuxResourceLimits.test.ts index f80959495a..124fe47d13 100644 --- a/src/process/LinuxResourceLimits.test.ts +++ b/src/process/LinuxResourceLimits.test.ts @@ -1,3 +1,4 @@ +import { BOUNDED_PRODUCER_LIMITS } from "../domain/producerResourceLimits.js"; import { expect, it, onTestFinished, vi } from "vitest"; import { writeFile, type FileHandle } from "node:fs/promises"; import { join } from "node:path"; @@ -8,7 +9,6 @@ import { readStableArtifact } from "../artifacts/readStableArtifact.js"; import { ok } from "../domain/result.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { - BOUNDED_PRODUCER_LIMITS, clampLinuxResourceLimits, prlimitArgumentVector, prlimitCeilingArguments, diff --git a/src/process/LinuxResourceLimits.ts b/src/process/LinuxResourceLimits.ts index 53a8a69d56..f6b7e09e79 100644 --- a/src/process/LinuxResourceLimits.ts +++ b/src/process/LinuxResourceLimits.ts @@ -16,16 +16,6 @@ const rows = { cpu_seconds: "Max cpu time", } as const; -/** Bound the resident input copy and retained text before allocation/collection. */ -export const BOUNDED_PRODUCER_LIMITS = { - input_bytes: 32 * 1024 * 1024, - output_bytes: 4 * 1024 * 1024, - command_timeout_ms: 30_000, - address_space_bytes: 1024 * 1024 * 1024, - file_bytes: 64 * 1024 * 1024, - cpu_seconds: 30, -} as const; - /** * Pin soft and hard ceilings together, ending at the prlimit option terminator. * Callers that append the producer command themselves use this prefix. diff --git a/src/rust/RustAuditableReader.ts b/src/rust/RustAuditableReader.ts index b03f315162..3e9b9d91e7 100644 --- a/src/rust/RustAuditableReader.ts +++ b/src/rust/RustAuditableReader.ts @@ -2,6 +2,7 @@ import { nodeCodecResult } from "../compression/NodeCodecResult.js"; import { inflateSync } from "node:zlib"; import { visit } from "jsonc-parser"; import { z } from "zod"; +import { RUST_PRODUCER_DEFAULTS } from "../domain/rust/rustResourceLimits.js"; import { cyclicDependencyIndices, type RustAuditableMetadata, @@ -9,16 +10,12 @@ import { } from "../domain/rust/rustBinary.js"; import { jsonValueByteLength } from "../domain/jsonValueByteLength.js"; -/** Selected-file admission limit shared with the stable artifact reader. */ -export const RUST_BINARY_INPUT_BYTES = 256 * 1024 * 1024; /** Compressed zlib stream limit. */ export const RUST_SECTION_BYTES = 1024 * 1024; /** Inflated JSON may grow by at most this multiple of the compressed stream. */ export const RUST_INFLATE_RATIO = 64; /** Absolute inflated JSON limit, applied together with the ratio. */ export const RUST_UNCOMPRESSED_BYTES = 8 * 1024 * 1024; -/** Complete normalized report limit. */ -export const RUST_OUTPUT_BYTES = 8 * 1024 * 1024; const JSON_DEPTH = 32; const packageSchema = z.strictObject({ @@ -60,10 +57,6 @@ export type RustAuditableConstraint = | { readonly resource: "transport"; readonly reported_limits: { readonly maximum_output_bytes: number }; - } - | { - readonly resource: "capacity"; - readonly reported_limits: { readonly maximum_rustc_hashes: number }; }; /** Admission refused before retained output or graph expansion. */ @@ -83,7 +76,10 @@ export const readAuditableSection = ( sectionName: RustAuditableMetadata["section_name"], fileOffset: number, bytes: Buffer, - options: { readonly peFileAlignment?: number | undefined } = {}, + options: { + readonly peFileAlignment?: number | undefined; + readonly maxOutputBytes?: number | undefined; + } = {}, ): RustAuditableMetadata => { if (bytes.length === 0) throw new RustAuditableFormatFailure( @@ -102,7 +98,10 @@ export const readAuditableSection = ( "Dependency section exceeds the 1 MiB compressed limit", ); const inflated = inflateSection(bytes, options.peFileAlignment); - const document = parseDocument(inflated.bytes); + const document = parseDocument( + inflated.bytes, + options.maxOutputBytes ?? RUST_PRODUCER_DEFAULTS.output_bytes, + ); const packages = normalizePackages(document.packages); const root = rootIndex(packages); return { @@ -207,7 +206,10 @@ const inflateFailure = (cause: unknown, maximum: number): Error => { return cause instanceof Error ? cause : new Error(String(cause)); }; -const parseDocument = (bytes: Buffer): z.infer => { +const parseDocument = ( + bytes: Buffer, + maxOutputBytes: number, +): z.infer => { let text: string; try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); @@ -274,7 +276,7 @@ const parseDocument = (bytes: Buffer): z.infer => { "Dependency JSON does not match the format schema", { cause: version.error }, ); - admitGraphExpansion(value); + admitGraphExpansion(value, maxOutputBytes); // One malformed package must fail before Zod retains an issue for every field. rejectMalformedPackages(value); const parsed = documentSchema.safeParse(value); @@ -303,7 +305,7 @@ const MIN_EDGE_OUTPUT_BYTES = JSON.stringify({ from_index: 0, to_index: 0, }).length; -const admitGraphExpansion = (value: unknown): void => { +const admitGraphExpansion = (value: unknown, maxOutputBytes: number): void => { if ( typeof value !== "object" || value === null || @@ -318,11 +320,11 @@ const admitGraphExpansion = (value: unknown): void => { let edges = 0; const admit = () => { const edgeBytes = edges * MIN_EDGE_OUTPUT_BYTES + Math.max(0, edges - 1); - if (minimumBytes + edgeBytes > RUST_OUTPUT_BYTES) + if (minimumBytes + edgeBytes > maxOutputBytes) throw new RustAuditableResourceFailure( { resource: "transport", - reported_limits: { maximum_output_bytes: RUST_OUTPUT_BYTES }, + reported_limits: { maximum_output_bytes: maxOutputBytes }, }, "Dependency graph exceeds the complete report byte budget before expansion", ); @@ -340,7 +342,7 @@ const admitGraphExpansion = (value: unknown): void => { pkg.features.every((feature) => typeof feature === "string") ) { const featureBudget = - RUST_OUTPUT_BYTES - minimumBytes - currentEdgeBytes() + 4; + maxOutputBytes - minimumBytes - currentEdgeBytes() + 4; const featureBytes = jsonValueByteLength( pkg.features as string[], featureBudget, diff --git a/src/rust/RustBinaryImage.ts b/src/rust/RustBinaryImage.ts index 238e55fa52..33eaf910d5 100644 --- a/src/rust/RustBinaryImage.ts +++ b/src/rust/RustBinaryImage.ts @@ -9,6 +9,8 @@ import type { RustAuditableMetadata, RustBinary, } from "../domain/rust/rustBinary.js"; +import { DEFAULT_RUSTC_HASH_LIMIT } from "../domain/rust/rustBinary.js"; +import { RUST_PRODUCER_DEFAULTS } from "../domain/rust/rustResourceLimits.js"; import { readAuditableSection, RustAuditableFormatFailure, @@ -17,8 +19,6 @@ import { const RUSTC_PREFIX = Buffer.from("/rustc/"); const HASH_BYTES = 40; -/** One toolchain normally leaves one hash; a small cap bounds retained output. */ -const MAX_RUSTC_HASHES = 64; /** Container identity plus optional dependency metadata and inferred rustc commits. */ export interface RustBinaryImage { @@ -32,7 +32,11 @@ export interface RustBinaryImage { } /** Read dependency metadata from container sections without executing the image. */ -export const readRustBinaryImage = (bytes: Buffer): RustBinaryImage => { +export const readRustBinaryImage = ( + bytes: Buffer, + maxRustcHashes = DEFAULT_RUSTC_HASH_LIMIT, + maxOutputBytes: number = RUST_PRODUCER_DEFAULTS.output_bytes, +): RustBinaryImage => { const container = readContainer(bytes); const dependency = dependencySection(container.format, container.sections); return { @@ -43,8 +47,8 @@ export const readRustBinaryImage = (bytes: Buffer): RustBinaryImage => { auditable_metadata: dependency === null ? null - : readDependency(bytes, container.format, dependency), - rustc_commit: rustcCommit(bytes, container), + : readDependency(bytes, container.format, dependency, maxOutputBytes), + rustc_commit: rustcCommit(bytes, container, maxRustcHashes, maxOutputBytes), ...(container.section_names_limitation === undefined ? {} : { section_names_limitation: container.section_names_limitation }), @@ -104,12 +108,14 @@ const readDependency = ( bytes: Buffer, format: RustBinaryImage["format"], section: ImageSection, + maxOutputBytes: number, ): RustAuditableMetadata => readAuditableSection( format === "macho" ? "__DATA,.dep-v0" : ".dep-v0", section.offset, bytes.subarray(section.offset, section.offset + section.size), { + maxOutputBytes, peFileAlignment: section.format === "pe" && section.virtual_size === 0 && @@ -123,9 +129,17 @@ const readDependency = ( const rustcCommit = ( bytes: Buffer, container: BinaryContainer, + maxHashes: number, + maxOutputBytes: number, ): RustBinaryImage["rustc_commit"] => { - const hashes: { hash: string; file_offset: number }[] = []; - const seen = new Set(); + const retention: RustcPathRetention = { + hashes: [], + seen: new Set(), + maxHashes, + maxBytes: maxOutputBytes, + bytes: 2, + reason: undefined, + }; const sections = container.sections .filter((section) => section.read_only) .map(({ offset, size }) => ({ offset, size })); @@ -137,11 +151,40 @@ const rustcCommit = ( : []; // Overlapping section and ELF load ranges are scanned once, so total work // stays bounded by the file size however many mappings reference a range. - for (const range of mergedReadOnlyFileRanges([...sections, ...loadRegions])) - collectHashes(bytes, range, seen, hashes); - return hashes.length === 0 ? null : { evidence_kind: "inferred", hashes }; + for (const range of mergedReadOnlyFileRanges([...sections, ...loadRegions])) { + collectHashes(bytes, range, retention); + if (retention.reason !== undefined) break; + } + return retention.hashes.length === 0 + ? null + : { + evidence_kind: "inferred", + hashes: retention.hashes, + ...(retention.reason !== undefined + ? { + coverage: { + status: "partial" as const, + reason: retention.reason, + effective_limit: + retention.reason === "report_bytes" + ? maxOutputBytes + : maxHashes, + }, + } + : {}), + }; }; +/** One owner for compiler-path uniqueness, retained output and exhaustion. */ +interface RustcPathRetention { + readonly hashes: { hash: string; file_offset: number }[]; + readonly seen: Set; + readonly maxHashes: number; + readonly maxBytes: number; + bytes: number; + reason: "max_rustc_hashes" | "report_bytes" | undefined; +} + const mergedReadOnlyFileRanges = ( ranges: readonly { readonly offset: number; readonly size: number }[], ): { readonly offset: number; readonly end: number }[] => { @@ -162,8 +205,7 @@ const mergedReadOnlyFileRanges = ( const collectHashes = ( bytes: Buffer, range: { readonly offset: number; readonly end: number }, - seen: Set, - hashes: { hash: string; file_offset: number }[], + retention: RustcPathRetention, ): void => { const region = bytes.subarray(range.offset, range.end); let cursor = 0; @@ -172,17 +214,20 @@ const collectHashes = ( if (at < 0) return; const start = at + RUSTC_PREFIX.length; const hash = hashAt(region, start); - if (hash !== null && !seen.has(hash)) { - if (seen.size >= MAX_RUSTC_HASHES) - throw new RustAuditableResourceFailure( - { - resource: "capacity", - reported_limits: { maximum_rustc_hashes: MAX_RUSTC_HASHES }, - }, - `Read-only data contains more than ${MAX_RUSTC_HASHES} distinct rustc commit paths.`, - ); - seen.add(hash); - hashes.push({ hash, file_offset: range.offset + at }); + if (hash !== null && !retention.seen.has(hash)) { + if (retention.seen.size >= retention.maxHashes) { + retention.reason = "max_rustc_hashes"; + return; + } + const entry = { hash, file_offset: range.offset + at }; + const entryBytes = Buffer.byteLength(JSON.stringify(entry)) + 1; + if (entryBytes > retention.maxBytes - retention.bytes) { + retention.reason = "report_bytes"; + return; + } + retention.bytes += entryBytes; + retention.seen.add(hash); + retention.hashes.push(entry); } cursor = at + 1; } diff --git a/src/rust/RustBinaryProvider.ts b/src/rust/RustBinaryProvider.ts index c993d5b003..c3d3ee7028 100644 --- a/src/rust/RustBinaryProvider.ts +++ b/src/rust/RustBinaryProvider.ts @@ -44,12 +44,17 @@ import { } from "./RustBinaryImage.js"; import { requireSelectedNativeImage } from "../native/metadata/MachoSliceSelection.js"; import { - RUST_BINARY_INPUT_BYTES, - RUST_OUTPUT_BYTES, RustAuditableFormatFailure, RustAuditableResourceFailure, } from "./RustAuditableReader.js"; +import { + resolveRustProducerLimits, + rustProducerLimitsSchema, + type RustProducerLimits, +} from "../domain/rust/rustResourceLimits.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; + const OPERATION = "inspect_rust_binary"; /** Stable in-process dependency inspection without subprocesses, temporary files, or a Rust toolchain. */ @@ -75,11 +80,26 @@ export class RustBinaryProvider implements RustBinaryPort { options?: ExecutionOptions, ): Promise> { let snapshot: StableArtifactSnapshot | undefined; + const selectedLimits = rustProducerLimitsSchema.safeParse( + input.producer_limits === undefined ? {} : input.producer_limits, + ); + if (!selectedLimits.success) + return err( + analysisInputErrorFromIssues( + OPERATION, + selectedLimits.error.issues.map((issue) => ({ + ...issue, + path: ["producer_limits", ...issue.path], + })), + input, + ), + ); + const limits = resolveRustProducerLimits(selectedLimits.data); try { return await this.#resources.run(async () => { snapshot = await readStableArtifact( input.path, - RUST_BINARY_INPUT_BYTES, + limits.input_bytes, options?.signal, undefined, this.#resources, @@ -92,10 +112,14 @@ export class RustBinaryProvider implements RustBinaryPort { input.architecture, ); const image = inFile( - readRustBinaryImage(selected.bytes), + readRustBinaryImage( + selected.bytes, + input.max_rustc_hashes, + limits.output_bytes, + ), selected.fileOffset, ); - const report = { + const report: RustBinary = { format: image.format, architecture: image.architecture, bits: image.bits, @@ -110,12 +134,17 @@ export class RustBinaryProvider implements RustBinaryPort { ...(selected.selection === null ? {} : { selected_slice: selected.selection }), - limitations: limitationsFor(image), + limitations: [ + ...limitationsFor(image), + `Selected producer_limits.input_bytes=${String(limits.input_bytes)} and output_bytes=${String(limits.output_bytes)} bytes; compressed/inflated dependency parser limits remain independent.`, + ], }; + retainCompilerPathsWithinReport(report, limits.output_bytes); if ( - Buffer.byteLength(JSON.stringify(report), "utf8") > RUST_OUTPUT_BYTES + Buffer.byteLength(JSON.stringify(report), "utf8") > + limits.output_bytes ) - throw outputLimit(input.path); + throw outputLimit(input.path, limits.output_bytes); const parsed = rustBinarySchema.safeParse(report); if (!parsed.success) throw new AnalysisOutputError( @@ -127,7 +156,7 @@ export class RustBinaryProvider implements RustBinaryPort { return ok(parsed.data); }); } catch (cause: unknown) { - return err(failureFor(cause, input.path, snapshot)); + return err(failureFor(cause, input.path, limits, snapshot)); } } } @@ -157,18 +186,21 @@ const inFile = (image: RustBinaryImage, fileOffset: number): RustBinaryImage => }, }; -const outputLimit = (path: string) => +const outputLimit = (path: string, maxOutputBytes: number) => new AnalysisResourceConstraintError( OPERATION, "transport", `Complete Rust metadata for ${path} exceeds the inline output budget; no partial success is returned.`, - { maximum_output_bytes: RUST_OUTPUT_BYTES }, - fixedAllowanceOptions("The Rust complete-report allowance", "report"), + { maximum_output_bytes: maxOutputBytes }, + { + retryable: false, + remediationAction: `Increase producer_limits.output_bytes above ${String(maxOutputBytes)} bytes on the next inspect_rust_binary call (CLI --producer-limits JSON); no restart is required. Input, dependency parser and transport limits remain independent. Repeating the unchanged request does not raise admission.`, + }, ); /** Resource-guard guidance delivered with every Rust inspection report. */ export const RUST_RESOURCE_GUARD_LIMITATION = - "Resource guards bound the selected file to 256 MiB, the compressed dependency stream to 1 MiB, inflated JSON to the smaller of 8 MiB and 64 times the consumed compressed-stream length, and the complete report to 8 MiB. Package and dependency counts are admitted against that output budget before schema cloning, graph objects, or reverse-edge allocation."; + "Default input and complete-report admission are 256 MiB and 8 MiB; producer_limits.input_bytes/output_bytes select larger or smaller values for the next call without restart. Independent parser guards bound the compressed dependency stream to 1 MiB and inflated JSON to the smaller of 8 MiB and 64 times the consumed compressed-stream length. Report admission applies before package schema cloning, graph objects, or reverse-edge allocation."; const limitationsFor = ( image: ReturnType, @@ -227,6 +259,14 @@ const rustcLimitation = ( return [ "No /rustc/<40-hex>/ path was found in read-only data, so a rustc commit hash is unknown.", ]; + if (commit.coverage?.reason === "report_bytes") + return [ + "Compiler-path retention exhausted the complete-report byte allowance; retained hashes are a prefix and compiler coverage is partial. Dependency observations are independent. Increase producer_limits.output_bytes on the next call (CLI --producer-limits JSON) to raise report admission; no restart is required. Increasing max_rustc_hashes alone does not raise it.", + ]; + if (commit.coverage?.status === "partial") + return [ + `Compiler-path retention reached max_rustc_hashes=${String(commit.coverage.effective_limit)}; the listed hashes are a prefix and compiler coverage is partial. Increase max_rustc_hashes on the next inspect_rust_binary call (CLI --max-rustc-hashes) to retain more paths; no restart is required. Dependency observations are independent. The complete-report byte allowance still applies.`, + ]; if (commit.hashes.length === 1) return [ "The rustc commit hash is inferred from a /rustc/<40-hex>/ path in read-only data. It does not by itself prove which compiler produced the binary.", @@ -236,9 +276,54 @@ const rustcLimitation = ( ]; }; +/** Optional inferred paths never displace independently observed dependency facts. */ +const retainCompilerPathsWithinReport = ( + report: RustBinary, + maxOutputBytes: number, +): void => { + const commit = report.rustc_commit; + if ( + commit === null || + Buffer.byteLength(JSON.stringify(report)) <= maxOutputBytes + ) + return; + const limitation = + "Compiler paths were omitted to preserve dependency facts within the complete-report byte allowance; compiler-path coverage is partial. Increase producer_limits.output_bytes on the next call (CLI --producer-limits JSON); no restart is required. Increasing max_rustc_hashes alone does not raise this allowance."; + const baseline = { + ...report, + rustc_commit: null, + limitations: [...report.limitations, limitation], + }; + const empty = { + ...commit, + hashes: [], + coverage: { + status: "partial" as const, + reason: "report_bytes" as const, + effective_limit: maxOutputBytes, + }, + }; + const available = + maxOutputBytes - + Buffer.byteLength(JSON.stringify({ ...baseline, rustc_commit: empty })); + let retainedBytes = 0; + let count = 0; + for (const hash of commit.hashes) { + const bytes = + Buffer.byteLength(JSON.stringify(hash)) + (count === 0 ? 0 : 1); + if (bytes > available - retainedBytes) break; + retainedBytes += bytes; + count++; + } + report.rustc_commit = + count === 0 ? null : { ...empty, hashes: commit.hashes.slice(0, count) }; + report.limitations = baseline.limitations; +}; + const failureForPrimary = ( facts: ArtifactFilePrimaryFacts, path: string, + limits: RustProducerLimits, ): AnalysisError => { const { cause, origin } = facts; if (origin === "owned-or-classified" && cause instanceof AnalysisError) @@ -251,18 +336,15 @@ const failureForPrimary = ( cause.constraint.reported_limits, { cause, - ...fixedAllowanceOptions( - cause.constraint.resource === "capacity" - ? "The rustc commit-path inventory" - : cause.constraint.resource === "transport" - ? "The Rust complete-report allowance" - : "This Rust inspection allowance", - cause.constraint.resource === "capacity" - ? "count" - : cause.constraint.resource === "transport" - ? "report" - : "parser", - ), + ...(cause.constraint.resource === "transport" + ? { + retryable: false, + remediationAction: `Increase producer_limits.output_bytes above ${String(limits.output_bytes)} bytes on the next call (CLI --producer-limits JSON); no restart is required. Compressed/inflated dependency parser and transport limits remain independent.`, + } + : fixedAllowanceOptions( + "This Rust dependency parser allowance", + "parser", + )), }, ); if (cause instanceof RustAuditableFormatFailure) @@ -277,15 +359,20 @@ const failureForPrimary = ( message: `Selected artifact ${path}: ${cause.message}`, }, ]); - return artifactFailure(facts, path); + return artifactFailure(facts, path, limits); }; const failureFor = ( cause: unknown, path: string, + limits: RustProducerLimits, snapshot?: StableArtifactSnapshot, ): AnalysisError => { - const primary = failureForPrimary(artifactFilePrimaryFacts(cause), path); + const primary = failureForPrimary( + artifactFilePrimaryFacts(cause), + path, + limits, + ); return projectStableArtifactFailure( cause, primary, @@ -299,13 +386,14 @@ const failureFor = ( const artifactFailure = ( facts: ArtifactFilePrimaryFacts, path: string, + limits: RustProducerLimits, ): AnalysisError => { const { cause, origin } = facts; if ( origin === "owned-or-classified" && cause instanceof ArtifactReaderFailure ) - return artifactReaderFailure(cause, path); + return artifactReaderFailure(cause, path, limits); if (cause instanceof Error && "code" in cause) { if (cause.code === "EACCES" || cause.code === "EPERM") return new AnalysisAccessDeniedError(OPERATION, path, cause.code, { @@ -332,6 +420,7 @@ const artifactFailure = ( const artifactReaderFailure = ( cause: ArtifactReaderFailure, path: string, + limits: RustProducerLimits, ): AnalysisError => { if (cause.reason === "integrity") return new AnalysisArtifactChangedError(OPERATION, path, cause.message, { @@ -346,10 +435,11 @@ const artifactReaderFailure = ( OPERATION, "file-size", cause.message, - { maximum_input_bytes: RUST_BINARY_INPUT_BYTES }, + { maximum_input_bytes: limits.input_bytes }, { cause, - ...fixedAllowanceOptions("the fixed Rust input allowance", "input"), + retryable: false, + remediationAction: `Increase producer_limits.input_bytes above ${String(limits.input_bytes)} bytes on the next inspect_rust_binary call (CLI --producer-limits JSON); no restart is required. Select the same complete artifact, without truncating its bytes. Report, dependency parser and host memory limits remain independent.`, }, ); if (cause.reason === "path") diff --git a/src/server/createServer.ts b/src/server/createServer.ts index e1ce7342dc..95aa20719e 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -850,13 +850,12 @@ export const createServer = ( recordEvidence, analysisAdmission, ); - registerWebSourceLocationTool( - server, - webSourceLocation, - toolLogger, + registerWebSourceLocationTool(server, webSourceLocation, { + logger: toolLogger, recordEvidence, - analysisAdmission, - ); + withAdmittedAnalysis: analysisAdmission, + evidenceById, + }); registerJavaScriptRecoveryTool( server, new JavaScriptRecoveryService(javascriptRecovery), diff --git a/src/server/registerFridaTools.ts b/src/server/registerFridaTools.ts index 4c2503b247..62ae43aaa4 100644 --- a/src/server/registerFridaTools.ts +++ b/src/server/registerFridaTools.ts @@ -1,3 +1,4 @@ +import type { FridaCapturePolicy } from "../domain/fridaCapturePolicy.js"; import type { McpServer, ServerContext } from "@modelcontextprotocol/server"; import type { Logger } from "pino"; @@ -67,6 +68,9 @@ export const registerFridaTools = ( async (input, context: ServerContext) => { const remote = remoteConnection(input.remote); const selected = { + ...(input.capture_limits === undefined + ? {} + : { captureLimits: input.capture_limits }), ...(input.device_id === undefined ? {} : { deviceId: input.device_id }), ...(remote === undefined ? {} : { remote }), }; @@ -222,6 +226,9 @@ export const registerFridaTools = ( async (input, context: ServerContext) => { const remote = remoteConnection(input.remote); const selected = { + ...(input.capture_limits === undefined + ? {} + : { captureLimits: input.capture_limits }), ...(input.device_id === undefined ? {} : { deviceId: input.device_id }), ...(remote === undefined ? {} : { remote }), }; @@ -310,6 +317,7 @@ const remoteConnection = ( }; const sessionOutput = (session: { + readonly captureLimits?: FridaCapturePolicy; readonly sessionId: string; readonly deviceId: string; readonly target: string; @@ -318,6 +326,9 @@ const sessionOutput = (session: { readonly state: "paused" | "running" | "detached"; }) => ({ session_id: session.sessionId, + ...(session.captureLimits === undefined + ? {} + : { capture_limits: session.captureLimits }), device_id: session.deviceId, target: session.target, pid: session.pid, diff --git a/src/server/registerWebSourceLocationTool.ts b/src/server/registerWebSourceLocationTool.ts index e952c2fa8d..33bea5de9d 100644 --- a/src/server/registerWebSourceLocationTool.ts +++ b/src/server/registerWebSourceLocationTool.ts @@ -1,3 +1,4 @@ +import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { WebSourceLocationService } from "../application/WebSourceLocationService.js"; @@ -5,28 +6,34 @@ import type { EvidenceWriter } from "../application/investigation/InvestigationR import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "pino"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; -import { createEvidenceToolHandler } from "./contractToolHandler.js"; +import { createRetainedSummaryToolHandler } from "./contractToolHandler.js"; import { applicationToolRegistrationOptions } from "./toolRegistrationOptions.js"; /** Bind source point tracing to its exact named contract and session Evidence owner. */ export const registerWebSourceLocationTool = ( server: EvidenceMcpServer, service: WebSourceLocationService, - logger: Logger, - recordEvidence?: EvidenceWriter["recordEvidence"], - withAdmittedAnalysis?: WithAdmittedAnalysis, + options: { + readonly logger: Logger; + readonly recordEvidence: EvidenceWriter["recordEvidence"] | undefined; + readonly withAdmittedAnalysis: WithAdmittedAnalysis | undefined; + readonly evidenceById: EvidenceLookup | undefined; + }, ): void => { + const { logger, recordEvidence, withAdmittedAnalysis, evidenceById } = + options; const contract = toolContract("trace_web_source_location"); - const handler = createEvidenceToolHandler( + const handler = createRetainedSummaryToolHandler( server, + contract, (input, options) => service.trace(input, options), logger, - recordEvidence, + { recordEvidence, evidenceById }, withAdmittedAnalysis, ); server.registerTool( contract.name, applicationToolRegistrationOptions(contract), - handler(contract), + handler, ); }; diff --git a/src/server/toolResult.ts b/src/server/toolResult.ts index 421f66392e..459b051776 100644 --- a/src/server/toolResult.ts +++ b/src/server/toolResult.ts @@ -78,27 +78,29 @@ const retainedEvidenceContinuation = ( /** The first page an owner offers when its complete retained result does not fit. */ const ownerFirstPage = (evidence: Evidence): JsonValue | undefined => - evidence.operation === "analyze_function" || - evidence.operation === "procedure_pseudo_code" - ? { kind: "native", facet: "pseudocode", offset: 0 } - : evidence.operation === "inspect_native_instruction" - ? { - kind: "native-instruction-references", - collection: "incoming_references", - offset: 0, - limit: 64, - } - : evidence.operation === "trace_application_feature" - ? { kind: "feature-trace", collection: "paths", offset: 0, limit: 64 } - : evidence.operation === "batch_decompile" - ? { - kind: "batch-decompile", - collection: "items", - offset: 0, - limit: 64, - } - : isNativeInventoryOperation(evidence.operation) - ? { kind: "native-inventory", offset: 0, limit: 64 } - : isNativeObservationOperation(evidence.operation) - ? { kind: "native-observation", offset: 0, limit: 64 } - : undefined; + evidence.operation === "trace_web_source_location" + ? { kind: "source-location", collection: "matches", offset: 0 } + : evidence.operation === "analyze_function" || + evidence.operation === "procedure_pseudo_code" + ? { kind: "native", facet: "pseudocode", offset: 0 } + : evidence.operation === "inspect_native_instruction" + ? { + kind: "native-instruction-references", + collection: "incoming_references", + offset: 0, + limit: 64, + } + : evidence.operation === "trace_application_feature" + ? { kind: "feature-trace", collection: "paths", offset: 0, limit: 64 } + : evidence.operation === "batch_decompile" + ? { + kind: "batch-decompile", + collection: "items", + offset: 0, + limit: 64, + } + : isNativeInventoryOperation(evidence.operation) + ? { kind: "native-inventory", offset: 0, limit: 64 } + : isNativeObservationOperation(evidence.operation) + ? { kind: "native-observation", offset: 0, limit: 64 } + : undefined; diff --git a/src/signals/SigrokRecordingProvider.ts b/src/signals/SigrokRecordingProvider.ts index d115e91e94..225c5a11b8 100644 --- a/src/signals/SigrokRecordingProvider.ts +++ b/src/signals/SigrokRecordingProvider.ts @@ -1,3 +1,7 @@ +import { + BOUNDED_PRODUCER_LIMITS as LIMITS, + resolveProducerLimits, +} from "../domain/producerResourceLimits.js"; import { SIGROK_PROVIDER_IDENTITY } from "./SigrokProviderIdentity.js"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { artifactFileCancellationCause } from "../artifacts/ArtifactFileOperation.js"; @@ -14,7 +18,6 @@ import { type SigrokSelection, } from "./sigrokSelectorRejection.js"; import { - BOUNDED_PRODUCER_LIMITS as LIMITS, prlimitArgumentVector, readLinuxResourceLimits, type LinuxResourceLimits, @@ -141,6 +144,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { input: DecodeSignalRecordingInput | InspectSignalDecodersInput, options?: ExecutionOptions, ): Promise> { + const limits = resolveProducerLimits(LIMITS, input.producer_limits); const environment = snapshotEnvironment(this.environment, this.platform); const operation = "path" in input ? OPERATION : "inspect_signal_decoders"; let phase: "configuration" | "workspace" | "producer" | "integrity" = @@ -191,7 +195,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { 64 * 1024 * 1024, options?.signal, ); - effectiveLimits = await this.#resources.readLimits(LIMITS); + effectiveLimits = await this.#resources.readLimits(limits); const limitArguments = prlimitArgumentVector( effectiveLimits, selectedCommand.invocationPath, @@ -225,8 +229,8 @@ export class SigrokRecordingProvider implements SignalRecordingPort { hostEnvironment: environment, }, { - timeoutMs: LIMITS.command_timeout_ms, - diagnosticBytes: LIMITS.output_bytes, + timeoutMs: limits.command_timeout_ms, + diagnosticBytes: limits.output_bytes, acceptNonZeroExit: true, }, { @@ -327,7 +331,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { sha256: limiterIdentity.sha256, arguments: limitArguments, }, - limits: { ...LIMITS, ...effectiveLimits }, + limits: { ...limits, ...effectiveLimits }, hardware_acquisition: "not-performed" as const, }); if (!("path" in input)) { @@ -355,8 +359,9 @@ export class SigrokRecordingProvider implements SignalRecordingPort { } else { const selected = await this.#resources.readSelectedArtifact({ path: input.path, - maximumBytes: LIMITS.input_bytes, - allowanceName: "the fixed sigrok input allowance", + maximumBytes: limits.input_bytes, + allowanceName: "the sigrok input allowance", + budgetParameter: "producer_limits.input_bytes", observedOutput: capturedOutput, signal: options?.signal, }); @@ -384,7 +389,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { phase = "integrity"; resourcePath = snapshotPath; if ( - (await this.#resources.read(snapshotPath, LIMITS.input_bytes)) + (await this.#resources.read(snapshotPath, limits.input_bytes)) .sha256 !== selected.sha256 ) throw new AnalysisArtifactChangedError( @@ -421,7 +426,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { "Annotations are the selected decoder's interpretation of the recorded samples and caller-selected settings, not independently verified protocol semantics. Empty output does not establish absence of traffic.", "Sample-number ranges and display text are preserved verbatim, not converted to timestamps or parsed as a lossless byte stream. All producer stderr is retained, including warnings on exit zero.", "Only a private copy of the selected file is decoded. Hardware discovery/acquisition and target execution are not requested. Caller-supplied tools, plugins and decoder modules are trusted; this is not an OS sandbox.", - "Linux prlimit enforces soft and hard ceilings before execution, preserving stricter inherited limits: at most 1 GiB address space per process, 64 MiB per file, 30 CPU seconds and no core dumps. Effective limits are reported. Resident input is capped at 32 MiB, each command at 30 wall-clock seconds and combined output at 4 MiB. Output overflow fails. These are per-process/per-file limits, not aggregate memory or filesystem quotas; caller-supplied tools and the limiter are trusted. REA retains the tighter inherited limit. This release has no runtime override.", + "Linux prlimit enforces the reported per-process address-space, per-file and CPU limits before execution with no core dumps, preserving tighter inherited soft and hard limits. Input, command deadline and retained output budgets are reported in limits; producer_limits can change their requested defaults on the next call without restart. Output overflow fails. Parser and transport bounds remain independent. These are per-process/per-file limits, not aggregate memory or filesystem quotas; caller-supplied tools and the limiter are trusted.", ], }), ); @@ -442,6 +447,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { effectiveLimits, capturedOutput, operation, + limits, ), ); } @@ -468,6 +474,10 @@ export class SigrokRecordingProvider implements SignalRecordingPort { effectiveLimits: LinuxResourceLimits | undefined, observedOutput: AnalysisCapturedOutput | undefined, operation = OPERATION, + limits: { + readonly output_bytes: number; + readonly command_timeout_ms: number; + } = LIMITS, ): AnalysisError { if (cause instanceof AnalysisError) return cause; if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined) @@ -482,6 +492,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort { effectiveLimits, observedOutput, operation, + limits, ), observedOutput, operation, @@ -514,9 +525,10 @@ export class SigrokRecordingProvider implements SignalRecordingPort { if (cause instanceof OwnedCommandFailure) return this.#resources.commandFailure( cause, - LIMITS, + limits, effectiveLimits, operation, + "producer_limits", ); if (phase === "workspace") return privateWorkspaceFailure( diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts index 2a118aadc7..6359107bfb 100644 --- a/src/sqlite/SqliteDatabaseLimits.ts +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -1,8 +1,10 @@ +import { SQLITE_DATABASE_OUTPUT_BYTES } from "../domain/sqlite/sqliteDatabase.js"; + /** Bound frozen input, retained JSON and worker allocation before parsing untrusted files. */ export const SQLITE_DATABASE_LIMITS = { concurrentInspections: 2, inputBytes: 256 * 1024 * 1024, - outputBytes: 16 * 1024 * 1024, + outputBytes: SQLITE_DATABASE_OUTPUT_BYTES, diagnosticBytes: 1024 * 1024, } as const; /** diff --git a/src/wasm/WabtArtifactProvider.test.ts b/src/wasm/WabtArtifactProvider.test.ts index 9189874193..d4eebb030b 100644 --- a/src/wasm/WabtArtifactProvider.test.ts +++ b/src/wasm/WabtArtifactProvider.test.ts @@ -4,17 +4,17 @@ import { join, resolve } from "node:path"; import { expect, it, onTestFinished, vi } from "vitest"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { WabtArtifactProvider } from "./WabtArtifactProvider.js"; -import { wasmArtifactSchema } from "../domain/wasm/wasmArtifact.js"; +import { WasmArtifactService } from "../application/wasm/WasmArtifactService.js"; +import { + wasmArtifactSchema, + WASM_RESOURCE_DEFAULTS, +} from "../domain/wasm/wasmArtifact.js"; import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; import { PrivateRuntimeRootUnavailableError } from "../process/PrivateRuntimeRootUnavailableError.js"; import { spawnOwnedProviderProcess } from "../process/ProviderProcess.js"; import { OwnedCommandFailure } from "../process/OwnedCommand.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; -import { - WABT_LIMITS, - admitWabtVersion, - WABT_ACCEPTED_RANGE, -} from "./WabtRelease.js"; +import { admitWabtVersion, WABT_ACCEPTED_RANGE } from "./WabtRelease.js"; // Admission must not outrun the `1.0.x` banners wasmArtifactSchema accepts. it("admits 1.0.x releases but not the 1.1.0 upper boundary", () => { @@ -221,6 +221,92 @@ const createWabtArtifactFixture = async () => { return { selectedRoot, path, bytes, dump }; }; +it.each([ + "max_input_bytes", + "max_tool_bytes", + "max_output_bytes", + "max_interface_bytes", + "max_artifact_paths", +] as const)( + "recovers the same complete artifact by changing only %s", + async (parameter) => { + const fixture = await createWabtArtifactFixture(); + const wat = await readFile( + new URL("../../tests/fixtures/wasm/decoded.wat", import.meta.url), + "utf8", + ); + const provider = new WabtArtifactProvider( + { REA_WABT_BIN_DIRECTORY: fixture.selectedRoot }, + (input) => { + const output = + input.arguments[0] === "--version" + ? "1.0.42\n" + : input.command.includes("objdump") + ? fixture.dump + : input.command.includes("wasm2wat") + ? wat + : ""; + return spawnOwnedProviderProcess({ + ...input, + hostEnvironment: nodeHostEnvironment(input.hostEnvironment), + command: process.execPath, + arguments: ["-e", `process.stdout.write(${JSON.stringify(output)})`], + }); + }, + ); + onTestFinished(() => provider.close()); + const service = new WasmArtifactService(provider); + const candidatePaths: string[] = []; + if (parameter === "max_artifact_paths") { + for (let i = 0; i < 17; i++) { + const candidate = join( + fixture.selectedRoot, + `candidate-${String(i)}.wasm`, + ); + await writeFile(candidate, fixture.bytes); + candidatePaths.push(candidate); + } + } + const input = { + path: fixture.path, + glue_paths: [], + candidate_paths: candidatePaths, + }; + const rejected = await service.inspect({ + ...input, + ...(parameter === "max_artifact_paths" ? {} : { [parameter]: 1 }), + }); + if (parameter === "max_interface_bytes") { + if (!rejected.ok) throw rejected.error; + expect(rejected.value.normalized_result).toMatchObject({ + interface: { state: "partial" }, + }); + } else { + if (rejected.ok) throw new Error("Expected byte admission to fail"); + expect(projectAnalysisError(rejected.error).remediation.action).toContain( + parameter, + ); + } + const requestedBudget = parameter === "max_artifact_paths" ? 17 : 10_000; + const recovered = await service.inspect({ + ...input, + [parameter]: requestedBudget, + }); + if (!recovered.ok) throw recovered.error; + const report = wasmArtifactSchema.parse(recovered.value.normalized_result); + expect(report.artifact.sha256).toBe( + createHash("sha256").update(fixture.bytes).digest("hex"), + ); + expect(report.wat.text).toBe(wat); + expect(report.interface?.state).toBe("complete"); + expect(report.candidates).toHaveLength(candidatePaths.length + 1); + expect(recovered.value.parameters[parameter]).toBe(requestedBudget); + expect(report.tool_profile.output_budget_bytes).toBe( + parameter === "max_output_bytes" ? 10_000 : 32 * 1024 * 1024, + ); + }, +); + it.each([ { label: "initially malformed", @@ -366,7 +452,7 @@ const assertModeOutcome = ( normalized_result: { tool_profile: { timeout_ms: 42_000, - output_budget_bytes: WABT_LIMITS.outputBytes, + output_budget_bytes: WASM_RESOURCE_DEFAULTS.outputBytes, }, }, }, @@ -1414,16 +1500,16 @@ it("reports WABT output beyond the budget as a resource constraint with a bounde expect(result.error).toMatchObject({ _tag: "AnalysisResourceConstraintError", resource: "memory", - reportedLimits: { max_output_bytes: WABT_LIMITS.outputBytes }, + reportedLimits: { max_output_bytes: WASM_RESOURCE_DEFAULTS.outputBytes }, }); const projection = projectAnalysisError(result.error); expect(projection).toMatchObject({ code: "resource_constraint", message: expect.stringContaining( - `${String(WABT_LIMITS.outputBytes)}-byte output budget`, + `${String(WASM_RESOURCE_DEFAULTS.outputBytes)}-byte output budget`, ), remediation: { - action: expect.stringContaining("retrying with the same input"), + action: expect.stringContaining("max_output_bytes"), }, }); expect(projection.details?.captured_output).toEqual({ @@ -1524,8 +1610,8 @@ it("bounds the excerpt when cleanup also fails after the output budget", () => { diagnostics: { previous_error: { failure_kind: "output-limit", - message: `WABT output exceeded the ${String(WABT_LIMITS.outputBytes)}-byte output budget`, - max_output_bytes: WABT_LIMITS.outputBytes, + message: `WABT output exceeded the ${String(WASM_RESOURCE_DEFAULTS.outputBytes)}-byte output budget`, + max_output_bytes: WASM_RESOURCE_DEFAULTS.outputBytes, }, captured_output: { stdout: "(module", stderr: "w".repeat(4096) }, }, @@ -1535,5 +1621,8 @@ it("bounds the excerpt when cleanup also fails after the output budget", () => { it("does not blame the input when a version probe exceeds the budget", () => { expect( wabtFailure(outputLimitFailure(), "configuration", "/module.wasm"), - ).toMatchObject({ _tag: "AnalysisOutputError" }); + ).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + reportedLimits: { max_output_bytes: 32 * 1024 * 1024 }, + }); }); diff --git a/src/wasm/WabtArtifactProvider.ts b/src/wasm/WabtArtifactProvider.ts index a7f529d73b..5993d23c18 100644 --- a/src/wasm/WabtArtifactProvider.ts +++ b/src/wasm/WabtArtifactProvider.ts @@ -41,6 +41,7 @@ import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { wasmArtifactSchema, DEFAULT_WASM_COMMAND_TIMEOUT_MS, + wasmResourceLimits, type InspectWasmArtifactInput, type WasmArtifact, } from "../domain/wasm/wasmArtifact.js"; @@ -57,7 +58,6 @@ import { import { WabtRuntime } from "./WabtRuntime.js"; import { WABT_PROVIDER_IDENTITY, - WABT_LIMITS, WABT_SOURCE_COMMIT, WABT_VERSION, admitWabtVersion, @@ -147,6 +147,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { environment: Readonly, options?: WasmArtifactExecutionOptions, ): Promise> { + const limits = wasmResourceLimits(input); let runtime: WabtRuntime | undefined; let phase: "configuration" | "artifact" | "producer" = "configuration"; let result: Result; @@ -211,7 +212,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { try { const snapshot = await readStableArtifact( path, - WABT_LIMITS.inputBytes, + limits.inputBytes, options?.signal, undefined, this.#resources, @@ -262,7 +263,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { await access(path, constants.X_OK); const binary = await readStableArtifact( path, - WABT_LIMITS.toolBytes, + limits.toolBytes, options?.signal, undefined, this.#resources, @@ -291,7 +292,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { if (snapshotHash === undefined) return; const bytes = await readStableArtifact( snapshotPath, - WABT_LIMITS.inputBytes, + limits.inputBytes, options?.signal, undefined, this.#resources, @@ -325,7 +326,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { }, { timeoutMs: commandTimeoutMs, - diagnosticBytes: WABT_LIMITS.outputBytes, + diagnosticBytes: limits.outputBytes, acceptNonZeroExit, }, { @@ -342,7 +343,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { }); const current = await readStableArtifact( command.path, - WABT_LIMITS.toolBytes, + limits.toolBytes, options?.signal, undefined, this.#resources, @@ -423,7 +424,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { ({ selection: _selection, ...command }) => command, ), timeout_ms: commandTimeoutMs, - output_budget_bytes: WABT_LIMITS.outputBytes, + output_budget_bytes: limits.outputBytes, }; capturedOutput = undefined; phase = "artifact"; @@ -586,7 +587,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { debugModuleNames, debugNameMetadata: decodedNameMetadata, interface: interfaceFacts, - } = parseWabtWat(wat.stdout.text); + } = parseWabtWat(wat.stdout.text, input.max_interface_bytes); // WAT framing reads module names, not every inner name map. A producer // warning may concern a skipped body or another custom section; without // section attribution it cannot establish complete debug-name coverage. @@ -649,7 +650,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { "Imports and exports retain exact top-level WABT WAT forms. Names remain escaped producer text; unescaped objdump display names are not parsed as unambiguous identifiers.", "JavaScript references are AST-observed static literals only. Candidate matches are associations, not proven data flow, URL-to-file identity or runtime loading. Computed references and aliases are not resolved.", "No target execution or implicit fetch. WABT tools are trusted caller-supplied executables; SHA-256 identifies their bytes, not publisher authenticity. No OS memory sandbox is claimed.", - `Each WABT command has a ${String(commandTimeoutMs)} ms deadline and a fixed 32 MiB combined output budget. The deadline is not a whole-inspection budget. Oversized output fails instead of producing truncated success.`, + `Each WABT command has a ${String(commandTimeoutMs)} ms deadline and a ${String(limits.outputBytes)} byte combined output budget (max_output_bytes). The deadline is not a whole-inspection budget. Oversized output fails instead of producing truncated success.`, ...(compatibleLimitation === undefined ? [] : [compatibleLimitation]), ...optionalLimitations, ]; @@ -694,6 +695,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { phase === "artifact" ? artifactInput.path : input.path, { commandTimeoutMs, + limits, inputField: artifactInput.field, observedOutput: capturedOutput, ...(versionProbe === undefined ? {} : { versionProbe }), diff --git a/src/wasm/WabtFailures.test.ts b/src/wasm/WabtFailures.test.ts index 7ad73b7e20..6f19c871b1 100644 --- a/src/wasm/WabtFailures.test.ts +++ b/src/wasm/WabtFailures.test.ts @@ -2,9 +2,27 @@ import { expect, it } from "vitest"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { wabtFailure } from "./WabtFailures.js"; -import { WABT_LIMITS } from "./WabtRelease.js"; +import { WASM_RESOURCE_DEFAULTS } from "../domain/wasm/wasmArtifact.js"; import { OwnedCommandFailure } from "../process/OwnedCommand.js"; +it.each(["configuration", "artifact"] as const)( + "keeps host allocation failure distinct from %s readiness and byte admission", + (phase) => { + const cause = Object.assign(new Error("Allocation failed"), { + code: "ENOMEM", + }); + const failure = projectAnalysisError( + wabtFailure(cause, phase, "/module.wasm"), + ); + expect(failure).toMatchObject({ + code: "resource_constraint", + retryable: false, + details: { resource: "memory" }, + }); + expect(failure.remediation.action).not.toContain("REA_WABT_BIN_DIRECTORY"); + }, +); + it.each(["configuration", "producer"] as const)( "retains timeout recovery under %s cleanup failure without making cleanup retryable", (phase) => { @@ -172,7 +190,7 @@ it.each([ ); it.each([false, true])( - "preserves the file-size boundary with cleanup=%s when an input exceeds its fixed allowance", + "preserves the file-size boundary with cleanup=%s when an input exceeds its allowance", (cleanup) => { const cause = new ArtifactReaderFailure( "limit", @@ -195,11 +213,13 @@ it.each([false, true])( retryable: false, details: { resource: "file-size", - reported_limits: { maximum_input_bytes: WABT_LIMITS.inputBytes }, + reported_limits: { + maximum_input_bytes: WASM_RESOURCE_DEFAULTS.inputBytes, + }, }, }); if (!cleanup) { - expect(projected.remediation.action).toContain("input byte allowance"); + expect(projected.remediation.action).toContain("max_input_bytes"); expect(projected.remediation.action).not.toContain("evidence reply"); } expect( diff --git a/src/wasm/WabtFailures.ts b/src/wasm/WabtFailures.ts index 1014abd281..b89b0bf1dc 100644 --- a/src/wasm/WabtFailures.ts +++ b/src/wasm/WabtFailures.ts @@ -3,6 +3,7 @@ import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { stableArtifactInputRecovery } from "../artifacts/StableArtifactFailure.js"; import { artifactFileCancellationCause, + artifactFilePrimaryFacts, bindArtifactFileCleanup, } from "../artifacts/ArtifactFileOperation.js"; import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; @@ -32,15 +33,24 @@ import { WabtAdmissionError } from "./WabtAdmissionError.js"; import { DEFAULT_WASM_COMMAND_TIMEOUT_MS, MAX_WASM_COMMAND_TIMEOUT_MS, + WASM_RESOURCE_DEFAULTS, + WASM_RESOURCE_MAXIMUMS, + type WasmResourceLimits, } from "../domain/wasm/wasmArtifact.js"; import { WABT_ACCEPTED_RANGE, - WABT_LIMITS, WABT_PROVIDER_IDENTITY, WABT_VERSION, } from "./WabtRelease.js"; const OUTPUT_LIMIT_EXCERPT_BYTES = 4096; -const OUTPUT_LIMIT_REASON = `WABT output exceeded the ${String(WABT_LIMITS.outputBytes)}-byte output budget`; + +/** Recovery names the exhausted per-call byte owner, independently of deadlines. */ +const byteBudgetRecovery = ( + parameter: "max_input_bytes" | "max_output_bytes" | "max_tool_bytes", + value: number, + maximum: number, +): string => + `${parameter} is ${String(value)} bytes. ${value < maximum ? `Raise ${parameter} above ${String(value)} bytes (positive integer, maximum ${String(maximum)}), or use --${parameter.replaceAll("_", "-")} on the CLI, then retry the same complete input after cleanup settles.` : `This is already the runtime representation maximum of ${String(maximum)} bytes; use a suitable smaller complete input or report this blocker.`} The new value applies to the next inspection without restarting REA. command_timeout_ms, other byte budgets and client delivery settings do not raise this allowance. Do not truncate required bytes. Repeating the unchanged request fails the same way.`; /** Facts owned before dispatch, even when a version command does not return successfully. */ export type WabtVersionProbe = { @@ -53,6 +63,7 @@ export type WabtVersionProbe = { type WabtFailureContext = { readonly commandTimeoutMs: number; + readonly limits?: WasmResourceLimits; readonly versionProbe?: WabtVersionProbe; readonly inputField?: readonly (string | number)[]; readonly observedOutput?: AnalysisCapturedOutput | undefined; @@ -104,6 +115,8 @@ const classifyWabtFailure = ( execution: WabtFailureContext | undefined, ): AnalysisError => { const versionProbe = execution?.versionProbe; + const limits = execution?.limits ?? WASM_RESOURCE_DEFAULTS; + const outputLimitReason = `WABT output exceeded the ${String(limits.outputBytes)}-byte output budget`; if (cause instanceof AnalysisError) return cause; const artifactCleanup = cause instanceof ArtifactReaderFailure ? cause.cleanup : undefined; @@ -114,6 +127,31 @@ const classifyWabtFailure = ( : { capturedOutput: execution.observedOutput }), }; const reason = cause instanceof Error ? cause.message : String(cause); + const physical = artifactFilePrimaryFacts(cause).cause; + if ( + physical instanceof Error && + (("code" in physical && + [ + "ENOMEM", + "ERR_MEMORY_ALLOCATION_FAILED", + "ERR_BUFFER_TOO_LARGE", + ].includes(String(physical.code))) || + (physical instanceof RangeError && + physical.message === "Array buffer allocation failed")) + ) + return new AnalysisResourceConstraintError( + "inspect_wasm_artifact", + "memory", + "The host could not allocate the admitted WABT acquisition buffer", + { max_input_bytes: limits.inputBytes, max_tool_bytes: limits.toolBytes }, + { + cause, + ...artifactOptions, + retryable: false, + remediationAction: + "Byte admission does not reserve host memory. Free host memory or select a suitable process with sufficient capacity, then retry after cleanup settles. Increasing byte budgets or command_timeout_ms does not repair this allocation failure; do not replace a correctly configured WABT installation for this reason.", + }, + ); if ( (cause instanceof ArtifactReaderFailure && cause.reason === "cancelled") || (cause instanceof Error && cause.name === "AbortError") @@ -161,7 +199,7 @@ const classifyWabtFailure = ( timeoutMs < MAX_WASM_COMMAND_TIMEOUT_MS ? `Increase command_timeout_ms above ${String(timeoutMs)} (integer milliseconds, maximum ${String(MAX_WASM_COMMAND_TIMEOUT_MS)}), or use rea inspect-wasm-artifact --command-timeout-ms on the CLI.` : `command_timeout_ms is already at the maximum supported ${String(MAX_WASM_COMMAND_TIMEOUT_MS)} ms; no higher command deadline is supported. Report this blocker or select another suitable complete artifact with caller approval.` - } This is a per-call setting; no server restart is needed. Allow the MCP client deadline to cover all version/analysis commands, file checks, cleanup and delivery; increasing only that client deadline does not change this budget. Input/output safety limits stay fixed, and detail: summary reduces delivery size, not analysis work. Resolve any incomplete cleanup before retrying; do not repeat the unchanged failed call.`, + } This is a per-call setting; no server restart is needed. Allow the MCP client deadline to cover all version/analysis commands, file checks, cleanup and delivery; increasing only that client deadline does not change this budget. max_input_bytes, max_tool_bytes and max_output_bytes are independent per-call controls; detail: summary reduces delivery size, not analysis work. Resolve any incomplete cleanup before retrying; do not repeat the unchanged failed call.`, }) : undefined; if (cause.cleanupFailure !== null) @@ -177,8 +215,13 @@ const classifyWabtFailure = ( : cause.reason === "output-limit" ? { failure_kind: "output-limit", - message: OUTPUT_LIMIT_REASON, - max_output_bytes: WABT_LIMITS.outputBytes, + message: outputLimitReason, + max_output_bytes: limits.outputBytes, + remediation_action: byteBudgetRecovery( + "max_output_bytes", + limits.outputBytes, + WASM_RESOURCE_MAXIMUMS.outputBytes, + ), } : reason, ...(capturedOutput === undefined @@ -194,15 +237,20 @@ const classifyWabtFailure = ( cause.cancellation === undefined ? cause : cause.cancellation.reason, }); if (timeout !== undefined) return timeout; - if (phase !== "configuration" && cause.reason === "output-limit") + if (cause.reason === "output-limit") return new AnalysisResourceConstraintError( "inspect_wasm_artifact", "memory", - OUTPUT_LIMIT_REASON, - { max_output_bytes: WABT_LIMITS.outputBytes }, + outputLimitReason, + { max_output_bytes: limits.outputBytes }, { ...options, - remediationAction: `The WABT output allowance is ${String(WABT_LIMITS.outputBytes)} bytes. This release has no runtime override. command_timeout_ms does not raise it. The WABT output budget is fixed, so retrying with the same input will fail the same way.`, + retryable: false, + remediationAction: byteBudgetRecovery( + "max_output_bytes", + limits.outputBytes, + WASM_RESOURCE_MAXIMUMS.outputBytes, + ), }, ); if (phase === "configuration" && cause.reason === "process") @@ -237,8 +285,12 @@ const classifyWabtFailure = ( { cause, ...artifactOptions, - userMessage: `Configured WABT tools exceed the ${String(WABT_LIMITS.toolBytes)}-byte tool allowance. This release has no runtime override.`, - remediationAction: `The WABT tool allowance is ${String(WABT_LIMITS.toolBytes)} bytes. This release has no runtime override. command_timeout_ms and REA_WABT_BIN_DIRECTORY do not raise it. Select tools within the allowance. Do not repeat the unchanged call.`, + userMessage: `Configured WABT tools exceed the ${String(limits.toolBytes)}-byte tool allowance (max_tool_bytes).`, + remediationAction: byteBudgetRecovery( + "max_tool_bytes", + limits.toolBytes, + WASM_RESOURCE_MAXIMUMS.toolBytes, + ), }, ); if (phase === "configuration") { @@ -276,12 +328,16 @@ const classifyWabtFailure = ( "inspect_wasm_artifact", "file-size", reason, - { maximum_input_bytes: WABT_LIMITS.inputBytes }, + { maximum_input_bytes: limits.inputBytes }, { cause, ...artifactOptions, retryable: false, - remediationAction: `Select a suitable stable artifact within the reported ${String(WABT_LIMITS.inputBytes)}-byte input byte allowance. This release has no runtime override. command_timeout_ms does not raise it. Do not truncate required bytes; retrying the unchanged oversized input will fail again.`, + remediationAction: byteBudgetRecovery( + "max_input_bytes", + limits.inputBytes, + WASM_RESOURCE_MAXIMUMS.inputBytes, + ), }, ); if ( diff --git a/src/wasm/WabtInterface.test.ts b/src/wasm/WabtInterface.test.ts index 72e3a2253f..99d0555980 100644 --- a/src/wasm/WabtInterface.test.ts +++ b/src/wasm/WabtInterface.test.ts @@ -121,36 +121,41 @@ it("preserves custom memory page sizes and treats unadmitted nested declarations expect(unknown.entries[0]?.signature).toBeNull(); }); -it("refuses typed name expansion within the source WABT byte cap before retaining interface output", () => { - const name = "x".repeat(6 * 1024 * 1024); - expect( - parseWabtWat(`(module (import "${name}" "f" (func)))`).interface, - ).toMatchObject({ +it("recovers typed name expansion with a larger selected interface allowance", () => { + const name = "x".repeat(500); + const wat = `(module (import "${name}" "f" (func)))`; + expect(parseWabtWat(wat, 1024).interface).toMatchObject({ state: "partial", entries: [], - issues: [expect.stringMatching(/representation budget/u)], + issues: expect.arrayContaining([ + expect.stringContaining("max_interface_bytes"), + ]), + }); + expect(parseWabtWat(wat, 8192).interface).toMatchObject({ + state: "complete", + entries: [{ direction: "import", module: name, name: "f" }], + issues: [], }); }); it("charges shared type signatures at every retained occurrence and preserves bounded partial facts", () => { - const params = Array(1000).fill("i32").join(" "); + const params = Array(64).fill("i32").join(" "); const exports = Array.from( - { length: 10000 }, + { length: 100 }, (_, i) => `(export "f${i}" (func 0))`, ).join(" "); const report = parseWabtWat( `(module (type (func (param ${params}))) (import "e" "f" (func (type 0))) ${exports})`, + 16384, ).interface; expect(report.state).toBe("partial"); expect(report.entries.length).toBeGreaterThan(1); - expect(report.entries.length).toBeLessThan(10001); + expect(report.entries.length).toBeLessThan(101); expect(report.entries[0]?.signature).toMatchObject({ kind: "func", params: expect.any(Array), }); - expect(Buffer.byteLength(JSON.stringify(report))).toBeLessThan( - 32 * 1024 * 1024, - ); + expect(Buffer.byteLength(JSON.stringify(report))).toBeLessThan(16384); }); it("keeps collided labels unknown in each index space and preserves numeric and unique identities", () => { diff --git a/src/wasm/WabtInterface.ts b/src/wasm/WabtInterface.ts index fc45d50c31..a2a17fd3c7 100644 --- a/src/wasm/WabtInterface.ts +++ b/src/wasm/WabtInterface.ts @@ -1,7 +1,7 @@ import { WabtAdmissionError } from "./WabtAdmissionError.js"; import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; import { wasmInterfaceEntryRepresentationBytes } from "../domain/wasm/wasmInterface.js"; -import { WasmInterfaceBudget } from "./WasmInterfaceBudget.js"; +import { WasmInterfaceBudget } from "../domain/wasm/wasmInterfaceBudget.js"; import type { WasmInterface, WasmInterfaceEntry, @@ -12,8 +12,8 @@ import { watString, watStringEnd } from "./WabtString.js"; import { watCommentEnd } from "./WabtComment.js"; /** Split direct WAT children only; function instruction bodies are never materialized. */ -const parts = (form: string): string[] => { - const budget = new WasmInterfaceBudget(); +const splitParts = (form: string, maximumBytes?: number): string[] => { + const budget = new WasmInterfaceBudget(maximumBytes); const items: string[] = []; let i = 1; while (i < form.length - 1) { @@ -54,42 +54,6 @@ const parts = (form: string): string[] => { } return items; }; -const head = (form: string): string | undefined => parts(form)[0]; -const admittedValues = new Set([ - "i32", - "i64", - "f32", - "f64", - "v128", - "funcref", - "externref", -]); -const isValueType = (type: string): type is WasmInterfaceValueType => - admittedValues.has(type); -const values = ( - form: string, - category: string, -): WasmInterfaceValueType[] | null => { - const result: WasmInterfaceValueType[] = []; - for (const item of parts(form)) { - if (!item.startsWith("(")) continue; - const group = parts(item); - if (group[0] !== category) continue; - for (const type of group.slice(1).filter((t) => !t.startsWith("$"))) { - if (!isValueType(type)) return null; - result.push(type); - } - } - return result; -}; -const functionType = ( - form: string, - kind: "func" | "tag", -): WasmInterfaceSignature | null => { - const params = values(form, "param"); - const results = values(form, "result"); - return params === null || results === null ? null : { kind, params, results }; -}; const name = (text: string | undefined): string => { if (text?.startsWith('"') !== true) throw new WabtAdmissionError("name_missing"); @@ -102,12 +66,54 @@ const integer = (text: string): string | null => { return BigInt(text).toString(); }; +const admittedValues = new Set([ + "i32", + "i64", + "f32", + "f64", + "v128", + "funcref", + "externref", +]); +const isValueType = (type: string): type is WasmInterfaceValueType => + admittedValues.has(type); + /** Resolve index spaces at the producer, not from retained raw WAT in consumers. */ const decodeWabtInterface = ( forms: readonly string[], result: WasmInterface, + maximumBytes: number | undefined, ): WasmInterface => { - const budget = new WasmInterfaceBudget(); + const budget = new WasmInterfaceBudget(maximumBytes); + const parts = (form: string) => splitParts(form, maximumBytes); + const head = (form: string): string | undefined => parts(form)[0]; + const values = ( + form: string, + category: string, + ): WasmInterfaceValueType[] | null => { + const result: WasmInterfaceValueType[] = []; + for (const item of parts(form)) { + if (!item.startsWith("(")) continue; + const group = parts(item); + if (group[0] !== category) continue; + for (const type of group.slice(1).filter((t) => !t.startsWith("$"))) { + if (!isValueType(type)) return null; + result.push(type); + } + } + return result; + }; + const functionType = ( + form: string, + kind: "func" | "tag", + ): WasmInterfaceSignature | null => { + const params = values(form, "param"); + const results = values(form, "result"); + return params === null || results === null + ? null + : { kind, params, results }; + }; + const types: (WasmInterfaceSignature | null)[] = []; const typeNames = new Map(); const definitionNames = new Map>(); @@ -319,14 +325,19 @@ const decodeWabtInterface = ( }; /** New typed coverage may be incomplete without discarding existing decoded WAT Evidence. */ -export const parseWabtInterface = (forms: readonly string[]): WasmInterface => { +export const parseWabtInterface = ( + forms: readonly string[], + maximumBytes?: number, +): WasmInterface => { const result: WasmInterface = { state: "complete", entries: [], issues: [] }; try { - return decodeWabtInterface(forms, result); + return decodeWabtInterface(forms, result, maximumBytes); } catch (cause: unknown) { if (!(cause instanceof AnalysisResourceConstraintError)) throw cause; result.state = "partial"; result.issues.push(cause.message); + if (cause.remediationAction !== undefined) + result.issues.push(cause.remediationAction); return result; } }; diff --git a/src/wasm/WabtRelease.ts b/src/wasm/WabtRelease.ts index 8d54b14df9..c80380978c 100644 --- a/src/wasm/WabtRelease.ts +++ b/src/wasm/WabtRelease.ts @@ -18,11 +18,6 @@ export const WABT_PROVIDER_IDENTITY = { name: "REA WABT artifact adapter", version: `wabt@${WABT_VERSION}`, } as const; -export const WABT_LIMITS = { - inputBytes: 8 * 1024 * 1024, - toolBytes: 64 * 1024 * 1024, - outputBytes: 32 * 1024 * 1024, -} as const; /** Admission of the banners reported by one configured WABT tool directory. */ export type WabtVersionAdmission = diff --git a/src/wasm/WabtWat.ts b/src/wasm/WabtWat.ts index a5305728d8..f9c909eec6 100644 --- a/src/wasm/WabtWat.ts +++ b/src/wasm/WabtWat.ts @@ -67,7 +67,7 @@ const moduleNames = ( }; /** Extract exact top-level WABT WAT forms; no raw WASM decoding or import/export name unescaping. */ -export const parseWabtWat = (text: string) => { +export const parseWabtWat = (text: string, maximumInterfaceBytes?: number) => { if (!/^\(module(?:\s|\))/u.test(text)) throw new WabtAdmissionError("unsupported_module_prefix"); const forms: string[] = []; @@ -160,6 +160,6 @@ export const parseWabtWat = (text: string) => { customSectionNames, debugModuleNames, debugNameMetadata, - interface: parseWabtInterface(forms), + interface: parseWabtInterface(forms, maximumInterfaceBytes), }; }; diff --git a/src/wasm/WasmInterfaceBudget.test.ts b/src/wasm/WasmInterfaceBudget.test.ts index 9f4edd5fde..09878685cd 100644 --- a/src/wasm/WasmInterfaceBudget.test.ts +++ b/src/wasm/WasmInterfaceBudget.test.ts @@ -1,15 +1,18 @@ import { expect, it } from "vitest"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; -import { WASM_INTERFACE_REPRESENTATION_BYTES } from "../domain/wasm/wasmInterface.js"; -import { WasmInterfaceBudget } from "./WasmInterfaceBudget.js"; +import { + DEFAULT_WASM_INTERFACE_REPRESENTATION_BYTES, + MAX_WASM_INTERFACE_REPRESENTATION_BYTES, +} from "../domain/wasm/wasmInterface.js"; +import { WasmInterfaceBudget } from "../domain/wasm/wasmInterfaceBudget.js"; it("refuses typed WASM interface growth without asking the caller to wait", () => { const budget = new WasmInterfaceBudget(); expect(() => - budget.reserve(WASM_INTERFACE_REPRESENTATION_BYTES + 1), + budget.reserve(DEFAULT_WASM_INTERFACE_REPRESENTATION_BYTES + 1), ).toThrowError(); try { - budget.reserve(WASM_INTERFACE_REPRESENTATION_BYTES + 1); + budget.reserve(DEFAULT_WASM_INTERFACE_REPRESENTATION_BYTES + 1); throw new Error("Expected a representation refusal"); } catch (error) { const projected = projectAnalysisError( @@ -20,17 +23,37 @@ it("refuses typed WASM interface growth without asking the caller to wait", () = details: { resource: "capacity", reported_limits: { - maximum_bytes: WASM_INTERFACE_REPRESENTATION_BYTES, + maximum_bytes: DEFAULT_WASM_INTERFACE_REPRESENTATION_BYTES, }, }, }); - expect(projected.remediation.action).toContain( - "typed WASM interface inventory", - ); - expect(projected.remediation.action).toContain("fixed byte allowance"); + expect(projected.remediation.action).toContain("max_interface_bytes"); expect(projected.remediation.action).not.toContain("fixed count"); expect(projected.remediation.action).not.toMatch( /Wait for an active analysis/u, ); } }); + +it("does not suggest an impossible increase at the runtime representation maximum", () => { + const budget = new WasmInterfaceBudget( + MAX_WASM_INTERFACE_REPRESENTATION_BYTES, + ); + try { + budget.reserve(MAX_WASM_INTERFACE_REPRESENTATION_BYTES + 1); + throw new Error("Expected serialization capacity refusal"); + } catch (error) { + const projected = projectAnalysisError( + error as Parameters[0], + ); + expect(projected).toMatchObject({ + details: { reported_limits: { source: "request" } }, + }); + expect(projected.remediation.action).toContain( + "already at the runtime representation maximum", + ); + expect(projected.remediation.action).not.toContain( + "Raise max_interface_bytes", + ); + } +}); diff --git a/src/wasm/WasmInterfaceBudget.ts b/src/wasm/WasmInterfaceBudget.ts deleted file mode 100644 index dc910ea2b3..0000000000 --- a/src/wasm/WasmInterfaceBudget.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; -import { fixedAllowanceOptions } from "../domain/fixedAllowanceRemediation.js"; -import { WASM_INTERFACE_REPRESENTATION_BYTES } from "../domain/wasm/wasmInterface.js"; - -/** Eager typed observations share the existing WABT output capacity, not an agent token estimate. */ -export class WasmInterfaceBudget { - #remaining = WASM_INTERFACE_REPRESENTATION_BYTES; - reserve(bytes: number): void { - if (!Number.isSafeInteger(bytes) || bytes < 0 || bytes > this.#remaining) - throw new AnalysisResourceConstraintError( - "inspect_wasm_artifact", - "capacity", - "Typed WASM interface decoding exceeds the WABT output representation budget", - { maximum_bytes: WASM_INTERFACE_REPRESENTATION_BYTES }, - fixedAllowanceOptions("The typed WASM interface inventory", "bytes"), - ); - this.#remaining -= bytes; - } -} diff --git a/src/windows/WindowsPrivateRuntime.test.ts b/src/windows/WindowsPrivateRuntime.test.ts index bdf6e541b3..db2a71d289 100644 --- a/src/windows/WindowsPrivateRuntime.test.ts +++ b/src/windows/WindowsPrivateRuntime.test.ts @@ -392,13 +392,14 @@ it("retains bound refusal and failed file close until the same owner recovers", expect(closes).toBe(2); }); -it.each(["valid", "oversized"] as const)( +it.each(["valid", "oversized", "oversized-close-failed"] as const)( "observes a %s Windows Ghidra endpoint through its owning runtime", async (kind) => { const { observeGhidraEndpoint } = await import("../ghidra/GhidraTransport.js"); const valid = Buffer.from('{"host":"127.0.0.1","port":65535}\n'); const operations: string[] = []; + let fileCloses = 0; nativeCall.mockImplementation((operation: string) => { operations.push(operation); if (operation === "runtime_create") return observation; @@ -408,6 +409,12 @@ it.each(["valid", "oversized"] as const)( size: kind === "valid" ? valid.length : valid.length + 1, }; if (operation === "read") return valid; + if ( + operation === "close" && + kind === "oversized-close-failed" && + fileCloses++ === 0 + ) + throw nativeFailure(5); return null; }); const runtime = WindowsPrivateRuntime.create("C:\\", "rea-"); @@ -430,8 +437,26 @@ it.each(["valid", "oversized"] as const)( value: { host: "127.0.0.1", port: 65535 }, }); else { - expect(result.ok).toBe(false); + expect(result).toMatchObject({ + ok: false, + error: { + kind: "start", + message: "Ghidra TCP endpoint is invalid", + cause: { reason: "limit" }, + }, + }); expect(operations).not.toContain("read"); + if (kind === "oversized-close-failed") { + if (result.ok) throw new Error("Expected endpoint admission failure"); + const cleanup = result.error.cleanupFailure; + expect(cleanup?.cleanupIncomplete).toBe(true); + expect(cleanup?.cleanupResources).toEqual([ + admittedFile.requestedPath, + ]); + expect(windowsPrivateRuntime(observation.path)).toBe(runtime); + await runtime.close(); + expect(cleanup?.cleanupIncomplete).toBe(false); + } } expect(operations).toContain("close"); } finally { diff --git a/tests/acceptance/analysisSettingsMcp.test.ts b/tests/acceptance/analysisSettingsMcp.test.ts index c855950f53..ca10a340a7 100644 --- a/tests/acceptance/analysisSettingsMcp.test.ts +++ b/tests/acceptance/analysisSettingsMcp.test.ts @@ -13,6 +13,75 @@ import { nativeMatchingNamedInventory } from "../fixtures/nativeFunctionMatching import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; import { parseEvidence } from "../../src/domain/evidence.js"; +it("recovers expanded ASAR listing admission on the same MCP connection independently of header bytes", async () => { + const root = await createTestTempDirectory("rea-listing-settings-"); + const archive = join(root, "small.asar"); + const payload = Buffer.from("answer"); + await writeFile( + archive, + Buffer.concat([ + asarHeaderBytes( + JSON.stringify({ + files: { "answer.txt": { size: payload.length, offset: "0" } }, + }), + ), + payload, + ]), + ); + const { client, call } = await connectLocalToolsMcp(); + try { + expect((await call("open_binary", { path: archive })).isError).not.toBe( + true, + ); + expect( + ( + await call("update_analysis_settings", { + changes: [ + { setting: "asar.max_listing_bytes", action: "set", value: 1 }, + ], + }) + ).isError, + ).not.toBe(true); + const refused = await call("inspect_artifact", {}); + expect(refused.isError).toBe(true); + const text = refused.content.find((item) => item.type === "text"); + if (text?.type !== "text") throw new Error("Missing projected error"); + expect(JSON.parse(text.text).error).toMatchObject({ + details: { + limit: { + setting: "asar.max_listing_bytes", + effective_value: 1, + agent_editable: true, + }, + }, + }); + expect( + ( + await call("update_analysis_settings", { + changes: [ + { setting: "asar.max_listing_bytes", action: "set", value: 4096 }, + ], + }) + ).isError, + ).not.toBe(true); + const recovered = await call("inspect_artifact", {}); + expect(recovered.isError, JSON.stringify(recovered)).not.toBe(true); + const inspected = artifactInspectionResultSchema.parse( + parseEvidence(recovered.structuredContent).normalized_result, + ); + const inventory = inspected.substeps.find( + (step) => step.operation === "inventory_artifact", + ); + expect( + inventory?.evidence.normalized_result.occurrences.some( + (row) => row.logical_path === "answer.txt", + ), + ).toBe(true); + } finally { + await client.close(); + } +}, 30000); + it("advertises exact contracts and recovers a full large-index archive over one MCP connection", async () => { const root = await createTestTempDirectory("rea-runtime-settings-"); const archive = join(root, "application.asar"); diff --git a/tests/acceptance/applications/javascriptAnalysisLimits.test.ts b/tests/acceptance/applications/javascriptAnalysisLimits.test.ts index b7a9215604..94399944aa 100644 --- a/tests/acceptance/applications/javascriptAnalysisLimits.test.ts +++ b/tests/acceptance/applications/javascriptAnalysisLimits.test.ts @@ -12,6 +12,11 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest, type SharedMcpCall } from "../../support/cli/cliFixture.js"; const cases = [ + { + name: "semantic evaluator precision", + args: ["--semantic-precision", '{"primitive_candidates":512}'], + controls: { semantic_precision: { primitive_candidates: 512 } }, + }, { name: "both limits", args: ["--max-heap-mb", "128", "--analysis-timeout-ms", "30000"], diff --git a/tests/acceptance/artifactReaderLimitsCli.test.ts b/tests/acceptance/artifactReaderLimitsCli.test.ts new file mode 100644 index 0000000000..65e9aa8a3c --- /dev/null +++ b/tests/acceptance/artifactReaderLimitsCli.test.ts @@ -0,0 +1,328 @@ +import { createHash } from "node:crypto"; +import { constants as bufferConstants } from "node:buffer"; +import { lstat, readFile, writeFile } from "node:fs/promises"; + +import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect } from "vitest"; + +import { + bunModuleGraphPayload, + elfAppendedBunPayload, +} from "../../src/artifacts/bun/BunStandalone.fixture.js"; +import { dotnetBundleFixture } from "../../src/artifacts/dotnet/DotnetBundle.fixture.js"; +import { nodeSeaElf } from "../../src/artifacts/nodeSea/NodeSea.fixture.js"; +import type { ArtifactReaderLimitsInput } from "../../src/domain/artifactReaderLimits.js"; +import { + artifactExtractionResultSchema, + artifactInventoryResultSchema, +} from "../../src/domain/artifactGraph.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { buildGodotPck } from "../fixtures/godotPck.js"; +import { workspaceCliTest as test } from "../support/cli/workspaceCliFixture.js"; + +const digest = (bytes: Buffer): string => + createHash("sha256").update(bytes).digest("hex"); + +const readInspection = (value: unknown) => { + const evidence = parseEvidence(value); + const result = artifactInspectionResultSchema.parse( + evidence.normalized_result, + ); + const source = result.substeps.find( + (step) => step.operation === "inventory_artifact", + )?.evidence; + if (source === undefined) throw new Error("Missing inventory Evidence"); + return { + evidence, + source, + inventory: artifactInventoryResultSchema.parse(source.normalized_result), + }; +}; + +interface ReaderCase { + readonly format: string; + readonly create: () => Buffer | Promise; + readonly lower: ArtifactReaderLimitsInput; + readonly raised: ArtifactReaderLimitsInput; + readonly parameter: string; + readonly value: number; +} + +const readers: readonly ReaderCase[] = [ + { + format: "bun-standalone", + create: () => + elfAppendedBunPayload( + bunModuleGraphPayload({ + modules: [ + { + name: "/first.js", + contents: Buffer.from("export const first=1;"), + }, + { + name: "/second.js", + contents: Buffer.from("export const second=2;"), + }, + ], + }), + ), + lower: { bun: { module_count: 1 } }, + raised: { bun: { module_count: 2 } }, + parameter: "artifact_limits.bun.module_count", + value: 1, + }, + { + format: "dotnet-bundle", + create: () => + dotnetBundleFixture([ + { path: "first.txt", bytes: Buffer.from("first") }, + { path: "second.txt", bytes: Buffer.from("second") }, + ]).bytes, + lower: { dotnet_bundle: { entry_count: 1 } }, + raised: { dotnet_bundle: { entry_count: 2 } }, + parameter: "artifact_limits.dotnet_bundle.entry_count", + value: 1, + }, + { + format: "node-sea", + create: () => nodeSeaElf({ execArgv: ["", ""] }), + lower: { node_sea: { argument_count: 1 } }, + raised: { node_sea: { argument_count: 2 } }, + parameter: "artifact_limits.node_sea.argument_count", + value: 1, + }, + { + format: "godot-pck", + create: () => + buildGodotPck(2, [{ path: "res://main.gd", data: "extends Node\n" }]), + lower: { godot: { directory_bytes: 1 } }, + raised: { godot: { directory_bytes: 4096 } }, + parameter: "artifact_limits.godot.directory_bytes", + value: 1, + }, + { + format: "zip", + create: async () => { + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("first.txt", new TextReader("first")); + await writer.add("second.txt", new TextReader("second")); + return Buffer.from(await writer.close()); + }, + lower: { expanded_bytes: 1 }, + raised: { expanded_bytes: 11 }, + parameter: "artifact_limits.expanded_bytes", + value: 1, + }, +]; + +test("rejects malformed directory dimensions before suggesting a budget increase", async ({ + cli, + workspace, +}) => { + const path = workspace.path("unrepresentable-directory.pck"); + const bytes = buildGodotPck(2, [{ path: "main.gd", data: "extends Node\n" }]); + // The v2 directory follows its 96-byte header and four-byte entry count. + // Change only the declared length; do not allocate the claimed path. + bytes.writeUInt32LE(0xffffffff, 100); + await writeFile(path, bytes); + const result = await cli.run({ + arguments: [ + "inspect-artifact", + path, + "--json", + "--artifact-limits", + JSON.stringify({ + godot: { directory_bytes: bufferConstants.MAX_STRING_LENGTH }, + }), + ], + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "artifact_operation_failed", + retryable: false, + details: { + reason: "format", + }, + }); + const failure = result.json as { + remediation: { action: string }; + message: string; + }; + expect(failure.remediation.action).not.toContain("Select a larger"); + expect(failure.message).not.toContain("Select a larger"); + expect((await readFile(path)).equals(bytes)).toBe(true); +}); + +for (const reader of readers) + test(`recovers the same ${reader.format} through selected CLI admission`, async ({ + cli, + workspace, + }) => { + const path = workspace.path("carrier.bin"); + const bytes = await reader.create(); + await writeFile(path, bytes); + const command = ["inspect-artifact", path, "--json"]; + const baseline = await cli.run({ arguments: command }); + expect(baseline.exitCode, baseline.stderr).toBe(0); + const original = readInspection(baseline.json); + expect(original.source.parameters).toEqual({ integrity_policy: "fail" }); + expect(original.inventory.effective_limits).toBeUndefined(); + + const refused = await cli.run({ + arguments: [ + ...command, + "--artifact-limits", + JSON.stringify(reader.lower), + ], + }); + expect(refused.exitCode).toBe(1); + expect(refused.json).toMatchObject({ + code: "truncated", + retryable: true, + details: { + reader_limit: { + parameter: reader.parameter, + effective_value: reader.value, + applies_at: "next_operation", + agent_editable: true, + rea_restart_required: false, + }, + }, + }); + const recovered = await cli.run({ + arguments: [ + ...command, + "--artifact-limits", + JSON.stringify(reader.raised), + ], + }); + expect(recovered.exitCode, recovered.stderr).toBe(0); + const inspected = readInspection(recovered.json); + expect(inspected.source.parameters).toEqual({ + integrity_policy: "fail", + artifact_limits: reader.raised, + }); + expect(inspected.inventory.effective_limits).toMatchObject(reader.raised); + expect(inspected.inventory.manifest).toMatchObject({ + root_format: reader.format, + root_sha256: digest(bytes), + root_artifact_id: original.inventory.manifest.root_artifact_id, + }); + expect(inspected.inventory.occurrences).toEqual( + original.inventory.occurrences, + ); + expect(digest(await readFile(path))).toBe(digest(bytes)); + }); + +test("admits the same SEA carrier above the legacy 4096-argument ceiling", async ({ + cli, + workspace, +}) => { + const path = workspace.path("many-arguments.bin"); + const bytes = nodeSeaElf({ + execArgv: Array.from({ length: 4097 }, () => ""), + }); + await writeFile(path, bytes); + const command = ["inspect-artifact", path, "--json"]; + const refused = await cli.run({ arguments: command }); + expect(refused.exitCode).toBe(1); + expect(refused.json).toMatchObject({ + code: "truncated", + details: { + reader_limit: { + parameter: "artifact_limits.node_sea.argument_count", + effective_value: 4096, + required_value: 4097, + }, + }, + }); + const recovered = await cli.run({ + arguments: [ + ...command, + "--artifact-limits", + '{"node_sea":{"argument_count":4097}}', + ], + }); + expect(recovered.exitCode, recovered.stderr).toBe(0); + expect(readInspection(recovered.json).inventory.manifest).toMatchObject({ + root_format: "node-sea", + root_sha256: digest(bytes), + }); +}); + +test("applies the selected reader budget before extraction and preserves its output", async ({ + cli, + workspace, +}) => { + const path = workspace.path("host.bin"); + const output = workspace.path("selected-output"); + const bytes = dotnetBundleFixture([ + { path: "first.txt", bytes: Buffer.from("first") }, + { path: "second.txt", bytes: Buffer.from("second") }, + ]).bytes; + await writeFile(path, bytes); + const command = ["extract-artifact", path, "--output-root", output, "--json"]; + const refused = await cli.run({ + arguments: [ + ...command, + "--artifact-limits", + '{"dotnet_bundle":{"entry_count":1}}', + ], + }); + expect(refused.exitCode).toBe(1); + expect(refused.json).toMatchObject({ + code: "truncated", + details: { + reader_limit: { parameter: "artifact_limits.dotnet_bundle.entry_count" }, + }, + }); + await expect(lstat(output)).rejects.toMatchObject({ code: "ENOENT" }); + const recovered = await cli.run({ + arguments: [ + ...command, + "--artifact-limits", + '{"dotnet_bundle":{"entry_count":2}}', + ], + }); + expect(recovered.exitCode, recovered.stderr).toBe(0); + const evidence = parseEvidence(recovered.json); + const extracted = artifactExtractionResultSchema.parse( + evidence.normalized_result, + ); + expect(evidence.parameters).toMatchObject({ + artifact_limits: { dotnet_bundle: { entry_count: 2 } }, + }); + expect(extracted.effective_limits).toMatchObject({ + dotnet_bundle: { entry_count: 2 }, + }); + expect(extracted.manifest.root_sha256).toBe(digest(bytes)); + expect( + await readFile(workspace.path("selected-output/first.txt"), "utf8"), + ).toBe("first"); + expect( + await readFile(workspace.path("selected-output/second.txt"), "utf8"), + ).toBe("second"); +}); + +test("rejects unknown reader-policy fields before opening the selected artifact", async ({ + cli, + workspace, +}) => { + for (const policy of [ + { unknown: 1 }, + { node_sea: { argument_count: 1, unknown: 1 } }, + ]) { + const invalid = await cli.run({ + arguments: [ + "inspect-artifact", + workspace.path("does-not-exist"), + "--artifact-limits", + JSON.stringify(policy), + "--json", + ], + }); + expect(invalid.exitCode).toBe(1); + expect(invalid.json).toMatchObject({ code: "VALIDATION_ERROR" }); + } +}); diff --git a/tests/acceptance/bunPathBudgetCli.test.ts b/tests/acceptance/bunPathBudgetCli.test.ts index 3768082dbc..c2a69fd4d0 100644 --- a/tests/acceptance/bunPathBudgetCli.test.ts +++ b/tests/acceptance/bunPathBudgetCli.test.ts @@ -1,7 +1,6 @@ import { writeFile } from "node:fs/promises"; import { expect } from "vitest"; import { workspaceCliTest as test } from "../support/cli/workspaceCliFixture.js"; -import { artifactFixedLimitFailure } from "../../src/artifacts/ArtifactReader.js"; import { bunModuleGraphPayload, elfAppendedBunPayload, @@ -31,17 +30,19 @@ for (const shared of [false, true]) }); if (!shared) { expect(result.exitCode).toBe(1); - const limit = artifactFixedLimitFailure( - "Artifact path tree", - 250_000, - "path nodes", - ); expect(result.json).toMatchObject({ code: "truncated", + retryable: true, details: { reason: "limit", boundary: "capacity", - detail: limit.message, + reader_limit: { + parameter: "artifact_limits.bun.path_nodes", + effective_value: 250_000, + required_value: 250_112, + unit: "count", + can_increase: true, + }, }, }); } else { diff --git a/tests/acceptance/dotnetBundleJourney.test.ts b/tests/acceptance/dotnetBundleJourney.test.ts index ffc59c6268..7a48589d15 100644 --- a/tests/acceptance/dotnetBundleJourney.test.ts +++ b/tests/acceptance/dotnetBundleJourney.test.ts @@ -6,7 +6,6 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { expect } from "vitest"; -import { artifactFixedLimitFailure } from "../../src/artifacts/ArtifactReader.js"; import { dotnetBundleFixture } from "../../src/artifacts/dotnet/DotnetBundle.fixture.js"; import { cArchiveExecutable } from "../../src/artifacts/pyinstaller/CArchive.fixture.js"; import { artifactExtractionResultSchema } from "../../src/domain/artifactGraph.js"; @@ -240,18 +239,21 @@ test("reports excessive bundle path depth as a public CLI limit", async ({ environment: { REA_LOG_LEVEL: "silent" }, }); expect(result.exitCode, result.stderr).toBe(1); - const failure = artifactFixedLimitFailure( - "Artifact path depth", - 256, - "path parts", - ); - expect(result.json).toMatchObject({ - message: failure.message, - details: { boundary: "capacity", detail: failure.message }, - }); expect(result.json).toMatchObject({ code: "truncated", category: "truncated", - details: { reason: "limit", truncated: true }, + retryable: true, + details: { + reason: "limit", + truncated: true, + boundary: "capacity", + reader_limit: { + parameter: "artifact_limits.dotnet_bundle.path_parts", + effective_value: 256, + required_value: 257, + unit: "count", + can_increase: true, + }, + }, }); }); diff --git a/tests/acceptance/godotPck.test.ts b/tests/acceptance/godotPck.test.ts index d16d4b86b2..0e69243b97 100644 --- a/tests/acceptance/godotPck.test.ts +++ b/tests/acceptance/godotPck.test.ts @@ -407,10 +407,17 @@ cliTest( expect(reply.exitCode, reply.stdout + reply.stderr).toBe(1); expect(reply.json).toMatchObject({ code: "truncated", + retryable: true, details: { reason: "limit", truncated: true, - detail: expect.stringContaining("expansion budget"), + reader_limit: { + parameter: "artifact_limits.expanded_bytes", + effective_value: 268_435_456, + required_value: 268_697_600, + unit: "bytes", + can_increase: true, + }, }, }); expect(await readdir(root)).toEqual(["aliased.pck"]); diff --git a/tests/acceptance/kernelModuleCli.test.ts b/tests/acceptance/kernelModuleCli.test.ts index 5eacc44488..2a66dc9072 100644 --- a/tests/acceptance/kernelModuleCli.test.ts +++ b/tests/acceptance/kernelModuleCli.test.ts @@ -9,6 +9,43 @@ import { import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; +cliTest( + "honors modinfo field retention budgets through CLI flags", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-kernel-module-text-cli-"); + const path = join(root, "text.ko"); + const key = "k".repeat(257); + const value = "v".repeat(4097); + await writeFile( + path, + kernelModuleFixture({ modinfo: Buffer.from(`${key}=${value}\0`) }), + ); + const result = await cli.run({ + arguments: [ + "inspect-kernel-module", + path, + "--max-modinfo-key-bytes", + "257", + "--max-modinfo-value-bytes", + "4097", + "--json", + ], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = toolContract("inspect_kernel_module").outputSchema.parse( + result.json, + ); + expect(evidence.normalized_result.modinfo?.entries[0]).toMatchObject({ + key, + value, + status: "observed", + reason: null, + omitted_fields: [], + }); + expect(evidence.normalized_result.coverage.status).toBe("complete"); + }, +); + cliTest( "reports license, duplicate aliases, and version CRCs through the public CLI", async ({ cli }) => { diff --git a/tests/acceptance/managedCliOutput.test.ts b/tests/acceptance/managedCliOutput.test.ts index 71bc9aa298..737e5bc55c 100644 --- a/tests/acceptance/managedCliOutput.test.ts +++ b/tests/acceptance/managedCliOutput.test.ts @@ -9,6 +9,83 @@ import { buildManagedPeFixture } from "../../src/dotnet/ManagedPe.fixture.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest, runFileCli } from "../support/cli/cliFixture.js"; +cliTest( + "recovers the same managed artifact through decimal CLI budgets and retains caller choices", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-managed-cli-budgets-"); + const path = join(root, "fixture.dll"); + const bytes = buildManagedPeFixture(); + await writeFile(path, bytes); + const command = ["inspect-managed-artifact", path]; + const refused = await cli.run({ + arguments: [ + ...command, + "--max-snapshot-bytes", + String(bytes.length - 1), + "--json", + ], + }); + expect(refused.exitCode).toBe(1); + expect(refused.json).toMatchObject({ + code: "resource_constraint", + details: { reported_limits: { max_snapshot_bytes: bytes.length - 1 } }, + }); + const admitted = ["--max-snapshot-bytes", String(bytes.length)]; + const partial = await cli.run({ + arguments: [ + ...command, + ...admitted, + "--max-metadata-representation-bytes", + "1", + "--json", + ], + }); + expect(partial.exitCode, partial.stderr).toBe(0); + expect(partial.json).toMatchObject({ + parameters: { + max_snapshot_bytes: bytes.length, + max_metadata_representation_bytes: 1, + }, + normalized_result: { + coverage: { + state: "partial", + issues: [expect.objectContaining({ code: "resource-limit" })], + }, + }, + }); + const complete = await cli.run({ + arguments: [ + ...command, + ...admitted, + "--max-metadata-representation-bytes", + "1048576", + "--json", + ], + }); + expect(complete.exitCode, complete.stderr).toBe(0); + expect(complete.json).toMatchObject({ + subject: { + digest: { sha256: createHash("sha256").update(bytes).digest("hex") }, + }, + parameters: { + max_snapshot_bytes: bytes.length, + max_metadata_representation_bytes: 1048576, + }, + normalized_result: { coverage: { state: "complete", issues: [] } }, + }); + for (const flag of [ + "--max-snapshot-bytes", + "--max-metadata-representation-bytes", + ]) { + const malformed = await cli.run({ + arguments: [...command, flag, "1e3", "--json"], + }); + expect(malformed.exitCode).toBe(1); + expect(malformed.json).toMatchObject({ code: "VALIDATION_ERROR" }); + } + }, +); + cliTest( "managed CLI preserves malformed prefix evidence without a complete CIL fingerprint", async ({ cli }) => { diff --git a/tests/acceptance/modelArtifactCli.test.ts b/tests/acceptance/modelArtifactCli.test.ts index 9b0fbb79ac..4ebf0d9a54 100644 --- a/tests/acceptance/modelArtifactCli.test.ts +++ b/tests/acceptance/modelArtifactCli.test.ts @@ -444,7 +444,7 @@ cliTest( ); cliTest( - "preserves exact large safetensors dimensions while bounding actual offsets", + "preserves exact large safetensors dimensions while rejecting impossible file offsets", async ({ cli }) => { const root = await createTestTempDirectory("rea-model-shape-precision-"); const path = join(root, "empty.safetensors"); @@ -524,6 +524,7 @@ cliTest( // the offset is converted to a host number. expect(offsetFailure.error).toMatchObject({ code: "target_unavailable", + category: "unsupported_target", details: { constraint: "file_format_rejected", reason: "Declared extent is outside the file", diff --git a/tests/acceptance/nodeSeaCli.test.ts b/tests/acceptance/nodeSeaCli.test.ts index a28623d3e5..6fd0d2be32 100644 --- a/tests/acceptance/nodeSeaCli.test.ts +++ b/tests/acceptance/nodeSeaCli.test.ts @@ -315,8 +315,20 @@ cliTest( arguments: ["inspect-artifact", source, "--json"], }); expect(result.exitCode).not.toBe(0); - expect(result.stdout + result.stderr).toContain( - "Node SEA asset count exceeds", - ); + expect(result.json).toMatchObject({ + code: "truncated", + retryable: true, + details: { + reason: "limit", + boundary: "capacity", + reader_limit: { + parameter: "artifact_limits.node_sea.asset_count", + effective_value: 100_000, + required_value: 100_001, + unit: "count", + can_increase: true, + }, + }, + }); }, ); diff --git a/tests/acceptance/rustBinaryCli.test.ts b/tests/acceptance/rustBinaryCli.test.ts index 44b7afe87a..8ba924ebb1 100644 --- a/tests/acceptance/rustBinaryCli.test.ts +++ b/tests/acceptance/rustBinaryCli.test.ts @@ -688,29 +688,33 @@ cliTest( ); cliTest( - "refuses distinct rustc paths above capacity with non-retryable guidance", + "retrieves a larger compiler-path prefix through the CLI without losing dependencies", async ({ cli }) => { const root = await createTestTempDirectory("rea-rust-cli-overflow-"); const path = join(root, "overflow.elf"); await writeFile(path, createRustHashOverflowImage()); const result = await cli.run({ - arguments: ["inspect-rust-binary", path, "--json"], + arguments: [ + "inspect-rust-binary", + path, + "--max-rustc-hashes", + "65", + "--producer-limits", + '{"input_bytes":8192,"output_bytes":16384}', + "--json", + ], }); - expect(result.exitCode).toBe(1); - expect(result.json).toMatchObject({ - code: "resource_constraint", - category: "resource_constraint", - retryable: false, - remediation: { - action: expect.stringMatching( - /rustc commit.path inventory.*fixed count/iu, - ), - }, - details: { - operation: "inspect_rust_binary", - resource: "capacity", - reported_limits: { maximum_rustc_hashes: 64 }, - }, + expect(result.exitCode).toBe(0); + const evidence = toolContract("inspect_rust_binary").outputSchema.parse( + result.json, + ); + const report = rustBinarySchema.parse(evidence.normalized_result); + expect(evidence.parameters).toMatchObject({ + max_rustc_hashes: 65, + producer_limits: { input_bytes: 8192, output_bytes: 16384 }, }); + expect(report.rustc_commit?.hashes).toHaveLength(65); + expect(report.rustc_commit).not.toHaveProperty("coverage"); + expect(report.auditable_metadata?.packages).toHaveLength(2); }, ); diff --git a/tests/acceptance/rustDependencyViews.test.ts b/tests/acceptance/rustDependencyViews.test.ts index 0d8c8915d4..c7802653ec 100644 --- a/tests/acceptance/rustDependencyViews.test.ts +++ b/tests/acceptance/rustDependencyViews.test.ts @@ -10,11 +10,61 @@ import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; import { createRustImage, + createRustHashOverflowImage, rustDependencyPayload, } from "../fixtures/rust/image.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; +cliTest( + "keeps compiler-prefix coverage in CLI summaries until the selected budget admits every hash", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-rust-compiler-summary-"); + const path = join(root, "compiler-paths.elf"); + await writeFile(path, createRustHashOverflowImage()); + for (const maxHashes of [64, 65]) { + const output = join(root, `source-${String(maxHashes)}.json`); + const delivered = await cli.run({ + arguments: [ + "inspect-rust-binary", + path, + "--detail", + "summary", + "--evidence_output", + output, + "--max-rustc-hashes", + String(maxHashes), + "--json", + ], + }); + expect(delivered.exitCode, delivered.stderr).toBe(0); + const summary = analysisViewResultSchema.parse( + (delivered.json as { summary: { normalized_result: unknown } }).summary + .normalized_result, + ); + if ( + summary.kind !== "summary" || + summary.parent_operation !== "inspect_rust_binary" + ) + throw new Error("Expected a Rust summary"); + expect(summary.summary.rustc_commit).toEqual({ + evidence_kind: "inferred", + hashes_count: maxHashes, + ...(maxHashes === 64 + ? { + coverage: { + status: "partial", + reason: "max_rustc_hashes", + effective_limit: 64, + }, + } + : {}), + }); + expect(summary.summary.auditable_metadata?.counts.packages).toBe(2); + } + }, +); + /** Both authenticated retention and portable imports query the same captured facts. */ const assertRetainedPages = async ( call: Awaited>["call"], diff --git a/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts index e6ded3b083..43f52aab7d 100644 --- a/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts +++ b/tests/acceptance/setup/registrationRuntimeEnvironmentCli.test.ts @@ -69,6 +69,11 @@ cliTest( }), REA_SWIFTC_COMPILE_TIMEOUT_MS: "180000", REA_LOG_LEVEL: "warn", + REA_ASAR_MAX_LISTING_BYTES: "134217728", + REA_GHIDRA_VALUE_FLOW_MAX_OPERATIONS: "6000", + REA_GHIDRA_VALUE_FLOW_MAX_INPUTS_PER_OPERATION: "128", + REA_GHIDRA_VALUE_FLOW_MAX_EDGES: "24000", + REA_GHIDRA_DECOMPILE_MAX_PAYLOAD_MIB: "128", REA_REFERENCE_SECRET_PATTERNS_JSON: '["explicit-private-value"]', REA_ANALYSIS_PROVIDER: "ghidra", REA_MCP_INPUT_SCHEMA_PROFILE: "compact", diff --git a/tests/acceptance/webSourceLocationCli.test.ts b/tests/acceptance/webSourceLocationCli.test.ts index a08d365762..cc85f089db 100644 --- a/tests/acceptance/webSourceLocationCli.test.ts +++ b/tests/acceptance/webSourceLocationCli.test.ts @@ -8,11 +8,113 @@ import { } from "@modelcontextprotocol/client/stdio"; import { fileURLToPath } from "node:url"; import { createHash } from "node:crypto"; -import { mkdir, symlink, writeFile } from "node:fs/promises"; +import { mkdir, symlink, writeFile, readFile, unlink } from "node:fs/promises"; import { join } from "node:path"; import { webSourceLocationFixture } from "../fixtures/webSourceLocation.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; -import { cliTest } from "../support/cli/cliFixture.js"; +import { cliTest, type TestCli } from "../support/cli/cliFixture.js"; + +const expectRetainedSourceMap = async ( + path: string, + evidenceId: string, + sha256: string, +) => { + const bytes = await readFile(path); + const bundle = JSON.parse(bytes.toString("utf8")) as { + records: { + evidence_id: string; + raw_result: { source_map_text?: string } | null; + }[]; + }; + const parent = bundle.records.find( + (record) => record.evidence_id === evidenceId, + ); + expect(parent?.evidence_id).toBe(evidenceId); + expect( + createHash("sha256") + .update(parent?.raw_result?.source_map_text ?? "") + .digest("hex"), + ).toBe(sha256); + return bytes.length; +}; + +const expectEqualPositionSummary = async ( + client: Client, + input: Record, +) => { + const compact = await client.callTool({ + name: "trace_web_source_location", + arguments: { + ...input, + detail: "summary", + }, + }); + expect(compact.isError).not.toBe(true); + expect(compact.structuredContent).toMatchObject({ + normalized_result: { + summary: { + match_count: 2, + matches: [ + { match_index: 0, source_index: 0 }, + { match_index: 1, source_index: 1 }, + ], + }, + }, + }); + expect(JSON.stringify(compact.structuredContent)).not.toContain('"text":'); +}; + +const expectCliSourceLocationSummary = async ( + cli: TestCli, + arguments_: readonly string[], + expected: { bundlePath: string; evidenceId: string; sha256: string }, +) => { + const { bundlePath, evidenceId, sha256 } = expected; + const missingSink = await cli.run({ + arguments: [...arguments_, "--detail", "summary"], + timeoutMs: 45000, + }); + expect(missingSink.exitCode).toBe(1); + expect(missingSink.json).toMatchObject({ + code: "invalid_request", + details: { issues: [{ path: ["evidence_output"] }] }, + }); + const summary = await cli.run({ + arguments: [ + ...arguments_, + "--detail", + "summary", + "--evidence_output", + bundlePath, + ], + timeoutMs: 45000, + }); + expect(summary.exitCode, summary.stderr).toBe(0); + expect(summary.stdout.length).toBeLessThan(30000); + const retainedBytes = await expectRetainedSourceMap( + bundlePath, + evidenceId, + sha256, + ); + expect(summary.json).toMatchObject({ + evidence_bundle: { path: bundlePath, bytes: retainedBytes }, + summary: { + normalized_result: { + parent_evidence_id: evidenceId, + summary: { + match_count: 1, + matches: [ + { + match_index: 0, + source_index: 0, + content: { state: "embedded", code_units: 8 }, + }, + ], + }, + }, + }, + }); +}; const expectInvalidSourceLocation = ( json: unknown, @@ -183,6 +285,12 @@ cliTest( }); expect(mcpResponse.isError, JSON.stringify(mcpResponse)).not.toBe(true); expect(mcpResponse.structuredContent).toEqual(response.json); + await expectEqualPositionSummary(client, { + manifest_path: manifest, + script_index: 0, + source_map: { path: map, url: fixture.sourceMap.url }, + generated_position: { line: 1, column: 0 }, + }); const pastColumn = await client.callTool({ name: "trace_web_source_location", arguments: { @@ -507,3 +615,230 @@ cliTest( }, 180_000, ); + +cliTest( + "raises local source-map admission and recovers the same complete map through the owned codec", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-source-map-budget-cli-"); + const fixture = webSourceLocationFixture( + JSON.stringify({ + version: 3, + sources: ["original.ts"], + sourcesContent: ["original"], + names: [], + mappings: "AAAA", + x_padding: "x".repeat(1024), + }), + ); + await mkdir(join(root, "files", "modules"), { recursive: true }); + await writeFile(join(root, "files", "modules", "main.js"), fixture.source); + const manifest = join(root, "manifest.json"); + const map = join(root, "app.js.map"); + await writeFile(manifest, JSON.stringify(fixture.manifest)); + await writeFile(map, fixture.sourceMap.text); + const args = [ + "trace-web-source-location", + manifest, + "0", + map, + fixture.sourceMap.url, + "1", + "0", + "--format", + "json", + ]; + const refused = await cli.run({ + arguments: [...args, "--source-map-bytes", "128"], + timeoutMs: 45000, + }); + expect(refused.exitCode).toBe(1); + expect((refused.json as { code?: unknown }).code).toBe("truncated"); + expect(refused.json).toMatchObject({ + code: "truncated", + details: { + detail: expect.stringContaining("source_map_limits.map_bytes"), + }, + }); + const recovered = await cli.run({ + arguments: [ + ...args, + "--source-map-bytes", + String(Buffer.byteLength(fixture.sourceMap.text)), + ], + timeoutMs: 45000, + }); + expect(recovered.exitCode, recovered.stderr).toBe(0); + expect(recovered.json).toMatchObject({ + parameters: { + source_map_limits: { + map_bytes: Buffer.byteLength(fixture.sourceMap.text), + }, + }, + normalized_result: { + source_map: { + sha256: createHash("sha256") + .update(fixture.sourceMap.text) + .digest("hex"), + }, + matches: [ + { reported_source: "original.ts", content: { text: "original" } }, + ], + }, + }); + }, +); + +cliTest( + "admits a complete map above 4 MiB by default while honoring a smaller caller budget", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-map-default-admission-"); + const fixture = webSourceLocationFixture( + JSON.stringify({ + version: 3, + sources: ["original.ts"], + sourcesContent: ["original"], + names: [], + mappings: "AAAA", + x_retained_extension: "x".repeat(6 * 1024 * 1024), + }), + ); + await mkdir(join(root, "files", "modules"), { recursive: true }); + await writeFile(join(root, "files", "modules", "main.js"), fixture.source); + const manifest = join(root, "manifest.json"); + const map = join(root, "app.js.map"); + await writeFile(manifest, JSON.stringify(fixture.manifest)); + await writeFile(map, fixture.sourceMap.text); + const arguments_ = [ + "trace-web-source-location", + manifest, + "0", + map, + fixture.sourceMap.url, + "1", + "0", + "--json", + ]; + const refused = await cli.run({ + arguments: [...arguments_, "--source-map-bytes", "4194304"], + timeoutMs: 45000, + }); + expect(refused.exitCode).toBe(1); + expect(JSON.stringify(refused.json)).toContain( + "source_map_limits.map_bytes", + ); + const completed = await cli.run({ + arguments: arguments_, + timeoutMs: 45000, + }); + expect(completed.exitCode, completed.stderr).toBe(0); + const evidence = completed.json as { + evidence_id: string; + normalized_result: unknown; + raw_result: { source_map_text: string }; + }; + expect(evidence.normalized_result).toMatchObject({ + source_map: { sha256: fixture.sourceMap.file.sha256 }, + matches: [{ source_index: 0, content: { text: "original" } }], + }); + expect( + createHash("sha256") + .update(evidence.raw_result.source_map_text) + .digest("hex"), + ).toBe(fixture.sourceMap.file.sha256); + await expectCliSourceLocationSummary(cli, arguments_, { + bundlePath: join(root, "cli-complete.json"), + evidenceId: evidence.evidence_id, + sha256: fixture.sourceMap.file.sha256, + }); + const { call } = await connectLocalToolsMcp(); + const input = { + manifest_path: manifest, + script_index: 0, + source_map: { path: map, url: fixture.sourceMap.url }, + generated_position: { line: 1, column: 0 }, + }; + const overflow = await call("trace_web_source_location", input); + expect(parseMcpToolError(overflow).error).toMatchObject({ + code: "resource_constraint", + details: { + reported_limits: { + boundary: "mcp-response", + evidence_reference: { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }, + continuation: { + name: "inspect_analysis_view", + arguments: { + view: { kind: "source-location", collection: "matches" }, + }, + }, + }, + }, + }); + await unlink(map); + const source = { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }; + const page = await call("inspect_analysis_view", { + source, + view: { + kind: "source-location", + collection: "matches", + offset: 0, + limit: 1, + }, + }); + expect(page.isError).not.toBe(true); + expect(page.structuredContent).toMatchObject({ + normalized_result: { + items: [ + { + match_index: 0, + source_index: 0, + original_position: { line: 1, column: 0 }, + }, + ], + coverage: { total: 1, exhausted: true }, + }, + }); + const text = await call("inspect_analysis_view", { + source, + view: { + kind: "source-location", + collection: "source-text", + match_index: 0, + offset: 0, + limit: 8, + }, + }); + expect(text.isError).not.toBe(true); + expect(text.structuredContent).toMatchObject({ + normalized_result: { item: { text: "original" } }, + }); + const exportPath = join(root, "session-complete.json"); + expect( + (await call("export_evidence_bundle", { path: exportPath })).isError, + ).not.toBe(true); + await expectRetainedSourceMap( + exportPath, + evidence.evidence_id, + fixture.sourceMap.file.sha256, + ); + const emptyMap = '{"version":3,"sources":[],"names":[],"mappings":""}'; + await writeFile(map, emptyMap); + const empty = await call("trace_web_source_location", { + ...input, + detail: "summary", + }); + expect(empty.isError).not.toBe(true); + expect(empty.structuredContent).toMatchObject({ + normalized_result: { + summary: { match_count: 0, matches: [] }, + coverage: { status: "empty", total: 0, exhausted: true }, + }, + }); + }, + 180_000, +); diff --git a/tests/boundary/binaryDiagnostics/pwntoolsLayoutProvider.test.ts b/tests/boundary/binaryDiagnostics/pwntoolsLayoutProvider.test.ts index b1e816758b..2a5eea5831 100644 --- a/tests/boundary/binaryDiagnostics/pwntoolsLayoutProvider.test.ts +++ b/tests/boundary/binaryDiagnostics/pwntoolsLayoutProvider.test.ts @@ -16,9 +16,10 @@ import { binaryLayoutFixture } from "../../fixtures/binaryDiagnostics/layout.js" import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; const unsupportedHost = process.platform !== "linux" || process.arch !== "x64"; -const requestSchema = z.strictObject({ +const requestSchema = z.object({ snapshot_path: z.string(), reply_path: z.string(), + producer_limits: z.object({ diagnostic_bytes: z.number() }), }); it.runIf(!unsupportedHost).each([ @@ -87,7 +88,9 @@ it.runIf(!unsupportedHost).each([ resource === "cpu" ? "CPU" : "file-size", ); if (resource === "file-size") { - expect(projected.remediation.action).toContain("outputBytes"); + expect(projected.remediation.action).toContain( + "producer_limits.file_bytes", + ); expect(projected.remediation.action).toContain("67108864"); expect(projected.remediation.action).toContain("file_size_bytes"); expect(projected.remediation.action).not.toContain("address-space"); @@ -226,9 +229,14 @@ it }, ); -it.runIf(!unsupportedHost).each([1048576, 1048577])( - "keeps complete diagnostics or fails explicitly when a valid reply emits %s bytes", - async (bytes) => { +it.runIf(!unsupportedHost).each([ + { bytes: 1048576, budget: undefined }, + { bytes: 1048577, budget: undefined }, + { bytes: 64, budget: 64 }, + { bytes: 65, budget: 64 }, +])( + "enforces selected diagnostic retention for a valid reply: %j", + async ({ bytes, budget }) => { const { path } = await fixture(); let ownedPath = ""; const provider = new PwntoolsLayoutProvider( @@ -240,6 +248,9 @@ it.runIf(!unsupportedHost).each([1048576, 1048577])( const request = requestSchema.parse( JSON.parse(await readFile(requestPath, "utf8")), ); + expect(request.producer_limits.diagnostic_bytes).toBe( + budget ?? 1048576, + ); const { artifact: _artifact, diagnostics: _diagnostics, @@ -263,8 +274,13 @@ it.runIf(!unsupportedHost).each([1048576, 1048577])( }); }, ); - const result = await provider.inspect({ path }); - if (bytes === 1048576) { + const result = await provider.inspect({ + path, + ...(budget === undefined + ? {} + : { producer_limits: { diagnostic_bytes: budget } }), + }); + if (bytes <= (budget ?? 1048576)) { if (!result.ok) throw result.error; expect(result.value.diagnostics.stdout).toHaveLength(bytes); expect(result.value.diagnostics.truncated).toBe(false); @@ -336,7 +352,7 @@ it.runIf(!unsupportedHost).each([ }, }); expect(projected.remediation.action).toContain("memory"); - expect(projected.remediation.action).toContain("address-space"); + expect(projected.remediation.action).toContain("address_space_bytes"); await expect(access(ownedPath)).rejects.toMatchObject({ code: "ENOENT" }); expect(await readFile(path, "utf8")).toBe("source-owned-seam-bytes"); }, diff --git a/tests/boundary/browser/historicalCaptureDecoder.test.ts b/tests/boundary/browser/historicalCaptureDecoder.test.ts index 3683ca9b34..c6bd965d35 100644 --- a/tests/boundary/browser/historicalCaptureDecoder.test.ts +++ b/tests/boundary/browser/historicalCaptureDecoder.test.ts @@ -1025,9 +1025,14 @@ it.each([ }, }, }); - expect(projected.remediation.action).toContain("no runtime override"); expect(projected.remediation.action).toContain( - "no completed capture Evidence", + `Increase max_reply_bytes above ${String(WEB_NETWORK_CAPTURE_LIMITS.outputBytes)}`, + ); + expect(projected.remediation.action).toContain( + "next inspect_web_network_capture call", + ); + expect(projected.remediation.action).toContain( + "do not recreate a completed report from a refused decode", ); expect(projected.remediation.action).not.toContain("rea doctor"); expect(result.error.partialObservation).toMatchObject({ diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index 1e238aa3ba..e25106d1da 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -1529,7 +1529,9 @@ describe("web source-map collection", () => { "included", "fetch_failed", ]); - expect(result.items[1]?.limitation).toContain("262144-record"); + expect(result.items[1]?.limitation).toContain( + "source_map_limits.decoded_records is 262144", + ); }); it("rejects deeply nested indexed maps before recursive flattening", async () => { @@ -1554,7 +1556,9 @@ describe("web source-map collection", () => { status: "fetch_failed", artifact: null, }); - expect(result.items[0]?.limitation).toContain("64-level"); + expect(result.items[0]?.limitation).toContain( + "source_map_limits.section_depth is 64", + ); }); it("retains every mapping from a sectioned source map", async () => { @@ -1938,7 +1942,7 @@ describe("expanded source-map evidence budget", () => { original_sources: [], }); expect(result.items[0]?.limitation).toContain( - "Resolved source-map identities exceed the 64 MiB decoder representation budget", + "source_map_limits.fetch_bytes is 67108864", ); }); }); @@ -1962,7 +1966,9 @@ describe("raw source-map structure preflight", () => { status: "fetch_failed", artifact: null, }); - expect(result.items[0]?.limitation).toContain("262144-record"); + expect(result.items[0]?.limitation).toContain( + "source_map_limits.decoded_records is 262144", + ); }); }); diff --git a/tests/boundary/browser/webSourceMapResponseIdentity.test.ts b/tests/boundary/browser/webSourceMapResponseIdentity.test.ts index 33e2a79a2a..9314b866a5 100644 --- a/tests/boundary/browser/webSourceMapResponseIdentity.test.ts +++ b/tests/boundary/browser/webSourceMapResponseIdentity.test.ts @@ -149,7 +149,7 @@ it("retains completed body identity when map interpretation reaches its deadline }); it("refuses oversized text before JSON interpretation while retaining body identity and later maps", async () => { - // JSON escaping expands these decoded control characters beyond the fixed + // JSON escaping expands these decoded control characters beyond the default // 32 MiB source-map output allowance while the response stays below 64 MiB. const oversized = Buffer.alloc(6 * 1024 * 1024, 1); const oversizedText = oversized.toString("utf8"); @@ -196,8 +196,10 @@ it("refuses oversized text before JSON interpretation while retaining body ident const limitation = result.value.items[0]?.limitation; expect(limitation).toMatch(/JSON[- ]escap/i); expect(limitation).toMatch(/metadata/i); - expect(limitation).toMatch(/shared/i); - expect(limitation).toMatch(/32\s*MiB/i); + expect(limitation).toContain( + "source_map_limits.output_bytes is 33554432 bytes", + ); + expect(limitation).toContain("Increase that per-call field"); expect(Buffer.byteLength(JSON.stringify(result.value))).toBeLessThanOrEqual( 32 * 1024 * 1024, ); diff --git a/tests/boundary/captures/captureProvider.test.ts b/tests/boundary/captures/captureProvider.test.ts index 84903df4df..24f599e2d5 100644 --- a/tests/boundary/captures/captureProvider.test.ts +++ b/tests/boundary/captures/captureProvider.test.ts @@ -142,6 +142,63 @@ else { if(readFileSync('capture', 'utf8') !== 'recording bytes') throw Error('wr }; }; +it("recovers the same capture with CLI producer budgets and retains a tighter inherited file limit", async () => { + const f = await fixture(); + const refused = await f.service.inspect({ + path: f.path, + fields: ["ip.addr"], + producer_limits: { input_bytes: 13 }, + }); + expect(refused).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); + if (refused.ok) throw new Error("Expected input-budget refusal"); + expect(analysisErrorRemediationAction(refused.error)).toContain( + "producer_limits.input_bytes", + ); + const cli = Cli.create("capture-producer-budget", { sync: false }); + registerCaptureCommands(cli, silentLogger, {}, f.service); + let stdout = ""; + await cli.serve( + [ + "inspect-packet-capture", + f.path, + "--fields", + "ip.addr", + "--producer-limits", + JSON.stringify({ + input_bytes: 15, + output_bytes: 1024, + command_timeout_ms: 1000, + address_space_bytes: 2147483648, + file_bytes: 2097152, + cpu_seconds: 45, + }), + "--format", + "json", + ], + { + stdout: (text) => { + stdout += text; + }, + exit: () => {}, + }, + ); + const evidence = parseEvidence(JSON.parse(stdout)); + const report = packetCaptureSchema.parse(evidence.normalized_result); + expect(report.packets).toHaveLength(1); + expect(report.limits).toEqual({ + input_bytes: 15, + output_bytes: 1024, + command_timeout_ms: 1000, + address_space_bytes: 2147483648, + file_bytes: 1048576, + cpu_seconds: 45, + }); + expect(report.resource_limiter.arguments).toContain("--cpu=45:45"); +}); + it("refuses new acquisition after provider shutdown", async () => { const f = await fixture(); await f.service.close(); @@ -754,7 +811,7 @@ it.each([ retryable: false, }); expect(analysisErrorRemediationAction(result.error)).toContain( - "The retained command-output allowance", + "producer_limits.output_bytes", ); expect(analysisErrorRemediationAction(result.error)).not.toMatch( /Review the reported worker memory limits/u, diff --git a/tests/boundary/cli/apktoolBooleanOptions.test.ts b/tests/boundary/cli/apktoolBooleanOptions.test.ts index 311d863a8c..6cbe726b20 100644 --- a/tests/boundary/cli/apktoolBooleanOptions.test.ts +++ b/tests/boundary/cli/apktoolBooleanOptions.test.ts @@ -75,3 +75,38 @@ it("documents the supported negative flag in command help", async () => { expect(result.exitCode).toBe(0); expect(result.stdout).toContain("--no-strings"); }); + +it("preserves selected string pages and budgets and rejects an invalid page before decoding", async () => { + const result = await invokeApktoolCli([ + "decode-android-resources", + "/targets/app.apk", + "--string-offset", + "2000", + "--string-limit", + "100", + "--max-total-strings", + "200", + "--max-string-table-bytes", + "12000000", + "--max-total-string-table-bytes", + "24000000", + "--json", + ]); + expect(result.exitCode).toBe(0); + expect(result.requests[0]?.input).toMatchObject({ + string_offset: 2000, + string_limit: 100, + max_total_strings: 200, + max_string_table_bytes: 12000000, + max_total_string_table_bytes: 24000000, + }); + const refused = await invokeApktoolCli([ + "decode-android-resources", + "/targets/app.apk", + "--string-limit", + "0", + "--json", + ]); + expect(refused.exitCode).not.toBe(0); + expect(refused.requests).toEqual([]); +}); diff --git a/tests/boundary/cli/artifactInputLimits.test.ts b/tests/boundary/cli/artifactInputLimits.test.ts index 04259cf033..a3132ec968 100644 --- a/tests/boundary/cli/artifactInputLimits.test.ts +++ b/tests/boundary/cli/artifactInputLimits.test.ts @@ -9,12 +9,12 @@ for (const { command, limit, recovery } of [ { command: "inspect-go-binary", limit: 256 * 1024 * 1024, - recovery: "input byte allowance", + recovery: "max_input_bytes", }, { command: "inspect-rust-binary", limit: 256 * 1024 * 1024, - recovery: "within the fixed Rust input allowance", + recovery: "producer_limits.input_bytes", }, { command: "inspect-il2cpp-metadata", @@ -24,11 +24,11 @@ for (const { command, limit, recovery } of [ { command: "inspect-wasm-artifact", limit: 8 * 1024 * 1024, - recovery: "input byte allowance", + recovery: "max_input_bytes", }, ]) { cliTest.skipIf(process.platform === "win32")( - `${command} identifies fixed input-size refusal and recovery`, + `${command} identifies input-size refusal and applicable recovery`, async ({ cli }) => { const root = await createTestTempDirectory("rea-artifact-input-limit-"); const selected = join(root, "oversized.bin"); diff --git a/tests/boundary/cli/cliInputSchemaRequired.test.ts b/tests/boundary/cli/cliInputSchemaRequired.test.ts index 1018bbe157..d803bc06ce 100644 --- a/tests/boundary/cli/cliInputSchemaRequired.test.ts +++ b/tests/boundary/cli/cliInputSchemaRequired.test.ts @@ -165,10 +165,13 @@ describe("CLI advertised input schemas", () => { : "./selected", ), }; - expect(validate(input)).toBe(count === 16); + // WASM path count is admitted by the workflow's selected + // max_artifact_paths budget; JavaScript recovery retains its own cap. + const accepted = command === "inspect-wasm-artifact" || count === 16; + expect(validate(input)).toBe(accepted); expect( parserSchema(cli, command, "options")?.safeParse(input).success, - ).toBe(count === 16); + ).toBe(accepted); } } const scenario = await advertisedSchema(cli, "capture-native-ui-scenario"); diff --git a/tests/boundary/cli/desktopCaptureCli.test.ts b/tests/boundary/cli/desktopCaptureCli.test.ts index b1ab339d1e..6ecc61ee7e 100644 --- a/tests/boundary/cli/desktopCaptureCli.test.ts +++ b/tests/boundary/cli/desktopCaptureCli.test.ts @@ -1,4 +1,5 @@ -import { readFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; +import { execFileSync } from "node:child_process"; import { join } from "node:path"; import { expect, it } from "vitest"; @@ -6,6 +7,7 @@ import { expect, it } from "vitest"; import { EDITED_NOTE, HOME, + capture, change, retained, seedEndCapture, @@ -189,3 +191,138 @@ it.each([ }); }, ); + +it.each([ + { label: "omitted", policy: undefined }, + { label: "empty", policy: {} }, + { label: "selected", policy: { text_lines: 2 } }, +])( + "preserves %s desktop view policy separately from the effective policy", + async ({ policy }) => { + const folder = await writeCaptureFolder(seedEndCapture(), undefined, [ + EDITED_NOTE, + ]); + const selected = + policy === undefined ? [] : ["--view-policy", JSON.stringify(policy)]; + for (const command of [ + "inspect-desktop-application-state", + "describe-desktop-application-task", + ]) { + const observed = await serve([command, folder, ...selected, "--json"]); + const evidence = parseEvidence(JSON.parse(observed.stdout)); + if (policy === undefined) + expect(evidence.parameters).not.toHaveProperty("view_policy"); + else expect(evidence.parameters.view_policy).toEqual(policy); + expect(evidence.normalized_result).toMatchObject({ + effective_view_policy: { + text_lines: policy?.text_lines ?? 5000, + file_bytes: 67108864, + }, + }); + } + }, +); + +it("recovers a selected captured view and exports the same policy into the Python reader", async () => { + const path = `${HOME}/Documents/budget.txt`; + const text = "one\ntwo\nthree"; + const local = capture(); + const first = local.state.checkpoints[0]; + if (first === undefined) throw new Error("missing fixture checkpoint"); + const folder = await writeCaptureFolder( + { + ...local, + state: { + ...local.state, + checkpoints: [ + { ...first, retained: [...first.retained, retained(path, text)] }, + ...local.state.checkpoints.slice(1), + ], + }, + }, + undefined, + [text], + ); + const inspect = async (lines: number) => { + const response = await serve([ + "inspect-desktop-application-state", + folder, + "--state-path", + path, + "--view-policy", + JSON.stringify({ text_lines: lines }), + "--json", + ]); + expect(response.exitCode).toBe(0); + return parseEvidence(JSON.parse(response.stdout)); + }; + const limited = await inspect(2); + const recovered = await inspect(3); + expect(limited.normalized_result).toMatchObject({ + paths: [ + { + path, + versions: [{ view: { lines: ["one", "two"], truncated: true } }], + }, + ], + }); + expect(recovered.normalized_result).toMatchObject({ + paths: [ + { + path, + versions: [ + { + checkpoint_index: 0, + sha256: retained(path, text).sha256, + view: { lines: ["one", "two", "three"], truncated: false }, + }, + ], + }, + ], + }); + const root = await createTestTempDirectory("desktop-view-policy-"); + const bundle = join(root, "bundle"); + const exported = await serve([ + "export-desktop-application", + folder, + bundle, + "--view-policy", + '{"text_lines":3}', + "--json", + ]); + expect(exported.exitCode).toBe(0); + const module = await readFile( + join(bundle, "envs/notes/notes_app.py"), + "utf8", + ); + const helpers = module + .split("@computer_app(")[0] + ?.split("\n") + .filter((line) => !line.includes("from agentenv_protocol")) + .join("\n"); + const script = join(root, "reader.py"); + await writeFile( + script, + `${helpers}\nprint(json.dumps(_text_view("notes.txt", "one\\ntwo\\nthree", 13)["view"]))\n`, + ); + const actual = JSON.parse( + execFileSync( + process.platform === "win32" ? "python" : "python3", + ["-I", "-B", script], + { encoding: "utf8" }, + ), + ); + expect(actual).toEqual({ + lines: ["one", "two", "three"], + line_count: 3, + truncated: false, + }); + const invalid = await serve([ + "inspect-desktop-application-state", + folder, + "--view-policy", + '{"text_lines":0}', + "--json", + ]); + expect(invalid.exitCode).toBe(1); +}); diff --git a/tests/boundary/cli/environmentBuildCli.test.ts b/tests/boundary/cli/environmentBuildCli.test.ts index d192b0a661..3d3500938b 100644 --- a/tests/boundary/cli/environmentBuildCli.test.ts +++ b/tests/boundary/cli/environmentBuildCli.test.ts @@ -49,11 +49,12 @@ cliTest( async ({ cli }) => { const root = await createTestTempDirectory("rea-environment-build-cli-"); const har = join(root, "grafana.har"); + const selectedCapture = "./grafana.har"; await writeFile(har, JSON.stringify(environmentManifestHar())); const built = await cli.run({ arguments: [ "environment-build", - "grafana.har", + selectedCapture, join(root, "built"), "--json", ], @@ -73,7 +74,7 @@ cliTest( const inspected = await cli.run({ arguments: [ "inspect-web-network-capture", - "grafana.har", + selectedCapture, "har", "--format", "json", @@ -81,6 +82,10 @@ cliTest( cwd: root, }); expect(inspected.exitCode, inspected.stderr).toBe(0); + expect(inspected.json).toMatchObject({ + caller_input_path: selectedCapture, + parameters: { capture_path: har }, + }); const capture = join(root, "capture.json"); await writeFile(capture, inspected.stdout); const request = join(root, "view.json"); @@ -127,7 +132,7 @@ cliTest( "utf8", ), ) as { caller_input_path?: string }; - expect(published.caller_input_path).toBe("grafana.har"); + expect(published.caller_input_path).toBe(selectedCapture); expect(built.json).toMatchObject({ tool_count: (exported.json as { tool_count: number }).tool_count, files: (exported.json as { files: unknown[] }).files, diff --git a/tests/boundary/cli/flutterCli.test.ts b/tests/boundary/cli/flutterCli.test.ts index 33593345ac..bb9b0e99cd 100644 --- a/tests/boundary/cli/flutterCli.test.ts +++ b/tests/boundary/cli/flutterCli.test.ts @@ -106,17 +106,36 @@ it("inspects the caller-selected nonstandard ABI through the public CLI", async path, "--abi", "ARM64-V8A", + "--pool-policy", + '{"contains":"package:animations/","uri_limit":1}', "--json", ]); expect(selected.exitCode).toBe(0); expect(JSON.parse(selected.stdout)).toMatchObject({ caller_input_path: path, - parameters: { path, abi: "ARM64-V8A" }, + parameters: { + path, + abi: "ARM64-V8A", + pool_policy: { contains: "package:animations/", uri_limit: 1 }, + }, normalized_result: { abi: "ARM64-V8A", + string_pool: { + package_uris: ["package:animations/src/fade_scale_transition.dart"], + package_uri_count_at_least: 1, + policy: { uri_limit: 1, contains: "package:animations/" }, + }, libapp: { snapshot_hash: "a".repeat(32), coverage: "complete" }, }, }); + const invalid = await serveCli([ + "inspect-dart-aot", + path, + "--pool-policy", + '{"uri_limit":0}', + "--json", + ]); + expect(JSON.parse(invalid.stdout).code).toBe("VALIDATION_ERROR"); }); /** Serve one command whose first snapshot-root removal fails and is retried on close. */ diff --git a/tests/boundary/cli/managedMemberComparisonCli.test.ts b/tests/boundary/cli/managedMemberComparisonCli.test.ts index ee2ef1fa5a..5680390577 100644 --- a/tests/boundary/cli/managedMemberComparisonCli.test.ts +++ b/tests/boundary/cli/managedMemberComparisonCli.test.ts @@ -35,12 +35,26 @@ cliTest( await writeFile(bad, bytes); const healthy = await cli.run({ - arguments: ["compare-managed-members", good, good, "--json"], + arguments: [ + "compare-managed-members", + good, + good, + "--max-snapshot-bytes", + String(bytes.length), + "--max-metadata-representation-bytes", + "1048576", + "--json", + ], }); expect(healthy.exitCode, healthy.stderr).toBe(0); expect(healthy.json).toMatchObject({ operation: "compare_managed_members", - parameters: { left_path: good, right_path: good }, + parameters: { + left_path: good, + right_path: good, + max_snapshot_bytes: bytes.length, + max_metadata_representation_bytes: 1048576, + }, }); for (const field of ["leftPath", "rightPath"] as const) { diff --git a/tests/boundary/cli/reverseEngineeringCommands.test.ts b/tests/boundary/cli/reverseEngineeringCommands.test.ts index 3c3bc92374..653803ec30 100644 --- a/tests/boundary/cli/reverseEngineeringCommands.test.ts +++ b/tests/boundary/cli/reverseEngineeringCommands.test.ts @@ -55,6 +55,7 @@ const runObjdump = async ( operation?: string; symbol?: string; commandTimeoutMs?: number; + maxOutputBytes?: number; } = {}, ) => { const commands = new Map< @@ -66,6 +67,7 @@ const runObjdump = async ( followDebugLinks: boolean; symbol?: string; commandTimeoutMs?: number; + maxOutputBytes?: number; }; }): Promise; } @@ -80,6 +82,7 @@ const runObjdump = async ( followDebugLinks: boolean; symbol?: string; commandTimeoutMs?: number; + maxOutputBytes?: number; }; }): Promise; }, @@ -103,6 +106,9 @@ const runObjdump = async ( options: { followDebugLinks: false, ...(selection.symbol === undefined ? {} : { symbol: selection.symbol }), + ...(selection.maxOutputBytes === undefined + ? {} + : { maxOutputBytes: selection.maxOutputBytes }), ...(selection.commandTimeoutMs === undefined ? {} : { commandTimeoutMs: selection.commandTimeoutMs }), @@ -127,17 +133,21 @@ it("forwards the selected native symbol to the shared workflow", async () => { }); }); -it("forwards a selected objdump command deadline and omits the field otherwise", async () => { +it("forwards selected objdump resource budgets and preserves omission", async () => { const inspectWithObjdump = vi.fn(async (_input: unknown) => ok(observation("complete")), ); const service = { inspectWithObjdump, close: async () => undefined }; - await runObjdump(service, "/tmp/sample", { commandTimeoutMs: 5_000 }); + await runObjdump(service, "/tmp/sample", { + commandTimeoutMs: 5_000, + maxOutputBytes: 1234, + }); expect(inspectWithObjdump.mock.calls[0]?.[0]).toEqual({ path: "/tmp/sample", operation: "file_headers", follow_debug_links: false, command_timeout_ms: 5_000, + max_output_bytes: 1234, }); await runObjdump(service, "/tmp/sample"); expect(inspectWithObjdump.mock.calls[1]?.[0]).toEqual({ diff --git a/tests/boundary/evm/evmoleInterfaceProvider.test.ts b/tests/boundary/evm/evmoleInterfaceProvider.test.ts index 1e98c38490..b87749fc79 100644 --- a/tests/boundary/evm/evmoleInterfaceProvider.test.ts +++ b/tests/boundary/evm/evmoleInterfaceProvider.test.ts @@ -23,6 +23,14 @@ const requestSchema = z.strictObject({ reply_path: z.string(), failure_marker_path: z.string(), encoding: z.enum(["raw", "hex"]), + producer_limits: z.object({ + input_bytes: z.number(), + output_bytes: z.number(), + diagnostic_bytes: z.number(), + address_space_bytes: z.number(), + file_bytes: z.number(), + cpu_seconds: z.number(), + }), }); const fixture = async () => { const workspace = await createTestWorkspace("rea-evm-interface-boundary-"); diff --git a/tests/boundary/evm/solidityMetadataWorker.test.ts b/tests/boundary/evm/solidityMetadataWorker.test.ts index 6b665ec0e3..27d654fd85 100644 --- a/tests/boundary/evm/solidityMetadataWorker.test.ts +++ b/tests/boundary/evm/solidityMetadataWorker.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { expect, it } from "vitest"; +import { selectedEvmProducerLimits } from "../../../src/domain/evm/evmInterface.js"; import { solidityMetadataSchema } from "../../../src/domain/evm/solidityMetadata.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -47,6 +48,7 @@ it.each([ reply_path: reply, failure_marker_path: join(directory, "resource.failure"), encoding: "hex", + producer_limits: selectedEvmProducerLimits(undefined), }), ); await promisify(execFile)( diff --git a/tests/boundary/filesystem/asarListingAdmission.test.ts b/tests/boundary/filesystem/asarListingAdmission.test.ts index 2705cac904..85fc33f538 100644 --- a/tests/boundary/filesystem/asarListingAdmission.test.ts +++ b/tests/boundary/filesystem/asarListingAdmission.test.ts @@ -7,6 +7,7 @@ import { expect, it } from "vitest"; import { AsarArtifactReader } from "../../../src/artifacts/AsarArtifactReader.js"; import { readAsarListing } from "../../../src/artifacts/AsarListing.js"; +import { resolveAsarHeaderAdmission } from "../../../src/artifacts/AsarHeader.js"; import { asarHeaderBytes, nestedAsarHeader, @@ -48,11 +49,42 @@ it("rejects path expansion before publishing any inventory, preserving the input const path = await archiveWith(bytes); await expect(inventoryArtifact(path)).rejects.toMatchObject({ reason: "limit", - message: "ASAR expanded listing exceeds its 64 MiB representation budget", + admission: { + kind: "asar-listing", + maximumBytes: 67108864, + source: "default", + }, }); expect(Buffer.compare(await readFile(path), bytes)).toBe(0); }); +it("recovers expanded listing admission independently of the unchanged header budget", async () => { + const bytes = asarHeaderBytes( + JSON.stringify({ files: { member: { size: 0, offset: "0" } } }), + ); + const path = await archiveWith(bytes); + const list = async (maximum: string) => { + const reader = new AsarArtifactReader(path, { + headerAdmission: resolveAsarHeaderAdmission({ + REA_ASAR_MAX_LISTING_BYTES: maximum, + }), + }); + try { + const paths: string[] = []; + for await (const entry of reader.entries()) paths.push(entry.path); + return paths; + } finally { + await reader.close(); + } + }; + await expect(list("128")).rejects.toMatchObject({ + reason: "limit", + admission: { kind: "asar-listing", maximumBytes: 128, source: "setting" }, + }); + expect(await list("1024")).toEqual(["member"]); + expect((await readFile(path)).equals(bytes)).toBe(true); +}); + it("reads deeply nested packed metadata without recursive listing or member lookup", async () => { const path = await archiveWith(nestedAsarHeader(3500, "d", 1)); const reader = new AsarArtifactReader(path); @@ -610,6 +642,10 @@ it("admits escaped integrity source retention before formatting its limitation", }), ).rejects.toMatchObject({ reason: "limit", - message: "ASAR expanded listing exceeds its 64 MiB representation budget", + admission: { + kind: "asar-listing", + maximumBytes: 67108864, + source: "default", + }, }); }); diff --git a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts index a9fab315d0..7c16297773 100644 --- a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts +++ b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts @@ -998,13 +998,13 @@ describe("direct analysis composed snapshot replay", () => { } }); -describe("Ghidra composed workflow snapshot replay", () => { +describe("Ghidra composed workflow runtime-budget admission", () => { it.each([ { tool: "binary_overview", arguments: {} }, { tool: "inspect_native_api", arguments: { procedure: "0x1000" } }, { tool: "trace_feature", arguments: { query: "fixture" } }, ] as const)( - "persists $tool with production Ghidra capability policies", + "recomputes $tool with production Ghidra capability policies", async ({ tool, arguments: parameters }) => { const fixture = await createSnapshotReplayFixture("rea-ghidra-workflow-"); const profile = createAnalysisProfile( @@ -1029,9 +1029,9 @@ describe("Ghidra composed workflow snapshot replay", () => { expect(first).toMatchObject({ operation: tool }); const snapshot = await readAnalysisSnapshot(fixture.snapshotPath); if (!snapshot.ok) throw snapshot.error; - expect(snapshot.value.workflow_entries).toEqual([ - expect.objectContaining({ operation: tool }), - ]); + // Ghidra dossiers are live: a saved composed answer must not bypass + // admission under the runtime budgets selected for the next invocation. + expect(snapshot.value.workflow_entries).toEqual([]); const initialCalls = [...fixture.calls]; const replay = await runDirectAnalysis( @@ -1041,14 +1041,32 @@ describe("Ghidra composed workflow snapshot replay", () => { parameters, { snapshotPath: fixture.snapshotPath }, ).then(projectCliAnalysisResult); - expect(replay).toEqual(first); - if (process.platform === "win32") { + expect(parseEvidence(replay).normalized_result).toEqual( + parseEvidence(first).normalized_result, + ); + if (tool !== "binary_overview") { + expect(fixture.starts).toHaveLength(2); + expect(fixture.calls.length).toBeGreaterThan(initialCalls.length); + if (tool === "inspect_native_api") + expect( + fixture.calls.filter( + (operation) => operation === "analyze_function", + ), + ).toHaveLength(2); + } else if (process.platform === "win32") { // The first provider call in a new CLI session admits its source. expect(fixture.starts).toHaveLength(2); expect(fixture.calls.length).toBeGreaterThan(initialCalls.length); } else { expect(fixture.starts).toHaveLength(1); expect(fixture.calls).toEqual(initialCalls); + expect(parseEvidence(replay).limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining( + "Loaded from a local REA analysis snapshot", + ), + ]), + ); } }, ); diff --git a/tests/boundary/filesystem/dotnetBundleArtifacts.test.ts b/tests/boundary/filesystem/dotnetBundleArtifacts.test.ts index 6f3740a319..e5e8d8ff7f 100644 --- a/tests/boundary/filesystem/dotnetBundleArtifacts.test.ts +++ b/tests/boundary/filesystem/dotnetBundleArtifacts.test.ts @@ -5,6 +5,7 @@ import { deflateRawSync } from "node:zlib"; import { expect, it, vi } from "vitest"; +import { DEFAULT_DOTNET_BUNDLE_READER_LIMITS } from "../../../src/domain/artifactReaderLimits.js"; import { artifactFixedLimitFailure } from "../../../src/artifacts/ArtifactReader.js"; import { DotnetBundleArtifactReader } from "../../../src/artifacts/DotnetBundleArtifactReader.js"; import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; @@ -24,64 +25,100 @@ import { import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; it.each([ - { - name: ".NET bundle entry count", - maximum: 100_000, - unit: "entries", - kind: "count", - }, - { - name: ".NET bundle decoded string", - maximum: 65_536, - unit: "UTF-8 bytes", - kind: "string", - }, - { - name: ".NET bundle member stored or expanded size", - maximum: 536_870_912, - unit: "bytes", - kind: "member", - }, - { - name: ".NET bundle exact integer range", - maximum: 9_007_199_254_740_991, - unit: "integer units", - kind: "integer", - }, -])( - "reports the effective fixed $name allowance", - async ({ name, maximum, unit, kind }) => { - const root = await createTestTempDirectory("rea-dotnet-fixed-allowance-"); + { field: "entry_count", maximum: 1, required: 2 }, + { field: "string_bytes", maximum: 13, required: 14 }, + { field: "member_bytes", maximum: 7, required: 8 }, + { field: "expanded_bytes", maximum: 15, required: 16 }, + { field: "path_parts", maximum: 1, required: 2 }, + { field: "path_nodes", maximum: 2, required: 3 }, + { field: "metadata_bytes", maximum: null, required: null }, +] as const)( + "recovers the same complete bundle after raising $field", + async ({ field, maximum, required }) => { + const root = await createTestTempDirectory("rea-dotnet-reader-admission-"); const path = join(root, "host.exe"); const fixture = dotnetBundleFixture([ - { - path: kind === "string" ? "a".repeat(maximum + 1) : "file", - bytes: Buffer.from("x"), - ...(kind === "member" || kind === "integer" - ? { declaredSize: maximum + 1 } - : {}), - }, + { path: "dir/first.txt", bytes: Buffer.from("aaaaaaaa") }, + { path: "dir/second.txt", bytes: Buffer.from("bbbbbbbb") }, ]); - if (kind === "count") - fixture.bytes.writeUInt32LE(maximum + 1, fixture.fields.entryCount); + const needed = required ?? fixture.bytes.length - fixture.headerOffset; + const selected = maximum ?? needed - 1; await writeFile(path, fixture.bytes); const handle = await open(path, "r"); try { - const expected = artifactFixedLimitFailure(name, maximum, unit); await expect( - readDotnetBundle(handle, fixture.bytes.length, fixture.headerOffset), + readDotnetBundle( + handle, + fixture.bytes.length, + fixture.headerOffset, + undefined, + { + ...DEFAULT_DOTNET_BUNDLE_READER_LIMITS, + [field]: selected, + }, + ), ).rejects.toMatchObject({ reason: "limit", boundary: "capacity", - message: expected.message, - userMessage: expected.message, + readerLimit: { + parameter: `artifact_limits.dotnet_bundle.${field}`, + effective_value: selected, + required_value: needed, + applies_at: "next_operation", + agent_editable: true, + rea_restart_required: false, + }, }); + const recovered = await readDotnetBundle( + handle, + fixture.bytes.length, + fixture.headerOffset, + undefined, + { ...DEFAULT_DOTNET_BUNDLE_READER_LIMITS, [field]: needed }, + ); + expect( + recovered.files.map((file) => ({ path: file.path, size: file.size })), + ).toEqual([ + { path: "dir/first.txt", size: 8 }, + { path: "dir/second.txt", size: 8 }, + ]); + expect(sha(await readFile(path))).toBe(sha(fixture.bytes)); } finally { await handle.close(); } }, ); +it("retains the exact integer representation ceiling independently of reader budgets", async () => { + const root = await createTestTempDirectory("rea-dotnet-integer-allowance-"); + const path = join(root, "host.exe"); + const fixture = dotnetBundleFixture([ + { + path: "file", + bytes: Buffer.from("x"), + declaredSize: 9_007_199_254_740_992, + }, + ]); + await writeFile(path, fixture.bytes); + const handle = await open(path, "r"); + try { + const expected = artifactFixedLimitFailure( + ".NET bundle exact integer range", + 9_007_199_254_740_991, + "integer units", + ); + await expect( + readDotnetBundle(handle, fixture.bytes.length, fixture.headerOffset), + ).rejects.toMatchObject({ + reason: "limit", + boundary: "capacity", + message: expected.message, + }); + } finally { + await handle.close(); + } +}); + it.each(["pyinstaller", "dotnet-bundle"] as const)( "keeps the %s outer container when a stored member is another executable container", async (format) => { diff --git a/tests/boundary/filesystem/evidenceBundleFiles.test.ts b/tests/boundary/filesystem/evidenceBundleFiles.test.ts index 271eefbe7e..afbc43315d 100644 --- a/tests/boundary/filesystem/evidenceBundleFiles.test.ts +++ b/tests/boundary/filesystem/evidenceBundleFiles.test.ts @@ -297,19 +297,22 @@ it("retains a malformed artifact parent and refuses publication through both del evidenceOutput: join(root, "cli.json"), }, ); - // Preserve every diagnostic from the owner; CLI delivery only adds closure. + // The one-shot CLI also explains that malformed embedded sources could not + // be collected before exit; the application still returns its typed cause. expect(cli).toMatchObject(projected); expect(cli).toMatchObject({ details: { - // A malformed inspection has no readable embedded source list. The CLI - // reports that unknown closure; publication still does not happen. + partial_observation: evidence, source_closure: { publication: "not_published", complete: false, + collection_failure: "Error", records: [], omitted_count: 0, omitted: [], - collection_failure: expect.any(String), + recovery: expect.stringContaining( + "this invocation retains nothing after exit", + ), }, }, }); diff --git a/tests/boundary/filesystem/flutterBuildProvider.test.ts b/tests/boundary/filesystem/flutterBuildProvider.test.ts index ceb4a45d05..1abea73cad 100644 --- a/tests/boundary/filesystem/flutterBuildProvider.test.ts +++ b/tests/boundary/filesystem/flutterBuildProvider.test.ts @@ -1987,3 +1987,94 @@ it("preserves a completed analysis failure through refused cleanup and late canc projectAnalysisError(control.error), ); }); + +it("recovers omitted Dart pool strings through selection and per-call admission without changing source identity", async () => { + const packages = Array.from( + { length: 1001 }, + (_, index) => `package:many/file${String(index).padStart(4, "0")}.dart`, + ); + const tail = "package:tail/omitted.dart"; + const longUri = `package:long/${"x".repeat(1100)}.dart`; + const longToken = `LongToken${"A".repeat(130)}`; + const tokens = Array.from( + { length: 2001 }, + (_, index) => `Class${String(index).padStart(4, "0")}`, + ); + const isolate = Buffer.concat([ + snapshotSection(REFERENCE_HASH), + Buffer.from( + `\0${[...packages, tail, longUri, ...tokens, longToken].join("\0")}\0`, + ), + ]); + const image = Buffer.alloc(0x2000 + isolate.length + 16); + buildElf64Image([ + { name: "_kDartVmSnapshotData", value: 0x1000, size: 52 }, + { name: "_kDartVmSnapshotInstructions", value: 0x1800, size: 16 }, + { name: "_kDartIsolateSnapshotData", value: 0x2000, size: isolate.length }, + { + name: "_kDartIsolateSnapshotInstructions", + value: 0x2000 + isolate.length, + size: 16, + }, + ]).copy(image); + snapshotSection(REFERENCE_HASH).copy(image, 0x1000); + isolate.copy(image, 0x2000); + const path = await buildApk("pool-headroom.apk", [ + { path: "lib/arm64-v8a/libapp.so", bytes: image }, + ]); + const provider = new FlutterBuildProvider(); + onTestFinished(() => provider.close()); + const service = new FlutterBuildAnalysisService(provider); + const inspect = async (pool_policy?: Record) => { + const result = await service.execute("inspect_dart_aot", { + path, + ...(pool_policy === undefined ? {} : { pool_policy }), + }); + if (!result.ok) throw result.error; + return { + evidence: result.value, + report: flutterResultSchemas.inspect_dart_aot.parse( + result.value.normalized_result, + ), + }; + }; + const limited = await inspect(); + expect(limited.report.string_pool.package_uris).toHaveLength(500); + expect(limited.report.string_pool.class_like_tokens).toHaveLength(2000); + expect(limited.report.string_pool.package_uris).not.toContain(tail); + expect(limited.report.string_pool.package_uris).not.toContain(longUri); + expect(limited.report.string_pool.class_like_tokens).not.toContain(longToken); + expect(limited.report.string_pool.coverage).toBe("partial"); + expect( + limited.evidence.limitations.some((text) => + text.includes("pool_policy.run_bytes=1024"), + ), + ).toBe(true); + const selected = await inspect({ contains: "package:tail/" }); + expect(selected.report.string_pool.package_uris).toEqual([tail]); + expect(selected.report.string_pool.package_uri_count_at_least).toBe(1); + const recovered = await inspect({ + uri_limit: 1003, + token_limit: 2002, + run_bytes: 2048, + token_bytes: 256, + }); + expect(recovered.report.string_pool.package_uris).toHaveLength(1003); + expect(recovered.report.string_pool.package_uris).toContain(tail); + expect(recovered.report.string_pool.package_uris).toContain(longUri); + expect(recovered.report.string_pool.class_like_tokens).toContain(longToken); + expect(recovered.report.string_pool.coverage).toBe("complete"); + expect(selected.report.target).toEqual(limited.report.target); + expect(recovered.report.libapp.sha256).toEqual(limited.report.libapp.sha256); + expect(recovered.evidence.parameters.pool_policy).toEqual({ + uri_limit: 1003, + token_limit: 2002, + run_bytes: 2048, + token_bytes: 256, + }); + const rejected = await service.execute("inspect_dart_aot", { + path, + pool_policy: { uri_limit: 0 }, + }); + expect(rejected.ok).toBe(false); +}); diff --git a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts index f30e3c724b..21f47ab6dc 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts @@ -1268,8 +1268,12 @@ it("reports semantic value resource limits in application graph coverage", async }), ]), ); - expect(graph.limitations).toContain( - "Primitive candidate budget exceeded (maximum 256 alternatives).", + expect(graph.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining( + "Primitive candidate budget exceeded (maximum 256 alternatives).", + ), + ]), ); expect(graph.limitations).toEqual( expect.arrayContaining([ diff --git a/tests/boundary/go/goBinaryImage.test.ts b/tests/boundary/go/goBinaryImage.test.ts index 07805f1c4a..3373eedd92 100644 --- a/tests/boundary/go/goBinaryImage.test.ts +++ b/tests/boundary/go/goBinaryImage.test.ts @@ -66,7 +66,9 @@ it("preserves exact Mach-O protection masks for Go fallback selection", () => { it("bounds cumulative header searches across overlapping ELF file spans", () => { const bytes = createGoOverlappingElfFixture(); expect(() => readGoBinaryImage(bytes)).toThrow(BinaryResourceFailure); - expect(() => readGoBinaryImage(bytes)).toThrow(/256 MiB.*search/); + expect(() => readGoBinaryImage(bytes)).toThrow( + /max_header_search_bytes=268435456/, + ); }); it("does not charge duplicate ELF search spans repeatedly", () => { diff --git a/tests/boundary/go/goBinaryProvider.test.ts b/tests/boundary/go/goBinaryProvider.test.ts index 89266cfe72..2403308689 100644 --- a/tests/boundary/go/goBinaryProvider.test.ts +++ b/tests/boundary/go/goBinaryProvider.test.ts @@ -1,6 +1,7 @@ import { writeFile, symlink } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; +import { GoBinaryService } from "../../../src/application/go/GoBinaryService.js"; import { GoBinaryProvider } from "../../../src/go/GoBinaryProvider.js"; import { AnalysisInputError, @@ -70,6 +71,8 @@ it("preserves the actual build-info byte guard as a resource failure", async () expect(result.error.reportedLimits).toEqual({ boundary: "build-info", maximum_bytes: 1024 * 1024, + parameter: "max_build_info_bytes", + unit: "bytes", }); const projected = projectAnalysisError(result.error); expect(projected.retryable).toBe(false); @@ -225,6 +228,7 @@ it("adjusts string_literals file offsets inside a universal Mach-O slice and rec aggregate_bytes: 26, max_items: 2, truncated: true, + limits_reached: ["max_string_literals"], items: [ { value: "ListResourceProviders", @@ -254,3 +258,87 @@ it("adjusts string_literals file offsets inside a universal Mach-O slice and rec expect(limitationsText).toContain("LC_DYLD_CHAINED_FIXUPS"); expect(limitationsText).toContain("truncated"); }); + +it.each([ + ["max_input_bytes", 4095, 8192], + ["max_report_bytes", 1, 16384], + ["max_build_info_bytes", 1, 4096], + ["max_header_search_bytes", 1, 4096], +] as const)( + "recovers the same Go artifact after increasing %s", + async (parameter, low, raised) => { + const root = await createTestTempDirectory("rea-go-budget-recovery-"); + const path = join(root, "application.elf"); + await writeFile(path, createGoBinaryFixture().bytes); + const service = new GoBinaryService(new GoBinaryProvider()); + try { + const refused = await service.inspect({ path, [parameter]: low }); + if (refused.ok) + throw new Error( + "Selected low budget must refuse this complete artifact", + ); + expect(projectAnalysisError(refused.error)).toMatchObject({ + code: "resource_constraint", + details: { reported_limits: { parameter, unit: "bytes" } }, + remediation: { action: expect.stringContaining(parameter) }, + }); + const recovered = await service.inspect({ path, [parameter]: raised }); + if (!recovered.ok) throw recovered.error; + expect(recovered.value.normalized_result).toMatchObject({ + artifact: { path, bytes: 4096 }, + build_info: { go_version: "go1.26.0" }, + }); + expect(recovered.value.parameters).toEqual({ path, [parameter]: raised }); + } finally { + await service.close(); + } + }, +); + +it.each([ + ["max_string_literals", 1], + ["max_string_literal_bytes", 3], + ["max_string_bytes", 5], + ["max_string_header_scan_bytes", 16], + ["max_string_candidates", 1], +] as const)( + "identifies and recovers the exact inferred-string limit %s", + async (parameter, low) => { + const fixture = createGoRodataStringFixture({ + format: "elf", + rodata: Buffer.from("alphabeta"), + headers: [ + { section: "data", offsetInSection: 0, ptr: 0x102000n, len: 5n }, + { section: "data", offsetInSection: 16, ptr: 0x102005n, len: 4n }, + ], + }); + const root = await createTestTempDirectory( + "rea-go-string-budget-recovery-", + ); + const path = join(root, "strings.elf"); + await writeFile(path, fixture.bytes); + const service = new GoBinaryService(new GoBinaryProvider()); + try { + const selected = { path, include_string_literals: true }; + const limited = await service.inspect({ ...selected, [parameter]: low }); + if (!limited.ok) throw limited.error; + expect(limited.value.normalized_result).toMatchObject({ + string_literals: { truncated: true, limits_reached: [parameter] }, + }); + const recovered = await service.inspect({ + ...selected, + [parameter]: 1048576, + }); + if (!recovered.ok) throw recovered.error; + expect(recovered.value.normalized_result).toMatchObject({ + string_literals: { + truncated: false, + items: [{ value: "alpha" }, { value: "beta" }], + }, + }); + expect(recovered.value.subject).toEqual(limited.value.subject); + } finally { + await service.close(); + } + }, +); diff --git a/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts b/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts index b27760bac2..2227bbf703 100644 --- a/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts +++ b/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts @@ -110,6 +110,61 @@ it("reuses one real owned analyzer while retaining distinct source identities an } }); +it("recovers finite primitive alternatives after changing precision on the same scope", async () => { + const scope = new ArtifactResourceScope(); + const expression = Array.from( + { length: 9 }, + (_, index) => + `(globalThis.choice${String(index)} ? 0 : ${String(2 ** index)})`, + ).join(" + "); + const file = source( + "precision.mjs", + `export function choose() { return ${expression}; }`, + ); + try { + for (const selected of [undefined, 512, undefined]) { + const worker = await createJavaScriptSourceAnalysis( + { + heapMb: 128, + timeoutMs: 30_000, + ...(selected === undefined + ? {} + : { semanticPrecision: { primitive_candidates: selected } }), + }, + 1, + { scope }, + ); + try { + const result = await worker.analyze(file, { + nodes: 100, + unknowns: 100, + }); + if (!result.ok) throw result.error.error; + const value = result.value.module?.callables[0]?.returnSites[0]?.value; + if (selected === undefined) { + expect(value).toMatchObject({ + status: "unknown", + resourceLimit: "primitive-candidates", + }); + } else { + expect(value?.status).toBe("union"); + if (value?.status !== "union") + throw new Error("Expected all finite alternatives"); + expect(value.values).toHaveLength(512); + expect(value.values).toContain(511); + expect(result.value.module?.coverage.precision).toMatchObject({ + primitive_candidates: 512, + }); + } + } finally { + await worker.close(); + } + } + } finally { + await scope.close(); + } +}); + it("reuses one owned worker across analyses on the same resource scope", async () => { const scope = new ArtifactResourceScope(); const launches: number[] = []; diff --git a/tests/boundary/javascript/sourceMapDecoder.test.ts b/tests/boundary/javascript/sourceMapDecoder.test.ts index 0eedf9723d..88d052691f 100644 --- a/tests/boundary/javascript/sourceMapDecoder.test.ts +++ b/tests/boundary/javascript/sourceMapDecoder.test.ts @@ -5,10 +5,7 @@ import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { webSourceMapReportSchema } from "../../../src/domain/webSourceLocation.js"; import { SourceMapDecoder } from "../../../src/javascript/sourceMaps/SourceMapDecoder.js"; -import { - SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, - type SourceMapDecoderDependencies, -} from "../../../src/javascript/sourceMaps/SourceMapCommand.js"; +import type { SourceMapDecoderDependencies } from "../../../src/javascript/sourceMaps/SourceMapCommand.js"; import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js"; @@ -17,6 +14,11 @@ import { webSourceLocationFixture, } from "../../fixtures/webSourceLocation.js"; +const expectedProcessMemoryLimitBytes = + process.platform === "win32" && process.versions.bun !== undefined + ? 704 * 1024 * 1024 + : null; + const realDecoder = (dependencies: SourceMapDecoderDependencies = {}) => new SourceMapDecoder({ ...dependencies, @@ -73,49 +75,64 @@ it("starts the codec deadline after cold ownership preparation", async () => { } }); -it("runs the real pinned codec with an independent heap and removes its private request directory", async () => { - let path: string | undefined; - const environment = { - ...process.env, - NODE_OPTIONS: "--require=/rea-file-that-does-not-exist", - }; - const decoder = realDecoder({ - environment, - createRuntime: async () => { - const root = await PrivateRuntimeRoot.create(); - path = root.path; - return root; - }, - }); - const response = await decoder.trace(input); - if (!response.ok) throw response.error; - expect(response.value).toMatchObject({ - engine: { id: "jridgewell-trace-mapping", version: "0.3.31" }, - matches: [{ content: { text: "original" } }], - }); - if (process.versions.bun === undefined) { - expect(response.value.runtime).toMatchObject({ - id: "node-v8", - version: process.version, +it.each([ + { heap: undefined, effectiveHeap: 192 }, + { heap: 384, effectiveHeap: 384 }, +])( + "runs the real pinned codec with selected heap $heap and removes its private request directory", + async ({ heap, effectiveHeap }) => { + let path: string | undefined; + const environment = { + ...process.env, + NODE_OPTIONS: "--require=/rea-file-that-does-not-exist", + }; + const decoder = realDecoder({ + environment, + createRuntime: async () => { + const root = await PrivateRuntimeRoot.create(); + path = root.path; + return root; + }, }); - expect(response.value.runtime.v8_heap_limit_bytes).toBeLessThanOrEqual( - 256 * 1024 * 1024, + const response = await decoder.trace({ + ...input, + ...(heap === undefined ? {} : { limits: { codec_heap_mib: heap } }), + }); + if (!response.ok) throw response.error; + expect(response.value).toMatchObject({ + engine: { id: "jridgewell-trace-mapping", version: "0.3.31" }, + matches: [{ content: { text: "original" } }], + }); + if (process.versions.bun === undefined) { + expect(response.value.runtime).toMatchObject({ + id: "node-v8", + version: process.version, + }); + expect(response.value.runtime.v8_heap_limit_bytes).toBeGreaterThan( + (effectiveHeap - 1) * 1024 * 1024, + ); + expect(response.value.runtime.v8_heap_limit_bytes).toBeLessThanOrEqual( + (effectiveHeap + 64) * 1024 * 1024, + ); + } else + expect(response.value.runtime).toEqual({ + id: "bun", + version: process.versions.bun, + node_compatibility_version: process.versions.node, + v8_heap_limit_bytes: null, + // Windows enforces a Job Object commit limit in place of the heap flags. + process_memory_limit_bytes: + process.platform === "win32" + ? (effectiveHeap + 512) * 1024 * 1024 + : null, + }); + expect(environment.NODE_OPTIONS).toBe( + "--require=/rea-file-that-does-not-exist", ); - } else - expect(response.value.runtime).toEqual({ - id: "bun", - version: process.versions.bun, - node_compatibility_version: process.versions.node, - v8_heap_limit_bytes: null, - // Windows enforces a Job Object commit limit in place of the heap flags. - process_memory_limit_bytes: SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, - }); - expect(environment.NODE_OPTIONS).toBe( - "--require=/rea-file-that-does-not-exist", - ); - if (path === undefined) throw new Error("runtime not acquired"); - await expect(access(path)).rejects.toMatchObject({ code: "ENOENT" }); -}); + if (path === undefined) throw new Error("runtime not acquired"); + await expect(access(path)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); it.each([ { cancel: false, denyCleanup: false }, @@ -451,7 +468,7 @@ it("rejects a codec report from a runtime other than the launching one", async ( runtime: { ...actual.value.runtime, process_memory_limit_bytes: - SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES === null ? 4096 : null, + expectedProcessMemoryLimitBytes === null ? 4096 : null, }, }); if (limit.ok) throw new Error("expected process limit mismatch"); @@ -463,7 +480,7 @@ it.each([true, false])( const response = await realDecoder({ launcher: async ({ memoryLimitBytes, ...options }) => { expect(memoryLimitBytes).toBe( - SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES ?? undefined, + expectedProcessMemoryLimitBytes ?? undefined, ); // Without a job limit any artifact runs the child; its handle reports // the notification an owned Windows job records before exit. @@ -492,8 +509,7 @@ it.each([true, false])( resource: "memory", reason: expect.stringContaining("process memory limit"), reportedLimits: { - codec_process_memory_limit_bytes: - SOURCE_MAP_CODEC_MEMORY_LIMIT_BYTES, + codec_process_memory_limit_bytes: expectedProcessMemoryLimitBytes, }, capturedOutput: { stderr: "panic: out of memory" }, } diff --git a/tests/boundary/mcp/dartAotSummaryMcp.test.ts b/tests/boundary/mcp/dartAotSummaryMcp.test.ts index 0d4cf5af4e..9794888856 100644 --- a/tests/boundary/mcp/dartAotSummaryMcp.test.ts +++ b/tests/boundary/mcp/dartAotSummaryMcp.test.ts @@ -26,17 +26,61 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; const dynamicSegmentOffset = (image: Buffer): number => Number(image.readBigUInt64LE(64 + 56 + 8)); -it("honors explicit ABI and old full requests, validates SDK schemas and reuses complete Evidence after APK removal", async () => { +/** Two exact ABI sources shared by full/retained and selected-summary workflows. */ +const aotViewFixture = async () => { const path = await createFlutterApk("views.apk", [ { path: "lib/arm64-v8a/libapp.so", bytes: aotFixtureImage() }, { path: "lib/x86_64/libapp.so", bytes: aotFixtureImage("b".repeat(32)) }, ]); const provider = createFlutterBuildAnalysisProvider(); - const execute = vi.spyOn(provider, "execute"); const harness = await createApplicationMcpHarness({ serverOptions: { flutterAnalysis: provider }, }); onTestFinished(harness.close); + return { path, provider, harness }; +}; + +it("retains the selected pool policy through MCP summary projection", async () => { + const { path, harness } = await aotViewFixture(); + const { client, session } = harness; + const definition = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_dart_aot", + ); + if (definition?.outputSchema === undefined) + throw new Error("Missing AOT output schema"); + const outputSchema: Record = definition.outputSchema; + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const filteredReply = await client.callTool({ + name: "inspect_dart_aot", + arguments: { + path, + detail: "summary", + pool_policy: { contains: "package:animations/", uri_limit: 1 }, + }, + }); + expect(filteredReply.isError).not.toBe(true); + expect( + ajv.validate(outputSchema, filteredReply.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const filteredView = dartAotSummaryViewResultSchema.parse( + parseEvidence(filteredReply.structuredContent).normalized_result, + ); + expect(filteredView.summary.string_pool.policy).toMatchObject({ + contains: "package:animations/", + uri_limit: 1, + }); + expect(filteredView.summary.string_pool.returned_counts.package_uris).toBe(1); + const filteredParent = session.evidenceById(filteredView.parent_evidence_id); + expect(filteredParent?.parameters.pool_policy).toEqual({ + contains: "package:animations/", + uri_limit: 1, + }); +}); + +it("honors explicit ABI and old full requests, validates SDK schemas and reuses complete Evidence after APK removal", async () => { + const { path, provider, harness } = await aotViewFixture(); + const execute = vi.spyOn(provider, "execute"); const { client, session } = harness; const catalog = (await client.listTools()).tools; const definition = catalog.find((tool) => tool.name === "inspect_dart_aot"); diff --git a/tests/boundary/mcp/fridaLifecycleMcp.test.ts b/tests/boundary/mcp/fridaLifecycleMcp.test.ts index 3c9d6573b2..572c32118b 100644 --- a/tests/boundary/mcp/fridaLifecycleMcp.test.ts +++ b/tests/boundary/mcp/fridaLifecycleMcp.test.ts @@ -30,7 +30,7 @@ import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js import { EvidenceIntegrityError } from "../../../src/domain/evidenceErrors.js"; import { evidenceBundleSchema } from "../../../src/domain/evidenceBundle.js"; import { fridaSessionStatusSchema } from "../../../src/domain/fridaSessionStatus.js"; -import type { Evidence } from "../../../src/domain/evidence.js"; +import { parseEvidence, type Evidence } from "../../../src/domain/evidence.js"; import { jsonObjectSchema } from "../../../src/domain/jsonValue.js"; import { nativeCallPartialObservationSchema } from "../../../src/domain/native/nativeCallPartialObservation.js"; import { err, ok } from "../../../src/domain/result.js"; @@ -257,6 +257,59 @@ const connectedFixture = async () => { return { ...native, manager, session, server, client, start, load }; }; +it.each(["start_frida_session", "instrument_with_frida"] as const)( + "selects capture retention through the public %s contract before observation", + async (operation) => { + const fixture = await connectedFixture(); + const description = "d".repeat(40_000); + fixture.controls.message = { + type: "send", + payload: "observed", + description, + }; + const result = await fixture.client.callTool({ + name: operation, + arguments: { + mode: "attach", + pid: 512, + capture_limits: { message_text_bytes: 50_000 }, + ...(operation === "instrument_with_frida" + ? { + source_kind: "inline", + source: "send('observed')", + duration_ms: 0, + } + : {}), + }, + }); + expect(result.isError).not.toBe(true); + let observed = result; + if (operation === "start_frida_session") { + const started = toolContract("start_frida_session").outputSchema.parse( + result.structuredContent, + ); + expect(started.capture_limits?.message_text_bytes).toBe(50_000); + observed = await fixture.load(started.session_id, "send('observed')"); + } + expect(observed.isError).not.toBe(true); + const evidence = parseEvidence(observed.structuredContent); + const facts = jsonObjectSchema.parse(evidence.normalized_result); + expect(facts.capture_limits).toMatchObject({ + message_text_bytes: 50_000, + session_bytes: 16_777_216, + }); + expect(facts.messages).toEqual([expect.objectContaining({ description })]); + expect(JSON.stringify(facts.messages)).not.toContain( + '"description_truncated":true', + ); + if (operation === "instrument_with_frida") { + expect(evidence.parameters.capture_limits).toEqual({ + message_text_bytes: 50_000, + }); + } + }, +); + it.each(["attached", "during-resume", "after-resume"] as const)( "preserves observed detachment through resume settlement: %s", async (timing) => { diff --git a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts index 7a113b7b78..fe1a09ec64 100644 --- a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts +++ b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts @@ -109,20 +109,21 @@ it.each([ strings_names: "unchanged", cfg: "unchanged", }); - const replay = await harness.session.execute( + const reobserved = await harness.session.execute( "analyze_function", evidence.parameters, ); - if (!replay.ok) throw replay.error; - expect(replay.value.relationshipCoverage).toEqual(coverage); - expect(harness.calls).toEqual(["analyze_function"]); + if (!reobserved.ok) throw reobserved.error; + expect(reobserved.value.result).toEqual(dossier); + expect(reobserved.value.relationshipCoverage).toEqual(coverage); + expect(harness.calls).toEqual(["analyze_function", "analyze_function"]); } finally { await harness.close(); } }, ); -it("retains terminal-call qualifications across function views and snapshot replay", async () => { +it("retains terminal-call qualifications across live dossiers and snapshot-backed function views", async () => { const limitation = "Ghidra Listing reports a terminal call at 0x401000 to EXTERNAL:0x20 __tls_get_addr (external=true, thunk=false, hasNoReturn=true); fallthrough is excluded by Ghidra's flow model. This records Ghidra's FunctionManager flag, not an independent verification that the callee cannot return."; const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); @@ -195,11 +196,19 @@ it("retains terminal-call qualifications across function views and snapshot repl if (!replay.ok) throw replay.error; expect(replay.value.result, name).toEqual(value); expect(replay.value.limitations, name).toContain(limitation); - expect(replay.value.limitations, name).toContainEqual( - expect.stringMatching(/snapshot.*did not re.run the provider/iu), + const snapshotLimitation = expect.stringMatching( + /snapshot.*did not re.run the provider/iu, ); + if (name === "analyze_function") + expect(replay.value.limitations, name).not.toContainEqual( + snapshotLimitation, + ); + else + expect(replay.value.limitations, name).toContainEqual( + snapshotLimitation, + ); } - expect(harness.calls).toEqual([...values.keys()]); + expect(harness.calls).toEqual([...values.keys(), "analyze_function"]); } finally { await harness.close(); } diff --git a/tests/boundary/mcp/ghidraNativeApiMcp.test.ts b/tests/boundary/mcp/ghidraNativeApiMcp.test.ts index 45e68b85f5..71b127a4dc 100644 --- a/tests/boundary/mcp/ghidraNativeApiMcp.test.ts +++ b/tests/boundary/mcp/ghidraNativeApiMcp.test.ts @@ -229,11 +229,23 @@ describe("Ghidra MCP native API evidence", () => { degraded, ); failed = false; - expect((await inspect()).normalized_result).toMatchObject({ + const recovered = await inspect(); + expect(recovered.normalized_result).toMatchObject({ boundary: { available: true }, }); - await inspect(); - expect(harness.calls).toEqual(["analyze_function", "analyze_function"]); + expect(recovered.evidence_id).not.toBe(degraded.evidence_id); + // A successful observation remains live too. A later refusal must not + // be hidden by replaying the recovered dossier or its composed answer. + failed = true; + expect(await inspect()).toEqual(degraded); + expect(harness.session.evidenceById(recovered.evidence_id)).toEqual( + recovered, + ); + expect(harness.calls).toEqual([ + "analyze_function", + "analyze_function", + "analyze_function", + ]); } finally { await harness.close(); } @@ -285,7 +297,7 @@ describe("Ghidra MCP native API evidence", () => { }, 30_000); }); -it("retains unavailable Ghidra native API observations across provenance and exact replay", async () => { +it("retains unavailable Ghidra native API observations across live reobservation and provenance", async () => { const residual = "The controlled provider omitted native API boundary facts."; // This wire fixture is controlled and never launches Ghidra. const harness = await connectGhidraMcp( @@ -352,23 +364,27 @@ it("retains unavailable Ghidra native API observations across provenance and exa }; const first = await inspect("0x401000"); - const replay = await inspect("0x401000"); - expect(replay.normalized_result).toEqual(first.normalized_result); - expect(replay.evidence_id).not.toBe(first.evidence_id); - expect(replay.limitations).toEqual( + const repeated = await inspect("0x401000"); + expect(repeated).toEqual(first); + expect(repeated.limitations).not.toEqual( expect.arrayContaining([ expect.stringMatching(/snapshot.*did not re.run the provider/iu), ]), ); const warmBundle = session.exportEvidenceBundle(); - const warmReplay = await inspect("0x401000"); - expect(warmReplay.evidence_id).toBe(replay.evidence_id); + const warmObservation = await inspect("0x401000"); + expect(warmObservation.evidence_id).toBe(first.evidence_id); expect(session.exportEvidenceBundle()).toEqual(warmBundle); + expect(harness.calls).toEqual([ + "analyze_function", + "analyze_function", + "analyze_function", + ]); const second = await inspect("0x401100"); expect(second.evidence_id).not.toBe(first.evidence_id); expect(session.evidenceById(first.evidence_id)).toEqual(first); - expect(session.evidenceById(replay.evidence_id)).toEqual(replay); + expect(session.evidenceById(repeated.evidence_id)).toEqual(repeated); const listed = await mcp.callTool({ name: "list_unknowns", @@ -378,17 +394,13 @@ it("retains unavailable Ghidra native API observations across provenance and exa const unknownPage = toolContract("list_unknowns").outputSchema.parse( listed.structuredContent, ).result; - expect(unknownPage.total).toBe(3); + expect(unknownPage.total).toBe(2); expect(unknownPage.items).toEqual( expect.arrayContaining([ expect.objectContaining({ question: expect.stringContaining(first.evidence_id), supporting_evidence_ids: [first.evidence_id], }), - expect.objectContaining({ - question: expect.stringContaining(replay.evidence_id), - supporting_evidence_ids: [replay.evidence_id], - }), expect.objectContaining({ question: expect.stringContaining(second.evidence_id), supporting_evidence_ids: [second.evidence_id], diff --git a/tests/boundary/mcp/goBinaryViews.test.ts b/tests/boundary/mcp/goBinaryViews.test.ts index 10c6476335..e0d43a0d08 100644 --- a/tests/boundary/mcp/goBinaryViews.test.ts +++ b/tests/boundary/mcp/goBinaryViews.test.ts @@ -407,6 +407,24 @@ goViewTest( }); await writeFile(path, fixture.bytes); + const limited = goBinarySummaryViewResultSchema.parse( + ( + await call("inspect_go_binary", { + path, + include_string_literals: true, + max_string_literals: 2, + detail: "summary", + }) + ).normalized_result, + ); + expect(limited.source_coverage.string_literals).toBe("truncated"); + expect(limited.summary.string_literals).toMatchObject({ + available_count: 2, + discovered_count: 3, + truncated: true, + limits_reached: ["max_string_literals"], + }); + const summary = goBinarySummaryViewResultSchema.parse( ( await call("inspect_go_binary", { diff --git a/tests/boundary/mcp/kernelModuleMcp.test.ts b/tests/boundary/mcp/kernelModuleMcp.test.ts index 10a3126acf..1eb0b8dd8c 100644 --- a/tests/boundary/mcp/kernelModuleMcp.test.ts +++ b/tests/boundary/mcp/kernelModuleMcp.test.ts @@ -22,6 +22,66 @@ const connect = async (mcp: TestMcp) => { return { client, session }; }; +mcpTest( + "keeps valid omitted modinfo fields distinct from malformed syntax and recovers on the same connection", + async ({ mcp }) => { + const { client, session } = await connect(mcp); + try { + const root = await createTestTempDirectory("rea-kernel-module-text-mcp-"); + const path = join(root, "text.ko"); + const key = "k".repeat(257); + const value = "v".repeat(4097); + await writeFile( + path, + kernelModuleFixture({ modinfo: Buffer.from(`${key}=${value}\0`) }), + ); + const inspect = async (budgets: Record = {}) => { + const response = await client.callTool({ + name: "inspect_kernel_module", + arguments: { path, ...budgets }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe( + true, + ); + return toolContract("inspect_kernel_module").outputSchema.parse( + response.structuredContent, + ); + }; + const initial = await inspect(); + const omitted = initial.normalized_result.modinfo?.entries[0]; + expect(omitted).toMatchObject({ + status: "observed", + reason: null, + key: null, + value: null, + omitted_fields: ["key", "value"], + raw_hex: null, + }); + expect(initial.normalized_result.coverage.status).toBe("partial"); + expect(initial.normalized_result.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("max_modinfo_key_bytes at least 257"), + expect.stringContaining("max_modinfo_value_bytes at least 4097"), + ]), + ); + const recovered = await inspect({ + max_modinfo_key_bytes: 257, + max_modinfo_value_bytes: 4097, + }); + expect(recovered.subject).toEqual(initial.subject); + expect(recovered.normalized_result.modinfo?.entries[0]).toEqual({ + ...omitted, + key, + value, + omitted_fields: [], + }); + expect(recovered.normalized_result.coverage.status).toBe("complete"); + } finally { + await session.close(); + } + }, +); + mcpTest( "advertises inspect_kernel_module and reads an ELF32 big-endian module", async ({ mcp }) => { diff --git a/tests/boundary/mcp/nodeSeaMcp.test.ts b/tests/boundary/mcp/nodeSeaMcp.test.ts index 0de49b09a2..31472f4c4a 100644 --- a/tests/boundary/mcp/nodeSeaMcp.test.ts +++ b/tests/boundary/mcp/nodeSeaMcp.test.ts @@ -8,10 +8,30 @@ import { buildNodeSeaBlob, } from "../../../src/artifacts/nodeSea/NodeSea.fixture.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; -import { z } from "zod"; import { artifactExtractionResultSchema } from "../../../src/domain/artifactGraph.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; +import { + analysisViewResultSchema, + artifactInventorySummaryViewResultSchema, +} from "../../../src/domain/analysisView/analysisView.js"; + +const expectBlobAdmissionFailure = ( + reply: Parameters[0], +) => { + expect(parseMcpToolError(reply).error).toMatchObject({ + code: "truncated", + retryable: true, + details: { + reader_limit: { + parameter: "artifact_limits.node_sea.blob_bytes", + effective_value: 1, + unit: "bytes", + can_increase: true, + }, + }, + }); +}; // The compiled stdio fixture checks advertised schemas against their dialect and // validates actual tool responses, avoiding a second source server module graph. @@ -36,20 +56,43 @@ it.each(["full", "compact"] as const)( try { const opened = await call("open_binary", { path }); expect(opened.isError, JSON.stringify(opened)).not.toBe(true); - const reply = await call("inspect_artifact", { detail: "summary" }); + const lowerPolicy = { node_sea: { blob_bytes: 1 } }; + const refused = await call("inspect_artifact", { + detail: "summary", + artifact_limits: lowerPolicy, + }); + expectBlobAdmissionFailure(refused); + const selectedPolicy = { node_sea: { blob_bytes: 4096 } }; + const reply = await call("inspect_artifact", { + detail: "summary", + artifact_limits: selectedPolicy, + }); expect(reply.isError, JSON.stringify(reply)).not.toBe(true); - const summary = z - .object({ - parent_evidence_id: z.string(), - summary: z.object({ root_format_metadata: z.unknown() }), - }) - .parse(parseEvidence(reply.structuredContent).normalized_result); + const summary = artifactInventorySummaryViewResultSchema.parse( + parseEvidence(reply.structuredContent).normalized_result, + ); + expect(summary.summary.effective_limits?.node_sea.blob_bytes).toBe(4096); expect(summary.summary.root_format_metadata).toMatchObject({ kind: "node-sea", carrier: "pe", fuse_state: "set", code_path: "main.js", }); + const bundleReply = await call("get_evidence_bundle", { + detail: "complete", + }); + expect(bundleReply.isError, JSON.stringify(bundleReply)).not.toBe(true); + const bundle = toolContract("get_evidence_bundle").outputSchema.parse( + bundleReply.structuredContent, + ).result; + if ("kind" in bundle) + throw new Error("Expected complete retained Evidence"); + const parent = parseEvidence( + bundle.records.find( + (record) => record.evidence_id === summary.parent_evidence_id, + ), + ); + expect(parent.parameters.artifact_limits).toEqual(selectedPolicy); const retained = await call("inspect_analysis_view", { source: { kind: "retained-evidence", @@ -75,6 +118,11 @@ it.each(["full", "compact"] as const)( }, ], }); + const loweredAfterRetention = await call("inspect_artifact", { + detail: "summary", + artifact_limits: lowerPolicy, + }); + expectBlobAdmissionFailure(loweredAfterRetention); const extracted = await call("extract_artifact", {}); expect(extracted.isError, JSON.stringify(extracted)).not.toBe(true); const result = artifactExtractionResultSchema.parse( diff --git a/tests/boundary/mcp/rustBinaryMcp.test.ts b/tests/boundary/mcp/rustBinaryMcp.test.ts index c158b2d3c5..345ab82f0e 100644 --- a/tests/boundary/mcp/rustBinaryMcp.test.ts +++ b/tests/boundary/mcp/rustBinaryMcp.test.ts @@ -99,7 +99,7 @@ mcpTest( ); mcpTest( - "preserves rustc hash capacity units through the public error contract", + "preserves dependency Evidence and compiler-prefix coverage through MCP", async ({ mcp }) => { const { client, session } = await connect(mcp); try { @@ -110,25 +110,17 @@ mcpTest( name: "inspect_rust_binary", arguments: { path }, }); - expect(response.isError).toBe(true); - expect(parseMcpToolError(response).error).toMatchObject({ - code: "resource_constraint", - category: "resource_constraint", - retryable: false, - remediation: { - action: expect.stringMatching( - /rustc commit.path inventory.*fixed count/iu, - ), - }, - details: { - operation: "inspect_rust_binary", - resource: "capacity", - reported_limits: { maximum_rustc_hashes: 64 }, - }, - }); - expect(parseMcpToolError(response).error.details).not.toHaveProperty( - "reported_limits.maximum_bytes", + expect(response.isError).not.toBe(true); + const evidence = toolContract("inspect_rust_binary").outputSchema.parse( + response.structuredContent, ); + const report = rustBinarySchema.parse(evidence.normalized_result); + expect(report.auditable_metadata?.packages).toHaveLength(2); + expect(report.rustc_commit?.coverage).toEqual({ + status: "partial", + reason: "max_rustc_hashes", + effective_limit: 64, + }); } finally { await session.close(); } diff --git a/tests/boundary/mcp/webSourceLocationMcp.test.ts b/tests/boundary/mcp/webSourceLocationMcp.test.ts index 0fbab7bc14..98828cc4e6 100644 --- a/tests/boundary/mcp/webSourceLocationMcp.test.ts +++ b/tests/boundary/mcp/webSourceLocationMcp.test.ts @@ -146,3 +146,43 @@ it("preserves missing Linux ps capability and recovery through source-map tracin expect(createRuntime).not.toHaveBeenCalled(); expect(launcher).not.toHaveBeenCalled(); }); + +it("rejects malformed delivery controls before acquiring source-map artifacts", async () => { + const load = vi.fn(async () => ok(webSourceLocationFixture())); + const harness = await createApplicationMcpHarness({ + serverOptions: { + webSourceLocation: new WebSourceLocationService( + { load }, + { + trace: async (input) => + ok(traceSourceMap(input.text, input.url, input.position)), + }, + ), + }, + }); + for (const delivery of [ + { + detail: "complete", + view: { kind: "source-location", collection: "matches" }, + }, + { + detail: "summary", + view: { + kind: "source-location", + collection: "source-text", + match_index: -1, + }, + }, + { + detail: "summary", + view: { kind: "source-location", collection: "matches", limit: 0 }, + }, + ]) { + const result = await harness.client.callTool({ + name: "trace_web_source_location", + arguments: { ...args, ...delivery }, + }); + expect(parseMcpToolError(result).error.code).toBe("invalid_request"); + } + expect(load).not.toHaveBeenCalled(); +}); diff --git a/tests/boundary/native/pwndbgReplyFinalization.test.ts b/tests/boundary/native/pwndbgReplyFinalization.test.ts index 494624529c..833cd161a5 100644 --- a/tests/boundary/native/pwndbgReplyFinalization.test.ts +++ b/tests/boundary/native/pwndbgReplyFinalization.test.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { resolvePwntoolsLimits } from "../../../src/domain/native/pwntoolsResourceLimits.js"; import { mkdir, writeFile } from "node:fs/promises"; import type { FileHandle } from "node:fs/promises"; import { join } from "node:path"; @@ -112,6 +113,7 @@ const fixture = async ( resources, options: { signal }, timeoutMs: 10_000, + producerLimits: resolvePwntoolsLimits(), runCommand: async (spawn) => { const output = await runOwnedCommand( { diff --git a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts index d961251998..9afbee3b27 100644 --- a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts +++ b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts @@ -26,14 +26,10 @@ import type { import { DEBUGGER_COMMAND_DEADLINE_SETTING, DEBUGGER_MAX_COMMAND_OUTPUT_BYTES_SETTING, - DEFAULT_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES_LIMIT, } from "../../../src/config/debuggerSessionLimits.js"; import { DEFAULT_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES } from "../../../src/domain/analysisResourceLimits.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; -import { - debuggerAdmissionRecovery, - stopOwnedDebuggerProcess, -} from "../../../src/process/ownedDebuggerSession.js"; +import { stopOwnedDebuggerProcess } from "../../../src/process/ownedDebuggerSession.js"; import { parseLinuxProcessStat } from "../../../src/process/LinuxProcessStat.js"; import { expectGdbSessionStarted } from "./gdbTestHelpers.js"; @@ -3551,6 +3547,12 @@ process.stdin.on('data', (chunk) => { if (result.ok) { expect(result.value.result.output_truncated).toBe(true); expect(result.value.result.records).toHaveLength(7); + expect(result.value.result.output_limits).toEqual([ + { kind: "command_bytes", maximum: 16_777_216 }, + ]); + expect(result.value.evidence.limitations.join(" ")).toContain( + "Set REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES", + ); } expect( manager.status(started.value.session_id)?.recent_mi_records_truncated, @@ -3707,22 +3709,17 @@ it("keeps the session usable after an oversized stream record is dropped", async expect(executed.value.result.console).toContain("before"); expect(executed.value.result.completion_status).toBe("complete"); expect(executed.value.result.output_truncated).toBe(true); - const output = DEFAULT_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES_LIMIT; - expect(executed.value.evidence.limitations.join(" ")).toContain( - `Retained MI records reached the ${String(output.value)} byte command-output cap or the 16384 record cap. ${debuggerAdmissionRecovery(output)} The record cap has no separate setting. Output completeness is unknown.`, - ); + expect(executed.value.result.output_limits).toEqual([ + { kind: "mi_line_bytes", maximum: 16_777_216 }, + ]); + expect(executed.value.evidence.normalized_result).toMatchObject({ + output_limits: [{ kind: "mi_line_bytes", maximum: 16_777_216 }], + }); const limitations = executed.value.evidence.limitations.join(" "); - expect(limitations).toContain( - "Retained MI records reached the 16777216 byte command-output cap or the 16384 record cap.", - ); - expect(limitations).toContain( - "The effective REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES value is 16777216 bytes (source default; default 16777216 bytes,", - ); - expect(limitations).toContain( - "Set REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES in the environment that starts REA", - ); - expect(limitations).toContain( - "The record cap has no separate setting. Output completeness is unknown.", + expect(limitations).toContain("MI line limit"); + expect(limitations).toContain("does not raise it"); + expect(limitations).not.toContain( + "Set REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES", ); } const second = await manager.execute(started.value.session_id, "after-drop"); @@ -3791,6 +3788,14 @@ process.stdin.on('data', (chunk) => { if (result.ok) { expect(result.value.result.records).toHaveLength(16_384); expect(result.value.result.output_truncated).toBe(true); + expect(result.value.result.output_limits).toEqual([ + { kind: "command_records", maximum: 16_384 }, + ]); + const limitations = result.value.evidence.limitations.join(" "); + expect(limitations).toContain("16384 record cap"); + expect(limitations).not.toContain( + "Set REA_DEBUGGER_MAX_COMMAND_OUTPUT_BYTES", + ); } }); diff --git a/tests/boundary/providers/desktopExportPython.test.ts b/tests/boundary/providers/desktopExportPython.test.ts index a475038489..99131c4fdc 100644 --- a/tests/boundary/providers/desktopExportPython.test.ts +++ b/tests/boundary/providers/desktopExportPython.test.ts @@ -1,16 +1,29 @@ +import { + DESKTOP_VIEW_DEFAULTS, + desktopViewPolicy, + type DesktopViewPolicy, +} from "../../../src/domain/desktop/desktopViewPolicy.js"; import { execFileSync, type ExecFileSyncOptionsWithStringEncoding, } from "node:child_process"; import { mkdtempSync, writeFileSync, unlinkSync, rmdirSync } from "node:fs"; import { tmpdir } from "node:os"; -import { chmod, mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { + chmod, + mkdir, + readFile, + readlink, + stat, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; import { PYTHON_HELPERS, + renderPythonHelpers, PYTHON_IMPORTS, } from "../../../src/domain/desktop/appExportPython.js"; import { @@ -358,7 +371,7 @@ it("restores a selected settings alias without replacing an existing path", asyn SEED_HOME = "/home/ada" TEXT_SEEDS = frozenset() SEED_MODES = {} -STATE_ALIASES = [("settings", ".config/notes", "directory")] +STATE_ALIASES = [("settings", ".config/notes", json.loads(sys.argv[3]))] HERE = Path(sys.argv[1]) ${PYTHON_HELPERS} home = Path(sys.argv[2]) @@ -366,7 +379,10 @@ _seed_home(home) _ensure_state_aliases(home) _ensure_state_aliases(home) `; - execPython(["-I", "-B", "-c", program, root, home], { encoding: "utf8" }); + execPython( + ["-I", "-B", "-c", program, root, home, JSON.stringify("directory")], + { encoding: "utf8" }, + ); expect(await readFile(join(home, "settings", "settings.ini"), "utf8")).toBe( "saved", ); @@ -380,10 +396,13 @@ _ensure_state_aliases(home) await mkdir(other); await writeFile(join(other, "settings"), "preserve"); expect(() => - execPython(["-I", "-B", "-c", program, root, other], { - encoding: "utf8", - stdio: "pipe", - }), + execPython( + ["-I", "-B", "-c", program, root, other, JSON.stringify("directory")], + { + encoding: "utf8", + stdio: "pipe", + }, + ), ).toThrow(); expect(await readFile(join(other, "settings"), "utf8")).toBe("preserve"); @@ -391,13 +410,29 @@ _ensure_state_aliases(home) await mkdir(empty); const emptySeed = join(root, "empty-seed"); await mkdir(emptySeed); - execPython(["-I", "-B", "-c", program, emptySeed, empty], { - encoding: "utf8", - }); + execPython( + ["-I", "-B", "-c", program, emptySeed, empty, JSON.stringify("directory")], + { + encoding: "utf8", + }, + ); await writeFile(join(empty, "settings", "new.ini"), "new settings"); expect( await readFile(join(empty, ".config", "notes", "new.ini"), "utf8"), ).toBe("new settings"); + + // An older capture does not establish the missing target's kind. + const unknown = join(root, "unknown-home"); + await mkdir(unknown); + execPython(["-I", "-B", "-c", program, emptySeed, unknown, "null"], { + encoding: "utf8", + }); + expect(await readlink(join(unknown, "settings"))).toBe( + join(unknown, ".config", "notes"), + ); + await expect(stat(join(unknown, ".config", "notes"))).rejects.toMatchObject({ + code: "ENOENT", + }); }); /** A stand-in xprop answering the root window list and each window's class and state from a table. */ @@ -509,11 +544,14 @@ it("refuses the JSON numbers REA's state view refuses, as JSON with an error rat }); /** The exported module's format, view and error for each named text, and REA's for the same bytes. */ -const textViews = (files: readonly (readonly [string, string])[]) => { +const textViews = ( + files: readonly (readonly [string, string])[], + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, +) => { const imports = PYTHON_IMPORTS.split("\n") .filter((line) => !line.includes("agentenv_protocol")) .join("\n"); - const program = `${imports}${PYTHON_HELPERS} + const program = `${imports}${renderPythonHelpers(policy)} for name, text in json.load(sys.stdin): view = _text_view(name, text, len(text.encode())) print(json.dumps({"format": view["format"], "view": view["view"], "error": view.get("error")})) @@ -528,10 +566,10 @@ for name, text in json.load(sys.stdin): .map((line) => JSON.parse(line) as unknown); const rea = files.map(([name, text]) => { const bytes = new TextEncoder().encode(text); - const format = detectStateFormat(name, bytes); + const format = detectStateFormat(name, bytes, policy); if (format !== "text" && format !== "json" && format !== "keyfile") throw new Error(`${name} reads as ${format}`); - const read = textualView(format, bytes); + const read = textualView(format, bytes, policy); return read.ok ? { format, view: read.value, error: null } : { format, view: null, error: read.error }; @@ -539,6 +577,88 @@ for name, text in json.load(sys.stdin): return { python, rea }; }; +it("applies the selected desktop view budgets equally in TypeScript and emitted Python", () => { + const files = [ + ["lines.txt", "one\ntwo\nthree"], + ["value.json", '{"key":"value"}'], + ] as const; + const limited = textViews( + files, + desktopViewPolicy({ text_lines: 2, parsed_bytes: 4 }), + ); + expect(limited.python).toEqual(limited.rea); + expect(limited.rea[0]?.view).toMatchObject({ + lines: ["one", "two"], + truncated: true, + line_count: 3, + }); + expect(limited.rea[1]?.error).toContain("limit for JSON"); + const recovered = textViews( + files, + desktopViewPolicy({ text_lines: 3, parsed_bytes: 64 }), + ); + expect(recovered.python).toEqual(recovered.rea); + expect(recovered.rea[0]?.view).toMatchObject({ + lines: ["one", "two", "three"], + truncated: false, + }); + expect(recovered.rea[1]).toEqual({ + format: "json", + view: { key: "value" }, + error: null, + }); +}); + +it("bounds the emitted reader's probe when an app grows a file after its size check", () => { + const imports = PYTHON_IMPORTS.split("\n") + .filter((line) => !line.includes("agentenv_protocol")) + .join("\n"); + const program = `${imports}${renderPythonHelpers(desktopViewPolicy({ file_bytes: 4 }))} +read_bytes = 0 +class ObservedReader: + def __init__(self, file): + self.file = file + def __enter__(self): + return self + def __exit__(self, *args): + self.file.close() + def read(self, size=-1): + global read_bytes + data = self.file.read(size) + read_bytes += len(data) + return data +class GrowingPath(type(Path())): + def open(self, mode="r", *args, **kwargs): + if mode == "rb": + Path(str(self)).write_bytes(b"x" * 20000) + return ObservedReader(super().open(mode, *args, **kwargs)) + return super().open(mode, *args, **kwargs) +with tempfile.TemporaryDirectory() as root: + small = Path(root) / "small.txt" + small.write_bytes(b"abcd") + ordinary = _view(small) + growing = GrowingPath(root) / "growing.txt" + growing.write_bytes(b"abcd") + grown = _view(growing) + print(json.dumps({"ordinary": ordinary, "grown": grown, "read_bytes": read_bytes, "grown_bytes": growing.stat().st_size})) +`; + const result = JSON.parse( + execPython(["-I", "-B", "-c", program], { encoding: "utf8" }), + ); + expect(result.ordinary).toMatchObject({ + format: "text", + view: { lines: ["abcd"] }, + truncated: false, + }); + expect(result.grown).toEqual({ + format: null, + view: null, + error: "larger than 4 bytes, the most the view reads", + }); + expect(result.read_bytes).toBe(5); + expect(result.grown_bytes).toBe(20000); +}); + it("reads text nested past Python's recursion limit as JSON only when all of it is, as REA does", () => { const brackets = (open: number, close: number) => `${"[".repeat(open)}${"]".repeat(close)}`; diff --git a/tests/boundary/providers/desktopExportSeedPolicy.test.ts b/tests/boundary/providers/desktopExportSeedPolicy.test.ts new file mode 100644 index 0000000000..d60b5c89a5 --- /dev/null +++ b/tests/boundary/providers/desktopExportSeedPolicy.test.ts @@ -0,0 +1,152 @@ +import { execFileSync } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { exportDesktopApplication } from "../../../src/application/desktop/DesktopApplicationExport.js"; +import { + HOME, + capture, + change, + checkpoint, + readers, + retained, + writeCaptureFolder, +} from "../../../src/application/desktop/desktopCapture.fixture.js"; +import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it.each([ + { + label: "scrubbed bytes over limit", + before: HOME, + bytes: 9, + readable: false, + }, + { + label: "scrubbed bytes exactly at limit", + before: HOME, + bytes: 10, + readable: true, + }, + { + label: "scrubbed bytes below limit", + before: HOME, + bytes: 11, + readable: true, + }, + { + label: "ordinary unmodified seed", + before: "old", + bytes: 9, + readable: true, + }, +])( + "admits task seed views like the shipped reader: $label", + async ({ before, bytes, readable }) => { + const scratch = await createTestTempDirectory("desktop-seed-policy-"); + const homePath = ".config/notes/seed.txt"; + const path = `${HOME}/${homePath}`; + const base = capture(); + const captured = { + ...base, + state: { + ...base.state, + checkpoints: [ + checkpoint("baseline", "baseline", [], [retained(path, before)]), + checkpoint( + "end", + "request", + [change(path, { before, after: "done" })], + [retained(path, "done")], + ), + ], + }, + }; + const folder = await writeCaptureFolder(captured, undefined, [ + before, + "done", + ]); + const directory = join(scratch, "plain"); + const request = { + capture: folder, + directory, + name: "notes", + tasks: [], + viewPolicy: { file_bytes: bytes }, + }; + // Export acquisition stays independent: an unreadable view still ships its seed. + const plain = await exportDesktopApplication(request, readers); + if (plain instanceof Error) throw plain; + expect(plain.parameters).toMatchObject({ + view_policy: { file_bytes: bytes }, + }); + const seed = join(directory, "envs/notes/seed", homePath); + expect(await readFile(seed, "utf8")).toBe( + before === HOME ? "/home/user" : before, + ); + + // Run the actual emitted module's helpers without importing the app framework + // or defining/launching the computer app class. + const driver = join(scratch, "read.py"); + await writeFile( + driver, + `import ast, json, sys +from pathlib import Path +source = Path(sys.argv[1]) +tree = ast.parse(source.read_text(), str(source)) +tree.body = [node for node in tree.body if not isinstance(node, ast.ClassDef) and not (isinstance(node, ast.ImportFrom) and (node.module or "").startswith("agentenv_protocol"))] +exec(compile(tree, str(source), "exec"), globals()) +print(json.dumps(_view(Path(sys.argv[2])))) +`, + ); + const emitted = JSON.parse( + execFileSync( + process.platform === "win32" ? "python" : "python3", + ["-I", "-B", driver, join(directory, "envs/notes/notes_app.py"), seed], + { encoding: "utf8" }, + ), + ) as { view: unknown; error?: string }; + expect(emitted.view !== null).toBe(readable); + if (!readable) + expect(emitted.error).toBe( + "larger than 9 bytes, the most the view reads", + ); + + const task = join(scratch, "task.json"); + await writeFile( + task, + JSON.stringify({ + name: "finish", + prompt: "Write done.", + checks: [ + { + id: "done", + kind: "lines_present", + file: homePath, + lines: ["done"], + }, + ], + }), + ); + const withTask = await exportDesktopApplication( + { ...request, directory: join(scratch, "task"), tasks: [task] }, + readers, + ); + if (!readable) { + expect(withTask).toBeInstanceOf(AnalysisInputError); + expect((withTask as AnalysisInputError).userMessage).toContain( + "No check fails on what the seed's views show", + ); + expect((withTask as AnalysisInputError).userMessage).toContain( + "larger than 9 bytes", + ); + } else { + if (withTask instanceof Error) throw withTask; + expect(withTask.normalized_result).toMatchObject({ + task: { failing_on_seed: ["done"] }, + }); + } + }, +); diff --git a/tests/boundary/providers/desktopSqliteViewBudget.test.ts b/tests/boundary/providers/desktopSqliteViewBudget.test.ts new file mode 100644 index 0000000000..b259c579c6 --- /dev/null +++ b/tests/boundary/providers/desktopSqliteViewBudget.test.ts @@ -0,0 +1,229 @@ +import { execFileSync } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { fileURLToPath } from "node:url"; + +import { expect, it, onTestFinished } from "vitest"; + +import { readSqliteStateView } from "../../../src/application/desktop/DesktopSqliteState.js"; +import { SqliteDatabaseService } from "../../../src/application/sqlite/SqliteDatabaseService.js"; +import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import { + PYTHON_IMPORTS, + renderPythonHelpers, +} from "../../../src/domain/desktop/appExportPython.js"; +import { desktopViewPolicy } from "../../../src/domain/desktop/desktopViewPolicy.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const imports = PYTHON_IMPORTS.split("\n") + .filter((line) => !line.includes("agentenv_protocol")) + .join("\n"); + +/** Execute emitted helpers or an independent oracle using only Python's standard library. */ +const python = async ( + scratch: string, + program: string, + args: string[] = [], +) => { + const script = join(scratch, "read.py"); + await writeFile(script, program); + return execFileSync( + process.platform === "win32" ? "python" : "python3", + ["-I", "-B", script, ...args], + { encoding: "utf8" }, + ); +}; + +/** One real DB and owned worker, read independently through both production readers. */ +const sqliteReaders = async (populate: (db: DatabaseSync) => void) => { + const scratch = await createTestTempDirectory("desktop-sqlite-view-budget-"); + const path = join(scratch, "state.db"); + const db = new DatabaseSync(path); + try { + db.exec("CREATE TABLE healthy(x); INSERT INTO healthy VALUES(1)"); + populate(db); + } finally { + db.close(); + } + const before = await readFile(path); + const worker = fileURLToPath( + new URL("../../../dist/sqlite/SqliteDatabaseWorker.js", import.meta.url), + ); + const provider = new SqliteDatabaseProvider(process.env, (spawn) => + spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((argument, index) => + index === 1 ? worker : argument, + ), + }), + ); + const service = new SqliteDatabaseService(provider); + onTestFinished(async () => { + await provider.close(); + expect((await readFile(path)).equals(before)).toBe(true); + }); + return { + scratch, + typescript: async (bytes: number) => { + const selected = await readSqliteStateView( + service, + path, + undefined, + desktopViewPolicy({ sqlite_view_bytes: bytes }), + ); + if (!selected.ok) throw new Error(selected.error); + return selected.view; + }, + emitted: async (bytes: number) => + JSON.parse( + await python( + scratch, + `${imports}${renderPythonHelpers(desktopViewPolicy({ sqlite_view_bytes: bytes }))}\nprint(json.dumps(_sqlite(Path(sys.argv[1]))))\n`, + [path], + ), + ) as { view: { tables: Record }; truncated: boolean }, + }; +}; + +it("bounds each projected SQLite table and recovers the same complete facts in both readers", async () => { + const fixture = await sqliteReaders((db) => { + db.exec("CREATE TABLE items(value)"); + const insert = db.prepare("INSERT INTO items VALUES(?)"); + insert.run("plain"); + insert.run("\u0000é😀"); + insert.run(Buffer.from([0, 255])); + }); + // Independent expected JSON includes UTF-8, escaping and blob expansion. + const expected = { + columns: ["value"], + primary_key: [], + rows: [["plain"], ["\u0000é😀"], [{ blob_hex: "00ff" }]], + truncated: false, + }; + const exact = Buffer.byteLength(JSON.stringify(expected)); + for (const bytes of [exact - 1, exact, exact + 1]) { + const typescript = await fixture.typescript(bytes); + const emitted = await fixture.emitted(bytes); + expect(emitted.view).toEqual(typescript); + expect(emitted.view.tables.healthy).toEqual({ + columns: ["x"], + primary_key: [], + rows: [[1]], + truncated: false, + }); + expect(emitted.truncated).toBe(bytes < exact); + expect(emitted.view.tables.items).toEqual( + bytes < exact + ? { + error: expect.stringContaining( + `sqlite_view_bytes=${String(bytes)} UTF-8 JSON bytes`, + ), + } + : expected, + ); + } +}); + +it("counts numeric facts using each reader's native compact JSON spelling", async () => { + const fixture = await sqliteReaders((db) => + db.exec( + "CREATE TABLE numbers(id INTEGER, whole REAL GENERATED ALWAYS AS (16.0) VIRTUAL, fraction REAL GENERATED ALWAYS AS (0.125) VIRTUAL, small REAL GENERATED ALWAYS AS (1e-7) VIRTUAL, large REAL GENERATED ALWAYS AS (1e20) VIRTUAL, neg REAL GENERATED ALWAYS AS (-0.0) VIRTUAL); INSERT INTO numbers(id) VALUES(1)", + ), + ); + // SQLite normalizes this generated -0.0 to zero; explicit signed-zero cell + // sizing is checked below without claiming the DB retained a missing sign. + const expected = { + columns: ["id", "whole", "fraction", "small", "large", "neg"], + primary_key: [], + rows: [[1, 16, 0.125, 1e-7, 1e20, 0]], + truncated: false, + }; + const typescriptBytes = Buffer.byteLength(JSON.stringify(expected)); + const pythonBytes = Number( + ( + await python( + fixture.scratch, + `import json +expected = {"columns": ["id", "whole", "fraction", "small", "large", "neg"], "primary_key": [], "rows": [[1, 16.0, 0.125, 1e-7, 1e20, 0.0]], "truncated": False} +print(len(json.dumps(expected, ensure_ascii=False, separators=(",", ":")).encode("utf-8"))) +`, + ) + ).trim(), + ); + expect(pythonBytes).not.toBe(typescriptBytes); + for (const bytes of new Set([ + typescriptBytes - 1, + typescriptBytes, + pythonBytes - 1, + pythonBytes, + ])) { + const error = { + error: expect.stringContaining(`sqlite_view_bytes=${String(bytes)}`), + }; + expect(await fixture.typescript(bytes)).toMatchObject({ + tables: { numbers: bytes >= typescriptBytes ? expected : error }, + }); + expect((await fixture.emitted(bytes)).view.tables.numbers).toEqual( + bytes >= pythonBytes ? expected : error, + ); + } +}); + +it("keeps the native SQLite record limit independent of a larger view allowance", async () => { + // A tiny stored declaration expands beyond the native hex/record allowance. + const fixture = await sqliteReaders((db) => + db.exec( + "CREATE TABLE native_limit(id, payload BLOB GENERATED ALWAYS AS (zeroblob(9 * 1024 * 1024)) VIRTUAL); INSERT INTO native_limit(id) VALUES(1)", + ), + ); + const bytes = 32 * 1024 * 1024; + const emitted = await fixture.emitted(bytes); + for (const view of [await fixture.typescript(bytes), emitted.view]) { + expect(view).toMatchObject({ + tables: { + healthy: { rows: [[1]] }, + native_limit: { error: expect.any(String) }, + }, + }); + expect(JSON.stringify(view)).not.toContain("exceeds sqlite_view_bytes"); + } + expect(emitted.view.tables.native_limit).toEqual({ + error: expect.stringContaining("string or blob too big"), + }); + const nativeError = (emitted.view.tables.native_limit as { error: string }) + .error; + expect(nativeError).toContain("fixed 16777216-byte native limit"); + expect(nativeError).toContain( + "Increasing sqlite_view_bytes does not raise this boundary", + ); + expect(nativeError).toContain("Keep the complete database"); + expect(nativeError).toContain("only if it preserves the required facts"); + expect(nativeError).not.toContain("Increase sqlite_view_bytes"); + expect(emitted.truncated).toBe(true); +}); + +it("refuses escaped cell overflow without allocating its full JSON spelling", async () => { + const scratch = await createTestTempDirectory("desktop-sqlite-json-parts-"); + const result = JSON.parse( + await python( + scratch, + `${imports}${renderPythonHelpers(desktopViewPolicy())} +import tracemalloc +value = chr(1) * 1048576 +tracemalloc.start() +size = _view_json_bytes({"rows": [[value]]}, 1024) +negative_zero = _view_json_bytes(_cell(-0.0, "utf-8"), 1024) +_, peak = tracemalloc.get_traced_memory() +tracemalloc.stop() +print(json.dumps({"size": size, "peak": peak, "negative_zero": negative_zero})) +`, + ), + ) as { size: number; peak: number; negative_zero: number }; + expect(result.size).toBeGreaterThan(1024); + expect(result.negative_zero).toBe(Buffer.byteLength(JSON.stringify("-0"))); + // Input exists before tracing. Whole-cell quoting previously made a six-MiB + // string merely to reject this one-KiB allowance. + expect(result.peak).toBeLessThan(1024 * 1024); +}); diff --git a/tests/boundary/providers/desktopTaskPython.test.ts b/tests/boundary/providers/desktopTaskPython.test.ts index d536f74627..65b468bc13 100644 --- a/tests/boundary/providers/desktopTaskPython.test.ts +++ b/tests/boundary/providers/desktopTaskPython.test.ts @@ -257,12 +257,15 @@ const linesPresent = (...lines: string[]): DesktopTaskCheck => ({ const graded = ({ results }: VerifierOutput) => results.map(({ result, settled, read_cut }) => [result, settled, read_cut]); -it("evaluates every kind of check in the exported verifier as REA does", async () => { +it("evaluates every kind of check without computer execution in the exported verifier as REA does", async () => { const checks: DesktopTaskCheck[] = CHECK_CASES.map(([check], index) => ({ ...check, id: `check-${String(index + 1)}`, })); - const output = await runVerifier(checks, [[0, CHECK_VIEWS]]); + const output = await runVerifier(checks, [[0, CHECK_VIEWS]], [], { + exec: false, + }); + expect(output.execs).toEqual([]); const [card, ...data] = output.called; expect(card).toBe("http://computer:8080"); diff --git a/tests/boundary/providers/desktopXmlWhitespacePython.test.ts b/tests/boundary/providers/desktopXmlWhitespacePython.test.ts index 2a79e439d6..6c8b3875e7 100644 --- a/tests/boundary/providers/desktopXmlWhitespacePython.test.ts +++ b/tests/boundary/providers/desktopXmlWhitespacePython.test.ts @@ -1,13 +1,21 @@ +import { + DESKTOP_VIEW_DEFAULTS, + desktopViewPolicy, + type DesktopViewPolicy, +} from "../../../src/domain/desktop/desktopViewPolicy.js"; import { execFileSync } from "node:child_process"; import { expect, it } from "vitest"; import { - PYTHON_HELPERS, + renderPythonHelpers, PYTHON_IMPORTS, } from "../../../src/domain/desktop/appExportPython.js"; import { xmlView } from "../../../src/domain/desktop/stateFormats.js"; /** Execute the unmodified generated XML reader with its standard-library imports. */ -const pythonViews = (texts: readonly string[]) => { +const pythonViews = ( + texts: readonly string[], + policy: DesktopViewPolicy = DESKTOP_VIEW_DEFAULTS, +) => { const imports = PYTHON_IMPORTS.split("\n") .filter((line) => !line.includes("agentenv_protocol")) .join("\n"); @@ -18,7 +26,7 @@ const pythonViews = (texts: readonly string[]) => { "-I", "-B", "-c", - `${imports}${PYTHON_HELPERS}\nprint(json.dumps([_xml_view(text) for text in json.load(sys.stdin)]))`, + `${imports}${renderPythonHelpers(policy)}\nprint(json.dumps([_xml_view(text) for text in json.load(sys.stdin)]))`, ], { input: JSON.stringify(texts), encoding: "utf8" }, ), @@ -37,7 +45,7 @@ it("matches the ECMAScript whitespace set for XML-valid characters and preserves const characters = "\t\n\r \u00a0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u2028\u2029\u202f\u205f\u3000\ufeff\u0085\u180e\u200b"; const texts = [...characters].map((text) => `${text}`); - expect(pythonViews(texts)).toEqual(texts.map(xmlView)); + expect(pythonViews(texts)).toEqual(texts.map((text) => xmlView(text))); }); it("keeps leaf text and xml:space-preserved text, reads a plain DOCTYPE as REA does, and refuses an unbound prefix", () => { @@ -48,7 +56,9 @@ it("keeps leaf text and xml:space-preserved text, reads a plain DOCTYPE as REA d '\u0085', ]; const actual = pythonViews(texts) as { root: { children: unknown[] } }[]; - const expected = texts.map(xmlView) as { root: { children: unknown[] } }[]; + const expected = texts.map((text) => xmlView(text)) as { + root: { children: unknown[] }; + }[]; expect(actual.map((view) => view.root.children)).toEqual( expected.map((view) => view.root.children), ); @@ -57,3 +67,11 @@ it("keeps leaf text and xml:space-preserved text, reads a plain DOCTYPE as REA d null, ]); }); + +it("keeps the deepest admitted XML view encodable with the same Python representation", () => { + const policy = desktopViewPolicy({ xml_depth: 494 }); + const text = `${''.repeat(495)}${"".repeat(495)}`; + const expected = xmlView(text, policy); + expect(expected).toMatchObject({ truncated: false }); + expect(pythonViews([text], policy)).toEqual([expected]); +}); diff --git a/tests/boundary/providers/managed/managedMemberComparison.test.ts b/tests/boundary/providers/managed/managedMemberComparison.test.ts index 490527b0e2..191d949347 100644 --- a/tests/boundary/providers/managed/managedMemberComparison.test.ts +++ b/tests/boundary/providers/managed/managedMemberComparison.test.ts @@ -1,3 +1,4 @@ +import { DEFAULT_MANAGED_SNAPSHOT_BYTES } from "../../../../src/domain/managed/managedInspectionLimits.js"; import { truncate, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -24,10 +25,7 @@ import { buildManagedNestedTypeFixture, buildManagedPeFixture, } from "../../../../src/dotnet/ManagedPe.fixture.js"; -import { - MAX_MANAGED_SNAPSHOT_BYTES, - readManagedSnapshot, -} from "../../../../src/dotnet/ManagedSnapshot.js"; +import { readManagedSnapshot } from "../../../../src/dotnet/ManagedSnapshot.js"; import * as boundedFileBytes from "../../../../src/process/BoundedFileBytes.js"; import { primaryAnalysisError } from "../../../../src/domain/analysisErrorBase.js"; import { buildManagedMemberCapacityFixture } from "../../../fixtures/managedMemberCapacity.js"; @@ -907,7 +905,7 @@ describe("managed comparison snapshot admission and cleanup", () => { const rightPath = join(directory, "right.dll"); const fixture = buildManagedPeFixture(); await writeFile(leftPath, Buffer.alloc(0)); - await truncate(leftPath, MAX_MANAGED_SNAPSHOT_BYTES + 1); + await truncate(leftPath, DEFAULT_MANAGED_SNAPSHOT_BYTES + 1); await writeFile(rightPath, fixture); const template = managedPeFixtureTarget(fixture); const read = vi.spyOn(boundedFileBytes, "readBoundedFileBytes"); @@ -928,7 +926,7 @@ describe("managed comparison snapshot admission and cleanup", () => { _tag: "AnalysisResourceConstraintError", operation: "compare_managed_members", reportedLimits: { - max_snapshot_bytes: MAX_MANAGED_SNAPSHOT_BYTES, + max_snapshot_bytes: DEFAULT_MANAGED_SNAPSHOT_BYTES, }, }); expect(read).not.toHaveBeenCalled(); diff --git a/tests/boundary/providers/managed/managedStaticProvider.test.ts b/tests/boundary/providers/managed/managedStaticProvider.test.ts index eb2aa8234a..4af7679749 100644 --- a/tests/boundary/providers/managed/managedStaticProvider.test.ts +++ b/tests/boundary/providers/managed/managedStaticProvider.test.ts @@ -1,3 +1,4 @@ +import { DEFAULT_MANAGED_SNAPSHOT_BYTES } from "../../../../src/domain/managed/managedInspectionLimits.js"; import { createHash } from "node:crypto"; import { truncate, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -15,10 +16,7 @@ import { } from "../../../../src/domain/managed/managedArtifact.js"; import { ManagedStaticProvider } from "../../../../src/dotnet/ManagedStaticProvider.js"; import { buildManagedPeFixture } from "../../../../src/dotnet/ManagedPe.fixture.js"; -import { - MAX_MANAGED_SNAPSHOT_BYTES, - readManagedSnapshot, -} from "../../../../src/dotnet/ManagedSnapshot.js"; +import { readManagedSnapshot } from "../../../../src/dotnet/ManagedSnapshot.js"; import { OwnedFileHandle } from "../../../../src/filesystem/OwnedFileHandle.js"; import { ArtifactResourceScope } from "../../../../src/artifacts/ArtifactResourceScope.js"; import { RegularFileCleanupFailure } from "../../../../src/filesystem/OwnedRegularFile.js"; @@ -26,6 +24,81 @@ import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjec import * as boundedFileBytes from "../../../../src/process/BoundedFileBytes.js"; import { primaryAnalysisError } from "../../../../src/domain/analysisErrorBase.js"; +it.each([ + "inspect_managed_artifact", + "inspect_managed_members", + "inspect_managed_native_boundaries", +] as const)( + "recovers the same managed target with per-call byte budgets: %s", + async (operation) => { + const root = await createTestTempDirectory("rea-managed-budget-recovery-"); + const path = join(root, "fixture.dll"); + const bytes = buildManagedPeFixture(); + await writeFile(path, bytes); + const target = await parseBinaryTarget(path); + if (!target.ok) throw target.error; + const client = new ManagedStaticProvider().createClient(target.value); + try { + const refused = await client.execute(operation, { + max_snapshot_bytes: bytes.length - 1, + }); + expect(refused).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + reportedLimits: { max_snapshot_bytes: bytes.length - 1 }, + }, + }); + if (refused.ok) throw new Error("Expected snapshot admission refusal"); + expect(projectAnalysisError(refused.error).remediation?.action).toContain( + "max_snapshot_bytes", + ); + const partial = await client.execute(operation, { + max_snapshot_bytes: bytes.length, + max_metadata_representation_bytes: 1, + }); + expect(partial.ok).toBe(true); + if (!partial.ok) throw partial.error; + expect(partial.value.result).toMatchObject({ + coverage: { + issues: [ + expect.objectContaining({ + code: "resource-limit", + detail: expect.stringContaining( + "max_metadata_representation_bytes=1", + ), + }), + ], + }, + }); + const recovered = await client.execute(operation, { + max_snapshot_bytes: bytes.length, + max_metadata_representation_bytes: 1024 * 1024, + }); + expect(recovered.ok).toBe(true); + if (!recovered.ok) throw recovered.error; + expect(recovered.value.result).toMatchObject({ + artifact: { sha256: createHash("sha256").update(bytes).digest("hex") }, + coverage: { state: "complete" }, + }); + // The warmed snapshot must not bypass a later caller's smaller admission. + expect( + await client.execute(operation, { + max_snapshot_bytes: bytes.length - 1, + }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); + expect( + await client.execute(operation, { max_snapshot_bytes: 0 }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); + } finally { + await client.close(); + } + }, +); + it.each( [false, true] .flatMap((warm) => @@ -205,7 +278,7 @@ describe("managed static snapshot capacity failure", () => { ); const path = join(directory, "sparse.exe"); await writeFile(path, Buffer.alloc(0)); - await truncate(path, MAX_MANAGED_SNAPSHOT_BYTES + 1); + await truncate(path, DEFAULT_MANAGED_SNAPSHOT_BYTES + 1); const resources = new ArtifactResourceScope(); const originalClose = OwnedFileHandle.prototype.close; let permitClose = false; diff --git a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts index 4e35f7fb78..b5177f8b29 100644 --- a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts +++ b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts @@ -18,6 +18,7 @@ import { z } from "zod"; import { AnalysisCancelledError, AnalysisCapabilityUnavailableError, + AnalysisResourceConstraintError, AnalysisTimeoutError, } from "../../../../src/domain/analysisErrorCore.js"; import { ProviderCleanupError } from "../../../../src/domain/providerCleanupError.js"; @@ -29,7 +30,8 @@ import { nativeCallObservationInputSchema } from "../../../../src/domain/native/ import { nativeCallPartialObservationSchema } from "../../../../src/domain/native/nativeCallPartialObservation.js"; import type { NativeCallTracer } from "../../../../src/native/LldbCallTracer.js"; import { LldbCallTracer } from "../../../../src/native/LldbCallTracer.js"; -import { err, ok } from "../../../../src/domain/result.js"; +import { LldbReadbackFiles } from "../../../../src/native/LldbReadbackFiles.js"; +import { err, ok, type Result } from "../../../../src/domain/result.js"; import { NativeCommandFailure } from "../../../../src/native/CommandRunner.js"; import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; import { createDeferred } from "../../../fixtures/binarySession.js"; @@ -184,6 +186,134 @@ const fixtureTracer = ( }; }; +describe("LLDB capture admission and readback", () => { + it("refuses an unrepresentable combined result before reading while retaining journal facts", async () => { + const original = LldbReadbackFiles.prototype.read; + const unexpectedRead = vi.fn(() => { + throw new Error( + "oversized result must be refused before allocation/read", + ); + }); + const read = vi + .spyOn(LldbReadbackFiles.prototype, "read") + .mockImplementation(async function ( + this: LldbReadbackFiles, + path: string, + consume: (handle: FileHandle) => Promise, + ): Promise> { + if (!path.endsWith("result.json")) + return original.call(this, path, consume) as Promise< + Result + >; + return ok( + await consume({ + stat: async () => ({ size: Number.MAX_SAFE_INTEGER }), + readFile: unexpectedRead, + read: unexpectedRead, + } as unknown as FileHandle), + ); + }); + const tracer = new LldbCallTracer( + {}, + async (_executable, arguments_) => { + const { config } = await configFromArguments(arguments_); + await writeFile( + config.observation_path, + `${JSON.stringify({ kind: "event", event: EVENT })}\n`, + ); + await writeFile(config.stdout_capture_path, "kept"); + await writeFile(config.stderr_capture_path, ""); + return { + kind: "exited", + targetLaunched: false, + exitCode: 0, + output: "", + targetIdentity: undefined, + cleanupFailure: undefined, + }; + }, + async () => ok(fakeTool), + ); + try { + const result = await tracer.trace(request); + if (result.ok) throw new Error("Expected runtime representation refusal"); + expect(result.error).toBeInstanceOf(AnalysisResourceConstraintError); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "resource_constraint", + retryable: false, + remediation: { + action: expect.stringContaining("capture_limits.trace_bytes"), + }, + }); + expect(result.error.partialObservation).toMatchObject({ + events: [EVENT], + process: { stdout: { text: "kept" } }, + }); + expect(unexpectedRead).not.toHaveBeenCalled(); + } finally { + read.mockRestore(); + await tracer.close(); + } + }); + + it("applies selected output retention to bridge configuration and completed readback", async () => { + const text = "selected output"; + const tracer = new LldbCallTracer( + {}, + async (_executable, arguments_) => { + const { configPath, config } = await configFromArguments(arguments_); + const configuration: unknown = JSON.parse( + await readFile(configPath, "utf8"), + ); + expect(configuration).toMatchObject({ + capture_limits: { output_bytes: 8, trace_bytes: 8388608 }, + }); + await writeFile(config.stdout_capture_path, text); + await writeFile(config.stderr_capture_path, ""); + await writeFile( + config.result_path, + JSON.stringify({ + ...successfulTracerOutput, + target_output: { + ...successfulTracerOutput.target_output, + stdout_bytes: Buffer.byteLength(text), + }, + }), + ); + return { + kind: "exited", + targetLaunched: false, + exitCode: 0, + output: "", + targetIdentity: undefined, + cleanupFailure: undefined, + }; + }, + async () => ok(fakeTool), + ); + try { + const selected = { + ...request, + input: nativeCallObservationInputSchema.parse({ + ...request.input, + capture_limits: { output_bytes: 8 }, + }), + }; + expect(selected.input.capture_limits).toEqual({ output_bytes: 8 }); + const result = await tracer.trace(selected); + if (!result.ok) throw result.error; + expect(result.value.stdout).toEqual({ + text: "selected", + bytes: 15, + truncated: true, + complete: true, + }); + } finally { + await tracer.close(); + } + }); +}); + describe("LLDB completed trace retention through finalization", () => { it.each([ "after launch", diff --git a/tests/boundary/providers/native/lldbRetainedObservations.test.ts b/tests/boundary/providers/native/lldbRetainedObservations.test.ts index 7fe6273e07..4f921759f7 100644 --- a/tests/boundary/providers/native/lldbRetainedObservations.test.ts +++ b/tests/boundary/providers/native/lldbRetainedObservations.test.ts @@ -4,7 +4,6 @@ import { err, ok, type Result } from "../../../../src/domain/result.js"; import { nativeCallPartialObservationSchema } from "../../../../src/domain/native/nativeCallPartialObservation.js"; import { - LLDB_OBSERVATION_JOURNAL_MAX_BYTES, parseLldbObservationJournal, projectLldbPartialObservation, readBoundedPrefix, @@ -168,13 +167,10 @@ describe("LLDB retained observation read failures", () => { const valid = JSON.stringify({ kind: "event", event }); const malformed = '{"kind":"unknown"}\n'; const malformedRows = Math.floor( - (LLDB_OBSERVATION_JOURNAL_MAX_BYTES - valid.length * 2 - 2) / - malformed.length, + (8 * 1024 * 1024 - valid.length * 2 - 2) / malformed.length, ); const journal = `${valid}\n${malformed.repeat(malformedRows)}${valid}\n`; - expect(Buffer.byteLength(journal)).toBeLessThanOrEqual( - LLDB_OBSERVATION_JOURNAL_MAX_BYTES, - ); + expect(Buffer.byteLength(journal)).toBeLessThanOrEqual(8 * 1024 * 1024); const partial = observation("tracer-failure", journal); expect(partial.events).toEqual([event, event]); expect(partial.limitations.length).toBeLessThanOrEqual(6); @@ -206,6 +202,43 @@ describe("LLDB retained observation read failures", () => { ); }); + it("reads the selected partial capture and journal budgets instead of fixed defaults", async () => { + const source = Buffer.from("retained-output"); + const first = await readPartialCapture( + "stdout", + readbackFilesFailingAfter(source, source.length, new Error("unused")), + 8, + ); + const raised = await readPartialCapture( + "stdout", + readbackFilesFailingAfter(source, source.length, new Error("unused")), + 32, + ); + if (!first.ok || !raised.ok) + throw new Error("Expected retained output reads"); + expect(first.value.capture.text).toBe("retained"); + expect(raised.value.capture.text).toBe("retained-output"); + const row = `${JSON.stringify({ kind: "event", event })}\n`; + const journal = Buffer.from(row.repeat(2)); + const limited = await readLldbObservationJournal( + "journal", + readbackFilesFailingAfter(journal, journal.length, new Error("unused")), + Buffer.byteLength(row), + ); + const complete = await readLldbObservationJournal( + "journal", + readbackFilesFailingAfter(journal, journal.length, new Error("unused")), + journal.length, + ); + if (!limited.ok || !complete.ok) + throw new Error("Expected retained journal reads"); + expect(limited.value.parsed.events).toEqual([event]); + expect(complete.value.parsed.events).toEqual([event, event]); + expect(limited.value.limitations).toContainEqual( + expect.stringContaining("capture_limits.trace_bytes"), + ); + }); + it("retains completed journal rows and the exact cause after a later read fails", async () => { const completeRow = `${JSON.stringify({ kind: "event", event })}\n`; const source = Buffer.from(`${completeRow}{"kind":"event"`); diff --git a/tests/boundary/providers/native/nativeCallObservation.test.ts b/tests/boundary/providers/native/nativeCallObservation.test.ts index 50194f501d..2178e9ae33 100644 --- a/tests/boundary/providers/native/nativeCallObservation.test.ts +++ b/tests/boundary/providers/native/nativeCallObservation.test.ts @@ -13,10 +13,7 @@ import type { AnalysisError } from "../../../../src/domain/analysisErrorBase.js" import { EvidenceIntegrityError } from "../../../../src/domain/evidenceErrors.js"; import { nativeCallPartialObservationSchema } from "../../../../src/domain/native/nativeCallPartialObservation.js"; import { nativeCallObservationResultSchema } from "../../../../src/domain/native/nativeCallObservation.js"; -import { - NATIVE_CALL_BREAKPOINT_LOCATION_BUDGET_LIMITATION, - NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION, -} from "../../../../src/native/NativeCallObservation.js"; +import { NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION } from "../../../../src/native/NativeCallObservation.js"; import { err, ok, type Result } from "../../../../src/domain/result.js"; import type { NativeCallTrace, @@ -304,7 +301,7 @@ describe("observe_native_calls projection: keeps locations within the metadata b }); }); -describe("observe_native_calls projection: reports partial observation coverage", () => { +describe("observe_native_calls projection: bounded breakpoint metadata", () => { it("retains normal exit and location counts when breakpoint metadata is bounded", async () => { const observed = await observe( new FixtureTracer( @@ -326,12 +323,16 @@ describe("observe_native_calls projection: reports partial observation coverage" terminated: true, }), ), - { breakpoints: BREAKPOINTS }, + { + breakpoints: BREAKPOINTS, + capture_limits: { breakpoint_location_bytes: 1024 }, + }, ); if (!observed.ok) throw observed.error; const result = nativeCallObservationResultSchema.parse( observed.value.result, ); + expect(result.capture_limits?.breakpoint_location_bytes).toBe(1024); expect(result.process).toMatchObject({ outcome: "exited", exit_status: 0, @@ -346,15 +347,19 @@ describe("observe_native_calls projection: reports partial observation coverage" resource_limit_reached: false, unresolved_breakpoints: [], }); - expect(result.limitations).toContain( - NATIVE_CALL_BREAKPOINT_LOCATION_BUDGET_LIMITATION, + expect(result.limitations).toContainEqual( + expect.stringContaining( + "capture_limits.breakpoint_location_bytes (1024 bytes)", + ), ); expect(result.limitations).not.toContain( NATIVE_CALL_BREAKPOINT_LOCATION_LIST_INCOMPLETE_LIMITATION, ); expect(result.limitations.join(" ")).not.toContain("process was killed"); }); +}); +describe("observe_native_calls projection: reports partial observation coverage", () => { it("marks event limits, unresolved breakpoints and unconfirmed exits", async () => { const tracer = new FixtureTracer( ok({ @@ -413,7 +418,7 @@ describe("observe_native_calls projection: reports partial observation coverage" /Breakpoints 1.*matched no code.*calls could not be observed/iu, ), expect.stringMatching( - /1048576 bytes.*per stream.*counted but not kept.*no runtime override/iu, + /capture_limits\.output_bytes.*1048576.*per stream.*counted but not kept.*earlier omitted output cannot be recovered/iu, ), expect.stringMatching( /could not confirm.*process 4242 exited.*check for it/iu, diff --git a/tests/boundary/rust/rustBinaryBudget.test.ts b/tests/boundary/rust/rustBinaryBudget.test.ts index 6368e01369..a593ed7bc9 100644 --- a/tests/boundary/rust/rustBinaryBudget.test.ts +++ b/tests/boundary/rust/rustBinaryBudget.test.ts @@ -1,49 +1,70 @@ -import { truncate, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { RustBinaryProvider } from "../../../src/rust/RustBinaryProvider.js"; -import { RUST_BINARY_INPUT_BYTES } from "../../../src/rust/RustAuditableReader.js"; -import { createRustHashOverflowImage } from "../../fixtures/rust/image.js"; +import { + createRustImage, + createRustHashOverflowImage, +} from "../../fixtures/rust/image.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -it("refuses extra rustc paths without asking the caller to wait", async () => { +it("reports a recoverable compiler prefix without refusing the dependency report", async () => { const root = await createTestTempDirectory("rea-rust-hash-budget-"); const path = join(root, "many-hashes.elf"); await writeFile(path, createRustHashOverflowImage()); const result = await new RustBinaryProvider().inspect({ path }); - if (result.ok) throw new Error("Expected a hash-capacity refusal"); - const projected = projectAnalysisError(result.error); - expect(projected).toMatchObject({ - retryable: false, - details: { - resource: "capacity", - reported_limits: { maximum_rustc_hashes: 64 }, - }, + if (!result.ok) throw result.error; + expect(result.value.rustc_commit?.coverage).toMatchObject({ + reason: "max_rustc_hashes", + effective_limit: 64, }); - expect(projected.remediation.action).toContain("rustc commit-path inventory"); - expect(projected.remediation.action).not.toMatch( - /Wait for an active analysis/u, + expect(result.value.auditable_metadata?.packages).toHaveLength(2); + expect(result.value.limitations).toContainEqual( + expect.stringContaining("--max-rustc-hashes"), ); }); -it("refuses an oversized Rust input without evidence-reply advice", async () => { +it("recovers the same complete Rust image after input and report budget changes", async () => { const root = await createTestTempDirectory("rea-rust-input-budget-"); - const path = join(root, "large.elf"); - await writeFile(path, Buffer.from([0x7f, 0x45, 0x4c, 0x46])); - await truncate(path, RUST_BINARY_INPUT_BYTES + 1); - const result = await new RustBinaryProvider().inspect({ path }); - if (result.ok) throw new Error("Expected an input-size refusal"); - const projected = projectAnalysisError(result.error); - expect(projected).toMatchObject({ - retryable: false, - details: { - resource: "file-size", - reported_limits: { maximum_input_bytes: RUST_BINARY_INPUT_BYTES }, - }, - }); - expect(projected.remediation.action).toContain("does not admit this input"); - expect(projected.remediation.action).not.toMatch( - /file-size allowance for the evidence reply/u, - ); + const path = join(root, "selected.elf"); + await writeFile(path, createRustImage({ format: "elf" })); + const provider = new RustBinaryProvider(); + try { + const smallInput = await provider.inspect({ + path, + producer_limits: { input_bytes: 4095 }, + }); + if (smallInput.ok) throw new Error("Expected input admission refusal"); + const projectedInput = projectAnalysisError(smallInput.error); + expect(projectedInput).toMatchObject({ + code: "resource_constraint", + details: { reported_limits: { maximum_input_bytes: 4095 } }, + }); + expect(projectedInput.remediation.action).toContain( + "producer_limits.input_bytes", + ); + const smallReport = await provider.inspect({ + path, + producer_limits: { input_bytes: 4096, output_bytes: 16 }, + }); + if (smallReport.ok) throw new Error("Expected report admission refusal"); + const projectedReport = projectAnalysisError(smallReport.error); + expect(projectedReport).toMatchObject({ + code: "resource_constraint", + details: { reported_limits: { maximum_output_bytes: 16 } }, + }); + expect(projectedReport.remediation.action).toContain( + "producer_limits.output_bytes", + ); + const recovered = await provider.inspect({ + path, + producer_limits: { input_bytes: 4096, output_bytes: 16384 }, + }); + if (!recovered.ok) throw recovered.error; + expect(recovered.value.auditable_metadata?.packages).toHaveLength(2); + expect(recovered.value.artifact.bytes).toBe(4096); + } finally { + await provider.close(); + } }); diff --git a/tests/boundary/rust/rustBinaryImage.test.ts b/tests/boundary/rust/rustBinaryImage.test.ts index dfb97e249a..330fc2a279 100644 --- a/tests/boundary/rust/rustBinaryImage.test.ts +++ b/tests/boundary/rust/rustBinaryImage.test.ts @@ -341,16 +341,23 @@ it("reads a rustc path from a read-only ELF section whose name exceeds 64 bytes" }); }); -it("refuses read-only data with more distinct rustc paths than it retains", () => { +it("retains dependency facts when compiler paths exceed the selected prefix", () => { const bytes = createRustHashOverflowImage(); - expect(() => readRustBinaryImage(bytes)).toThrow( - expect.objectContaining({ - constraint: { - resource: "capacity", - reported_limits: { maximum_rustc_hashes: 64 }, - }, - }), - ); + const partial = readRustBinaryImage(bytes); + expect(partial.auditable_metadata?.packages.map(({ name }) => name)).toEqual([ + "itoa", + "sample", + ]); + expect(partial.rustc_commit?.hashes).toHaveLength(64); + expect(partial.rustc_commit?.coverage).toEqual({ + status: "partial", + reason: "max_rustc_hashes", + effective_limit: 64, + }); + const recovered = readRustBinaryImage(bytes, 65); + expect(recovered.rustc_commit?.hashes).toHaveLength(65); + expect(recovered.rustc_commit).not.toHaveProperty("coverage"); + expect(recovered.auditable_metadata).toEqual(partial.auditable_metadata); }); it.each([ diff --git a/tests/boundary/signals/recordingProvider.test.ts b/tests/boundary/signals/recordingProvider.test.ts index 9884f00423..2caae452ef 100644 --- a/tests/boundary/signals/recordingProvider.test.ts +++ b/tests/boundary/signals/recordingProvider.test.ts @@ -213,6 +213,42 @@ it("refuses new acquisition after provider shutdown", async () => { expect(f.calls()).toBe(0); }); +it("recovers the same recording with caller producer budgets and preserves inherited enforcement", async () => { + const f = await fixture(); + const refused = await f.service.decode({ + path: f.path, + decoders: "uart", + producer_limits: { input_bytes: 1 }, + }); + expect(refused).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); + if (refused.ok) throw new Error("Expected input-budget refusal"); + expect(analysisErrorRemediationAction(refused.error)).toContain( + "producer_limits.input_bytes", + ); + const selected = { + input_bytes: 1024, + output_bytes: 2048, + command_timeout_ms: 1000, + address_space_bytes: 2147483648, + file_bytes: 2097152, + cpu_seconds: 45, + }; + const recovered = await f.service.decode({ + path: f.path, + decoders: "uart", + producer_limits: selected, + }); + if (!recovered.ok) throw recovered.error; + const report = signalRecordingSchema.parse(recovered.value.normalized_result); + expect(report.annotations_text).not.toBe(""); + expect(report.selection.producer_limits).toEqual(selected); + expect(report.limits).toEqual({ ...selected, file_bytes: 1048576 }); + expect(report.resource_limiter.arguments).toContain("--cpu=45:45"); +}); + it("pages literal output without changing decoder arguments or normalizing text", async () => { const f = await fixture("many"); for (const [options, first, count] of [ @@ -1297,10 +1333,11 @@ it.each([ throw new Error("Expected command-output allowance refusal"); expect(result.error).toMatchObject({ resource: "memory", + reportedLimits: { output_bytes: 4194304 }, retryable: false, }); expect(analysisErrorRemediationAction(result.error)).toContain( - "The retained command-output allowance", + "producer_limits.output_bytes", ); expect(analysisErrorRemediationAction(result.error)).not.toMatch( /Review the reported worker memory limits/u, diff --git a/tests/composition/reverseEngineering.test.ts b/tests/composition/reverseEngineering.test.ts index 2b6f7f5cb1..2f2a5d5042 100644 --- a/tests/composition/reverseEngineering.test.ts +++ b/tests/composition/reverseEngineering.test.ts @@ -7,10 +7,7 @@ import { PassThrough } from "node:stream"; import { describe, expect, it } from "vitest"; -import { - digestFile, - REVERSE_COMMAND_OUTPUT_LIMITATION, -} from "../../src/application/reverse/ReverseEngineeringService.js"; +import { digestFile } from "../../src/application/reverse/ReverseEngineeringService.js"; import { TestReverseEngineeringService as ReverseEngineeringService } from "../support/reverseEngineeringService.js"; import { providerSnapshot } from "../support/reverseEngineeringCompositionFixtures.js"; import { createReverseEngineeringService } from "../../src/composition/reverseEngineering.js"; @@ -34,6 +31,70 @@ it("documents the shared combined capture limit for both command tools", () => { expect(contract.description).toContain("command_timeout_ms"); }); +it.each(["objdump", "rizin"] as const)( + "recovers a bounded %s capture after the caller raises output admission", + async (provider) => { + const directory = await mkdtemp( + join(tmpdir(), "rea-reverse-output-admission-"), + ); + const path = join(directory, "input.bin"); + await writeFile(path, "selected artifact"); + const profile = { + executable: process.execPath, + arguments: ["-e", "process.stdout.write('complete-output')"], + identity: { + id: "fixture-command", + name: "Fixture command", + version: "1", + }, + }; + const service = new ReverseEngineeringService({ + commands: { objdump: () => profile, rizin: () => profile }, + }); + const inspect = (max_output_bytes: number) => + provider === "objdump" + ? service.inspectWithObjdump({ + operation: "architectures", + max_output_bytes, + }) + : service.executeRizinCommand({ + path, + command: "iI", + max_output_bytes, + }); + try { + const limited = await inspect(8); + if (!limited.ok) throw limited.error; + expect(limited.value.normalized_result).toMatchObject({ + stdout: "complete", + output_budget_bytes: 8, + output_budget_source: "request", + output_truncated: true, + completion_status: "unknown", + }); + expect(limited.value.parameters).toMatchObject({ max_output_bytes: 8 }); + expect(limited.value.limitations).toContainEqual( + expect.stringContaining("max_output_bytes=8"), + ); + const recovered = await inspect(32); + if (!recovered.ok) throw recovered.error; + expect(recovered.value.normalized_result).toMatchObject({ + stdout: "complete-output", + output_budget_bytes: 32, + output_budget_source: "request", + output_truncated: false, + completion_status: "complete", + }); + expect(recovered.value.parameters).toMatchObject({ + max_output_bytes: 32, + }); + } finally { + await service.close(); + await rm(directory, { recursive: true, force: true }); + } + }, +); + describe("reverse engineering command services", () => { it("uses compatible local DWARF link flags and disables debuginfod through the environment", async () => { const directory = await mkdtemp(join(tmpdir(), "rea-reverse-tools-")); @@ -177,8 +238,8 @@ describe("reverse engineering command services", () => { output_truncated: true, completion_status: "unknown", }); - expect(result.value.limitations).toContain( - REVERSE_COMMAND_OUTPUT_LIMITATION, + expect(result.value.limitations).toContainEqual( + expect.stringContaining("max_output_bytes=16777216"), ); } } finally { diff --git a/tests/composition/reverseEngineeringAdmission.test.ts b/tests/composition/reverseEngineeringAdmission.test.ts index 3ca64cf576..a6ea37fe6a 100644 --- a/tests/composition/reverseEngineeringAdmission.test.ts +++ b/tests/composition/reverseEngineeringAdmission.test.ts @@ -6,101 +6,105 @@ import { expect, it } from "vitest"; import { TestReverseEngineeringService } from "../support/reverseEngineeringService.js"; -it("bounds concurrent reverse-engineering provider calls", async () => { - const directory = await mkdtemp(join(tmpdir(), "rea-reverse-capacity-")); - const path = join(directory, "fixture.bin"); - await writeFile(path, "fixture"); - let release!: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - let calls = 0; - let mode: "success" | "failure" | "timeout" = "success"; - const service = new TestReverseEngineeringService({ - run: async () => { - calls += 1; - if (mode === "failure") - throw Object.assign(new Error("provider failed"), { code: "ENOENT" }); - if (mode === "timeout") - throw Object.assign(new Error("provider timed out"), { - code: "ETIMEDOUT", - }); - await gate; - return { stdout: "", stderr: "" }; - }, - }); - - try { - const active = Array.from({ length: 8 }, () => - service.inspectWithObjdump({ path, operation: "file_headers" }), - ); - const preAborted = new AbortController(); - preAborted.abort(); - await expect( - service.inspectWithObjdump( - { path, operation: "file_headers" }, - { signal: preAborted.signal }, - ), - ).resolves.toMatchObject({ - ok: false, - error: { _tag: "AnalysisCancelledError" }, +it.each([ + { budget: undefined, capacity: 8 }, + { budget: 40 * 1024 * 1024, capacity: 3 }, +])( + "bounds provider concurrency and aggregate capture admission at budget $budget", + async ({ budget, capacity }) => { + const directory = await mkdtemp(join(tmpdir(), "rea-reverse-capacity-")); + const path = join(directory, "fixture.bin"); + await writeFile(path, "fixture"); + const input = { + path, + operation: "file_headers" as const, + ...(budget === undefined ? {} : { max_output_bytes: budget }), + }; + let release!: () => void; + const gate = new Promise((resolve) => { + release = resolve; }); - await expect( - service.inspectWithObjdump({ path, operation: "file_headers" }), - ).resolves.toMatchObject({ - ok: false, - error: { - _tag: "AnalysisResourceConstraintError", - resource: "transport", - reportedLimits: { - max_concurrent_operations: 8, - active_operations: 8, - }, + let calls = 0; + let mode: "success" | "failure" | "timeout" = "success"; + const service = new TestReverseEngineeringService({ + run: async () => { + calls += 1; + if (mode === "failure") + throw Object.assign(new Error("provider failed"), { code: "ENOENT" }); + if (mode === "timeout") + throw Object.assign(new Error("provider timed out"), { + code: "ETIMEDOUT", + }); + await gate; + return { stdout: "", stderr: "" }; }, }); - release(); - const results = await Promise.all(active); - expect(results.every((result) => result.ok)).toBe(true); - expect(calls).toBe(8); - mode = "failure"; - const failed = await Promise.all( - Array.from({ length: 8 }, () => - service.inspectWithObjdump({ path, operation: "file_headers" }), - ), - ); - expect(failed.every((result) => !result.ok)).toBe(true); + try { + const active = Array.from({ length: capacity }, () => + service.inspectWithObjdump(input), + ); + const preAborted = new AbortController(); + preAborted.abort(); + await expect( + service.inspectWithObjdump(input, { signal: preAborted.signal }), + ).resolves.toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + await expect(service.inspectWithObjdump(input)).resolves.toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + resource: "transport", + reportedLimits: { + max_concurrent_operations: 8, + active_operations: capacity, + max_active_capture_bytes: 134217728, + }, + }, + }); + release(); + const results = await Promise.all(active); + expect(results.every((result) => result.ok)).toBe(true); + expect(calls).toBe(capacity); - mode = "timeout"; - const timedOut = await Promise.all( - Array.from({ length: 8 }, () => - service.inspectWithObjdump({ path, operation: "file_headers" }), - ), - ); - expect(timedOut.every((result) => !result.ok)).toBe(true); - - const aborted = new AbortController(); - aborted.abort(); - const cancelled = await Promise.all( - Array.from({ length: 8 }, () => - service.inspectWithObjdump( - { path, operation: "file_headers" }, - { signal: aborted.signal }, + mode = "failure"; + const failed = await Promise.all( + Array.from({ length: capacity }, () => + service.inspectWithObjdump(input), ), - ), - ); - expect(cancelled.every((result) => !result.ok)).toBe(true); + ); + expect(failed.every((result) => !result.ok)).toBe(true); - mode = "success"; - await expect( - service.inspectWithObjdump({ path, operation: "file_headers" }), - ).resolves.toMatchObject({ ok: true }); - } finally { - release(); - await service.close(); - await rm(directory, { recursive: true, force: true }); - } -}); + mode = "timeout"; + const timedOut = await Promise.all( + Array.from({ length: capacity }, () => + service.inspectWithObjdump(input), + ), + ); + expect(timedOut.every((result) => !result.ok)).toBe(true); + + const aborted = new AbortController(); + aborted.abort(); + const cancelled = await Promise.all( + Array.from({ length: capacity }, () => + service.inspectWithObjdump(input, { signal: aborted.signal }), + ), + ); + expect(cancelled.every((result) => !result.ok)).toBe(true); + + mode = "success"; + await expect(service.inspectWithObjdump(input)).resolves.toMatchObject({ + ok: true, + }); + } finally { + release(); + await service.close(); + await rm(directory, { recursive: true, force: true }); + } + }, +); it("preserves the exact caller reason when objdump admission is already aborted", async () => { const directory = await mkdtemp(join(tmpdir(), "rea-reverse-preabort-")); diff --git a/tests/composition/reverseEngineeringEdgeCases.test.ts b/tests/composition/reverseEngineeringEdgeCases.test.ts index 425d7cf559..2ed095f6df 100644 --- a/tests/composition/reverseEngineeringEdgeCases.test.ts +++ b/tests/composition/reverseEngineeringEdgeCases.test.ts @@ -14,10 +14,7 @@ import { AnalysisCapabilityUnavailableError, AnalysisResourceConstraintError, } from "../../src/domain/analysisErrorCore.js"; -import { - ReverseEngineeringService as CoreReverseEngineeringService, - REVERSE_COMMAND_OUTPUT_LIMITATION, -} from "../../src/application/reverse/ReverseEngineeringService.js"; +import { ReverseEngineeringService as CoreReverseEngineeringService } from "../../src/application/reverse/ReverseEngineeringService.js"; import { createReverseEngineeringCommandPort } from "../../src/composition/reverseEngineering.js"; import { objdumpCommand } from "../../src/objdump/ObjdumpCommand.js"; import { @@ -78,7 +75,7 @@ it.each(["objdump", "rizin"] as const)( artifact_changed: null, }, limitations: expect.arrayContaining([ - REVERSE_COMMAND_OUTPUT_LIMITATION, + expect.stringContaining("max_output_bytes=16777216"), ]), }, }, diff --git a/tests/fixtures/rust/image.ts b/tests/fixtures/rust/image.ts index 90bab317b2..2e3dbdf91d 100644 --- a/tests/fixtures/rust/image.ts +++ b/tests/fixtures/rust/image.ts @@ -294,15 +294,14 @@ const writeMachoSegment = ({ /** Read-only ELF data exceeding the distinct rustc-hash retention capacity. */ export const createRustHashOverflowImage = (): Buffer => { - const bytes = createRustImage({ format: "elf", omitSection: true }); - // Without .dep-v0 the read-only section header is the second one, at 384. - // Section names end before byte 800, so the hash block starts there. - bytes.writeBigUInt64LE(800n, 384 + 24); - bytes.writeBigUInt64LE(3296n, 384 + 32); - bytes.fill(0, 800); + const bytes = Buffer.alloc(8192); + createRustImage({ format: "elf" }).copy(bytes); + // Preserve dependency bytes and names; move only the read-only hash section. + bytes.writeBigUInt64LE(4096n, 448 + 24); + bytes.writeBigUInt64LE(65n * 48n, 448 + 32); for (let index = 0; index < 65; index += 1) { const hash = index.toString(16).padStart(40, "0"); - bytes.write(`/rustc/${hash}/`, 800 + index * 48, "latin1"); + bytes.write(`/rustc/${hash}/`, 4096 + index * 48, "latin1"); } return bytes; }; diff --git a/tests/support/browser/analyzeWebBundleFixture.ts b/tests/support/browser/analyzeWebBundleFixture.ts index db1443f908..fcf215d4fb 100644 --- a/tests/support/browser/analyzeWebBundleFixture.ts +++ b/tests/support/browser/analyzeWebBundleFixture.ts @@ -18,6 +18,6 @@ export const analyzeWebBundleFixture = ( return buildCapturedWebBundleAnalysis( inspection, sourceMaps, - sourceScripts.map(analyzeCapturedWebScript), + sourceScripts.map((script) => analyzeCapturedWebScript(script)), ); };