From 60149282fcfc05ac32fbc94434c107710120da6f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 1 Nov 2026 00:15:28 +0800 Subject: [PATCH] test: strengthen integration oracles and verification (#8295) * test: strengthen integration oracles and verification * test: bound lifecycle gates and strengthen workflow checks * test: verify complete paged and extracted results * test: preserve bytecode and cancellation evidence * test: strengthen retained evidence and fixture cleanup * test: require cross-projection bridge identities * test: retain fixture owners through cleanup failures * test: verify graph examples and project selectors * test: tighten provider and lifecycle regressions * test: consolidate hex coverage and bound cutter cleanup * test: bound subprocess output waits and strengthen graph checks * test: check mcp recovery and preserve cleanup owners * test: verify structured mcp argument failures * test: preserve artifact cleanup and gdb limit evidence * test: preserve evidence identities and bound provider gates * test: retain browser and comparison cleanup owners * test: bound startup and strengthen fixture oracles * test: bound lifecycle gates and verify recovered evidence * test: verify public outcomes and lifecycle ownership * test: assert precise evidence and release lifecycle gates * test: verify selected facts and bound cleanup waits * test: check retained evidence and decoded records * test: verify final reports and retain cleanup ownership * test: bound reviewed startup gates and retain cleanup * test: verify PE errors and preserve MCP cleanup * test: preserve MCP fixture cleanup ownership * test: release provider gates before late startup --- .github/workflows/real-browser.yml | 5 + .github/workflows/real-packet-capture.yml | 2 +- package.json | 2 +- scripts/ci/require-provider-tests.mjs | 3 + scripts/ci/scopes.mjs | 17 + scripts/fixtures/browser-request-site.mjs | 6 + scripts/lib/browser-request-e2e.mjs | 21 +- scripts/run-development-tests.mjs | 74 ++- scripts/run-exclusive-command.mjs | 30 +- scripts/verify-pcap.mjs | 4 +- src/apkSignature/ApkSignatureProvider.test.ts | 62 -- src/apkSignature/apkV1Signers.test.ts | 2 +- src/apktool/ApktoolCommandRecovery.test.ts | 31 +- src/application/Doctor.lifecycle.test.ts | 54 +- src/application/Setup.configuration.test.ts | 10 +- .../analysisView/AnalysisViewService.test.ts | 26 +- .../analysisView/BrowserScenarioViews.test.ts | 35 +- .../analysisView/evidenceFileSource.test.ts | 6 +- .../binary/AnalysisProviderRegistry.test.ts | 199 ++++--- .../binary/BinarySessionLiveness.test.ts | 287 +++++---- .../binary/DeferredAnalysisClient.test.ts | 70 ++- .../binary/LazyAnalysisProvider.test.ts | 32 +- .../BinaryLayoutService.test.ts | 17 +- .../PeCertificatesService.test.ts | 14 - .../PeImportsService.test.ts | 14 - .../RecordedCrashService.test.ts | 17 +- .../environment/environmentBuild.test.ts | 13 +- .../environment/environmentState.test.ts | 1 + src/application/erlang/BeamService.test.ts | 15 - ...JavaScriptSemanticGraphSeedQueries.test.ts | 14 +- .../sqlite/SqliteDatabaseService.test.ts | 31 - src/artifacts/HexRecordArtifactReader.test.ts | 92 +-- src/artifacts/Uf2ArtifactReader.test.ts | 71 +-- src/artifacts/ZipArtifactReader.test.ts | 21 - src/browser/CdpSafeMetadata.test.ts | 2 - src/browser/PlaywrightScenarioRequest.test.ts | 82 +-- src/browser/history/SazMetadata.test.ts | 46 +- src/cliJsonPaths.test.ts | 6 +- src/contracts/analysisDetail.test.ts | 103 +++- src/doctorRuntime.test.ts | 46 +- src/domain/analysisView/analysisView.test.ts | 10 +- .../environmentManifestView.test.ts | 3 + src/domain/callPath.test.ts | 1 + src/domain/electron/electronFuses.test.ts | 10 +- src/domain/evidenceCompletionLedger.test.ts | 12 +- src/domain/explicitSensitiveFailure.test.ts | 10 +- src/domain/firmware/espImage.test.ts | 76 +-- src/domain/firmware/hexRecords.test.ts | 15 - src/domain/functionComparison.test.ts | 12 +- ...tSemanticAnalysis.builtin_defaults.test.ts | 6 +- .../javascriptSemanticAnalysis.calls.test.ts | 111 ++-- ...avascriptSemanticAnalysis.dataflow.test.ts | 6 +- ...criptSemanticAnalysis.fingerprints.test.ts | 2 + ...Analysis.initialization_boundaries.test.ts | 7 +- ...vascriptSemanticAnalysis.structure.test.ts | 18 +- .../javascriptSemanticCallResolution.test.ts | 1 + ...avascriptSemanticSharedChildWrites.test.ts | 6 +- .../javascriptSymbolicBindings.test.ts | 28 +- src/domain/native/nativeAotPe.budget.test.ts | 4 +- src/domain/referenceSourceGraph.test.ts | 3 + src/domain/webBundleAnalyzer.test.ts | 1 - src/domain/webScriptExportPlan.test.ts | 2 +- src/dotnet/ManagedMemberPointers.test.ts | 56 -- src/electron/ElectronFuseCleanup.test.ts | 24 +- src/erlang/BeamParser.test.ts | 83 --- src/evm/EvmBytecodeCarrier.test.ts | 25 +- .../devicetree/DevicetreeParser.test.ts | 6 - src/ghidra/GhidraDolProfile.test.ts | 2 + src/ghidra/GhidraInventoryValues.test.ts | 6 +- src/ghidra/GhidraLanguageOverride.test.ts | 2 + src/ghidra/GhidraMipsProfile.test.ts | 2 + src/ghidra/GhidraPowerPcProfile.test.ts | 2 + src/ghidra/GhidraProvider.behavior.test.ts | 18 +- src/ghidra/GhidraRiscvProfile.test.ts | 2 + src/ghidra/GhidraSessionError.test.ts | 8 +- .../extensions/GhidraExtensions.test.ts | 100 ++-- src/hopper/HopperNameInventory.test.ts | 27 +- src/hopper/HopperProvider.test.ts | 4 +- src/hopper/HopperRegexSearch.test.ts | 5 +- src/ida/IdaDoctor.test.ts | 8 +- src/ida/IdaProvider.test.ts | 27 +- src/ida/IdaSessionClient.test.ts | 7 +- ...avaScriptRuntimeScope.cancellation.test.ts | 60 +- src/inspector/V8InspectorEndpoint.test.ts | 2 +- ...V8InspectorProvider.closeOwnership.test.ts | 108 +++- src/native/NativeMachoOperand.test.ts | 13 +- src/native/SignatureTargetBinding.test.ts | 51 +- src/native/elf/ElfNoteParser.test.ts | 15 - src/process/ArtifactAnalysisResources.test.ts | 23 +- src/process/OwnedWorkerRuntime.test.ts | 83 ++- .../ProcessOwnership.observation.test.ts | 20 +- src/process/ProviderProcessOutput.test.ts | 42 +- src/process/RegexMatcher.test.ts | 87 --- .../capture/FilesystemCoverage.test.ts | 2 +- .../capture/ProcessCaptureLifecycle.test.ts | 5 +- src/python/PythonPycParser.test.ts | 62 -- src/server/BundleResourcesMcp.test.ts | 58 +- .../environment/serveEnvironment.test.ts | 15 +- .../environment/serveEnvironmentState.test.ts | 46 +- src/server/http/serveStreamableHttp.test.ts | 118 ++-- src/server/registerGuidedWorkflowTool.test.ts | 87 +-- tests/acceptance/analysisSettingsMcp.test.ts | 27 + tests/acceptance/apkSignatureV1Cli.test.ts | 162 ++++-- .../apkSignatureViewsCliMcp.test.ts | 29 +- .../adbPackageInventoryCoverage.test.ts | 29 +- .../applications/appBundleEncodingCli.test.ts | 170 +++--- .../bundleResourceTargetIdentity.test.ts | 49 +- .../javascriptCancellationCli.test.ts | 76 ++- .../javascriptFunctionNameCli.test.ts | 10 +- .../javascriptGraphQueryCoverage.test.ts | 40 +- .../javascriptNativeBindingOwnership.test.ts | 18 + .../javascriptNextRouteContext.test.ts | 2 + .../javascriptWindowOptionsCli.test.ts | 18 + .../mobileDirectoryComponents.test.ts | 106 +++- .../applications/packageSubpathsCli.test.ts | 1 + .../applications/peResources.test.ts | 12 +- tests/acceptance/applications/runtime.test.ts | 2 + .../webScriptPortablePathsCli.test.ts | 3 + .../acceptance/artifactClassification.test.ts | 148 ++--- .../artifactComparisonAsarUnpacked.test.ts | 14 +- .../artifactInventoryJvmClass.test.ts | 147 ----- .../artifactZipExtractionPaths.test.ts | 11 +- .../artifactZipLeadingBytes.test.ts | 3 + tests/acceptance/cliTimings.test.ts | 6 + .../acceptance/derivedEvidenceReplay.test.ts | 5 + tests/acceptance/devicetreeCli.test.ts | 33 +- tests/acceptance/dotnetBundleJourney.test.ts | 2 + tests/acceptance/electronFusesCli.test.ts | 3 + tests/acceptance/erlangBeamCli.test.ts | 134 ++++- tests/acceptance/espImageCli.test.ts | 87 +++ .../acceptance/firmwareContainersCli.test.ts | 3 + tests/acceptance/fitImageCli.test.ts | 3 + tests/acceptance/flutterAnalysis.test.ts | 4 + tests/acceptance/hexRecordsCli.test.ts | 52 ++ tests/acceptance/il2cppMetadataCli.test.ts | 4 +- tests/acceptance/inesContainer.test.ts | 8 + .../javascriptRouteBudgetRecovery.test.ts | 10 + tests/acceptance/javascriptRoutesCli.test.ts | 25 +- .../localToolsMcpProviderIsolation.test.ts | 59 ++ .../acceptance/managedMemberPointers.test.ts | 11 + .../acceptance/managedMemberRecovery.test.ts | 14 +- tests/acceptance/pcapOutputUtf8CliMcp.test.ts | 5 + tests/acceptance/pcapVerifier.test.ts | 62 ++ tests/acceptance/peCertificatesCli.test.ts | 40 +- tests/acceptance/peImportsCli.test.ts | 51 +- tests/acceptance/peLoadConfigCli.test.ts | 23 +- .../process/processFinalization.test.ts | 31 +- .../process/processTerminalWidth.test.ts | 15 +- .../providers/hopperStartupRecovery.test.ts | 57 +- tests/acceptance/pyinstallerArchive.test.ts | 9 +- tests/acceptance/pythonPycCli.test.ts | 12 +- .../clientConfigurationDiagnosticsCli.test.ts | 5 +- .../acceptance/sourcePathFailuresCli.test.ts | 11 + tests/acceptance/sqliteDatabaseCli.test.ts | 42 +- .../acceptance/tools/harBoundaryFacts.test.ts | 10 +- .../tools/mobileGraphExamples.test.ts | 31 +- .../tools/nativeFunctionMatching.test.ts | 7 +- tests/acceptance/tools/qlogCapture.test.ts | 100 +++- tests/acceptance/uImageCli.test.ts | 30 +- .../wasmInterfaceComparison.test.ts | 49 ++ .../android/androidJniLibraries.test.ts | 9 +- ...BrowserProvider.lifecycle_failures.test.ts | 6 +- .../cdpBrowserProvider.redirectErrors.test.ts | 6 +- .../browser/cdpEndpointResponseBounds.test.ts | 111 ++-- .../browser/cdpJsonShapeComparison.test.ts | 2 +- .../browser/historicalWebCapture.test.ts | 9 +- .../browser/webSourceMapFetcher.test.ts | 100 ++-- .../boundary/captures/captureProvider.test.ts | 16 +- .../boundary/captures/fieldDiscovery.test.ts | 11 +- tests/boundary/captures/packetOutput.test.ts | 70 ++- .../cli/cliInputSchemaRequired.test.ts | 1 + tests/boundary/cli/cliOptionAliases.test.ts | 17 +- tests/boundary/cli/cliOutput.test.ts | 20 +- .../boundary/cli/cliWorkerCompletion.test.ts | 37 +- tests/boundary/cli/developmentTests.test.ts | 91 ++- tests/boundary/cli/fridaCli.test.ts | 7 +- .../boundary/cli/githubActionsExample.test.ts | 2 +- tests/boundary/cli/guidanceCli.test.ts | 3 + .../cli/idaDoctorRegistration.test.ts | 4 +- .../cli/idaWorkspaceDiscovery.test.ts | 8 +- tests/boundary/cli/jebCli.test.ts | 7 +- tests/boundary/cli/jsonInputStatus.test.ts | 5 +- .../cli/managedMemberSummaryCli.test.ts | 8 + .../cli/nativeMachoLinkerOptionsCli.test.ts | 2 +- .../cli/nativeMachoObjectSegmentCli.test.ts | 5 +- .../cli/reverseEngineeringCommands.test.ts | 12 +- .../cli/sharedJavaScriptCliSession.test.ts | 5 +- .../evm/evmLimiterAvailability.test.ts | 40 +- .../filesystem/android/application.test.ts | 17 +- .../filesystem/asarReaderCleanup.test.ts | 26 +- .../boundary/filesystem/ciChangeScope.test.ts | 111 +++- .../filesystem/ciRequiredLanes.test.ts | 45 ++ .../filesystem/ciTestProjects.test.ts | 53 +- .../filesystem/ciVerifiedPackage.test.ts | 138 ++++- .../clientRegistrationStatus.test.ts | 1 + .../directAnalysisSnapshotReplay.test.ts | 1 + .../filesystem/fileHandleChunks.test.ts | 40 +- .../filesystem/flutterBuildProvider.test.ts | 2 +- .../filesystem/ghidraPspExtension.test.ts | 2 + .../boundary/filesystem/inesArtifact.test.ts | 165 +++--- ...javascriptApplicationEmptyLiterals.test.ts | 50 +- .../javascriptRuntimeReconciliation.test.ts | 39 +- .../viteOutputDependencies.test.ts | 49 +- .../keyedArchiveNumberMetadata.test.ts | 13 - .../boundary/filesystem/peLoadConfig.test.ts | 19 - .../setupClientConfigCleanupOwnership.test.ts | 29 +- tests/boundary/filesystem/setupConfig.test.ts | 2 +- .../setupSkillCleanupOwnership.test.ts | 25 +- .../filesystem/setupSkillPublication.test.ts | 2 +- .../filesystem/webScriptExport.test.ts | 133 ++--- .../il2cpp/il2cppMappingOwnership.test.ts | 4 +- .../il2cpp/il2cppMetadataProvider.test.ts | 63 +- .../javascriptRuntimeObservation.test.ts | 20 +- .../wakaruIntegration.test.ts | 75 ++- .../javascriptSourceAnalysisWorker.test.ts | 26 +- .../javascript/workerOrderAudit.test.ts | 38 +- tests/boundary/mcp/advertisedMcpTool.test.ts | 43 ++ .../boundary/mcp/agentKnownAnswerMcp.test.ts | 1 + tests/boundary/mcp/analysisViewMcp.test.ts | 32 +- .../mcp/artifactInspectionCoverageMcp.test.ts | 136 +++-- .../mcp/artifactMcp.inlineInventory.test.ts | 369 ++++++------ tests/boundary/mcp/artifactMcp.test.ts | 268 +++++---- tests/boundary/mcp/browserMcp.test.ts | 167 ++++-- tests/boundary/mcp/browserScenarioMcp.test.ts | 279 ++++----- .../mcp/browserSourceCoordinates.test.ts | 145 ++--- .../mcp/compactInputSchemaMcp.test.ts | 124 ++-- .../mcp/comparisonUnknownsMcp.test.ts | 23 +- tests/boundary/mcp/deferredRuntime.test.ts | 14 +- tests/boundary/mcp/electronMcp.test.ts | 8 +- tests/boundary/mcp/espImageMcp.test.ts | 28 +- .../mcp/evidenceInputContractsMcp.test.ts | 70 +-- tests/boundary/mcp/evmInterfaceMcp.test.ts | 306 +++++----- .../mcp/firmwareContainersMcp.test.ts | 9 + tests/boundary/mcp/fitImageMcp.test.ts | 31 +- .../mcp/fixedEvidenceConsumersMcp.test.ts | 19 +- tests/boundary/mcp/flutterMcp.test.ts | 18 +- .../mcp/functionComparisonCoverageMcp.test.ts | 226 +++---- .../mcp/functionComparisonMcp.test.ts | 329 +++++------ ...unctionComparisonStringDecodingMcp.test.ts | 37 +- tests/boundary/mcp/gdbCancellationMcp.test.ts | 83 ++- tests/boundary/mcp/ghidraMcpHarness.ts | 47 +- .../mcp/ghidraMcpHarnessLifecycle.test.ts | 126 ++++ tests/boundary/mcp/goBinaryMcp.test.ts | 46 +- tests/boundary/mcp/godotPckMcp.test.ts | 194 +++--- tests/boundary/mcp/hexRecordsMcp.test.ts | 16 +- .../mcp/investigationAnalysisMcp.test.ts | 11 +- .../mcp/javascriptFactsViewMcp.test.ts | 8 +- .../mcp/javascriptSourceSelection.test.ts | 37 +- tests/boundary/mcp/jebMcp.test.ts | 226 +++---- tests/boundary/mcp/largeBytePattern.test.ts | 22 +- .../mcp/managedArtifactSummaryMcp.test.ts | 28 +- tests/boundary/mcp/managedMcp.test.ts | 364 ++++++------ tests/boundary/mcp/mcpContractHarness.ts | 9 +- tests/boundary/mcp/mcpHttpOptions.test.ts | 17 +- .../mcp/mobileApplicationGraphMcp.test.ts | 21 +- .../mcp/nativeDispatchBoundTarget.test.ts | 5 +- .../mcp/nativeInventoryViewsMcp.test.ts | 1 + .../mcp/nativeLoadImageViewsMcp.test.ts | 13 +- .../mcp/nativeMachoSummaryMcp.test.ts | 9 +- tests/boundary/mcp/nodeSeaMcp.test.ts | 47 +- tests/boundary/mcp/officialArguments.test.ts | 79 ++- tests/boundary/mcp/officialTools.test.ts | 55 +- .../mcp/optionalProviderIsolation.test.ts | 80 ++- .../boundary/mcp/outputValidatorReuse.test.ts | 60 +- tests/boundary/mcp/packetCaptureMcp.test.ts | 18 +- tests/boundary/mcp/peCertificatesMcp.test.ts | 185 +++--- tests/boundary/mcp/progressMcp.test.ts | 67 ++- .../mcp/pyinstallerArtifactMcp.test.ts | 435 +++++++------- tests/boundary/mcp/pythonPycMcp.test.ts | 550 +++++++++--------- .../retainedApplicationEvidenceMcp.test.ts | 29 +- .../reverseEngineeringEnvironmentMcp.test.ts | 18 +- .../mcp/reverseEngineeringPartialMcp.test.ts | 2 + .../mcp/rizinDebuggerCancellationMcp.test.ts | 33 +- .../mcp/sessionAvailabilityRecovery.test.ts | 2 + .../mcp/sourceToBundleViewsMcp.test.ts | 25 +- tests/boundary/mcp/streamableHttpMcp.test.ts | 10 + .../mcp/webScriptSelectionMcp.test.ts | 56 +- .../mcp/workflowSnapshotReplayMcp.test.ts | 110 ++-- tests/boundary/pcap/pcapProvider.test.ts | 90 +-- .../pcap/pcapSnapshotCancellation.test.ts | 3 + .../boundary/process/ProcessEvidence.test.ts | 2 - .../process/androidAnalysisMcp.test.ts | 86 +-- .../androidSearchEvidenceBudget.test.ts | 10 +- .../boundary/process/jadxIntegration.test.ts | 2 + .../packageVerifierCommandSemantics.test.ts | 26 +- .../process/processCaptureProgress.test.ts | 79 ++- tests/boundary/process/processCli.test.ts | 8 +- .../processFilesystemCoverageAdapters.test.ts | 18 +- .../process/processFixtureLifecycle.test.ts | 251 ++++++++ .../process/selectedExecutableCleanup.test.ts | 72 ++- .../process/verifierTerminalOwner.test.ts | 49 +- .../GdbSessionManagerCoverage.test.ts | 72 ++- .../GdbSessionReviewRegressions.test.ts | 9 +- .../RizinDebugSessionCoverage.fixtures.ts | 7 +- .../RizinDebugSessionCoverage.part1.test.ts | 15 +- .../RizinDebugSessionCoverage.part2.test.ts | 16 - .../RizinDebugSessionCoverage.part3.test.ts | 98 +++- .../RizinDebugSessionCoverage.part4.test.ts | 50 +- ...RizinDebugSessionReviewRegressions.test.ts | 41 +- .../providers/ghidra/ghidraClient.test.ts | 26 +- .../hopper/hopperInstructions.test.ts | 7 +- .../hopper/linuxPrivateDisplay.test.ts | 3 +- .../providers/kuna/kunaProjectCache.test.ts | 100 +++- .../providers/kuna/kunaQueryWork.test.ts | 29 +- .../providers/kuna/kunaSessionFixture.ts | 17 +- .../kuna/kunaWorkspaceRollback.test.ts | 53 +- .../native/lldbReadbackFiles.test.ts | 1 + .../nativeMachoUuidlessInspection.test.ts | 66 +-- .../providers/native/nativeProvider.test.ts | 64 +- .../providers/startupCloseWaiters.test.ts | 40 +- .../linuxHopperPublisherVerification.test.ts | 25 +- .../boundary/wasm/interfaceComparison.test.ts | 44 -- .../analysis-sessions/binarySession.test.ts | 6 + .../investigationRecords.test.ts | 26 +- .../managedApplicationGraph.test.ts | 6 +- .../analysis-sessions/providerBinding.test.ts | 57 +- .../browserScenarioStartupRedaction.test.ts | 26 +- .../composition/directAnalysisCleanup.test.ts | 4 +- tests/composition/reverseEngineering.test.ts | 25 +- .../reverseEngineeringCommandPort.test.ts | 6 - .../reverseEngineeringEdgeCases.test.ts | 141 +++-- .../conformance/ida/providerContracts.test.ts | 5 + .../evaluation/providerRequiredCases.test.ts | 7 +- tests/fixtures/advertisedMcpTool.ts | 5 +- tests/fixtures/cutter_bridge_protocol_test.py | 9 +- tests/fixtures/jebMcp.ts | 20 +- tests/fixtures/lifecycleAcknowledgement.ts | 3 +- tests/fixtures/localToolsMcp.ts | 18 +- tests/fixtures/pcap/offlineHttpCapture.ts | 8 +- .../bundleComparisonFailures.test.ts | 4 +- .../configurationWriteDiagnostics.test.ts | 2 +- .../cutterDescriptorLifetime.test.ts | 42 +- .../darwinCompilerCleanup.test.ts | 76 ++- .../derivedOperationFailures.test.ts | 1 + .../ghidraEndpointLifetime.test.ts | 3 +- .../ghidraImportSnapshotLifetime.test.ts | 2 + ...cessObservationCommandFinalization.test.ts | 24 +- .../setupConfigurationConflicts.test.ts | 4 +- .../analysis-efficiency/test_codex_context.py | 2 +- tests/support/mcp/artifactSession.ts | 16 +- tests/support/process/processFixture.ts | 39 +- .../support/publishedDirectoryCloseFailure.ts | 61 ++ 342 files changed, 8844 insertions(+), 6279 deletions(-) delete mode 100644 tests/acceptance/artifactInventoryJvmClass.test.ts create mode 100644 tests/acceptance/localToolsMcpProviderIsolation.test.ts create mode 100644 tests/acceptance/pcapVerifier.test.ts create mode 100644 tests/boundary/mcp/advertisedMcpTool.test.ts create mode 100644 tests/boundary/mcp/ghidraMcpHarnessLifecycle.test.ts create mode 100644 tests/boundary/process/processFixtureLifecycle.test.ts delete mode 100644 tests/boundary/wasm/interfaceComparison.test.ts create mode 100644 tests/support/publishedDirectoryCloseFailure.ts diff --git a/.github/workflows/real-browser.yml b/.github/workflows/real-browser.yml index fc44a5bb9d..3c8ea8c03c 100644 --- a/.github/workflows/real-browser.yml +++ b/.github/workflows/real-browser.yml @@ -53,3 +53,8 @@ jobs: env: REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome REA_BROWSER_NO_SANDBOX: "true" + # The preceding verifier builds the runtime used by this request journey. + - run: dbus-run-session -- node scripts/verify-browser-request.mjs + env: + REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome + REA_BROWSER_NO_SANDBOX: "true" diff --git a/.github/workflows/real-packet-capture.yml b/.github/workflows/real-packet-capture.yml index a4bf3eead8..3d33c6ae42 100644 --- a/.github/workflows/real-packet-capture.yml +++ b/.github/workflows/real-packet-capture.yml @@ -54,7 +54,7 @@ jobs: mkdir -p .cache rm -f "$report" node scripts/run-exact-tests.mjs run --reporter=default --reporter=json --outputFile.json="$report" \ - tests/boundary/captures/fieldDiscovery.test.ts src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/pcap/pcapProvider.test.ts tests/boundary/pcap/pcapProviderCrossPlatform.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts + tests/boundary/captures/fieldDiscovery.test.ts src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/pcap/pcapProvider.test.ts tests/boundary/pcap/pcapProviderCrossPlatform.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts tests/acceptance/pcapVerifier.test.ts node scripts/ci/require-provider-tests.mjs packet "$report" - run: node scripts/verify-packet-capture.mjs env: diff --git a/package.json b/package.json index c9b4e8aac5..85bf319342 100644 --- a/package.json +++ b/package.json @@ -291,7 +291,7 @@ "verify:native-ui:permissions": "npm run build:cached && node scripts/verify-native-ui.mjs --allow-permission-denial", "test:focused": "node scripts/run-development-tests.mjs focused", "test:sqlite": "npm run test:prepare && npm run test:sqlite:run", - "test:sqlite:run": "vitest run src/application/sqlite/ tests/boundary/sqlite/ tests/acceptance/sqliteDatabaseCli.test.ts tests/boundary/mcp/serverLifecycleMcp.test.ts", + "test:sqlite:run": "node scripts/run-exact-tests.mjs run tests/acceptance/sqliteDatabaseCli.test.ts tests/boundary/mcp/serverLifecycleMcp.test.ts && vitest run src/application/sqlite/ tests/boundary/sqlite/", "build:windows-native": "node scripts/build-windows-native.mjs", "verify:windows-native:artifact": "node scripts/verify-windows-native-artifact.mjs", "prepublishOnly": "npm run verify:windows-native:artifact", diff --git a/scripts/ci/require-provider-tests.mjs b/scripts/ci/require-provider-tests.mjs index d4f1571a24..cd273c9f20 100644 --- a/scripts/ci/require-provider-tests.mjs +++ b/scripts/ci/require-provider-tests.mjs @@ -19,6 +19,9 @@ const descriptors = [ ); const requirements = { packet: { + "tests/acceptance/pcapVerifier.test.ts": [ + "verifies complete and unresolved HTTP coverage with real Wireshark tools", + ], "tests/boundary/captures/resourceLimits.test.ts": [ ...resources, "reports a real kernel file-limit termination with its effective ceiling", diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 96118044ab..c0dff3ed66 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -28,6 +28,8 @@ const PATH_SCOPES = { "tests/boundary/cli/captureCleanup.test.ts", "tests/boundary/mcp/packetCaptureMcp.test.ts", "tests/boundary/pcap/**", + "tests/acceptance/pcapVerifier.test.ts", + "tests/fixtures/pcap/offlineHttpCapture.ts", "src/captures/**", "src/pcap/**", "src/domain/pcap/**", @@ -43,6 +45,7 @@ const PATH_SCOPES = { "src/server/registerCaptureTools.ts", ...CAPTURE_SHARED_INPUTS, "scripts/verify-packet-capture.mjs", + "scripts/verify-pcap.mjs", ".github/workflows/real-packet-capture.yml", ], signal_recording: [ @@ -351,20 +354,30 @@ const PATH_SCOPES = { ".github/workflows/real-browser.yml", "scripts/verify-real-browser.mjs", "scripts/verify-browser-network.mjs", + "scripts/verify-browser-request.mjs", "scripts/verify-browser-scripts.mjs", "scripts/verify-browser-modules.mjs", "scripts/verify-browser-dom-destinations.mjs", "scripts/lib/browser-*.mjs", "scripts/fixtures/browser-network-site.mjs", + "scripts/fixtures/browser-request-site.mjs", "scripts/fixtures/browser-script-site.mjs", "scripts/fixtures/browser-module-site.mjs", "scripts/fixtures/browser-verifier-site.mjs", "scripts/fixtures/page-cdp-proxy.mjs", "src/browser/**", "src/domain/browser*.ts", + "src/domain/webCaptureDiffSchemas.ts", + "src/domain/webScreenshot.ts", "src/domain/webScript*.ts", "src/domain/webModule*.ts", + "src/domain/analysisView/environment*.ts", + "src/domain/analysisView/{EnvironmentManifest*,EnvironmentState*}.ts", + "src/domain/analysisView/{scenarioCaptureRecords,recordedResponseBody,recordedExchangeBody,recordedMediaType,webNetworkRouteTemplates,WebNetworkProjectionAccumulator}.ts", + "src/application/BrowserScenario*.ts", + "src/application/environment/**", "src/application/WebModule*.ts", + "src/composition/browserScenario.ts", "src/composition/webModules.ts", "src/artifacts/readStableArtifact.ts", "src/artifacts/ArtifactPaths.ts", @@ -374,6 +387,8 @@ const PATH_SCOPES = { "src/server/registerWebModuleTool.ts", "src/server/createServer.ts", "src/cli/webModuleCommands.ts", + "src/cliBrowserScenarioCommands.ts", + "src/cli/environmentCommands.ts", "src/cli.ts", "src/cliCommandNames.ts", "src/contracts/toolEffects.ts", @@ -564,6 +579,8 @@ const PATH_SCOPES = { ".github/workflows/real-web-source-map.yml", "scripts/verify-browser-source-maps.mjs", "scripts/lib/browser-source-map-e2e.mjs", + "scripts/lib/browser-nested-source-map-e2e.mjs", + "scripts/lib/browser-runtime-fixture-lifecycle.mjs", "scripts/fixtures/browser-source-map-site.mjs", "src/javascript/sourceMaps/**", "src/browser/assets/**", diff --git a/scripts/fixtures/browser-request-site.mjs b/scripts/fixtures/browser-request-site.mjs index a8b4343849..7711a02ec6 100644 --- a/scripts/fixtures/browser-request-site.mjs +++ b/scripts/fixtures/browser-request-site.mjs @@ -94,6 +94,12 @@ export async function startBrowserRequestSite() { writes.push({ method: request.method, path: url.pathname, + query: url.search, + body, + contentType: request.headers["content-type"], + cookie: + request.headers.cookie?.includes(`csrftoken=${requestFixtureCsrf}`) ?? + false, csrf: request.headers["x-csrftoken"] === requestFixtureCsrf, bearer: request.headers.authorization === `Bearer ${requestFixtureToken}`, diff --git a/scripts/lib/browser-request-e2e.mjs b/scripts/lib/browser-request-e2e.mjs index 64b80c9b09..80ed2f983d 100644 --- a/scripts/lib/browser-request-e2e.mjs +++ b/scripts/lib/browser-request-e2e.mjs @@ -67,10 +67,13 @@ const writeSession = (executable, origin) => read("list-1", "/api/items"), read("list-2", "/api/items"), read("item-1", "/api/items/1"), - write("create-1", "POST", "/api/items", { - json: { title: "Scratch item 1" }, - status: 201, - }), + { + ...write("create-1", "POST", "/api/items", { + json: { title: "Scratch item 1" }, + status: 201, + }), + query: [{ name: "org", value: { source: "literal", value: "1" } }], + }, read("created-1", "/api/items/3"), read("list-created-1", "/api/items"), write("create-2", "POST", "/api/items", { @@ -337,6 +340,16 @@ export async function verifyBrowserRequestSteps( assertRequestsAttributed(scenario, capture); assert.equal(site.writes.length, 6); assert.ok(site.writes.every(({ csrf, bearer }) => csrf && bearer)); + assert.deepEqual(site.writes[0], { + method: "POST", + path: "/api/items", + query: "?org=1", + body: { title: "Scratch item 1" }, + contentType: "application/json", + cookie: true, + csrf: true, + bearer: true, + }); assertPageValuesRedacted(evidence, capture); const capturePath = join(root.path, "items.capture.json"); diff --git a/scripts/run-development-tests.mjs b/scripts/run-development-tests.mjs index 499f73c479..e182b14658 100644 --- a/scripts/run-development-tests.mjs +++ b/scripts/run-development-tests.mjs @@ -1,6 +1,7 @@ import { spawn } from "node:child_process"; import { stat } from "node:fs/promises"; import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; import { developmentTestPlan, @@ -10,6 +11,7 @@ import { let child; let forceTimer; let shutdownSignal; +let childOwnsTree = false; const signalExitCodes = { SIGINT: 130, SIGTERM: 143 }; for (const signal of Object.keys(signalExitCodes)) process.on(signal, () => { @@ -17,8 +19,10 @@ for (const signal of Object.keys(signalExitCodes)) shutdownSignal = signal; if (child !== undefined && child.exitCode === null) { child.kill(signal); - forceTimer = setTimeout(() => child?.kill("SIGKILL"), 1_000); - forceTimer.unref?.(); + if (!childOwnsTree) { + forceTimer = setTimeout(() => child?.kill("SIGKILL"), 1_000); + forceTimer.unref?.(); + } } }); @@ -55,28 +59,38 @@ try { throw new Error( "Test prerequisites require the locked dependency installation", ); - const prepared = await run(process.execPath, [ - "scripts/run-exclusive.mjs", - "artifacts", + const prepared = await run( process.execPath, - resolve("scripts/run-turbo.mjs"), - "run", - ...new Set([ - ...(plan.needsBuild ? ["artifacts:skills"] : []), - ...plan.artifactTasks, - ]), - ]); + [ + "scripts/run-exclusive.mjs", + "artifacts", + process.execPath, + resolve("scripts/run-turbo.mjs"), + "run", + ...new Set([ + ...(plan.needsBuild ? ["artifacts:skills"] : []), + ...plan.artifactTasks, + ]), + ], + false, + "existing", + ); if (prepared.code !== 0) process.exitCode = prepared.code; } if (!process.exitCode && !shutdownSignal) { - const tested = await run(process.execPath, [ - resolve( - request.paths.length > 0 - ? "scripts/run-exact-tests.mjs" - : "node_modules/vitest/vitest.mjs", - ), - ...plan.vitestArguments, - ]); + const tested = await run( + process.execPath, + [ + resolve( + request.paths.length > 0 + ? "scripts/run-exact-tests.mjs" + : "node_modules/vitest/vitest.mjs", + ), + ...plan.vitestArguments, + ], + false, + "test", + ); process.exitCode = tested.code; } } @@ -86,15 +100,27 @@ try { } if (shutdownSignal) process.exitCode = signalExitCodes[shutdownSignal]; -function run(executable, arguments_, capture = false) { +function run(executable, arguments_, capture = false, supervision) { if (shutdownSignal) return Promise.resolve({ code: signalExitCodes[shutdownSignal], output: "", }); - child = spawn(executable, arguments_, { - stdio: capture ? ["ignore", "pipe", "pipe"] : "inherit", - }); + // Source-only tests do not require the built Windows Job Object artifact. + const wrap = supervision === "test" && process.platform !== "win32"; + childOwnsTree = supervision === "existing" || wrap; + child = spawn( + wrap ? process.execPath : executable, + wrap + ? [ + fileURLToPath(new URL("./run-exclusive.mjs", import.meta.url)), + "--unlocked", + executable, + ...arguments_, + ] + : arguments_, + { stdio: capture ? ["ignore", "pipe", "pipe"] : "inherit" }, + ); let output = ""; if (capture) { child.stdout.on("data", (data) => (output += data.toString())); diff --git a/scripts/run-exclusive-command.mjs b/scripts/run-exclusive-command.mjs index 6d7fc37bcf..515a2aeca0 100644 --- a/scripts/run-exclusive-command.mjs +++ b/scripts/run-exclusive-command.mjs @@ -33,7 +33,7 @@ if (!process.connected) { process.exitCode = signalExitCodes.SIGTERM; } else if ( typeof lockName !== "string" || - !/^[a-z][a-z0-9-]*$/u.test(lockName) || + !(lockName === "--unlocked" || /^[a-z][a-z0-9-]*$/u.test(lockName)) || command.length === 0 ) { console.error("Usage: run-exclusive.mjs [args...]"); @@ -45,19 +45,21 @@ if (!process.connected) { async function run() { try { - await mkdir(lockRoot, { recursive: true }); - // SQLite owns acquisition and OS-backed lock release, including process - // death. Keep the database pathname stable; deleting it would split owners. - database = new DatabaseSync(join(lockRoot, `${lockName}.sqlite`)); - try { - database.exec("BEGIN EXCLUSIVE"); - } catch (cause) { - if (isBusyError(cause)) - throw new Error( - `Another ${lockName} command is already running. Wait for it to finish.`, - { cause }, - ); - throw cause; + if (lockName !== "--unlocked") { + await mkdir(lockRoot, { recursive: true }); + // SQLite owns acquisition and OS-backed lock release, including process + // death. Keep the database pathname stable; deleting it would split owners. + database = new DatabaseSync(join(lockRoot, `${lockName}.sqlite`)); + try { + database.exec("BEGIN EXCLUSIVE"); + } catch (cause) { + if (isBusyError(cause)) + throw new Error( + `Another ${lockName} command is already running. Wait for it to finish.`, + { cause }, + ); + throw cause; + } } if (shutdownSignal === undefined) { const result = await runCommand(); diff --git a/scripts/verify-pcap.mjs b/scripts/verify-pcap.mjs index 6c0bdf5eb0..721db7b237 100644 --- a/scripts/verify-pcap.mjs +++ b/scripts/verify-pcap.mjs @@ -79,7 +79,9 @@ await withVerifierCleanup(async () => { (entry) => entry.response_address_count > entry.response_addresses.length, ); - assert.equal(result.coverage, partial ? "partial" : "complete"); + // Count discrepancies prove partial coverage, but matching counts do not + // rule out unresolved HTTP targets or unavailable DNS identity extraction. + if (partial) assert.equal(result.coverage, "partial"); assert.deepEqual( await hashVerifierFile(path, resources), identity, diff --git a/src/apkSignature/ApkSignatureProvider.test.ts b/src/apkSignature/ApkSignatureProvider.test.ts index f8568b38a4..ef44e3bf99 100644 --- a/src/apkSignature/ApkSignatureProvider.test.ts +++ b/src/apkSignature/ApkSignatureProvider.test.ts @@ -11,9 +11,6 @@ import { APK_SIGNATURE_V3_ID, LEAF_CERTIFICATE, LEAF_SHA256, - LEAF_SUBJECT, - OID_ECDSA_SHA256, - OID_SHA256, OPENSSL_BLOCK_ISSUER_SERIAL, ROOT_CERTIFICATE, ROOT_SHA256, @@ -149,18 +146,6 @@ const reportOf = async (path: string) => { }; describe("ApkSignatureProvider without the Android SDK", () => { - it("retains the declared digest's signature algorithm ID and bytes", async () => { - const report = await reportOf( - await write( - "algorithm-id.apk", - syntheticSigned([{ id: V2, value: bogusV2Value() }]), - ), - ); - expect(report.schemes.v2?.signers?.[0]?.digests[0]).toMatchObject({ - signature_algorithm_id: 0x0103, - declared_digest_hex: "ab".repeat(32), - }); - }); it("reports an unsigned ZIP as having no scheme", async () => { const r = await reportOf( await write( @@ -174,53 +159,6 @@ describe("ApkSignatureProvider without the Android SDK", () => { expect(r.posture.v1_only).toBe(false); }); - it("detects v1-only signing and names the signer certificate its block declares", async () => { - for (const stored of [false, true]) { - const r = await reportOf( - await write( - `v1-${String(stored)}.apk`, - await zipWith([ - { name: "classes.dex", data: "x" }, - { name: "META-INF/MANIFEST.MF", data: "Manifest-Version: 1.0" }, - { name: "META-INF/CERT.SF", data: "Signature-Version: 1.0" }, - { - name: "META-INF/CERT.EC", - data: OPENSSL_BLOCK_ISSUER_SERIAL, - stored, - }, - ]), - ), - ); - expect(r.schemes.v1.present).toBe(true); - expect(r.posture.v1_only).toBe(true); - expect(r.posture.strongest_scheme).toBe("v1"); - expect(r.posture.v1_signers_match_block_signers).toBeNull(); - const [signer] = r.schemes.v1.signers; - expect(signer).toMatchObject({ - signature_file: "META-INF/CERT.SF", - block_file: "META-INF/CERT.EC", - certificate: { - sha256_fingerprint: LEAF_SHA256, - subject: LEAF_SUBJECT, - public_key_algorithm: "ec", - }, - certificates: [ - { sha256_fingerprint: LEAF_SHA256, subject: LEAF_SUBJECT }, - { sha256_fingerprint: ROOT_SHA256 }, - ], - signer_infos: [ - { - identified_by: "issuer-and-serial", - certificate_sha256: LEAF_SHA256, - digest_algorithm_oid: OID_SHA256, - signature_algorithm_oid: OID_ECDSA_SHA256, - }, - ], - decode_error: null, - }); - } - }); - it("decodes a v2 signer, keeps an unparseable certificate null, and flags a digest mismatch", async () => { const r = await reportOf( await write( diff --git a/src/apkSignature/apkV1Signers.test.ts b/src/apkSignature/apkV1Signers.test.ts index ee3ca6982c..359becbbc9 100644 --- a/src/apkSignature/apkV1Signers.test.ts +++ b/src/apkSignature/apkV1Signers.test.ts @@ -60,7 +60,7 @@ describe("v1 signer output projection", () => { expect(new X509Certificate(certificate).subject).toContain('\\"'); const signer = signerOf(certificate); const facts = v1SignerFacts(signer); - expect(facts.certificate?.subject).not.toBeNull(); + expect(facts.certificate?.subject).toEqual(expect.any(String)); expect(projectedV1SignerBytes(signer)).toBeGreaterThanOrEqual( Buffer.byteLength(JSON.stringify(facts)), ); diff --git a/src/apktool/ApktoolCommandRecovery.test.ts b/src/apktool/ApktoolCommandRecovery.test.ts index 661176302f..e8e6fb6ce1 100644 --- a/src/apktool/ApktoolCommandRecovery.test.ts +++ b/src/apktool/ApktoolCommandRecovery.test.ts @@ -1,9 +1,10 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; import { afterEach, expect, it, vi } from "vitest"; import { createDeferred } from "../../tests/fixtures/binarySession.js"; -import { AnalysisError } from "../domain/analysisErrorBase.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { DarwinProcessOwnershipInspectionError } from "../process/DarwinProcessRunTokenReader.js"; @@ -319,6 +320,7 @@ it("retries failed command peers without repeating successful helper cleanup", a it("joins preparation during close and retains a late helper handoff", async () => { const client = provider(); const ready = createDeferred(); + const entered = createDeferred(); const cleanup = new ProviderCleanupError( "process-ownership", ["late-helper"], @@ -328,6 +330,7 @@ it("joins preparation during close and retains a late helper handoff", async () cleanup.resolveCleanup(); }); vi.mocked(prepareOwnedCommand).mockImplementationOnce(async () => { + entered.resolve(); await ready.promise; throw new DarwinProcessOwnershipInspectionError( "physical preparation failure", @@ -335,12 +338,13 @@ it("joins preparation during close and retains a late helper handoff", async () ); }); const pending = inspect(client); - await vi.waitFor(() => expect(prepareOwnedCommand).toHaveBeenCalledOnce()); let closed = false; - const closing = client.close().then(() => { - closed = true; - }); - try { + let closing: Promise | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered.promise, pending, "command preparation"); + closing = client.close().then(() => { + closed = true; + }); await Promise.resolve(); expect(closed).toBe(false); ready.resolve(); @@ -351,11 +355,16 @@ it("joins preparation during close and retains a late helper handoff", async () expect(close).toHaveBeenCalledOnce(); expect(result.error.cleanupIncomplete).toBe(false); expect(runOwnedCommand).not.toHaveBeenCalled(); - } finally { - ready.resolve(); - await pending; - await closing; - } + }, [ + () => ready.resolve(), + async () => { + await pending; + }, + async () => { + await closing; + }, + () => client.close(), + ]); }); it("preserves an opaque cancellation reason before acquiring a command", async () => { diff --git a/src/application/Doctor.lifecycle.test.ts b/src/application/Doctor.lifecycle.test.ts index f37fa4c847..9a8d86ca47 100644 --- a/src/application/Doctor.lifecycle.test.ts +++ b/src/application/Doctor.lifecycle.test.ts @@ -1,4 +1,6 @@ import { expect, it } from "vitest"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { createDoctorHostFixture } from "./Doctor.fixture.js"; @@ -47,18 +49,24 @@ it("joins a late provider inspection and retries only its unresolved cleanup own }, }); const inspection = host.providerInspections!(); - const signal = await entered; - const closing = host.close!().then( - () => { - closed = true; - return undefined; - }, - (cause: unknown) => { - closed = true; - return cause; - }, - ); - try { + let closing: Promise | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + entered.then(() => {}), + inspection, + "provider inspection", + ); + const signal = await entered; + closing = host.close!().then( + () => { + closed = true; + return undefined; + }, + (cause: unknown) => { + closed = true; + return cause; + }, + ); expect(signal.aborted).toBe(true); expect(closed).toBe(false); handOff(); @@ -78,14 +86,20 @@ it("joins a late provider inspection and retries only its unresolved cleanup own await expect(host.providerInspections!()).rejects.toThrow( "scope is closed", ); - } finally { - denied = false; - handOff(); - finish(); - await inspection; - await closing; - await host.close!(); - } + }, [ + () => { + denied = false; + handOff(); + finish(); + }, + async () => { + await inspection; + }, + async () => { + await closing; + }, + () => host.close!(), + ]); }); it("retains a completed healthy report when cancellation arrives during final cleanup", async () => { diff --git a/src/application/Setup.configuration.test.ts b/src/application/Setup.configuration.test.ts index c502299ffe..bea24fec81 100644 --- a/src/application/Setup.configuration.test.ts +++ b/src/application/Setup.configuration.test.ts @@ -216,7 +216,12 @@ describe("optional Hopper installation failures", () => { host, ); expect(result.code).toBe("download_failed"); - expect(result.remediation).toMatch(/^Download failed\. ./); + expect(result.remediation).toContain("Download failed."); + expect(result.remediation).toContain("could not be updated"); + expect(result.clients.cursor).toMatchObject({ + status: "failed", + reason: "write", + }); expect(host.configurations).toBe(1); }); @@ -234,7 +239,8 @@ describe("optional Hopper installation failures", () => { host, ); expect(result.code).toBe("skill_install_failed"); - expect(result.remediation).toMatch(/^Download failed\. ./); + expect(result.remediation).toContain("Download failed."); + expect(result.remediation).toContain("injected skill write failure"); expect(host.skillInstalls).toBe(1); }); }); diff --git a/src/application/analysisView/AnalysisViewService.test.ts b/src/application/analysisView/AnalysisViewService.test.ts index 87a725ef9c..0775951e6d 100644 --- a/src/application/analysisView/AnalysisViewService.test.ts +++ b/src/application/analysisView/AnalysisViewService.test.ts @@ -540,9 +540,19 @@ it("resolves same-session retained JavaScript Evidence without re-running analys lookup, ); if (!item.ok) throw item.error; - expect(JSON.stringify(item.value.normalized_result)).toContain( - "export const secret = 1;", - ); + expect(item.value.normalized_result).toMatchObject({ + kind: "item", + item: { + path: "renderer.js", + observations: expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + source: "export const secret = 1;\n", + }), + }), + ]), + }, + }); }); it("rejects stale retained IDs and unsupported parent operations", async () => { @@ -859,10 +869,14 @@ it("seals an inferred environment state whose values are the recorded ones", asy name: "grafana-example", parent_evidence_id: parent.evidence_id, timeline: "recorded-start-time", + entities: expect.arrayContaining([ + expect.objectContaining({ + type: "/api/folders", + id: "fg1jo91e57e2ob", + fields: expect.objectContaining({ title: "probe folder" }), + }), + ]), }); - expect(JSON.stringify(sealed.value.normalized_result)).toContain( - "probe folder", - ); expect(parseEvidence(sealed.value)).toEqual(sealed.value); }); diff --git a/src/application/analysisView/BrowserScenarioViews.test.ts b/src/application/analysisView/BrowserScenarioViews.test.ts index 4a01e56046..79bfd43fb1 100644 --- a/src/application/analysisView/BrowserScenarioViews.test.ts +++ b/src/application/analysisView/BrowserScenarioViews.test.ts @@ -375,16 +375,47 @@ describe("browser scenario retained boundaries", () => { rows: [{ transaction_id: "request-2" }], }, }); + expect(result).toMatchObject({ + items: { + rows: [ + { + events: expect.not.arrayContaining([ + expect.objectContaining({ + event: expect.objectContaining({ kind: "network-content" }), + }), + ]), + omitted_events: expect.arrayContaining([ + expect.objectContaining({ kind: "network-content" }), + ]), + }, + ], + }, + }); expect(JSON.stringify(result)).not.toContain('"content":'); const selected = await view(capture, { ...request, event_kinds: ["network-content"], }); - expect(JSON.stringify(selected)).toContain('"content":'); expect(selected).toMatchObject({ items: { rows: [ { + transaction_id: "request-2", + events: [ + expect.objectContaining({ + event: expect.objectContaining({ + kind: "network-content", + transaction_id: "request-2", + phase: "response", + body: expect.objectContaining({ + state: "captured", + content: Buffer.from('{"id":2,"login":"probe"}').toString( + "base64", + ), + }), + }), + }), + ], omitted_events: expect.arrayContaining([ { item_index: expect.any(Number), kind: "request" }, ]), @@ -393,7 +424,9 @@ describe("browser scenario retained boundaries", () => { }, }); }); +}); +describe("browser scenario network observation", () => { it("distinguishes unrequested and legacy-unknown network observation from an observed empty inventory", () => { const base = scenarioCaptureFixture([], { request_body: false, diff --git a/src/application/analysisView/evidenceFileSource.test.ts b/src/application/analysisView/evidenceFileSource.test.ts index 855e1ddf45..3e251d868e 100644 --- a/src/application/analysisView/evidenceFileSource.test.ts +++ b/src/application/analysisView/evidenceFileSource.test.ts @@ -326,7 +326,9 @@ it("keeps bundle-selection errors useful when operation labels exceed the reply if (read.ok) throw new Error("Expected a record-selection error"); const encoded = JSON.stringify(read); expect(Buffer.byteLength(encoded, "utf8")).toBeLessThan(8192); - for (const record of records) expect(encoded).toContain(record.evidence_id); - expect(encoded).toContain("operation omitted"); + for (const record of records) + expect(read.error.userMessage).toContain( + `${record.evidence_id} (operation omitted)`, + ); expect(encoded).not.toContain(operation); }); diff --git a/src/application/binary/AnalysisProviderRegistry.test.ts b/src/application/binary/AnalysisProviderRegistry.test.ts index ef4b44394e..a0497b9b11 100644 --- a/src/application/binary/AnalysisProviderRegistry.test.ts +++ b/src/application/binary/AnalysisProviderRegistry.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../../tests/fixtures/lifecycleAcknowledgement.js"; import { AnalysisProviderRegistry } from "./AnalysisProviderRegistry.js"; import { @@ -651,14 +653,15 @@ describe("analysis provider registry: profile and cleanup failures", () => { selected = true; return result; }); - await pending; - controller.abort(reason); let closed = false; - const closing = registry.closeOwnedResources().then((result) => { - closed = true; - return result; - }); - try { + let closing: ReturnType | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(pending, selecting, "profile probe"); + controller.abort(reason); + closing = registry.closeOwnedResources().then((result) => { + closed = true; + return result; + }); await new Promise((resolve) => setImmediate(resolve)); expect(selected).toBe(false); expect(closed).toBe(false); @@ -674,10 +677,18 @@ describe("analysis provider registry: profile and cleanup failures", () => { expect(await closing).toEqual(ok(null)); expect(registry.hasPendingOwnedResources()).toBe(false); expect(alpha.created).toEqual([]); - } finally { - finish?.(); - await Promise.all([selecting, closing]); - } + }, [ + () => { + controller.abort(reason); + finish?.(); + }, + async () => { + await Promise.allSettled([selecting, closing]); + }, + async () => { + expect(await registry.closeOwnedResources()).toEqual(ok(null)); + }, + ]); }); }); @@ -970,41 +981,62 @@ describe("automatic profile cleanup ownership", () => { }; }), ); - const result = await registry.select(DATABASE_TARGET, "auto", { + const selecting = registry.select(DATABASE_TARGET, "auto", { signal: controller.signal, }); - if (result.ok) throw new Error("Mixed failed profiles admitted"); - expect(result.error.cleanupIncomplete).toBe(true); - expect(result.error.cleanupResources).toEqual([ - "alpha-file", - "beta-file", + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + admitted, + selecting, + "profile children admission", + ); + const result = await selecting; + if (result.ok) throw new Error("Mixed failed profiles admitted"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toEqual([ + "alpha-file", + "beta-file", + ]); + expect(result.error.diagnostics?.profile_errors).toEqual( + expect.arrayContaining([ + expect.objectContaining({ provider_id: "alpha" }), + expect.objectContaining({ provider_id: "beta" }), + ]), + ); + const close1 = await registry.closeOwnedResources(); + expect(close1.ok).toBe(false); + released.add(first); + const close2 = await registry.closeOwnedResources(); + expect(close2.ok).toBe(false); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toEqual([ + first === "alpha" ? "beta-file" : "alpha-file", + ]); + expect(close1.ok ? [] : close1.error.cleanupResources).toEqual([ + first === "alpha" ? "beta-file" : "alpha-file", + ]); + released.add(first === "alpha" ? "beta" : "alpha"); + expect(await registry.closeOwnedResources()).toEqual(ok(null)); + expect(result.error.cleanupIncomplete).toBe(false); + if (close1.ok || close2.ok) + throw new Error("Cleanup unexpectedly succeeded"); + expect(close1.error.cleanupIncomplete).toBe(false); + expect(close2.error.cleanupIncomplete).toBe(false); + expect(registry.hasPendingOwnedResources()).toBe(false); + }, [ + () => { + released.add("alpha"); + released.add("beta"); + controller.abort(reason); + admitChildren?.(); + }, + async () => { + await Promise.allSettled([selecting]); + }, + async () => { + expect(await registry.closeOwnedResources()).toEqual(ok(null)); + }, ]); - expect(result.error.diagnostics?.profile_errors).toEqual( - expect.arrayContaining([ - expect.objectContaining({ provider_id: "alpha" }), - expect.objectContaining({ provider_id: "beta" }), - ]), - ); - const close1 = await registry.closeOwnedResources(); - expect(close1.ok).toBe(false); - released.add(first); - const close2 = await registry.closeOwnedResources(); - expect(close2.ok).toBe(false); - expect(result.error.cleanupIncomplete).toBe(true); - expect(result.error.cleanupResources).toEqual([ - first === "alpha" ? "beta-file" : "alpha-file", - ]); - expect(close1.ok ? [] : close1.error.cleanupResources).toEqual([ - first === "alpha" ? "beta-file" : "alpha-file", - ]); - released.add(first === "alpha" ? "beta" : "alpha"); - expect(await registry.closeOwnedResources()).toEqual(ok(null)); - expect(result.error.cleanupIncomplete).toBe(false); - if (close1.ok || close2.ok) - throw new Error("Cleanup unexpectedly succeeded"); - expect(close1.error.cleanupIncomplete).toBe(false); - expect(close2.error.cleanupIncomplete).toBe(false); - expect(registry.hasPendingOwnedResources()).toBe(false); }, ); }); @@ -1034,10 +1066,12 @@ it("registers before production and gates fresh selections until shutdown has jo }, ]); const selecting = registry.select(DATABASE_TARGET, "alpha"); - await entered; - const closing = registry.closeOwnedResources(); - const next = registry.select(DATABASE_TARGET, "alpha"); - try { + let closing: ReturnType | undefined; + let next: ReturnType | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, selecting, "profile finalization"); + closing = registry.closeOwnedResources(); + next = registry.select(DATABASE_TARGET, "alpha"); await new Promise((resolve) => setImmediate(resolve)); expect(calls).toBe(1); expect(registry.hasPendingOwnedResources()).toBe(true); @@ -1050,11 +1084,15 @@ it("registers before production and gates fresh selections until shutdown has jo expect((await next).ok).toBe(true); expect(calls).toBe(2); expect(registry.hasPendingOwnedResources()).toBe(false); - } finally { - finish?.(); - await Promise.all([selecting, closing, next]); - await registry.closeOwnedResources(); - } + }, [ + () => finish?.(), + async () => { + await Promise.allSettled([selecting, closing, next]); + }, + async () => { + expect(await registry.closeOwnedResources()).toEqual(ok(null)); + }, + ]); }); it("joins cancelled readiness finalization and shutdown before admitting another observation", async () => { @@ -1094,15 +1132,17 @@ it("joins cancelled readiness finalization and shutdown before admitting another settled = true; return cause; }); - await entered; - controller.abort(reason); let closed = false; - const closing = registry.closeOwnedResources().then((value) => { - closed = true; - return value; - }); - const next = registry.candidates(); - try { + let closing: ReturnType | undefined; + let next: ReturnType | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, observing, "readiness observation"); + controller.abort(reason); + closing = registry.closeOwnedResources().then((value) => { + closed = true; + return value; + }); + next = registry.candidates(); await new Promise((resolve) => setImmediate(resolve)); expect(settled).toBe(false); expect(closed).toBe(false); @@ -1125,11 +1165,18 @@ it("joins cancelled readiness finalization and shutdown before admitting another expect(await registry.candidates()).toHaveLength(1); expect(calls).toBe(2); expect(registry.hasPendingOwnedResources()).toBe(false); - } finally { - finish(); - await Promise.allSettled([observing, closing, next]); - await registry.closeOwnedResources(); - } + }, [ + () => { + controller.abort(reason); + finish(); + }, + async () => { + await Promise.allSettled([observing, closing, next]); + }, + async () => { + expect(await registry.closeOwnedResources()).toEqual(ok(null)); + }, + ]); }); it.each(["alpha", "beta"])( @@ -1208,8 +1255,12 @@ it("joins every candidate and retains physical failure with exact mixed cleanup settled = true; return cause; }); - await entered; - try { + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + entered, + observing, + "mixed readiness observation", + ); await new Promise((resolve) => setImmediate(resolve)); expect(settled).toBe(false); finish(); @@ -1245,11 +1296,15 @@ it("joins every candidate and retains physical failure with exact mixed cleanup details: { diagnostics: { reason: "physical observation failed" } }, }); expect(registry.hasPendingOwnedResources()).toBe(false); - } finally { - finish(); - await observing; - await registry.closeOwnedResources(); - } + }, [ + finish, + async () => { + await observing; + }, + async () => { + expect(await registry.closeOwnedResources()).toEqual(ok(null)); + }, + ]); }); it("registers selection before async readiness can race shutdown", async () => { diff --git a/src/application/binary/BinarySessionLiveness.test.ts b/src/application/binary/BinarySessionLiveness.test.ts index a39c4ebec1..9bf4bb7357 100644 --- a/src/application/binary/BinarySessionLiveness.test.ts +++ b/src/application/binary/BinarySessionLiveness.test.ts @@ -1,8 +1,10 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../../tests/fixtures/lifecycleAcknowledgement.js"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import { elf } from "../../domain/binaryTarget.fixture.js"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, onTestFinished } from "vitest"; import { createAnalysisExecution, @@ -42,76 +44,68 @@ import { SessionProviderRouter } from "./SessionProviderRouter.js"; */ it("binds omitted provider subjects to the selected target", async () => { const { session } = openSession(); - try { - const opened = await session.open(process.execPath); - expect(opened.ok).toBe(true); - const execution = await session.execute("address_name", {}); - expect(execution.ok).toBe(true); - if (!execution.ok || !opened.ok) return; - expect(execution.value.subject).toEqual({ - path: opened.value.path, - sha256: opened.value.sha256, - format: opened.value.format, - ...(opened.value.architecture === undefined - ? {} - : { architecture: opened.value.architecture }), - }); - } finally { - await session.close(); - } + const opened = await session.open(process.execPath); + expect(opened.ok).toBe(true); + const execution = await session.execute("address_name", {}); + expect(execution.ok).toBe(true); + if (!execution.ok || !opened.ok) return; + expect(execution.value.subject).toEqual({ + path: opened.value.path, + sha256: opened.value.sha256, + format: opened.value.format, + ...(opened.value.architecture === undefined + ? {} + : { architecture: opened.value.architecture }), + }); }); it("preserves an explicitly unbound provider observation", async () => { const { session } = openSession({ explicitNullSubject: true }); - try { - expect((await session.open(process.execPath)).ok).toBe(true); - const execution = await session.execute("address_name", {}); - expect(execution.ok).toBe(true); - if (execution.ok) expect(execution.value.subject).toBeNull(); - } finally { - await session.close(); - } + expect((await session.open(process.execPath)).ok).toBe(true); + const execution = await session.execute("address_name", {}); + expect(execution.ok).toBe(true); + if (execution.ok) expect(execution.value.subject).toBeNull(); }); it("retains live source Evidence without replaying a prior observation", async () => { const { session, deep } = openSession({ live: true }); - try { - expect((await session.open(process.execPath)).ok).toBe(true); - expect( - ( - await session.execute("address_name", { - document: "0", - address: "0x9999", - }) - ).ok, - ).toBe(true); - const child = deep.created[0]; - if (child === undefined) throw new Error("expected provider child"); - let reads = 0; - child.execute = () => - Promise.resolve( - ok(createAnalysisExecution(`live ${++reads}`, deepIdentity)), - ); - const parameters = { document: "0", address: "0x1000" }; - const first = await session.execute("address_name", parameters); - const second = await session.execute("address_name", parameters); - expect(first.ok && first.value.result).toBe("live 1"); - expect(second.ok && second.value.result).toBe("live 2"); - if (!first.ok || !second.ok) return; - expect(first.value.evidenceId).toBeDefined(); - expect(second.value.evidenceId).toBeDefined(); - expect(first.value.evidenceId).not.toBe(second.value.evidenceId); - expect(session.exportEvidenceBundle().records).toEqual( - expect.arrayContaining([ - expect.objectContaining({ normalized_result: "live 1" }), - expect.objectContaining({ normalized_result: "live 2" }), - ]), + expect((await session.open(process.execPath)).ok).toBe(true); + expect( + ( + await session.execute("address_name", { + document: "0", + address: "0x9999", + }) + ).ok, + ).toBe(true); + const child = deep.created[0]; + if (child === undefined) throw new Error("expected provider child"); + let reads = 0; + child.execute = () => + Promise.resolve( + ok(createAnalysisExecution(`live ${++reads}`, deepIdentity)), ); - const snapshot = session.exportAnalysisSnapshot(); - if (snapshot.ok) expect(snapshot.value.entries).toHaveLength(0); - } finally { - await session.close(); - } + const parameters = { document: "0", address: "0x1000" }; + const first = await session.execute("address_name", parameters); + const second = await session.execute("address_name", parameters); + expect(first.ok && first.value.result).toBe("live 1"); + expect(second.ok && second.value.result).toBe("live 2"); + if (!first.ok || !second.ok) return; + expect(first.value.evidenceId).toBeDefined(); + expect(second.value.evidenceId).toBeDefined(); + expect(first.value.evidenceId).not.toBe(second.value.evidenceId); + expect(session.exportEvidenceBundle().records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ normalized_result: "live 1" }), + expect.objectContaining({ normalized_result: "live 2" }), + ]), + ); + const snapshot = session.exportAnalysisSnapshot(); + expect(snapshot.ok, snapshot.ok ? undefined : snapshot.error.message).toBe( + true, + ); + if (!snapshot.ok) return; + expect(snapshot.value.entries).toEqual([]); }); it("keeps unbound observations out of persisted target snapshots", async () => { @@ -119,93 +113,84 @@ it("keeps unbound observations out of persisted target snapshots", async () => { const directory = await createTestTempDirectory("rea-unbound-snapshot-"); const path = join(directory, "snapshot.json"); const parameters = { document: "0", address: "0x1000" }; - try { - expect((await session.open(process.execPath)).ok).toBe(true); - const bound = await session.execute("address_name", parameters); - expect(bound.ok).toBe(true); - const child = deep.created[0]; - if (child === undefined || !bound.ok) - throw new Error("Expected bound read"); - child.execute = () => - Promise.resolve( - ok( - createAnalysisExecution("unbound observation", deepIdentity, { - subject: null, - }), - ), - ); - const execution = await session.execute("address_name", { - ...parameters, - address: "0x2000", - }); - expect(execution.ok).toBe(true); - if (!execution.ok) return; - expect(execution.value.subject).toBeNull(); - expect(session.exportEvidenceBundle().records).toEqual( - expect.arrayContaining([ - expect.objectContaining({ + expect((await session.open(process.execPath)).ok).toBe(true); + const bound = await session.execute("address_name", parameters); + expect(bound.ok).toBe(true); + const child = deep.created[0]; + if (child === undefined || !bound.ok) throw new Error("Expected bound read"); + child.execute = () => + Promise.resolve( + ok( + createAnalysisExecution("unbound observation", deepIdentity, { subject: null, - normalized_result: "unbound observation", }), - ]), + ), ); - const snapshot = session.exportAnalysisSnapshot(); - expect(snapshot.ok, snapshot.ok ? undefined : snapshot.error.message).toBe( - true, + const execution = await session.execute("address_name", { + ...parameters, + address: "0x2000", + }); + expect(execution.ok).toBe(true); + if (!execution.ok) return; + expect(execution.value.subject).toBeNull(); + expect(session.exportEvidenceBundle().records).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + subject: null, + normalized_result: "unbound observation", + }), + ]), + ); + const snapshot = session.exportAnalysisSnapshot(); + expect(snapshot.ok, snapshot.ok ? undefined : snapshot.error.message).toBe( + true, + ); + if (!snapshot.ok) return; + expect(snapshot.value.entries).toHaveLength(1); + expect(snapshot.value.entries[0]?.parameters).toEqual(parameters); + const unboundReplay = await session.execute("address_name", { + ...parameters, + address: "0x2000", + }); + expect(unboundReplay.ok).toBe(true); + if (unboundReplay.ok) { + expect(unboundReplay.value.subject).toBeNull(); + expect(unboundReplay.value.evidenceId).toBe(execution.value.evidenceId); + } + expect( + (await session.writeAnalysisSnapshot(snapshot.value, path, false)).ok, + ).toBe(true); + expect((await session.close()).ok).toBe(true); + expect( + ( + await session.open(process.execPath, { + loadSnapshot: () => session.readAnalysisSnapshot(path), + }) + ).ok, + ).toBe(true); + const replay = await session.execute("address_name", parameters); + expect(replay.ok).toBe(true); + if (replay.ok) { + expect(replay.value.evidenceId).toBe(bound.value.evidenceId); + expect(replay.value.limitations).toContain( + "Loaded from a local REA analysis snapshot; this call did not re-run the provider.", ); - if (!snapshot.ok) return; - expect(snapshot.value.entries).toHaveLength(1); - expect(snapshot.value.entries[0]?.parameters).toEqual(parameters); - const unboundReplay = await session.execute("address_name", { - ...parameters, - address: "0x2000", - }); - expect(unboundReplay.ok).toBe(true); - if (unboundReplay.ok) { - expect(unboundReplay.value.subject).toBeNull(); - expect(unboundReplay.value.evidenceId).toBe(execution.value.evidenceId); - } - expect( - (await session.writeAnalysisSnapshot(snapshot.value, path, false)).ok, - ).toBe(true); - expect((await session.close()).ok).toBe(true); - expect( - ( - await session.open(process.execPath, { - loadSnapshot: () => session.readAnalysisSnapshot(path), - }) - ).ok, - ).toBe(true); - const replay = await session.execute("address_name", parameters); - expect(replay.ok).toBe(true); - if (replay.ok) { - expect(replay.value.evidenceId).toBe(bound.value.evidenceId); - expect(replay.value.limitations).toContain( - "Loaded from a local REA analysis snapshot; this call did not re-run the provider.", - ); - } - } finally { - await session.close(); } }); describe("BinarySession open liveness", () => { it("retires a locally blocked owner even when its native health is idle", async () => { const { session, deep } = openSession(); - try { - expect((await session.open(process.execPath)).ok).toBe(true); - expect((await session.execute("address_name", {})).ok).toBe(true); - const retained = deep.created[0]; - if (retained === undefined) throw new Error("Expected an acquired child"); - retained.reuseReadiness = () => "blocked"; - expect(retained.operationHealthSnapshot?.()?.state).toBe("idle"); - expect((await session.open(process.execPath)).ok).toBe(true); - expect(retained.closeCalled).toBe(true); - expect((await session.execute("address_name", {})).ok).toBe(true); - expect(deep.created).toHaveLength(2); - } finally { - await session.close(); - } + expect((await session.open(process.execPath)).ok).toBe(true); + expect((await session.execute("address_name", {})).ok).toBe(true); + const retained = deep.created[0]; + if (retained === undefined) throw new Error("Expected an acquired child"); + retained.reuseReadiness = () => "blocked"; + expect(retained.operationHealthSnapshot?.()?.state).toBe("idle"); + expect((await session.open(process.execPath)).ok).toBe(true); + expect(retained.closeCalled).toBe(true); + expect((await session.execute("address_name", {})).ok).toBe(true); + expect(deep.created).toHaveLength(2); }); it("reuses the retained client while the child stays live", async () => { const { session, deep } = openSession(); @@ -368,6 +353,7 @@ it("clears retired runtime availability after cancelled-switch restoration", asy }); const { session, deep } = openSession({ failOperationInitially: true, + releaseBeforeCleanup: releaseClose, beforeFirstClose: async () => { closeStarted(); await closeGate; @@ -394,11 +380,17 @@ it("clears retired runtime availability after cancelled-switch restoration", asy ); await writeFile(alternate, elf(2, 1, 62)); const switching = session.open(alternate, { signal: controller.signal }); - await closing; - controller.abort(); - releaseClose(); - - const cancelled = await switching; + const cancelled = await withVerifierCleanup(async () => { + await awaitAcknowledgement(closing, switching, "provider close"); + controller.abort(); + releaseClose(); + return await switching; + }, [ + releaseClose, + async () => { + await switching; + }, + ]); expect(cancelled).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" }, @@ -435,7 +427,6 @@ it("clears retired runtime availability after cancelled-switch restoration", asy reason: "available", remediation: null, }); - await session.close(); }); const sessionRunId = ( @@ -498,6 +489,7 @@ const openSession = ( operationHealth?: boolean; failOperationInitially?: boolean; beforeFirstClose?: () => Promise; + releaseBeforeCleanup?: () => void; explicitNullSubject?: boolean; live?: boolean; } = {}, @@ -527,6 +519,11 @@ const openSession = ( ]), new InvestigationRecords(), ); + onTestFinished(async () => { + options.releaseBeforeCleanup?.(); + const closed = await session.close(); + if (!closed.ok) throw closed.error; + }); return { session, deep: { created }, onProbe }; }; diff --git a/src/application/binary/DeferredAnalysisClient.test.ts b/src/application/binary/DeferredAnalysisClient.test.ts index 740273dc64..587a230ba8 100644 --- a/src/application/binary/DeferredAnalysisClient.test.ts +++ b/src/application/binary/DeferredAnalysisClient.test.ts @@ -1,4 +1,6 @@ import { expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../../tests/fixtures/lifecycleAcknowledgement.js"; import { DeferredAnalysisClient } from "../../providers/DeferredAnalysisClient.js"; import { createAnalysisExecution, @@ -164,8 +166,12 @@ it("keeps a shared creation alive for an interested peer", async () => { reason = { request: "one peer leaves" }; const first = client.execute("health", {}, { signal: controller.signal }); const second = client.execute("health", {}); - try { - await entered.promise; + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + entered.promise, + first, + "shared client creation", + ); controller.abort(reason); const cancelled = await first; if (cancelled.ok) throw new Error("Expected peer cancellation"); @@ -181,11 +187,17 @@ it("keeps a shared creation alive for an interested peer", async () => { expect(creates).toBe(1); expect(await client.close()).toEqual(ok(null)); expect(closes).toBe(1); - } finally { - loading.resolve(child()); - await Promise.allSettled([first, second]); - await client.close(); - } + }, [ + () => controller.abort(), + () => loading.resolve(child()), + async () => { + await Promise.allSettled([first, second]); + }, + async () => { + const closed = await client.close(); + if (!closed.ok) throw closed.error; + }, + ]); }); it("joins a late acquired client's final cleanup before cancellation settles", async () => { @@ -207,8 +219,8 @@ it("joins a late acquired client's final cleanup before cancellation settles", a settled = true; return result; }); - try { - await entered.promise; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered.promise, pending, "client creation"); controller.abort(reason); loading.resolve( child(async () => { @@ -218,7 +230,7 @@ it("joins a late acquired client's final cleanup before cancellation settles", a return ok(null); }), ); - await closeEntered.promise; + await awaitAcknowledgement(closeEntered.promise, pending, "client cleanup"); expect(settled).toBe(false); expect(client.operationHealthSnapshot()?.state).toBe("idle"); expect(client.reuseReadiness()).toBe("blocked"); @@ -229,12 +241,18 @@ it("joins a late acquired client's final cleanup before cancellation settles", a expect(result.error.cleanupIncomplete).toBe(false); expect(await client.close()).toEqual(ok(null)); expect(closes).toBe(1); - } finally { - loading.resolve(child()); - closing.resolve(); - await pending; - await client.close(); - } + }, [ + () => controller.abort(), + () => loading.resolve(child()), + () => closing.resolve(), + async () => { + await pending; + }, + async () => { + const closed = await client.close(); + if (!closed.ok) throw closed.error; + }, + ]); }); it("keeps a producer-authored physical failure primary even when it is the caller's abort reason", async () => { @@ -256,8 +274,8 @@ it("keeps a producer-authored physical failure primary even when it is the calle {}, { signal: controller.signal }, ); - try { - await entered.promise; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered.promise, pending, "client creation"); controller.abort(physical); finishing.resolve(); const result = await pending; @@ -272,11 +290,17 @@ it("keeps a producer-authored physical failure primary even when it is the calle expect(result.error.cause).toBe(physical); expect(physical.cause).toBeUndefined(); expect(projectAnalysisError(result.error).code).toBe("execution_failure"); - } finally { - finishing.resolve(); - await pending; - await client.close(); - } + }, [ + () => controller.abort(), + () => finishing.resolve(), + async () => { + await pending; + }, + async () => { + const closed = await client.close(); + if (!closed.ok) throw closed.error; + }, + ]); }); it("retains a failed late client's exact close owner and restores cancellation after confirmed retry", async () => { diff --git a/src/application/binary/LazyAnalysisProvider.test.ts b/src/application/binary/LazyAnalysisProvider.test.ts index 497ebafcb9..ae2180437b 100644 --- a/src/application/binary/LazyAnalysisProvider.test.ts +++ b/src/application/binary/LazyAnalysisProvider.test.ts @@ -1,4 +1,6 @@ import { expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../../tests/fixtures/lifecycleAcknowledgement.js"; import { parseBinaryTarget } from "../BinaryTargetResolver.js"; import type { @@ -7,7 +9,6 @@ import type { } from "../AnalysisProvider.js"; import { createAnalysisExecution } from "../AnalysisProvider.js"; import { LazyAnalysisProvider } from "./LazyAnalysisProvider.js"; -import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import { ok as resultOk } from "../../domain/result.js"; const identity: ProviderIdentity = { @@ -66,14 +67,15 @@ it("joins a cancelled unabortable module load and refuses late client acquisitio const entered = new Promise((resolve) => { enter = resolve; }); + const loading = new Promise((resolve) => { + finishLoading = resolve; + }); const provider = new LazyAnalysisProvider({ identity, capabilities: [], load: () => { enter(); - return new Promise((resolve) => { - finishLoading = resolve; - }); + return loading; }, }); const controller = new AbortController(); @@ -86,9 +88,9 @@ it("joins a cancelled unabortable module load and refuses late client acquisitio settled = true; return value; }); - await entered; - controller.abort(reason); - try { + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, pending, "provider module load"); + controller.abort(reason); await new Promise((resolve) => setImmediate(resolve)); expect(settled).toBe(false); finishLoading( @@ -105,11 +107,17 @@ it("joins a cancelled unabortable module load and refuses late client acquisitio expect(result.error.cause).toBe(reason); expect(creates).toBe(0); expect(await client.close()).toEqual(resultOk(null)); - } finally { - finishLoading(providerWithClient()); - await pending; - await client.close(); - } + }, [ + () => controller.abort(), + () => finishLoading(providerWithClient()), + async () => { + await pending; + }, + async () => { + const closed = await client.close(); + if (!closed.ok) throw closed.error; + }, + ]); }); it("reports that a lazy client was closed before its first operation", async () => { diff --git a/src/application/binaryDiagnostics/BinaryLayoutService.test.ts b/src/application/binaryDiagnostics/BinaryLayoutService.test.ts index 96f563cfdc..f8a4c6b878 100644 --- a/src/application/binaryDiagnostics/BinaryLayoutService.test.ts +++ b/src/application/binaryDiagnostics/BinaryLayoutService.test.ts @@ -121,7 +121,7 @@ it("preserves the caller's exact abort reason before invocation", async () => { expect(calls).toBe(0); }); -it("cancels before invocation and after an acquired observation", async () => { +it("retains the acquired observation when cancellation arrives during inspection", async () => { const controller = new AbortController(); const service = new BinaryLayoutService({ inspectAndClose(input, options) { @@ -154,21 +154,6 @@ it("cancels before invocation and after an acquired observation", async () => { normalized_result: binaryLayoutFixture(), }); } - const untouched = new BinaryLayoutService({ - inspectAndClose(input, options) { - return this.inspect(input, options); - }, - identity: BINARY_LAYOUT_TEST_PROVIDER, - inspect: () => { - throw new Error("must not run"); - }, - }); - expect( - await untouched.inspect( - { path: "/source.elf" }, - { signal: controller.signal }, - ), - ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); }); it.each(["different-path", "invalid-digest", "outside-range"])( diff --git a/src/application/binaryDiagnostics/PeCertificatesService.test.ts b/src/application/binaryDiagnostics/PeCertificatesService.test.ts index ab2b3369a5..f0bc7d12c4 100644 --- a/src/application/binaryDiagnostics/PeCertificatesService.test.ts +++ b/src/application/binaryDiagnostics/PeCertificatesService.test.ts @@ -49,20 +49,6 @@ it("rejects a relative path and extra arguments before reading", async () => { expect(calls).toBe(0); }); -it("records the selected PE and an absent certificate table", async () => { - const result = await new PeCertificatesService({ - identity, - inspect: () => Promise.resolve(ok(report)), - }).inspect({ path: report.artifact.path }); - expect(result.ok).toBe(true); - if (!result.ok) throw result.error; - expect(result.value).toMatchObject({ - confidence: "observed", - subject: { format: "pe", architecture: "x86" }, - normalized_result: report, - }); -}); - it("discards an observation if the caller cancels while the provider completes", async () => { const controller = new AbortController(); const service = new PeCertificatesService({ diff --git a/src/application/binaryDiagnostics/PeImportsService.test.ts b/src/application/binaryDiagnostics/PeImportsService.test.ts index 0b8a777333..58a342bbd5 100644 --- a/src/application/binaryDiagnostics/PeImportsService.test.ts +++ b/src/application/binaryDiagnostics/PeImportsService.test.ts @@ -62,20 +62,6 @@ it("rejects a relative path and extra arguments before reading", async () => { expect(calls).toBe(0); }); -it("records the selected PE and an absent import table", async () => { - const result = await new PeImportsService({ - identity, - inspect: () => Promise.resolve(ok(report)), - }).inspect({ path: report.artifact.path }); - expect(result.ok).toBe(true); - if (!result.ok) throw result.error; - expect(result.value).toMatchObject({ - confidence: "observed", - subject: { format: "pe", architecture: "x86_64" }, - normalized_result: report, - }); -}); - it.each(["imports", "delay_imports", "examined_dlls"] as const)( "rejects contradictory complete %s coverage", async (field) => { diff --git a/src/application/binaryDiagnostics/RecordedCrashService.test.ts b/src/application/binaryDiagnostics/RecordedCrashService.test.ts index 82fe247adb..f9d723c5d1 100644 --- a/src/application/binaryDiagnostics/RecordedCrashService.test.ts +++ b/src/application/binaryDiagnostics/RecordedCrashService.test.ts @@ -128,7 +128,7 @@ it("preserves the caller's exact abort reason before invocation", async () => { expect(calls).toBe(0); }); -it("cancels before invocation and after an acquired observation", async () => { +it("retains the acquired observation when cancellation arrives during inspection", async () => { const controller = new AbortController(); const service = new RecordedCrashService({ inspectAndClose(input, options) { @@ -161,21 +161,6 @@ it("cancels before invocation and after an acquired observation", async () => { normalized_result: recordedCrashFixture(), }); } - const untouched = new RecordedCrashService({ - inspectAndClose(input, options) { - return this.inspect(input, options); - }, - identity: RECORDED_CRASH_TEST_PROVIDER, - inspect: () => { - throw new Error("must not run"); - }, - }); - expect( - await untouched.inspect( - { path: "/source.elf" }, - { signal: controller.signal }, - ), - ).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } }); }); it.each([ diff --git a/src/application/environment/environmentBuild.test.ts b/src/application/environment/environmentBuild.test.ts index 1bc6c6588f..5db742ae07 100644 --- a/src/application/environment/environmentBuild.test.ts +++ b/src/application/environment/environmentBuild.test.ts @@ -378,10 +378,15 @@ describe("buildEnvironment cancellation and input checks", () => { const built = await buildEnvironment(input); if (built.ok) throw new Error("Expected a cancellation"); const projected = projectAnalysisError(built.error); - expect(projected.code).toBe("cancelled"); - expect(JSON.stringify(projected)).toContain( - `"output_directory":${JSON.stringify(input.directory)}`, - ); + expect(projected).toMatchObject({ + code: "cancelled", + details: { + diagnostics: { + output_directory: input.directory, + bundle_written: true, + }, + }, + }); expect(await listFiles(input.directory!)).toHaveLength(7); }); diff --git a/src/application/environment/environmentState.test.ts b/src/application/environment/environmentState.test.ts index bc58e3fb0e..7c1799487d 100644 --- a/src/application/environment/environmentState.test.ts +++ b/src/application/environment/environmentState.test.ts @@ -89,6 +89,7 @@ describe("loadEnvironmentState", () => { const notEvidence = loadEnvironmentState({}, source); if (notEvidence.ok || !(notEvidence.error instanceof AnalysisInputError)) throw new Error("Expected an input refusal"); + expect(notEvidence.error.issues.length).toBeGreaterThan(0); expect( notEvidence.error.issues.every((issue) => issue.path[0] === "state"), ).toBe(true); diff --git a/src/application/erlang/BeamService.test.ts b/src/application/erlang/BeamService.test.ts index 031f771436..7df93684ae 100644 --- a/src/application/erlang/BeamService.test.ts +++ b/src/application/erlang/BeamService.test.ts @@ -61,21 +61,6 @@ it("rejects a relative path and extra arguments before reading", async () => { expect(calls).toBe(0); }); -it("records the selected BEAM form", async () => { - const result = await new BeamService({ - close: async () => {}, - identity, - inspect: () => Promise.resolve(ok(report)), - }).inspect({ path: report.artifact.path }); - expect(result.ok).toBe(true); - if (!result.ok) throw result.error; - expect(result.value).toMatchObject({ - confidence: "observed", - subject: { format: "file" }, - normalized_result: report, - }); -}); - it("discards an observation if the caller cancels while the provider completes", async () => { const controller = new AbortController(); const reason = { source: "caller-request", operation: "cancel" }; diff --git a/src/application/javascript/JavaScriptSemanticGraphSeedQueries.test.ts b/src/application/javascript/JavaScriptSemanticGraphSeedQueries.test.ts index 598bae4251..5699a607f8 100644 --- a/src/application/javascript/JavaScriptSemanticGraphSeedQueries.test.ts +++ b/src/application/javascript/JavaScriptSemanticGraphSeedQueries.test.ts @@ -77,9 +77,13 @@ it("queries an empty event name as an exact retained event identity", () => { }), ); expect(trace.seed_node_ids.length).toBeGreaterThan(0); - expect( - trace.nodes - .filter(({ node_id }) => trace.seed_node_ids.includes(node_id)) - .every(({ properties }) => properties.event_name === ""), - ).toBe(true); + const seeds = trace.nodes.filter(({ node_id }) => + trace.seed_node_ids.includes(node_id), + ); + expect(seeds.map(({ node_id }) => node_id).sort()).toEqual( + [...trace.seed_node_ids].sort(), + ); + expect(seeds.every(({ properties }) => properties.event_name === "")).toBe( + true, + ); }); diff --git a/src/application/sqlite/SqliteDatabaseService.test.ts b/src/application/sqlite/SqliteDatabaseService.test.ts index 0b39c5c1ee..1be65ea4dd 100644 --- a/src/application/sqlite/SqliteDatabaseService.test.ts +++ b/src/application/sqlite/SqliteDatabaseService.test.ts @@ -63,25 +63,6 @@ it.each([ }, ); -it("preserves the selected worker budget in provider input and Evidence", async () => { - const report = { ...fixture(), worker_timeout_ms: 1234 }; - const service = new SqliteDatabaseService({ - inspect: async (input) => { - expect(input.worker_timeout_ms).toBe(1234); - return ok(execution(report)); - }, - }); - const result = await service.inspect({ - path: "/selected.db", - worker_timeout_ms: 1234, - }); - if (!result.ok) throw result.error; - expect(result.value.parameters).toMatchObject({ worker_timeout_ms: 1234 }); - expect(result.value.normalized_result).toMatchObject({ - worker_timeout_ms: 1234, - }); -}); - it.each([ { selected: undefined, reported: undefined }, { selected: undefined, reported: 1234 }, @@ -109,18 +90,6 @@ it.each([ }, ); -it("retains the actual default without inventing an explicit caller selection", async () => { - const service = new SqliteDatabaseService({ - inspect: async () => ok(execution(fixture())), - }); - const result = await service.inspect({ path: "/selected.db" }); - if (!result.ok) throw result.error; - expect(result.value.normalized_result).toMatchObject({ - worker_timeout_ms: 30_000, - }); - expect(result.value.parameters).not.toHaveProperty("worker_timeout_ms"); -}); - it("accepts sqlite_schema rows without requiring a fabricated declaration", async () => { const report = { ...fixture(), diff --git a/src/artifacts/HexRecordArtifactReader.test.ts b/src/artifacts/HexRecordArtifactReader.test.ts index b88a53e7c0..7b6721837e 100644 --- a/src/artifacts/HexRecordArtifactReader.test.ts +++ b/src/artifacts/HexRecordArtifactReader.test.ts @@ -1,4 +1,4 @@ -import { readFile, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; @@ -14,9 +14,6 @@ import { import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js"; -const INVENTORY_LIMITATION = - "Intel HEX/S-record inventory decodes data records into address-contiguous regions. It does not establish CPU, byte order, memory map, or that a declared start address is executable; gaps are unwritten, not zero."; - const writeImage = async (name: string, text: string): Promise => { const directory = await createTestTempDirectory("rea-hex-records-"); const path = join(directory, name); @@ -24,60 +21,6 @@ const writeImage = async (name: string, text: string): Promise => { return path; }; -it("inventories Intel HEX regions with root and region facts", async () => { - const snapshot = await scanCanonicalArtifactInventory( - await writeImage("firmware.hex", INTEL_HEX_I32_GAP), - ); - expect(snapshot.manifest.root_format).toBe("intel-hex"); - expect(snapshot.coverage).toMatchObject({ - scope: "container-entries", - status: "complete", - }); - expect(snapshot.limitations).toContain(INVENTORY_LIMITATION); - const root = snapshot.occurrences.find((row) => row.logical_path === "."); - expect(root).toMatchObject({ - artifact_kind: "container", - format_metadata: { - kind: "hex-records", - variant: "I32HEX", - start_address: 0x0800_0101, - region_count: 2, - }, - }); - expect( - snapshot.occurrences - .filter((row) => row.logical_path !== ".") - .map((row) => [row.logical_path, row.declared_size, row.format_metadata]), - ).toEqual([ - [ - "0x08000000-0x0800001f.bin", - 32, - { - kind: "hex-record-region", - start_address: 0x0800_0000, - end_address: 0x0800_001f, - length: 32, - record_count: 2, - first_line: 2, - last_line: 3, - }, - ], - [ - "0x08000100-0x08000103.bin", - 4, - { - kind: "hex-record-region", - start_address: 0x0800_0100, - end_address: 0x0800_0103, - length: 4, - record_count: 1, - first_line: 4, - last_line: 4, - }, - ], - ]); -}); - it("reports unknown coverage and termination facts as limitations", async () => { const truncated = await scanCanonicalArtifactInventory( await writeImage("truncated.s19", SREC_S19.replace("S9030000FC\n", "")), @@ -103,19 +46,7 @@ it("reports unknown coverage and termination facts as limitations", async () => ); }); -it("fails closed on malformed or over-limit records", async () => { - await expect( - scanCanonicalArtifactInventory( - await writeImage( - "checksum.hex", - INTEL_HEX_I32_GAP.replace("BEEFC3", "BEEFC4"), - ), - ), - ).rejects.toMatchObject({ - reason: "format", - message: - "Intel HEX is malformed: line 4: record type 00 checksum 0xc4 does not match computed 0xc3", - }); +it("fails closed on over-limit records", async () => { const runs = [intelHexRecord(0, 0, [1])]; for (let index = 1; index <= 65_536; index += 1) { const address = index * 2; @@ -137,7 +68,7 @@ it("leaves an invalid first record to plain file classification", async () => { expect(snapshot.manifest.root_format).toBe("file"); }); -it("extracts byte-exact regions that re-inventory as raw files", async () => { +it("re-inventories extracted regions as raw files", async () => { const directory = await createTestTempDirectory("rea-hex-extract-"); const path = join(directory, "firmware.hex"); await writeFile(path, INTEL_HEX_I32_GAP); @@ -151,23 +82,6 @@ it("extracts byte-exact regions that re-inventory as raw files", async () => { integrityPolicy: "fail", resourceScope, }); - expect(extracted.containment_verified).toBe(true); - expect( - Buffer.compare( - await readFile( - join(extracted.output_root, "0x08000000-0x0800001f.bin"), - ), - Buffer.from(Array.from({ length: 32 }, (_, index) => index)), - ), - ).toBe(0); - expect( - Buffer.compare( - await readFile( - join(extracted.output_root, "0x08000100-0x08000103.bin"), - ), - Buffer.from("deadbeef", "hex"), - ), - ).toBe(0); const reinventoried = await scanCanonicalArtifactInventory( extracted.output_root, ); diff --git a/src/artifacts/Uf2ArtifactReader.test.ts b/src/artifacts/Uf2ArtifactReader.test.ts index 3cf1cafe38..37d7df4733 100644 --- a/src/artifacts/Uf2ArtifactReader.test.ts +++ b/src/artifacts/Uf2ArtifactReader.test.ts @@ -1,11 +1,9 @@ -import { readFile, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it } from "vitest"; import { Uf2ArtifactReader } from "./Uf2ArtifactReader.js"; -import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; -import { extractArtifact } from "./extraction/ArtifactExtraction.js"; import { RP2040_FAMILY, RP2XXX_ABSOLUTE_FAMILY, @@ -135,78 +133,13 @@ it("leaves coverage unknown when numbered blocks are absent", async () => { ]); }); -it("fails closed on a conflicting rewrite and leaves short files unrecognized", async () => { - const rewrite = uf2Block({ - address: 0x1000_0010, - payload: [0xff], - blockNo: 0, - numBlocks: 1, - familyId: RP2040_FAMILY, - }); - await expect( - scanCanonicalArtifactInventory( - await writeImage( - "conflict.uf2", - uf2File([...uf2Image(RP2040_FAMILY, 0x1000_0000, APP), rewrite]), - ), - ), - ).rejects.toMatchObject({ - reason: "format", - message: - "UF2 is malformed: family 0xe48bff56 (RP2040) address 0x10000010 is written with different values by blocks 0-2 (0x75) and block 3 (0xff)", - }); +it("leaves short files unrecognized", async () => { const truncated = await scanCanonicalArtifactInventory( await writeImage("short.uf2", ABSOLUTE.subarray(0, 511)), ); expect(truncated.manifest.root_format).toBe("file"); }); -it("extracts byte-exact regions into family directories", async () => { - const directory = await createTestTempDirectory("rea-uf2-extract-"); - const path = join(directory, "firmware.uf2"); - await writeFile( - path, - uf2File([ABSOLUTE, ...uf2Image(RP2040_FAMILY, 0x1000_0000, APP)]), - ); - const resourceScope = new ArtifactResourceScope(); - try { - const extracted = await extractArtifact({ - inputPath: path, - inputFormat: "uf2", - outputRoot: join(directory, "out"), - environment: process.env, - integrityPolicy: "fail", - resourceScope, - }); - expect(extracted.containment_verified).toBe(true); - expect(extracted.coverage.status).toBe("unknown"); - expect( - Buffer.compare( - await readFile( - join( - extracted.output_root, - "family-0xe48bff56/0x10000000-0x10000257.bin", - ), - ), - APP, - ), - ).toBe(0); - expect( - Buffer.compare( - await readFile( - join( - extracted.output_root, - "family-0xe48bff57/0x10ffff00-0x10ffffff.bin", - ), - ), - Buffer.alloc(256, 0xef), - ), - ).toBe(0); - } finally { - await resourceScope.close(); - } -}); - it("rejects a forged region entry and a changed carrier", async () => { const path = await writeImage( "identity.uf2", diff --git a/src/artifacts/ZipArtifactReader.test.ts b/src/artifacts/ZipArtifactReader.test.ts index 15c14fcd0a..a117ea9ba3 100644 --- a/src/artifacts/ZipArtifactReader.test.ts +++ b/src/artifacts/ZipArtifactReader.test.ts @@ -593,27 +593,6 @@ describe("ZipArtifactReader decompression budget", () => { } }); - it("streams a valid small member through open", async () => { - const directory = await createTestTempDirectory("rea-zip-budget-valid-"); - const path = join(directory, "valid.zip"); - const writer = new ZipWriter(new Uint8ArrayWriter()); - await writer.add("member.bin", new TextReader("valid member bytes")); - await writeFile(path, await writer.close()); - const reader = new ZipArtifactReader(path, "zip"); - try { - const entries = []; - for await (const entry of reader.entries()) entries.push(entry); - const entry = entries[0]; - if (entry === undefined) throw new Error("Expected a ZIP member"); - const chunks: Buffer[] = []; - for await (const chunk of await reader.open(entry)) - chunks.push(Buffer.from(chunk)); - expect(Buffer.concat(chunks).toString()).toBe("valid member bytes"); - } finally { - await reader.close(); - } - }); - it("rejects under-declared member output before delivering its complete actual bytes", async () => { const directory = await createTestTempDirectory("rea-zip-underdeclared-"); const path = join(directory, "underdeclared.zip"); diff --git a/src/browser/CdpSafeMetadata.test.ts b/src/browser/CdpSafeMetadata.test.ts index 6e310a487f..779dae41a8 100644 --- a/src/browser/CdpSafeMetadata.test.ts +++ b/src/browser/CdpSafeMetadata.test.ts @@ -220,8 +220,6 @@ describe("safe CDP response metadata", () => { "agent-header-secret", ]) expect(serialized).not.toContain(secret); - expect(serialized).toContain("link-secret"); - expect(serialized).toContain("private.example.test/x"); }); it("reports well-known agent resources as untrusted observations", () => { diff --git a/src/browser/PlaywrightScenarioRequest.test.ts b/src/browser/PlaywrightScenarioRequest.test.ts index b962012c9c..76194c53a2 100644 --- a/src/browser/PlaywrightScenarioRequest.test.ts +++ b/src/browser/PlaywrightScenarioRequest.test.ts @@ -73,32 +73,30 @@ describe("request step schema", () => { ); }); - it("accepts a same-origin request with query, headers and a JSON body", () => { + it("accepts a declared secret as a request header value", () => { const parsed = scenario( [ requestStep({ - query: [{ name: "org", value: { source: "literal", value: "1" } }], headers: [ - { - name: "X-CSRFToken", - value: { source: "cookie", name: "csrftoken" }, - }, { name: "Authorization", value: { source: "secret", secret_id: "token" }, prefix: "Bearer ", }, ], - json: { title: "Scratch folder 1" }, - expected_status: [200, 201], }), ], [{ secret_id: "token", environment_variable: "APP_TOKEN" }], ); expect(parsed.actions[0]).toMatchObject({ action: "request", - method: "POST", - path: "/api/folders", + headers: [ + { + name: "Authorization", + value: { source: "secret", secret_id: "token" }, + prefix: "Bearer ", + }, + ], }); }); @@ -220,66 +218,12 @@ describe("request step execution", () => { vi.unstubAllGlobals(); }); - it("sends the declared request through the page's fetch with its cookies", async () => { - const calls = stubFetch(201); - const parsed = scenario([ - requestStep({ - query: [{ name: "org", value: { source: "literal", value: "1" } }], - json: { title: "Scratch folder 1" }, - }), - ]); - await perform(fakePage({}), parsed.actions[0]!, resolved(parsed)); - expect(calls).toEqual([ - { - url: `${ORIGIN}/api/folders?org=1`, - init: expect.objectContaining({ - method: "POST", - credentials: "same-origin", - headers: [["content-type", "application/json"]], - body: '{"title":"Scratch folder 1"}', - }), - }, - ]); - }); - - it("reads header values from a cookie and from stored JSON, and redacts them afterwards", async () => { + it("sends requests with same-origin credentials", async () => { const calls = stubFetch(200); - const parsed = scenario([ - requestStep({ - headers: [ - { - name: "X-CSRFToken", - value: { source: "cookie", name: "csrftoken" }, - }, - { - name: "Authorization", - value: { - source: "local_storage", - key: "auth", - json_pointer: "/token", - }, - prefix: "Bearer ", - }, - ], - }), - ]); - const secrets = resolved(parsed); - await perform( - fakePage({ - cookies: [{ name: "csrftoken", value: "csrf-value-123" }], - storage: { auth: JSON.stringify({ token: "jwt-value-456" }) }, - }), - parsed.actions[0]!, - secrets, - ); - expect(calls[0]!.init.headers).toEqual([ - ["X-CSRFToken", "csrf-value-123"], - ["Authorization", "Bearer jwt-value-456"], - ]); - const redacted = secrets.redact("csrf-value-123 and jwt-value-456"); - expect(redacted).not.toContain("csrf-value-123"); - expect(redacted).not.toContain("jwt-value-456"); - expect(redacted).toContain("[REDACTED:page:create.X-CSRFToken]"); + const parsed = scenario([requestStep({})]); + await perform(fakePage({}), parsed.actions[0]!, resolved(parsed)); + expect(calls).toHaveLength(1); + expect(calls[0]!.init.credentials).toBe("same-origin"); }); it("refuses to run off the start origin, and fails on an unexpected status or a missing cookie", async () => { diff --git a/src/browser/history/SazMetadata.test.ts b/src/browser/history/SazMetadata.test.ts index 708858360c..76f88adf45 100644 --- a/src/browser/history/SazMetadata.test.ts +++ b/src/browser/history/SazMetadata.test.ts @@ -47,15 +47,27 @@ it.each([ }); it("retains U+FFFD in well-formed metadata as an ordinary XML character", () => { - const decoded = JSON.stringify( - decodeSazMetadata( - Buffer.from( - '�', - ), - ), + const decoded = decodeSazMetadata( + Buffer.from('�'), ); - expect(decoded).toContain('{"name":"V","value":"caf�"}'); - expect(decoded).toContain('"value":"�"'); + expect(decoded).toMatchObject({ + name: "#document", + children: [ + { + name: "Session", + children: [ + { + name: "SessionFlag", + attributes: [ + { name: "N", value: "ui-comments" }, + { name: "V", value: "caf�" }, + ], + }, + { name: "#text", value: "�" }, + ], + }, + ], + }); }); it.each([ @@ -74,10 +86,20 @@ it.each([ it("preserves valid comments, CDATA, declarations, and predefined entities", () => { const xml = 'A & B'; - const decoded = JSON.stringify(decodeSazMetadata(Buffer.from(xml))); - expect(decoded).toContain("note & literal"); - expect(decoded).toContain("A & B"); - expect(decoded).toContain("C & D"); + const decoded = decodeSazMetadata(Buffer.from(xml)); + expect(decoded).toMatchObject({ + children: [ + { name: "xml" }, + { + name: "Session", + children: [ + { name: "#comment", value: " note & literal " }, + { name: "Value", children: [{ name: "#text", value: "A & B" }] }, + { name: "C", children: [{ name: "#cdata-section", value: "C & D" }] }, + ], + }, + ], + }); }); it("rejects DTDs without resolving entities", () => { diff --git a/src/cliJsonPaths.test.ts b/src/cliJsonPaths.test.ts index f5bb133380..56d881cbf9 100644 --- a/src/cliJsonPaths.test.ts +++ b/src/cliJsonPaths.test.ts @@ -108,12 +108,14 @@ describe("CLI JSON path normalization", () => { "FILE:electron", "https://example.test/app", ])("keeps URL %j distinguishable from a local path", (value) => { - const input = { + const expected = { executable_path: value, application_path: resolve("/synthetic/app"), }; + const input = { ...expected }; const normalized = resolveCliJsonPaths(input, paths); - expect(normalized).toEqual(input); + expect(normalized).toEqual(expected); + expect(input).toEqual(expected); const parsed = electronActiveObservationInputSchema.safeParse(normalized); expect(parsed.success).toBe(false); if (parsed.success) throw new Error("Expected URL path rejection"); diff --git a/src/contracts/analysisDetail.test.ts b/src/contracts/analysisDetail.test.ts index 275abc7ce9..18292692b0 100644 --- a/src/contracts/analysisDetail.test.ts +++ b/src/contracts/analysisDetail.test.ts @@ -1,14 +1,18 @@ import { expect, it } from "vitest"; import { z } from "zod"; +import { Ajv2020 } from "ajv/dist/2020.js"; import { binaryLayoutSummaryViewResultSchema, javascriptApplicationSummaryViewResultSchema, projectAnalysisView, + analysisViewRequestSchema, } from "../domain/analysisView/analysisView.js"; import { + analysisViewCaptureEvidence, analysisViewJavaScriptEvidence, analysisViewLayoutEvidence, + environmentStateCaptureFixture, } from "../../tests/fixtures/analysisView.js"; import { toolContract } from "./toolContracts.js"; @@ -29,25 +33,6 @@ const projectSummary = (parent: { ); it("advertises only that operation's summary from analysis detail outputs", () => { - for (const name of [ - "inspect_binary_layout", - "analyze_javascript_application", - ] as const) { - const schema = JSON.stringify( - z.toJSONSchema(toolContract(name).outputSchema), - ); - expect(schema).not.toContain("api-surface"); - expect(schema).not.toContain("environment-manifest"); - expect(schema).not.toContain("environment-state"); - expect(schema).not.toContain("inspect_web_network_capture"); - } - const viewSchema = JSON.stringify( - z.toJSONSchema(toolContract("inspect_analysis_view").outputSchema), - ); - expect(viewSchema).toContain("api-surface"); - expect(viewSchema).toContain("environment-manifest"); - expect(viewSchema).toContain("environment-state"); - const layout = projectSummary(analysisViewLayoutEvidence()); const javascript = projectSummary(analysisViewJavaScriptEvidence()); if (!layout.ok || !javascript.ok) throw new Error("Expected summaries"); @@ -61,4 +46,84 @@ it("advertises only that operation's summary from analysis detail outputs", () = expect( binaryLayoutSummaryViewResultSchema.safeParse(javascript.value).success, ).toBe(false); + + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const validators = Object.fromEntries( + ( + [ + "inspect_binary_layout", + "analyze_javascript_application", + "inspect_analysis_view", + ] as const + ).map((name) => [ + name, + ajv.compile(z.toJSONSchema(toolContract(name).outputSchema)), + ]), + ); + const layoutReceipt = { + ...analysisViewLayoutEvidence(), + normalized_result: layout.value, + }; + const javascriptReceipt = { + ...analysisViewJavaScriptEvidence(), + normalized_result: javascript.value, + }; + expect(validators.inspect_binary_layout!(layoutReceipt)).toBe(true); + expect(validators.analyze_javascript_application!(javascriptReceipt)).toBe( + true, + ); + const parent = analysisViewCaptureEvidence(environmentStateCaptureFixture()); + const captureParent = { + evidenceId: parent.evidence_id, + operation: parent.operation, + normalizedResult: parent.normalized_result, + parameters: parent.parameters, + subject: parent.subject, + limitations: parent.limitations, + }; + const captureSummary = projectAnalysisView(captureParent, { + kind: "summary", + }); + if (!captureSummary.ok) throw captureSummary.error; + const captureReceipt = { ...parent, normalized_result: captureSummary.value }; + expect(validators.inspect_analysis_view!(captureReceipt)).toBe(true); + expect( + validators.inspect_binary_layout!({ + ...layoutReceipt, + normalized_result: captureSummary.value, + }), + ).toBe(false); + expect( + validators.analyze_javascript_application!({ + ...javascriptReceipt, + normalized_result: captureSummary.value, + }), + ).toBe(false); + for (const kind of [ + "api-surface", + "environment-manifest", + "environment-state", + ] as const) { + const projected = projectAnalysisView( + captureParent, + analysisViewRequestSchema.parse({ kind }), + ); + if (!projected.ok) throw projected.error; + const receipt = { ...parent, normalized_result: projected.value }; + expect(validators.inspect_analysis_view!(receipt), kind).toBe(true); + expect( + validators.inspect_binary_layout!({ + ...layoutReceipt, + normalized_result: projected.value, + }), + kind, + ).toBe(false); + expect( + validators.analyze_javascript_application!({ + ...javascriptReceipt, + normalized_result: projected.value, + }), + kind, + ).toBe(false); + } }); diff --git a/src/doctorRuntime.test.ts b/src/doctorRuntime.test.ts index a3d9a377d3..2ca01b0e8d 100644 --- a/src/doctorRuntime.test.ts +++ b/src/doctorRuntime.test.ts @@ -2,6 +2,8 @@ import { chmod, mkdir, realpath, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it as test, vi } from "vitest"; +import { withVerifierCleanup } from "../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../tests/fixtures/lifecycleAcknowledgement.js"; import { createSystemDoctorHost } from "./doctorRuntime.js"; import { DOCTOR_PROVIDER_IDS } from "./application/DoctorProviderIds.js"; @@ -273,8 +275,8 @@ test.each([true, false])( .finally(() => { settled = true; }); - try { - await entered; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, pending, "display selection"); expect(selected.mock.calls).toHaveLength(1); if (cancel) { controller.abort(reason); @@ -295,13 +297,17 @@ test.each([true, false])( expect.objectContaining({ name: "hopper-demo-runtime", ok: true }), ); } - } finally { - release(); - await pending; - await system.close?.(); - inspected.mockRestore(); - selected.mockRestore(); - } + }, [ + release, + async () => { + await pending; + }, + async () => { + await system.close?.(); + }, + () => inspected.mockRestore(), + () => selected.mockRestore(), + ]); }, 5_000, ); @@ -340,8 +346,8 @@ test("joins display finalization before preserving a late cancellation", async ( .finally(() => { settled = true; }); - try { - await entered; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, pending, "display finalization"); controller.abort(reason); await Promise.resolve(); expect(settled).toBe(false); @@ -351,13 +357,17 @@ test("joins display finalization before preserving a late cancellation", async ( expect(failure).toBeInstanceOf(AnalysisCancelledError); expect((failure as Error).cause).toBe(reason); expect(inspected).not.toHaveBeenCalled(); - } finally { - release(); - await pending; - await system.close?.(); - inspected.mockRestore(); - selected.mockRestore(); - } + }, [ + release, + async () => { + await pending; + }, + async () => { + await system.close?.(); + }, + () => inspected.mockRestore(), + () => selected.mockRestore(), + ]); }); test("rejects a pre-aborted caller before the Hopper display check", async () => { diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index d52c2433fc..a26e5658fd 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -782,9 +782,15 @@ it("projects JavaScript summary and preserves selected module observations", () expect(item.value.item).toMatchObject({ path: "renderer.js", kind: "javascript-asset", + observations: expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + source: "export const secret = 1;\n", + text: "should not leak", + }), + }), + ]), }); - expect(JSON.stringify(item.value.item)).toContain("export const secret = 1;"); - expect(JSON.stringify(item.value.item)).toContain("should not leak"); const page = projectAnalysisView(parent, { kind: "page", collection: "modules", diff --git a/src/domain/analysisView/environmentManifestView.test.ts b/src/domain/analysisView/environmentManifestView.test.ts index 995b737667..fdbdbbe93b 100644 --- a/src/domain/analysisView/environmentManifestView.test.ts +++ b/src/domain/analysisView/environmentManifestView.test.ts @@ -351,6 +351,9 @@ it("derives value-free body shapes, observed query names and record pointers", ( "status", ]); const silent = manifest([], environmentManifestCaptureFixture(), false); + expect(silent.tools.map((tool) => tool.name)).toEqual( + projected.tools.map((tool) => tool.name), + ); expect( silent.tools.every((tool) => tool.query_parameter_names.length === 0), ).toBe(true); diff --git a/src/domain/callPath.test.ts b/src/domain/callPath.test.ts index 2e0c590cee..238cd5add9 100644 --- a/src/domain/callPath.test.ts +++ b/src/domain/callPath.test.ts @@ -180,6 +180,7 @@ describe("call path reconstruction", () => { expect( result.paths.map((path) => path.nodes.map(({ address }) => address)), ).toEqual([["0x1000", "0x2000", "0x4000"]]); + expect(result.paths[0]?.edges).toHaveLength(2); expect( result.paths[0]?.edges.every((edge) => edge.evidence_links.length > 0), ).toBe(true); diff --git a/src/domain/electron/electronFuses.test.ts b/src/domain/electron/electronFuses.test.ts index c074ae1763..9468a96017 100644 --- a/src/domain/electron/electronFuses.test.ts +++ b/src/domain/electron/electronFuses.test.ts @@ -155,15 +155,7 @@ describe("Electron fuse wire decoding", () => { }); }); - it("does not treat a missing sentinel or a short wire as disabled fuses", () => { - const missing = describeElectronFuseFindings([slice([])], "mach-o"); - expect(missing.runtime_implications).toEqual([]); - expect(missing.limitations.join(" ")).toContain( - "unknown, not that every fuse is disabled", - ); - expect(missing.limitations.join(" ")).toContain( - "fuse state for that slice is unknown, not disabled", - ); + it("does not treat a short wire as disabled fuses", () => { const truncated = describeElectronFuseFindings( [ slice([ diff --git a/src/domain/evidenceCompletionLedger.test.ts b/src/domain/evidenceCompletionLedger.test.ts index 08e848349d..f5e8c6d21a 100644 --- a/src/domain/evidenceCompletionLedger.test.ts +++ b/src/domain/evidenceCompletionLedger.test.ts @@ -56,6 +56,10 @@ describe("Evidence completion ledger", () => { ]); expect(left).toEqual(right); + expect(left.records).toEqual([ + record("scenario.a", "unknown", [evidenceId("3")]), + record("scenario.z", "pass", [evidenceId("1"), evidenceId("2")]), + ]); expect(left.ledger_id).toMatch(/^ecl_[a-f0-9]{64}$/u); }); @@ -67,9 +71,13 @@ describe("Evidence completion ledger", () => { expect( createEvidenceCompletionLedger([ - record("scenario.many-evidence-items", "pass", evidenceIds), + record( + "scenario.many-evidence-items", + "pass", + [...evidenceIds].reverse(), + ), ]).records[0]?.evidence_ids, - ).toHaveLength(evidenceIds.length); + ).toEqual(evidenceIds); }); it("rejects missing, malformed, and duplicate Evidence IDs", () => { diff --git a/src/domain/explicitSensitiveFailure.test.ts b/src/domain/explicitSensitiveFailure.test.ts index 2871960ce3..c02093672e 100644 --- a/src/domain/explicitSensitiveFailure.test.ts +++ b/src/domain/explicitSensitiveFailure.test.ts @@ -113,11 +113,15 @@ it("preserves cleanup failure authority and resources while excluding diagnostic const projected = projectAnalysisError( redactExplicitFailure(error, ["private", "REDACTED"]), ); - expect(projected.code).toBe("cleanup_incomplete"); + expect(projected).toMatchObject({ + code: "cleanup_incomplete", + details: { + resources: expect.arrayContaining(["pid:123"]), + diagnostics: { exit_code: 2 }, + }, + }); expect(JSON.stringify(projected)).not.toContain("private"); expect(JSON.stringify(projected)).not.toContain("REDACTED"); - expect(JSON.stringify(projected)).toContain("pid:123"); - expect(JSON.stringify(projected)).toContain('"exit_code":2'); }); it("redacts cleanup primaries and mirrors the source error's live status", () => { diff --git a/src/domain/firmware/espImage.test.ts b/src/domain/firmware/espImage.test.ts index 380f248095..394ec8b644 100644 --- a/src/domain/firmware/espImage.test.ts +++ b/src/domain/firmware/espImage.test.ts @@ -108,82 +108,8 @@ describe("ESP image recognition", () => { }); describe("ESP image decoding", () => { - it("reports header, chip, segment, integrity and descriptor facts", () => { - const decoded = image(APP); - expect(decoded.metadata).toEqual({ - kind: "esp-image", - header: { - segment_count: 4, - spi_mode: 2, - spi_speed: 0, - spi_size: 2, - entry_addr: 0x4038_0000, - wp_pin: 0xee, - spi_pin_drv: "000000", - chip_id: 5, - min_chip_rev: 0, - min_chip_rev_full: 0, - max_chip_rev_full: 199, - reserved: "00000000", - hash_appended: true, - }, - chip: { target: "esp32c3", isa: "riscv", source: "ESP-IDF v6.1" }, - // Segments end at 456; the checksum ends the next 16-byte block. - checksum: { - offset: 463, - stored: espChecksum(ESP32C3_APP_SEGMENTS), - computed: espChecksum(ESP32C3_APP_SEGMENTS), - status: "valid", - }, - appended_sha256: { - offset: 464, - stored: sha256(APP.subarray(0, 464)), - computed: sha256(APP.subarray(0, 464)), - status: "valid", - }, - image_end: 496, - signature_sector_offset: null, - undescribed_trailing: [], - descriptor: { - kind: "app", - offset: 32, - observed_length: 256, - secure_version: 0, - version: "1.2.3", - project_name: "rea_esp_fixture", - time: "12:34:56", - date: "Oct 27 2026", - idf_ver: "v6.1", - app_elf_sha256: "ab".repeat(32), - min_efuse_blk_rev_full: 0, - max_efuse_blk_rev_full: 199, - mmu_page_size: 16, - spi_flash_mode: 2, - }, - flash_dump: null, - }); - expect( - decoded.segments.map(({ dataOffset, availableLength, metadata }) => [ - metadata.index, - metadata.load_addr, - metadata.data_len, - metadata.header_offset, - dataOffset, - availableLength, - metadata.classification, - ]), - ).toEqual([ - [0, 0x3c00_0020, 288, 24, 32, 288, "mapped"], - [1, 0x3fc8_8000, 16, 320, 328, 16, "loaded"], - [2, 0x4038_0000, 32, 344, 352, 32, "loaded"], - [3, 0x4200_0020, 64, 384, 392, 64, "mapped"], - ]); - expect(decoded.coverage).toBe("complete"); - expect(decoded.limitations).toEqual([]); - expect(decoded.signatureBlocks).toEqual([]); - }); - it("decodes identically from any chunking", () => { + expect(image(APP).metadata.flash_dump).toBeNull(); const signed = signedEspImage(APP, ["rsa", null, "ecdsa-p384"]); const expected = image(signed, signed.length); for (const chunkBytes of [1, 3, 16, 4096]) diff --git a/src/domain/firmware/hexRecords.test.ts b/src/domain/firmware/hexRecords.test.ts index 6a3e0b040c..e24bce67dc 100644 --- a/src/domain/firmware/hexRecords.test.ts +++ b/src/domain/firmware/hexRecords.test.ts @@ -399,21 +399,6 @@ describe("hex record recognition", () => { }); }); -it.each([3, 5])( - "rejects a nonzero reserved offset in Intel HEX start record %i", - (type) => { - const text = [ - intelHexRecord(0, 0x20, [0xa5]), - intelHexRecord(type, 0x1234, [0, 0, 0, 1]), - intelHexRecord(1, 0), - ].join("\n"); - expect(decode(text)).toMatchObject({ - ok: false, - error: { kind: "format" }, - }); - }, -); - it("keeps data offsets, start payloads and the ignored EOF offset distinct", () => { const parsed = image( [ diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index cdb2f9f8df..f1cdd12cfe 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -596,7 +596,11 @@ describe("function collection collation ties", () => { const calls = result.dimensions.find( ({ dimension }) => dimension === "calls", ); - expect(calls).toMatchObject({ status: "changed" }); + expect(calls).toMatchObject({ + status: "changed", + left_digest: expect.any(String), + right_digest: expect.any(String), + }); expect(calls?.left_digest).not.toBe(calls?.right_digest); }); @@ -1143,7 +1147,11 @@ describe("function comparison identity and malformed CFG", () => { const pseudocode = comparison.dimensions.find( ({ dimension }) => dimension === "pseudocode", ); - expect(pseudocode).toMatchObject({ status: "changed" }); + expect(pseudocode).toMatchObject({ + status: "changed", + left_digest: expect.any(String), + right_digest: expect.any(String), + }); expect(pseudocode?.left_digest).not.toBe(pseudocode?.right_digest); }); }); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts index 6626bf03f7..5adc82e75a 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.builtin_defaults.test.ts @@ -175,9 +175,9 @@ it.each([ require('node:fs').readFileSync('also-wrong'); `); expect(ir.configurationOperations).toEqual([]); - expect( - ir.bindings.find(({ name }) => name === "fs")?.provenance.status, - ).not.toBe("module"); + const binding = ir.bindings.find(({ name }) => name === "fs"); + expect(binding).toBeDefined(); + expect(binding?.provenance.status).not.toBe("module"); }); it("keeps local loader reassignment separate from the ambient loader", () => { diff --git a/src/domain/javascript/javascriptSemanticAnalysis.calls.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.calls.test.ts index 148cf73fef..0f80b47dca 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.calls.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.calls.test.ts @@ -336,25 +336,25 @@ describe("JavaScript semantic analysis: computed method keys", () => { }); }); -describe("JavaScript semantic analysis: calls 2", () => { - it("does not treat a shadowed Promise binding as the intrinsic", () => { - const ir = analyzeJavaScriptSemantics(` +it("does not treat a shadowed Promise binding as the intrinsic", () => { + const ir = analyzeJavaScriptSemantics(` function run(Promise) { return Promise.resolve(1).then(work); } `); - expect(ir.promiseOperations).toEqual([ - expect.objectContaining({ - kind: "chain", - method: "then", - ownership: "returned", - sourcePromiseIds: [], - sourceResolution: "unresolved", - }), - ]); - }); + expect(ir.promiseOperations).toEqual([ + expect.objectContaining({ + kind: "chain", + method: "then", + ownership: "returned", + sourcePromiseIds: [], + sourceResolution: "unresolved", + }), + ]); +}); +describe("JavaScript semantic analysis: callable fingerprints", () => { it("fingerprints formatting and local-name changes identically", () => { const left = analyzeJavaScriptSemantics(` function calculate(value) { @@ -371,9 +371,16 @@ describe("JavaScript semantic analysis: calls 2", () => { expect(left.functionFingerprints).toHaveLength(1); expect(right.functionFingerprints).toHaveLength(1); + expect(changed.functionFingerprints).toHaveLength(1); expect(left.functionFingerprints[0]?.components).toEqual( right.functionFingerprints[0]?.components, ); + expect( + changed.functionFingerprints[0]?.components.literalSetSha256, + ).toEqual(expect.any(String)); + expect(left.functionFingerprints[0]?.components.literalSetSha256).toEqual( + expect.any(String), + ); expect( changed.functionFingerprints[0]?.components.literalSetSha256, ).not.toBe(left.functionFingerprints[0]?.components.literalSetSha256); @@ -432,11 +439,13 @@ describe("JavaScript semantic analysis: calls 2", () => { ), ); - expect( - fingerprint(ordered, "ordered")?.components.relationShapeSha256, - ).not.toBe( - fingerprint(reversed, "ordered")?.components.relationShapeSha256, - ); + const orderedDigest = fingerprint(ordered, "ordered")?.components + .relationShapeSha256; + const reversedDigest = fingerprint(reversed, "ordered")?.components + .relationShapeSha256; + expect(orderedDigest).toEqual(expect.any(String)); + expect(reversedDigest).toEqual(expect.any(String)); + expect(orderedDigest).not.toBe(reversedDigest); expect(fingerprint(ordered, "timerOwner")?.components.effects).toContain( "timer", ); @@ -451,9 +460,10 @@ describe("JavaScript semantic analysis: calls 2", () => { fingerprint(ordered, "networkOwner")?.components.effects, ).not.toContain("timer"); }); +}); - it("retains ambiguous local callees and explicit dynamic frontiers", () => { - const ir = analyzeJavaScriptSemantics(` +it("retains ambiguous local callees and explicit dynamic frontiers", () => { + const ir = analyzeJavaScriptSemantics(` function left(value) { return value; } function right(value) { return value; } const selected = chooseLeft ? left : right; @@ -462,37 +472,36 @@ describe("JavaScript semantic analysis: calls 2", () => { external(input); `); - const selected = ir.callSites.find( - ({ resolution }) => resolution === "ambiguous", - ); - expect(selected?.calleeCallableIds).toEqual( - [ - onlyCallable(ir, "left").callableId, - onlyCallable(ir, "right").callableId, - ].sort(), - ); - expect( - ir.argumentFlows.filter( - ({ callSiteId }) => callSiteId === selected?.callSiteId, - ), - ).toHaveLength(2); - expect(ir.frontiers).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - kind: "dynamic-call", - reason: expect.stringMatching(/ambiguous/iu), - }), - expect.objectContaining({ - kind: "dynamic-property", - reason: expect.stringMatching(/computed member/iu), - }), - expect.objectContaining({ - kind: "dynamic-call", - reason: "Unresolved call target external.", - }), - ]), - ); - }); + const selected = ir.callSites.find( + ({ resolution }) => resolution === "ambiguous", + ); + expect(selected?.calleeCallableIds).toEqual( + [ + onlyCallable(ir, "left").callableId, + onlyCallable(ir, "right").callableId, + ].sort(), + ); + expect( + ir.argumentFlows.filter( + ({ callSiteId }) => callSiteId === selected?.callSiteId, + ), + ).toHaveLength(2); + expect(ir.frontiers).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "dynamic-call", + reason: expect.stringMatching(/ambiguous/iu), + }), + expect.objectContaining({ + kind: "dynamic-property", + reason: expect.stringMatching(/computed member/iu), + }), + expect.objectContaining({ + kind: "dynamic-call", + reason: "Unresolved call target external.", + }), + ]), + ); }); describe("JavaScript semantic rest parameter flow", () => { diff --git a/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts index 6ec889bff4..f2331fcd2e 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.dataflow.test.ts @@ -473,9 +473,9 @@ describe("JavaScript semantic analysis: mapped arguments writes", () => { .returnSites[0]?.value; if (value?.status !== "object") throw new Error("Expected an object return value"); - expect( - value.properties.find(({ name }) => name === "value")?.value.status, - ).not.toBe("literal"); + const property = value.properties.find(({ name }) => name === "value"); + expect(property).toBeDefined(); + expect(property?.value.status).not.toBe("literal"); }); }); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.fingerprints.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.fingerprints.test.ts index ce88236a50..3714ac44d8 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.fingerprints.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.fingerprints.test.ts @@ -59,6 +59,8 @@ it("keeps a nested function's directive out of its owner fingerprint", () => { const second = analyze('"ordinary"'); expect(first.size).toBe(2); expect(second.get("outer")).toEqual(first.get("outer")); + expect(first.get("inner")?.literalSetSha256).toEqual(expect.any(String)); + expect(second.get("inner")?.literalSetSha256).toEqual(expect.any(String)); expect(second.get("inner")?.literalSetSha256).not.toBe( first.get("inner")?.literalSetSha256, ); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.initialization_boundaries.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.initialization_boundaries.test.ts index 748a83477a..767ad1b740 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.initialization_boundaries.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.initialization_boundaries.test.ts @@ -53,9 +53,9 @@ it.each([ ])( "indexes the legacy block-function write on its outer owner: %s", (source) => { - expect( - onlyCallable(analyze(source), "read").returnSites[0]?.value.status, - ).not.toBe("literal"); + const returnSite = onlyCallable(analyze(source), "read").returnSites[0]; + expect(returnSite).toBeDefined(); + expect(returnSite?.value.status).not.toBe("literal"); }, ); @@ -246,5 +246,6 @@ it("owns a class declaration's self reference separately from its outer variable it("does not initialize a class self name during computed-key evaluation", () => { const ir = analyze(`class Service { [Service()]() {} }`, "fixture.mjs"); + expect(ir.callSites[0]).toBeDefined(); expect(ir.callSites[0]?.resolution).not.toBe("exact"); }); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.structure.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.structure.test.ts index ad050270b8..c412ac9413 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.structure.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.structure.test.ts @@ -210,6 +210,7 @@ describe("JavaScript semantic analysis: loop lexical names", () => { ({ name, role }) => name === "value" && role === "read", ); expect(reads).toHaveLength(2); + expect(reads[0]?.bindingId).toEqual(expect.any(String)); expect(reads[0]?.bindingId).not.toBe(reads[1]?.bindingId); expect(reads[1]?.bindingId).toBe(topLevelBinding(ir, "value").bindingId); }); @@ -224,6 +225,7 @@ describe("JavaScript semantic analysis: loop lexical names", () => { ({ name, role }) => name === "value" && role === "read", ); expect(reads).toHaveLength(2); + expect(reads[0]?.bindingId).toEqual(expect.any(String)); expect(reads[0]?.bindingId).not.toBe( topLevelBinding(ir, "value").bindingId, ); @@ -242,11 +244,14 @@ describe("JavaScript semantic analysis: loop lexical names", () => { const binding = onlyBinding(ir, "value"); const callable = ir.callables.find(({ name }) => name === "run"); expect(binding.scopeId).toBe(callable?.bodyScopeId); - expect( - ir.references - .filter(({ name, role }) => name === "value" && role === "read") - .every(({ bindingId }) => bindingId === binding.bindingId), - ).toBe(true); + const reads = ir.references.filter( + ({ name, role }) => name === "value" && role === "read", + ); + expect(reads).toHaveLength(2); + expect(reads.map(({ bindingId }) => bindingId)).toEqual([ + binding.bindingId, + binding.bindingId, + ]); }); }); @@ -268,6 +273,9 @@ describe("JavaScript semantic analysis: class expression lexical names", () => { ({ name, role }) => name === "Service" && role === "read", ); expect(references).toHaveLength(2); + expect(references[1]?.bindingId).toBe( + topLevelBinding(ir, "Service").bindingId, + ); expect(references[0]?.bindingId).not.toBe(references[1]?.bindingId); expect(references[0]?.resolution).toBe("resolved"); const classScope = ir.scopes.find(({ kind }) => kind === "class"); diff --git a/src/domain/javascript/javascriptSemanticCallResolution.test.ts b/src/domain/javascript/javascriptSemanticCallResolution.test.ts index af65fb35ac..e96889419a 100644 --- a/src/domain/javascript/javascriptSemanticCallResolution.test.ts +++ b/src/domain/javascript/javascriptSemanticCallResolution.test.ts @@ -54,6 +54,7 @@ it("orders exported callable initializers within their enclosing declaration", ( resolution: "exact", calleeCallableIds: [original.callableId], }); + expect(ir.callSites[1]).toBeDefined(); expect(ir.callSites[1]?.resolution).not.toBe("exact"); }); diff --git a/src/domain/javascript/javascriptSemanticSharedChildWrites.test.ts b/src/domain/javascript/javascriptSemanticSharedChildWrites.test.ts index 9c2c36f1d0..2ecac92443 100644 --- a/src/domain/javascript/javascriptSemanticSharedChildWrites.test.ts +++ b/src/domain/javascript/javascriptSemanticSharedChildWrites.test.ts @@ -202,12 +202,14 @@ it.each([ : (child: { loader: unknown }) => { if (!preserveHost) child.loader = {}; }; + let thrown: unknown; try { runInNewContext(source, { module, external }); } catch (error) { - expect(error).toMatchObject({ name: "TypeError" }); - expect(preserveHost).toBe(false); + thrown = error; } + if (preserveHost) expect(thrown).toBeUndefined(); + else expect(thrown).toMatchObject({ name: "TypeError" }); expect(calls).toEqual(preserveHost ? ["./uncertain.cjs"] : []); } }, diff --git a/src/domain/javascript/javascriptSymbolicBindings.test.ts b/src/domain/javascript/javascriptSymbolicBindings.test.ts index 6ff6988d64..170a6cf8fd 100644 --- a/src/domain/javascript/javascriptSymbolicBindings.test.ts +++ b/src/domain/javascript/javascriptSymbolicBindings.test.ts @@ -368,13 +368,31 @@ it("shares completed factory control flow across repeated loads within one analy }, }); expect(results).toHaveLength(64); - expect( - results.every((result) => - symbolicVariants(result).every( - (variant, index) => variant === symbolicVariants(results[0]!)[index], + const variants = symbolicVariants(results[0]!); + expect(variants).toHaveLength(2); + expect(variants).toEqual( + expect.arrayContaining( + [true, false].map((positive) => + expect.objectContaining({ + kind: "namespace", + conditions: expect.arrayContaining([ + { + node: expect.objectContaining({ + type: "Identifier", + name: "flag", + }), + positive, + }, + ]), + }), ), ), - ).toBe(true); + ); + for (const result of results) { + const actual = symbolicVariants(result); + expect(actual).toHaveLength(variants.length); + actual.forEach((variant, index) => expect(variant).toBe(variants[index])); + } expect( evaluated.mock.calls.filter(([node]) => t.isIdentifier(node, { name: "flag" }), diff --git a/src/domain/native/nativeAotPe.budget.test.ts b/src/domain/native/nativeAotPe.budget.test.ts index b7fb540ee5..39ed862fef 100644 --- a/src/domain/native/nativeAotPe.budget.test.ts +++ b/src/domain/native/nativeAotPe.budget.test.ts @@ -201,7 +201,7 @@ describe("NativeAOT detail and summary byte accounting", () => { budget, ); const complete = detail(REPORT_BUDGET); - expect(complete?.derived_memory).not.toBeNull(); + expect(complete?.derived_memory).toEqual(expect.any(Object)); let low = 0; let high = Buffer.byteLength(JSON.stringify(complete)); while (low < high) { @@ -211,7 +211,7 @@ describe("NativeAOT detail and summary byte accounting", () => { } const atBoundary = detail(low); const belowBoundary = detail(low - 1); - expect(atBoundary?.derived_memory).not.toBeNull(); + expect(atBoundary?.derived_memory).toEqual(expect.any(Object)); expect(Buffer.byteLength(JSON.stringify(atBoundary))).toBeLessThanOrEqual( low, ); diff --git a/src/domain/referenceSourceGraph.test.ts b/src/domain/referenceSourceGraph.test.ts index 37cfa4f8f8..bbcf6a48fe 100644 --- a/src/domain/referenceSourceGraph.test.ts +++ b/src/domain/referenceSourceGraph.test.ts @@ -228,6 +228,9 @@ describe("historical source graph", () => { it("omits absent local diagnostics and rejects explicitly undefined non-JSON metadata", () => { const input = graphInput(); const graph = createHistoricalSourceGraph(input); + expect(graph.entries.map((entry) => entry.path)).toEqual( + input.entries.map((entry) => entry.path), + ); expect( graph.entries.every( (entry) => !Object.hasOwn(entry, "local_diagnostics"), diff --git a/src/domain/webBundleAnalyzer.test.ts b/src/domain/webBundleAnalyzer.test.ts index 1c086eb01c..c2241777c0 100644 --- a/src/domain/webBundleAnalyzer.test.ts +++ b/src/domain/webBundleAnalyzer.test.ts @@ -467,7 +467,6 @@ describe("web bundle analyzer", () => { expect.objectContaining({ value: "Vue" }), ]), ); - expect(JSON.stringify(result)).toContain("authorization=secret"); expect(result.completeness.status).toBe("complete"); }); diff --git a/src/domain/webScriptExportPlan.test.ts b/src/domain/webScriptExportPlan.test.ts index 8c774cd4d8..ca0cf3d20f 100644 --- a/src/domain/webScriptExportPlan.test.ts +++ b/src/domain/webScriptExportPlan.test.ts @@ -38,7 +38,7 @@ describe("captured script path projection", () => { const paths = planned.map(({ record }) => record.content.state === "exported" ? record.content.relative_path : "", ); - expect(paths[0]).toMatch(/^origins\/[a-f0-9]{64}\/app\/main.js$/u); + expect(paths[0]).toMatch(/^origins\/[a-f0-9]{64}\/app\/main\.js$/u); expect(paths[1]?.replace("lib/value.mjs", "main.js")).toBe(paths[0]); expect(paths[2]).not.toBe(paths[0]); }); diff --git a/src/dotnet/ManagedMemberPointers.test.ts b/src/dotnet/ManagedMemberPointers.test.ts index 35764177f7..736151bd99 100644 --- a/src/dotnet/ManagedMemberPointers.test.ts +++ b/src/dotnet/ManagedMemberPointers.test.ts @@ -20,62 +20,6 @@ const metadataLayout = (bytes: Buffer) => { }; describe("managed member pointer ownership", () => { - it.each([ - { table: "fields" as const, prefix: "0x040000" }, - { table: "methods" as const, prefix: "0x060000" }, - ])("maps reordered pointer slots to physical $table", ({ table, prefix }) => { - const bytes = buildManagedMemberPointerFixture(); - const result = inspectManagedMembersBytes( - bytes, - managedPeFixtureTarget(bytes), - ); - - expect(result.coverage).toEqual({ state: "complete", issues: [] }); - expect(result.types.map((type) => type.full_name)).toEqual([ - "", - "Fixture.A", - "Fixture.B", - ]); - expect( - result[table].map(({ token, declaring_type, declaring_type_token }) => ({ - token, - declaring_type, - declaring_type_token, - })), - ).toEqual([ - { - token: `${prefix}01`, - declaring_type: "Fixture.B", - declaring_type_token: "0x02000003", - }, - { - token: `${prefix}02`, - declaring_type: "Fixture.A", - declaring_type_token: "0x02000002", - }, - ]); - }); - - it("resolves a local custom-attribute constructor through MethodPtr", () => { - const bytes = buildManagedMemberPointerFixture(); - const target = managedPeFixtureTarget(bytes); - const artifact = inspectManagedArtifactBytes(bytes, target); - const members = inspectManagedMembersBytes(bytes, target); - - expect(artifact.metadata.stream_names).toContain("#-"); - expect(artifact.coverage).toEqual({ state: "complete", issues: [] }); - expect(artifact.attributes).toEqual([ - expect.objectContaining({ - parent_token: "0x20000001", - constructor_token: "0x06000001", - type_name: "Fixture.B", - }), - ]); - expect(artifact.attributes[0]?.type_name).toBe( - members.methods[0]?.declaring_type, - ); - }); - it.each([ { label: "identity pointer tables", diff --git a/src/electron/ElectronFuseCleanup.test.ts b/src/electron/ElectronFuseCleanup.test.ts index efb5399cc0..9833db8dbc 100644 --- a/src/electron/ElectronFuseCleanup.test.ts +++ b/src/electron/ElectronFuseCleanup.test.ts @@ -5,6 +5,8 @@ import { syncBuiltinESMExports } from "node:module"; import * as fileSystem from "node:fs/promises"; import { dirname, join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; import { thinFuseImage } from "../../tests/fixtures/electron/fuseImage.js"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; @@ -375,8 +377,8 @@ it("awaits an admitted preflight's final close before projecting cancellation", settled = true; return result; }); - try { - await closeEntered; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(closeEntered, running, "preflight close"); await new Promise((resolve) => setImmediate(resolve)); expect(settled).toBe(false); releaseClose(); @@ -396,13 +398,17 @@ it("awaits an admitted preflight's final close before projecting cancellation", await provider.close(); expect(result.error.cleanupIncomplete).toBe(false); expect(retry.error.cleanupIncomplete).toBe(false); - } finally { - releaseClose(); - denied = false; - await running; - await provider.close(); - await fixture.release(); - } + }, [ + () => { + releaseClose(); + denied = false; + }, + async () => { + await running; + }, + () => provider.close(), + () => fixture.release(), + ]); }); it.each(["admission", "close"] as const)( diff --git a/src/erlang/BeamParser.test.ts b/src/erlang/BeamParser.test.ts index 07775fab3c..ce304aa202 100644 --- a/src/erlang/BeamParser.test.ts +++ b/src/erlang/BeamParser.test.ts @@ -33,54 +33,6 @@ const parse = (bytes: Buffer, input = defaults) => input, ); -it("reads atoms, exports, imports, lambdas and the Code header", () => { - const report = parse(erlangBeamFixture("demo")); - expect(report.atom_encoding).toBe("utf8"); - expect(report.module).toBe("demo"); - expect(report.atoms?.map((atom) => atom.name)).toEqual([ - "demo", - "module_info", - "start", - "erlang", - "get_module_info", - ]); - expect(report.exports).toEqual([ - { function_index: 3, function: "start", arity: 0, label: 2 }, - { function_index: 2, function: "module_info", arity: 0, label: 1 }, - { function_index: 2, function: "module_info", arity: 1, label: 3 }, - ]); - expect(report.imports).toEqual([ - { - module_index: 4, - module: "erlang", - function_index: 5, - function: "get_module_info", - arity: 1, - }, - ]); - expect(report.lambdas?.[0]).toMatchObject({ - function: "start", - arity: 1, - label: 4, - old_unique: 42, - }); - expect(report.code).toMatchObject({ - instruction_set: 0, - opcode_max: 169, - labels: 4, - functions: 2, - extra_header_bytes: 0, - instructions: { bytes: 4 }, - }); - expect(report.chunks.map((chunk) => chunk.name)).toEqual([ - "AtU8", - "Code", - "ExpT", - "ImpT", - "FunT", - ]); -}); - it("selects the last table and Code payload for every repeated chunk name", () => { const original = erlangBeamFixture("selected"); const decoy = erlangBeamFixture("decoy"); @@ -236,41 +188,6 @@ it.each([Buffer.from([0]), Buffer.from([0, 0, 0, 1])])( }, ); -// Expected values come from beam_lib:chunks/2 on the same file under OTP 29. -it("reads a compiler-produced OTP 29 BEAM with the compact atom table", () => { - const report = parse(erlangBeamOtp29Fixture()); - expect(report.module).toBe("rea_beam_fixture"); - expect(report.atoms?.map((atom) => atom.name)).toEqual([ - "rea_beam_fixture", - "hello", - "lists", - "reverse", - "module_info", - "erlang", - "get_module_info", - ]); - expect(report.exports).toEqual([ - { function_index: 5, function: "module_info", arity: 1, label: 6 }, - { function_index: 5, function: "module_info", arity: 0, label: 4 }, - { function_index: 2, function: "hello", arity: 1, label: 2 }, - ]); - expect( - report.imports?.map((row) => [row.module, row.function, row.arity]), - ).toEqual([ - ["lists", "reverse", 1], - ["erlang", "get_module_info", 1], - ["erlang", "get_module_info", 2], - ]); - expect(report.code).toMatchObject({ - instruction_set: 0, - opcode_max: 181, - labels: 7, - functions: 3, - extra_header_bytes: 0, - instructions: { bytes: 49 }, - }); -}); - it("accepts extended compact atom lengths and rejects truncated or overflowing values", () => { const compactAtom = (length: Buffer, name: Buffer): Buffer => erlangBeamChunksFixture([ diff --git a/src/evm/EvmBytecodeCarrier.test.ts b/src/evm/EvmBytecodeCarrier.test.ts index 5f0bd50ea9..69ae9b388a 100644 --- a/src/evm/EvmBytecodeCarrier.test.ts +++ b/src/evm/EvmBytecodeCarrier.test.ts @@ -1,3 +1,5 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; import { expect, it } from "vitest"; import { decodeEvmBytecodeCarrier, @@ -22,14 +24,23 @@ it.each(["0x1", "60 00", "0xzz", "\uFEFF6000", "\u00a06000"])( ); }, ); -it("rejects a long interior separator run in linear time", () => { - const text = `60${" ".repeat(200_000)}00`; - const started = performance.now(); - expect(() => decodeEvmBytecodeCarrier(Buffer.from(text), "hex")).toThrow( - EvmCarrierFailure, +it("rejects a long interior separator run without pathological backtracking", async () => { + const moduleUrl = new URL("./EvmBytecodeCarrier.ts", import.meta.url).href; + const source = ` + import assert from "node:assert/strict"; + import { decodeEvmBytecodeCarrier, EvmCarrierFailure } from ${JSON.stringify(moduleUrl)}; + const text = "60" + " ".repeat(1_000_000) + "00"; + assert.throws(() => decodeEvmBytecodeCarrier(Buffer.from(text), "hex"), EvmCarrierFailure); + `; + await promisify(execFile)( + process.execPath, + ["--input-type=module", "-e", source], + { + timeout: 15_000, + killSignal: "SIGKILL", + }, ); - expect(performance.now() - started).toBeLessThan(1_000); -}); +}, 20_000); it("distinguishes raw bytes, empty code and malformed UTF-8", () => { const bytes = Uint8Array.from([0xff, 0x00]); const decoded = decodeEvmBytecodeCarrier(bytes, "raw"); diff --git a/src/firmware/devicetree/DevicetreeParser.test.ts b/src/firmware/devicetree/DevicetreeParser.test.ts index 51e990f59b..d0635a236e 100644 --- a/src/firmware/devicetree/DevicetreeParser.test.ts +++ b/src/firmware/devicetree/DevicetreeParser.test.ts @@ -88,11 +88,6 @@ describe("flattened devicetree", () => { ] as const)("rejects %s", (_name, mutate) => expect(() => parse(mutate(devicetreeFixture()))).toThrow(), ); - it("distinguishes unsupported versions", () => { - expect(() => parse(devicetreeFixture(undefined, 18))).toThrow( - expect.objectContaining({ kind: "unsupported" }), - ); - }); it("bounds depth, nodes and properties", () => { let root: FixtureNode = { name: "leaf" }; for (let i = 0; i < 10; i++) root = { name: `n${i}`, children: [root] }; @@ -190,7 +185,6 @@ it("leaves conflicting phandle declarations unresolved", () => { ]); }); it.each([ - ["empty", Buffer.alloc(0)], ["short", Buffer.from([0, 0])], ["multiple cells", cells(1, 2)], ])( diff --git a/src/ghidra/GhidraDolProfile.test.ts b/src/ghidra/GhidraDolProfile.test.ts index b7b0dd5bd4..0de31e0799 100644 --- a/src/ghidra/GhidraDolProfile.test.ts +++ b/src/ghidra/GhidraDolProfile.test.ts @@ -109,6 +109,8 @@ describe("GameCube DOL Ghidra profile", () => { ); if (!left.ok) throw left.error; if (!right.ok) throw right.error; + expect(left.value.profile?.digest).toEqual(expect.any(String)); + expect(right.value.profile?.digest).toEqual(expect.any(String)); expect(left.value.profile?.digest).not.toBe(right.value.profile?.digest); }); diff --git a/src/ghidra/GhidraInventoryValues.test.ts b/src/ghidra/GhidraInventoryValues.test.ts index 94b0aa5f72..d11449babe 100644 --- a/src/ghidra/GhidraInventoryValues.test.ts +++ b/src/ghidra/GhidraInventoryValues.test.ts @@ -110,9 +110,7 @@ describe("native load-image result projection", () => { ); expect(parsed.ok).toBe(true); if (!parsed.ok) throw parsed.error; - expect(JSON.stringify(parsed.value.value)).toBe( - JSON.stringify(observations), - ); + expect(parsed.value.value).toEqual(observations); expect(parsed.value.rawResult).toBeUndefined(); }); @@ -129,7 +127,7 @@ describe("native load-image result projection", () => { ); expect(parsed.ok).toBe(true); if (!parsed.ok) throw parsed.error; - expect(JSON.stringify(parsed.value.value)).toContain('"entry_context":[]'); + expect(parsed.value.value).toMatchObject({ entry_context: [] }); }); }); diff --git a/src/ghidra/GhidraLanguageOverride.test.ts b/src/ghidra/GhidraLanguageOverride.test.ts index 76919a39c2..2657b23b9e 100644 --- a/src/ghidra/GhidraLanguageOverride.test.ts +++ b/src/ghidra/GhidraLanguageOverride.test.ts @@ -419,6 +419,8 @@ describe("Ghidra language override profile", () => { }); expect(parameters).not.toHaveProperty("language_selection"); expect(ghidraProfileLanguageOverride(parameters)).toBeUndefined(); + expect(automatic.value.profile?.digest).toEqual(expect.any(String)); + expect(configured.value.profile?.digest).toEqual(expect.any(String)); expect(automatic.value.profile?.digest).not.toBe( configured.value.profile?.digest, ); diff --git a/src/ghidra/GhidraMipsProfile.test.ts b/src/ghidra/GhidraMipsProfile.test.ts index c9c637fad0..27045790ce 100644 --- a/src/ghidra/GhidraMipsProfile.test.ts +++ b/src/ghidra/GhidraMipsProfile.test.ts @@ -126,6 +126,8 @@ describe("bounded Ghidra MIPS admission", () => { }, gnu_attributes: null, }); + expect(left.value.profile?.digest).toEqual(expect.any(String)); + expect(right.value.profile?.digest).toEqual(expect.any(String)); expect(left.value.profile?.digest).not.toBe(right.value.profile?.digest); const windows = new GhidraProvider( config, diff --git a/src/ghidra/GhidraPowerPcProfile.test.ts b/src/ghidra/GhidraPowerPcProfile.test.ts index 01f9e00779..67877e22c2 100644 --- a/src/ghidra/GhidraPowerPcProfile.test.ts +++ b/src/ghidra/GhidraPowerPcProfile.test.ts @@ -124,6 +124,8 @@ describe("bounded Ghidra PowerPC admission", () => { ); if (!left.ok) throw left.error; if (!right.ok) throw right.error; + expect(left.value.profile?.digest).toEqual(expect.any(String)); + expect(right.value.profile?.digest).toEqual(expect.any(String)); expect(left.value.profile?.digest).not.toBe(right.value.profile?.digest); }); diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index b4a7b1ca5c..e3c41ad9df 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -482,6 +482,7 @@ describe("Ghidra raw-binary profile", () => { expect(resolved.ok && shifted.ok).toBe(true); if (!resolved.ok || !shifted.ok || resolved.value.profile === null) throw new Error("expected raw-binary profiles"); + expect(shifted.value.profile?.digest).toEqual(expect.any(String)); expect(shifted.value.profile?.digest).not.toBe( resolved.value.profile.digest, ); @@ -4864,14 +4865,14 @@ it("reports every Windows existing-project capability unavailable before open", REA_GHIDRA_PROJECT_PATH: join(process.cwd(), "selected.gpr"), }); try { + const capabilities = ghidra.capabilities(); + expect(capabilities.length).toBeGreaterThan(0); expect( - ghidra - .capabilities() - .every( - (capability) => - !capability.available && - capability.availabilityCode === "unsupported_host", - ), + capabilities.every( + (capability) => + !capability.available && + capability.availabilityCode === "unsupported_host", + ), ).toBe(true); const readiness = await ghidra.inspectReadinessSnapshot(); expect(readiness.availability).toMatchObject({ @@ -4879,6 +4880,9 @@ it("reports every Windows existing-project capability unavailable before open", code: "unsupported_host", reason: EXISTING_PROJECT_WINDOWS_UNAVAILABLE, }); + expect(readiness.capabilities.map(({ operation }) => operation)).toEqual( + capabilities.map(({ operation }) => operation), + ); expect( readiness.capabilities.every( (capability) => diff --git a/src/ghidra/GhidraRiscvProfile.test.ts b/src/ghidra/GhidraRiscvProfile.test.ts index f8c8dcd228..486044c1b9 100644 --- a/src/ghidra/GhidraRiscvProfile.test.ts +++ b/src/ghidra/GhidraRiscvProfile.test.ts @@ -166,6 +166,8 @@ const registerRiscvLanguageCommitmentCases = () => { elf_class: 32, rvc_declared: false, }); + expect(wide.value.profile?.digest).toEqual(expect.any(String)); + expect(narrow.value.profile?.digest).toEqual(expect.any(String)); expect(wide.value.profile?.digest).not.toBe(narrow.value.profile?.digest); }); diff --git a/src/ghidra/GhidraSessionError.test.ts b/src/ghidra/GhidraSessionError.test.ts index fd461f2898..b95e67b0b6 100644 --- a/src/ghidra/GhidraSessionError.test.ts +++ b/src/ghidra/GhidraSessionError.test.ts @@ -52,7 +52,13 @@ it("reports truncated cyclic causes in serializable diagnostics", () => { cause, ).diagnostics; - expect(JSON.stringify(diagnostics)).toContain('"cause_truncated":true'); + const serialized = JSON.parse(JSON.stringify(diagnostics)) as { + failure_cause: unknown; + }; + let nested = serialized.failure_cause; + while (typeof nested === "object" && nested !== null && "cause" in nested) + nested = nested.cause; + expect(nested).toMatchObject({ cause_truncated: true }); }); it("keeps bounded compiler and nested launcher outcomes in diagnostics", () => { diff --git a/src/ghidra/extensions/GhidraExtensions.test.ts b/src/ghidra/extensions/GhidraExtensions.test.ts index 02242f45bd..a3f5b835a4 100644 --- a/src/ghidra/extensions/GhidraExtensions.test.ts +++ b/src/ghidra/extensions/GhidraExtensions.test.ts @@ -292,30 +292,26 @@ describe("Ghidra extension profile and producer validation", () => { it("requires exact inventory, artifact digest and consistent producer status", async () => { const { extension } = await artifact(); const valid = result(extension.sha256); - const inventoryMismatch = - "Ghidra extension result inventory differs from its committed profile."; expect(validateGhidraExtensionResults([extension], [valid])).toBeNull(); - expect(validateGhidraExtensionResults([extension], [])).toBe( - inventoryMismatch, - ); - expect(validateGhidraExtensionResults([], [valid])).toBe(inventoryMismatch); - expect(validateGhidraExtensionResults([extension], [valid, valid])).toBe( - inventoryMismatch, + expect(validateGhidraExtensionResults([extension], [])).toContain( + "inventory", ); + expect(validateGhidraExtensionResults([], [valid])).toContain("inventory"); expect( - validateGhidraExtensionResults( - [extension], - [{ ...valid, sha256: "b".repeat(64) }], - ), - ).toBe("Ghidra extension identity mismatch: nativeaot"); + validateGhidraExtensionResults([extension], [valid, valid]), + ).toContain("inventory"); + const identityMismatch = validateGhidraExtensionResults( + [extension], + [{ ...valid, sha256: "b".repeat(64) }], + ); + expect(identityMismatch).toContain("identity"); + expect(identityMismatch).toContain(extension.id); expect( validateGhidraExtensionResults( [extension], [{ ...valid, status: "complete" }], ), - ).toBe( - "NativeAOT extension returned a malformed or inconsistent producer report.", - ); + ).toContain("producer report"); }); it("requires matching explicit configuration for imported executable profiles", async () => { const { extension, path } = await artifact(); @@ -329,39 +325,28 @@ describe("Ghidra extension profile and producer validation", () => { ).toBeNull(); expect( validateGhidraExtensionProfile([extension], config(), target, "linux"), - ).toBe( - "Configured Ghidra extension inventory differs from the resolved analysis profile.", + ).toContain("inventory"); + const unsupportedHost = validateGhidraExtensionProfile( + [extension], + config(path), + target, + "darwin", ); - expect( - validateGhidraExtensionProfile( - [extension], - config(path), - target, - "darwin", - ), - ).toBe( - "NativeAOT recovery is verified on Linux only; omit REA_GHIDRA_NATIVEAOT_JAR for ordinary native analysis.", - ); - expect( - validateGhidraExtensionProfile( - [{ ...extension, entry_class: "arbitrary.Code" }], + expect(unsupportedHost).toContain("Linux"); + expect(unsupportedHost).toContain("REA_GHIDRA_NATIVEAOT_JAR"); + for (const changed of [ + { ...extension, entry_class: "arbitrary.Code" }, + { ...extension, configured_path: "/another.jar" }, + ]) { + const invalid = validateGhidraExtensionProfile( + [changed], config(path), target, "linux", - ), - ).toBe( - "Ghidra extension profile lacks matching explicit local configuration: nativeaot", - ); - expect( - validateGhidraExtensionProfile( - [{ ...extension, configured_path: "/another.jar" }], - config(path), - target, - "linux", - ), - ).toBe( - "Ghidra extension profile lacks matching explicit local configuration: nativeaot", - ); + ); + expect(invalid).toContain("configuration"); + expect(invalid).toContain(extension.id); + } }); it("does not classify PE/CLI or ReadyToRun as NativeAOT", async () => { const resolved = await resolveGhidraExtensions( @@ -465,8 +450,8 @@ describe("Ghidra extension recovery coverage", () => { }, }, }; - expect(validateGhidraExtensionResults([extension], [incomplete])).toBe( - "NativeAOT recovery coverage contradicts its reported completion status.", + expect(validateGhidraExtensionResults([extension], [incomplete])).toContain( + "coverage", ); expect( validateGhidraExtensionResults( @@ -485,9 +470,7 @@ describe("Ghidra extension recovery coverage", () => { [extension], [{ ...positive, result: { ...positive.result, discovery: null } }], ), - ).toBe( - "NativeAOT extension returned a malformed or inconsistent producer report.", - ); + ).toContain("producer report"); }); it("preserves real loader failures without requiring a successful analyzer report", async () => { const { extension } = await artifact(); @@ -512,14 +495,13 @@ describe("Ghidra extension recovery coverage", () => { const incomplete = result(extension.sha256); incomplete.result.status = "complete"; incomplete.result.reason = null; - expect( - validateGhidraExtensionResults( - [extension], - [{ ...incomplete, status: "complete", reason: null }], - ), - ).toBe( - "NativeAOT recovery omitted its supported format, method-table inventory or derived-memory identity.", + const rejection = validateGhidraExtensionResults( + [extension], + [{ ...incomplete, status: "complete", reason: null }], ); + expect(rejection).toContain("format"); + expect(rejection).toContain("method-table"); + expect(rejection).toContain("derived-memory"); }); it.each(["not_applicable", "unsupported", "failed"] as const)( "rejects contradictory %s reports without discarding discovery context", @@ -544,7 +526,7 @@ describe("Ghidra extension recovery coverage", () => { [extension], [{ ...report, result: { ...report.result, method_tables: 1 } }], ), - ).toBe("NativeAOT non-recovery status carries recovered metadata."); + ).toContain("metadata"); for (const reason of [null, "", " "] as const) expect( validateGhidraExtensionResults( @@ -557,7 +539,7 @@ describe("Ghidra extension recovery coverage", () => { }, ], ), - ).toBe("NativeAOT non-recovery status omits its required reason."); + ).toContain("reason"); }, ); }); diff --git a/src/hopper/HopperNameInventory.test.ts b/src/hopper/HopperNameInventory.test.ts index 0d129f1587..e220be62cc 100644 --- a/src/hopper/HopperNameInventory.test.ts +++ b/src/hopper/HopperNameInventory.test.ts @@ -87,10 +87,12 @@ describe("Hopper name inventory labels", () => { it("leaves malformed provider inventories untouched for the consumer to reject", () => { const value = [{ address: 1, value: "__objc_class_HuntRoot_class" }]; + const original = structuredClone(value); expect(projectHopperNameInventory(value)).toEqual({ - result: value, + result: original, changed: false, }); + expect(value).toEqual(original); }); it("preserves source rows and the exact raw reply through the provider boundary", async () => { @@ -110,6 +112,7 @@ describe("Hopper name inventory labels", () => { objc_class_label: "hopper_legacy_class_object", }, ]; + const originalSource = structuredClone(source); const callTool = vi .spyOn(HopperClient.prototype, "callTool") .mockResolvedValue(ok(source)); @@ -126,14 +129,13 @@ describe("Hopper name inventory labels", () => { format: "analysis-database", }); - const result = await client.execute("list_names", {}); - - expect(callTool).toHaveBeenCalledOnce(); - expect(result.ok).toBe(true); - if (result.ok) { + try { + const result = await client.execute("list_names", {}); + expect(callTool).toHaveBeenCalledOnce(); + if (!result.ok) throw result.error; expect(result.value.result).toEqual([ { - ...source[0], + ...originalSource[0], objc_class_label: "hopper_legacy_class_object", }, { @@ -142,9 +144,14 @@ describe("Hopper name inventory labels", () => { provider_metadata: { ordinal: 2 }, }, ]); - expect(result.value.rawResult).toEqual(source); + expect(result.value.rawResult).toEqual(originalSource); + expect(source).toEqual(originalSource); + } finally { + try { + await client.close(); + } finally { + callTool.mockRestore(); + } } - await client.close(); - callTool.mockRestore(); }); }); diff --git a/src/hopper/HopperProvider.test.ts b/src/hopper/HopperProvider.test.ts index e3cf5f1876..7eb41e3853 100644 --- a/src/hopper/HopperProvider.test.ts +++ b/src/hopper/HopperProvider.test.ts @@ -230,6 +230,7 @@ describe("Hopper relationship coverage", () => { : operation === "procedure_callees" ? ["0x2000"] : []; + const originalSource = structuredClone(source); const callTool = vi .spyOn(HopperClient.prototype, "callTool") .mockImplementation((method) => @@ -277,7 +278,8 @@ describe("Hopper relationship coverage", () => { procedure: "0x1000", }); if (!observed.ok) throw observed.error; - expect(observed.value.result).toEqual(source); + expect(observed.value.result).toEqual(originalSource); + expect(source).toEqual(originalSource); if (operation === "procedure_callers") { expect(observed.value.rawResult).toMatchObject({ native_result: { diff --git a/src/hopper/HopperRegexSearch.test.ts b/src/hopper/HopperRegexSearch.test.ts index ea1b3abc58..d1ce0b62aa 100644 --- a/src/hopper/HopperRegexSearch.test.ts +++ b/src/hopper/HopperRegexSearch.test.ts @@ -61,6 +61,7 @@ it.each(["search_strings", "search_procedures"] as const)( selected, { ...selected, value: "matched alternative" }, ]; + const originalSource = structuredClone(source); const callTool = vi .spyOn(HopperClient.prototype, "callTool") .mockResolvedValue(ok(source)); @@ -83,12 +84,12 @@ it.each(["search_strings", "search_procedures"] as const)( case_sensitive: true, }); expect(result.ok).toBe(true); - if (result.ok) expect(result.value.result).toEqual([selected]); + if (result.ok) expect(result.value.result).toEqual([originalSource[1]]); expect(callTool).toHaveBeenCalledOnce(); expect(callTool.mock.calls[0]?.[0]).toBe( operation === "search_strings" ? "list_strings" : "list_procedures", ); - expect(source[1]).toEqual(selected); + expect(source).toEqual(originalSource); } finally { await client.close(); callTool.mockRestore(); diff --git a/src/ida/IdaDoctor.test.ts b/src/ida/IdaDoctor.test.ts index d3c23635d3..641649ae8e 100644 --- a/src/ida/IdaDoctor.test.ts +++ b/src/ida/IdaDoctor.test.ts @@ -32,9 +32,9 @@ it.each([true, false])( workspaceRoot: rawRoot, }), ); - const before = await readdir(root); + const before = (await readdir(root)).sort(); const inspection = await inspectIdaRegistration(path, resourceScope); - const after = await readdir(root); + const after = (await readdir(root)).sort(); expect(inspection.available).toBe(exists); expect( inspection.checks.find(({ name }) => name === "private_workspace") @@ -62,7 +62,7 @@ it.skipIf(process.platform === "win32")( workspaceRoot: selected, }), ); - const before = await readdir(root); + const before = (await readdir(root)).sort(); const inspection = await inspectIdaRegistration(path, resourceScope); expect(inspection.available).toBe(false); const check = inspection.checks.find( @@ -71,7 +71,7 @@ it.skipIf(process.platform === "win32")( expect(check).toMatchObject({ ok: false, code: "invalid_configuration" }); expect(check?.detail).toContain(selected); expect(check?.detail).toContain("ENOTDIR"); - expect(await readdir(root)).toEqual(before); + expect((await readdir(root)).sort()).toEqual(before); expect(await readdir(workspace)).toEqual([]); }, ); diff --git a/src/ida/IdaProvider.test.ts b/src/ida/IdaProvider.test.ts index ebcd1878dd..bc563d15fc 100644 --- a/src/ida/IdaProvider.test.ts +++ b/src/ida/IdaProvider.test.ts @@ -112,7 +112,7 @@ it.each(["missing", "file", "directory", "omitted"] as const)( throw new Error("Readiness must not connect to IDA"); }); const provider = new IdaProvider(config.value, {}, connect); - const before = await readdir(root); + const before = (await readdir(root)).sort(); try { const snapshot = await provider.inspectReadinessSnapshot(); if (kind === "missing" || kind === "file") { @@ -135,7 +135,7 @@ it.each(["missing", "file", "directory", "omitted"] as const)( )?.reason, ).toContain("direct call edges"); } else expect(snapshot.availability.status).toBe("available"); - expect(await readdir(root)).toEqual(before); + expect((await readdir(root)).sort()).toEqual(before); expect(connect).not.toHaveBeenCalled(); } finally { const closed = await provider.closeOwnedResources(); @@ -200,20 +200,20 @@ describe("IDA provider target support", () => { const config = parseConfig({}); if (!config.ok) throw new Error("expected valid configuration"); const provider = new IdaProvider(config.value, {}); - expect( - provider.inspectTargetSupport({ - path: "/tmp/chre_pal.so", - sha256: "a".repeat(64), - kind: "executable", - format: "elf", - architecture: "hexagon", - availableArchitectures: ["hexagon"], - }), - ).toMatchObject({ + const support = provider.inspectTargetSupport({ + path: "/tmp/chre_pal.so", + sha256: "a".repeat(64), + kind: "executable", + format: "elf", + architecture: "hexagon", + availableArchitectures: ["hexagon"], + }); + expect(support).toMatchObject({ status: "unsupported", code: "architecture_unsupported", - reason: expect.stringMatching(/hexagon.*Ghidra/su), }); + expect(support.reason).toContain("hexagon"); + expect(support.reason).toContain("Ghidra"); }); it("admits only the families IDA admits today, so new families stay refused", () => { const config = parseConfig({}); @@ -336,6 +336,7 @@ it.each(["http", "stdio"] as const)( ); const changed = await provider.resolveAnalysisProfile(target); if (!changed.ok) throw changed.error; + expect(changed.value.profile?.digest).toEqual(expect.any(String)); expect(changed.value.profile?.digest).not.toBe(profiles[0]); } finally { await provider.closeOwnedResources(); diff --git a/src/ida/IdaSessionClient.test.ts b/src/ida/IdaSessionClient.test.ts index f3ac679990..b17bdf2ae2 100644 --- a/src/ida/IdaSessionClient.test.ts +++ b/src/ida/IdaSessionClient.test.ts @@ -1938,9 +1938,11 @@ describe("headless IDA cleanup verification", () => { }); it("releases a privately owned worker after malformed open output using verified session inventory", async () => { const { producer, client, root } = await fixture("headless"); + let ownedId: string | undefined; producer.beforeCall = async (name, args) => { if (name === "idb_open") { - producer.database = z.string().parse(args.preferred_session_id); + ownedId = z.string().parse(args.preferred_session_id); + producer.database = ownedId; producer.inputPath = z.string().parse(args.input_path); } }; @@ -1953,9 +1955,10 @@ describe("headless IDA cleanup verification", () => { : undefined; expect((await client.execute("health", {})).ok).toBe(false); expect((await client.close()).ok).toBe(true); + expect(ownedId).toEqual(expect.any(String)); expect( producer.calls.find(({ name }) => name === "idb_close")?.args.database, - ).toMatch(/^rea-/u); + ).toBe(ownedId); expect(await readdir(root)).toEqual(["target.elf"]); }); }); diff --git a/src/inspector/JavaScriptRuntimeScope.cancellation.test.ts b/src/inspector/JavaScriptRuntimeScope.cancellation.test.ts index 4dd0057176..ef1faf1c6f 100644 --- a/src/inspector/JavaScriptRuntimeScope.cancellation.test.ts +++ b/src/inspector/JavaScriptRuntimeScope.cancellation.test.ts @@ -4,6 +4,8 @@ import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { afterEach, expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; const fileAccess = vi.hoisted(() => ({ path: undefined as string | undefined, @@ -248,31 +250,43 @@ it("drains pending Electron discovery before shutdown cleanup completes", async const listing = provider.listTargets( listElectronTargetsInputSchema.parse({ cdp_endpoint: fake.endpoint }), ); - await started; let closed = false; - const closing = provider.close().then(() => { - closed = true; - }); - const closedBeforeDiscovery = await Promise.race([ - closing.then(() => true), - new Promise((resolve) => setImmediate(() => resolve(false))), - ]); - expect(closedBeforeDiscovery).toBe(false); - expect(closed).toBe(false); - releaseDiscovery(); + let closing: Promise | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(started, listing, "Electron discovery"); + closing = provider.close().then(() => { + closed = true; + }); + const closedBeforeDiscovery = await Promise.race([ + closing.then(() => true), + new Promise((resolve) => setImmediate(() => resolve(false))), + ]); + expect(closedBeforeDiscovery).toBe(false); + expect(closed).toBe(false); + releaseDiscovery(); - const [listed] = await Promise.all([listing, closing]); - expect(closed).toBe(true); - expect(listed.ok).toBe(true); - expect(fileAccess.calls).toContain("stat"); - const rejected = await provider.listTargets( - listElectronTargetsInputSchema.parse({ cdp_endpoint: fake.endpoint }), - ); - expect(rejected.ok).toBe(false); - if (rejected.ok) - throw new Error("Expected closed provider admission failure"); - expect(rejected.error.cleanupIncomplete).toBe(false); - expect(rejected.error.cleanupResources).toEqual([]); + const [listed] = await Promise.all([listing, closing]); + expect(closed).toBe(true); + expect(listed.ok).toBe(true); + expect(fileAccess.calls).toContain("stat"); + const rejected = await provider.listTargets( + listElectronTargetsInputSchema.parse({ cdp_endpoint: fake.endpoint }), + ); + expect(rejected.ok).toBe(false); + if (rejected.ok) + throw new Error("Expected closed provider admission failure"); + expect(rejected.error.cleanupIncomplete).toBe(false); + expect(rejected.error.cleanupResources).toEqual([]); + }, [ + releaseDiscovery, + async () => { + await listing; + }, + async () => { + await closing; + }, + () => provider.close(), + ]); }); it("keeps ordinary and malformed file-location authorization unchanged", async () => { diff --git a/src/inspector/V8InspectorEndpoint.test.ts b/src/inspector/V8InspectorEndpoint.test.ts index c99fba45ea..63eb761e01 100644 --- a/src/inspector/V8InspectorEndpoint.test.ts +++ b/src/inspector/V8InspectorEndpoint.test.ts @@ -78,8 +78,8 @@ it.each([ expect(diagnostic).toMatchObject({ code: "target_unavailable", message: expect.stringContaining(message), + remediation: { action: expect.stringContaining(action) }, }); - expect(JSON.stringify(diagnostic)).toContain(action); expect(inspector.commands).toEqual([]); } finally { await inspector.close(); diff --git a/src/inspector/V8InspectorProvider.closeOwnership.test.ts b/src/inspector/V8InspectorProvider.closeOwnership.test.ts index 3500411f19..ef592d4996 100644 --- a/src/inspector/V8InspectorProvider.closeOwnership.test.ts +++ b/src/inspector/V8InspectorProvider.closeOwnership.test.ts @@ -6,6 +6,11 @@ import { pathToFileURL } from "node:url"; import { expect, it, onTestFinished, vi } from "vitest"; import { WebSocketServer } from "ws"; +import { + withVerifierCleanup, + closeVerifierResources, +} from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; import { CdpConnection } from "../browser/CdpConnection.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; @@ -74,6 +79,8 @@ const inspectorCloseFixture = async () => { const firstClose = new Promise((_resolve, reject) => { rejectFirstClose = reject; }); + // Cleanup may reject the gate before a transport consumes it. + void firstClose.catch(() => undefined); const originalClose = CdpConnection.prototype.close; const closeSpy = vi .spyOn(CdpConnection.prototype, "close") @@ -89,14 +96,20 @@ const inspectorCloseFixture = async () => { }); onTestFinished(async () => { - closeSpy.mockRestore(); - if (closeCalls > 0) rejectFirstClose?.(new Error("fixture finalization")); - await new V8InspectorProvider(undefined, admission).close(); - for (const socket of sockets.clients) socket.terminate(); - await new Promise((resolve) => sockets.close(() => resolve())); - if (server.listening) - await new Promise((resolve) => server.close(() => resolve())); - await rm(root, { recursive: true, force: true }); + await closeVerifierResources([ + () => rejectFirstClose?.(new Error("fixture finalization")), + () => closeSpy.mockRestore(), + () => new V8InspectorProvider(undefined, admission).close(), + () => { + for (const socket of sockets.clients) socket.terminate(); + }, + () => new Promise((resolve) => sockets.close(() => resolve())), + async () => { + if (server.listening) + await new Promise((resolve) => server.close(() => resolve())); + }, + () => rm(root, { recursive: true, force: true }), + ]); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const address = server.address(); @@ -141,17 +154,33 @@ it("retains cancelled final-close ownership and retries the exact transport befo const observing = observeJavaScriptRuntime(provider(), input, { signal: controller.signal, }); - await closeStarted; - controller.abort(cancellationReason); - let firstObservationSettled = false; - void observing.finally(() => { - firstObservationSettled = true; - }); - await Promise.resolve(); - expect(firstObservationSettled).toBe(false); - rejectFirstClose?.(new Error("controlled final socket close failure")); - - const result = await observing; + const result = await withVerifierCleanup(async () => { + await awaitAcknowledgement( + closeStarted, + observing, + "inspector final close", + ); + controller.abort(cancellationReason); + let firstObservationSettled = false; + void observing.then( + () => { + firstObservationSettled = true; + }, + () => { + firstObservationSettled = true; + }, + ); + await Promise.resolve(); + expect(firstObservationSettled).toBe(false); + rejectFirstClose(new Error("controlled final socket close failure")); + return await observing; + }, [ + () => controller.abort(new Error("fixture finalization")), + () => rejectFirstClose(new Error("fixture finalization")), + async () => { + await observing; + }, + ]); expect(result.ok).toBe(false); if (result.ok) throw new Error("Expected cleanup-incomplete cancellation"); const error = result.error; @@ -202,15 +231,26 @@ it("retains cancelled final-close ownership and retries the exact transport befo const activeObservation = observeJavaScriptRuntime(provider(), activeInput, { signal: activeController.signal, }); - try { - await waitForConnections(4); + const queuedController = new AbortController(); + let shutdown: Promise | undefined; + let queuedObservation: + | ReturnType + | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + waitForConnections(4), + activeObservation, + "active inspector connection", + ); let shutdownSettled = false; - const shutdown = provider() + shutdown = provider() .close() .then(() => { shutdownSettled = true; }); - const queuedObservation = observeJavaScriptRuntime(provider(), input); + queuedObservation = observeJavaScriptRuntime(provider(), input, { + signal: queuedController.signal, + }); await new Promise((resolve) => setImmediate(resolve)); expect(shutdownSettled).toBe(false); expect(fixture.acceptedConnections).toBe(4); @@ -221,12 +261,26 @@ it("retains cancelled final-close ownership and retries the exact transport befo expect(activeResult.ok).toBe(false); await shutdown; expect(shutdownSettled).toBe(true); - await waitForConnections(5); + await awaitAcknowledgement( + waitForConnections(5), + queuedObservation, + "queued inspector connection", + ); const queuedResult = await queuedObservation; expect(queuedResult.ok).toBe(true); expect(fixture.acceptedConnections).toBe(5); expect(closeSpy).toHaveBeenCalledTimes(8); - } finally { - activeController.abort(new Error("fixture finalization")); - } + }, [ + () => activeController.abort(new Error("fixture finalization")), + () => queuedController.abort(new Error("fixture finalization")), + async () => { + await activeObservation; + }, + async () => { + await shutdown; + }, + async () => { + await queuedObservation; + }, + ]); }); diff --git a/src/native/NativeMachoOperand.test.ts b/src/native/NativeMachoOperand.test.ts index 8ff35a26ba..828b1be89d 100644 --- a/src/native/NativeMachoOperand.test.ts +++ b/src/native/NativeMachoOperand.test.ts @@ -61,7 +61,7 @@ const tracer: NativeCallTracer = { }; describe("private otool operand lifecycle", () => { - it("uses a private alias only for otool and removes it after all commands settle", async () => { + it("uses a private alias for native commands and removes it after all commands settle", async () => { const path = await mkdtemp(join(tmpdir(), "rea-macho-operand-test-")); const selected = await parenthesizedTarget(); const target = selected.target.path; @@ -90,10 +90,13 @@ describe("private otool operand lifecycle", () => { try { const result = await client.execute("inspect_macho", {}); if (!result.ok) throw result.error; - expect(JSON.stringify(result.value.result)).toContain( - join(path, "artifact.macho"), - ); const alias = join(path, "artifact.macho"); + const report = inspectMachoSchema.parse(result.value.result); + expect( + report.provenance + .find((entry) => entry.tool === "lipo") + ?.command.at(-1), + ).toBe(alias); expect(calls.some(({ operand }) => operand === alias)).toBe(true); expect(calls.some(({ operand }) => operand === target)).toBe(false); await expect(stat(path)).rejects.toMatchObject({ code: "ENOENT" }); @@ -166,7 +169,9 @@ describe("private otool operand lifecycle", () => { await rm(selected.directory, { recursive: true, force: true }); } }); +}); +describe("private operand command cleanup", () => { it("retains the alias until command cleanup is confirmed", async () => { const path = await mkdtemp(join(tmpdir(), "rea-macho-operand-test-")); let unresolved = true; diff --git a/src/native/SignatureTargetBinding.test.ts b/src/native/SignatureTargetBinding.test.ts index f361ebf90f..f6ca2b5553 100644 --- a/src/native/SignatureTargetBinding.test.ts +++ b/src/native/SignatureTargetBinding.test.ts @@ -12,6 +12,8 @@ import { import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, it, vi, onTestFinished } from "vitest"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; @@ -246,22 +248,41 @@ it("waits for an in-flight signature binding before provider close", async () => ); const client = provider.createClient(target); const operation = client.execute("inspect_signature", {}); - await entered; - let closeSettled = false; - const closing = client.close().finally(() => { - closeSettled = true; - }); - await Promise.resolve(); - expect(closeSettled).toBe(false); - releaseRootFactory(); - await expect(operation).resolves.toMatchObject({ - ok: false, - error: { _tag: "AnalysisArtifactChangedError" }, - }); - await expect(closing).resolves.toMatchObject({ ok: true, value: null }); - expect(closeSettled).toBe(true); - await expect(stat(roots[0]!)).rejects.toMatchObject({ code: "ENOENT" }); + let closing: ReturnType | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + entered, + operation, + "signature root acquisition", + ); + closing = client.close().finally(() => { + closeSettled = true; + }); + await Promise.resolve(); + expect(closeSettled).toBe(false); + releaseRootFactory(); + await expect(operation).resolves.toMatchObject({ + ok: false, + error: { _tag: "AnalysisArtifactChangedError" }, + }); + await expect(closing).resolves.toMatchObject({ ok: true, value: null }); + expect(closeSettled).toBe(true); + await expect(stat(roots[0]!)).rejects.toMatchObject({ code: "ENOENT" }); + }, [ + releaseRootFactory, + async () => { + await operation; + }, + async () => { + const result = await closing; + if (result !== undefined && !result.ok) throw result.error; + }, + async () => { + const result = await client.close(); + if (!result.ok) throw result.error; + }, + ]); }); it("releases retained descriptors before retrying their containing root", async () => { diff --git a/src/native/elf/ElfNoteParser.test.ts b/src/native/elf/ElfNoteParser.test.ts index 844edfbb36..8a82254c3e 100644 --- a/src/native/elf/ElfNoteParser.test.ts +++ b/src/native/elf/ElfNoteParser.test.ts @@ -3,7 +3,6 @@ import { readFileSync } from "node:fs"; import { expect, it } from "vitest"; import { inspectElfNotesInputSchema } from "../../domain/native/elfNotes.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; -import { BinaryFormatFailure } from "../metadata/BinaryContainer.js"; import { decodeElfSectionNameTable } from "../metadata/ElfSectionNameTable.js"; import { readElfDirectoryFromBuffer } from "../metadata/NativeSectionDirectory.js"; import { @@ -243,20 +242,6 @@ it("keeps note facts but marks section names unavailable for unknown compression ); }); -it("tags valid extended section numbering as unsupported", () => { - let failure: unknown; - try { - parse(elfExtendedSectionCountFixture()); - } catch (cause) { - failure = cause; - } - expect(failure).toBeInstanceOf(BinaryFormatFailure); - expect(failure).toMatchObject({ - kind: "unsupported", - message: "ELF extended numbering is unsupported.", - }); -}); - it.each([1, 2])( "admits algorithm %i section names when max_file_bytes includes expansion", (compressionType) => { diff --git a/src/process/ArtifactAnalysisResources.test.ts b/src/process/ArtifactAnalysisResources.test.ts index bca2754e3d..421621f7d1 100644 --- a/src/process/ArtifactAnalysisResources.test.ts +++ b/src/process/ArtifactAnalysisResources.test.ts @@ -7,6 +7,8 @@ import { withArtifactRegularFile } from "../artifacts/ArtifactFileOperation.js"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; import { OwnedCommandFailure } from "./OwnedCommand.js"; import { EventEmitter } from "node:events"; import { PassThrough } from "node:stream"; @@ -837,7 +839,9 @@ it("observes cancellation after pending descriptor cleanup before admitting repl { message: "busy", details: {} }, ); let calls = 0; - try { + const caller = new AbortController(); + let pending: ReturnType | undefined; + await withVerifierCleanup(async () => { const first = await resources.run(async () => ok(await resources.read("fixture", 1)), ); @@ -845,13 +849,12 @@ it("observes cancellation after pending descriptor cleanup before admitting repl ok: false, error: { cleanupIncomplete: true }, }); - const caller = new AbortController(); const reason = { request: "cancel while old descriptor closes" }; - const pending = resources.run(async () => { + pending = resources.run(async () => { calls++; return ok("Must not run"); }, caller.signal); - await started; + await awaitAcknowledgement(started, pending, "descriptor cleanup"); caller.abort(reason); releaseCleanup(); const result = await pending; @@ -859,10 +862,14 @@ it("observes cancellation after pending descriptor cleanup before admitting repl expect(result.error.cause).toBe(reason); expect(calls).toBe(0); if (!first.ok) expect(first.error.cleanupIncomplete).toBe(false); - } finally { - releaseCleanup(); - await resources.close(); - } + }, [ + () => caller.abort(), + releaseCleanup, + async () => { + await pending; + }, + () => resources.close(), + ]); }); it.each([false, true])( diff --git a/src/process/OwnedWorkerRuntime.test.ts b/src/process/OwnedWorkerRuntime.test.ts index 73e400b366..3a9ff3886f 100644 --- a/src/process/OwnedWorkerRuntime.test.ts +++ b/src/process/OwnedWorkerRuntime.test.ts @@ -1,6 +1,8 @@ import { access } from "node:fs/promises"; import { randomUUID } from "node:crypto"; import { expect, it } from "vitest"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { cleanupOwnedProcessGroup } from "./ProcessOwnership.js"; @@ -48,22 +50,34 @@ it("waits for an admitted late handshake before closing its process and root", a announceInitialize(); await held; }); - await initializing; - await waitFor(() => runtime.supervisor !== undefined); - const rootPath = runtime.rootPath; - expect(rootPath).toBeDefined(); - const closing = runtime.close().then(() => { - closeSettled = true; - }); - await new Promise((resolve) => setImmediate(resolve)); - expect(closeSettled).toBe(false); - expect(runtime.supervisor).toBeDefined(); - releaseInitialize(); - await expect(starting).rejects.toThrow("closing"); - await closing; - expect(runtime.supervisor).toBeUndefined(); - expect(runtime.rootPath).toBeUndefined(); - await expect(access(rootPath as string)).rejects.toThrow(); + let closing: Promise | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(initializing, starting, "worker initialization"); + await waitFor(() => runtime.supervisor !== undefined); + const rootPath = runtime.rootPath; + expect(rootPath).toBeDefined(); + closing = runtime.close().then(() => { + closeSettled = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(closeSettled).toBe(false); + expect(runtime.supervisor).toBeDefined(); + releaseInitialize(); + await expect(starting).rejects.toThrow("closing"); + await closing; + expect(runtime.supervisor).toBeUndefined(); + expect(runtime.rootPath).toBeUndefined(); + await expect(access(rootPath as string)).rejects.toThrow(); + }, [ + releaseInitialize, + async () => { + await starting.catch(() => undefined); + }, + async () => { + await closing; + }, + () => runtime.close(), + ]); }); it("resolves a bound cleanup error after shared pending-owner retry succeeds", async () => { @@ -87,22 +101,31 @@ it("resolves a bound cleanup error after shared pending-owner retry succeeds", a }, }), ); - await runtime.start(startOptions()); - const rootPath = runtime.rootPath; - const error = new ProviderCleanupError("owned-worker-test", [], { - reason: "controlled cleanup retry", - }); - await expect(runtime.stop()).resolves.toMatchObject({ status: "incomplete" }); - runtime.bindCleanupFailure(error); - expect(error.cleanupIncomplete).toBe(true); - expect(error.cleanupResources).toContain(rootPath); + await withVerifierCleanup(async () => { + await runtime.start(startOptions()); + const rootPath = runtime.rootPath; + const error = new ProviderCleanupError("owned-worker-test", [], { + reason: "controlled cleanup retry", + }); + await expect(runtime.stop()).resolves.toMatchObject({ + status: "incomplete", + }); + runtime.bindCleanupFailure(error); + expect(error.cleanupIncomplete).toBe(true); + expect(error.cleanupResources).toContain(rootPath); - await OwnedWorkerRuntime.retryPending(); + await OwnedWorkerRuntime.retryPending(); - expect(error.cleanupIncomplete).toBe(false); - expect(error.cleanupResources).toEqual([]); - expect(runtime.rootPath).toBeUndefined(); - await expect(access(rootPath as string)).rejects.toThrow(); + expect(error.cleanupIncomplete).toBe(false); + expect(error.cleanupResources).toEqual([]); + expect(runtime.rootPath).toBeUndefined(); + await expect(access(rootPath as string)).rejects.toThrow(); + }, [ + () => { + failCleanup = false; + }, + () => runtime.close(), + ]); }); it("observes request-stream errors and resets them for a fresh child", async () => { diff --git a/src/process/ProcessOwnership.observation.test.ts b/src/process/ProcessOwnership.observation.test.ts index f8b1330a87..a2cb4641e8 100644 --- a/src/process/ProcessOwnership.observation.test.ts +++ b/src/process/ProcessOwnership.observation.test.ts @@ -1,4 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; import { cleanupOwnedProcessGroup, type ProcessOwnershipHost, @@ -287,14 +289,22 @@ describe("multi-group ownership observation cancellation", () => { return outcome; }); let settledBeforeRelease = false; - try { - await secondListStarted; + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + secondListStarted, + observedOutcome, + "liveness recheck", + ); controller.abort(reason); await new Promise((resolve) => setImmediate(resolve)); settledBeforeRelease = settled; - } finally { - releaseSecondList?.(); - } + }, [ + () => controller.abort(reason), + () => releaseSecondList?.(), + async () => { + await observedOutcome; + }, + ]); const outcome = await observedOutcome; expect(settledBeforeRelease).toBe(true); diff --git a/src/process/ProviderProcessOutput.test.ts b/src/process/ProviderProcessOutput.test.ts index f8a71a749a..d8ce4e7acb 100644 --- a/src/process/ProviderProcessOutput.test.ts +++ b/src/process/ProviderProcessOutput.test.ts @@ -1,6 +1,8 @@ import { EventEmitter } from "node:events"; import { PassThrough } from "node:stream"; import { expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; import { ProviderProcessSupervisor, type ProcessDiagnosticBudget, @@ -29,19 +31,33 @@ it("retains final stderr between verified exit and output close during stop", as }, }); const stopped = owner.stop(); - expect(owner.stop()).toBe(stopped); - await started; - // Let cleanup observe exit before delivering the separately acknowledged output. - await new Promise((resolve) => setImmediate(resolve)); - producer.stderr.end("final launcher diagnostic"); - producer.stdout.end(); - producer.emit("close", 2, null); - await expect(stopped).resolves.toEqual({ status: "verified-cleanup" }); - expect(owner.snapshot()).toMatchObject({ - stderr: { text: "final launcher diagnostic" }, - exitCode: 2, - }); - expect(await owner.waitForOutputClose(0)).toBe(true); + await withVerifierCleanup(async () => { + expect(owner.stop()).toBe(stopped); + await awaitAcknowledgement(started, stopped, "process cleanup"); + // Let cleanup observe exit before delivering the separately acknowledged output. + await new Promise((resolve) => setImmediate(resolve)); + producer.stderr.end("final launcher diagnostic"); + producer.stdout.end(); + producer.emit("close", 2, null); + await expect(stopped).resolves.toEqual({ status: "verified-cleanup" }); + expect(owner.snapshot()).toMatchObject({ + stderr: { text: "final launcher diagnostic" }, + exitCode: 2, + }); + expect(await owner.waitForOutputClose(0)).toBe(true); + }, [ + () => { + producer.stderr.end(); + producer.stdout.end(); + if (!owner.observedOutputClose()) producer.emit("close", 2, null); + }, + async () => { + await stopped; + }, + () => owner.dispose(), + () => producer.stderr.destroy(), + () => producer.stdout.destroy(), + ]); }); it("bounds late output retention and drain without changing verified cleanup", async () => { diff --git a/src/process/RegexMatcher.test.ts b/src/process/RegexMatcher.test.ts index 60552860ed..3fe5c8eb5d 100644 --- a/src/process/RegexMatcher.test.ts +++ b/src/process/RegexMatcher.test.ts @@ -100,24 +100,6 @@ it.each([ nodeOptions: '"--max-old-space-size=256', heap: effective(80, true), }, - { - name: "leaves a percentage flag unestablished instead of the 128 MiB request", - execArgv: ["--max-old-space-size-percentage=50"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, - { - name: "lets a percentage flag win over a later absolute flag", - execArgv: ["--max-old-space-size-percentage=50", "--max-old-space-size=64"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, - { - name: "lets a percentage flag win over an earlier absolute flag", - execArgv: ["--max-old-space-size=64", "--max-old-space-size-percentage=50"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, { name: "lets a NODE_OPTIONS percentage win over an execArgv absolute flag", execArgv: ["--max-old-space-size=64"], @@ -164,36 +146,12 @@ it.each([ nodeOptions: undefined, heap: { kind: "unknown" } as const, }, - { - name: "leaves an env-file launch unestablished when NODE_OPTIONS was absent", - execArgv: ["--env-file=rea.env"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, - { - name: "leaves an env-file-if-exists launch unestablished when NODE_OPTIONS was absent", - execArgv: ["--env-file-if-exists=.env"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, - { - name: "does not let an absolute flag hide NODE_OPTIONS supplied by an env file", - execArgv: ["--env-file=rea.env", "--max-old-space-size=64"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, { name: "recognizes an env-file flag split from its path", execArgv: ["--env-file", "rea.env"], nodeOptions: undefined, heap: { kind: "unknown" } as const, }, - { - name: "keeps an inherited NODE_OPTIONS value when an env file cannot override it", - execArgv: ["--env-file=rea.env"], - nodeOptions: "--max-old-space-size=64", - heap: effective(64, true), - }, { name: "lets a later absolute flag win when an inherited NODE_OPTIONS blocks the env file", execArgv: ["--max-old-space-size=32", "--env-file-if-exists=rea.env"], @@ -206,33 +164,6 @@ it.each([ nodeOptions: "", heap: effective(128, false), }, - { - name: "leaves a config-file percentage source unestablished", - execArgv: ["--experimental-config-file=node.config.json"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, - { - name: "leaves a default config-file launch unestablished", - execArgv: ["--experimental-default-config-file"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, - { - name: "recognizes a config-file flag split from its path", - execArgv: ["--experimental-config-file", "node.config.json"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, - { - name: "does not let an absolute flag hide a config-file heap option", - execArgv: [ - "--experimental-config-file=node.config.json", - "--max-old-space-size=64", - ], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, { name: "does not let a recovered NODE_OPTIONS hide a config-file heap option", execArgv: ["--experimental-default-config-file"], @@ -245,30 +176,12 @@ it.each([ nodeOptions: "--experimental-config-file=node.config.json", heap: { kind: "unknown" } as const, }, - { - name: "leaves a positive --max-heap-size unestablished instead of the 128 MiB request", - execArgv: ["--max-heap-size=64"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, { name: "recognizes an underscore --max-heap-size", execArgv: ["--max_heap_size=+064"], nodeOptions: undefined, heap: { kind: "unknown" } as const, }, - { - name: "leaves a NODE_OPTIONS --max-heap-size unestablished", - execArgv: [], - nodeOptions: "--max-heap-size=64", - heap: { kind: "unknown" } as const, - }, - { - name: "does not let an old-space flag hide a positive --max-heap-size", - execArgv: ["--max-heap-size=64", "--max-old-space-size=256"], - nodeOptions: undefined, - heap: { kind: "unknown" } as const, - }, { name: "does not let a later --max-heap-size hide an old-space flag", execArgv: ["--max-old-space-size=64"], diff --git a/src/process/capture/FilesystemCoverage.test.ts b/src/process/capture/FilesystemCoverage.test.ts index 2d5a3d891b..62a4ff0c5e 100644 --- a/src/process/capture/FilesystemCoverage.test.ts +++ b/src/process/capture/FilesystemCoverage.test.ts @@ -173,7 +173,7 @@ it.skipIf(process.platform === "win32")( expect( snapshot.files.find(({ path }) => path === "root_0:readable.txt") ?.sha256, - ).not.toBeNull(); + ).toEqual(expect.any(String)); expect( snapshot.files.find(({ path }) => path === "root_0:unreadable.txt"), ).toMatchObject({ type: "file", size: 17, sha256: null }); diff --git a/src/process/capture/ProcessCaptureLifecycle.test.ts b/src/process/capture/ProcessCaptureLifecycle.test.ts index d70e14c348..781ae11c59 100644 --- a/src/process/capture/ProcessCaptureLifecycle.test.ts +++ b/src/process/capture/ProcessCaptureLifecycle.test.ts @@ -529,15 +529,16 @@ it("cancels the bounded-wait timer once deliveries settle after an exit", async it("keeps a non-empty reason when a signal delivery throws without a message", async () => { const fake = fakeTerminal(); + let signalStarted = false; const throwing = { ...fake, signalTarget: async (): Promise => { + signalStarted = true; throw new Error(); }, }; const pending = start(throwing, { timeout_ms: 100, finalization_ms: 5_000 }); - await waitFor(() => throwing.signals.length >= 0); - await new Promise((resolve) => setTimeout(resolve, 150)); + await waitFor(() => signalStarted); throwing.deliverExit({ exitCode: 0, signal: 0 }); const exit = observed(await pending); diff --git a/src/python/PythonPycParser.test.ts b/src/python/PythonPycParser.test.ts index 246d6cb569..72a67a0f7e 100644 --- a/src/python/PythonPycParser.test.ts +++ b/src/python/PythonPycParser.test.ts @@ -1095,68 +1095,6 @@ it("rejects code keys and members whose consts are unhashable", () => { expect(hashable.coverage.status).toBe("complete"); }); -it("uses the code equality fields for each CPython layout", () => { - const codeCollections = ( - layout: "py38" | "py311", - different: boolean, - ): [Buffer, Buffer] => { - const lineA = marshalCode(layout, { - name: "f", - filename: "fixture.py", - firstlineno: 1, - linetable: Buffer.from([2, 1]), - }); - const lineB = marshalCode(layout, { - name: "f", - filename: "fixture.py", - firstlineno: 1, - linetable: Buffer.from([2, different ? 2 : 1]), - }); - const frozenSet = Buffer.concat([ - Buffer.from([0x3e]), - u32(2), - lineA, - lineB, - ]); - const dict = Buffer.concat([ - Buffer.from([0x7b]), - lineA, - Buffer.from([0x4e]), - lineB, - Buffer.from([0x54, 0x30]), - ]); - return [frozenSet, dict]; - }; - for (const [magic, layout] of [ - [3413, "py38"], - [3439, "py38"], - ] as const) { - const [frozenSet, dict] = codeCollections(layout, true); - const report = parse( - Buffer.concat([ - pythonPycHeader(magic), - marshalCode(layout, { consts: [frozenSet, dict] }), - ]), - ); - expect(root(report).consts).toEqual([ - { kind: "frozenset", length: 1 }, - { kind: "dict", length: 1 }, - ]); - } - - const [frozenSet, dict] = codeCollections("py311", true); - const py312 = parse( - Buffer.concat([ - pythonPycHeader(3531), - marshalCode("py311", { consts: [frozenSet, dict] }), - ]), - ); - expect(root(py312).consts).toEqual([ - { kind: "frozenset", length: 2 }, - { kind: "dict", length: 2 }, - ]); -}); - it("keeps the first dictionary key when a later key compares equal", () => { const key = (filename: string): Buffer => marshalCode("py311", { diff --git a/src/server/BundleResourcesMcp.test.ts b/src/server/BundleResourcesMcp.test.ts index a187b2c321..377cb71d02 100644 --- a/src/server/BundleResourcesMcp.test.ts +++ b/src/server/BundleResourcesMcp.test.ts @@ -1,9 +1,8 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { Client } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; import type { AnySchema } from "ajv"; -import { afterEach, expect, it } from "vitest"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; +import { expect, it } from "vitest"; +import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { z } from "zod"; @@ -19,9 +18,10 @@ import { SessionProviderRouter } from "../application/binary/SessionProviderRout import { toolContract } from "../contracts/toolContracts.js"; import { createEvidence } from "../domain/evidence.js"; import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js"; -import { createServer } from "./createServer.js"; import { createArtifactBinarySession } from "../composition/binary.js"; import { build } from "plist"; +import { createApplicationMcpHarness } from "../../tests/fixtures/applicationMcpHarness.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; const identity: ProviderIdentity = { id: "bundle-resource-mcp-fixture", @@ -29,14 +29,8 @@ const identity: ProviderIdentity = { version: "1", }; -const resources: Array<{ close(): Promise }> = []; - -afterEach(async () => { - await Promise.all(resources.splice(0).map((resource) => resource.close())); -}); - it("routes bundle resource MCP calls with null executable identity and retained Evidence", async () => { - const parent = await mkdtemp(join(tmpdir(), "rea-bundle-resource-mcp-")); + const parent = await createTestTempDirectory("rea-bundle-resource-mcp-"); const bundle = join(parent, "Fixture.app"); await mkdir(bundle); const plistPath = join(bundle, "Info.plist"); @@ -82,22 +76,11 @@ it("routes bundle resource MCP calls with null executable identity and retained new InvestigationRecords(), { createClient: () => ({ client: resourceClient, capabilities: [] }) }, ); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "bundle-resource-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - resources.push({ - close: async () => { - await client.close(); - await server.close(); - await session.close(); - await rm(parent, { recursive: true, force: true }); - }, + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "bundle-resource-test", version: "1" }, }); - await server.connect(serverTransport); - await client.connect(clientTransport); - await expectResourceSchemaBranches(client); const unbound = await client.callTool({ @@ -192,8 +175,8 @@ it("routes bundle resource MCP calls with null executable identity and retained }); it("exposes the production resource session through the MCP SDK for a keyless bundle", async () => { - const parent = await mkdtemp( - join(tmpdir(), "rea-bundle-resource-production-mcp-"), + const parent = await createTestTempDirectory( + "rea-bundle-resource-production-mcp-", ); const bundle = join(parent, "Fixture.app"); const resourceDirectory = join(bundle, "Resources"); @@ -210,23 +193,10 @@ it("exposes the production resource session through the MCP SDK for a keyless bu ); const session = await createArtifactBinarySession({}, process.platform); - const server = createServer({ kind: "session", session }); - const client = new Client({ - name: "production-bundle-resource-test", - version: "1", + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "production-bundle-resource-test", version: "1" }, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - resources.push({ - close: async () => { - await client.close(); - await server.close(); - await session.close(); - await rm(parent, { recursive: true, force: true }); - }, - }); - await server.connect(serverTransport); - await client.connect(clientTransport); const opened = await client.callTool({ name: "open_bundle_resources", diff --git a/src/server/environment/serveEnvironment.test.ts b/src/server/environment/serveEnvironment.test.ts index 8a88b40bc3..0ecb0f93be 100644 --- a/src/server/environment/serveEnvironment.test.ts +++ b/src/server/environment/serveEnvironment.test.ts @@ -118,9 +118,18 @@ describe("replay value fidelity over MCP", () => { name: "get_users_by_id", arguments: { id: "3" }, }); - expect(JSON.stringify(ownKeys.structuredContent)).toContain( - '"__proto__":{"retained":true}', - ); + const content = ownKeys.structuredContent; + if ( + typeof content !== "object" || + content === null || + !("body" in content) + ) + throw new Error("Expected a structured replay body"); + const body = content.body; + expect(body).toEqual(JSON.parse(proto)); + if (typeof body !== "object" || body === null) + throw new Error("Expected an object replay body"); + expect(Object.hasOwn(body, "__proto__")).toBe(true); } finally { await transport.terminateSession(); await client.close(); diff --git a/src/server/environment/serveEnvironmentState.test.ts b/src/server/environment/serveEnvironmentState.test.ts index b39800502d..d27f3b341e 100644 --- a/src/server/environment/serveEnvironmentState.test.ts +++ b/src/server/environment/serveEnvironmentState.test.ts @@ -431,9 +431,28 @@ describe("state data plane intake", () => { ]); expect(batch.json).toMatchObject([ { id: 1, result: {} }, - { id: 2, result: { parts: [{ kind: "data" }] } }, + { + id: 2, + result: { + parts: [ + { + kind: "data", + data: { + rea_state: { + entities: expect.arrayContaining([ + expect.objectContaining({ + type: "/api/users", + id: "2", + fields: expect.objectContaining({ login: "renamed" }), + }), + ]), + }, + }, + }, + ], + }, + }, ]); - expect(JSON.stringify(batch.json)).toContain('"login":"renamed"'); const notified = await plane.post({ jsonrpc: "2.0", method: "data/reset" }); expect(notified.status).toBe(202); expect(loginOf(plane.store)).toBe("probe"); @@ -542,10 +561,29 @@ describe("state data plane timing and order", () => { const replies = (await batch).json; expect(replies).toMatchObject([ { id: 1, result: {} }, - { id: 2, result: { parts: [{ kind: "data" }] } }, + { + id: 2, + result: { + parts: [ + { + kind: "data", + data: { + rea_state: { + entities: expect.arrayContaining([ + expect.objectContaining({ + type: "/api/users", + id: "2", + fields: expect.objectContaining({ login: "renamed" }), + }), + ]), + }, + }, + }, + ], + }, + }, ]); // The get arrived with the add, before the reset, so it reports the loaded state. - expect(JSON.stringify(replies)).toContain('"login":"renamed"'); await expect(reset).resolves.toMatchObject({ result: {} }); expect(loginOf(plane.store)).toBe("probe"); }); diff --git a/src/server/http/serveStreamableHttp.test.ts b/src/server/http/serveStreamableHttp.test.ts index a8b1f87c52..f07c71890b 100644 --- a/src/server/http/serveStreamableHttp.test.ts +++ b/src/server/http/serveStreamableHttp.test.ts @@ -13,6 +13,8 @@ import { expectForeignHostRefused, rawHttpRequest, } from "../../../tests/support/http/rawRequest.js"; +import { awaitAcknowledgement } from "../../../tests/fixtures/lifecycleAcknowledgement.js"; +import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs"; import { hostHeaderAllowlist, McpSessionSlot, @@ -1104,21 +1106,25 @@ it.each(["replacement", "DELETE"])( const first = await connect(served.url); let next: Awaited> | undefined; let replacement: ReturnType | undefined; + let deletion: + | ReturnType + | undefined; + const request = first.client + .callTool({ name: "slow", arguments: {} }) + .catch(() => undefined); try { - void first.client - .callTool({ name: "slow", arguments: {} }) - .catch(() => undefined); - await started; - const deletion = + await awaitAcknowledgement(started, request, "tool start"); + deletion = ending === "DELETE" ? first.transport.terminateSession() : undefined; - if (deletion !== undefined) await aborted; + if (deletion !== undefined) + await awaitAcknowledgement(aborted, deletion, "tool cancellation"); let connected = false; replacement = connect(served.url).then((value) => { connected = true; next = value; return value; }); - await aborted; + await awaitAcknowledgement(aborted, replacement, "tool cancellation"); await new Promise((resolve) => setImmediate(resolve)); expect(connections).toBe(1); expect(connected).toBe(false); @@ -1132,9 +1138,15 @@ it.each(["replacement", "DELETE"])( ]); } finally { release(); - await replacement?.catch(() => undefined); - await first.client.close(); - await next?.client.close(); + await closeVerifierResources([ + () => first.client.close(), + async () => { + await Promise.allSettled([request, deletion, replacement]); + }, + async () => { + await next?.client.close(); + }, + ]); } }, ); @@ -1162,7 +1174,11 @@ it.each(["ready", "failure"] as const)( }); const settled = starting.catch(() => undefined); try { - await factoryStarted.promise; + await awaitAcknowledgement( + factoryStarted.promise, + starting, + "factory start", + ); expect((await fetch(new URL("/pending", url))).status).toBe(503); expect(handled).not.toHaveBeenCalled(); factoryGate.release(); @@ -1222,14 +1238,19 @@ it.each(["open stream", "late response"])( const response = fetch(new URL("/stream", served.url)).catch( () => undefined, ); - await entered.promise; - if (timing === "open stream") await response; - let closed = false; - const closing = served.prepareShutdown().then(() => { - closed = true; - }); + let closing: Promise | undefined; try { - await cancelling.promise; + await awaitAcknowledgement(entered.promise, response, "route start"); + if (timing === "open stream") await response; + let closed = false; + closing = served.prepareShutdown().then(() => { + closed = true; + }); + await awaitAcknowledgement( + cancelling.promise, + closing, + "body cancellation", + ); await new Promise((resolve) => setImmediate(resolve)); expect(closed).toBe(false); expect(shutdownHooks()).toBe(0); @@ -1238,8 +1259,14 @@ it.each(["open stream", "late response"])( expect(shutdownHooks()).toBe(1); } finally { cleanup.release(); - await closing; - await (await response)?.body?.cancel().catch(() => undefined); + await closeVerifierResources([ + async () => { + await (closing ?? served.close()); + }, + async () => { + await (await response)?.body?.cancel().catch(() => undefined); + }, + ]); } }, ); @@ -1586,13 +1613,14 @@ it.each(["prepareShutdown", "close"] as const)( const response = fetch(new URL("/pending", served.url)).catch( () => undefined, ); - await started; - let closed = false; - const closing = served[method]().then(() => { - closed = true; - }); + let closing: Promise | undefined; try { - await cleanupStarted; + await awaitAcknowledgement(started, response, "route start"); + let closed = false; + closing = served[method]().then(() => { + closed = true; + }); + await awaitAcknowledgement(cleanupStarted, closing, "route finalizer"); await new Promise((resolve) => setImmediate(resolve)); expect(closed).toBe(false); expect(shutdownHooks()).toBe(0); @@ -1601,8 +1629,14 @@ it.each(["prepareShutdown", "close"] as const)( expect(shutdownHooks()).toBe(1); } finally { release(); - await closing; - await response; + await closeVerifierResources([ + async () => { + await (closing ?? served.close()); + }, + async () => { + await response; + }, + ]); } }, ); @@ -1909,7 +1943,7 @@ it.each(["replacement", "DELETE", "shutdown"])( ); try { // With no stream prefetch, this proves the SDK is waiting on the old body. - await readStarted; + await awaitAcknowledgement(readStarted, pending, "request body read"); if (ending === "shutdown") await slot.close(); else { if (ending === "DELETE") { @@ -2034,12 +2068,14 @@ it.each(["after timeout", "during drain"])( }, }); const first = await connect(served.url); + const request = first.client + .callTool({ name: "slow", arguments: {} }) + .catch(() => undefined); + let replacement: Promise | undefined; + let pendingClose: Promise | undefined; try { - void first.client - .callTool({ name: "slow", arguments: {} }) - .catch(() => undefined); - await started; - const replacement = fetch(served.url, { + await awaitAcknowledgement(started, request, "tool start"); + replacement = fetch(served.url, { method: "POST", headers: { "content-type": "application/json", @@ -2051,8 +2087,8 @@ it.each(["after timeout", "during drain"])( if (shutdownTiming === "during drain") return undefined; throw cause; }); - await drainStarted; - const pendingClose = + await awaitAcknowledgement(drainStarted, replacement, "session drain"); + pendingClose = shutdownTiming === "during drain" ? served.close() : undefined; const refused = await replacement; if (shutdownTiming === "after timeout") expect(refused?.status).toBe(503); @@ -2068,7 +2104,15 @@ it.each(["after timeout", "during drain"])( expect(shutdowns).toBe(1); } finally { release(); - await first.client.close(); + await closeVerifierResources([ + () => first.client.close(), + async () => { + await Promise.allSettled([request, replacement]); + }, + async () => { + await pendingClose; + }, + ]); } }, ); diff --git a/src/server/registerGuidedWorkflowTool.test.ts b/src/server/registerGuidedWorkflowTool.test.ts index 0b275074b5..e237ca208f 100644 --- a/src/server/registerGuidedWorkflowTool.test.ts +++ b/src/server/registerGuidedWorkflowTool.test.ts @@ -1,7 +1,11 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { afterEach, expect, it, vi } from "vitest"; +import { InMemoryTransport } from "@modelcontextprotocol/client"; +import { expect, vi } from "vitest"; import type { AnalysisOperationPort } from "../application/AnalysisProvider.js"; import { parseMcpToolError } from "../../tests/fixtures/mcpToolError.js"; +import { + mcpTest as it, + type TestMcp, +} from "../../tests/support/mcp/mcpFixture.js"; import { PROMPT_CONTRACTS } from "../contracts/promptContracts.js"; import { createCatalogDigests } from "../catalogIdentity.js"; import { TOOL_CONTRACTS, toolContract } from "../contracts/toolContracts.js"; @@ -10,20 +14,7 @@ import { registerGuidedWorkflowTool } from "./registerGuidedWorkflowTool.js"; import { registerGuidedPrompts } from "./registerPrompts.js"; import { ToolResultDelivery } from "./ToolResultDelivery.js"; -const resources: Array<{ - readonly client: Client; - readonly server: EvidenceMcpServer; -}> = []; - -afterEach(async () => { - await Promise.all( - resources - .splice(0) - .flatMap(({ client, server }) => [client.close(), server.close()]), - ); -}); - -const fixture = async (families?: ReadonlySet) => { +const fixture = async (mcp: TestMcp, families?: ReadonlySet) => { const info = { name: "guided-workflow-fixture", version: "1" }; const server = new EvidenceMcpServer( info, @@ -49,15 +40,29 @@ const fixture = async (families?: ReadonlySet) => { execute, } as unknown as AnalysisOperationPort); registerGuidedWorkflowTool(server); - const client = new Client({ name: "guided-workflow-client", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); - resources.push({ client, server }); + const client = await mcp.connect(server, { + name: "guided-workflow-client", + version: "1", + }); return { client, server, execute }; }; +it("closes the server when client transport startup fails", async ({ mcp }) => { + const pair = InMemoryTransport.createLinkedPair(); + const failure = new Error("client transport startup failed"); + vi.spyOn(InMemoryTransport, "createLinkedPair").mockReturnValueOnce(pair); + vi.spyOn(pair[0], "start").mockRejectedValueOnce(failure); + try { + await expect(fixture(mcp)).rejects.toBe(failure); + await expect( + pair[1].send({ jsonrpc: "2.0", method: "probe" }), + ).rejects.toThrow(); + } finally { + await Promise.all(pair.map((transport) => transport.close())); + vi.restoreAllMocks(); + } +}); + const textOf = (result: { readonly content: readonly { readonly type: string; @@ -65,8 +70,10 @@ const textOf = (result: { }[]; }) => result.content.find(({ type }) => type === "text")?.text ?? ""; -it("lists workflows and renders text equivalent to the native MCP prompt", async () => { - const { client, execute } = await fixture(); +it("lists workflows and renders text equivalent to the native MCP prompt", async ({ + mcp, +}) => { + const { client, execute } = await fixture(mcp); expect(toolContract("guided_workflows").kind).toBe("application"); const listedTools = await client.listTools(); expect(listedTools.tools.map(({ name }) => name)).toContain( @@ -125,8 +132,10 @@ it("lists workflows and renders text equivalent to the native MCP prompt", async expect(execute).not.toHaveBeenCalled(); }); -it("returns actionable errors and leaves native completion available", async () => { - const { client, execute } = await fixture(); +it("returns actionable errors and leaves native completion available", async ({ + mcp, +}) => { + const { client, execute } = await fixture(mcp); await expect( client.getPrompt({ name: "investigate_feature", arguments: {} }), ).rejects.toThrow(); @@ -152,8 +161,8 @@ it("returns actionable errors and leaves native completion available", async () expect(execute).toHaveBeenCalledTimes(1); }); -it("rejects unexpected top-level tool arguments", async () => { - const { client, execute } = await fixture(); +it("rejects unexpected top-level tool arguments", async ({ mcp }) => { + const { client, execute } = await fixture(mcp); const unexpected = await client.callTool({ name: "guided_workflows", arguments: { action: "list", extra: true }, @@ -175,8 +184,10 @@ it("preserves prompt identity while catalog identity includes the added tool", ( ); }); -it("retains numeric nested argument paths through the MCP application boundary", async () => { - const { client } = await fixture(); +it("retains numeric nested argument paths through the MCP application boundary", async ({ + mcp, +}) => { + const { client } = await fixture(mcp); const result = await client.callTool({ name: "guided_workflows", arguments: { @@ -196,8 +207,8 @@ it("retains numeric nested argument paths through the MCP application boundary", ); }); -it("reports every forbidden list field in one MCP failure", async () => { - const { client } = await fixture(); +it("reports every forbidden list field in one MCP failure", async ({ mcp }) => { + const { client } = await fixture(mcp); const result = await client.callTool({ name: "guided_workflows", arguments: { @@ -215,8 +226,10 @@ it("reports every forbidden list field in one MCP failure", async () => { ); }); -it("refuses misspelled context fields rather than losing the caller's selection", async () => { - const { client } = await fixture(); +it("refuses misspelled context fields rather than losing the caller's selection", async ({ + mcp, +}) => { + const { client } = await fixture(mcp); const result = await client.callTool({ name: "guided_workflows", arguments: { @@ -242,8 +255,10 @@ it("refuses misspelled context fields rather than losing the caller's selection" ).rejects.toThrow(); }); -it("lists the same workflows as scoped native prompts and refuses unavailable workflows", async () => { - const { client } = await fixture(new Set(["application"])); +it("lists the same workflows as scoped native prompts and refuses unavailable workflows", async ({ + mcp, +}) => { + const { client } = await fixture(mcp, new Set(["application"])); const native = await client.listPrompts(); const listed = await client.callTool({ name: "guided_workflows", diff --git a/tests/acceptance/analysisSettingsMcp.test.ts b/tests/acceptance/analysisSettingsMcp.test.ts index 6acbd7fe4e..c855950f53 100644 --- a/tests/acceptance/analysisSettingsMcp.test.ts +++ b/tests/acceptance/analysisSettingsMcp.test.ts @@ -2,6 +2,7 @@ import { analysisSettingsResultSchema } from "../../src/contracts/analysisSettin import { expect, it } from "vitest"; import type { AnySchema } from "ajv"; import { Ajv2020 } from "ajv/dist/2020.js"; +import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { join } from "node:path"; @@ -9,6 +10,8 @@ import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { INLINE_EVIDENCE_MCP_ENVIRONMENT } from "../../scripts/lib/mcp-verification-environment.mjs"; import { asarHeaderBytes } from "../fixtures/asarHeader.js"; import { nativeMatchingNamedInventory } from "../fixtures/nativeFunctionMatching.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; it("advertises exact contracts and recovers a full large-index archive over one MCP connection", async () => { const root = await createTestTempDirectory("rea-runtime-settings-"); @@ -83,6 +86,30 @@ it("advertises exact contracts and recovers a full large-index archive over one expect(recovered.structuredContent).toMatchObject({ normalized_result: { coverage: { status: "complete-within-substeps" } }, }); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(recovered.structuredContent).normalized_result, + ); + const inventoryStep = inspection.substeps.find( + (substep) => substep.operation === "inventory_artifact", + ); + if (inventoryStep === undefined) + throw new Error("Missing inventory substep"); + const inventory = inventoryStep.evidence.normalized_result; + for (const [path, bytes] of [ + ["main.js", first], + ["second.js", second], + ] as const) { + const occurrence = inventory.occurrences.find( + (row) => row.logical_path === path, + ); + expect(occurrence, path).toBeDefined(); + expect( + inventory.nodes.find( + (node) => node.artifact_id === occurrence?.artifact_id, + )?.sha256, + path, + ).toBe(createHash("sha256").update(bytes).digest("hex")); + } const analyzed = await invoke( "analyze_javascript_application", analysisRequest, diff --git a/tests/acceptance/apkSignatureV1Cli.test.ts b/tests/acceptance/apkSignatureV1Cli.test.ts index 34da933f51..1399f045ad 100644 --- a/tests/acceptance/apkSignatureV1Cli.test.ts +++ b/tests/acceptance/apkSignatureV1Cli.test.ts @@ -1,9 +1,9 @@ import { apkSignatureResultSchema } from "../../src/domain/apkSignature/apkSignatureReport.js"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { expect } from "vitest"; +import { expect, onTestFinished } from "vitest"; +import { closeVerifierResources } from "../../scripts/lib/verifier-cleanup.mjs"; import { AnalysisProviderRegistry } from "../../src/application/binary/AnalysisProviderRegistry.js"; import { composeBinarySession } from "../../src/application/binary/BinarySessionComposition.js"; @@ -17,9 +17,12 @@ import { LEAF_CERTIFICATE, LEAF_SHA256, LEAF_SUBJECT, + OID_ECDSA_SHA256, + OID_SHA256, OPENSSL_BLOCK_ISSUER_SERIAL, OPENSSL_BLOCK_KEY_ID, ROOT_CERTIFICATE, + ROOT_SHA256, signedDataBlock, signerInfo, tlv, @@ -32,6 +35,7 @@ import { import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { cliTest } from "../support/cli/cliFixture.js"; +import { createTestMcp } from "../support/mcp/mcpFixture.js"; const completeEvidence = (value: unknown) => { const evidence = toolContract("inspect_apk_signature").outputSchema.parse( @@ -45,6 +49,59 @@ const completeEvidence = (value: unknown) => { }; }; +const connectSignerMcp = async () => { + const session = composeBinarySession(new AnalysisProviderRegistry([])); + const mcp = createTestMcp(); + onTestFinished(() => + closeVerifierResources([ + () => mcp.close(), + async () => { + const result = await session.close(); + if (!result.ok) throw result.error; + }, + ]), + ); + return await mcp.connect(createServer({ kind: "session", session }), { + name: "apk-v1-signer-parity", + version: "1", + }); +}; + +const expectV1Only = ( + report: ReturnType["normalized_result"], +) => { + expect(report.schemes.v1.present).toBe(true); + expect(report.posture).toMatchObject({ + v1_only: true, + strongest_scheme: "v1", + v1_signers_match_block_signers: null, + }); + expect(report.schemes.v1.signers).toMatchObject([ + { + signature_file: "META-INF/CERT.SF", + block_file: "META-INF/CERT.EC", + certificate: { + sha256_fingerprint: LEAF_SHA256, + subject: LEAF_SUBJECT, + public_key_algorithm: "ec", + }, + certificates: [ + { sha256_fingerprint: LEAF_SHA256, subject: LEAF_SUBJECT }, + { sha256_fingerprint: ROOT_SHA256 }, + ], + signer_infos: [ + { + identified_by: "issuer-and-serial", + certificate_sha256: LEAF_SHA256, + digest_algorithm_oid: OID_SHA256, + signature_algorithm_oid: OID_ECDSA_SHA256, + }, + ], + decode_error: null, + }, + ]); +}; + cliTest( "names the v1 signer certificate and a v1/v2 signer mismatch identically through CLI and MCP", async ({ cli }) => { @@ -57,6 +114,20 @@ cliTest( ]); const v1Only = join(root, "v1-only.apk"); await writeFile(v1Only, v1Zip); + const v1Stored = join(root, "v1-stored.apk"); + await writeFile( + v1Stored, + await zipWith([ + { name: "classes.dex", data: "dex" }, + { name: "META-INF/MANIFEST.MF", data: "Manifest-Version: 1.0" }, + { name: "META-INF/CERT.SF", data: "Signature-Version: 1.0" }, + { + name: "META-INF/CERT.EC", + data: OPENSSL_BLOCK_ISSUER_SERIAL, + stored: true, + }, + ]), + ); // The v2 block names a different certificate than the JAR signature. const mixed = join(root, "v1-v2-mismatch.apk"); await writeFile( @@ -118,23 +189,14 @@ cliTest( return completeEvidence(result.json); }; const cliV1 = await viaCli(v1Only); + const cliStored = await viaCli(v1Stored); const cliMixed = await viaCli(mixed); const cliRotated = await viaCli(rotated); const cliRotatedWithV2 = await viaCli(rotatedWithV2); const cliMisrotated = await viaCli(misrotated); - expect(cliV1.normalized_result.schemes.v1.signers).toMatchObject([ - { - signature_file: "META-INF/CERT.SF", - block_file: "META-INF/CERT.EC", - certificate: { sha256_fingerprint: LEAF_SHA256, subject: LEAF_SUBJECT }, - decode_error: null, - }, - ]); - expect(cliV1.normalized_result.posture).toMatchObject({ - v1_only: true, - v1_signers_match_block_signers: null, - }); + expectV1Only(cliV1.normalized_result); + expectV1Only(cliStored.normalized_result); expect(cliMixed.normalized_result.posture).toMatchObject({ v1_only: false, v1_signers_match_block_signers: false, @@ -157,47 +219,33 @@ cliTest( ).toBe(false); expect(cliV1.limitations.join(" ")).toContain("NOT verified"); - const session = composeBinarySession(new AnalysisProviderRegistry([])); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "apk-v1-signer-parity", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const tool = (await client.listTools()).tools.find( - (t) => t.name === "inspect_apk_signature", + const client = await connectSignerMcp(); + const tool = (await client.listTools()).tools.find( + (t) => t.name === "inspect_apk_signature", + ); + const outputSchema: Record | undefined = + tool?.outputSchema; + if (tool === undefined || outputSchema === undefined) + throw new Error("inspect_apk_signature must advertise an output schema"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + for (const [path, fromCli] of [ + [v1Only, cliV1], + [mixed, cliMixed], + [rotated, cliRotated], + [rotatedWithV2, cliRotatedWithV2], + ] as const) { + const reply = await client.callTool( + { name: tool.name, arguments: { path, detail: "complete" } }, + { toolDefinition: tool }, ); - const outputSchema: Record | undefined = - tool?.outputSchema; - if (tool === undefined || outputSchema === undefined) - throw new Error( - "inspect_apk_signature must advertise an output schema", - ); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); - for (const [path, fromCli] of [ - [v1Only, cliV1], - [mixed, cliMixed], - [rotated, cliRotated], - [rotatedWithV2, cliRotatedWithV2], - ] as const) { - const reply = await client.callTool( - { name: tool.name, arguments: { path, detail: "complete" } }, - { toolDefinition: tool }, - ); - expect(reply.isError, JSON.stringify(reply)).not.toBe(true); - expect( - ajv.validate(outputSchema, reply.structuredContent), - JSON.stringify(ajv.errors), - ).toBe(true); - const evidence = parseEvidence(reply.structuredContent); - expect(evidence.normalized_result).toEqual(fromCli.normalized_result); - expect(evidence.evidence_id).toBe(fromCli.evidence_id); - } - } finally { - await client.close(); - await server.close(); - await session.close(); + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + expect( + ajv.validate(outputSchema, reply.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const evidence = parseEvidence(reply.structuredContent); + expect(evidence.normalized_result).toEqual(fromCli.normalized_result); + expect(evidence.evidence_id).toBe(fromCli.evidence_id); } }, ); @@ -289,9 +337,11 @@ cliTest( expect( evidence.normalized_result.schemes.v1.signers[0]?.decode_error, ).toContain("malformed PKCS#7"); - expect(evidence.limitations).toContain( - "v1_signers_match_block_signers is null: one or more v1 signer identities are unknown; inspect the v1 signer records for details.", + const comparisonLimitation = evidence.limitations.find((limitation) => + limitation.includes("v1_signers_match_block_signers"), ); + expect(comparisonLimitation).toContain("v1 signer identit"); + expect(comparisonLimitation).toContain("unknown"); } }, ); diff --git a/tests/acceptance/apkSignatureViewsCliMcp.test.ts b/tests/acceptance/apkSignatureViewsCliMcp.test.ts index e2ce8b1034..743b67022a 100644 --- a/tests/acceptance/apkSignatureViewsCliMcp.test.ts +++ b/tests/acceptance/apkSignatureViewsCliMcp.test.ts @@ -13,6 +13,7 @@ import { } from "../../src/domain/apkSignature/apkSignatureQuery.js"; import { apkSignatureResultSchema } from "../../src/domain/apkSignature/apkSignatureReport.js"; import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; import { @@ -358,16 +359,32 @@ cliTest( "--json", ], }); - expect(invalid.exitCode).not.toBe(0); - expect(invalid.stdout).toContain("evidence_output"); - expect(invalid.stdout).not.toContain("ENOENT"); + expect(invalid.exitCode).toBe(1); + expect(invalid.json).toMatchObject({ + code: "invalid_request", + details: { + operation: "inspect_apk_signature", + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ["evidence_output"], + reason: "invalid_value", + }), + ]), + }, + }); const ambiguous = await call("inspect_apk_signature", { path: join(root, "missing.apk"), detail: "complete", view: { collection: "signers" }, }); - expect(ambiguous.isError).toBe(true); - expect(JSON.stringify(ambiguous)).toContain("view"); - expect(JSON.stringify(ambiguous)).not.toContain("ENOENT"); + expect(parseMcpToolError(ambiguous).error).toMatchObject({ + code: "invalid_request", + details: { + operation: "inspect_apk_signature", + issues: expect.arrayContaining([ + expect.objectContaining({ path: ["view"], reason: "invalid_value" }), + ]), + }, + }); }, ); diff --git a/tests/acceptance/applications/adbPackageInventoryCoverage.test.ts b/tests/acceptance/applications/adbPackageInventoryCoverage.test.ts index 1b0fea8265..69f6523d50 100644 --- a/tests/acceptance/applications/adbPackageInventoryCoverage.test.ts +++ b/tests/acceptance/applications/adbPackageInventoryCoverage.test.ts @@ -79,20 +79,21 @@ process.exit(91); expect.stringContaining("unparsed_lines"), ); - if (operation !== "resolve_adb_packages") continue; - expect(evidence.normalized_result).toMatchObject({ - matches: [ - { - package_name: "com.example.app", - base_apk_device_path: "/data/app/example/base.apk", - }, - ], - exact_match: false, - }); - const reply = await call(operation, { - serial: "emulator-5554", - query: "example", - }); + const installedPackage = { + package_name: "com.example.app", + base_apk_device_path: "/data/app/example/base.apk", + }; + expect(evidence.normalized_result).toMatchObject( + operation === "resolve_adb_packages" + ? { matches: [installedPackage], exact_match: false } + : { packages: [installedPackage], scope: "all" }, + ); + const reply = await call( + operation, + operation === "resolve_adb_packages" + ? { serial: "emulator-5554", query: "example" } + : { serial: "emulator-5554" }, + ); expect(reply.isError).not.toBe(true); const structured = toolContract(operation).outputSchema.parse( reply.structuredContent, diff --git a/tests/acceptance/applications/appBundleEncodingCli.test.ts b/tests/acceptance/applications/appBundleEncodingCli.test.ts index 133be6b625..f0cafe53d7 100644 --- a/tests/acceptance/applications/appBundleEncodingCli.test.ts +++ b/tests/acceptance/applications/appBundleEncodingCli.test.ts @@ -1,11 +1,10 @@ import { mkdir, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; -import { Client } from "@modelcontextprotocol/client"; -import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; -import { describe, expect, onTestFinished } from "vitest"; +import { describe, expect } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { cliTest } from "../../support/cli/cliFixture.js"; @@ -61,7 +60,7 @@ const encodingFixture = () => { valid: false, }, ]; - return { utf16, cases }; + return cases; }; describe.skipIf(process.platform !== "darwin")( @@ -81,21 +80,40 @@ describe.skipIf(process.platform !== "darwin")( join(contents, "MacOS", "App"), ]); expect(compiled.exitCode, compiled.stderr).toBe(0); - const { utf16, cases } = encodingFixture(); + const cases = encodingFixture(); + const { call } = await connectLocalToolsMcp({ + PATH: "/usr/bin:/bin", + }); + const cliFailureReasons: Readonly> = { + "declaration mismatch": + "XML encoding declaration disagrees with its bytes", + "truncated UTF-16": + "XML text bytes are invalid for the detected encoding", + "malformed UTF-8 executable": + "XML text bytes are invalid for the detected encoding", + }; for (const { name, bytes, valid } of cases) { await writeFile(join(contents, "Info.plist"), bytes); - const result = await cli.run({ - arguments: ["inspect-plist", app, "--json"], - environment: { - REA_LOG_LEVEL: "silent", - REA_ANALYSIS_PROVIDER: "auto", - }, - }); - expect( - result.exitCode, - `${name}: ${result.stdout}\n${result.stderr}`, - ).toBe(valid ? 0 : 1); - if (valid) + const opened = await call("open_binary", { path: app }); + if (valid) { + expect( + opened.isError, + `${name}: ${JSON.stringify(opened)}`, + ).not.toBe(true); + expect(opened.structuredContent, name).toMatchObject({ + result: { + path: join(contents, "MacOS", "App"), + format: "mach-o", + }, + }); + const result = await cli.run({ + arguments: ["inspect-plist", app, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect( + result.exitCode, + `${name}: ${result.stdout}\n${result.stderr}`, + ).toBe(0); expect(result.json).toMatchObject({ normalized_result: { value: { @@ -105,83 +123,53 @@ describe.skipIf(process.platform !== "darwin")( bundle: { executable: "App" }, }, }); - else - expect(result.json).toMatchObject({ - error: "Analysis failed", - details: { - path: join(contents, "Info.plist"), - reason: expect.stringContaining( - "app Info.plist is malformed, unsupported, or lacks CFBundleExecutable", - ), + const inspected = await call("inspect_plist", {}); + expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); + expect(inspected.structuredContent).toMatchObject({ + normalized_result: { + value: { + CFBundleExecutable: "App", + CFBundleIdentifier: "dev.rea.encoding", + }, }, }); - } - const transport = new StdioClientTransport({ - command: process.execPath, - args: [resolve("scripts/rea.mjs"), "mcp"], - env: { - PATH: process.env.PATH ?? "", - REA_LOG_LEVEL: "silent", - REA_ANALYSIS_PROVIDER: "auto", - }, - stderr: "pipe", - }); - const client = new Client({ - name: "bundle-encoding-parity", - version: "1", - }); - onTestFinished(async () => { - try { - await client.close(); - } finally { - await transport.close(); + const closed = await call("close_binary", {}); + expect(closed.isError, JSON.stringify(closed)).not.toBe(true); + } else { + expect(opened.isError, name).toBe(true); + expect(parseMcpToolError(opened), name).toMatchObject({ + error: { + details: { + path: join(contents, "Info.plist"), + reason: expect.stringContaining( + "app Info.plist is malformed, unsupported, or lacks CFBundleExecutable", + ), + }, + }, + }); + const cliFailureReason = cliFailureReasons[name]; + if (cliFailureReason !== undefined) { + const result = await cli.run({ + arguments: ["inspect-plist", app, "--json"], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(1); + expect(result.json).toMatchObject({ + code: "capability_unavailable", + details: { + provider_id: "native-macos", + operation: "inspect_plist", + reason: expect.stringContaining(join(contents, "Info.plist")), + }, + }); + expect(result.json).toMatchObject({ + details: { + reason: expect.stringContaining(cliFailureReason), + }, + }); + } } - }); - await client.connect(transport); - await writeFile( - join(contents, "Info.plist"), - Buffer.concat([ - Buffer.from([0xfe, 0xff]), - Buffer.from(utf16).swap16(), - ]), - ); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path: app }, - }); - expect(opened.isError, JSON.stringify(opened)).not.toBe(true); - const inspected = await client.callTool({ - name: "inspect_plist", - arguments: {}, - }); - expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); - expect(inspected.structuredContent).toMatchObject({ - normalized_result: { - value: { - CFBundleExecutable: "App", - CFBundleIdentifier: "dev.rea.encoding", - }, - }, - }); - await writeFile( - join(contents, "Info.plist"), - Buffer.concat([Buffer.from([0xff, 0xfe]), utf16.subarray(0, -1)]), - ); - const invalid = await client.callTool({ - name: "open_binary", - arguments: { path: app }, - }); - expect(invalid.isError).toBe(true); - expect(parseMcpToolError(invalid)).toMatchObject({ - error: { - details: { - path: join(contents, "Info.plist"), - reason: expect.stringContaining( - "app Info.plist is malformed, unsupported, or lacks CFBundleExecutable", - ), - }, - }, - }); + } }, 45_000, ); diff --git a/tests/acceptance/applications/bundleResourceTargetIdentity.test.ts b/tests/acceptance/applications/bundleResourceTargetIdentity.test.ts index 7a857f24e4..f3e885e6ba 100644 --- a/tests/acceptance/applications/bundleResourceTargetIdentity.test.ts +++ b/tests/acceptance/applications/bundleResourceTargetIdentity.test.ts @@ -3,9 +3,13 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; +import { interfaceBuilderAnalysisSchema } from "../../../src/domain/apple/interfaceBuilderGraph.js"; +import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; +import { inspectPlistSchema } from "../../../src/domain/native/nativeInspection.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { cliTest } from "../../support/cli/cliFixture.js"; cliTest.for([ @@ -61,8 +65,13 @@ cliTest.for([ "CFBundleExecutableProbeMarkerChangedController", ); const stale = await call(operation, parameters); - expect(stale.isError).toBe(true); - expect(JSON.stringify(stale)).toContain("integrity_mismatch"); + expect(parseMcpToolError(stale).error).toMatchObject({ + code: + operation === "inspect_plist" + ? "artifact_changed" + : "artifact_operation_failed", + category: "integrity_mismatch", + }); expect((await call("open_binary", { path: app })).isError).not.toBe(true); const after = await call(operation, parameters); @@ -71,13 +80,37 @@ cliTest.for([ expect(refreshed.subject?.digest.sha256).toBe( createHash("sha256").update(changed).digest("hex"), ); - if (operation !== "inspect_asset_catalog") - expect(JSON.stringify(refreshed.normalized_result)).toContain( - "ChangedController", + const nibHash = createHash("sha256").update(changedNib).digest("hex"); + if (operation === "inspect_plist") + expect( + inspectPlistSchema.parse(refreshed.normalized_result).value, + ).toMatchObject({ Marker: "ChangedController" }); + if (operation === "decode_interface_builder") { + const report = interfaceBuilderAnalysisSchema.parse( + refreshed.normalized_result, ); - if (operation !== "inspect_asset_catalog" && operation !== "inspect_plist") - expect(JSON.stringify(refreshed.normalized_result)).toContain( - createHash("sha256").update(changedNib).digest("hex"), + expect(report.documents).toContainEqual( + expect.objectContaining({ + relative_path: "Contents/Resources/Main.nib", + archive_sha256: nibHash, + }), ); + expect(report.graph.nodes).toContainEqual( + expect.objectContaining({ + attributes: expect.objectContaining({ + class_name: "ChangedController", + }), + }), + ); + } + if (operation === "inspect_keyed_archive") { + const report = keyedArchiveResultSchema.parse( + refreshed.normalized_result, + ); + expect(report.archive_sha256).toBe(nibHash); + expect(report.objects).toContainEqual( + expect.objectContaining({ class_name: "ChangedController" }), + ); + } }, ); diff --git a/tests/acceptance/applications/javascriptCancellationCli.test.ts b/tests/acceptance/applications/javascriptCancellationCli.test.ts index 473ae34362..0e7ce3c5ab 100644 --- a/tests/acceptance/applications/javascriptCancellationCli.test.ts +++ b/tests/acceptance/applications/javascriptCancellationCli.test.ts @@ -1,42 +1,35 @@ -import { execFile, spawn } from "node:child_process"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it, onTestFinished } from "vitest"; +import { expect } from "vitest"; import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { processTest } from "../../support/process/processFixture.js"; -it("returns a nonzero exit for a streamed JSON analysis failure", async () => { - const root = await createTestTempDirectory("rea-cli-js-missing-"); - const result = await new Promise<{ code: number | string; stdout: string }>( - (resolve, reject) => { - execFile( - process.execPath, - [ - "scripts/rea.mjs", - "analyze-javascript-application", - join(root, "missing"), - "--json", - ], - (error, stdout) => { - if (error !== null && error.code === undefined) { - reject(error); - return; - } - resolve({ code: error?.code ?? 0, stdout }); - }, - ); - }, - ); - expect(result.code).toBe(1); - expect(JSON.parse(result.stdout)).toMatchObject({ - code: "invalid_request", - details: { issues: [{ path: ["input_path"], reason: "invalid_value" }] }, - }); -}); +processTest( + "returns a nonzero exit for a streamed JSON analysis failure", + async ({ processes }) => { + const root = await createTestTempDirectory("rea-cli-js-missing-"); + const result = await processes.run( + process.execPath, + [ + "scripts/rea.mjs", + "analyze-javascript-application", + join(root, "missing"), + "--json", + ], + { timeoutMs: 20_000 }, + ); + expect(result.exitCode).toBe(1); + expect(JSON.parse(result.stdout)).toMatchObject({ + code: "invalid_request", + details: { issues: [{ path: ["input_path"], reason: "invalid_value" }] }, + }); + }, +); // `analyze` routes a JavaScript directory to the same workflow and must keep // its typed cancellation contract. -it.skipIf(process.platform === "win32").each([ +processTest.skipIf(process.platform === "win32").for([ { command: "analyze-javascript-application", phase: "analyze_javascript_application", @@ -59,7 +52,7 @@ it.skipIf(process.platform === "win32").each([ }, ])( "returns typed cancellation and exit 130 through $command during $phase", - async ({ command, phase, parsedFiles }) => { + async ({ command, phase, parsedFiles }, { processes }) => { const root = await createTestTempDirectory("rea-cli-js-cancellation-"); await writeFile(join(root, "main.js"), "export const observed = 1;\n"); const nodeArguments: string[] = []; @@ -88,17 +81,14 @@ it.skipIf(process.platform === "win32").each([ ); nodeArguments.push("--import", preload); } - const child = spawn( - process.execPath, - [...nodeArguments, "scripts/rea.mjs", command, root, "--json"], - { - stdio: ["ignore", "pipe", "pipe"], - }, - ); - onTestFinished(() => { - if (child.exitCode === null && child.signalCode === null) - child.kill("SIGTERM"); - }); + const child = processes.spawn(process.execPath, [ + ...nodeArguments, + "scripts/rea.mjs", + command, + root, + "--json", + ]); + child.stdin.end(); let stdout = "", partial = "", requested = false; diff --git a/tests/acceptance/applications/javascriptFunctionNameCli.test.ts b/tests/acceptance/applications/javascriptFunctionNameCli.test.ts index 9887373c0e..b402c5d7f3 100644 --- a/tests/acceptance/applications/javascriptFunctionNameCli.test.ts +++ b/tests/acceptance/applications/javascriptFunctionNameCli.test.ts @@ -43,12 +43,16 @@ it.each([ parseEvidence(JSON.parse(stdout)).normalized_result, ); const graph = result.semantic_graph; + const calls = graph.nodes.filter( + ({ kind, identity }) => + kind === "call-site" && identity.source_range?.start.line === callLine, + ); + expect(calls).toHaveLength(1); + const call = calls[0]!; const targets = graph.relations .filter( ({ relation, source_node_id }) => - relation === "calls" && - graph.nodes.find(({ node_id }) => node_id === source_node_id) - ?.identity.source_range?.start.line === callLine, + relation === "calls" && source_node_id === call.node_id, ) .map( ({ target_node_id }) => diff --git a/tests/acceptance/applications/javascriptGraphQueryCoverage.test.ts b/tests/acceptance/applications/javascriptGraphQueryCoverage.test.ts index c28b0add64..47adda924c 100644 --- a/tests/acceptance/applications/javascriptGraphQueryCoverage.test.ts +++ b/tests/acceptance/applications/javascriptGraphQueryCoverage.test.ts @@ -55,8 +55,46 @@ cliTest( parseEvidence(traced.structuredContent).normalized_result, ); expect(trace.summary.total_seed_matches).toBeGreaterThan(0); - if (seed.kind === "property") + expect(trace.seed).toEqual(seed); + expect(trace.seed_node_ids).toHaveLength( + trace.summary.total_seed_matches, + ); + const seedNodes = trace.seed_node_ids.map((id) => + trace.nodes.find(({ node_id }) => node_id === id), + ); + if (seed.kind === "event") { + expect(seedNodes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "event", + properties: expect.objectContaining({ event_name: "" }), + }), + expect.objectContaining({ + kind: "listener", + properties: expect.objectContaining({ event_name: "" }), + }), + ]), + ); + } else { expect(trace.summary.relevant_unknowns).toBeGreaterThan(0); + expect(seedNodes).toContainEqual( + expect.objectContaining({ + kind: "expression", + properties: expect.objectContaining({ + property_path: ["speed"], + property_slot_resolution: "unresolved", + }), + }), + ); + const seedUnknown = trace.unknowns.find( + ({ node_id }) => + node_id !== null && trace.seed_node_ids.includes(node_id), + ); + expect(seedUnknown).toMatchObject({ + family: "object-flow", + reason: "ambiguous-target", + }); + } } }, ); diff --git a/tests/acceptance/applications/javascriptNativeBindingOwnership.test.ts b/tests/acceptance/applications/javascriptNativeBindingOwnership.test.ts index 3ef72c408e..78d8af7b1e 100644 --- a/tests/acceptance/applications/javascriptNativeBindingOwnership.test.ts +++ b/tests/acceptance/applications/javascriptNativeBindingOwnership.test.ts @@ -122,6 +122,24 @@ cliTest.for(["CLI", "stdio MCP"] as const)( }), }), ]); + for (const source of [ + "ordinary.cjs", + "optional.cjs", + "arithmetic.cjs", + "logical.cjs", + ]) + expect(dependencies).toContainEqual( + expect.objectContaining({ + relation: "loads", + properties: expect.objectContaining({ operation: "load" }), + evidence: expect.objectContaining({ + location: expect.objectContaining({ + available: true, + value: expect.objectContaining({ source }), + }), + }), + }), + ); for (const edge of dependencies) if (edge.properties.operation === "load") expect(edge.relation).toBe("loads"); diff --git a/tests/acceptance/applications/javascriptNextRouteContext.test.ts b/tests/acceptance/applications/javascriptNextRouteContext.test.ts index 4788845f1d..6e96b4be01 100644 --- a/tests/acceptance/applications/javascriptNextRouteContext.test.ts +++ b/tests/acceptance/applications/javascriptNextRouteContext.test.ts @@ -132,6 +132,8 @@ cliTest.for(["CLI", "stdio MCP"] as const)( directory: "app.asar/app/widget/app", package: { path: `app.asar/${nestedManifest}` }, }); + expect(nested?.id).toEqual(expect.any(String)); + expect(recovered?.id).toEqual(expect.any(String)); expect(recovered?.id).not.toBe(nested?.id); expect( restored.route_inventory?.routes.map(({ path, source_count }) => ({ diff --git a/tests/acceptance/applications/javascriptWindowOptionsCli.test.ts b/tests/acceptance/applications/javascriptWindowOptionsCli.test.ts index 27e82ebefb..43a85f22ff 100644 --- a/tests/acceptance/applications/javascriptWindowOptionsCli.test.ts +++ b/tests/acceptance/applications/javascriptWindowOptionsCli.test.ts @@ -62,6 +62,24 @@ cliTest( properties: { source_path: "main.js", options_status: "object-literal", + web_preferences: expect.arrayContaining([ + { + name: "nodeIntegration", + value: { status: "literal", value: true, expression: null }, + }, + { + name: "contextIsolation", + value: { status: "literal", value: false, expression: null }, + }, + { + name: "preload", + value: { + status: "literal", + value: "./preload.js", + expression: null, + }, + }, + ]), resolution_sources: [ { source: "main.js", start: { line: 5 } }, { source: "main.js", start: { line: 4 } }, diff --git a/tests/acceptance/applications/mobileDirectoryComponents.test.ts b/tests/acceptance/applications/mobileDirectoryComponents.test.ts index fab73ee2b5..57bbb233f0 100644 --- a/tests/acceptance/applications/mobileDirectoryComponents.test.ts +++ b/tests/acceptance/applications/mobileDirectoryComponents.test.ts @@ -1,4 +1,5 @@ import { readFile, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { join } from "node:path"; import { @@ -37,7 +38,22 @@ const harmony = { nestedCode: "ets/modules.abc/actual.abc", runtimeFamilies: ["ark", "flutter", "javascript", "native"], nestedRuntime: "ark", - bridgeCount: 1, + nestedCategory: "bytecode", + fileComponents: { + manifests: ["module.json"], + bytecode: ["ets/modules.abc"], + resources: ["resources.index", "resources/base"], + native_libraries: ["libs/arm64-v8a/libflutter.so"], + javascript: ["assets/main.js"], + signing: ["META-INF/TEST.p7b"], + }, + fileBridges: [ + { + managed_path: "ets/modules.abc", + native_path: "libs/arm64-v8a/libflutter.so", + basis: "napi-library-convention", + }, + ], }; const fixtures = [ @@ -57,7 +73,28 @@ const fixtures = [ nestedCode: "classes.dex/actual.dex", runtimeFamilies: ["dalvik-art", "flutter", "java-kotlin", "native"], nestedRuntime: "dalvik-art", - bridgeCount: 2, + nestedCategory: "dex", + fileComponents: { + manifests: ["AndroidManifest.xml"], + resources: ["resources.arsc"], + dex: ["classes.dex"], + jvm_classes: ["Main.class"], + native_libraries: ["lib/arm64-v8a/libflutter.so"], + javascript: [], + signing: ["META-INF/TEST.RSA"], + }, + fileBridges: [ + { + managed_path: "classes.dex", + native_path: "lib/arm64-v8a/libflutter.so", + basis: "flutter-convention", + }, + { + managed_path: "Main.class", + native_path: "lib/arm64-v8a/libflutter.so", + basis: "flutter-convention", + }, + ], }, { family: "Apple", @@ -72,7 +109,25 @@ const fixtures = [ nestedCode: "Payload/Fixture.app/main.js/actual.js", runtimeFamilies: ["javascript", "native"], nestedRuntime: "javascript", - bridgeCount: 1, + nestedCategory: "javascript", + fileComponents: { + bundle_metadata: ["Payload/Fixture.app/Info.plist"], + executables: ["Payload/Fixture.app/libentry.dylib"], + frameworks: [], + native_libraries: ["Payload/Fixture.app/libentry.dylib"], + javascript: ["Payload/Fixture.app/main.js"], + signing: [], + privileged_helpers: [], + launchd_plists: [], + helpers: [], + }, + fileBridges: [ + { + source_path: "Payload/Fixture.app/main.js", + native_path: "Payload/Fixture.app/libentry.dylib", + basis: "javascript-and-native-content", + }, + ], }, harmony, { @@ -165,22 +220,27 @@ cliTest.for(fixtures)( const childPackages = fixture.paths.filter((member) => /\.(?:hap|hsp)$/u.test(member), ); - const categories = Object.values(projection.components); - for (const category of categories) - expect(new Set(category.map(({ path }) => path)).size).toBe( - category.length, - ); - const componentPaths = [ - ...new Set(categories.flat().map((component) => component.path)), - ].sort(); - expect(componentPaths, mode).toEqual( + expect( + Object.fromEntries( + Object.entries(projection.components).map( + ([category, components]) => [ + category, + components.map(({ path }) => path), + ], + ), + ), + mode, + ).toEqual( mode === "files" - ? fixture.paths - .filter((member) => !childPackages.includes(member)) - .sort() - : mode === "nested" - ? [fixture.nestedCode] - : [], + ? fixture.fileComponents + : Object.fromEntries( + Object.keys(fixture.fileComponents).map((category) => [ + category, + mode === "nested" && category === fixture.nestedCategory + ? [fixture.nestedCode] + : [], + ]), + ), ); expect(projection.runtime_families, mode).toEqual( mode === "files" @@ -189,8 +249,8 @@ cliTest.for(fixtures)( ? [fixture.nestedRuntime] : [], ); - expect(projection.bridge_candidates, mode).toHaveLength( - mode === "files" ? fixture.bridgeCount : 0, + expect(projection.bridge_candidates, mode).toEqual( + mode === "files" ? fixture.fileBridges : [], ); expect(projection.source_evidence_ids).toEqual([ inventoryEvidence.evidence_id, @@ -347,7 +407,11 @@ cliTest.for([ expect(child).toMatchObject({ artifact_id: null, sha256: null }); else { expect(child.artifact_id).toMatch(/^art_/u); - expect(child.sha256).toMatch(/^[a-f0-9]{64}$/u); + expect(child.sha256).toBe( + createHash("sha256") + .update("opaque fixture content") + .digest("hex"), + ); } } if (encrypted) expect(Object.values(graph.components).flat()).toEqual([]); diff --git a/tests/acceptance/applications/packageSubpathsCli.test.ts b/tests/acceptance/applications/packageSubpathsCli.test.ts index 50e6a80fbc..7237018d41 100644 --- a/tests/acceptance/applications/packageSubpathsCli.test.ts +++ b/tests/acceptance/applications/packageSubpathsCli.test.ts @@ -222,6 +222,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)( expect(preloads[0]?.identity).toMatchObject({ key: "electron-preload:module-specifier:node_modules/exported/parser.cjs", }); + expect(preloads[0]?.observations.length).toBeGreaterThan(0); for (const observation of preloads[0]?.observations ?? []) expect(observation).toMatchObject({ properties: { diff --git a/tests/acceptance/applications/peResources.test.ts b/tests/acceptance/applications/peResources.test.ts index 154101e0d4..6544701a4f 100644 --- a/tests/acceptance/applications/peResources.test.ts +++ b/tests/acceptance/applications/peResources.test.ts @@ -453,9 +453,15 @@ cliTest( { name: sessionDefinition.name, arguments: {} }, { toolDefinition: sessionDefinition }, ); - expect(JSON.stringify(session.structuredContent)).toContain( - "inspect_pe_resources", - ); + expect(session.isError, JSON.stringify(session)).not.toBe(true); + const status = toolContract("binary_session").outputSchema.parse( + session.structuredContent, + ).result; + expect( + status.tool_availability.groups.find((group) => + group.names.includes("inspect_pe_resources"), + ), + ).toMatchObject({ available: true }); await expectPeInventoriesAcrossAdapters(cli, call, root); await expectPeEmptyBranchesAcrossAdapters(cli, call, root); await expectPeWordAlignedRecordsAcrossAdapters({ diff --git a/tests/acceptance/applications/runtime.test.ts b/tests/acceptance/applications/runtime.test.ts index 1e1ee8abe1..c8dedddf63 100644 --- a/tests/acceptance/applications/runtime.test.ts +++ b/tests/acceptance/applications/runtime.test.ts @@ -83,6 +83,8 @@ const callCurrentDocument = async ( cwd: process.cwd(), env: { PATH: process.env.PATH ?? "", + // The fixture exercises Hopper even when other providers are installed. + REA_ANALYSIS_PROVIDER: "hopper", HOPPER_LAUNCHER_PATH: process.execPath, ...(target === undefined ? {} diff --git a/tests/acceptance/applications/webScriptPortablePathsCli.test.ts b/tests/acceptance/applications/webScriptPortablePathsCli.test.ts index b952defcf3..4261694c54 100644 --- a/tests/acceptance/applications/webScriptPortablePathsCli.test.ts +++ b/tests/acceptance/applications/webScriptPortablePathsCli.test.ts @@ -38,6 +38,9 @@ cliTest( const manifest = webScriptExportManifestSchema.parse( JSON.parse(await readFile(join(output, "manifest.json"), "utf8")), ); + expect(manifest.scripts.map(({ url }) => url)).toEqual( + scripts.map(({ url }) => url), + ); for (const [index, record] of manifest.scripts.entries()) { expect(record.content.state).toBe("exported"); if (record.content.state !== "exported") diff --git a/tests/acceptance/artifactClassification.test.ts b/tests/acceptance/artifactClassification.test.ts index 7436a49679..02b7700b9c 100644 --- a/tests/acceptance/artifactClassification.test.ts +++ b/tests/acceptance/artifactClassification.test.ts @@ -13,6 +13,7 @@ import { import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; import { parseEvidence } from "../../src/domain/evidence.js"; +import { javaClassFile } from "../fixtures/javaClassFile.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; @@ -20,92 +21,91 @@ cliTest.for(["jar", "zip"])( "distinguishes JVM classes from universal Mach-O files in %s inventory", async (extension, { cli }) => { const root = await createTestTempDirectory("rea-class-inventory-"); - const path = join(root, `mixed.${extension}`); // The large class can hold the entire 65-entry table suggested by its // version bytes. Checking only the table length does not distinguish it. const classes = [ - ["Hello.class", javaClass()], - ["Big.class", javaClass("x".repeat(4_000))], - ["renamed.bin", javaClass()], + ["Hello.class", javaClassFile()], + ["pkg/Big.class", javaClassFile(4_000)], + ["renamed.bin", javaClassFile()], ] as const; - const writer = new ZipWriter(new Uint8ArrayWriter()); - for (const [name, bytes] of classes) - await writer.add(name, new Uint8ArrayReader(bytes)); - const nativePaths: string[] = []; + const nativeEntries: [string, Uint8Array][] = []; for (const wide of [false, true]) { for (const littleEndian of [false, true]) { - const name = `fat${wide ? 64 : 32}-${littleEndian ? "le" : "be"}`; - nativePaths.push(name); - await writer.add( - name, - new Uint8ArrayReader( - fatImage( - [{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }], - wide, - littleEndian, - ), + nativeEntries.push([ + `fat${wide ? 64 : 32}-${littleEndian ? "le" : "be"}`, + fatImage( + [{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }], + wide, + littleEndian, ), - ); + ]); } } - await writeFile(path, await writer.close()); - const reply = await cli.run({ - arguments: ["inspect-artifact", path, "--json"], - }); - expect(reply.exitCode, reply.stdout + reply.stderr).toBe(0); - const inspection = artifactInspectionResultSchema.parse( - parseEvidence(reply.json).normalized_result, - ); - const inventory = artifactInventoryResultSchema.parse( - inspection.substeps[0]?.evidence.normalized_result, - ); - for (const name of nativePaths) { - expect(inventory.occurrences).toContainEqual( - expect.objectContaining({ - logical_path: name, - artifact_kind: "executable", - artifact_format: "mach-o-universal", - limitations: [], - }), + // Exercise a JAR without FAT controls as well as mixed JAR and ZIP files. + const scenarios = extension === "jar" ? [false, true] : [true]; + for (const includeNative of scenarios) { + const path = join( + root, + `${includeNative ? "mixed" : "classes"}.${extension}`, ); - } - for (const [name, bytes] of classes) { - const occurrence = inventory.occurrences.find( - (item) => item.logical_path === name, + const selectedClasses = includeNative ? classes : classes.slice(0, 2); + const writer = new ZipWriter(new Uint8ArrayWriter()); + if (!includeNative) + await writer.add( + "META-INF/MANIFEST.MF", + new Uint8ArrayReader(Buffer.from("Manifest-Version: 1.0\n\n")), + ); + for (const [name, bytes] of [ + ...selectedClasses, + ...(includeNative ? nativeEntries : []), + ]) + await writer.add(name, new Uint8ArrayReader(bytes)); + await writeFile(path, await writer.close()); + + const reply = await cli.run({ + arguments: ["inspect-artifact", path, "--json"], + }); + expect(reply.exitCode, reply.stdout + reply.stderr).toBe(0); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(reply.json).normalized_result, ); - expect(occurrence).toBeDefined(); - expect.soft(occurrence?.artifact_kind, name).not.toBe("executable"); - expect.soft(occurrence?.artifact_format, name).not.toMatch(/^mach-o/u); - const sha256 = createHash("sha256").update(bytes).digest("hex"); - const node = inventory.nodes.find((item) => item.sha256 === sha256); - expect(node).toBeDefined(); - expect.soft(node?.format, name).not.toMatch(/^mach-o/u); + const inventory = artifactInventoryResultSchema.parse( + inspection.substeps[0]?.evidence.normalized_result, + ); + expect( + inventory.occurrences + .filter(({ logical_path }) => logical_path.endsWith(".class")) + .map(({ logical_path }) => logical_path), + ).toEqual(["Hello.class", "pkg/Big.class"]); + if (includeNative) { + for (const [name] of nativeEntries) { + expect(inventory.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: name, + artifact_kind: "executable", + artifact_format: "mach-o-universal", + limitations: [], + }), + ); + } + } + for (const [name, bytes] of selectedClasses) { + const occurrence = inventory.occurrences.find( + (item) => item.logical_path === name, + ); + expect(occurrence, name).toMatchObject({ + artifact_kind: "resource", + artifact_format: "file", + }); + const node = inventory.nodes.find( + (item) => item.artifact_id === occurrence?.artifact_id, + ); + expect(node, name).toMatchObject({ + format: "file", + sha256: createHash("sha256").update(bytes).digest("hex"), + }); + } } }, ); - -/** Minimal valid Java 21 class, with an optional unused UTF-8 constant. */ -const javaClass = (extra?: string): Buffer => { - const utf8 = (value: string): Buffer => { - const text = Buffer.from(value); - const header = Buffer.alloc(3); - header[0] = 1; - header.writeUInt16BE(text.length, 1); - return Buffer.concat([header, text]); - }; - const constants = [ - Buffer.from([7, 0, 2]), - utf8("Hello"), - Buffer.from([7, 0, 4]), - utf8("java/lang/Object"), - ...(extra === undefined ? [] : [utf8(extra)]), - ]; - const header = Buffer.from("cafebabe000000410000", "hex"); - header.writeUInt16BE(constants.length + 1, 8); - return Buffer.concat([ - header, - ...constants, - Buffer.from("0021000100030000000000000000", "hex"), - ]); -}; diff --git a/tests/acceptance/artifactComparisonAsarUnpacked.test.ts b/tests/acceptance/artifactComparisonAsarUnpacked.test.ts index de6430732b..476632caa9 100644 --- a/tests/acceptance/artifactComparisonAsarUnpacked.test.ts +++ b/tests/acceptance/artifactComparisonAsarUnpacked.test.ts @@ -172,12 +172,16 @@ it("compares ASAR companion declarations from public MCP inventories and preserv await writer.add("member.bin", new TextReader("identical child bytes")); await writeFile(zipPath, await writer.close()); const zip = await inspect(zipPath); + const zipOccurrences = artifactInventoryResultSchema.parse( + zip.normalized_result, + ).occurrences; + expect(zipOccurrences).toContainEqual( + expect.objectContaining({ logical_path: "member.bin" }), + ); expect( - artifactInventoryResultSchema - .parse(zip.normalized_result) - .occurrences.every( - (occurrence) => !Object.hasOwn(occurrence, "unpacked"), - ), + zipOccurrences.every( + (occurrence) => !Object.hasOwn(occurrence, "unpacked"), + ), ).toBe(true); const unchanged = await compare(zip, zip); expect(unchanged).toMatchObject({ diff --git a/tests/acceptance/artifactInventoryJvmClass.test.ts b/tests/acceptance/artifactInventoryJvmClass.test.ts deleted file mode 100644 index 51f3e01ebf..0000000000 --- a/tests/acceptance/artifactInventoryJvmClass.test.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { execFile } from "node:child_process"; -import { writeFile } from "node:fs/promises"; -import { join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; - -import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; -import { expect, it } from "vitest"; - -import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; -import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; -import { parseEvidence } from "../../src/domain/evidence.js"; -import { - fatImage, - CPU, - machoImage, -} from "../../src/artifacts/apple/MachoImage.fixture.js"; -import { javaClassFile } from "../fixtures/javaClassFile.js"; -import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; - -const execute = promisify(execFile); -const cli = fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)); - -const archive = async ( - entries: readonly (readonly [string, Uint8Array])[], -): Promise => { - const writer = new ZipWriter(new Uint8ArrayWriter()); - for (const [name, bytes] of entries) - await writer.add(name, new Uint8ArrayReader(bytes)); - return Buffer.from(await writer.close()); -}; - -const inspectArchive = async (path: string) => { - const result = await execute(process.execPath, [ - cli, - "inspect-artifact", - path, - "--json", - ]); - const inspection = artifactInspectionResultSchema.parse( - parseEvidence(JSON.parse(result.stdout)).normalized_result, - ); - const evidence = inspection.substeps[0]?.evidence; - if (evidence === undefined) - throw new Error("CLI inspection did not retain inventory Evidence"); - return artifactInventoryResultSchema.parse(evidence.normalized_result); -}; - -it("keeps valid JVM classes from being classified as Mach-O while identifying FAT controls", async () => { - const directory = await createTestTempDirectory("rea-jvm-inventory-"); - const javaEntries = [ - ["Hello.class", javaClassFile()], - ["pkg/Big.class", javaClassFile(4000)], - ] as const; - const fatControls = [ - [ - "fat-be32.bin", - fatImage([{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }]), - ], - [ - "fat-le32.bin", - fatImage( - [{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }], - false, - true, - ), - ], - [ - "fat-be64.bin", - fatImage( - [{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }], - true, - ), - ], - [ - "fat-le64.bin", - fatImage( - [{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }], - true, - true, - ), - ], - ] as const; - - const jarPath = join(directory, "classes.jar"); - await writeFile( - jarPath, - await archive([ - ["META-INF/MANIFEST.MF", Buffer.from("Manifest-Version: 1.0\n\n")], - ...javaEntries, - ]), - ); - const jar = await inspectArchive(jarPath); - expect( - jar.occurrences - .filter(({ logical_path }) => logical_path.endsWith(".class")) - .map(({ logical_path, artifact_kind, artifact_format }) => ({ - logical_path, - artifact_kind, - artifact_format, - })), - ).toEqual([ - { - logical_path: "Hello.class", - artifact_kind: "resource", - artifact_format: "file", - }, - { - logical_path: "pkg/Big.class", - artifact_kind: "resource", - artifact_format: "file", - }, - ]); - const jarClassIds = new Set( - jar.occurrences - .filter(({ logical_path }) => logical_path.endsWith(".class")) - .map(({ artifact_id }) => artifact_id), - ); - expect( - jar.nodes - .filter(({ artifact_id }) => jarClassIds.has(artifact_id)) - .map(({ format }) => format) - .sort(), - ).toEqual(["file", "file"]); - - const mixedPath = join(directory, "mixed.zip"); - await writeFile(mixedPath, await archive([...javaEntries, ...fatControls])); - const mixed = await inspectArchive(mixedPath); - const formatsByPath = new Map( - mixed.occurrences.map( - ({ logical_path, artifact_kind, artifact_format }) => [ - logical_path, - { artifact_kind, artifact_format }, - ], - ), - ); - for (const [name] of javaEntries) - expect(formatsByPath.get(name)).toEqual({ - artifact_kind: "resource", - artifact_format: "file", - }); - for (const [name] of fatControls) - expect(formatsByPath.get(name)).toEqual({ - artifact_kind: "executable", - artifact_format: "mach-o-universal", - }); -}); diff --git a/tests/acceptance/artifactZipExtractionPaths.test.ts b/tests/acceptance/artifactZipExtractionPaths.test.ts index e40d653f2f..011b651c93 100644 --- a/tests/acceptance/artifactZipExtractionPaths.test.ts +++ b/tests/acceptance/artifactZipExtractionPaths.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -56,8 +57,14 @@ cliTest( collisions.map(({ artifact_id }) => artifact_id === null ? null : nodesById.get(artifact_id), ), - ).size, - ).toBe(2); + ), + ).toEqual( + new Set( + ["NFC bytes", "NFD bytes"].map((bytes) => + createHash("sha256").update(bytes).digest("hex"), + ), + ), + ); for (const integrityPolicy of ["fail", "record-and-continue"] as const) { const result = await cli.run({ diff --git a/tests/acceptance/artifactZipLeadingBytes.test.ts b/tests/acceptance/artifactZipLeadingBytes.test.ts index 4e3add055e..65783e1c57 100644 --- a/tests/acceptance/artifactZipLeadingBytes.test.ts +++ b/tests/acceptance/artifactZipLeadingBytes.test.ts @@ -67,6 +67,9 @@ cliTest( expect( await readFile(join(extraction.output_root, "a.txt"), "utf8"), ).toBe("hello world\n"); + expect( + await readFile(join(extraction.output_root, "d/b.txt"), "utf8"), + ).toBe("x\n"); } finally { await rm(extraction.output_root, { force: true, recursive: true }); } diff --git a/tests/acceptance/cliTimings.test.ts b/tests/acceptance/cliTimings.test.ts index ac7cbd9ef1..822a8d5c17 100644 --- a/tests/acceptance/cliTimings.test.ts +++ b/tests/acceptance/cliTimings.test.ts @@ -41,13 +41,19 @@ cliTest( expect(successful.json).toMatchObject({ normalized_result: { coverage: { status: "complete" } }, }); + expect(successful.stdout).toBe(defaultRun.stdout); timingRecord(successful.stderr); + const defaultFailure = await cli.run({ + arguments: ["inspect-elf-notes", invalid, "--json"], + }); + expect(defaultFailure.exitCode).toBe(1); const failed = await cli.run({ arguments: ["inspect-elf-notes", invalid, "--json", "--timings"], }); expect(failed.exitCode).toBe(1); expect(failed.json).toMatchObject({ code: "invalid_request" }); + expect(failed.stdout).toBe(defaultFailure.stdout); timingRecord(failed.stderr); }, ); diff --git a/tests/acceptance/derivedEvidenceReplay.test.ts b/tests/acceptance/derivedEvidenceReplay.test.ts index 332bb040de..36c71c79cf 100644 --- a/tests/acceptance/derivedEvidenceReplay.test.ts +++ b/tests/acceptance/derivedEvidenceReplay.test.ts @@ -149,6 +149,11 @@ it.each([ right, }); expect(invalid.isError).toBe(true); + expect(parseMcpToolError(invalid).error).toMatchObject({ + code: "evidence_integrity_mismatch", + category: "integrity_mismatch", + retryable: false, + }); const afterReplay = await restored.call("get_evidence_bundle", {}); expect( parseEvidenceBundle( diff --git a/tests/acceptance/devicetreeCli.test.ts b/tests/acceptance/devicetreeCli.test.ts index b82039b3ce..5f19d7fc44 100644 --- a/tests/acceptance/devicetreeCli.test.ts +++ b/tests/acceptance/devicetreeCli.test.ts @@ -66,7 +66,15 @@ cliTest( result.json, ).normalized_result; if (!("references" in report)) throw new Error("Expected complete report"); - expect(report.references).toMatchObject([{ phandle: 3, target: null }]); + expect(report.references).toMatchObject([ + { source: "/consumer", phandle: 3, target: null }, + ]); + expect(report.nodes[1]?.properties[0]).toMatchObject({ + name: "phandle", + bytes: 0, + hex: "", + derived: { cells: null }, + }); expect(report.limitations).toContain( "phandle at /target is not a single cell; this node's reference identity remains unresolved.", ); @@ -99,7 +107,28 @@ cliTest( kind: "summary", summary: { unresolved_reference_count: 1 }, }); - expect(await readFile(output, "utf8")).toContain('"nodes"'); + const bundle = z + .object({ records: z.array(z.unknown()).length(1) }) + .parse(JSON.parse(await readFile(output, "utf8"))); + const retained = toolContract("inspect_devicetree").outputSchema.parse( + bundle.records[0], + ).normalized_result; + if (!("nodes" in retained)) throw new Error("Expected complete evidence"); + expect(retained.nodes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: "/serial@1000", + properties: expect.arrayContaining([ + expect.objectContaining({ + name: "reg", + derived: expect.objectContaining({ + reg: [{ address: "0x1000", size: "0x100" }], + }), + }), + ]), + }), + ]), + ); }, ); for (const [name, build, flags, code] of [ diff --git a/tests/acceptance/dotnetBundleJourney.test.ts b/tests/acceptance/dotnetBundleJourney.test.ts index 85d7768c72..ffc59c6268 100644 --- a/tests/acceptance/dotnetBundleJourney.test.ts +++ b/tests/acceptance/dotnetBundleJourney.test.ts @@ -122,6 +122,8 @@ for (const format of ["dotnet-bundle", "pyinstaller"] as const) { XDG_CACHE_HOME: workspace.path("cache"), XDG_STATE_HOME: workspace.path("state"), REA_LOG_LEVEL: "silent", + HOPPER_LAUNCHER_PATH: workspace.path("unavailable-hopper"), + GHIDRA_INSTALL_DIR: workspace.path("unavailable-ghidra"), }; const outputs: string[] = []; const client = new Client({ diff --git a/tests/acceptance/electronFusesCli.test.ts b/tests/acceptance/electronFusesCli.test.ts index a43b39b539..893bf5c2fb 100644 --- a/tests/acceptance/electronFusesCli.test.ts +++ b/tests/acceptance/electronFusesCli.test.ts @@ -285,6 +285,9 @@ cliTest( expect(evidence.limitations.join(" ")).toContain( "unknown, not that every fuse is disabled", ); + expect(evidence.limitations.join(" ")).toContain( + "fuse state for that slice is unknown, not disabled", + ); }, ); diff --git a/tests/acceptance/erlangBeamCli.test.ts b/tests/acceptance/erlangBeamCli.test.ts index 36cfdd4676..a2286f6dbe 100644 --- a/tests/acceptance/erlangBeamCli.test.ts +++ b/tests/acceptance/erlangBeamCli.test.ts @@ -65,22 +65,67 @@ const extendedFirstAtomLength = (): Buffer => { return extended; }; -cliTest("reports BEAM exports through the public CLI", async ({ cli }) => { - const root = await createTestTempDirectory("rea-beam-cli-"); - const path = join(root, "app.beam"); - await writeFile(path, erlangBeamFixture("cli_mod")); - const result = await cli.run({ - arguments: ["inspect-erlang-beam", path, "--json"], - }); - expect(result.exitCode, result.stderr).toBe(0); - const evidence = completeBeamEvidenceSchema.parse(result.json); - expect(evidence.normalized_result.module).toBe("cli_mod"); - expect( - evidence.normalized_result.exports?.some( - (entry) => entry.function === "start" && entry.arity === 0, - ), - ).toBe(true); -}); +cliTest( + "reports BEAM tables and code through the public CLI", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-beam-cli-"); + const path = join(root, "app.beam"); + await writeFile(path, erlangBeamFixture("cli_mod")); + const result = await cli.run({ + arguments: ["inspect-erlang-beam", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const evidence = completeBeamEvidenceSchema.parse(result.json); + expect(evidence).toMatchObject({ + confidence: "observed", + subject: { local_path: path, format: "file" }, + }); + expect(evidence.normalized_result).toMatchObject({ + atom_encoding: "utf8", + module: "cli_mod", + atoms: [ + { name: "cli_mod" }, + { name: "module_info" }, + { name: "start" }, + { name: "erlang" }, + { name: "get_module_info" }, + ], + lambdas: [ + expect.objectContaining({ + function: "start", + arity: 1, + label: 4, + old_unique: 42, + }), + ], + code: { + instruction_set: 0, + opcode_max: 169, + labels: 4, + functions: 2, + extra_header_bytes: 0, + instructions: { bytes: 4 }, + }, + }); + expect(evidence.normalized_result.exports).toEqual([ + { function_index: 3, function: "start", arity: 0, label: 2 }, + { function_index: 2, function: "module_info", arity: 0, label: 1 }, + { function_index: 2, function: "module_info", arity: 1, label: 3 }, + ]); + expect(evidence.normalized_result.imports).toEqual([ + { + module_index: 4, + module: "erlang", + function_index: 5, + function: "get_module_info", + arity: 1, + }, + ]); + expect( + evidence.normalized_result.chunks.map((chunk) => chunk.name), + ).toEqual(["AtU8", "Code", "ExpT", "ImpT", "FunT"]); + }, +); cliTest( "inspects empty optional BEAM chunks through CLI and MCP without losing source evidence", @@ -191,8 +236,13 @@ cliTest( }); const atomChunks = report.chunks.filter(({ name }) => name === "AtU8"); expect(atomChunks).toHaveLength(2); - expect(atomChunks[0]?.payload.sha256).not.toBe( - atomChunks[1]?.payload.sha256, + expect(atomChunks[0]?.payload.sha256).toBe( + createHash("sha256").update(decoy.subarray(8, 14)).digest("hex"), + ); + expect(atomChunks[1]?.payload.sha256).toBe( + createHash("sha256") + .update(original.subarray(20, 20 + original.readUInt32BE(16))) + .digest("hex"), ); expect(atomChunks[0]?.payload.location.offset).toBe(20); expect(atomChunks[1]?.payload.location.offset).toBe(20 + decoy.length); @@ -220,6 +270,41 @@ cliTest( }; const control = await inspect(controlPath); const extended = await inspect(extendedPath); + // Golden facts from beam_lib:chunks/2 on the compiler-produced OTP 29 file. + expect(control.normalized_result.module).toBe("rea_beam_fixture"); + expect(control.normalized_result.atoms?.map((atom) => atom.name)).toEqual([ + "rea_beam_fixture", + "hello", + "lists", + "reverse", + "module_info", + "erlang", + "get_module_info", + ]); + expect(control.normalized_result.exports).toEqual([ + { function_index: 5, function: "module_info", arity: 1, label: 6 }, + { function_index: 5, function: "module_info", arity: 0, label: 4 }, + { function_index: 2, function: "hello", arity: 1, label: 2 }, + ]); + expect( + control.normalized_result.imports?.map((row) => [ + row.module, + row.function, + row.arity, + ]), + ).toEqual([ + ["lists", "reverse", 1], + ["erlang", "get_module_info", 1], + ["erlang", "get_module_info", 2], + ]); + expect(control.normalized_result.code).toMatchObject({ + instruction_set: 0, + opcode_max: 181, + labels: 7, + functions: 3, + extra_header_bytes: 0, + instructions: { bytes: 49 }, + }); expect(extended.normalized_result.module).toBe("rea_beam_fixture"); expect(extended.normalized_result.atoms).toEqual( control.normalized_result.atoms, @@ -236,9 +321,16 @@ cliTest( expect(extended.normalized_result.artifact.sha256).not.toBe( control.normalized_result.artifact.sha256, ); - expect(extended.normalized_result.chunks[0]?.payload.sha256).not.toBe( - control.normalized_result.chunks[0]?.payload.sha256, - ); + for (const [evidence, bytes] of [ + [control, controlBytes], + [extended, extendedBytes], + ] as const) { + expect(evidence.normalized_result.chunks[0]?.payload.sha256).toBe( + createHash("sha256") + .update(bytes.subarray(20, 20 + bytes.readUInt32BE(16))) + .digest("hex"), + ); + } }, ); diff --git a/tests/acceptance/espImageCli.test.ts b/tests/acceptance/espImageCli.test.ts index 21a330a480..a24c13372d 100644 --- a/tests/acceptance/espImageCli.test.ts +++ b/tests/acceptance/espImageCli.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, onTestFinished } from "vitest"; @@ -7,6 +8,7 @@ import { ESP32C3_APP_SEGMENTS, ESP32C3_BOOTLOADER_SEGMENTS, esp8266Image, + espChecksum, espImage, } from "../fixtures/espImage.js"; import { ESP8266_REFUSAL } from "../../src/domain/firmware/espImage.js"; @@ -15,7 +17,91 @@ import { artifactInspectionResultSchema } from "../../src/domain/artifactInspect import { artifactExtractionResultSchema, artifactInventoryResultSchema, + type ArtifactInventoryResult, } from "../../src/domain/artifactGraph.js"; +import { ESP_IMAGE_LIMITATIONS } from "../../src/domain/firmware/espImageFacts.js"; + +const assertAppFacts = (graph: ArtifactInventoryResult, bytes: Buffer) => { + const digest = createHash("sha256") + .update(bytes.subarray(0, 464)) + .digest("hex"); + expect( + graph.occurrences.find((row) => row.logical_path === ".")?.format_metadata, + ).toEqual({ + kind: "esp-image", + header: { + segment_count: 4, + spi_mode: 2, + spi_speed: 0, + spi_size: 2, + entry_addr: 0x4038_0000, + wp_pin: 0xee, + spi_pin_drv: "000000", + chip_id: 5, + min_chip_rev: 0, + min_chip_rev_full: 0, + max_chip_rev_full: 199, + reserved: "00000000", + hash_appended: true, + }, + chip: { target: "esp32c3", isa: "riscv", source: "ESP-IDF v6.1" }, + checksum: { + offset: 463, + stored: espChecksum(ESP32C3_APP_SEGMENTS), + computed: espChecksum(ESP32C3_APP_SEGMENTS), + status: "valid", + }, + appended_sha256: { + offset: 464, + stored: digest, + computed: digest, + status: "valid", + }, + image_end: 496, + signature_sector_offset: null, + flash_dump: null, + undescribed_trailing: [], + descriptor: { + kind: "app", + offset: 32, + observed_length: 256, + secure_version: 0, + version: "1.2.3", + project_name: "rea_esp_fixture", + time: "12:34:56", + date: "Oct 27 2026", + idf_ver: "v6.1", + app_elf_sha256: "ab".repeat(32), + min_efuse_blk_rev_full: 0, + max_efuse_blk_rev_full: 199, + mmu_page_size: 16, + spi_flash_mode: 2, + }, + }); + const segments = graph.occurrences.filter((row) => row.logical_path !== "."); + expect( + segments.map((row) => { + const meta = row.format_metadata; + if (meta?.kind !== "esp-image-segment") + throw new Error("Expected ESP segment metadata"); + return [ + meta.index, + meta.load_addr, + meta.data_len, + meta.header_offset, + row.source_location?.offset, + row.source_location?.length, + meta.classification, + ]; + }), + ).toEqual([ + [0, 0x3c00_0020, 288, 24, 32, 288, "mapped"], + [1, 0x3fc8_8000, 16, 320, 328, 16, "loaded"], + [2, 0x4038_0000, 32, 344, 352, 32, "loaded"], + [3, 0x4200_0020, 64, 384, 392, 64, "mapped"], + ]); + expect(graph.limitations).toEqual([ESP_IMAGE_LIMITATIONS.inventoryScope]); +}; const cases = [ { @@ -53,6 +139,7 @@ cliTest.for(cases)( ); expect(graph.manifest.root_format).toBe("esp-image"); expect(graph.coverage.status).toBe("complete"); + if (role === "app") assertAppFacts(graph, bytes); expect( graph.occurrences.find((row) => row.logical_path === ".") ?.format_metadata, diff --git a/tests/acceptance/firmwareContainersCli.test.ts b/tests/acceptance/firmwareContainersCli.test.ts index eca7b2cf4a..fbdc8d3cc9 100644 --- a/tests/acceptance/firmwareContainersCli.test.ts +++ b/tests/acceptance/firmwareContainersCli.test.ts @@ -202,6 +202,9 @@ cliTest( arguments: ["inspect-artifact", source, "--json"], }); expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + details: { operation: "inspect_artifact", reason: "format" }, + }); expect(result.stdout + result.stderr).toContain( "UF2 is malformed: family 0xe48bff56 (RP2040) address 0x10000000 is written with different values by blocks 0-1 (0x01) and block 2 (0xff)", ); diff --git a/tests/acceptance/fitImageCli.test.ts b/tests/acceptance/fitImageCli.test.ts index 1b7fd9957e..9bb4f8bb5a 100644 --- a/tests/acceptance/fitImageCli.test.ts +++ b/tests/acceptance/fitImageCli.test.ts @@ -202,7 +202,10 @@ cliTest.for([true, false])( (row) => row.logical_path === "images/linked.bin", ); expect(linked?.format_metadata).toMatchObject({ image_data: "target" }); + expect(target?.artifact_id).toEqual(expect.any(String)); expect(linked?.artifact_id).toBe(target?.artifact_id); + expect(linked?.source_location?.offset).toEqual(expect.any(Number)); + expect(target?.source_location?.offset).toEqual(expect.any(Number)); expect(linked?.source_location?.offset).not.toBe( target?.source_location?.offset, ); diff --git a/tests/acceptance/flutterAnalysis.test.ts b/tests/acceptance/flutterAnalysis.test.ts index f1a8177b5d..1c8ea71347 100644 --- a/tests/acceptance/flutterAnalysis.test.ts +++ b/tests/acceptance/flutterAnalysis.test.ts @@ -157,6 +157,10 @@ cliTest( "identify_flutter_build", ).outputSchema.parse(result.json).normalized_result; expect(identified.coverage).toBe("complete"); + expect(identified.abis.map(({ abi }) => abi).sort()).toEqual([ + "arm64-v8a", + "x86_64", + ]); for (const abi of identified.abis) { expect(abi.libflutter).toMatchObject({ build_id: "0123456789abcdef", diff --git a/tests/acceptance/hexRecordsCli.test.ts b/tests/acceptance/hexRecordsCli.test.ts index 7acba11a72..f085181eba 100644 --- a/tests/acceptance/hexRecordsCli.test.ts +++ b/tests/acceptance/hexRecordsCli.test.ts @@ -68,6 +68,54 @@ cliTest.for(cases)( .filter((row) => row.logical_path !== ".") .map((row) => row.logical_path), ).toEqual(Object.keys(regions)); + if (encoding === "intel-hex") { + expect(graph.coverage.scope).toBe("container-entries"); + expect(graph.limitations).toContain( + "Intel HEX/S-record inventory decodes data records into address-contiguous regions. It does not establish CPU, byte order, memory map, or that a declared start address is executable; gaps are unwritten, not zero.", + ); + expect( + graph.occurrences.find((row) => row.logical_path === "."), + ).toMatchObject({ + artifact_kind: "container", + format_metadata: { region_count: 2 }, + }); + expect( + graph.occurrences + .filter((row) => row.logical_path !== ".") + .map((row) => [ + row.logical_path, + row.declared_size, + row.format_metadata, + ]), + ).toEqual([ + [ + "0x08000000-0x0800001f.bin", + 32, + { + kind: "hex-record-region", + start_address: 0x0800_0000, + end_address: 0x0800_001f, + length: 32, + record_count: 2, + first_line: 2, + last_line: 3, + }, + ], + [ + "0x08000100-0x08000103.bin", + 4, + { + kind: "hex-record-region", + start_address: 0x0800_0100, + end_address: 0x0800_0103, + length: 4, + record_count: 1, + first_line: 4, + last_line: 4, + }, + ], + ]); + } expect(inspection.next_probes.map((probe) => probe.operation)).toEqual([ "open_binary", ]); @@ -98,6 +146,10 @@ cliTest("rejects a malformed record through the CLI", async ({ cli }) => { arguments: ["inspect-artifact", source, "--json"], }); expect(result.exitCode).not.toBe(0); + expect(result.json).toMatchObject({ + code: "artifact_operation_failed", + details: { operation: "inspect_artifact", reason: "format" }, + }); expect(result.stdout + result.stderr).toContain( "Intel HEX is malformed: line 4: record type 00 checksum 0xc4 does not match computed 0xc3", ); diff --git a/tests/acceptance/il2cppMetadataCli.test.ts b/tests/acceptance/il2cppMetadataCli.test.ts index 1d7b847c2e..0327902d1f 100644 --- a/tests/acceptance/il2cppMetadataCli.test.ts +++ b/tests/acceptance/il2cppMetadataCli.test.ts @@ -751,7 +751,9 @@ cliTest( const evidence = saved.value.records[0]; if (evidence === undefined) throw new Error("Missing exported source Evidence"); - expect(JSON.stringify(overwrite.json)).toContain(evidence.evidence_id); + expect(overwrite.json).toMatchObject({ + details: { partial_observation: { evidence_id: evidence.evidence_id } }, + }); expect( il2cppMetadataSchema.parse(evidence.normalized_result).methods, ).toEqual(FIXTURE_METHODS); diff --git a/tests/acceptance/inesContainer.test.ts b/tests/acceptance/inesContainer.test.ts index fceccf9b74..589d4d63cc 100644 --- a/tests/acceptance/inesContainer.test.ts +++ b/tests/acceptance/inesContainer.test.ts @@ -129,6 +129,7 @@ cliTest( const digest = createHash("sha256").update(bytes).digest("hex"); expect(inventory.ines).toMatchObject({ header: { + header_hex: bytes.subarray(0, 16).toString("hex"), mapper: 19, trainer_present: true, prg_rom_units_16k: 2, @@ -162,7 +163,14 @@ cliTest( .update(bytes.subarray(offset, offset + length)) .digest("hex"), ); + expect(nodes.get(occurrence.artifact_id)?.size).toBe(length); + if (logical_path === "header.bin") { + expect(nodes.get(occurrence.artifact_id)?.format).toBe("file"); + } } + expect(inventory.ines?.reconstruction.region_order).toEqual( + expected.map(([logicalPath]) => logicalPath), + ); const { call } = await connectLocalToolsMcp(); const opened = await call("open_binary", { path }); diff --git a/tests/acceptance/javascriptRouteBudgetRecovery.test.ts b/tests/acceptance/javascriptRouteBudgetRecovery.test.ts index 5ac467c568..6d7fe16845 100644 --- a/tests/acceptance/javascriptRouteBudgetRecovery.test.ts +++ b/tests/acceptance/javascriptRouteBudgetRecovery.test.ts @@ -93,6 +93,16 @@ cliTest( ], }); expect(narrow.exitCode, narrow.stderr).toBe(0); + expect(narrow.json).toMatchObject({ + normalized_result: { + items: [ + expect.objectContaining({ + classification: "unknown", + confidence: "low", + }), + ], + }, + }); expect(narrow.json).toMatchObject({ normalized_result: { items: [expect.not.objectContaining({ path: expect.anything() })], diff --git a/tests/acceptance/javascriptRoutesCli.test.ts b/tests/acceptance/javascriptRoutesCli.test.ts index 2a2bc8d3ec..982d06a2b0 100644 --- a/tests/acceptance/javascriptRoutesCli.test.ts +++ b/tests/acceptance/javascriptRoutesCli.test.ts @@ -13,6 +13,16 @@ const routeViewFromEvidence = (evidence: unknown) => parseEvidence(evidence).normalized_result, ); +const expectAccountRoutes = (items: unknown[]) => { + expect(items).toHaveLength(2); + expect(items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: "/account" }), + expect.objectContaining({ path: "/account/:id" }), + ]), + ); +}; + const interceptedSource = "export default function Photo(){return null;}"; const independentPaths = [ "/checks/independent", @@ -74,7 +84,10 @@ cliTest( evidence_bundle: { path: string }; }; const view = routeViewFromEvidence(envelope.summary); - expect(view.items).toHaveLength(2); + expect(view.items).toEqual([ + expect.objectContaining({ path: "/reports" }), + expect.objectContaining({ path: "/reports/aliased" }), + ]); expect(view.coverage).toMatchObject({ total: 6, next_offset: 2 }); expect(view.sources[0]!.sha256).toMatch(/^[a-f0-9]{64}$/u); await rm(path, { recursive: true }); @@ -110,7 +123,13 @@ cliTest( ], }); expect(repeated.exitCode, repeated.stdout + repeated.stderr).toBe(0); - expect(routeViewFromEvidence(repeated.json).items).toHaveLength(2); + const nextPage = routeViewFromEvidence(repeated.json); + expect(nextPage.parent_evidence_id).toBe(view.parent_evidence_id); + expect(nextPage.coverage).toMatchObject({ total: 6, next_offset: 4 }); + expect(nextPage.items).toEqual([ + expect.objectContaining({ path: "/reports/help" }), + expect.objectContaining({ path: "/reports/history" }), + ]); }, ); cliTest( @@ -182,7 +201,7 @@ it.each(["full", "compact"] as const)( }); expect(response.isError).not.toBe(true); const selected = routeViewFromEvidence(response.structuredContent); - expect(selected.items).toHaveLength(2); + expectAccountRoutes(selected.items); await rm(path, { recursive: true }); const occurrences = await call("inspect_analysis_view", { source: { diff --git a/tests/acceptance/localToolsMcpProviderIsolation.test.ts b/tests/acceptance/localToolsMcpProviderIsolation.test.ts new file mode 100644 index 0000000000..13316e0398 --- /dev/null +++ b/tests/acceptance/localToolsMcpProviderIsolation.test.ts @@ -0,0 +1,59 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { delimiter, join } from "node:path"; + +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; +import { parseMcpToolError } from "../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +it("isolates Ghidra PATH discovery while honoring an explicit installation override", async () => { + const root = await createTestTempDirectory("rea-local-tools-provider-"); + await mkdir(join(root, "Ghidra"), { recursive: true }); + await mkdir(join(root, "support"), { recursive: true }); + await writeFile( + join(root, "Ghidra", "application.properties"), + "application.version=12.1.4\napplication.java.min=21\n", + ); + await writeFile( + join(root, process.platform === "win32" ? "ghidraRun.bat" : "ghidraRun"), + "", + { mode: 0o700 }, + ); + await writeFile( + join( + root, + "support", + process.platform === "win32" ? "analyzeHeadless.bat" : "analyzeHeadless", + ), + "", + { mode: 0o700 }, + ); + const path = [root, process.env.PATH ?? ""].join(delimiter); + const probe = async (environment: Readonly>) => { + const { call } = await connectLocalToolsMcp({ PATH: path, ...environment }); + const response = await call("open_binary", { + path: process.execPath, + provider_id: "ghidra", + }); + expect(response.isError).toBe(true); + const error = parseMcpToolError(response); + const details = z + .object({ + rejections: z.array( + z.object({ provider_id: z.string(), diagnostics: z.unknown() }), + ), + }) + .parse(error.error.details); + const rejection = details.rejections.find( + (item) => item.provider_id === "ghidra", + ); + if (rejection === undefined) throw new Error("Missing Ghidra rejection"); + return z.object({ install_dir: z.string() }).parse(rejection.diagnostics) + .install_dir; + }; + + expect(await probe({})).toBe(process.execPath); + expect(await probe({ GHIDRA_INSTALL_DIR: root })).toBe(root); +}); diff --git a/tests/acceptance/managedMemberPointers.test.ts b/tests/acceptance/managedMemberPointers.test.ts index 53fad72573..1d395ba482 100644 --- a/tests/acceptance/managedMemberPointers.test.ts +++ b/tests/acceptance/managedMemberPointers.test.ts @@ -51,8 +51,10 @@ cliTest( }); if (name === "inspect_managed_artifact") { expect(mcpEvidence.normalized_result).toMatchObject({ + metadata: { stream_names: expect.arrayContaining(["#-"]) }, attributes: [ expect.objectContaining({ + parent_token: "0x20000001", constructor_token: "0x06000001", type_name: "Fixture.B", }), @@ -60,24 +62,33 @@ cliTest( }); } else { expect(mcpEvidence.normalized_result).toMatchObject({ + types: [ + expect.objectContaining({ full_name: "" }), + expect.objectContaining({ full_name: "Fixture.A" }), + expect.objectContaining({ full_name: "Fixture.B" }), + ], fields: [ expect.objectContaining({ token: "0x04000001", declaring_type: "Fixture.B", + declaring_type_token: "0x02000003", }), expect.objectContaining({ token: "0x04000002", declaring_type: "Fixture.A", + declaring_type_token: "0x02000002", }), ], methods: [ expect.objectContaining({ token: "0x06000001", declaring_type: "Fixture.B", + declaring_type_token: "0x02000003", }), expect.objectContaining({ token: "0x06000002", declaring_type: "Fixture.A", + declaring_type_token: "0x02000002", }), ], }); diff --git a/tests/acceptance/managedMemberRecovery.test.ts b/tests/acceptance/managedMemberRecovery.test.ts index ab9e3db2ca..88d31f562d 100644 --- a/tests/acceptance/managedMemberRecovery.test.ts +++ b/tests/acceptance/managedMemberRecovery.test.ts @@ -86,28 +86,34 @@ cliTest( expect(report.types).toHaveLength(3); expect(report.methods).toEqual([]); const empty = report.types.find((type) => type.method_list.count === 0); - expect(empty).toBeDefined(); + if (empty === undefined) throw new Error("Missing empty fixture type"); const selectedCli = await cli.run({ arguments: [ "inspect-managed-members", path, "--type", - empty!.full_name, + empty.full_name, "--json", ], }); expect(selectedCli.exitCode, selectedCli.stderr).toBe(0); const selected = await artifactMcpCall(client, { name: "inspect_managed_members", - arguments: { path, type_names: [empty!.full_name] }, + arguments: { path, type_names: [empty.full_name] }, }); expect(selected.isError).not.toBe(true); const selectedEvidence = parseEvidence(selected.structuredContent); expect(selectedEvidence.normalized_result).toEqual( parseEvidence(selectedCli.json).normalized_result, ); - expect(selectedEvidence.normalized_result).toMatchObject({ + const selectedReport = managedMemberInspectionSchema.parse( + selectedEvidence.normalized_result, + ); + expect(selectedReport).toMatchObject({ coverage: { state: "complete", issues: [] }, + selection: { type_names: [empty.full_name], unmatched_type_names: [] }, + types: [{ token: empty.token, full_name: empty.full_name }], + methods: [], }); const summaryReply = await artifactMcpCall(client, { diff --git a/tests/acceptance/pcapOutputUtf8CliMcp.test.ts b/tests/acceptance/pcapOutputUtf8CliMcp.test.ts index 5c16585544..cea2f229df 100644 --- a/tests/acceptance/pcapOutputUtf8CliMcp.test.ts +++ b/tests/acceptance/pcapOutputUtf8CliMcp.test.ts @@ -111,6 +111,11 @@ cliTest.skipIf(process.platform === "win32")( stage: "version", invalid_utf8_streams: [scenario.stream], }, + captured_output: { + stdout: expect.stringContaining( + scenario.tool === "tshark" ? "TShark" : "Capinfos", + ), + }, }, }); diff --git a/tests/acceptance/pcapVerifier.test.ts b/tests/acceptance/pcapVerifier.test.ts new file mode 100644 index 0000000000..d2f739c24b --- /dev/null +++ b/tests/acceptance/pcapVerifier.test.ts @@ -0,0 +1,62 @@ +import { execFile } from "node:child_process"; +import { existsSync } from "node:fs"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { pcapResultSchemas } from "../../src/domain/pcap/pcapAnalysis.js"; +import { offlineHttpCapture } from "../fixtures/pcap/offlineHttpCapture.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); + +it.skipIf( + process.platform !== "linux" || + !existsSync("/usr/bin/tshark") || + !existsSync("/usr/bin/capinfos"), +)( + "verifies complete and unresolved HTTP coverage with real Wireshark tools", + async () => { + const root = await createTestTempDirectory("rea-pcap-verifier-test-"); + const capture = join(root, "capture.pcap"); + for (const unresolved of [false, true]) { + await writeFile( + capture, + offlineHttpCapture( + "/without-host", + 1, + unresolved ? null : "example.test", + ), + ); + const environment = { + ...process.env, + REA_TSHARK_COMMAND: "/usr/bin/tshark", + REA_CAPINFOS_COMMAND: "/usr/bin/capinfos", + }; + const cli = await execute( + process.execPath, + ["scripts/rea.mjs", "inspect-pcap-capture", capture, "--json"], + { env: environment, timeout: 60_000, maxBuffer: 8 * 1024 * 1024 }, + ); + const result = pcapResultSchemas.inspect_pcap_capture.parse( + parseEvidence(JSON.parse(cli.stdout)).normalized_result, + ); + expect(result.coverage).toBe(unresolved ? "partial" : "complete"); + expect(result.http_request_uri_count).toBe(unresolved ? 0 : 1); + expect(result.http_request_uris).toEqual( + unresolved ? [] : ["http://example.test/without-host"], + ); + await execute( + process.execPath, + ["scripts/verify-pcap.mjs", "--pcap", capture], + { + env: environment, + timeout: 60_000, + maxBuffer: 8 * 1024 * 1024, + }, + ); + } + }, + 120_000, +); diff --git a/tests/acceptance/peCertificatesCli.test.ts b/tests/acceptance/peCertificatesCli.test.ts index c963922ff1..860041988e 100644 --- a/tests/acceptance/peCertificatesCli.test.ts +++ b/tests/acceptance/peCertificatesCli.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; @@ -33,7 +34,16 @@ cliTest.for(["pubkey", "signature"])( expect(invalid.exitCode).toBe(1); expect(invalid.json).toMatchObject({ code: "invalid_request", - details: { operation: "inspect_pe_certificates" }, + details: { + operation: "inspect_pe_certificates", + issues: [ + expect.objectContaining({ + path: ["path"], + reason: "invalid_format", + message: expect.stringContaining("padding"), + }), + ], + }, }); // Only zero the advertised unused bit; all ASN.1 lengths stay unchanged. const last = at + marker.length - 1; @@ -126,10 +136,9 @@ cliTest( "resolves a relative PE path against the working directory", async ({ cli }) => { const root = await createTestTempDirectory("rea-pe-cert-cli-rel-"); - await writeFile( - join(root, "unsigned.exe"), - peWithoutCertificateTableFixture(), - ); + const path = join(root, "unsigned.exe"); + const bytes = peWithoutCertificateTableFixture(); + await writeFile(path, bytes); const result = await cli.run({ arguments: ["inspect-pe-certificates", "unsigned.exe", "--json"], cwd: root, @@ -138,8 +147,25 @@ cliTest( const evidence = toolContract("inspect_pe_certificates").outputSchema.parse( result.json, ); - expect(evidence.normalized_result.table).toBeNull(); - expect(evidence.subject?.local_path).toBe(join(root, "unsigned.exe")); + expect(evidence).toMatchObject({ + confidence: "observed", + subject: { local_path: path, format: "pe", architecture: "x86" }, + normalized_result: { + artifact: { + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + format: "pe32", + machine: 0x14c, + table: null, + entries: [], + coverage: { status: "complete", examined_entries: 0, certificates: 0 }, + limitations: expect.arrayContaining([ + expect.stringContaining("IMAGE_DIRECTORY_ENTRY_SECURITY is absent"), + ]), + }, + }); }, ); diff --git a/tests/acceptance/peImportsCli.test.ts b/tests/acceptance/peImportsCli.test.ts index 502f261fe0..e2ced905e7 100644 --- a/tests/acceptance/peImportsCli.test.ts +++ b/tests/acceptance/peImportsCli.test.ts @@ -1,4 +1,5 @@ import { readFile, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { join } from "node:path"; import { expect } from "vitest"; import { toolContract } from "../../src/contracts/toolContracts.js"; @@ -35,7 +36,15 @@ for (const bits of [32, 64] as const) { expect(result.exitCode).toBe(1); expect(result.json).toMatchObject({ code: "invalid_request", - details: { operation: "inspect_pe_imports" }, + details: { + operation: "inspect_pe_imports", + issues: expect.arrayContaining([ + expect.objectContaining({ + path: ["path"], + reason: "invalid_format", + }), + ]), + }, }); const response = await call("inspect_pe_imports", { path }); expect(response.isError).toBe(true); @@ -168,7 +177,9 @@ cliTest( "resolves a relative PE path against the working directory", async ({ cli }) => { const root = await createTestTempDirectory("rea-pe-imports-cli-rel-"); - await writeFile(join(root, "unsigned.exe"), peWithoutImportTableFixture()); + const path = join(root, "unsigned.exe"); + const bytes = peWithoutImportTableFixture(); + await writeFile(path, bytes); const result = await cli.run({ arguments: ["inspect-pe-imports", "unsigned.exe", "--json"], cwd: root, @@ -177,8 +188,33 @@ cliTest( const evidence = toolContract("inspect_pe_imports").outputSchema.parse( result.json, ); - expect(evidence.normalized_result.import_directory).toBeNull(); - expect(evidence.subject?.local_path).toBe(join(root, "unsigned.exe")); + expect(evidence).toMatchObject({ + confidence: "observed", + subject: { + local_path: path, + format: "pe", + architecture: "x86_64", + digest: { sha256: createHash("sha256").update(bytes).digest("hex") }, + }, + normalized_result: { + artifact: { path, bytes: bytes.length }, + format: "pe32-plus", + machine: 0x8664, + bits: 64, + architecture: "amd64", + import_directory: null, + delay_import_directory: null, + imports: [], + delay_imports: [], + imphash: { algorithm: "pefile", md5: null }, + coverage: { + status: "complete", + examined_dlls: 0, + imports: 0, + delay_imports: 0, + }, + }, + }); }, ); @@ -192,7 +228,12 @@ cliTest("reports a truncated PE as invalid input", async ({ cli }) => { expect(result.exitCode).toBe(1); expect(result.json).toMatchObject({ code: "invalid_request", - details: { operation: "inspect_pe_imports" }, + details: { + operation: "inspect_pe_imports", + issues: expect.arrayContaining([ + expect.objectContaining({ path: ["path"], reason: "invalid_format" }), + ]), + }, }); }); diff --git a/tests/acceptance/peLoadConfigCli.test.ts b/tests/acceptance/peLoadConfigCli.test.ts index 72243d33ea..c520828ff1 100644 --- a/tests/acceptance/peLoadConfigCli.test.ts +++ b/tests/acceptance/peLoadConfigCli.test.ts @@ -94,5 +94,26 @@ cliTest("fails closed at the callback limit", async ({ cli }) => { ], }); expect(result.exitCode).toBe(1); - expect(JSON.stringify(result.json)).toContain("Raise max_callbacks above 1"); + expect(result.json).toMatchObject({ + code: "resource_constraint", + retryable: false, + details: { reported_limits: { limit: "callbacks", maximum: 1 } }, + remediation: { + action: expect.stringContaining("Raise max_callbacks above 1"), + }, + }); + const retried = await cli.run({ + arguments: [ + "inspect-pe-load-config", + path, + "--max-callbacks", + "3", + "--json", + ], + }); + expect(retried.exitCode, retried.stderr).toBe(0); + const report = toolContract("inspect_pe_load_config").outputSchema.parse( + retried.json, + ).normalized_result; + expect(report.tls?.callbacks).toHaveLength(3); }); diff --git a/tests/acceptance/process/processFinalization.test.ts b/tests/acceptance/process/processFinalization.test.ts index dc35f2d829..75abe7f582 100644 --- a/tests/acceptance/process/processFinalization.test.ts +++ b/tests/acceptance/process/processFinalization.test.ts @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; import { access, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -188,10 +189,11 @@ const completedCaptureFromOutcome = (outcome: CaptureOutcome) => { return partial.capture; }; -const expectCooperativeFinalization = ( +const expectCooperativeFinalization = async ( outcome: CaptureOutcome, + root: string, expectedExitCode = 0, -): void => { +): Promise => { const capture = completedCaptureFromOutcome(outcome); expect(capture.exit.reason, "the deadline remains the exit reason").toBe( "timeout", @@ -220,10 +222,13 @@ const expectCooperativeFinalization = ( capture.frames.map(({ data }) => data).join(""), "finalization output is captured", ).toContain("finalized"); + const reportBytes = await readFile(join(root, "final.json")); + expect(JSON.parse(reportBytes.toString("utf8"))).toEqual({ phase: "final" }); expect( - capture.files_after.find(({ path }) => path.endsWith("final.json"))?.sha256, + capture.files_after.find(({ path }) => path.endsWith(":final.json")) + ?.sha256, "the final filesystem snapshot hashes the final report", - ).toMatch(/^[0-9a-f]{64}$/u); + ).toBe(createHash("sha256").update(reportBytes).digest("hex")); expect( capture.manifest.scenario.finalization_ms, "the scenario commitment retains finalization_ms", @@ -282,7 +287,7 @@ cliCaptureTest.for(["CLI", "MCP"] as const)( adapter === "CLI" ? await captureViaCli(cli, root, scenario) : await captureViaMcp(scenario); - expectCooperativeFinalization(outcome); + await expectCooperativeFinalization(outcome, root); }, ); @@ -301,7 +306,7 @@ cliCaptureTest.for(["CLI", "MCP"] as const)( adapter === "CLI" ? await captureViaCli(cli, root, scenario) : await captureViaMcp(scenario); - expectCooperativeFinalization(outcome, 3); + await expectCooperativeFinalization(outcome, root, 3); }, ); @@ -394,23 +399,17 @@ cliCaptureTest.for(["CLI", "MCP"] as const)( adapter === "CLI" ? await captureViaCli(cli, root, scenario) : await captureViaMcp(scenario); - if (outcome.kind === "partial") { - expect( - finalizationFromPartial(outcome.partial), - "the cleanup-flake partial observation has no default finalization", - ).toBeUndefined(); - return; - } + const capture = completedCaptureFromOutcome(outcome); expect( - outcome.capture.exit, + capture.exit, "default captures have no finalization evidence", ).not.toHaveProperty("finalization"); expect( - outcome.capture.manifest.scenario, + capture.manifest.scenario, "the committed scenario keeps its legacy identity", ).not.toHaveProperty("finalization_ms"); expect( - outcome.capture.manifest.comparison_contract, + capture.manifest.comparison_contract, "the comparison contract keeps its legacy identity", ).not.toHaveProperty("finalization_ms"); }, diff --git a/tests/acceptance/process/processTerminalWidth.test.ts b/tests/acceptance/process/processTerminalWidth.test.ts index 0de31db8e3..377a772b43 100644 --- a/tests/acceptance/process/processTerminalWidth.test.ts +++ b/tests/acceptance/process/processTerminalWidth.test.ts @@ -92,10 +92,20 @@ cliTest.skipIf(CAPTURE_SKIP_REASON)( executable: process.execPath, arguments: [ "-e", - "setTimeout(() => process.stdout.write('ab'), 100); setTimeout(() => {}, 180)", + ` + const keepAlive = setInterval(() => {}, 1000); + const resized = () => { + if (process.stdout.columns === 2) { + clearInterval(keepAlive); + process.stdout.write('ab'); + } + }; + process.stdout.on('resize', resized); + resized(); + `, ], terminal: { columns: 3, rows: 2, scrollback: 0 }, - events: [{ type: "resize", at_ms: 20, columns: 2, rows: 2 }], + events: [{ type: "resize", at_ms: 1_000, columns: 2, rows: 2 }], timeout_ms: 5_000, }), ); @@ -106,6 +116,7 @@ cliTest.skipIf(CAPTURE_SKIP_REASON)( const capture = parseProcessCapture( parseEvidence(result.json).normalized_result, ); + expect(capture.exit).toMatchObject({ code: 0, reason: "exited" }); expect(capture.rendered_frames.some((frame) => frame.columns === 2)).toBe( true, ); diff --git a/tests/acceptance/providers/hopperStartupRecovery.test.ts b/tests/acceptance/providers/hopperStartupRecovery.test.ts index 135685642d..1f7facf76c 100644 --- a/tests/acceptance/providers/hopperStartupRecovery.test.ts +++ b/tests/acceptance/providers/hopperStartupRecovery.test.ts @@ -1,7 +1,6 @@ import { spawn, type ChildProcess } from "node:child_process"; import { access, copyFile, readFile, rm, writeFile } from "node:fs/promises"; import { dirname, resolve } from "node:path"; -import { setTimeout as delay } from "node:timers/promises"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; @@ -9,12 +8,49 @@ import { expect, onTestFinished, vi } from "vitest"; import { z } from "zod"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { workspaceCliTest } from "../../support/cli/workspaceCliFixture.js"; import { acquireHopperTargetLease } from "../../../src/process/HopperLease.js"; const statusSchema = z.object({ result: z.object({ open: z.boolean() }) }); const test = workspaceCliTest.skipIf(process.platform !== "darwin"); +const stopChild = async (child: ChildProcess) => { + if (child.exitCode !== null || child.signalCode !== null) return; + const exited = new Promise((resolveExit) => + child.once("exit", () => resolveExit()), + ); + child.kill("SIGKILL"); + await exited; +}; + +const closeWithLateBridge = async ( + mcp: Client, + startBridge: () => ChildProcess, +): Promise => { + let closeStarted!: () => void; + const started = new Promise((resolveStarted) => { + closeStarted = resolveStarted; + }); + const closing = mcp.callTool( + { name: "close_binary", arguments: {} }, + { onprogress: () => closeStarted() }, + ); + let closeAcknowledged = false; + try { + await awaitAcknowledgement(started, closing, "late bridge close progress"); + closeAcknowledged = true; + // A verified close may remove the socket before the fixture can poll it. + // The successful close and child exit establish the owned bridge outcome. + const owned = startBridge(); + expect((await closing).isError).not.toBe(true); + return owned; + } finally { + if (!closeAcknowledged) await mcp.close(); + await Promise.allSettled([closing]); + } +}; + test.for(["target", "loader"] as const)( "preserves a prepared target or refuses literal-unsafe %s before handoff", { timeout: 90_000 }, @@ -229,16 +265,8 @@ test.for(["handoff", "not-found"] as const)( const mcp = new Client({ name: "hopper-late-readiness", version: "1" }); const children: ChildProcess[] = []; let runtime: string | undefined; - const stop = async (child: ChildProcess) => { - if (child.exitCode !== null || child.signalCode !== null) return; - const exited = new Promise((resolveExit) => - child.once("exit", () => resolveExit()), - ); - child.kill("SIGKILL"); - await exited; - }; onTestFinished(async () => { - await Promise.all(children.map(stop)); + await Promise.all(children.map(stopChild)); await mcp.close(); await transport.close(); if (runtime !== undefined) @@ -335,15 +363,10 @@ test.for(["handoff", "not-found"] as const)( ); expect(wrong.exitCode).toBeNull(); await access(runtime); - await stop(wrong); + await stopChild(wrong); await rm(socket, { force: true }); } - const closing = call("close_binary"); - await delay(150); - // A verified close may remove the socket before the fixture can poll it. - // The successful close and child exit establish the owned bridge outcome. - const owned = lateBridge(runId); - expect((await closing).isError).not.toBe(true); + const owned = await closeWithLateBridge(mcp, () => lateBridge(runId)); await vi.waitFor(() => expect(owned.exitCode).toBe(0)); await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); await writeFile(mode, "bridge"); diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts index 4f50f3b82a..46085f2a79 100644 --- a/tests/acceptance/pyinstallerArchive.test.ts +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -482,8 +482,13 @@ it("extracts CArchive members through extract-artifact", async () => { hello?.bytes_written ?? 0, ); expect( - await readFile(join(extracted.output_root, "PYZ.pyz", "app.pyc")), - ).toHaveLength(16 + Buffer.from("app-body").length); + (await readFile(join(extracted.output_root, "PYZ.pyz", "app.pyc"))).equals( + Buffer.concat([ + Buffer.from("610d0d0a000000000000000000000000", "hex"), + Buffer.from("app-body"), + ]), + ), + ).toBe(true); expect( Buffer.compare( await readFile(join(extracted.output_root, "hello")), diff --git a/tests/acceptance/pythonPycCli.test.ts b/tests/acceptance/pythonPycCli.test.ts index 4f3b56b39a..a8982f5e50 100644 --- a/tests/acceptance/pythonPycCli.test.ts +++ b/tests/acceptance/pythonPycCli.test.ts @@ -458,6 +458,7 @@ cliTest( expect(result.exitCode, result.stderr).toBe(0); const evidence = completePycEvidenceSchema.parse(result.json); expect(evidence.normalized_result.code_objects[0]?.firstlineno).toBe(-1); + expect(evidence.normalized_result.code_objects[1]?.firstlineno).toBe(-7); expect(evidence.normalized_result.code_objects[0]?.consts).toContainEqual({ kind: "code", name: "inner", @@ -557,13 +558,10 @@ cliTest( }); expect(result.exitCode, result.stderr).toBe(0); const evidence = completePycEvidenceSchema.parse(result.json); - expect( - evidence.normalized_result.code_objects[0]?.consts.map((value) => - value.kind === "frozenset" || value.kind === "dict" - ? value.length - : null, - ), - ).toEqual(testCase.lengths); + expect(evidence.normalized_result.code_objects[0]?.consts).toEqual([ + { kind: "frozenset", length: testCase.lengths[0] }, + { kind: "dict", length: testCase.lengths[1] }, + ]); } }, ); diff --git a/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts b/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts index a3ed9425f0..54402c7925 100644 --- a/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts +++ b/tests/acceptance/setup/clientConfigurationDiagnosticsCli.test.ts @@ -224,6 +224,9 @@ cliTest.skipIf(process.platform === "win32" || process.getuid?.() === 0)( await chmod(locked, 0o700); } expect(await readFile(target, "utf8")).toBe(original); - expect(await readdir(home)).toEqual([".claude.json", "owned-denied"]); + expect((await readdir(home)).sort()).toEqual([ + ".claude.json", + "owned-denied", + ]); }, ); diff --git a/tests/acceptance/sourcePathFailuresCli.test.ts b/tests/acceptance/sourcePathFailuresCli.test.ts index e0821d37b4..1ae2d7c45c 100644 --- a/tests/acceptance/sourcePathFailuresCli.test.ts +++ b/tests/acceptance/sourcePathFailuresCli.test.ts @@ -57,6 +57,17 @@ cliTest.skipIf(process.platform === "win32")( expect(result.json).toMatchObject({ code: "invalid_request", category: "invalid_input", + details: { + operation: "import_reference_source", + diagnostics: { reference_source_code: "invalid-root" }, + issues: [ + expect.objectContaining({ + path: ["root"], + reason: "invalid_value", + message: expect.stringContaining(path), + }), + ], + }, }); } const valid = await cli.run({ diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index 49f268f3e3..3308c15681 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -332,6 +332,14 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( const { call } = await connectLocalToolsMcp(); const inventory = await call("inspect_sqlite_database", { path }); expect(inventory.isError).not.toBe(true); + const inventoryEvidence = parseEvidence(inventory.structuredContent); + expect(inventoryEvidence.parameters).not.toHaveProperty( + "worker_timeout_ms", + ); + expect( + sqliteDatabaseSchema.parse(inventoryEvidence.normalized_result) + .worker_timeout_ms, + ).toBe(30_000); const schema = z .object({ normalized_result: z.object({ @@ -422,13 +430,33 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( rows: { columns: ["id", ""], returned_rows: 1 }, }, }); - for (const input of [ - { path, columns: ["id"] }, - { path, schema_scope: "selected_table" }, - { path, table: "selected", columns: [] }, - { path, table: "selected", columns: ["ID"] }, - ]) - expect((await call("inspect_sqlite_database", input)).isError).toBe(true); + for (const { input, issue } of [ + { + input: { path, columns: ["id"] }, + issue: { path: ["columns"], reason: "invalid_value" }, + }, + { + input: { path, schema_scope: "selected_table" }, + issue: { path: ["schema_scope"], reason: "invalid_value" }, + }, + { + input: { path, table: "selected", columns: [] }, + issue: { path: ["columns"], reason: "out_of_range", minimum: 1 }, + }, + { + input: { path, table: "selected", columns: ["ID"] }, + issue: { path: ["columns"], reason: "invalid_format" }, + }, + ]) { + const response = await call("inspect_sqlite_database", input); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "invalid_request", + details: { + operation: "inspect_sqlite_database", + issues: expect.arrayContaining([expect.objectContaining(issue)]), + }, + }); + } expect(Buffer.compare(await readFile(path), before)).toBe(0); }, ); diff --git a/tests/acceptance/tools/harBoundaryFacts.test.ts b/tests/acceptance/tools/harBoundaryFacts.test.ts index ced659752e..398c04347f 100644 --- a/tests/acceptance/tools/harBoundaryFacts.test.ts +++ b/tests/acceptance/tools/harBoundaryFacts.test.ts @@ -87,10 +87,16 @@ cliTest( ], }); } else { - expect(JSON.stringify(result)).toContain(marker); expect(result).toMatchObject({ items: [ - {}, + { + reported: { + name: { payload: marker }, + type: { body: marker }, + request: { reason: { payload: marker } }, + response: { method: { payload: marker } }, + }, + }, { reported: { name: scalarMarker, diff --git a/tests/acceptance/tools/mobileGraphExamples.test.ts b/tests/acceptance/tools/mobileGraphExamples.test.ts index 50da8f07a3..c30b13c3f7 100644 --- a/tests/acceptance/tools/mobileGraphExamples.test.ts +++ b/tests/acceptance/tools/mobileGraphExamples.test.ts @@ -20,30 +20,42 @@ const graphExamples = [ command: "project-android-application-graph", operation: "project_android_application_graph", example: ANDROID_APPLICATION_GRAPH_EXAMPLE, + rootFormat: "apk", + digestDigit: "a", }, { name: "IPA", command: "project-apple-application-graph", operation: "project_apple_application_graph", example: APPLE_APPLICATION_GRAPH_EXAMPLE, + rootFormat: "ipa", + digestDigit: "b", }, { name: "macOS", command: "project-apple-application-graph", operation: "project_apple_application_graph", example: MACOS_APPLICATION_GRAPH_EXAMPLE, + rootFormat: "directory", + digestDigit: "c", + applicationRoots: ["."], }, { name: "Harmony", command: "project-harmony-application-graph", operation: "project_harmony_application_graph", example: HARMONY_APPLICATION_GRAPH_EXAMPLE, + rootFormat: "hap", + digestDigit: "d", }, ] as const; cliTest.for(graphExamples)( "CLI executes the canonical $name graph example without target files", - async ({ command, operation, example }, { cli }) => { + async ( + { command, operation, example, rootFormat, digestDigit, ...facts }, + { cli }, + ) => { const root = await createTestTempDirectory("rea-mobile-graph-example-"); const input = JSON.stringify(example); const path = join(root, "example.json"); @@ -56,9 +68,26 @@ cliTest.for(graphExamples)( expect(result.exitCode, result.stdout).toBe(0); const evidence = parseEvidence(result.json); expect(evidence.operation).toBe(operation); + const sourceId = example.inventory_evidence[0]!.evidence_id; + expect(evidence.evidence_links).toEqual([sourceId]); expect(evidence.normalized_result).toMatchObject({ + root_sha256: digestDigit.repeat(64), + root_format: rootFormat, + source_evidence_ids: [sourceId], coverage: { status: "partial", inventory_complete: true }, }); + if ("applicationRoots" in facts) { + expect(evidence.normalized_result).toMatchObject({ + application_roots: facts.applicationRoots, + bundles: [ + expect.objectContaining({ + path: ".", + role: "application", + role_basis: "path-convention", + }), + ], + }); + } } }, ); diff --git a/tests/acceptance/tools/nativeFunctionMatching.test.ts b/tests/acceptance/tools/nativeFunctionMatching.test.ts index daf427cbf1..dd1fdceabe 100644 --- a/tests/acceptance/tools/nativeFunctionMatching.test.ts +++ b/tests/acceptance/tools/nativeFunctionMatching.test.ts @@ -327,11 +327,14 @@ cliTest.for([{ profile: "full" }, { profile: "compact" }] as const)( "compare_functions", ); await verifyPortableCli(cli, fixture, input, evidence.evidence_id); + const missingId = `ev_${"a".repeat(64)}`; const missing = await call("match_native_functions", { - left: reference(`ev_${"a".repeat(64)}`), + left: reference(missingId), right: reference(right.evidence_id), }); - expect(missing.isError).toBe(true); + expect(parseMcpToolError(missing)).toMatchObject({ + error: { details: { evidence_id: missingId, reason: "missing" } }, + }); }, ); diff --git a/tests/acceptance/tools/qlogCapture.test.ts b/tests/acceptance/tools/qlogCapture.test.ts index 6c5a8bb0d1..cc0c739e2a 100644 --- a/tests/acceptance/tools/qlogCapture.test.ts +++ b/tests/acceptance/tools/qlogCapture.test.ts @@ -45,6 +45,24 @@ const qlog = (events: unknown[]) => ], }); +const expectRecoveredCapture = async ( + call: Awaited>["call"], + path: string, +): Promise => { + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "qlog", + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const capture = webNetworkCaptureSchema.parse( + parseEvidence(response.structuredContent).normalized_result, + ); + expect(capture).toMatchObject({ + format: "qlog", + records: [{ reported: { name: "transport:packet_sent" } }], + }); +}; + it("does not classify hidden custom event data as incomplete transport facts", async () => { const root = await createTestTempDirectory("rea-qlog-hidden-extension-"); const path = join(root, "input.qlog"); @@ -408,12 +426,41 @@ it("fails whole-capture validation before selection and survives malformed, unsu const root = await createTestTempDirectory("rea-qlog-errors-"); const path = join(root, "input.qlog"); const { call, client } = await connectLocalToolsMcp(); - for (const [text, code] of [ - [qlog([event, { ...event, time: "invalid" }]), "invalid_request"], - [qlog([event]).slice(0, -1), "invalid_request"], + for (const [text, expected] of [ + [ + qlog([event, { ...event, time: "invalid" }]), + { + code: "invalid_request", + details: { + operation: "inspect_web_network_capture", + issues: [ + { + path: ["capture_path", "/traces/0/events/1"], + reason: "invalid_format", + }, + ], + }, + }, + ], + [ + qlog([event]).slice(0, -1), + { + code: "invalid_request", + details: { + operation: "inspect_web_network_capture", + issues: [{ path: ["capture_path", ""], reason: "invalid_format" }], + }, + }, + ], [ qlog([event]).replace('"qlog_version":"0.3"', '"qlog_version":"0.4"'), - "capability_unavailable", + { + code: "capability_unavailable", + details: { + operation: "inspect_web_network_capture", + provider_id: "qlog", + }, + }, ], [ qlog([ @@ -422,7 +469,20 @@ it("fails whole-capture validation before selection and survives malformed, unsu extension: JSON.parse("[".repeat(65) + "0" + "]".repeat(65)), }, ]), - "invalid_request", + { + code: "invalid_request", + details: { + operation: "inspect_web_network_capture", + issues: [ + expect.objectContaining({ + reason: "out_of_range", + expected: { + maximum_capture_nesting: WEB_NETWORK_CAPTURE_LIMITS.depth, + }, + }), + ], + }, + }, ], ] as const) { await writeFile(path, text); @@ -434,7 +494,7 @@ it("fails whole-capture validation before selection and survives malformed, unsu record_ordinals: [0], }), ), - ).toMatchObject({ error: { code } }); + ).toMatchObject({ error: expected }); const failed: unknown = await execute(process.execPath, [ cli, "inspect-web-network-capture", @@ -444,8 +504,16 @@ it("fails whole-capture validation before selection and survives malformed, unsu ]).catch((cause: unknown) => cause); expect(failed).toMatchObject({ code: 1, - stdout: expect.stringContaining(`"code":"${code}"`), + stdout: expect.any(String), }); + if ( + typeof failed !== "object" || + failed === null || + !("stdout" in failed) || + typeof failed.stdout !== "string" + ) + throw new Error("CLI failure omitted JSON output"); + expect(JSON.parse(failed.stdout)).toMatchObject(expected); } await truncate(path, WEB_NETWORK_CAPTURE_LIMITS.inputBytes + 1); expect( @@ -458,14 +526,7 @@ it("fails whole-capture validation before selection and survives malformed, unsu ).toMatchObject({ error: { code: "invalid_request" } }); await writeFile(path, qlog([event])); await client.ping(); - expect( - ( - await call("inspect_web_network_capture", { - capture_path: path, - format: "qlog", - }) - ).isError, - ).not.toBe(true); + await expectRecoveredCapture(call, path); }, 60_000); runtimeTest.skipIf(process.versions.bun !== undefined || !hasMemoryBound)( @@ -504,14 +565,7 @@ runtimeTest.skipIf(process.versions.bun !== undefined || !hasMemoryBound)( }); await writeFile(path, qlog([event])); await client.ping(); - expect( - ( - await call("inspect_web_network_capture", { - capture_path: path, - format: "qlog", - }) - ).isError, - ).not.toBe(true); + await expectRecoveredCapture(call, path); }, 60_000, ); diff --git a/tests/acceptance/uImageCli.test.ts b/tests/acceptance/uImageCli.test.ts index 027844dda5..fd69591c9d 100644 --- a/tests/acceptance/uImageCli.test.ts +++ b/tests/acceptance/uImageCli.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -19,6 +20,7 @@ cliTest( const root = await createTestTempDirectory("rea-uimage-cli-"); const partA = Buffer.from("first-part"); const partB = Buffer.from("second-part"); + const payload = buildMultiPayload([partA, partB]); const source = join(root, "firmware.bin"); await writeFile( source, @@ -30,7 +32,7 @@ cliTest( loadAddress: 0x8000, entryPoint: 0x8000, name: "multi-test", - payload: buildMultiPayload([partA, partB]), + payload, }), ); const inspected = await cli.run({ @@ -69,10 +71,30 @@ cliTest( parseEvidence(extracted.json).normalized_result, ); expect(result.containment_verified).toBe(true); + const expectedArtifacts = [ + ["image-0.bin", partA], + ["image-1.bin", partB], + ["payload.bin", payload], + ] as const; expect( - (await readFile(join(result.output_root, "payload.bin"))).equals( - buildMultiPayload([partA, partB]), - ), + result.artifacts + .map(({ relative_path, bytes_written, sha256 }) => ({ + relative_path, + bytes_written, + sha256, + })) + .sort((left, right) => + left.relative_path.localeCompare(right.relative_path), + ), + ).toEqual( + expectedArtifacts.map(([relative_path, bytes]) => ({ + relative_path, + bytes_written: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + })), + ); + expect( + (await readFile(join(result.output_root, "payload.bin"))).equals(payload), ).toBe(true); expect( (await readFile(join(result.output_root, "image-0.bin"))).equals(partA), diff --git a/tests/acceptance/wasmInterfaceComparison.test.ts b/tests/acceptance/wasmInterfaceComparison.test.ts index 9a12455a58..75a39c7745 100644 --- a/tests/acceptance/wasmInterfaceComparison.test.ts +++ b/tests/acceptance/wasmInterfaceComparison.test.ts @@ -7,6 +7,7 @@ import { createEvidence, parseEvidence } from "../../src/domain/evidence.js"; import { wasmInterfaceComparisonSchema } from "../../src/contracts/wasm/wasmComparisonContracts.js"; import { composeBinarySession } from "../../src/application/binary/BinarySessionComposition.js"; import { AnalysisProviderRegistry } from "../../src/application/binary/AnalysisProviderRegistry.js"; +import { parseWabtWat } from "../../src/wasm/WabtWat.js"; import { createServer } from "../../src/server/createServer.js"; import { wasmArtifactFixture } from "../fixtures/wasm/artifact.js"; import { cliTest } from "../support/cli/cliFixture.js"; @@ -152,3 +153,51 @@ cliTest( } }, ); + +cliTest( + "does not report colliding debug identifiers as unchanged after a signature change", + async ({ cli }) => { + const input = (parameter: string) => { + const report = wasmArtifactFixture(); + report.interface = parseWabtWat( + `(module (func $same (param i32)) (func $same (param ${parameter})) (export "first" (func $same)) (export "second" (func $same)))`, + ).interface; + return createEvidence( + { + path: report.artifact.path, + sha256: report.artifact.sha256, + format: "file", + }, + { id: "fixture", name: "typed fixture", version: "1" }, + { + operation: "inspect_wasm_artifact", + parameters: { path: report.artifact.path }, + result: report, + confidence: "observed", + }, + ); + }; + const root = await createTestTempDirectory("rea-wasm-colliding-names-"); + const path = join(root, "input.json"); + await writeFile( + path, + JSON.stringify({ + left: input("i32"), + right: input("i64"), + scope: "both", + }), + ); + const result = await cli.run({ + arguments: ["compare-wasm-interfaces", path, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const report = wasmInterfaceComparisonSchema.parse( + parseEvidence(result.json).normalized_result, + ); + expect(report.changes.map(({ state }) => state)).toEqual([ + "unknown", + "unknown", + ]); + expect(report.summary.unchanged).toBe(0); + }, +); diff --git a/tests/boundary/android/androidJniLibraries.test.ts b/tests/boundary/android/androidJniLibraries.test.ts index be9b28ddb0..734d141ef8 100644 --- a/tests/boundary/android/androidJniLibraries.test.ts +++ b/tests/boundary/android/androidJniLibraries.test.ts @@ -310,7 +310,14 @@ it("retains the live reader after failed close and gates the next read on its cl expect(source.resources.hasPending).toBe(true); allowClose = true; const recovered = await read(source); - expect(recovered.libraries[0]?.facts).not.toBeNull(); + expect(recovered.libraries).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: LIBRARY, + facts: expect.objectContaining({ tables: expect.any(Array) }), + }), + ]), + ); expect(open).toHaveBeenCalledTimes(2); expect(source.resources.hasPending).toBe(false); expect(first.cleanupIncomplete).toBe(false); diff --git a/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts b/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts index 50d39ded4e..e9dcedc9bc 100644 --- a/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts +++ b/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts @@ -335,6 +335,10 @@ describe("CdpBrowserProvider: cancellation and session events", () => { type: "same_document_navigation", url: `${browser.allowedOrigin}/reloaded?token=session-secret#spa-secret`, }), + expect.objectContaining({ + type: "redirect", + url: `${browser.allowedOrigin}/redirected?token=redirect-secret`, + }), expect.objectContaining({ type: "load_failed", detail: "net::ERR_CONNECTION_REFUSED", @@ -342,8 +346,6 @@ describe("CdpBrowserProvider: cancellation and session events", () => { ]), ); expect(result.value.completeness.status).toBe("attach_limited"); - expect(JSON.stringify(result.value)).toContain("session-secret#spa-secret"); - expect(JSON.stringify(result.value)).toContain("redirect-secret"); expect(browser.commands.map(({ method }) => method)).toContain( "Target.detachFromTarget", ); diff --git a/tests/boundary/browser/cdpBrowserProvider.redirectErrors.test.ts b/tests/boundary/browser/cdpBrowserProvider.redirectErrors.test.ts index 64735b1a4e..3a277c0a64 100644 --- a/tests/boundary/browser/cdpBrowserProvider.redirectErrors.test.ts +++ b/tests/boundary/browser/cdpBrowserProvider.redirectErrors.test.ts @@ -78,9 +78,9 @@ describe("CdpBrowserProvider: redirect errors: redirect response validation", () }, ], }); - expect( - inspection.network.requests[0]?.body_shapes.request, - ).not.toBeNull(); + expect(inspection.network.requests[0]?.body_shapes.request).toEqual( + expect.any(Object), + ); expect(inspection.network.requests[0]?.body_shapes.response).toBeNull(); expect(inspection.network.requests[0]?.body_shapes.status).toBe( "partial", diff --git a/tests/boundary/browser/cdpEndpointResponseBounds.test.ts b/tests/boundary/browser/cdpEndpointResponseBounds.test.ts index b0ff728de1..ff8dd821ee 100644 --- a/tests/boundary/browser/cdpEndpointResponseBounds.test.ts +++ b/tests/boundary/browser/cdpEndpointResponseBounds.test.ts @@ -2,6 +2,8 @@ import { createServer } from "node:http"; import type { RequestListener, Server } from "node:http"; import { afterEach, describe, expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { discoverCdpEndpoint, @@ -41,10 +43,7 @@ describe("CDP discovery response UTF-8", () => { malformed[malformed.indexOf("target-slot")] = field === "title" ? 0xff : 0xc3; let sendMalformed = true; - let responseClosed: (() => void) | undefined; - const closed = new Promise((resolve) => { - responseClosed = resolve; - }); + let responseClosed = false; const server = await startServer((request, response) => { if (request.url === "/json/version") { response.end( @@ -58,7 +57,10 @@ describe("CDP discovery response UTF-8", () => { }), ); } else if (request.url === "/json/list") { - if (sendMalformed) response.once("close", () => responseClosed?.()); + if (sendMalformed) + response.once("close", () => { + responseClosed = true; + }); response.end(sendMalformed ? malformed : valid); } else { response.writeHead(404).end(); @@ -72,7 +74,7 @@ describe("CDP discovery response UTF-8", () => { operation: "list_browser_targets", reason: "invalid_endpoint_response", }); - await closed; + await expect.poll(() => responseClosed, { timeout: 2_000 }).toBe(true); sendMalformed = false; await expect( discoverCdpEndpoint(server.url.href, "list_browser_targets"), @@ -127,12 +129,11 @@ describe("CDP discovery response bounds", () => { }); it("rejects and closes an oversized chunked response without parsing a prefix", async () => { - let responseClosed: (() => void) | undefined; - const closed = new Promise((resolve) => { - responseClosed = resolve; - }); + let responseClosed = false; const server = await startServer((_request, response) => { - response.once("close", () => responseClosed?.()); + response.once("close", () => { + responseClosed = true; + }); response.write(JSON.stringify("x".repeat(15))); }); @@ -145,47 +146,56 @@ describe("CDP discovery response bounds", () => { _tag: "BrowserObservationError", reason: "payload_limit", }); - await closed; + await expect.poll(() => responseClosed, { timeout: 2_000 }).toBe(true); }); it("ends a response that exceeds the total elapsed deadline", async () => { - let responseClosed: (() => void) | undefined; + let responseClosed = false; let requestStarted: (() => void) | undefined; - const closed = new Promise((resolve) => { - responseClosed = resolve; - }); const started = new Promise((resolve) => { requestStarted = resolve; }); const server = await startServer((_request, response) => { - response.once("close", () => responseClosed?.()); + response.once("close", () => { + responseClosed = true; + }); requestStarted?.(); }); - const pending = readCdpJson(server.url, "list_browser_targets", undefined, { - bytes: 16, - timeoutMs: 50, - }); - const timeoutAssertion = expect(pending).rejects.toMatchObject({ - _tag: "BrowserObservationError", - reason: "timeout", - }); - await started; - - await timeoutAssertion; - await closed; + const controller = new AbortController(); + const pending = readCdpJson( + server.url, + "list_browser_targets", + controller.signal, + { + bytes: 16, + timeoutMs: 50, + }, + ); + await withVerifierCleanup(async () => { + await awaitAcknowledgement(started, pending, "CDP request"); + await expect(pending).rejects.toMatchObject({ + _tag: "BrowserObservationError", + reason: "timeout", + }); + await expect.poll(() => responseClosed, { timeout: 2_000 }).toBe(true); + }, [ + () => controller.abort(), + async () => { + await pending.catch(() => undefined); + }, + ]); }); it("preserves caller cancellation and closes the response", async () => { - let responseClosed: (() => void) | undefined; + let responseClosed = false; let requestStarted: (() => void) | undefined; - const closed = new Promise((resolve) => { - responseClosed = resolve; - }); const started = new Promise((resolve) => { requestStarted = resolve; }); const server = await startServer((_request, response) => { - response.once("close", () => responseClosed?.()); + response.once("close", () => { + responseClosed = true; + }); requestStarted?.(); }); const controller = new AbortController(); @@ -198,23 +208,28 @@ describe("CDP discovery response bounds", () => { timeoutMs: 1_000, }, ); - await started; - controller.abort(); - - await expect(pending).rejects.toMatchObject({ - _tag: "AnalysisCancelledError", - operation: "list_browser_targets", - }); - await closed; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(started, pending, "CDP request"); + controller.abort(); + await expect(pending).rejects.toMatchObject({ + _tag: "AnalysisCancelledError", + operation: "list_browser_targets", + }); + await expect.poll(() => responseClosed, { timeout: 2_000 }).toBe(true); + }, [ + () => controller.abort(), + async () => { + await pending.catch(() => undefined); + }, + ]); }); it("destroys an unsuccessful HTTP response", async () => { - let responseClosed: (() => void) | undefined; - const closed = new Promise((resolve) => { - responseClosed = resolve; - }); + let responseClosed = false; const server = await startServer((_request, response) => { - response.once("close", () => responseClosed?.()); + response.once("close", () => { + responseClosed = true; + }); response.writeHead(503).flushHeaders(); }); @@ -227,7 +242,7 @@ describe("CDP discovery response bounds", () => { _tag: "BrowserObservationError", reason: "invalid_endpoint_response", }); - await closed; + await expect.poll(() => responseClosed, { timeout: 2_000 }).toBe(true); }); }); diff --git a/tests/boundary/browser/cdpJsonShapeComparison.test.ts b/tests/boundary/browser/cdpJsonShapeComparison.test.ts index 4c9b747cd4..4568d59af0 100644 --- a/tests/boundary/browser/cdpJsonShapeComparison.test.ts +++ b/tests/boundary/browser/cdpJsonShapeComparison.test.ts @@ -378,7 +378,7 @@ const comparableCaptures = ( expect(request?.body_shapes.status).toBe("included"); expect( request?.body_shapes[source === "request" ? "request" : "response"], - ).not.toBeNull(); + ).toEqual(expect.any(Object)); } return [before, after]; }; diff --git a/tests/boundary/browser/historicalWebCapture.test.ts b/tests/boundary/browser/historicalWebCapture.test.ts index cf29cdf1f3..538438a051 100644 --- a/tests/boundary/browser/historicalWebCapture.test.ts +++ b/tests/boundary/browser/historicalWebCapture.test.ts @@ -291,7 +291,9 @@ it("reads actual upstream HAR packages in an owned process and retains selected const fixture = historicalHar(); const first = fixture.log.entries[0]; if (first === undefined) throw new Error("fixture missing"); - fixture.log.entries.push(structuredClone(first)); + const second = structuredClone(first); + second.request.url = "https://example.test/b"; + fixture.log.entries.push(second); const text = JSON.stringify(fixture); const path = join(root, "capture.har"); await writeFile(path, text); @@ -303,7 +305,10 @@ it("reads actual upstream HAR packages in an owned process and retains selected }); expect(result.value.normalized_result).toMatchObject({ total_records: 2, - records: [{ ordinal: 1 }, { ordinal: 0 }], + records: [ + { ordinal: 1, reported: { request: { url: second.request.url } } }, + { ordinal: 0, reported: { request: { url: first.request.url } } }, + ], runtime_attribution: "unknown", }); }); diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index 2117969338..1e238aa3ba 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -3,6 +3,8 @@ import { createServer } from "node:http"; import { describe, expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { fetchWebSourceMaps } from "../../../src/browser/WebSourceMapFetcher.js"; import { fetchWebSourceMapsOk } from "../../fixtures/fetchWebSourceMapsOk.js"; import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; @@ -765,16 +767,15 @@ describe("web source-map fetching and validation: enforcing fetch limits and val }); } }); +}); +describe("source-map response cancellation", () => { it("cancels a pending response reader on caller abort", async () => { let markPullStarted: (() => void) | undefined; const pullStarted = new Promise((resolve) => { markPullStarted = resolve; }); - let markCancelled: (() => void) | undefined; - const cancelled = new Promise((resolve) => { - markCancelled = resolve; - }); + let cancelled = false; const body = new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode("{")); @@ -783,7 +784,7 @@ describe("web source-map fetching and validation: enforcing fetch limits and val markPullStarted?.(); }, cancel() { - markCancelled?.(); + cancelled = true; }, }); const controller = new AbortController(); @@ -792,13 +793,23 @@ describe("web source-map fetching and validation: enforcing fetch limits and val fetch: async () => new Response(body), }); - await pullStarted; - controller.abort(abortReason); - - const result = await pending; - expect(result.ok).toBe(false); - if (!result.ok) expect(result.error.cause).toBe(abortReason); - await cancelled; + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + pullStarted, + pending, + "source-map response pull", + ); + controller.abort(abortReason); + const result = await pending; + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.cause).toBe(abortReason); + await expect.poll(() => cancelled).toBe(true); + }, [ + () => controller.abort(), + async () => { + await pending; + }, + ]); }); }); @@ -810,10 +821,7 @@ describe("completed source-map cancellation", () => { const pullStarted = new Promise((resolve) => { markPullStarted = resolve; }); - let markCancelled: (() => void) | undefined; - const cancelled = new Promise((resolve) => { - markCancelled = resolve; - }); + let cancelled = false; const secondBody = new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode("{")); @@ -822,7 +830,7 @@ describe("completed source-map cancellation", () => { markPullStarted?.(); }, cancel() { - markCancelled?.(); + cancelled = true; }, }); const firstBody = JSON.stringify({ @@ -857,32 +865,42 @@ describe("completed source-map cancellation", () => { }, ); - await pullStarted; - controller.abort(reason); - - const result = await pending; - expect(result.ok).toBe(false); - if (result.ok) return; - const failure = result.error; - expect(failure).toBeInstanceOf(AnalysisCancelledError); - expect(failure.cause).toBe(reason); - expect(failure.partialObservation).toMatchObject({ - kind: "web-source-map-fetch", - source_maps: { - requested: 2, - processed: 1, - items: [ - { - status: "included", - artifact: { - text: firstBody, - sha256: createHash("sha256").update(firstBody).digest("hex"), + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + pullStarted, + pending, + "second source-map response pull", + ); + controller.abort(reason); + const result = await pending; + expect(result.ok).toBe(false); + if (result.ok) return; + const failure = result.error; + expect(failure).toBeInstanceOf(AnalysisCancelledError); + expect(failure.cause).toBe(reason); + expect(failure.partialObservation).toMatchObject({ + kind: "web-source-map-fetch", + source_maps: { + requested: 2, + processed: 1, + items: [ + { + status: "included", + artifact: { + text: firstBody, + sha256: createHash("sha256").update(firstBody).digest("hex"), + }, }, - }, - ], + ], + }, + }); + await expect.poll(() => cancelled).toBe(true); + }, [ + () => controller.abort(), + async () => { + await pending; }, - }); - await cancelled; + ]); }, ); }); diff --git a/tests/boundary/captures/captureProvider.test.ts b/tests/boundary/captures/captureProvider.test.ts index 8195a1fc57..84903df4df 100644 --- a/tests/boundary/captures/captureProvider.test.ts +++ b/tests/boundary/captures/captureProvider.test.ts @@ -16,6 +16,7 @@ import { Cli } from "incur"; import { registerCaptureCommands } from "../../../src/cli/captureCommands.js"; import { silentLogger } from "../../../src/logger.js"; import { analysisErrorRemediationAction } from "../../../src/domain/analysisErrorPresentation.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; // A command-boundary simulator: real process ownership/filesystem, not real TShark coverage. const fixture = async (mode = "ok", failCleanup = false) => { @@ -205,9 +206,12 @@ it("returns canonical alias output through the public packet-capture CLI", async ], { stdout: (text) => (stdout += text), exit: () => {} }, ); - const result: unknown = JSON.parse(stdout); - expect(JSON.stringify(result)).toContain('"bootp.type"'); - expect(JSON.stringify(result)).toContain('"dhcp.type":["1"]'); + const evidence = parseEvidence(JSON.parse(stdout)); + const report = packetCaptureSchema.parse(evidence.normalized_result); + expect(report.selection.fields).toEqual(["bootp.type"]); + expect(report.packets).toEqual([ + { "frame.number": ["1"], "dhcp.type": ["1"] }, + ]); }); it("bounds admission without queuing additional snapshots or delaying cancellation", async () => { @@ -634,8 +638,14 @@ it("discovers installed protocols and field definitions without a capture", asyn { records: [ { + kind: "field", + name: "Other Vendor", abbreviation: "usb.idVendor", field_type: "FT_UINT32", + parent_protocol: "usb", + display_base: "BASE_HEX", + bitmask: "0x0", + description: "Second registration", source_line: 3, }, ], diff --git a/tests/boundary/captures/fieldDiscovery.test.ts b/tests/boundary/captures/fieldDiscovery.test.ts index 79a27f7bfb..3bc664b76c 100644 --- a/tests/boundary/captures/fieldDiscovery.test.ts +++ b/tests/boundary/captures/fieldDiscovery.test.ts @@ -4,7 +4,7 @@ import { parseFieldDiscovery } from "../../../src/captures/parseFieldDiscovery.j it("keeps duplicate registrations and source ordinals instead of overwriting abbreviations", () => { const text = - "P\tUSB\tusb\r\nF\tVendor\tusb.idVendor\tFT_UINT16\tusb\tBASE_HEX\t0x0\t\r\nF\tVendor\tusb.idVendor\tFT_UINT32\tusb\tBASE_HEX\t0x0\t"; + "P\tUSB\tusb\r\nF\tVendor\tusb.idVendor\tFT_UINT16\tusb\tBASE_HEX\t0x0\t\r\nF\tOther Vendor\tusb.idVendor\tFT_UINT32\tusb\tBASE_DEC\t0xff00\tSecond registration"; const result = parseFieldDiscovery( text, inspectPacketFieldsInputSchema.parse({ @@ -13,12 +13,17 @@ it("keeps duplicate registrations and source ordinals instead of overwriting abb offset: 1, }), ); - expect(result.records).toMatchObject([ + expect(result.records).toEqual([ { source_line: 3, + kind: "field", + name: "Other Vendor", abbreviation: "usb.idVendor", field_type: "FT_UINT32", - description: "", + parent_protocol: "usb", + display_base: "BASE_DEC", + bitmask: "0xff00", + description: "Second registration", }, ]); expect(result.output_window).toEqual({ diff --git a/tests/boundary/captures/packetOutput.test.ts b/tests/boundary/captures/packetOutput.test.ts index 4b32ea6e18..afbfda9dd5 100644 --- a/tests/boundary/captures/packetOutput.test.ts +++ b/tests/boundary/captures/packetOutput.test.ts @@ -1,5 +1,7 @@ import { expect, it } from "vitest"; +import { ZodError } from "zod"; import { parsePacketOutput } from "../../../src/captures/parsePacketOutput.js"; +import { AnalysisOutputError } from "../../../src/domain/analysisErrorCore.js"; it("preserves repeated field values and missing fields while discarding export index metadata", () => { const output = JSON.stringify([ @@ -21,21 +23,61 @@ it("preserves repeated field values and missing fields while discarding export i ]); }); -it.each( - [ - [{ "ip.addr": ["192.0.2.1"] }], - [{ "frame.number": ["1"], "ip.addr": "192.0.2.1" }], - [{ "frame.number": ["0"] }], - [{ "frame.number": ["1", "2"] }], - [{ "frame.number": ["2"] }, { "frame.number": ["2"] }], - [{ "frame.number": ["2"] }, { "frame.number": ["1"] }], - ].map((layers) => [layers]), -)("rejects malformed, repeated or reordered producer frames: %j", (layers) => { - expect(() => +it.each([ + { + layers: { "ip.addr": ["192.0.2.1"] }, + issue: { + code: "invalid_type", + path: [0, "_source", "layers", "frame.number"], + }, + }, + { + layers: { "frame.number": ["1"], "ip.addr": "192.0.2.1" }, + issue: { code: "invalid_type", path: [0, "_source", "layers", "ip.addr"] }, + }, + { + layers: { "frame.number": ["0"] }, + issue: { + code: "invalid_format", + path: [0, "_source", "layers", "frame.number", 0], + }, + }, + { + layers: { "frame.number": ["1", "2"] }, + issue: { code: "too_big", path: [0, "_source", "layers", "frame.number"] }, + }, +])("rejects malformed producer fields: $layers", ({ layers, issue }) => { + let failure: unknown; + try { + parsePacketOutput(JSON.stringify([{ _source: { layers } }])); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(ZodError); + if (!(failure instanceof ZodError)) throw failure; + expect(failure.issues).toEqual( + expect.arrayContaining([expect.objectContaining(issue)]), + ); +}); + +it.each([ + [["2"], ["2"]], + [["2"], ["1"]], +])("rejects repeated or reordered producer frames: %j then %j", (...frames) => { + let failure: unknown; + try { parsePacketOutput( - JSON.stringify(layers.map((value) => ({ _source: { layers: value } }))), - ), - ).toThrow(); + JSON.stringify( + frames.map((frame) => ({ + _source: { layers: { "frame.number": frame } }, + })), + ), + ); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(AnalysisOutputError); + expect(failure).toMatchObject({ operation: "inspect_packet_capture" }); }); it("preserves canonical field names emitted for a selected alias", () => { diff --git a/tests/boundary/cli/cliInputSchemaRequired.test.ts b/tests/boundary/cli/cliInputSchemaRequired.test.ts index 78117cbab1..1a65c77a51 100644 --- a/tests/boundary/cli/cliInputSchemaRequired.test.ts +++ b/tests/boundary/cli/cliInputSchemaRequired.test.ts @@ -247,6 +247,7 @@ describe("CLI advertised input schemas", () => { cli as unknown as CliInstance, "group", ); + expect(advertised.args?.properties.scope).toBeDefined(); expect(advertised.args?.required ?? []).not.toContain("scope"); }); }); diff --git a/tests/boundary/cli/cliOptionAliases.test.ts b/tests/boundary/cli/cliOptionAliases.test.ts index a829094f2d..4c6238230b 100644 --- a/tests/boundary/cli/cliOptionAliases.test.ts +++ b/tests/boundary/cli/cliOptionAliases.test.ts @@ -1,8 +1,12 @@ import { Cli } from "incur"; import { describe, expect, it } from "vitest"; +import { z } from "zod"; import { createCli } from "../../../src/cli.js"; -import { createCliInventory } from "../../../scripts/lib/catalog-cli.mjs"; +import { + cliCommandOptionNames, + createCliInventory, +} from "../../../scripts/lib/catalog-cli.mjs"; type CliInstance = ReturnType; type CommandMap = NonNullable>; @@ -59,9 +63,20 @@ describe("CLI option aliases", () => { // Incur parses `-abc` as stacked one-letter flags, so help must never // advertise a multi-letter single-dash spelling. const help = await serve(cli, [command, "--help"]); + expect(help.exitCode, `${command} --help`).toBe(0); + expect(help.stdout, `${command} --help`).not.toBe(""); for (const match of help.stdout.matchAll(/(?:^|[\s,])-([a-z][\w-]+)/gmu)) invalid.push(`${command} help: -${match[1] ?? ""}`); const options = /^Options:\n((?: {2}.*\n?)*)/mu.exec(help.stdout)?.[1]; + const entry = registeredCommands.get(cli)?.get(command); + expect(entry, `${command} registration`).toBeDefined(); + const schema: unknown = entry && Reflect.get(entry, "options"); + if (schema !== undefined) + expect(schema, `${command} options schema`).toBeInstanceOf(z.ZodObject); + if (cliCommandOptionNames(cli, command).length > 0) { + expect(options, `${command} --help options`).toBeDefined(); + expect(options, `${command} --help option entries`).toMatch(/^ --/mu); + } for (const line of options?.split("\n") ?? []) if (/^ {2}--\S+(?: <[^>]+>)?\s*$/u.test(line)) undescribed.push(`${command} ${line.trim()}`); diff --git a/tests/boundary/cli/cliOutput.test.ts b/tests/boundary/cli/cliOutput.test.ts index 0d0401ef4f..79d6279bf8 100644 --- a/tests/boundary/cli/cliOutput.test.ts +++ b/tests/boundary/cli/cliOutput.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -651,8 +652,7 @@ describe("compiled CLI output boundary", () => { exitCode: 1, json: { code: "VALIDATION_ERROR", - message: - "REA could not read the command arguments. Check the named argument and its expected type, then try again.", + message: expect.stringMatching(/\S/u), fieldErrors: [ { path: "path", @@ -751,11 +751,17 @@ describe("compiled CLI artifact diagnostics", () => { arguments: ["--json", "inspect-artifact", archive], }); expect(result.exitCode).toBe(1); - const output = JSON.stringify(result.json); - expect(output).toContain('"logical_path":"main.js"'); - expect(output).toMatch(/"declared_sha256":"[a-f0-9]{64}"/u); - expect(output).toMatch(/"calculated_sha256":"[a-f0-9]{64}"/u); - expect(output).toContain('"unpacked":true'); + const sha256 = (value: string) => + createHash("sha256").update(value).digest("hex"); + expect(result.json).toMatchObject({ + code: "artifact_integrity_mismatch", + details: { + logical_path: "main.js", + declared_sha256: sha256("console.log('ok');\n"), + calculated_sha256: sha256("changed();\n"), + unpacked: true, + }, + }); }, CLI_INTEGRATION_TIMEOUT_MS, ); diff --git a/tests/boundary/cli/cliWorkerCompletion.test.ts b/tests/boundary/cli/cliWorkerCompletion.test.ts index f0ed533307..dc879f65a0 100644 --- a/tests/boundary/cli/cliWorkerCompletion.test.ts +++ b/tests/boundary/cli/cliWorkerCompletion.test.ts @@ -136,8 +136,23 @@ processTest( "publishes one fatal CLI response and does not dispatch commands queued behind it", async ({ processes }) => { const root = await createTestTempDirectory("rea-cli-worker-fatal-"); - const result = await processes.run(process.execPath, [worker], { - input: Buffer.concat([ + const child = processes.spawn(process.execPath, [worker]); + const closed = new Promise((resolve) => { + child.once("close", () => resolve()); + }); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf8"); + }); + const frames: unknown[] = []; + child.stdout.on( + "data", + createCliSessionFrameReader((frame: unknown) => { + frames.push(frame); + }), + ); + child.stdin.end( + Buffer.concat([ encodeCliSessionFrame({ id: 1, arguments: null, cwd: root, env: {} }), encodeCliSessionFrame({ id: 2, @@ -146,15 +161,17 @@ processTest( env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" }, }), ]), - timeoutMs: 15_000, - }); - expect(result.signal, result.stderr).toBeNull(); - expect(result.exitCode, result.stderr).toBe(1); - expect(result.stderr).toContain( + ); + expect(await waitForExit(child, 15_000), stderr).toBe(true); + await closed; + expect(child.signalCode, stderr).toBeNull(); + expect(child.exitCode, stderr).toBe(1); + expect(stderr).toContain( "CLI session command is missing its arguments, cwd, or environment", ); - expect(result.stdout).toContain('"id":1'); - expect(result.stdout).toContain('"fatal":true'); - expect(result.stdout).not.toContain('"id":2'); + expect(frames).toEqual([ + { ready: true }, + expect.objectContaining({ id: 1, fatal: true, exitCode: 1 }), + ]); }, ); diff --git a/tests/boundary/cli/developmentTests.test.ts b/tests/boundary/cli/developmentTests.test.ts index 93e3d9ccca..c29a19dcde 100644 --- a/tests/boundary/cli/developmentTests.test.ts +++ b/tests/boundary/cli/developmentTests.test.ts @@ -6,7 +6,9 @@ import { symlink, writeFile, } from "node:fs/promises"; +import { spawn } from "node:child_process"; import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; import { describe, expect, it } from "vitest"; @@ -52,8 +54,11 @@ const fixture = async () => { for (const path of [ "scripts/run-development-tests.mjs", "scripts/run-exact-tests.mjs", + "scripts/run-exclusive.mjs", + "scripts/run-exclusive-command.mjs", "scripts/lib/development-tests.mjs", "scripts/lib/test-projects.mjs", + "scripts/lib/windows-owned-command.mjs", ]) await copyFile(join(process.cwd(), path), join(root, path)); await symlink( @@ -267,11 +272,18 @@ describe("development test runner exact selection", () => { it("preserves all configured projects for an exact file", async () => { const root = await fixture(); await writeFile( - join(root, "vitest.config.mjs"), - 'export default { test: { projects: ["domain", "evaluation"].map(name => ({ test: { name, include: ["src/value.test.ts"] } })) } };\n', + join(root, "src/value.test.ts"), + 'import { it } from "vitest"; import { writeFileSync } from "node:fs"; it("records the project", () => writeFileSync(`selected-${process.env.REA_TEST_PROJECT}.txt`, "ran"));\n', ); - const result = await testCommand(root, "focused", "src/value.test.ts"); - expect(result.stdout).toMatch(/Test Files\s+2 passed/u); + await writeFile( + join(root, "vitest.config.mjs"), + 'export default { test: { projects: ["domain", "evaluation"].map(name => ({ test: { name, env: { REA_TEST_PROJECT: name }, include: ["src/value.test.ts"] } })) } };\n', + ); + await testCommand(root, "focused", "src/value.test.ts"); + for (const name of ["domain", "evaluation"]) + expect(await readFile(join(root, `selected-${name}.txt`), "utf8")).toBe( + "ran", + ); }); it("fails an exact file with no tests even when another selected file passes", async () => { @@ -287,6 +299,77 @@ describe("development test runner exact selection", () => { }); }); +describe("development test runner cancellation", () => { + it.runIf(process.platform !== "win32")( + "waits for the owned test process group on cancellation", + async () => { + const root = await fixture(); + const ready = join(root, "descendant-ready"); + const activity = join(root, "descendant-activity"); + const release = join(root, "descendant-release"); + await writeFile( + join(root, "descendant.cjs"), + ` + const fs = require("node:fs"); + process.on("SIGTERM", () => {}); + fs.writeFileSync("descendant-activity", "active\\n"); + fs.writeFileSync("descendant-ready", "ready"); + const timer = setInterval(() => { + if (fs.existsSync("descendant-release")) { clearInterval(timer); clearTimeout(deadline); return; } + fs.appendFileSync("descendant-activity", "active\\n"); + }, 20); + const deadline = setTimeout(() => clearInterval(timer), 10000); + `, + ); + await writeFile( + join(root, "src/value.test.ts"), + 'import { it } from "vitest"; import { spawn } from "node:child_process"; it("waits", async () => { spawn(process.execPath, ["descendant.cjs"], { stdio: "ignore" }); await new Promise(() => {}); }, 30000);', + ); + const runner = spawn( + process.execPath, + ["scripts/run-development-tests.mjs", "local", "src/value.test.ts"], + { cwd: root, stdio: "ignore" }, + ); + const closed = new Promise<{ + code: number | null; + signal: string | null; + }>((resolve) => + runner.once("exit", (code, signal) => resolve({ code, signal })), + ); + try { + let started = false; + for (let attempt = 0; attempt < 200; attempt += 1) { + try { + await readFile(ready); + started = true; + break; + } catch (cause) { + if ( + !(cause instanceof Error) || + !("code" in cause) || + cause.code !== "ENOENT" + ) + throw cause; + await delay(50); + } + } + expect(started).toBe(true); + runner.kill("SIGTERM"); + expect(await closed).toEqual({ code: 143, signal: null }); + const stopped = await readFile(activity, "utf8"); + await delay(100); + expect(await readFile(activity, "utf8")).toBe(stopped); + } finally { + await writeFile(release, "stop"); + if (runner.exitCode === null && runner.signalCode === null) + runner.kill("SIGTERM"); + await closed; + } + }, + 20_000, + ); +}); + describe("development test runner public CLI", () => { it("runs an explicitly selected source test on a clean tree without build artifacts", async () => { const root = await fixture(); diff --git a/tests/boundary/cli/fridaCli.test.ts b/tests/boundary/cli/fridaCli.test.ts index a68caee857..f6f3c3c7a2 100644 --- a/tests/boundary/cli/fridaCli.test.ts +++ b/tests/boundary/cli/fridaCli.test.ts @@ -407,7 +407,6 @@ it("keeps Frida script-file identity in original bytes when UTF-8 decoding repla .update(decodedSource, "utf8") .digest("hex"); const outputText = JSON.stringify(output); - expect(outputText).toContain(rawDigest); if (name === "ff.js" || name === "fe.js") { expect(rawDigest).not.toBe(loadedSourceDigest); expect(output).toMatchObject({ @@ -419,10 +418,10 @@ it("keeps Frida script-file identity in original bytes when UTF-8 decoding repla loaded_source_sha256: loadedSourceDigest, }, }, + limitations: expect.arrayContaining([ + expect.stringMatching(/replacement|original bytes/iu), + ]), }); - expect(outputText).toMatch( - /limitations.*(?:replacement|original bytes)/iu, - ); } else { expect(output).toMatchObject({ normalized_result: { source_sha256: rawDigest }, diff --git a/tests/boundary/cli/githubActionsExample.test.ts b/tests/boundary/cli/githubActionsExample.test.ts index 7a895d2e7c..a8e7df0a38 100644 --- a/tests/boundary/cli/githubActionsExample.test.ts +++ b/tests/boundary/cli/githubActionsExample.test.ts @@ -197,7 +197,7 @@ describe.skipIf(process.platform === "win32")( const result = await run(analyze); expect(result.signal).toBeNull(); const raw = await readFile(join(reports, "analysis.json"), "utf8"); - expect(await readdir(reports)).toEqual([ + expect((await readdir(reports)).sort()).toEqual([ "analysis.json", "analysis.stderr", "exit-code.txt", diff --git a/tests/boundary/cli/guidanceCli.test.ts b/tests/boundary/cli/guidanceCli.test.ts index b80f737330..f169bd17eb 100644 --- a/tests/boundary/cli/guidanceCli.test.ts +++ b/tests/boundary/cli/guidanceCli.test.ts @@ -98,6 +98,7 @@ it("reports no matches and rejects invalid CLI inputs", async () => { it("continues a CLI search with a query- and bundle-bound cursor", async () => { const first = await run(["guidance", "--query", "module", "--limit", "1"]); + expect(first.exitCode, first.stdout).toBe(0); const firstPage = JSON.parse(first.stdout) as { readonly next_cursor?: string; readonly matches: readonly { readonly locator: string }[]; @@ -116,6 +117,8 @@ it("continues a CLI search with a query- and bundle-bound cursor", async () => { const nextPage = JSON.parse(next.stdout) as { readonly matches: readonly { readonly locator: string }[]; }; + expect(firstPage.matches[0]?.locator).toEqual(expect.any(String)); + expect(nextPage.matches[0]?.locator).toEqual(expect.any(String)); expect(nextPage.matches[0]?.locator).not.toBe(firstPage.matches[0]?.locator); const wrongQuery = await run([ "guidance", diff --git a/tests/boundary/cli/idaDoctorRegistration.test.ts b/tests/boundary/cli/idaDoctorRegistration.test.ts index 592827b707..46c005e2a6 100644 --- a/tests/boundary/cli/idaDoctorRegistration.test.ts +++ b/tests/boundary/cli/idaDoctorRegistration.test.ts @@ -60,7 +60,7 @@ cliTest.skipIf(process.platform !== "darwin").for([true, false])( }), ); const registrationBytes = await readFile(registration); - const before = await readdir(physical); + const before = (await readdir(physical)).sort(); const result = await cli.run({ arguments: ["doctor", "--provider", "ida", "--json"], environment: { HOME: root, REA_IDA_MCP_CONFIG: registration }, @@ -85,7 +85,7 @@ cliTest.skipIf(process.platform !== "darwin").for([true, false])( expect( Buffer.compare(await readFile(registration), registrationBytes), ).toBe(0); - expect(await readdir(physical)).toEqual(before); + expect((await readdir(physical)).sort()).toEqual(before); if (usable) { expect(await readdir(parent)).toEqual([]); expect(await readFile(lexical, "utf8")).toBe("lexical parent is a file"); diff --git a/tests/boundary/cli/idaWorkspaceDiscovery.test.ts b/tests/boundary/cli/idaWorkspaceDiscovery.test.ts index a16095405d..78f90d2d81 100644 --- a/tests/boundary/cli/idaWorkspaceDiscovery.test.ts +++ b/tests/boundary/cli/idaWorkspaceDiscovery.test.ts @@ -18,7 +18,7 @@ cliTest.skipIf(process.platform === "win32")( await writeFile(command, `#!/bin/sh\nprintf launched > '${marker}'\n`); await chmod(command, 0o700); const registration = join(root, "ida.json"); - const before = await readdir(root); + const before = (await readdir(root)).sort(); for (const [workspaceRoot, usable] of [ [join(root, "missing"), false], [file, false], @@ -94,9 +94,9 @@ cliTest.skipIf(process.platform === "win32")( ]), }); } - expect((await readdir(root)).filter((name) => name !== "ida.json")).toEqual( - before, - ); + expect( + (await readdir(root)).filter((name) => name !== "ida.json").sort(), + ).toEqual(before); await expect(access(marker)).rejects.toMatchObject({ code: "ENOENT" }); }, ); diff --git a/tests/boundary/cli/jebCli.test.ts b/tests/boundary/cli/jebCli.test.ts index 545696eef3..2ec76b0fa8 100644 --- a/tests/boundary/cli/jebCli.test.ts +++ b/tests/boundary/cli/jebCli.test.ts @@ -91,6 +91,7 @@ it.each( tool: "open_project", argv: ["open-jeb-project", "sample.apk", "--json"], input: { path: "sample.apk" }, + providerInput: { file_path: "sample.apk" }, }, ].flatMap((request) => ["metadata-refusal", "primary-refusal", "success"].map((stage) => ({ @@ -100,7 +101,7 @@ it.each( ), )( "preserves $operation attribution and primary payload with $stage", - async ({ operation, tool, argv, input, stage }) => { + async ({ operation, tool, argv, input, providerInput, stage }) => { const refusal = { success: false, message: "Selected stage refused", @@ -158,6 +159,10 @@ it.each( expect(fixture.calls).toEqual( stage === "primary-refusal" ? [tool] : [tool, "get_client_information"], ); + expect(fixture.requests[0]).toEqual({ + name: tool, + arguments: providerInput ?? input, + }); } finally { await processes.shutdown(); await fixture.close(); diff --git a/tests/boundary/cli/jsonInputStatus.test.ts b/tests/boundary/cli/jsonInputStatus.test.ts index bd5033f2c3..bdbec616f5 100644 --- a/tests/boundary/cli/jsonInputStatus.test.ts +++ b/tests/boundary/cli/jsonInputStatus.test.ts @@ -47,7 +47,10 @@ describe("compiled CLI JSON input failure status", () => { expect(result.json).toMatchObject({ code: "invalid_request", category: "invalid_input", - details: { operation }, + details: { + operation, + issues: [{ path: [], reason: "invalid_format", expected: "JSON" }], + }, }); expect(result.stderr).toBe(""); expect(result.exitCode).toBe(1); diff --git a/tests/boundary/cli/managedMemberSummaryCli.test.ts b/tests/boundary/cli/managedMemberSummaryCli.test.ts index e6cb12db7a..f95fb94c1d 100644 --- a/tests/boundary/cli/managedMemberSummaryCli.test.ts +++ b/tests/boundary/cli/managedMemberSummaryCli.test.ts @@ -321,6 +321,14 @@ cliTest( const report = managedMemberInspectionSchema.parse( parent.normalized_result, ); + // The fixture declares three MemberRef rows. Paging against the produced + // report alone would also pass if the inspector silently dropped rows. + expect(report.member_refs.map(({ token }) => token)).toEqual([ + "0x0a000001", + "0x0a000002", + "0x0a000003", + ]); + expect(summary.summary.reported_counts.member_refs).toBe(3); expect(summary.summary.selection).toEqual({ type_names: typeNames, unmatched_type_names: ["Missing.Type"], diff --git a/tests/boundary/cli/nativeMachoLinkerOptionsCli.test.ts b/tests/boundary/cli/nativeMachoLinkerOptionsCli.test.ts index 2b24a97582..706863201e 100644 --- a/tests/boundary/cli/nativeMachoLinkerOptionsCli.test.ts +++ b/tests/boundary/cli/nativeMachoLinkerOptionsCli.test.ts @@ -47,7 +47,7 @@ describe.skipIf(process.platform !== "darwin")( loadCommands.items .filter((command) => command.kind === "LC_LINKER_OPTION") .map(({ fields }) => fields), - ).toEqual([ + ).toMatchObject([ { cmd: "LC_LINKER_OPTION", cmdsize: 40, diff --git a/tests/boundary/cli/nativeMachoObjectSegmentCli.test.ts b/tests/boundary/cli/nativeMachoObjectSegmentCli.test.ts index 50d720cab9..83eb8fd40f 100644 --- a/tests/boundary/cli/nativeMachoObjectSegmentCli.test.ts +++ b/tests/boundary/cli/nativeMachoObjectSegmentCli.test.ts @@ -32,7 +32,10 @@ describe.skipIf(process.platform !== "darwin")( }); expect(result.exitCode, result.stdout + result.stderr).toBe(0); const inspected = evidenceSchema.parse(result.json).normalized_result; - expect(inspected.load_commands.items[0]?.fields.segname).toBe(""); + const segment = inspected.load_commands.items.find( + ({ kind }) => kind === "LC_SEGMENT_64" || kind === "LC_SEGMENT", + ); + expect(segment?.fields["segname #1"]).toBe(""); expect(inspected.segments).toMatchObject({ items: [ { diff --git a/tests/boundary/cli/reverseEngineeringCommands.test.ts b/tests/boundary/cli/reverseEngineeringCommands.test.ts index c49d9f9833..ee9a7d5eab 100644 --- a/tests/boundary/cli/reverseEngineeringCommands.test.ts +++ b/tests/boundary/cli/reverseEngineeringCommands.test.ts @@ -10,7 +10,6 @@ import { AnalysisTimeoutError, AnalysisOutputError, } from "../../../src/domain/analysisErrorCore.js"; -import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { createEvidence, evidenceSchema, @@ -347,7 +346,9 @@ it("preserves an independent timeout after cleanup succeeds", async () => { }, }); expect(error.cleanupIncomplete).toBe(false); - expect(projectAnalysisError(error).details).not.toHaveProperty("cleanup"); + expect((result as { details: unknown }).details).not.toHaveProperty( + "cleanup", + ); }); it("does not project cancellation cleanup as incomplete after a successful retry", async () => { @@ -362,11 +363,12 @@ it("does not project cancellation cleanup as incomplete after a successful retry }), }; - await runObjdump(service); + const result = await runObjdump(service); expect(error.cleanupIncomplete).toBe(false); - expect(projectAnalysisError(error).details).toMatchObject({ - cleanup: "complete", + expect(result).toMatchObject({ + code: "cancelled", + details: { cleanup: "complete" }, }); }); diff --git a/tests/boundary/cli/sharedJavaScriptCliSession.test.ts b/tests/boundary/cli/sharedJavaScriptCliSession.test.ts index 6ef66cc142..2b23dfd24a 100644 --- a/tests/boundary/cli/sharedJavaScriptCliSession.test.ts +++ b/tests/boundary/cli/sharedJavaScriptCliSession.test.ts @@ -899,7 +899,10 @@ test("closes the real JavaScript CLI worker through its resource cleanup", async expect(worker?.exitCode).toBeNull(); expect(worker?.signalCode).toBeNull(); await session.close(); - expect(worker?.exitCode).toBe(0); + expect( + worker?.exitCode, + `worker signal after cleanup: ${String(worker?.signalCode)}`, + ).toBe(0); expect(worker?.signalCode).toBeNull(); expect(session.processId()).toBeUndefined(); }); diff --git a/tests/boundary/evm/evmLimiterAvailability.test.ts b/tests/boundary/evm/evmLimiterAvailability.test.ts index ec1e1d5655..d8ad789adf 100644 --- a/tests/boundary/evm/evmLimiterAvailability.test.ts +++ b/tests/boundary/evm/evmLimiterAvailability.test.ts @@ -1,11 +1,13 @@ import { expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import { buildCapabilityInventory } from "../../../src/application/CapabilityInventory.js"; import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; import { EVM_PRLIMIT_COMMAND_DEFAULT, evmLimiterAvailability, } from "../../../src/evm/EvmoleInterfaceProvider.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; const linux = { platform: "linux" as const, arch: "x64" }; const status = { open: false, capabilities: [] }; @@ -148,15 +150,22 @@ it("yields the event loop while the filesystem owner probe is pending", async () }, }, ); - await started; - let timerRan = false; - setImmediate(() => { - timerRan = true; - }); - await new Promise((resolve) => setImmediate(resolve)); - expect(timerRan).toBe(true); - release(); - expect(await probe).toMatchObject({ status: "available" }); + await withVerifierCleanup(async () => { + await awaitAcknowledgement(started, probe, "filesystem access"); + let timerRan = false; + setImmediate(() => { + timerRan = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(timerRan).toBe(true); + release(); + expect(await probe).toMatchObject({ status: "available" }); + }, [ + release, + async () => { + await probe; + }, + ]); }); it.each(["access", "stat"] as const)( @@ -194,17 +203,20 @@ it.each(["access", "stat"] as const)( }, ); - try { - await started; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(started, probe, `${stage} filesystem probe`); controller.abort(abortReason); release(); const error = await probe.catch((cause: unknown) => cause); expect(error).toBeInstanceOf(AnalysisCancelledError); expect(error).toMatchObject({ operation: "binary_session" }); expect((error as AnalysisCancelledError).cause).toBe(abortReason); - } finally { - release(); - } + }, [ + release, + async () => { + await probe.catch(() => undefined); + }, + ]); }, ); diff --git a/tests/boundary/filesystem/android/application.test.ts b/tests/boundary/filesystem/android/application.test.ts index 4f4bb04408..fe7ce32a32 100644 --- a/tests/boundary/filesystem/android/application.test.ts +++ b/tests/boundary/filesystem/android/application.test.ts @@ -19,6 +19,7 @@ import { import { projectAndroidApplicationEvidence } from "../../../../src/application/android/AndroidApplicationService.js"; import { androidApplicationProjectionResultSchema } from "../../../../src/domain/android/androidApplication.js"; +import { artifactInventoryResultSchema } from "../../../../src/domain/artifactGraph.js"; describe("Android application projection", () => { it("projects deterministic APK components and explicit bridge hypotheses", async () => { @@ -321,11 +322,19 @@ describe("case-distinct APK resource names", () => { ); expect(projection.components.manifests).toHaveLength(1); expect(projection.components.dex).toHaveLength(1); - expect(JSON.stringify(inventory)).toContain("res/2F.xml"); - expect(JSON.stringify(inventory)).toContain("res/2f.xml"); - expect(JSON.stringify(inventory)).toContain( - "English (en-US) Unicode case folding", + const graph = artifactInventoryResultSchema.parse( + inventory.normalized_result, ); + for (const logicalPath of ["res/2F.xml", "res/2f.xml"]) { + expect(graph.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: logicalPath, + limitations: expect.arrayContaining([ + expect.stringContaining("English (en-US) Unicode case folding"), + ]), + }), + ); + } }); }); diff --git a/tests/boundary/filesystem/asarReaderCleanup.test.ts b/tests/boundary/filesystem/asarReaderCleanup.test.ts index 15202aab31..47acbf5b2d 100644 --- a/tests/boundary/filesystem/asarReaderCleanup.test.ts +++ b/tests/boundary/filesystem/asarReaderCleanup.test.ts @@ -12,6 +12,8 @@ import { createPackage } from "@electron/asar"; import { describe, expect, it } from "vitest"; import { AsarArtifactReader } from "../../../src/artifacts/AsarArtifactReader.js"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const CONTENT = "module.exports = 'owned';\n"; @@ -174,12 +176,14 @@ describe("ASAR reader cleanup", () => { }, }); - try { + let opening: ReturnType | undefined; + let closing: ReturnType | undefined; + await withVerifierCleanup(async () => { const entry = await packedMember(reader); pauseMemberOpen = true; - const opening = reader.open(entry); - await memberOpenStarted; - const closing = reader.close(); + opening = reader.open(entry); + await awaitAcknowledgement(memberOpenStarted, opening, "member open"); + closing = reader.close(); releaseMemberOpen(); const output = await opening; const memberHandle = latestHandle(opened); @@ -189,10 +193,16 @@ describe("ASAR reader cleanup", () => { expect(output.destroyed).toBe(true); await expectSnapshotRemoved(memberHandle.fd, root); - } finally { - releaseMemberOpen(); - await reader.close(); - } + }, [ + releaseMemberOpen, + async () => { + await opening; + }, + async () => { + await closing; + }, + () => reader.close(), + ]); }); }); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 05206d096e..1c5b9908a5 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -108,8 +108,30 @@ it.each([ ["bridge/pwndbg/test_core_context.py", "python"], ["tests/support/analysis-efficiency/test_signal_oracle.py", "python"], ["scripts/run-python-tests.py", "python"], + ["scripts/verify-browser-request.mjs", "browser"], + ["scripts/lib/browser-request-e2e.mjs", "browser"], + ["scripts/fixtures/browser-request-site.mjs", "browser"], + ["scripts/lib/browser-nested-source-map-e2e.mjs", "web_source_map"], + ["scripts/lib/browser-runtime-fixture-lifecycle.mjs", "web_source_map"], + ["src/cliBrowserScenarioCommands.ts", "browser"], + ["src/application/BrowserScenarioCaptureService.ts", "browser"], + ["src/composition/browserScenario.ts", "browser"], + ["src/cli/environmentCommands.ts", "browser"], + ["src/application/environment/environmentBuild.ts", "browser"], + ["src/domain/analysisView/environmentCoverageView.ts", "browser"], + ["src/domain/analysisView/scenarioCaptureRecords.ts", "browser"], + ["src/domain/analysisView/recordedResponseBody.ts", "browser"], + ["src/domain/analysisView/recordedExchangeBody.ts", "browser"], + ["src/domain/analysisView/recordedMediaType.ts", "browser"], + ["src/domain/analysisView/webNetworkRouteTemplates.ts", "browser"], + ["src/domain/analysisView/EnvironmentManifestAccumulator.ts", "browser"], + ["src/domain/analysisView/EnvironmentStateAccumulator.ts", "browser"], + ["src/domain/analysisView/WebNetworkProjectionAccumulator.ts", "browser"], ["src/captures/CaptureProvider.ts", "packet_capture"], ["tests/boundary/mcp/packetCaptureMcp.test.ts", "packet_capture"], + ["tests/acceptance/pcapVerifier.test.ts", "packet_capture"], + ["tests/fixtures/pcap/offlineHttpCapture.ts", "packet_capture"], + ["scripts/verify-pcap.mjs", "packet_capture"], ["src/signals/RecordingProvider.ts", "signal_recording"], ["tests/boundary/mcp/signalRecordingMcp.test.ts", "signal_recording"], ["src/process/LinuxResourceLimits.ts", "packet_capture"], @@ -124,6 +146,19 @@ it.each([ }); }); +it.each(["src/domain/webCaptureDiffSchemas.ts", "src/domain/webScreenshot.ts"])( + "selects real browser verification for %s without capture lanes", + async (path) => { + const { directory, base } = await fixture(); + const head = await change(directory, [path]); + expect(await classify(directory, base, head)).toMatchObject({ + browser: true, + packet_capture: false, + signal_recording: false, + }); + }, +); + it("reuses only scheduled qualification and preserves every explicit full request", async () => { const { directory, base } = await fixture(); const reused = await classify(directory, base, base, "schedule", false, true); @@ -1047,7 +1082,7 @@ it.each([ code: true, runtime: true, agentenv: true, - browser: false, + browser: true, package: false, }, ], @@ -1250,37 +1285,40 @@ it.each([ }, ); -it("rejects skipped selected work while allowing unselected lanes to skip", async () => { - const command = fileURLToPath( - new URL("../../../scripts/ci/required.mjs", import.meta.url), - ); - const { CI_FLAGS } = await loadScopes(); - const jobs = { - changes: { - result: "success", - outputs: { - ...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])), - tier: "fast", - evm: "true", +it.each(["evm", "browser"])( + "rejects skipped selected %s work while allowing unselected lanes to skip", + async (scope) => { + const command = fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ); + const { CI_FLAGS } = await loadScopes(); + const jobs = { + changes: { + result: "success", + outputs: { + ...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])), + tier: "fast", + [scope]: "true", + }, }, - }, - static: { result: "success" }, - test: { result: "success" }, - evm: { result: "success" }, - browser: { result: "skipped" }, - }; - const run = (result: string) => - execFileAsync(process.execPath, [command], { - env: { - ...process.env, - CI_JOBS: JSON.stringify({ ...jobs, evm: { result } }), - }, - }); - await expect(run("success")).resolves.toMatchObject({ stderr: "" }); - for (const result of ["failure", "cancelled", "skipped"]) { - await expect(run(result)).rejects.toMatchObject({ code: 1 }); - } -}); + static: { result: "success" }, + test: { result: "success" }, + evm: { result: "skipped" }, + browser: { result: "skipped" }, + }; + const run = (result: string) => + execFileAsync(process.execPath, [command], { + env: { + ...process.env, + CI_JOBS: JSON.stringify({ ...jobs, [scope]: { result } }), + }, + }); + await expect(run("success")).resolves.toMatchObject({ stderr: "" }); + for (const result of ["failure", "cancelled", "skipped"]) { + await expect(run(result)).rejects.toMatchObject({ code: 1 }); + } + }, +); it.each([ [ @@ -1665,6 +1703,17 @@ const stepRuns = z.object({ ), }); +it("runs the browser request journey in the real browser lane", async () => { + const workflow = stepRuns.parse(await workflowFile("real-browser.yml")); + const commands = workflow.jobs["real-browser"]?.steps?.map( + (step) => step.run, + ); + expect(commands).toContain("dbus-run-session -- npm run verify:browser"); + expect(commands).toContain( + "dbus-run-session -- node scripts/verify-browser-request.mjs", + ); +}); + it("owns every explicit native-host test path with the lane that runs it", async () => { const { CI_LANES } = await loadScopes(); const ci = stepRuns.parse(await workflowFile("ci.yml")); diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index f0282a37a4..3b41b3fe7e 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -93,6 +93,51 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and expect(downloads[0]?.with).toEqual({ name, path: "." }); }); +it("requires each named SQLite compatibility test while retaining both directory suites", async () => { + const { scripts } = z + .object({ scripts: z.record(z.string(), z.string()) }) + .parse( + JSON.parse( + await readFile( + new URL("../../../package.json", import.meta.url), + "utf8", + ), + ), + ); + const commands = z + .string() + .parse(scripts["test:sqlite:run"]) + .split("&&") + .map((command) => command.trim().split(/\s+/u)); + const exact = commands.find( + (command) => + command[0] === "node" && + command[1] === "scripts/run-exact-tests.mjs" && + command[2] === "run", + ); + expect(exact).toEqual( + expect.arrayContaining([ + "tests/acceptance/sqliteDatabaseCli.test.ts", + "tests/boundary/mcp/serverLifecycleMcp.test.ts", + ]), + ); + const directories = commands.find( + (command) => command[0] === "vitest" && command[1] === "run", + ); + expect(directories).toEqual( + expect.arrayContaining([ + "src/application/sqlite/", + "tests/boundary/sqlite/", + ]), + ); + const steps = z + .array(z.object({ run: z.string().optional() })) + .parse((await workflow("sqlite-runtime.yml")).jobs.sqlite?.steps); + expect(steps.some((step) => step.run === "npm run test:sqlite:run")).toBe( + true, + ); +}); + it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => { const { jobs } = await workflow(); const CI_FLAGS = await scopeFlags(); diff --git a/tests/boundary/filesystem/ciTestProjects.test.ts b/tests/boundary/filesystem/ciTestProjects.test.ts index 34e9bc60c6..0e6c0baec2 100644 --- a/tests/boundary/filesystem/ciTestProjects.test.ts +++ b/tests/boundary/filesystem/ciTestProjects.test.ts @@ -1,5 +1,5 @@ import { execFileSync } from "node:child_process"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; @@ -112,14 +112,53 @@ describe("canonical test-project ownership", () => { const request = parseDevelopmentTestRequest("focused", [testPath]); expect(developmentTestPlan(request)).toMatchObject({ needsBuild: true }); }); +}); - it("uses canonical selectors in Vitest while retaining independent runner policy", async () => { - const config = await readFile(join(repository, "vitest.config.ts"), "utf8"); - expect(config).toContain( - "[...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]", +describe("Vitest project configuration", () => { + it("uses canonical test-project selectors in Vitest", async () => { + const root = await createTestTempDirectory("rea-vitest-project-selectors-"); + const script = join(root, "project-selectors.mjs"); + await writeFile( + script, + ` + const { default: config } = await import(process.argv[2]); + console.log(JSON.stringify(config.test.projects.map(({ test }) => ({ + name: test.name, + include: test.include, + exclude: test.exclude ?? [], + })))); + `, + ); + const result = await execFileOutput( + process.execPath, + [ + "--import", + "tsx", + script, + pathToFileURL(join(repository, "vitest.config.ts")).href, + ], + { + cwd: repository, + timeout: 10_000, + maxBuffer: 64 * 1024, + stopSignal: "SIGTERM", + }, + ); + const selectors = JSON.parse(result.stdout) as Array<{ + name: string; + include: string[]; + exclude: string[]; + }>; + const canonical = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS].map( + ({ name, include, exclude }) => ({ + name, + include, + exclude: exclude ?? [], + }), + ); + expect(selectors.sort((a, b) => a.name.localeCompare(b.name))).toEqual( + canonical.sort((a, b) => a.name.localeCompare(b.name)), ); - expect(config).toContain("PROJECT_POLICIES"); - expect(config).not.toContain('"src/{contracts,domain}/**/*.test.ts"'); }); it.each([ diff --git a/tests/boundary/filesystem/ciVerifiedPackage.test.ts b/tests/boundary/filesystem/ciVerifiedPackage.test.ts index 57e44ad730..06d0591a81 100644 --- a/tests/boundary/filesystem/ciVerifiedPackage.test.ts +++ b/tests/boundary/filesystem/ciVerifiedPackage.test.ts @@ -38,26 +38,66 @@ it("validates a tarball against the exact checkout and current workflow attempt" it("rejects a workflow SHA that differs from the checked out HEAD", async () => { const fixture = await createFixture(); - await expect(validate(fixture, "c".repeat(40))).rejects.toThrow(); + await expectFailure( + fixture, + validate(fixture, "c".repeat(40)), + "Verified package checked out source SHA mismatch", + ); }); it.each([ - ["wrong checkout SHA", { sourceSha: "c".repeat(40) }], - ["wrong workflow run", { runId: 124 }], - ["future workflow attempt", { runAttempt: 3 }], - ["wrong verifier platform", { platform: "darwin" }], - ["wrong verifier architecture", { architecture: "arm64" }], - ["wrong package digest", { sha256: "c".repeat(64) }], - ["old manifest schema", { schemaVersion: 1 }], - ["wrong package version", { packageVersion: "1.2.4" }], - ["wrong Node version", { nodeVersion: "v24.0.0" }], - ["wrong release channel", { releaseTag: "next" }], - ["invalid release channel", { releaseTag: "latest\noutput=injection" }], - ["invalid package version", { packageVersion: "invalid\noutput=injection" }], -])("rejects %s", async (_label, override) => { + [ + "wrong checkout SHA", + { sourceSha: "c".repeat(40) }, + "Verified package source SHA mismatch", + ], + ["wrong workflow run", { runId: 124 }, "workflow run ID mismatch"], + [ + "future workflow attempt", + { runAttempt: 3 }, + "workflow run attempt is from the future", + ], + [ + "wrong verifier platform", + { platform: "darwin" }, + "verifier platform mismatch", + ], + [ + "wrong verifier architecture", + { architecture: "arm64" }, + "verifier architecture mismatch", + ], + [ + "wrong package digest", + { sha256: "c".repeat(64) }, + "tarball SHA-256 mismatch", + ], + ["old manifest schema", { schemaVersion: 1 }, "manifest schema version"], + [ + "wrong package version", + { packageVersion: "1.2.4" }, + "package version mismatch", + ], + [ + "wrong Node version", + { nodeVersion: "v24.0.0" }, + "Node.js version mismatch", + ], + ["wrong release channel", { releaseTag: "next" }, "release tag mismatch"], + [ + "invalid release channel", + { releaseTag: "latest\noutput=injection" }, + "manifest has an invalid release tag", + ], + [ + "invalid package version", + { packageVersion: "invalid\noutput=injection" }, + "package version mismatch", + ], +])("rejects %s", async (_label, override, diagnostic) => { const fixture = await createFixture({ override }); - await expect(validate(fixture)).rejects.toThrow(); + await expectFailure(fixture, validate(fixture), diagnostic); }); it("accepts an earlier successful attempt from the same workflow run", async () => { @@ -69,20 +109,45 @@ it("accepts an earlier successful attempt from the same workflow run", async () }); it.each([ - ["malformed manifest", "{"], - ["manifest with missing fields", JSON.stringify({ schemaVersion: 2 })], -])("rejects %s", async (_label, contents) => { + [ + "malformed manifest", + "{", + "Could not read verified package manifest: SyntaxError", + ], + [ + "manifest with missing fields", + JSON.stringify({ schemaVersion: 2 }), + "manifest has missing or unknown fields", + ], +])("rejects %s", async (_label, contents, diagnostic) => { const fixture = await createFixture({ manifestContents: contents }); - await expect(validate(fixture)).rejects.toThrow(); + await expectFailure(fixture, validate(fixture), diagnostic); }); it("rejects an absent manifest", async () => { const fixture = await createFixture({ writeManifest: false }); - await expect(validate(fixture)).rejects.toThrow(); + await expectFailure( + fixture, + validate(fixture), + "Could not read verified package manifest: Error: ENOENT", + ); }); +async function expectFailure( + fixture: Fixture, + operation: Promise<{ stdout: string; stderr: string }>, + diagnostic: string, +) { + await expect(operation).rejects.toMatchObject({ + code: 1, + stdout: "", + stderr: expect.stringContaining(diagnostic), + }); + expect(await readFile(fixture.output, "utf8")).toBe(""); +} + async function createFixture({ override = {}, manifestContents, @@ -232,20 +297,31 @@ it.each([ ); it.each([ - ["old schema", { schemaVersion: 1 }], - ["wrong source", { sourceSha: "c".repeat(40) }], - ["wrong run", { runId: 124 }], - ["future attempt", { runAttempt: 3 }], - ["wrong Node", { nodeVersion: "v24.0.0" }], - ["wrong version", { packageVersion: "1.2.4" }], - ["wrong archive", { sha256: "c".repeat(64) }], - ["unknown channel", { releaseTag: "beta\noutput=injection" }], + ["old schema", { schemaVersion: 1 }, "manifest schema version"], + [ + "wrong source", + { sourceSha: "c".repeat(40) }, + "Verified package source SHA mismatch", + ], + ["wrong run", { runId: 124 }, "workflow run ID mismatch"], + [ + "future attempt", + { runAttempt: 3 }, + "workflow run attempt is from the future", + ], + ["wrong Node", { nodeVersion: "v24.0.0" }, "Node.js version mismatch"], + ["wrong version", { packageVersion: "1.2.4" }, "package version mismatch"], + ["wrong archive", { sha256: "c".repeat(64) }, "tarball SHA-256 mismatch"], + [ + "unknown channel", + { releaseTag: "beta\noutput=injection" }, + "manifest has an invalid release tag", + ], ] as const)( "refuses issuer-side %s without installed dependencies", - async (_label, override) => { + async (_label, override, diagnostic) => { const fixture = await createFixture({ override }); - await expect(preparePublication(fixture)).rejects.toThrow(); - expect(await readFile(fixture.output, "utf8")).toBe(""); + await expectFailure(fixture, preparePublication(fixture), diagnostic); }, ); diff --git a/tests/boundary/filesystem/clientRegistrationStatus.test.ts b/tests/boundary/filesystem/clientRegistrationStatus.test.ts index 4e76e62b53..269f0ef68a 100644 --- a/tests/boundary/filesystem/clientRegistrationStatus.test.ts +++ b/tests/boundary/filesystem/clientRegistrationStatus.test.ts @@ -192,6 +192,7 @@ describe("client registration environment validation", () => { const byClient = new Map(statuses.map((status) => [status.client, status])); // The failing resolver must not invalidate non-Kimi/Codex registrations. + expect(byClient.has("cursor")).toBe(true); expect(byClient.get("cursor")?.state).not.toBe("invalid"); expect(byClient.get("codex")).toMatchObject({ state: "invalid", diff --git a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts index df6992a22b..a9e3abdef5 100644 --- a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts +++ b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts @@ -647,6 +647,7 @@ it.each([ expect(refreshed.limitations).toEqual( expect.arrayContaining(first.limitations), ); + expect(refreshed.analysis_profile?.digest).toEqual(expect.any(String)); expect(refreshed.analysis_profile?.digest).not.toBe(priorProfile.digest); expect(refreshed.normalized_result).toEqual(first.normalized_result); const starts = [...fixture.starts]; diff --git a/tests/boundary/filesystem/fileHandleChunks.test.ts b/tests/boundary/filesystem/fileHandleChunks.test.ts index 4c823d3d02..e4dd4f80cc 100644 --- a/tests/boundary/filesystem/fileHandleChunks.test.ts +++ b/tests/boundary/filesystem/fileHandleChunks.test.ts @@ -1,6 +1,8 @@ import type { FileHandle } from "node:fs/promises"; import { buffer } from "node:stream/consumers"; import { expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { FileReadCancelled, readFileHandleChunks, @@ -36,21 +38,29 @@ it.each(["complete", "fail"])( .finally(() => { settled = true; }); - await entered; - controller.abort(null); - await new Promise((resolve) => setImmediate(resolve)); - expect(settled).toBe(false); - expect(source.destroyed).toBe(false); - const failure = new Error("controlled native read failure"); - if (outcome === "fail") reject(failure); - else release({ bytesRead: 0, buffer: Buffer.alloc(0) }); - const error = await result; - if (outcome === "fail") expect(error).toBe(failure); - else { - expect(error).toBeInstanceOf(FileReadCancelled); - if (error instanceof FileReadCancelled) expect(error.reason).toBeNull(); - } - expect(read).toHaveBeenCalledTimes(1); + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, result, "descriptor read"); + controller.abort(null); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + expect(source.destroyed).toBe(false); + const failure = new Error("controlled native read failure"); + if (outcome === "fail") reject(failure); + else release({ bytesRead: 0, buffer: Buffer.alloc(0) }); + const error = await result; + if (outcome === "fail") expect(error).toBe(failure); + else { + expect(error).toBeInstanceOf(FileReadCancelled); + if (error instanceof FileReadCancelled) expect(error.reason).toBeNull(); + } + expect(read).toHaveBeenCalledTimes(1); + }, [ + () => controller.abort(null), + () => release({ bytesRead: 0, buffer: Buffer.alloc(0) }), + async () => { + await result; + }, + ]); }, ); diff --git a/tests/boundary/filesystem/flutterBuildProvider.test.ts b/tests/boundary/filesystem/flutterBuildProvider.test.ts index 5cefb09d1d..40aaf9093c 100644 --- a/tests/boundary/filesystem/flutterBuildProvider.test.ts +++ b/tests/boundary/filesystem/flutterBuildProvider.test.ts @@ -398,7 +398,7 @@ it.each([ const workspace = stage === "root" ? null - : (expect.stringMatching(/\/rea-flutter-[^/]+$/u) as unknown); + : (expect.stringMatching(/[/\\]rea-flutter-[^/\\]+$/u) as unknown); const location = { workspace_path: workspace, path: workspace }; expect(projectAnalysisError(result.error)).toMatchObject( code === "ENOSPC" diff --git a/tests/boundary/filesystem/ghidraPspExtension.test.ts b/tests/boundary/filesystem/ghidraPspExtension.test.ts index 0ac7f3564b..dd01e2ba86 100644 --- a/tests/boundary/filesystem/ghidraPspExtension.test.ts +++ b/tests/boundary/filesystem/ghidraPspExtension.test.ts @@ -145,6 +145,7 @@ describe.skipIf(process.platform === "win32")( if (!first.ok) throw first.error; const again = await provider.resolveAnalysisProfile(target); if (!again.ok) throw again.error; + expect(first.value.profile?.digest).toEqual(expect.any(String)); expect(again.value.profile?.digest).toBe(first.value.profile?.digest); expect(first.value.profile?.parameters).toMatchObject({ architecture: "mips", @@ -184,6 +185,7 @@ describe.skipIf(process.platform === "win32")( ); const changed = await provider.resolveAnalysisProfile(target); if (!changed.ok) throw changed.error; + expect(changed.value.profile?.digest).toEqual(expect.any(String)); expect(changed.value.profile?.digest).not.toBe( first.value.profile?.digest, ); diff --git a/tests/boundary/filesystem/inesArtifact.test.ts b/tests/boundary/filesystem/inesArtifact.test.ts index 0307b83f7a..0903caf487 100644 --- a/tests/boundary/filesystem/inesArtifact.test.ts +++ b/tests/boundary/filesystem/inesArtifact.test.ts @@ -133,95 +133,88 @@ it("honors explicit database selection before parsing NES-signature bytes", asyn }); } }); -it.each([true, false])( - "inventories and reconstructs all bytes with trainer=%s", - async (trainer) => { - const root = await createTestTempDirectory("rea-ines-inventory-"); - const path = join(root, "fixture.nes"); - const bytes = inesFixture({ - trainer, - chrUnits: trainer ? 1 : 0, - trailing: Buffer.from("retained tail"), +it("inventories and reconstructs all bytes without trainer or CHR ROM", async () => { + const root = await createTestTempDirectory("rea-ines-inventory-"); + const path = join(root, "fixture.nes"); + const bytes = inesFixture({ + chrUnits: 0, + trailing: Buffer.from("retained tail"), + }); + await writeFile(path, bytes); + const inventory = await inventoryArtifact(path); + expect(inventory.ines?.header).toMatchObject({ + header_hex: bytes.subarray(0, 16).toString("hex"), + mapper: 19, + trainer_present: false, + chr_rom_units_8k: 0, + }); + expect(inventory.ines?.reconstruction).toMatchObject({ + bytes: bytes.length, + sha256: sha256(bytes), + verified: true, + }); + const children = inventory.occurrences.filter( + ({ logical_path }) => logical_path !== ".", + ); + const ordered = [...children].sort( + (a, b) => + (a.source_location?.offset ?? 0) - (b.source_location?.offset ?? 0), + ); + const nodes = new Map( + inventory.nodes.map((node) => [node.artifact_id, node]), + ); + const headerRegion = children.find( + ({ logical_path }) => logical_path === "header.bin", + ); + if ( + headerRegion?.artifact_id === undefined || + headerRegion.artifact_id === null + ) + throw new Error("Missing header region"); + expect(nodes.get(headerRegion.artifact_id)?.format).toBe("file"); + let offset = 0; + for (const occurrence of ordered) { + const range = occurrence.source_location; + if (range === null || occurrence.artifact_id === null) + throw new Error("Missing region identity"); + expect(range.offset).toBe(offset); + const original = bytes.subarray(range.offset, range.offset + range.length); + expect(nodes.get(occurrence.artifact_id)).toMatchObject({ + sha256: sha256(original), + size: range.length, }); - await writeFile(path, bytes); - const inventory = await inventoryArtifact(path); - expect(inventory.ines?.header).toMatchObject({ - header_hex: bytes.subarray(0, 16).toString("hex"), - mapper: 19, - trainer_present: trainer, - chr_rom_units_8k: trainer ? 1 : 0, - }); - expect(inventory.ines?.reconstruction).toMatchObject({ - bytes: bytes.length, - sha256: sha256(bytes), - verified: true, - }); - const children = inventory.occurrences.filter( - ({ logical_path }) => logical_path !== ".", - ); - const ordered = [...children].sort( - (a, b) => - (a.source_location?.offset ?? 0) - (b.source_location?.offset ?? 0), - ); - const nodes = new Map( - inventory.nodes.map((node) => [node.artifact_id, node]), - ); - const headerRegion = children.find( - ({ logical_path }) => logical_path === "header.bin", - ); - if ( - headerRegion?.artifact_id === undefined || - headerRegion.artifact_id === null - ) - throw new Error("Missing header region"); - expect(nodes.get(headerRegion.artifact_id)?.format).toBe("file"); - let offset = 0; - for (const occurrence of ordered) { - const range = occurrence.source_location; - if (range === null || occurrence.artifact_id === null) - throw new Error("Missing region identity"); - expect(range.offset).toBe(offset); - const original = bytes.subarray( - range.offset, - range.offset + range.length, - ); - expect(nodes.get(occurrence.artifact_id)).toMatchObject({ - sha256: sha256(original), - size: range.length, - }); - offset += range.length; - } - expect(offset).toBe(bytes.length); - expect(inventory.ines?.reconstruction.region_order).toEqual( - ordered.map(({ logical_path }) => logical_path), - ); - expect(await inventoryArtifact(path)).toEqual(inventory); + offset += range.length; + } + expect(offset).toBe(bytes.length); + expect(inventory.ines?.reconstruction.region_order).toEqual( + ordered.map(({ logical_path }) => logical_path), + ); + expect(await inventoryArtifact(path)).toEqual(inventory); - const scope = new ArtifactResourceScope(); - const outputRoot = join(root, "extracted"); - try { - const extraction = await extractArtifact({ - inputPath: path, - inputFormat: "ines", - outputRoot, - environment: process.env, - integrityPolicy: "fail", - resourceScope: scope, - }); - expect(extraction.artifacts).toHaveLength(children.length); - const reconstructed = Buffer.concat( - await Promise.all( - ordered.map(({ logical_path }) => - readFile(join(outputRoot, logical_path)), - ), + const scope = new ArtifactResourceScope(); + const outputRoot = join(root, "extracted"); + try { + const extraction = await extractArtifact({ + inputPath: path, + inputFormat: "ines", + outputRoot, + environment: process.env, + integrityPolicy: "fail", + resourceScope: scope, + }); + expect(extraction.artifacts).toHaveLength(children.length); + const reconstructed = Buffer.concat( + await Promise.all( + ordered.map(({ logical_path }) => + readFile(join(outputRoot, logical_path)), ), - ); - expect(reconstructed.equals(bytes)).toBe(true); - } finally { - await scope.close(); - } - }, -); + ), + ); + expect(reconstructed.equals(bytes)).toBe(true); + } finally { + await scope.close(); + } +}); it.each([ { diff --git a/tests/boundary/filesystem/javascript-applications/javascriptApplicationEmptyLiterals.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptApplicationEmptyLiterals.test.ts index a575e41a8a..3923c2acbb 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptApplicationEmptyLiterals.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptApplicationEmptyLiterals.test.ts @@ -93,19 +93,49 @@ it("keeps bridge entry paths stable through worker validation and API seed match nested: { write: nested }, }); `); - const semanticPathIds = new Set( - (output.semantic_graph?.nodes ?? []) - .map(({ properties }) => properties.path_id) - .filter((pathId): pathId is string => typeof pathId === "string"), + const semanticProperties = (output.semantic_graph?.nodes ?? []).map( + ({ properties }) => properties, ); - const applicationPathIds = new Set( - output.graph.nodes.flatMap(({ observations }) => - observations - .map(({ properties }) => properties.path_id) + const applicationProperties = output.graph.nodes.flatMap(({ observations }) => + observations.map(({ properties }) => properties), + ); + const pathIds = (properties: typeof semanticProperties) => + new Set( + properties + .map(({ path_id }) => path_id) .filter((pathId): pathId is string => typeof pathId === "string"), - ), + ); + expect(pathIds(semanticProperties)).toEqual(pathIds(applicationProperties)); + const semanticPaths = semanticProperties.filter( + ({ step_kind }) => step_kind === "terminal", + ); + const applicationPaths = applicationProperties.filter( + ({ step_kind }) => step_kind === "terminal", + ); + const expectedPaths = [ + expect.objectContaining({ + path_id: expect.any(String), + entry_path: ["desktop", "nested.write"], + channel: "literal", + }), + expect.objectContaining({ + path_id: expect.any(String), + entry_path: ["desktop", "nested", "write"], + channel: "nested", + }), + ]; + expect(semanticPaths).toEqual(expect.arrayContaining(expectedPaths)); + expect(applicationPaths).toEqual(expect.arrayContaining(expectedPaths)); + const pathIdentities = (paths: typeof semanticPaths) => + new Set( + paths.map(({ path_id, entry_path, channel }) => + JSON.stringify([path_id, entry_path, channel]), + ), + ); + expect(new Set(semanticPaths.map(({ path_id }) => path_id)).size).toBe(2); + expect(pathIdentities(applicationPaths)).toEqual( + pathIdentities(semanticPaths), ); - expect(semanticPathIds).toEqual(applicationPathIds); const matches = findApplicationFeatureSeeds( output.graph.nodes, diff --git a/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts index 03d3909a96..83519455f4 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptRuntimeReconciliation.test.ts @@ -186,15 +186,36 @@ it("matches renderer, frame, script bytes, and worker without claiming execution expect(result.limitations.join(" ")).toMatch( /not reported as executed|not code reachability/u, ); - expect( - result.graph.edges - .filter(({ relation }) => relation === "observed_as") - .every( - ({ evidence }) => - evidence.authority === "cross-layer-reconciliation" && - evidence.state === "inferred", - ), - ).toBe(true); + const observedAsEdges = result.graph.edges.filter( + ({ relation }) => relation === "observed_as", + ); + expect(observedAsEdges).toHaveLength(2); + for (const [entityKind, basis] of [ + ["script", "content-and-location"], + ["worker", "artifact-path"], + ] as const) { + const reconciliation = result.reconciliations.find( + ({ entity_kind }) => entity_kind === entityKind, + ); + if (reconciliation === undefined || reconciliation.status !== "matched") + throw new TypeError(`Expected matched ${entityKind} reconciliation`); + expect(observedAsEdges).toContainEqual( + expect.objectContaining({ + source_node_id: reconciliation.static_node_id, + target_node_id: reconciliation.runtime_node_id, + relation: "observed_as", + properties: expect.objectContaining({ + reconciliation_id: reconciliation.reconciliation_id, + static_layer_id: reconciliation.static_layer_id, + basis, + }), + evidence: expect.objectContaining({ + authority: "cross-layer-reconciliation", + state: "inferred", + }), + }), + ); + } const worker = result.reconciliations.find( ({ entity_kind: kind }) => kind === "worker", ); diff --git a/tests/boundary/filesystem/javascript-applications/viteOutputDependencies.test.ts b/tests/boundary/filesystem/javascript-applications/viteOutputDependencies.test.ts index dafdf609be..cc4f6ca038 100644 --- a/tests/boundary/filesystem/javascript-applications/viteOutputDependencies.test.ts +++ b/tests/boundary/filesystem/javascript-applications/viteOutputDependencies.test.ts @@ -52,19 +52,45 @@ describe("Vite build-output preload dependencies", () => { resolution_status: "resolved", resolved_path: "assets/lazy.js", }); - expect(JSON.stringify(result.graph)).toContain( - "no ancestor of the importing file holds it", + const missing = result.graph.edges.find( + ({ properties }) => + properties.specifier === "vite-output:assets/missing.js", + ); + expect(missing?.evidence.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("no ancestor of the importing file holds it"), + ]), ); }); it("does not report a dependency as an external package when only the assets directory is selected", async () => { const root = await viteBuild(); const result = await reconstructJavaScriptFixture(join(root, "assets")); - const serialized = JSON.stringify(result.graph); - - expect(serialized).not.toContain('"resolution_status":"external"'); - expect(serialized).toContain( - "A Vite preload dependency is relative to the build output directory", + expect( + result.graph.edges.map(({ properties }) => properties.resolution_status), + ).not.toContain("external"); + expect( + result.graph.nodes.flatMap(({ observations }) => + observations.map(({ properties }) => properties.resolution_status), + ), + ).not.toContain("external"); + // Also guard nested graph metadata so the original graph-wide exclusion + // remains in force if a producer adds another resolution-status location. + expect(JSON.stringify(result.graph)).not.toContain( + '"resolution_status":"external"', + ); + const edges = result.graph.edges.filter( + ({ properties }) => properties.specifier === "vite-output:assets/lazy.js", + ); + expect(edges).toHaveLength(1); + expect(typeof edges[0]?.properties.resolution_status).toBe("string"); + expect(edges[0]?.properties.resolution_status).not.toBe("external"); + expect(edges[0]?.evidence.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining( + "A Vite preload dependency is relative to the build output directory", + ), + ]), ); }); @@ -90,10 +116,11 @@ describe("Vite build-output preload dependencies", () => { resolution_status: "unavailable", candidate_path: "dist/assets/lazy.css", }); - const serialized = JSON.stringify(edge); - expect(serialized).toContain("dist/assets/lazy.css"); - expect(serialized).not.toContain( - "no ancestor of the importing file holds it", + expect(edge?.evidence.limitations).toEqual(expect.any(Array)); + expect(edge?.evidence.limitations).not.toEqual( + expect.arrayContaining([ + expect.stringContaining("no ancestor of the importing file holds it"), + ]), ); }); }); diff --git a/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts index 7ab92903b6..691645d13f 100644 --- a/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts +++ b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts @@ -129,19 +129,6 @@ cliTest( }, ); -it.each(archiveNumberMetadataCases)( - "filesystem preserves numeric meaning for $name", - async (item) => { - const fixture = await keyedArchiveNumberMetadataFixture(item); - const result = await inspectBundleKeyedArchive({ - bundlePath: fixture.root, - targetSha256: fixture.digest, - parameters: { path: "archive.plist" }, - }); - expectKeyedArchiveNumberMetadata(result, item, fixture.digest); - }, -); - cliTest.for(archiveNumberMetadataCases)( "built CLI preserves numeric meaning for $name", async (item, { cli }) => { diff --git a/tests/boundary/filesystem/peLoadConfig.test.ts b/tests/boundary/filesystem/peLoadConfig.test.ts index e252340568..9348d064fd 100644 --- a/tests/boundary/filesystem/peLoadConfig.test.ts +++ b/tests/boundary/filesystem/peLoadConfig.test.ts @@ -57,25 +57,6 @@ describe("PE import stable filesystem boundary", () => { expect(cancelled.error._tag).toBe("AnalysisCancelledError"); }); - it("names the exhausted callback budget", async () => { - const root = await createTestTempDirectory("rea-pe-config-limit-"); - const path = join(root, "app.exe"); - await writeFile(path, peLoadConfigImage()); - const service = createPeLoadConfigService(); - try { - const result = await service.inspect({ path, max_callbacks: 1 }); - if (result.ok) throw new Error("Expected callback budget failure"); - expect(result.error).toMatchObject({ - _tag: "AnalysisResourceConstraintError", - remediationAction: expect.stringContaining( - "Raise max_callbacks above 1", - ), - }); - } finally { - await service.close(); - } - }); - it.each(["control", "close-failure", "close-cancel", "close-both"] as const)( "retains completed PE acquisition through %s", async (mode) => { diff --git a/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts b/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts index 068e415ceb..45cd83da93 100644 --- a/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts +++ b/tests/boundary/filesystem/setupClientConfigCleanupOwnership.test.ts @@ -36,6 +36,8 @@ import { runUninstall, } from "../../../src/application/Uninstall.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); @@ -832,14 +834,14 @@ describe("setup client configuration cleanup ownership", () => { }); let hostCloseFinished = false; - let alignment: Promise | undefined; - try { - alignment = host.clientNeedsConfigure(client, {}, ["rea", "mcp"]); - await readStarted; + const alignment = host.clientNeedsConfigure(client, {}, ["rea", "mcp"]); + let closeResult: ReturnType> | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(readStarted, alignment, "configuration read"); const close = host.close; if (close === undefined) throw new Error("Setup host close is missing"); - const closeResult = close().then((result) => { + closeResult = close().then((result) => { hostCloseFinished = true; return result; }); @@ -860,11 +862,18 @@ describe("setup client configuration cleanup ownership", () => { await expect(alignment).resolves.toBe(true); expect(await closeResult).toBeUndefined(); expect(hostCloseFinished).toBe(true); - } finally { - unblockRead(); - await alignment?.catch(() => undefined); - await host.close?.(); - } + }, [ + unblockRead, + async () => { + await alignment; + }, + async () => { + await closeResult; + }, + async () => { + await host.close?.(); + }, + ]); }); }); diff --git a/tests/boundary/filesystem/setupConfig.test.ts b/tests/boundary/filesystem/setupConfig.test.ts index 9612e19e2d..4a0acf0c7f 100644 --- a/tests/boundary/filesystem/setupConfig.test.ts +++ b/tests/boundary/filesystem/setupConfig.test.ts @@ -666,7 +666,7 @@ it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( "began reading a different", ); expect(await readFile(configPath, "utf8")).toBe(original); - expect(await readdir(home)).toEqual(["client.json", "private"]); + expect((await readdir(home)).sort()).toEqual(["client.json", "private"]); } finally { await chmod(locked, 0o700); } diff --git a/tests/boundary/filesystem/setupSkillCleanupOwnership.test.ts b/tests/boundary/filesystem/setupSkillCleanupOwnership.test.ts index f142fe5edc..f4b2868f0a 100644 --- a/tests/boundary/filesystem/setupSkillCleanupOwnership.test.ts +++ b/tests/boundary/filesystem/setupSkillCleanupOwnership.test.ts @@ -28,6 +28,8 @@ import { systemUninstallHost, } from "../../../src/application/Uninstall.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); @@ -247,11 +249,12 @@ describe("skill regular-file cleanup ownership", () => { return handle; }); - try { - const reading = readSkillFile(manifest, resources); - await readStarted; + const reading = readSkillFile(manifest, resources); + let closing: Promise | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(readStarted, reading, "skill read"); let closed = false; - const closing = resources.close().then(() => { + closing = resources.close().then(() => { closed = true; }); await Promise.resolve(); @@ -260,10 +263,16 @@ describe("skill regular-file cleanup ownership", () => { expect(await reading).toBe("stale managed skill\n"); await closing; expect(closed).toBe(true); - } finally { - unblockRead(); - await resources.close(); - } + }, [ + unblockRead, + async () => { + await reading; + }, + async () => { + await closing; + }, + () => resources.close(), + ]); }); }); diff --git a/tests/boundary/filesystem/setupSkillPublication.test.ts b/tests/boundary/filesystem/setupSkillPublication.test.ts index d99da39e7c..f487694aa0 100644 --- a/tests/boundary/filesystem/setupSkillPublication.test.ts +++ b/tests/boundary/filesystem/setupSkillPublication.test.ts @@ -130,7 +130,7 @@ describe("skill publication ownership", () => { expect(result.error.remediation).not.toContain("Check permissions"); expect(result.error.rollbackFailures).toEqual([]); expect(await readFile(manifest, "utf8")).toBe("concurrent user file\n"); - expect(await readdir(dirname(manifest))).toEqual([ + expect((await readdir(dirname(manifest))).sort()).toEqual([ "SKILL.md", "references", ]); diff --git a/tests/boundary/filesystem/webScriptExport.test.ts b/tests/boundary/filesystem/webScriptExport.test.ts index 7008f15d04..9cbbccf650 100644 --- a/tests/boundary/filesystem/webScriptExport.test.ts +++ b/tests/boundary/filesystem/webScriptExport.test.ts @@ -47,6 +47,7 @@ import { scriptCaptureEvidenceFixture, scriptScenarioFixture, } from "../../fixtures/webScriptCapture.js"; +import { injectPublishedDirectoryCloseFailure } from "../../support/publishedDirectoryCloseFailure.js"; vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); @@ -390,78 +391,72 @@ describe("captured script output owner retry", () => { }); }); - it("returns completed output when retry closes a descriptor after commit", async () => { - const { input, resources } = await setup(); - const createTree = SafeOutputTree.create.bind(SafeOutputTree); - const create = vi - .spyOn(SafeOutputTree, "create") - .mockImplementationOnce(async (outputRoot, platform) => { - const tree = await createTree(outputRoot, platform); - const commit = tree.commit.bind(tree); - vi.spyOn(tree, "commit").mockImplementation(async () => { - await commit(); - throw new Error("injected descriptor close failure"); - }); - return tree; - }); - try { - const result = await exportWebScripts(input, resources); - if (!result.ok) throw result.error; - const published = webScriptExportResultSchema.parse( - result.value.normalized_result, + // Windows commit does not open directory descriptors for fsync. + it.skipIf(process.platform === "win32")( + "returns completed output when retry closes a descriptor after commit", + async () => { + const { input, resources } = await setup(); + const failure = await injectPublishedDirectoryCloseFailure( + input.output_directory, ); - expect(published.manifest.path).toBe( - join(input.output_directory, "manifest.json"), - ); - await expect(access(published.manifest.path)).resolves.toBeUndefined(); - } finally { - create.mockRestore(); - } - }); + try { + const result = await exportWebScripts(input, resources); + if (!result.ok) throw result.error; + const published = webScriptExportResultSchema.parse( + result.value.normalized_result, + ); + expect(published.manifest.path).toBe( + join(input.output_directory, "manifest.json"), + ); + expect(failure.wasInjected()).toBe(true); + expect(failure.owners).toHaveLength(2); + expect(failure.owners[1]).toBe(failure.owners[0]); + expect(failure.owners[0]?.handle.fd).toBe(-1); + await expect(access(published.manifest.path)).resolves.toBeUndefined(); + } finally { + failure.restore(); + } + }, + ); - it("retains published output and its result while cleanup remains unresolved", async () => { - const { input, resources } = await setup(); - const createTree = SafeOutputTree.create.bind(SafeOutputTree); - let cleanupAttempts = 0; - const create = vi - .spyOn(SafeOutputTree, "create") - .mockImplementationOnce(async (outputRoot, platform) => { - const tree = await createTree(outputRoot, platform); - const commit = tree.commit.bind(tree); - vi.spyOn(tree, "commit").mockImplementation(async () => { - await commit(); - throw new Error("injected descriptor close failure"); + it.skipIf(process.platform === "win32")( + "retains published output and its result while cleanup remains unresolved", + async () => { + const { input, resources } = await setup(); + const failure = await injectPublishedDirectoryCloseFailure( + input.output_directory, + true, + ); + try { + const result = await exportWebScripts(input, resources); + if (result.ok) throw new Error("Expected unresolved cleanup"); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.userMessage).not.toContain("Remove the residual"); + expect(result.error.partialObservation).toMatchObject({ + kind: "web-script-export", + result: { + output_directory: input.output_directory, + manifest: { path: join(input.output_directory, "manifest.json") }, + }, }); - const rollback = tree.rollback.bind(tree); - vi.spyOn(tree, "rollback").mockImplementation(async () => { - cleanupAttempts += 1; - if (cleanupAttempts === 1) - throw new Error("injected cleanup refusal"); - return rollback(); - }); - return tree; - }); - try { - const result = await exportWebScripts(input, resources); - if (result.ok) throw new Error("Expected unresolved cleanup"); - expect(result.error.cleanupIncomplete).toBe(true); - expect(result.error.userMessage).not.toContain("Remove the residual"); - expect(result.error.partialObservation).toMatchObject({ - kind: "web-script-export", - result: { - output_directory: input.output_directory, - manifest: { path: join(input.output_directory, "manifest.json") }, - }, - }); - await expect( - access(join(input.output_directory, "manifest.json")), - ).resolves.toBeUndefined(); - await resources.close(); - expect(cleanupAttempts).toBe(2); - } finally { - create.mockRestore(); - } - }); + await expect( + access(join(input.output_directory, "manifest.json")), + ).resolves.toBeUndefined(); + expect(failure.wasInjected()).toBe(true); + expect(failure.owners[0]?.handle.fd).toBeGreaterThanOrEqual(0); + failure.allowCleanup(); + await resources.close(); + expect(failure.owners.length).toBeGreaterThan(1); + expect( + failure.owners.every((owner) => owner === failure.owners[0]), + ).toBe(true); + expect(failure.owners[0]?.handle.fd).toBe(-1); + } finally { + failure.allowCleanup(); + failure.restore(); + } + }, + ); }); describe("captured script publication failures and cleanup", () => { diff --git a/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts b/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts index a166ca7dfa..65cac5407e 100644 --- a/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts +++ b/tests/boundary/il2cpp/il2cppMappingOwnership.test.ts @@ -93,7 +93,9 @@ it("links one code-loaded type table among short runs across many mappings", () }); expect(result.types_field).toBeNull(); expect(result.types_count).toBe(4); - expect(result.return_types.every((row) => row.name === "int")).toBe(true); + expect(result.return_types.map((row) => row.name)).toEqual( + Array.from({ length: 5 }, () => "int"), + ); expect(Buffer.compare(bytes, snapshot)).toBe(0); }); diff --git a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts index c15d9c897b..e97f4d9d52 100644 --- a/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts +++ b/tests/boundary/il2cpp/il2cppMetadataProvider.test.ts @@ -3,6 +3,7 @@ import { createHash } from "node:crypto"; import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import { AnalysisCancelledError, AnalysisCapabilityUnavailableError, @@ -24,6 +25,7 @@ import { createLinkedIl2cppMetadataFixture, } from "../../fixtures/il2cpp/metadata.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; afterEach(() => { vi.restoreAllMocks(); @@ -325,12 +327,13 @@ it("bounds concurrent snapshots and retains active decoding until close complete return release; }); const active = provider.inspect({ path }); - await entered; let closed = false; - const closing = provider.close().then(() => { - closed = true; - }); - try { + let closing: Promise | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, active, "metadata decoding"); + closing = provider.close().then(() => { + closed = true; + }); const refused = await provider.inspect({ path }); expect(refused.ok).toBe(false); if (!refused.ok) { @@ -346,18 +349,30 @@ it("bounds concurrent snapshots and retains active decoding until close complete } await setImmediate(); expect(closed).toBe(false); - } finally { resume(); await closing; - } - expect((await active).ok).toBe(true); - expect(closed).toBe(true); - const afterClose = await provider.inspect({ path }); - expect(afterClose.ok).toBe(false); - if (!afterClose.ok) { - expect(afterClose.error).toBeInstanceOf(AnalysisCapabilityUnavailableError); - expect(projectAnalysisError(afterClose.error).retryable).toBe(false); - } + expect((await active).ok).toBe(true); + expect(closed).toBe(true); + const afterClose = await provider.inspect({ path }); + expect(afterClose.ok).toBe(false); + if (!afterClose.ok) { + expect(afterClose.error).toBeInstanceOf( + AnalysisCapabilityUnavailableError, + ); + expect(projectAnalysisError(afterClose.error).retryable).toBe(false); + } + }, [ + resume, + async () => { + await closing; + }, + async () => { + await active; + }, + async () => { + await provider.close(); + }, + ]); }); it("keeps occupied capacity retryable while the reader remains open", async () => { @@ -377,8 +392,8 @@ it("keeps occupied capacity retryable while the reader remains open", async () = return release; }); const active = provider.inspect({ path }); - await entered; - try { + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, active, "metadata decoding"); const refused = await provider.inspect({ path }); expect(refused.ok).toBe(false); if (!refused.ok) { @@ -394,11 +409,15 @@ it("keeps occupied capacity retryable while the reader remains open", async () = }, }); } - } finally { - resume(); - await active; - await provider.close(); - } + }, [ + resume, + async () => { + await active; + }, + async () => { + await provider.close(); + }, + ]); }); it.each([false, true])( diff --git a/tests/boundary/inspector/javascriptRuntimeObservation.test.ts b/tests/boundary/inspector/javascriptRuntimeObservation.test.ts index 11ff4aad28..d969ef7fe7 100644 --- a/tests/boundary/inspector/javascriptRuntimeObservation.test.ts +++ b/tests/boundary/inspector/javascriptRuntimeObservation.test.ts @@ -410,7 +410,11 @@ test("captures complete metadata with two enable commands", async () => { expect(new Set(fake.commands.map(({ method }) => method))).toEqual( new Set(["Runtime.enable", "Debugger.enable"]), ); - expect(JSON.stringify(result.value)).toContain(outside); + expect(result.value.scripts.items).toContainEqual( + expect.objectContaining({ + location: expect.objectContaining({ file_path: outside }), + }), + ); expect(result.value.unavailable_without_instrumentation).toContain( "Electron IPC messages and handlers", ); @@ -759,9 +763,17 @@ describe("passive V8 Inspector evidence", () => { reason: "runtime-location-unresolved", }), ); - expect(JSON.stringify(result.graph)).toContain(rawUrl); - expect(JSON.stringify(result.graph)).toContain("v8-hash"); - expect(JSON.stringify(result.graph)).toContain('"reported_url":""'); + const runtimeProperties = result.graph.nodes + .filter(({ kind }) => kind === "runtime-script-instance") + .flatMap(({ observations }) => + observations.map(({ properties }) => properties), + ); + expect(runtimeProperties).toContainEqual( + expect.objectContaining({ reported_url: rawUrl, cdp_hash: "v8-hash" }), + ); + expect(runtimeProperties).toContainEqual( + expect.objectContaining({ reported_url: "", cdp_hash: null }), + ); }); }); diff --git a/tests/boundary/javascript-recovery/wakaruIntegration.test.ts b/tests/boundary/javascript-recovery/wakaruIntegration.test.ts index 52dd80eef8..6b35c89702 100644 --- a/tests/boundary/javascript-recovery/wakaruIntegration.test.ts +++ b/tests/boundary/javascript-recovery/wakaruIntegration.test.ts @@ -21,6 +21,7 @@ import { recoveryFixture, } from "../../fixtures/javascript-recovery/provider.js"; import { readWithoutFifoWriter } from "../../fixtures/fifoInput.js"; +import { injectPublishedDirectoryCloseFailure } from "../../support/publishedDirectoryCloseFailure.js"; const registerRecoveryProvenanceCases = () => { it("publishes exact bytes and mappings with source identity and no runtime-equivalence claim", async () => { @@ -340,18 +341,9 @@ describe.skipIf(process.platform !== "linux" || process.arch !== "x64")( it("returns completed recovery when retry closes a descriptor after commit", async () => { const fixture = await recoveryFixture(); - const createTree = SafeOutputTree.create.bind(SafeOutputTree); - const create = vi - .spyOn(SafeOutputTree, "create") - .mockImplementationOnce(async (outputRoot, platform) => { - const tree = await createTree(outputRoot, platform); - const commit = tree.commit.bind(tree); - vi.spyOn(tree, "commit").mockImplementation(async () => { - await commit(); - throw new Error("injected descriptor close failure"); - }); - return tree; - }); + const failure = await injectPublishedDirectoryCloseFailure( + fixture.output, + ); try { const result = await fixture.service.recover({ path: fixture.path, @@ -361,36 +353,29 @@ describe.skipIf(process.platform !== "linux" || process.arch !== "x64")( const published = javascriptRecoveryResultSchema.parse( result.value.normalized_result, ); + expect(failure.wasInjected()).toBe(true); + expect(failure.owners).toHaveLength(2); + expect(failure.owners[1]).toBe(failure.owners[0]); + expect(failure.owners[0]?.handle.fd).toBe(-1); await expect(access(published.manifest.path)).resolves.toBeUndefined(); expect(published.source.published_copy.path).toContain(fixture.output); } finally { - create.mockRestore(); + failure.allowCleanup(); + try { + await fixture.provider.close?.(); + } finally { + failure.restore(); + } } await assertRecoveryCleanup(fixture.launches); }); it("retains published recovery and its result while cleanup remains unresolved", async () => { const fixture = await recoveryFixture(); - const createTree = SafeOutputTree.create.bind(SafeOutputTree); - let cleanupAttempts = 0; - const create = vi - .spyOn(SafeOutputTree, "create") - .mockImplementationOnce(async (outputRoot, platform) => { - const tree = await createTree(outputRoot, platform); - const commit = tree.commit.bind(tree); - vi.spyOn(tree, "commit").mockImplementation(async () => { - await commit(); - throw new Error("injected descriptor close failure"); - }); - const rollback = tree.rollback.bind(tree); - vi.spyOn(tree, "rollback").mockImplementation(async () => { - cleanupAttempts += 1; - if (cleanupAttempts === 1) - throw new Error("injected cleanup refusal"); - return rollback(); - }); - return tree; - }); + const failure = await injectPublishedDirectoryCloseFailure( + fixture.output, + true, + ); try { const result = await fixture.service.recover({ path: fixture.path, @@ -412,13 +397,25 @@ describe.skipIf(process.platform !== "linux" || process.arch !== "x64")( await expect( access(join(fixture.output, "manifest.json")), ).resolves.toBeUndefined(); + expect(failure.wasInjected()).toBe(true); + expect(failure.owners[0]?.handle.fd).toBeGreaterThanOrEqual(0); + if (fixture.provider.close === undefined) + throw new Error("Expected Wakaru cleanup capability"); + failure.allowCleanup(); + await fixture.provider.close(); + expect(failure.owners.length).toBeGreaterThan(1); + expect( + failure.owners.every((owner) => owner === failure.owners[0]), + ).toBe(true); + expect(failure.owners[0]?.handle.fd).toBe(-1); } finally { - create.mockRestore(); + failure.allowCleanup(); + try { + await fixture.provider.close?.(); + } finally { + failure.restore(); + } } - if (fixture.provider.close === undefined) - throw new Error("Expected Wakaru cleanup capability"); - await fixture.provider.close(); - expect(cleanupAttempts).toBe(2); await expect( access(join(fixture.output, "manifest.json")), ).resolves.toBeUndefined(); @@ -571,7 +568,7 @@ describe.skipIf(process.platform !== "linux" || process.arch !== "x64")( const code = projectAnalysisError(result.error).code; if (mode === "secondary-change" || mode === "redistribute") expect(code).toBe("artifact_changed"); - else expect(code).toMatch(/execution_failure|unreadable_output/u); + else expect(["execution_failure", "unreadable_output"]).toContain(code); const reason = mode === "secondary-foreign" ? "does not bind the selected input snapshot" diff --git a/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts b/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts index 271f4f5335..0d5a140fad 100644 --- a/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts +++ b/tests/boundary/javascript/javascriptSourceAnalysisWorker.test.ts @@ -4,6 +4,8 @@ import { readFile, readdir } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; import { expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { JavaScriptAnalysisWorker } from "../../../src/javascript/analysis/JavaScriptAnalysisWorker.js"; import { createJavaScriptSourceAnalysis } from "../../../src/artifacts/javascript/JavaScriptSourceAnalysis.js"; import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; @@ -225,7 +227,7 @@ it("keeps a worker parked while a limit change closes the previous process", asy let replacement: | ReturnType | undefined; - try { + await withVerifierCleanup(async () => { const initial = await createJavaScriptSourceAnalysis(limits, 1, { scope, launcher, @@ -252,7 +254,7 @@ it("keeps a worker parked while a limit change closes the previous process", asy 1, { scope, launcher }, ); - await closeStarted; + await awaitAcknowledgement(closeStarted, replacement, "worker close"); const concurrent = await createJavaScriptSourceAnalysis(limits, 1, { scope, @@ -288,13 +290,19 @@ it("keeps a worker parked while a limit change closes the previous process", asy await scope.close(); await exits.get(parkedPid); expect(exited.has(parkedPid)).toBe(true); - } finally { - JavaScriptAnalysisWorker.prototype.close = originalClose; - releaseClose(); - const port = await replacement?.catch(() => undefined); - await port?.close().catch(() => undefined); - await scope.close(); - } + }, [ + releaseClose, + async () => { + const port = await replacement?.catch(() => undefined); + await port?.close(); + }, + async () => { + await scope.close(); + }, + () => { + JavaScriptAnalysisWorker.prototype.close = originalClose; + }, + ]); }); it("does not start a replacement when closing the previous worker fails", async () => { diff --git a/tests/boundary/javascript/workerOrderAudit.test.ts b/tests/boundary/javascript/workerOrderAudit.test.ts index a32896d5c6..1b83588760 100644 --- a/tests/boundary/javascript/workerOrderAudit.test.ts +++ b/tests/boundary/javascript/workerOrderAudit.test.ts @@ -20,6 +20,8 @@ import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess. import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; import { staticCheckpointFacts } from "../../fixtures/javascriptWorkerCheckpoint.js"; +// The observer is attached before the protocol reader. A queued microtask +// runs after both readers have synchronously processed the current chunk. const observeWorkerFrames = ( stdout: Readable | null | undefined, observe: (frame: JavaScriptWorkerMessage, beforeEof: boolean) => void, @@ -205,9 +207,12 @@ it("clears a consumed resource frame before the next reusable worker job", async `${ready} let count=0; require("node:readline").createInterface({input:process.stdin}).on("line", line => { const {id}=JSON.parse(line); const first=++count===1; const response={kind:"failure",id,reason:first?"resource":"output",resource:first?"memory":null,limits:null,message:first?"first resource response":"second job output response"}; process.stdout.write(JSON.stringify(response)+"\\n"); if(!first)setInterval(()=>{},1000); });`, ], }); - launched.process.stdout?.on("data", (chunk: Buffer | string) => { - if (String(chunk).includes("second job output response")) - setTimeout(() => controller.abort("abort after second response"), 5); + observeWorkerFrames(launched.process.stdout, (frame) => { + if ( + frame.kind === "failure" && + frame.message === "second job output response" + ) + queueMicrotask(() => controller.abort("abort after second response")); }); return launched; }, @@ -288,9 +293,12 @@ it("retains coalesced static evidence when abort follows a resource frame", asyn `${ready} const fs=require("node:fs"); const {createHash}=require("node:crypto"); const staticFacts=JSON.parse(${JSON.stringify(encodedStatic)}); require("node:readline").createInterface({input:process.stdin}).on("line", line => { const request=JSON.parse(line); if(!request.file)return; const text=JSON.stringify(staticFacts); fs.writeFileSync(request.id+".static.json",text); const frames=[{kind:"static",id:request.id,bytes:Buffer.byteLength(text),sha256:createHash("sha256").update(text).digest("hex")},{kind:"failure",id:request.id,reason:"resource",resource:"memory",limits:null,message:"resource failure before abort"}]; process.stdout.write(frames.map(frame=>JSON.stringify(frame)).join("\\n")+"\\n"); });`, ], }); - launched.process.stdout?.on("data", (chunk: Buffer | string) => { - if (String(chunk).includes("resource failure before abort")) - setTimeout(() => controller.abort(null), 5); + observeWorkerFrames(launched.process.stdout, (frame) => { + if ( + frame.kind === "failure" && + frame.message === "resource failure before abort" + ) + queueMicrotask(() => controller.abort(null)); }); return launched; }, @@ -1039,10 +1047,13 @@ it("preserves a complete current-job failure parsed before caller abort", async `${ready} process.stdin.once("data", chunk => { const id=JSON.parse(String(chunk)).id; process.stdout.write(JSON.stringify({kind:"phase",id,phase:"parse"})+"\\n"); setTimeout(() => process.stdout.write(JSON.stringify({kind:"failure",id,reason:"output",resource:null,limits:null,message:"complete current-job failure"})+"\\n"),1); setInterval(()=>{},1000); });`, ], }); - launched.process.stdout?.on("data", (chunk: Buffer | string) => { - if (String(chunk).includes("complete current-job failure")) { + observeWorkerFrames(launched.process.stdout, (frame) => { + if ( + frame.kind === "failure" && + frame.message === "complete current-job failure" + ) { completeFrameSeen = true; - setTimeout(() => controller.abort("abort after parsed failure"), 5); + queueMicrotask(() => controller.abort("abort after parsed failure")); } }); return launched; @@ -1081,10 +1092,13 @@ it("does not let a wrong-job failure frame override later caller abort", async ( `${ready} process.stdin.once("data", () => { const id="00000000-0000-4000-8000-000000000000"; process.stdout.write(JSON.stringify({kind:"failure",id,reason:"output",resource:null,limits:null,message:"unrelated job failure"})+"\\n"); setInterval(()=>{},1000); });`, ], }); - launched.process.stdout?.on("data", (chunk: Buffer | string) => { - if (String(chunk).includes("unrelated job failure")) { + observeWorkerFrames(launched.process.stdout, (frame) => { + if ( + frame.kind === "failure" && + frame.message === "unrelated job failure" + ) { completeFrameSeen = true; - setTimeout(() => controller.abort(reason), 5); + queueMicrotask(() => controller.abort(reason)); } }); return launched; diff --git a/tests/boundary/mcp/advertisedMcpTool.test.ts b/tests/boundary/mcp/advertisedMcpTool.test.ts new file mode 100644 index 0000000000..76b88bb5a7 --- /dev/null +++ b/tests/boundary/mcp/advertisedMcpTool.test.ts @@ -0,0 +1,43 @@ +import { expect, it, vi } from "vitest"; + +import { advertisedMcpToolSchemas } from "../../fixtures/advertisedMcpTool.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; + +it.each(["first", "last"] as const)( + "rejects an ambiguous advertised schema when the duplicate is %s", + async (position) => { + const { client } = await createApplicationMcpHarness(); + const schemas = await advertisedMcpToolSchemas(client, "binary_session"); + const result = await client.callTool( + { name: "binary_session", arguments: {} }, + { toolDefinition: schemas.toolDefinition }, + ); + expect(result.isError).not.toBe(true); + expect( + schemas.ajv.validate(schemas.outputSchema, result.structuredContent), + ).toBe(true); + expect(schemas.ajv.validate(schemas.outputSchema, { bogus: true })).toBe( + false, + ); + + const listing = await client.listTools(); + const permissive = { + ...schemas.toolDefinition, + outputSchema: { type: "object" }, + }; + expect(schemas.ajv.validate(permissive.outputSchema, { bogus: true })).toBe( + true, + ); + vi.spyOn(client, "listTools").mockResolvedValue({ + ...listing, + tools: + position === "first" + ? [permissive, ...listing.tools] + : [...listing.tools, permissive], + }); + + await expect( + advertisedMcpToolSchemas(client, "binary_session"), + ).rejects.toThrow("binary_session has multiple advertised definitions"); + }, +); diff --git a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts index 6a4d050d08..5e0354bd4a 100644 --- a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts +++ b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts @@ -136,6 +136,7 @@ describe("known-answer packaged artifact digest binding", () => { arguments: { input_path: targets.javascript }, }); const evidence = resultEvidence(result); + expect(evidence.subject?.digest.sha256).toEqual(expect.any(String)); expect(evidence.subject?.digest.sha256).not.toBe( targets.javascriptSha256, ); diff --git a/tests/boundary/mcp/analysisViewMcp.test.ts b/tests/boundary/mcp/analysisViewMcp.test.ts index 47a50ac94f..ed43d45aac 100644 --- a/tests/boundary/mcp/analysisViewMcp.test.ts +++ b/tests/boundary/mcp/analysisViewMcp.test.ts @@ -13,7 +13,6 @@ import { } from "../../fixtures/historicalHar.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import { afterAll, expect, it, onTestFinished } from "vitest"; import { z } from "zod"; @@ -34,7 +33,6 @@ import { createJavaScriptApplicationGraph, createJavaScriptApplicationNode, } from "../../../src/domain/javascript/javascriptApplicationGraph.js"; -import { createServer } from "../../../src/server/createServer.js"; import { connectGhidraMcp } from "./ghidraMcpHarness.js"; import { analysisViewJavaScriptEvidence, @@ -52,6 +50,7 @@ import { } from "../../fixtures/analysisView.js"; import { BOOK_ROWS } from "../../fixtures/environmentStateWrites.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createLinkedIl2cppMetadataFixture } from "../../fixtures/il2cpp/metadata.js"; import { createIl2cppBinaryFixture, @@ -80,35 +79,24 @@ const connect = async ( const session = createTestBinarySession(() => { throw new Error("selected views must not start a deep provider"); }); - const server = createServer( - { kind: "session", session }, - { + return createApplicationMcpHarness({ + session, + clientIdentity: { name: "analysis-view-mcp", version: "1" }, + schemaValidator: schemaValidation, + serverOptions: { // These schema conformance cases exercise full nested constraints. environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: "full", - ...(maximumInputElements === undefined - ? {} - : { REA_MCP_MAX_INPUT_ELEMENTS: String(maximumInputElements) }), + REA_MCP_MAX_INPUT_ELEMENTS: + maximumInputElements === undefined + ? process.env.REA_MCP_MAX_INPUT_ELEMENTS + : String(maximumInputElements), }, ...(binaryLayout === undefined ? {} : { binaryLayout }), ...(webNetworkCapture === undefined ? {} : { webNetworkCapture }), }, - ); - const client = new Client( - { name: "analysis-view-mcp", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); - return { client, session }; }; it("selects every IL2CPP native type reference by rendered name with stable row identity", async () => { diff --git a/tests/boundary/mcp/artifactInspectionCoverageMcp.test.ts b/tests/boundary/mcp/artifactInspectionCoverageMcp.test.ts index 0c674ff6d9..e1a6a48b4e 100644 --- a/tests/boundary/mcp/artifactInspectionCoverageMcp.test.ts +++ b/tests/boundary/mcp/artifactInspectionCoverageMcp.test.ts @@ -2,9 +2,11 @@ import { execFileSync } from "node:child_process"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { afterAll, describe, expect, it } from "vitest"; +import { afterAll, describe, expect } from "vitest"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; +import { mcpTest, type TestMcp } from "../../support/mcp/mcpFixture.js"; +import { ownArtifactSession } from "../../support/mcp/artifactSession.js"; +import type { OnTestFinishedHandler } from "vitest"; import { machoImage } from "../../../src/artifacts/apple/MachoImage.fixture.js"; import { createArtifactBinarySession } from "../../../src/composition/binary.js"; @@ -33,29 +35,30 @@ const writeBundle = async (withFifo: boolean): Promise => { return bundle; }; -const connect = async () => { +const connect = async ( + mcp: TestMcp, + onTestFinished: (handler: OnTestFinishedHandler) => void, +) => { const session = await createArtifactBinarySession(process.env); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client( + const client = await mcp.connect( + server, { name: "inspection-coverage", version: "1" }, - { jsonSchemaValidator: schemaValidation }, + schemaValidation, ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const definition = (await client.listTools()).tools.find( (tool) => tool.name === "inspect_analysis_view", ); expect(definition?.outputSchema).toBeDefined(); const outputSchema: Record = definition!.outputSchema!; - return { client, server, outputSchema }; + return { client, outputSchema }; }; describe.skipIf(process.platform === "win32")( "inspect_artifact traversal coverage", () => { - it.each([ + mcpTest.for([ { withFifo: true, status: "partial", @@ -68,69 +71,64 @@ describe.skipIf(process.platform === "win32")( }, ] as const)( "derives inspection and summary coverage from inventory traversal (FIFO: $withFifo)", - async ({ withFifo, status, sourceCoverage }) => { + { timeout: 30_000 }, + async ({ withFifo, status, sourceCoverage }, { mcp, onTestFinished }) => { const path = await writeBundle(withFifo); - const { client, server, outputSchema } = await connect(); - try { - const opened = await client.callTool({ - name: "open_binary", - arguments: { path }, - }); - expect(opened.isError, JSON.stringify(opened)).not.toBe(true); - const inspected = await client.callTool({ - name: "inspect_artifact", - arguments: {}, - }); - expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); - const parent = parseEvidence(inspected.structuredContent); - const inspection = artifactInspectionResultSchema.parse( - parent.normalized_result, - ); - const inventory = inspection.substeps[0]!.evidence.normalized_result; - expect(inventory.coverage).toEqual(sourceCoverage); - expect(inspection.coverage.status).toBe(status); + const { client, outputSchema } = await connect(mcp, onTestFinished); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const inspected = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); + const parent = parseEvidence(inspected.structuredContent); + const inspection = artifactInspectionResultSchema.parse( + parent.normalized_result, + ); + const inventory = inspection.substeps[0]!.evidence.normalized_result; + expect(inventory.coverage).toEqual(sourceCoverage); + expect(inspection.coverage.status).toBe(status); - const summaries = [ - await client.callTool({ - name: "inspect_analysis_view", - arguments: { - source: { - kind: "retained-evidence", - evidence_id: parent.evidence_id, - }, - view: { kind: "summary" }, + const summaries = [ + await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, }, - }), - await client.callTool({ - name: "inspect_artifact", - arguments: { detail: "summary" }, - }), - ]; - for (const reply of summaries) { - expect(reply.isError, JSON.stringify(reply)).not.toBe(true); - expect( - schemaValidation.assertions.validate( - outputSchema, - reply.structuredContent, - ), - JSON.stringify(schemaValidation.assertions.errors), - ).toBe(true); - const view = analysisViewResultSchema.parse( - parseEvidence(reply.structuredContent).normalized_result, - ); - if (view.kind !== "summary" || !("source_complete" in view.summary)) - throw new Error("Expected an artifact inventory summary"); - expect(view.summary.source_complete).toBe(true); - expect(view.summary.source_coverage).toEqual(sourceCoverage); - expect(view.summary.inspection?.coverage.status).toBe(status); - expect(view.unknowns.includes(TRAVERSAL_UNKNOWN)).toBe(withFifo); - } - } finally { - await client.close(); - await server.close(); + view: { kind: "summary" }, + }, + }), + await client.callTool({ + name: "inspect_artifact", + arguments: { detail: "summary" }, + }), + ]; + for (const reply of summaries) { + expect(reply.isError, JSON.stringify(reply)).not.toBe(true); + expect( + schemaValidation.assertions.validate( + outputSchema, + reply.structuredContent, + ), + JSON.stringify(schemaValidation.assertions.errors), + ).toBe(true); + const view = analysisViewResultSchema.parse( + parseEvidence(reply.structuredContent).normalized_result, + ); + if (view.kind !== "summary" || !("source_complete" in view.summary)) + throw new Error("Expected an artifact inventory summary"); + expect(view.summary.source_complete).toBe(true); + expect(view.summary.source_coverage).toEqual(sourceCoverage); + expect(view.summary.inspection?.coverage.status).toBe(status); + expect(view.unknowns.includes(TRAVERSAL_UNKNOWN)).toBe(withFifo); } }, - 30_000, ); }, ); diff --git a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts index 34a29e4cdb..3824bea0bb 100644 --- a/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts +++ b/tests/boundary/mcp/artifactMcp.inlineInventory.test.ts @@ -3,9 +3,11 @@ import { INLINE_EVIDENCE_MCP_ENVIRONMENT } from "../../../scripts/lib/mcp-verifi import { readFile, unlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { Client } from "@modelcontextprotocol/client"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; -import { afterAll, expect, it } from "vitest"; +import { afterAll, expect } from "vitest"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; +import { ownArtifactSession } from "../../support/mcp/artifactSession.js"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { z } from "zod"; import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; @@ -86,29 +88,27 @@ const assertInventorySubjectValidation = async ( } }; -it("preserves keyed archive identities and rejects inconsistent inventory carrier subjects", async () => { - const directory = await createTestTempDirectory("rea-keyed-mcp-"); - const path = join(directory, "archive.plist"); - await writeFile( - path, - await readFile( - new URL( - "../../fixtures/golden/keyed-archive/foundation.xml", - import.meta.url, +mcpTest( + "preserves keyed archive identities and rejects inconsistent inventory carrier subjects", + async ({ mcp, onTestFinished }) => { + const directory = await createTestTempDirectory("rea-keyed-mcp-"); + const path = join(directory, "archive.plist"); + await writeFile( + path, + await readFile( + new URL( + "../../fixtures/golden/keyed-archive/foundation.xml", + import.meta.url, + ), ), - ), - ); - const session = createTestBinarySession(new ArtifactProvider(process.env)); - const server = createServer({ kind: "session", session }); - const client = new Client( - { name: "keyed-mcp-test", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); + ); + const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect( + createServer({ kind: "session", session }), + { name: "keyed-mcp-test", version: "1" }, + schemaValidation, + ); const opened = await client.callTool({ name: "open_binary", arguments: { path }, @@ -188,12 +188,8 @@ it("preserves keyed archive identities and rejects inconsistent inventory carrie if (inventory === undefined) throw new Error("Missing inventory Evidence"); await assertInventorySubjectValidation(client, inventory); await assertInventorySubjectValidation(client, parent); - } finally { - await client.close(); - await server.close(); - await session.close(); - } -}); + }, +); const createInventoryViewReader = async ( client: Client, @@ -338,9 +334,51 @@ const assertSummaryDelivery = async ( ); }; -it.each(["full", "compact"] as const)( +const assertOccurrencePages = async ( + readView: Awaited>, + occurrences: z.output["occurrences"], +) => { + const paths = ["files/0.txt", "files/519.txt", "missing.txt"]; + const matched = occurrences.filter((row) => paths.includes(row.logical_path)); + expect(matched.map((row) => row.logical_path)).toEqual(paths.slice(0, 2)); + const selected = await readView({ + kind: "inventory", + collection: "occurrences", + logical_paths: paths, + offset: 0, + limit: 1, + }); + expect(selected.result).toMatchObject({ + kind: "inventory", + source_complete: true, + items: matched.slice(0, 1), + coverage: { examined: 1, total: 2, next_offset: 1, exhausted: false }, + }); + const next = await readView({ + kind: "inventory", + collection: "occurrences", + logical_paths: paths, + offset: 1, + limit: 1, + }); + expect(next.result).toMatchObject({ + items: matched.slice(1, 2), + coverage: { examined: 1, next_offset: null, exhausted: true }, + }); + const none = await readView({ + kind: "inventory", + collection: "occurrences", + logical_paths: [], + }); + expect(none.result).toMatchObject({ + items: [], + coverage: { total: 0, exhausted: true }, + }); +}; + +mcpTest.for(["full", "compact"] as const)( "keeps complete ZIP output and reads retained inventory views after the source is removed (%s)", - async (profile) => { + async (profile, { mcp, onTestFinished }) => { const directory = await createTestTempDirectory("rea-artifact-inline-"); const archive = join(directory, "many.zip"); const writer = new ZipWriter(new Uint8ArrayWriter()); @@ -353,167 +391,122 @@ it.each(["full", "compact"] as const)( await writeFile(archive, await writer.close()); const session = createTestBinarySession(new ArtifactProvider(process.env)); - const server = createServer( - { kind: "session", session }, + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect( + createServer( + { kind: "session", session }, + { + environment: { + ...process.env, + ...INLINE_EVIDENCE_MCP_ENVIRONMENT, + REA_MCP_INPUT_SCHEMA_PROFILE: profile, + }, + }, + ), + { name: "artifact-inline-test", version: "1" }, + schemaValidation, + ); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: archive }, + }); + expect(opened.isError).not.toBe(true); + const result = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(result.isError, JSON.stringify(result.structuredContent)).not.toBe( + true, + ); + const parent = parseEvidence(result.structuredContent); + await assertSummaryDelivery(client, session, parent); + const parsedInspection = artifactInspectionResultSchema.parse( + parent.normalized_result, + ); + const inventoryEvidence = parsedInspection.substeps[0]?.evidence; + if (inventoryEvidence === undefined) + throw new Error("Missing inventory Evidence"); + const graph = artifactInventoryResultSchema.parse( + inventoryEvidence.normalized_result, + ); + expect(graph).toMatchObject({ + integrity_contradictions: [], + coverage: { scope: "container-entries", status: "complete" }, + }); + expect(graph.occurrences).toHaveLength(fileCount + 1); + // Follow-ups must use retained source facts, not reopen this archive. + await unlink(archive); + const readView = await createInventoryViewReader(client, parent, profile); + const summary = await readView({ kind: "summary" }); + expect(summary.result).toMatchObject({ + kind: "summary", + parent_evidence_id: parent.evidence_id, + artifact: { path: archive, sha256: graph.manifest.root_sha256 }, + summary: { + manifest: graph.manifest, + source_complete: true, + available_counts: { occurrences: fileCount + 1 }, + observed_entry_kind_counts: { file: fileCount + 1 }, + inventory_evidence_ids: [inventoryEvidence.evidence_id], + }, + }); + expect(summary.evidence.evidence_links).toEqual([parent.evidence_id]); + await assertOccurrencePages(readView, graph.occurrences); + + const partial = createEvidence( + { path: archive, sha256: graph.manifest.root_sha256, format: "zip" }, + inventoryEvidence.provider, { - environment: { - ...process.env, - ...INLINE_EVIDENCE_MCP_ENVIRONMENT, - REA_MCP_INPUT_SCHEMA_PROFILE: profile, + operation: "inventory_artifact", + parameters: {}, + result: { + ...graph, + occurrences: graph.occurrences.slice(0, 1), }, }, ); - const client = new Client( - { name: "artifact-inline-test", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path: archive }, - }); - expect(opened.isError).not.toBe(true); - const result = await client.callTool({ - name: "inspect_artifact", - arguments: {}, - }); - expect(result.isError, JSON.stringify(result.structuredContent)).not.toBe( - true, - ); - const parent = parseEvidence(result.structuredContent); - await assertSummaryDelivery(client, session, parent); - const parsedInspection = artifactInspectionResultSchema.parse( - parent.normalized_result, - ); - const inventoryEvidence = parsedInspection.substeps[0]?.evidence; - if (inventoryEvidence === undefined) - throw new Error("Missing inventory Evidence"); - const graph = artifactInventoryResultSchema.parse( - inventoryEvidence.normalized_result, - ); - expect(graph).toMatchObject({ - integrity_contradictions: [], - coverage: { scope: "container-entries", status: "complete" }, - }); - expect(graph.occurrences).toHaveLength(fileCount + 1); - // Follow-ups must use retained source facts, not reopen this archive. - await unlink(archive); - const readView = await createInventoryViewReader(client, parent, profile); - const summary = await readView({ kind: "summary" }); - expect(summary.result).toMatchObject({ - kind: "summary", - parent_evidence_id: parent.evidence_id, - artifact: { path: archive, sha256: graph.manifest.root_sha256 }, - summary: { - manifest: graph.manifest, - source_complete: true, - available_counts: { occurrences: fileCount + 1 }, - observed_entry_kind_counts: { file: fileCount + 1 }, - inventory_evidence_ids: [inventoryEvidence.evidence_id], + const partialReply = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "inline", evidence: partial }, + view: { + kind: "inventory", + collection: "occurrences", + logical_paths: ["missing.txt"], }, - }); - expect(summary.evidence.evidence_links).toEqual([parent.evidence_id]); - const paths = ["files/0.txt", "files/519.txt", "missing.txt"]; - const selected = await readView({ - kind: "inventory", - collection: "occurrences", - logical_paths: paths, - offset: 0, - limit: 1, - }); - expect(selected.result).toMatchObject({ - kind: "inventory", - source_complete: true, - items: graph.occurrences - .filter((row) => paths.includes(row.logical_path)) - .slice(0, 1), - coverage: { examined: 1, total: 2, next_offset: 1, exhausted: false }, - }); - const next = await readView({ - kind: "inventory", - collection: "occurrences", - logical_paths: paths, - offset: 1, - limit: 1, - }); - expect(next.result).toMatchObject({ - coverage: { examined: 1, next_offset: null, exhausted: true }, - }); - const none = await readView({ - kind: "inventory", - collection: "occurrences", - logical_paths: [], - }); - expect(none.result).toMatchObject({ - items: [], - coverage: { total: 0, exhausted: true }, - }); + }, + }); + expect( + parseEvidence(partialReply.structuredContent).normalized_result, + ).toMatchObject({ + source_complete: false, + items: [], + unknowns: expect.arrayContaining([ + expect.stringContaining("Missing matches do not establish absence"), + ]), + }); - const partial = createEvidence( - { path: archive, sha256: graph.manifest.root_sha256, format: "zip" }, - inventoryEvidence.provider, - { - operation: "inventory_artifact", - parameters: {}, - result: { - ...graph, - occurrences: graph.occurrences.slice(0, 1), - }, - }, - ); - const partialReply = await client.callTool({ - name: "inspect_analysis_view", - arguments: { - source: { kind: "inline", evidence: partial }, - view: { - kind: "inventory", - collection: "occurrences", - logical_paths: ["missing.txt"], - }, - }, - }); - expect( - parseEvidence(partialReply.structuredContent).normalized_result, - ).toMatchObject({ - source_complete: false, - items: [], - unknowns: expect.arrayContaining([ - expect.stringContaining("Missing matches do not establish absence"), - ]), - }); - - await assertPortableRoundtrip(client, directory, parent, summary.result); - } finally { - await client.close(); - await server.close(); - await session.close(); - } + await assertPortableRoundtrip(client, directory, parent, summary.result); }, ); -it("refuses summary delivery without a retained Evidence owner and preserves complete delivery", async () => { - const directory = await createTestTempDirectory("rea-inventory-fixed-"); - const archive = join(directory, "fixture.zip"); - const writer = new ZipWriter(new Uint8ArrayWriter()); - await writer.add("main.js", new TextReader("console.log('fixed');\n")); - await writeFile(archive, await writer.close()); - const session = createTestBinarySession(new ArtifactProvider(process.env)); - const opened = await session.open(archive); - if (!opened.ok) throw opened.error; - const server = createServer({ kind: "fixed", analysis: session }); - const client = new Client( - { name: "artifact-fixed-test", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); +mcpTest( + "refuses summary delivery without a retained Evidence owner and preserves complete delivery", + async ({ mcp, onTestFinished }) => { + const directory = await createTestTempDirectory("rea-inventory-fixed-"); + const archive = join(directory, "fixture.zip"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("main.js", new TextReader("console.log('fixed');\n")); + await writeFile(archive, await writer.close()); + const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); + const opened = await session.open(archive); + if (!opened.ok) throw opened.error; + const client = await mcp.connect( + createServer({ kind: "fixed", analysis: session }), + { name: "artifact-fixed-test", version: "1" }, + schemaValidation, + ); const summary = await client.callTool({ name: "inspect_artifact", arguments: { detail: "summary" }, @@ -534,9 +527,5 @@ it("refuses summary delivery without a retained Evidence owner and preserves com ).toMatchObject({ substeps: [expect.objectContaining({ operation: "inventory_artifact" })], }); - } finally { - await client.close(); - await server.close(); - await session.close(); - } -}); + }, +); diff --git a/tests/boundary/mcp/artifactMcp.test.ts b/tests/boundary/mcp/artifactMcp.test.ts index deb282cee5..907937d320 100644 --- a/tests/boundary/mcp/artifactMcp.test.ts +++ b/tests/boundary/mcp/artifactMcp.test.ts @@ -29,10 +29,10 @@ import { import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { Client } from "@modelcontextprotocol/client"; import { createPackageWithOptions } from "@electron/asar"; import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; -import { expect, it, vi, type MockInstance } from "vitest"; +import { expect, vi, type MockInstance } from "vitest"; import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -42,6 +42,8 @@ import { } from "../../fixtures/zipNames.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { mcpTest as it } from "../../support/mcp/mcpFixture.js"; +import { ownArtifactSession } from "../../support/mcp/artifactSession.js"; import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js"; import { createServer } from "../../../src/server/createServer.js"; @@ -57,7 +59,10 @@ const integrityMessage = (unpacked: boolean): string => : "Artifact bytes contradict declared integrity." } If expected, rerun inspect_artifact with integrity_policy=record-and-continue (CLI: --integrity-policy record-and-continue) to retain observed bytes as untrusted; otherwise get a fresh copy.`; -it("refuses an artifact parent and summary when its actual inventory source cannot be recorded", async () => { +it("refuses an artifact parent and summary when its actual inventory source cannot be recorded", async ({ + mcp, + onTestFinished, +}) => { const root = await createTestTempDirectory( "rea-artifact-source-refusal-mcp-", ); @@ -66,10 +71,12 @@ it("refuses an artifact parent and summary when its actual inventory source cann await zip.add("main.js", new TextReader("console.log('source');\n")); await writeFile(archive, await zip.close()); const session = await createArtifactBinarySession({}); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "artifact-source-refusal", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const client = await mcp.connect(server, { + name: "artifact-source-refusal", + version: "1", + }); const attempted: Evidence[] = []; const failure = new EvidenceIntegrityError( "controlled inventory source recording refusal", @@ -77,8 +84,6 @@ it("refuses an artifact parent and summary when its actual inventory source cann const nativeRecord = session.recordEvidence.bind(session); let recording: MockInstance | undefined; try { - await server.connect(serverTransport); - await client.connect(clientTransport); const opened = await client.callTool({ name: "open_binary", arguments: { path: archive }, @@ -112,31 +117,17 @@ it("refuses an artifact parent and summary when its actual inventory source cann expect(session.exportEvidenceBundle().records).toEqual(before); } finally { recording?.mockRestore(); - const closed = await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); - const failures: unknown[] = []; - for (const result of closed) - if (result.status === "rejected") failures.push(result.reason); - const sessionClosed = closed[2]; - if (sessionClosed.status === "fulfilled" && !sessionClosed.value.ok) - failures.push(sessionClosed.value.error); - if (failures.length > 0) - throw new AggregateError( - failures, - "Artifact recorder refusal cleanup failed", - ); + await mcp.close(); } }); -it.each([ +it.for([ { unpacked: false, label: "embedded" }, { unpacked: true, label: "unpacked" }, ])( "returns actionable local details for an $label ASAR integrity error", - async ({ unpacked }) => { + { timeout: 20_000 }, + async ({ unpacked }, { mcp, onTestFinished }) => { const root = await createTestTempDirectory("rea-asar-integrity-mcp-"); const source = join(root, "source"); const original = "console.log('ok');\n"; @@ -160,13 +151,13 @@ it.each([ } const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "asar-integrity-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const client = await mcp.connect(server, { + name: "asar-integrity-test", + version: "1", + }); try { - await server.connect(serverTransport); - await client.connect(clientTransport); const opened = await client.callTool({ name: "open_binary", arguments: { path: archive }, @@ -194,17 +185,15 @@ it.each([ }, }); } finally { - await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); + await mcp.close(); } }, - 20_000, ); -it("extracts an active archive through MCP when requested", async () => { +it("extracts an active archive through MCP when requested", async ({ + mcp, + onTestFinished, +}) => { const root = await createTestTempDirectory("rea-artifact-extract-mcp-"); const archive = join(root, "fixture.zip"); const zip = new ZipWriter(new Uint8ArrayWriter()); @@ -212,14 +201,14 @@ it("extracts an active archive through MCP when requested", async () => { await writeFile(archive, await zip.close()); const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "artifact-extract-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const client = await mcp.connect(server, { + name: "artifact-extract-test", + version: "1", + }); let outputRoot: string | undefined; try { - await server.connect(serverTransport); - await client.connect(clientTransport); const opened = await client.callTool({ name: "open_binary", arguments: { path: archive }, @@ -240,13 +229,19 @@ it("extracts an active archive through MCP when requested", async () => { ); expect(normalized.artifacts).toHaveLength(1); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); - if (outputRoot !== undefined) - await rm(outputRoot, { recursive: true, force: true }); + try { + await mcp.close(); + } finally { + if (outputRoot !== undefined) + await rm(outputRoot, { recursive: true, force: true }); + } } }); -it("preserves raw legacy ZIP name bytes and CP437 path interpretation through MCP", async () => { +it("preserves raw legacy ZIP name bytes and CP437 path interpretation through MCP", async ({ + mcp, + onTestFinished, +}) => { const root = await createTestTempDirectory("rea-legacy-zip-name-mcp-"); const archive = join(root, "legacy-name.zip"); const rawName = Buffer.from([0xc7, 0xd1, 0xb1, 0xdb, 0x2e, 0x74, 0x78, 0x74]); @@ -261,13 +256,13 @@ it("preserves raw legacy ZIP name bytes and CP437 path interpretation through MC ); const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "legacy-zip-name-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const client = await mcp.connect(server, { + name: "legacy-zip-name-test", + version: "1", + }); try { - await server.connect(serverTransport); - await client.connect(clientTransport); const opened = await client.callTool({ name: "open_binary", arguments: { path: archive }, @@ -294,11 +289,14 @@ it("preserves raw legacy ZIP name bytes and CP437 path interpretation through MC }), ); } finally { - await Promise.all([client.close(), server.close(), session.close()]); + await mcp.close(); } }); -it("extracts a macOS app bundle through MCP", async () => { +it("extracts a macOS app bundle through MCP", async ({ + mcp, + onTestFinished, +}) => { const root = await createTestTempDirectory("rea-artifact-extract-app-mcp-"); const contents = join(root, "Fixture.app", "Contents"); await mkdir(join(contents, "MacOS"), { recursive: true }); @@ -313,17 +311,14 @@ it("extracts a macOS app bundle through MCP", async () => { await writeFile(join(contents, "MacOS", "Fixture"), header); const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ + const client = await mcp.connect(server, { name: "artifact-extract-app-test", version: "1", }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); let outputRoot: string | undefined; try { - await server.connect(serverTransport); - await client.connect(clientTransport); const opened = await client.callTool({ name: "open_binary", arguments: { path: join(root, "Fixture.app") }, @@ -343,13 +338,19 @@ it("extracts a macOS app bundle through MCP", async () => { await readFile(join(outputRoot, "Contents", "Info.plist"), "utf8"), ).toBe(plist); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); - if (outputRoot !== undefined) - await rm(outputRoot, { recursive: true, force: true }); + try { + await mcp.close(); + } finally { + if (outputRoot !== undefined) + await rm(outputRoot, { recursive: true, force: true }); + } } }); -it("records an explicitly continued mismatch, preserves verified siblings, and never reports equivalence", async () => { +it("records an explicitly continued mismatch, preserves verified siblings, and never reports equivalence", async ({ + mcp, + onTestFinished, +}) => { const root = await createTestTempDirectory("rea-asar-continue-mcp-"); const source = join(root, "source"); await mkdir(source); @@ -372,13 +373,13 @@ it("records an explicitly continued mismatch, preserves verified siblings, and n await writeFile(archive, bytes); const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "asar-continue-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const client = await mcp.connect(server, { + name: "asar-continue-test", + version: "1", + }); try { - await server.connect(serverTransport); - await client.connect(clientTransport); await client.callTool({ name: "open_binary", arguments: { path: archive }, @@ -464,7 +465,11 @@ it("records an explicitly continued mismatch, preserves verified siblings, and n }); const { coverage: _coverage, ...inventoryWithoutCoverage } = inventory; const missingCoverage = createEvidence( - { path: archive, sha256: inventory.manifest.root_sha256, format: "asar" }, + { + path: archive, + sha256: inventory.manifest.root_sha256, + format: "asar", + }, compact.provider, { operation: "inventory_artifact", @@ -480,26 +485,26 @@ it("records an explicitly continued mismatch, preserves verified siblings, and n true, ); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); + await mcp.close(); } }); -it("rejects inline artifact Evidence whose content does not match its ID", async () => { +it("rejects inline artifact Evidence whose content does not match its ID", async ({ + mcp, + onTestFinished, +}) => { const session = createTestBinarySession(() => ({ health: () => Promise.resolve(), execute: () => Promise.resolve(observed(null)), close: () => Promise.resolve(resultOk(null)), })); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ + const client = await mcp.connect(server, { name: "artifact-authority-test", version: "1", }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); try { - await server.connect(serverTransport); - await client.connect(clientTransport); const result = await client.callTool({ name: "compare_artifacts", arguments: { @@ -518,26 +523,26 @@ it("rejects inline artifact Evidence whose content does not match its ID", async session.evidenceById(ARTIFACT_COMPARISON_EXAMPLE.right.evidence_id), ).toBeUndefined(); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); + await mcp.close(); } }); -it("compares inline inventory Evidence without prior session calls", async () => { +it("compares inline inventory Evidence without prior session calls", async ({ + mcp, + onTestFinished, +}) => { const session = createTestBinarySession(() => ({ health: () => Promise.resolve(), execute: () => Promise.resolve(observed(null)), close: () => Promise.resolve(resultOk(null)), })); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ + const client = await mcp.connect(server, { name: "artifact-ownership-test", version: "1", }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); try { - await server.connect(serverTransport); - await client.connect(clientTransport); const result = await client.callTool({ name: "compare_artifacts", arguments: ARTIFACT_COMPARISON_EXAMPLE, @@ -557,11 +562,14 @@ it("compares inline inventory Evidence without prior session calls", async () => session.evidenceById(ARTIFACT_COMPARISON_EXAMPLE.right.evidence_id), ).toBeDefined(); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); + await mcp.close(); } }); -it("returns full artifact graphs inline and compares changed inventories", async () => { +it("returns full artifact graphs inline and compares changed inventories", async ({ + mcp, + onTestFinished, +}) => { const directory = await createTestTempDirectory("rea-artifact-mcp-"); const archive = join(directory, "fixture.ipa"); const changedArchive = join(directory, "changed.ipa"); @@ -575,13 +583,13 @@ it("returns full artifact graphs inline and compares changed inventories", async ); await writeFile(changedArchive, await changedWriter.close()); const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "artifact-mcp-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const client = await mcp.connect(server, { + name: "artifact-mcp-test", + version: "1", + }); try { - await server.connect(serverTransport); - await client.connect(clientTransport); const opened = await client.callTool({ name: "open_binary", arguments: { path: archive }, @@ -692,11 +700,14 @@ it("returns full artifact graphs inline and compares changed inventories", async }, }); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); + await mcp.close(); } }); -it("compares summary Evidence through retained complete inspection parents", async () => { +it("compares summary Evidence through retained complete inspection parents", async ({ + mcp, + onTestFinished, +}) => { const directory = await createTestTempDirectory("rea-artifact-summary-mcp-"); const archive = join(directory, "one.zip"); const changedArchive = join(directory, "two.zip"); @@ -709,16 +720,13 @@ it("compares summary Evidence through retained complete inspection parents", asy await writeFile(changedArchive, await changedWriter.close()); const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ + const client = await mcp.connect(server, { name: "artifact-summary-mcp-test", version: "1", }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); try { - await server.connect(serverTransport); - await client.connect(clientTransport); const summaries = []; for (const path of [archive, changedArchive]) { const opened = await client.callTool({ @@ -753,7 +761,7 @@ it("compares summary Evidence through retained complete inspection parents", asy .parse(right.normalized_result).parent_evidence_id; expect(result.evidence_links).toEqual([leftParentId, rightParentId]); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); + await mcp.close(); } }); @@ -930,18 +938,21 @@ const withForeignMetadataAndNestedZipMember = ( ); }; -it("does not apply PyInstaller declaration unknowns to foreign metadata or nested ZIP members", async () => { +it("does not apply PyInstaller declaration unknowns to foreign metadata or nested ZIP members", async ({ + mcp, + onTestFinished, +}) => { const root = await createTestTempDirectory("rea-pyinstaller-scope-mcp-"); const archive = join(root, "sample"); await writeFile(archive, pyInstallerComparisonArchive("m", false)); const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "pyinstaller-scope-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const client = await mcp.connect(server, { + name: "pyinstaller-scope-test", + version: "1", + }); try { - await server.connect(serverTransport); - await client.connect(clientTransport); const observed = await inspectArtifactInventoryThroughMcp(client, archive); const foreign = withForeignMetadataAndNestedZipMember(observed); const response = await compareArtifactEvidenceThroughMcp( @@ -958,11 +969,14 @@ it("does not apply PyInstaller declaration unknowns to foreign metadata or neste changes: [], }); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); + await mcp.close(); } }); -it("compares PyInstaller declaration facts without inventing payload identity", async () => { +it("compares PyInstaller declaration facts without inventing payload identity", async ({ + mcp, + onTestFinished, +}) => { const root = await createTestTempDirectory("rea-pyinstaller-compare-mcp-"); const leftPath = join(root, "module"); const rightPath = join(root, "script"); @@ -977,13 +991,13 @@ it("compares PyInstaller declaration facts without inventing payload identity", writeFile(shiftedPath, pyInstallerComparisonArchive("m", false, true)), ]); const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "pyinstaller-compare-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); + const client = await mcp.connect(server, { + name: "pyinstaller-compare-test", + version: "1", + }); try { - await server.connect(serverTransport); - await client.connect(clientTransport); const left = await inspectArtifactInventoryThroughMcp(client, leftPath); const right = await inspectArtifactInventoryThroughMcp(client, rightPath); const clean = await inspectArtifactInventoryThroughMcp( @@ -1103,11 +1117,14 @@ it("compares PyInstaller declaration facts without inventing payload identity", ], }); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); + await mcp.close(); } }); -it("discovers artifact target prerequisites and preserves typed MCP refusals", async () => { +it("discovers artifact target prerequisites and preserves typed MCP refusals", async ({ + mcp, + onTestFinished, +}) => { const root = await createTestTempDirectory("rea-artifact-availability-mcp-"); const zip = new ZipWriter(new Uint8ArrayWriter()); await zip.add("a.txt", new TextReader("hello")); @@ -1137,13 +1154,12 @@ it("discovers artifact target prerequisites and preserves typed MCP refusals", a const session = createTestBinarySession( new ArtifactProvider(process.env, "darwin"), ); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ + const client = await mcp.connect(server, { name: "artifact-availability-test", version: "1", }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); const operations = [ "decode_interface_builder", "inspect_asset_catalog", @@ -1161,8 +1177,6 @@ it("discovers artifact target prerequisites and preserves typed MCP refusals", a return result.result.tool_availability.groups; }; try { - await server.connect(serverTransport); - await client.connect(clientTransport); const before = await availability(); for (const name of operations) expect(before.find((group) => group.names.includes(name))).toMatchObject({ @@ -1251,11 +1265,14 @@ it("discovers artifact target prerequisites and preserves typed MCP refusals", a await client.callTool({ name: "close_binary", arguments: {} }); } } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); + await mcp.close(); } }); -it("attributes ASAR integrity contradictions to their actual inputs and withholds correlation support", async () => { +it("attributes ASAR integrity contradictions to their actual inputs and withholds correlation support", async ({ + mcp, + onTestFinished, +}) => { const root = await createTestTempDirectory("rea-asar-attribution-"); const source = join(root, "source"); await mkdir(source); @@ -1277,12 +1294,13 @@ it("attributes ASAR integrity contradictions to their actual inputs and withhold const currentArchive = join(root, "current.asar"); await createPackageWithOptions(source, currentArchive, {}); const session = createTestBinarySession(new ArtifactProvider(process.env)); + ownArtifactSession(session, onTestFinished); const server = createServer({ kind: "session", session }); - const client = new Client({ name: "artifact-attribution", version: "1" }); - const [ct, st] = InMemoryTransport.createLinkedPair(); + const client = await mcp.connect(server, { + name: "artifact-attribution", + version: "1", + }); try { - await server.connect(st); - await client.connect(ct); const observations = []; for (const path of [ originalArchive, @@ -1334,7 +1352,7 @@ it("attributes ASAR integrity contradictions to their actual inputs and withhold presence, ); } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); + await mcp.close(); } }); diff --git a/tests/boundary/mcp/browserMcp.test.ts b/tests/boundary/mcp/browserMcp.test.ts index 103df01633..cc251e4479 100644 --- a/tests/boundary/mcp/browserMcp.test.ts +++ b/tests/boundary/mcp/browserMcp.test.ts @@ -1,10 +1,12 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { crc32, deflateSync } from "node:zlib"; import { parseEvidence } from "../../../src/domain/evidence.js"; -import { ok as resultOk } from "../../../src/domain/result.js"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { err, ok as resultOk } from "../../../src/domain/result.js"; +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; +import type { Client } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { afterAll, afterEach, expect, it } from "vitest"; +import { afterAll, afterEach, expect, it, vi } from "vitest"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; @@ -20,16 +22,27 @@ import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaP import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +import { createTestMcp } from "../../support/mcp/mcpFixture.js"; import { startFakeCdpBrowser, type FakeCdpBrowser, } from "../../fixtures/fakeCdpBrowser.js"; const schemaValidation = createMcpSchemaValidation(); -afterAll(() => schemaValidation.clear()); - -const resources: Array<{ close(): Promise }> = []; -const browsers: FakeCdpBrowser[] = []; +const resources = new ArtifactResourceScope(); +const retainResource = (resource: string, close: () => Promise): void => { + resources.retain({ kind: "lease", resource, lease: { release: close } }); +}; +const retainBrowser = (browser: FakeCdpBrowser): void => { + retainResource(browser.endpoint, () => browser.close()); +}; +afterAll(async () => { + try { + await resources.close(); + } finally { + schemaValidation.clear(); + } +}); const INTEGRATION_TEST_TIMEOUT_MS = 20_000; it("preserves Link parameter order and context through the advertised MCP contract", async () => { @@ -80,7 +93,7 @@ it("preserves Link parameter order and context through the advertised MCP contra ]; }, }); - browsers.push(browser); + retainBrowser(browser); const connected = await connectBrowser(); const advertised = (await connected.client.listTools()).tools.find( ({ name }) => name === "inspect_web_page", @@ -216,7 +229,7 @@ cliTest( ] : undefined, }); - browsers.push(browser); + retainBrowser(browser); const { client, session } = await connectBrowser(); const args = { cdp_endpoint: browser.endpoint, @@ -271,11 +284,49 @@ cliTest( INTEGRATION_TEST_TIMEOUT_MS, ); -afterEach(async () => { - await Promise.all( - resources.splice(0).map(async (resource) => resource.close()), +afterEach(() => resources.retryCleanup()); + +it("closes the browser and retains failed MCP and session cleanup for retry", async () => { + const { client, session } = await connectBrowser(); + const browser = await startFakeCdpBrowser(); + retainBrowser(browser); + const browserClose = vi.spyOn(browser, "close"); + const sdkFailure = new Error("controlled SDK close failure"); + const sessionFailure = new ProviderCleanupError( + "fixture", + ["session-owner"], + { + reason: "controlled session close failure", + }, ); - await Promise.all(browsers.splice(0).map(async (browser) => browser.close())); + const clientClose = vi + .spyOn(client, "close") + .mockRejectedValueOnce(sdkFailure); + const sessionClose = vi + .spyOn(session, "close") + .mockResolvedValueOnce(err(sessionFailure)); + try { + const failure: unknown = await resources + .retryCleanup() + .catch((cause: unknown) => cause); + expect(failure).toMatchObject({ + cause: expect.objectContaining({ + errors: expect.arrayContaining([sdkFailure, sessionFailure]), + }), + }); + await expect(fetch(`${browser.endpoint}/json/version`)).rejects.toThrow(); + expect(clientClose).toHaveBeenCalledTimes(1); + expect(sessionClose).toHaveBeenCalledTimes(1); + await resources.retryCleanup(); + expect(clientClose).toHaveBeenCalledTimes(2); + expect(sessionClose).toHaveBeenCalledTimes(2); + expect(browserClose).toHaveBeenCalledTimes(1); + } finally { + clientClose.mockRestore(); + sessionClose.mockRestore(); + await resources.retryCleanup(); + browserClose.mockRestore(); + } }); it( @@ -287,7 +338,7 @@ it( sensitiveShapes: true, eventsBeforeReply: ["Network.enable"], }); - browsers.push(browser); + retainBrowser(browser); const connected = await connectBrowser(); const tools = await connected.client.listTools(); @@ -505,7 +556,20 @@ const verifySessionAndComparisonTools = async ( arguments: { before: { inspection: capture } }, }); expect(incompleteComparison.isError).toBe(true); - const screenshot = await connected.client.callTool({ + await verifyScreenshotTools(connected.client, browser); + expect(inspected.structuredContent).toMatchObject({ + normalized_result: expect.any(Object), + operation: "inspect_web_page", + predicate_type: expect.any(String), + parameters: expect.any(Object), + }); +}; + +const verifyScreenshotTools = async ( + client: Client, + browser: FakeCdpBrowser, +): Promise => { + const screenshot = await client.callTool({ name: "capture_web_screenshot", arguments: { cdp_endpoint: browser.endpoint, @@ -515,7 +579,7 @@ const verifySessionAndComparisonTools = async ( }); expect(screenshot.isError).not.toBe(true); const screenshotArtifact = artifactOf(screenshot.structuredContent); - const visual = await connected.client.callTool({ + const visual = await client.callTool({ name: "compare_web_screenshots", arguments: { before: screenshotArtifact, @@ -527,13 +591,31 @@ const verifySessionAndComparisonTools = async ( normalized_result: { status: "identical", changed_pixels: 0 }, }); - await assertScreenshotPngFailures(connected.client); - expect(inspected.structuredContent).toMatchObject({ - normalized_result: expect.any(Object), - operation: "inspect_web_page", - predicate_type: expect.any(String), - parameters: expect.any(Object), + const before = singlePixelScreenshot([0, 0, 0]); + const after = singlePixelScreenshot([10, 0, 0]); + const changed = await client.callTool({ + name: "compare_web_screenshots", + arguments: { before, after, channel_threshold: 2 }, }); + expect(changed.isError).not.toBe(true); + expect(changed.structuredContent).toMatchObject({ + parameters: { + before_artifact_sha256: before.sha256, + after_artifact_sha256: after.sha256, + channel_threshold: 2, + }, + normalized_result: { + status: "different", + channel_threshold: 2, + compared_pixels: 1, + changed_pixels: 1, + changed_ratio: 1, + maximum_channel_delta: 10, + mean_absolute_channel_delta: 2.5, + }, + }); + + await assertScreenshotPngFailures(client); }; const pngChunk = (type: string, data: Buffer): Buffer => { @@ -548,6 +630,21 @@ const pngChunk = (type: string, data: Buffer): Buffer => { return chunk; }; +const singlePixelScreenshot = (rgb: readonly [number, number, number]) => { + const header = Buffer.alloc(13); + header.writeUInt32BE(1, 0); + header.writeUInt32BE(1, 4); + header.set([8, 2, 0, 0, 0], 8); + return createWebScreenshotArtifact( + Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + pngChunk("IHDR", header), + pngChunk("IDAT", deflateSync(Buffer.from([0, ...rgb]))), + pngChunk("IEND", Buffer.alloc(0)), + ]), + ); +}; + const assertScreenshotPngFailures = async (client: Client): Promise => { const header = Buffer.alloc(13); header.writeUInt32BE(1, 0); @@ -624,7 +721,7 @@ cliTest( mappings: "AAAA", }), }); - browsers.push(browser); + retainBrowser(browser); const connected = await connectBrowser(); const result = await connected.client.callTool({ name: "analyze_web_bundle", @@ -706,7 +803,7 @@ it.each(["full", "compact"] as const)( "records a browser-socket endpoint redacted and reconciles its Evidence (%s)", async (profile) => { const browser = await startFakeCdpBrowser(); - browsers.push(browser); + retainBrowser(browser); const connected = await connectBrowser(profile); const tool = (await connected.client.listTools()).tools.find( ({ name }) => name === "inspect_web_page", @@ -760,7 +857,7 @@ it.each([ "selects the exact opaque target ID through %s", async (name, observationMs) => { const browser = await startFakeCdpBrowser(); - browsers.push(browser); + retainBrowser(browser); const { client } = await connectBrowser(); const args = { cdp_endpoint: browser.endpoint, @@ -786,7 +883,7 @@ it.each([ it("does not attach to a target outside the request's allowed origin scope", async () => { const browser = await startFakeCdpBrowser(); - browsers.push(browser); + retainBrowser(browser); const connected = await connectBrowser(); const result = await connected.client.callTool({ name: "inspect_web_page", @@ -851,6 +948,12 @@ const connectBrowser = async (profile?: "full" | "compact") => { execute: () => Promise.resolve(observed(null)), close: () => Promise.resolve(resultOk(null)), })); + retainResource("binary session", async () => { + const result = await session.close(); + if (!result.ok) throw result.error; + }); + const mcp = createTestMcp(); + retainResource("MCP connection", () => mcp.close()); const server = createServer( { kind: "session", session }, { @@ -862,15 +965,11 @@ const connectBrowser = async (profile?: "full" | "compact") => { }), }, ); - const client = new Client( + const client = await mcp.connect( + server, { name: "browser-mcp-test", version: "1" }, - { jsonSchemaValidator: schemaValidation }, + schemaValidation, ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - resources.push(client, server, session); - await server.connect(serverTransport); - await client.connect(clientTransport); return { client, session }; }; @@ -914,7 +1013,7 @@ it.each([24, 32, "crc"])( commandResult: ({ method }) => method === "Page.captureScreenshot" ? { data } : undefined, }); - browsers.push(browser); + retainBrowser(browser); const { client } = await connectBrowser(); const result = await client.callTool({ name: "capture_web_screenshot", diff --git a/tests/boundary/mcp/browserScenarioMcp.test.ts b/tests/boundary/mcp/browserScenarioMcp.test.ts index bbcdd6abb7..171c96e386 100644 --- a/tests/boundary/mcp/browserScenarioMcp.test.ts +++ b/tests/boundary/mcp/browserScenarioMcp.test.ts @@ -1,5 +1,4 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { afterEach, describe, expect, onTestFinished, test } from "vitest"; +import { describe, expect, test } from "vitest"; import { Ajv2020 } from "ajv/dist/2020.js"; import { z } from "zod"; @@ -19,7 +18,7 @@ import { import type { AnalysisError } from "../../../src/domain/analysisErrorBase.js"; import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; import { err, ok, type Result } from "../../../src/domain/result.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { observed } from "../../fixtures/analysisExecution.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { advertisedInputJsonSchema } from "../../../src/contracts/toolSchemaMetadata.js"; @@ -185,24 +184,14 @@ test.each(["full", "compact"] as const)( const session = createTestBinarySession(() => { throw new Error("Schema discovery must not start a deep provider"); }); - const server = createServer( - { kind: "session", session }, - { + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "browser-action-patterns", version: "1" }, + serverOptions: { browserScenarioCapture: provider, environment: { REA_MCP_INPUT_SCHEMA_PROFILE: profile }, }, - ); - const client = new Client({ - name: "browser-action-patterns", - version: "1", }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - onTestFinished(async () => { - await Promise.all([client.close(), server.close(), session.close()]); - }); - await server.connect(serverTransport); - await client.connect(clientTransport); const tool = (await client.listTools()).tools.find( ({ name }) => name === "capture_browser_scenario", ); @@ -259,21 +248,16 @@ test.each(["full", "compact"] as const)( ); describe("browser scenario MCP tool", () => { - const resources: Array<{ close(): Promise }> = []; - - afterEach(async () => { - await Promise.all(resources.splice(0).map((item) => item.close())); - }); - test("runs the explicit scenario request without a permission grant", async () => { const provider = new FakeBrowserScenarioProvider(); const session = createTestBinarySession(() => ({ execute: () => Promise.resolve(observed(null)), close: () => Promise.resolve(ok(null)), })); - const server = createServer( - { kind: "session", session }, - { + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "browser-scenario-test", version: "1" }, + serverOptions: { browserObservation: new CdpBrowserProvider(), browserScenarioCapture: provider, availabilityPolicy: () => ({ @@ -281,13 +265,7 @@ describe("browser scenario MCP tool", () => { investigationInputRoots: 0, }), }, - ); - const client = new Client({ name: "browser-scenario-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - resources.push(client, server, session); - await server.connect(serverTransport); - await client.connect(clientTransport); + }); const minimal = await client.callTool({ name: "capture_browser_scenario", @@ -402,50 +380,45 @@ test("records a completed late-close cancellation as server Evidence", async () execute: () => Promise.resolve(observed(null)), close: () => Promise.resolve(ok(null)), })); - const server = createServer( - { kind: "session", session }, - { + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "scenario-late-close", version: "1" }, + serverOptions: { browserScenarioCapture: provider, availabilityPolicy: () => ({ processCaptureEnabled: false, investigationInputRoots: 0, }), }, - ); - const client = new Client({ name: "scenario-late-close", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); - try { - const response = await client.callTool({ - name: "capture_browser_scenario", - arguments: minimalScenario(), - }); - const { error } = parseMcpToolError(response); - const reference = z - .object({ - partial_observation: z.object({ - kind: z.literal("retained-evidence"), - evidence_id: z.string(), - }), - }) - .parse(error.details).partial_observation; - const retained = session.evidenceById(reference.evidence_id); - if (retained === undefined) throw new Error("Evidence was not retained"); - expect(retained.operation).toBe("capture_browser_scenario"); - expect(retained.normalized_result).toMatchObject({ - browser: { cleanup: "terminated-owned-process" }, - }); - expect( - (retained.normalized_result as BrowserScenarioCapture).steps.every( - ({ status }) => status === "completed", - ), - ).toBe(true); - await expect(client.ping()).resolves.toEqual({}); - } finally { - await Promise.all([client.close(), server.close(), session.close()]); - } + }); + const response = await client.callTool({ + name: "capture_browser_scenario", + arguments: minimalScenario(), + }); + const { error } = parseMcpToolError(response); + const reference = z + .object({ + partial_observation: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }) + .parse(error.details).partial_observation; + const retained = session.evidenceById(reference.evidence_id); + if (retained === undefined) throw new Error("Evidence was not retained"); + expect(retained.operation).toBe("capture_browser_scenario"); + expect(retained.normalized_result).toMatchObject({ + browser: { cleanup: "terminated-owned-process" }, + }); + expect( + (retained.normalized_result as BrowserScenarioCapture).steps.map( + ({ step_id, status }) => ({ step_id, status }), + ), + ).toEqual([ + { step_id: "scenario_start", status: "completed" }, + { step_id: "settle", status: "completed" }, + ]); + await expect(client.ping()).resolves.toEqual({}); }); test("delivers a retained summary and pages its exact steps and events", async () => { @@ -454,104 +427,96 @@ test("delivers a retained summary and pages its exact steps and events", async ( execute: () => Promise.resolve(observed(null)), close: () => Promise.resolve(ok(null)), })); - const server = createServer( - { kind: "session", session }, - { + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "scenario-summary", version: "1" }, + serverOptions: { browserScenarioCapture: provider, availabilityPolicy: () => ({ processCaptureEnabled: false, investigationInputRoots: 0, }), }, + }); + const tools = (await client.listTools()).tools; + const advertised = tools.find( + ({ name }) => name === "capture_browser_scenario", + ); + if (advertised?.outputSchema === undefined) + throw new Error("Missing advertised scenario output schema"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const validateInput = ajv.compile(advertised.inputSchema); + const validateOutput = ajv.compile( + z.record(z.string(), z.unknown()).parse(advertised.outputSchema), + ); + const request = { ...minimalScenario(), detail: "summary" }; + expect(validateInput(request), JSON.stringify(validateInput.errors)).toBe( + true, ); - const client = new Client({ name: "scenario-summary", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); - try { - const tools = (await client.listTools()).tools; - const advertised = tools.find( - ({ name }) => name === "capture_browser_scenario", - ); - if (advertised?.outputSchema === undefined) - throw new Error("Missing advertised scenario output schema"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); - const validateInput = ajv.compile(advertised.inputSchema); - const validateOutput = ajv.compile( - z.record(z.string(), z.unknown()).parse(advertised.outputSchema), - ); - const request = { ...minimalScenario(), detail: "summary" }; - expect(validateInput(request), JSON.stringify(validateInput.errors)).toBe( - true, - ); - const response = await client.callTool({ - name: "capture_browser_scenario", - arguments: request, - }); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - expect( - validateOutput(response.structuredContent), - JSON.stringify(validateOutput.errors), - ).toBe(true); - const summary = z - .object({ - normalized_result: z.object({ - kind: z.literal("summary"), - parent_evidence_id: z.string(), - summary: z.object({ - steps: z.array(z.object({ step_id: z.string() })), - }), + const response = await client.callTool({ + name: "capture_browser_scenario", + arguments: request, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect( + validateOutput(response.structuredContent), + JSON.stringify(validateOutput.errors), + ).toBe(true); + const summary = z + .object({ + normalized_result: z.object({ + kind: z.literal("summary"), + parent_evidence_id: z.string(), + summary: z.object({ + steps: z.array(z.object({ step_id: z.string() })), }), - }) - .parse(response.structuredContent).normalized_result; - expect(summary.summary.steps.map(({ step_id }) => step_id)).toEqual([ - "scenario_start", - "settle", - ]); - // Delivery detail is neither sent to the provider nor recorded as a scenario parameter. - expect(provider.scenarios[0]).not.toHaveProperty("detail"); - const parent = session.evidenceById(summary.parent_evidence_id); - expect(parent?.operation).toBe("capture_browser_scenario"); - expect(parent?.parameters).not.toHaveProperty("detail"); + }), + }) + .parse(response.structuredContent).normalized_result; + expect(summary.summary.steps.map(({ step_id }) => step_id)).toEqual([ + "scenario_start", + "settle", + ]); + // Delivery detail is neither sent to the provider nor recorded as a scenario parameter. + expect(provider.scenarios[0]).not.toHaveProperty("detail"); + const parent = session.evidenceById(summary.parent_evidence_id); + expect(parent?.operation).toBe("capture_browser_scenario"); + expect(parent?.parameters).not.toHaveProperty("detail"); - const page = await client.callTool({ - name: "inspect_analysis_view", - arguments: { - source: { - kind: "retained-evidence", - evidence_id: summary.parent_evidence_id, - }, - view: { - kind: "browser-scenario", - collection: "steps", - step_indices: [1], - }, + const page = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, }, - }); - expect(page.isError, JSON.stringify(page)).not.toBe(true); - expect(page.structuredContent).toMatchObject({ - normalized_result: { + view: { kind: "browser-scenario", - parent_evidence_id: summary.parent_evidence_id, - items: { - collection: "steps", - rows: [{ step_index: 1, step_id: "settle", artifacts: {} }], - }, - coverage: { total: 1, exhausted: true }, + collection: "steps", + step_indices: [1], }, - }); + }, + }); + expect(page.isError, JSON.stringify(page)).not.toBe(true); + expect(page.structuredContent).toMatchObject({ + normalized_result: { + kind: "browser-scenario", + parent_evidence_id: summary.parent_evidence_id, + items: { + collection: "steps", + rows: [{ step_index: 1, step_id: "settle", artifacts: {} }], + }, + coverage: { total: 1, exhausted: true }, + }, + }); - const invalid = await client.callTool({ - name: "capture_browser_scenario", - arguments: { ...minimalScenario(), detail: "brief" }, - }); - expect(parseMcpToolError(invalid)).toMatchObject({ - error: { code: "invalid_request" }, - }); - expect(provider.scenarios).toHaveLength(1); - } finally { - await Promise.all([client.close(), server.close(), session.close()]); - } + const invalid = await client.callTool({ + name: "capture_browser_scenario", + arguments: { ...minimalScenario(), detail: "brief" }, + }); + expect(parseMcpToolError(invalid)).toMatchObject({ + error: { code: "invalid_request" }, + }); + expect(provider.scenarios).toHaveLength(1); }); diff --git a/tests/boundary/mcp/browserSourceCoordinates.test.ts b/tests/boundary/mcp/browserSourceCoordinates.test.ts index 058a2d54f7..3724412a2f 100644 --- a/tests/boundary/mcp/browserSourceCoordinates.test.ts +++ b/tests/boundary/mcp/browserSourceCoordinates.test.ts @@ -1,10 +1,6 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { expect, it } from "vitest"; +import { expect, it, onTestFinished } from "vitest"; import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; -import { ok } from "../../../src/domain/result.js"; -import { createServer } from "../../../src/server/createServer.js"; -import { observed } from "../../fixtures/analysisExecution.js"; -import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js"; it("reports malformed source coordinates as unknown through MCP", async () => { @@ -53,13 +49,10 @@ it("reports malformed source coordinates as unknown through MCP", async () => { return undefined; }, }); - const session = createTestBinarySession(() => ({ - execute: () => Promise.resolve(observed(null)), - close: () => Promise.resolve(ok(null)), - })); - const server = createServer( - { kind: "session", session }, - { + onTestFinished(() => browser.close()); + const { client } = await createApplicationMcpHarness({ + clientIdentity: { name: "source-coordinate-test", version: "1" }, + serverOptions: { environment: { ...process.env }, browserObservation: new CdpBrowserProvider(), availabilityPolicy: () => ({ @@ -67,75 +60,63 @@ it("reports malformed source coordinates as unknown through MCP", async () => { investigationInputRoots: 0, }), }, - ); - const client = new Client({ name: "source-coordinate-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const args = { - cdp_endpoint: browser.endpoint, - allowed_origins: [browser.allowedOrigin], - target_id: "allowed-page", - observation_ms: 0, - }; - const page = await client.callTool({ - name: "inspect_web_page", - arguments: args, - }); - expect(page.isError).not.toBe(true); - expect(page.structuredContent).toMatchObject({ - normalized_result: { - console: { - events: [expect.objectContaining({ line: null, column: null })], - }, - network: { - requests: expect.arrayContaining([ - expect.objectContaining({ - request_id: "bad-coordinates", - initiator: expect.objectContaining({ line: null, column: null }), - }), - ]), - }, - completeness: { - excluded: expect.arrayContaining([ - expect.objectContaining({ - section: "console_events", - reason: "invalid_protocol_value", - }), - expect.objectContaining({ - section: "network_initiators", - reason: "invalid_protocol_value", - }), - ]), - }, + }); + const args = { + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + }; + const page = await client.callTool({ + name: "inspect_web_page", + arguments: args, + }); + expect(page.isError).not.toBe(true); + expect(page.structuredContent).toMatchObject({ + normalized_result: { + console: { + events: [expect.objectContaining({ line: null, column: null })], }, - }); - const tools = await client.callTool({ - name: "discover_webmcp_tools", - arguments: args, - }); - expect(tools.isError).not.toBe(true); - expect(tools.structuredContent).toMatchObject({ - normalized_result: { - tools: { - items: [expect.objectContaining({ registration_source: null })], - }, - completeness: { - excluded: expect.arrayContaining([ - expect.objectContaining({ - section: "webmcp_tools", - reason: "invalid_protocol_value", - }), - ]), - }, + network: { + requests: expect.arrayContaining([ + expect.objectContaining({ + request_id: "bad-coordinates", + initiator: expect.objectContaining({ line: null, column: null }), + }), + ]), }, - }); - } finally { - await client.close(); - await server.close(); - await session.close(); - await browser.close(); - } + completeness: { + excluded: expect.arrayContaining([ + expect.objectContaining({ + section: "console_events", + reason: "invalid_protocol_value", + }), + expect.objectContaining({ + section: "network_initiators", + reason: "invalid_protocol_value", + }), + ]), + }, + }, + }); + const tools = await client.callTool({ + name: "discover_webmcp_tools", + arguments: args, + }); + expect(tools.isError).not.toBe(true); + expect(tools.structuredContent).toMatchObject({ + normalized_result: { + tools: { + items: [expect.objectContaining({ registration_source: null })], + }, + completeness: { + excluded: expect.arrayContaining([ + expect.objectContaining({ + section: "webmcp_tools", + reason: "invalid_protocol_value", + }), + ]), + }, + }, + }); }, 20_000); diff --git a/tests/boundary/mcp/compactInputSchemaMcp.test.ts b/tests/boundary/mcp/compactInputSchemaMcp.test.ts index 6c5650469a..8b916cca73 100644 --- a/tests/boundary/mcp/compactInputSchemaMcp.test.ts +++ b/tests/boundary/mcp/compactInputSchemaMcp.test.ts @@ -1,7 +1,6 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, onTestFinished } from "vitest"; import { z } from "zod"; import { @@ -25,6 +24,7 @@ import { processScenarioSchema } from "../../../src/domain/process/processScenar import { EvidenceMcpServer } from "../../../src/server/EvidenceMcpServer.js"; import { createToolResultDelivery } from "../../../src/server/toolResult.js"; import { toolRegistrationOptions } from "../../../src/server/toolRegistrationOptions.js"; +import { createTestMcp } from "../../support/mcp/mcpFixture.js"; import { arrayItemSchemaError, strictGatewayItemSchemaErrors, @@ -83,16 +83,13 @@ const unsupportedDependencyKeywords = ( const listToolsThroughTransport = async ( server: EvidenceMcpServer, ): Promise => { - const client = new Client({ name: "compact-profile", version: "0" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - return (await client.listTools()).tools; - } finally { - await Promise.allSettled([client.close(), server.close()]); - } + const mcp = createTestMcp(); + onTestFinished(() => mcp.close()); + const client = await mcp.connect(server, { + name: "compact-profile", + version: "0", + }); + return (await client.listTools()).tools; }; const registerCatalog = (server: EvidenceMcpServer): void => { @@ -513,36 +510,33 @@ describe("MCP compact input schema canonical enforcement", () => { }; }, ); - const client = new Client({ name: "compact-validation", version: "0" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const advertised = (await client.listTools()).tools.find( - ({ name }) => name === contract.name, - ); - if (advertised === undefined) - throw new Error("Process capture tool was not advertised"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); - const validate = ajv.compile(advertised.inputSchema); - const valid = { executable: "node", environment: { APP_MODE: "test" } }; - const reserved = { - executable: "node", - environment: { REA_PROCESS_RUN_ID: "caller-value" }, - }; - expect(validate(valid)).toBe(true); - expect(processScenarioSchema.safeParse(valid).success).toBe(true); - expect(processScenarioSchema.safeParse(reserved).success).toBe(false); - const result = await client.callTool({ - name: contract.name, - arguments: reserved, - }); - expect(result.isError).toBe(true); - expect(handlerCalled).toBe(false); - } finally { - await Promise.allSettled([client.close(), server.close()]); - } + const mcp = createTestMcp(); + onTestFinished(() => mcp.close()); + const client = await mcp.connect(server, { + name: "compact-validation", + version: "0", + }); + const advertised = (await client.listTools()).tools.find( + ({ name }) => name === contract.name, + ); + if (advertised === undefined) + throw new Error("Process capture tool was not advertised"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const validate = ajv.compile(advertised.inputSchema); + const valid = { executable: "node", environment: { APP_MODE: "test" } }; + const reserved = { + executable: "node", + environment: { REA_PROCESS_RUN_ID: "caller-value" }, + }; + expect(validate(valid)).toBe(true); + expect(processScenarioSchema.safeParse(valid).success).toBe(true); + expect(processScenarioSchema.safeParse(reserved).success).toBe(false); + const result = await client.callTool({ + name: contract.name, + arguments: reserved, + }); + expect(result.isError).toBe(true); + expect(handlerCalled).toBe(false); }); it("compacts plain standard schemas that are not Zod instances", async () => { @@ -625,30 +619,28 @@ it.each(["full", "compact"] as const)( calls++; return { content: [{ type: "text", text: "accepted" }] }; }); - const client = new Client({ name: "resource-scope", version: "1" }); - const [ct, st] = InMemoryTransport.createLinkedPair(); - try { - await server.connect(st); - await client.connect(ct); - const definition = (await client.listTools()).tools.find( - ({ name }) => name === "resource_fixture", - ); - if (definition === undefined) throw new Error("Missing resource fixture"); - const advertised = new Ajv2020({ strict: false }).compile( - definition.inputSchema, - ); - for (const value of [7, "seven"]) { - const args = { child: { value } }; - expect(advertised(args)).toBe(canonical(args)); - const result = await client.callTool({ - name: "resource_fixture", - arguments: args, - }); - expect(result.isError === true).toBe(!canonical(args)); - } - expect(calls).toBe(1); - } finally { - await Promise.allSettled([client.close(), server.close()]); + const mcp = createTestMcp(); + onTestFinished(() => mcp.close()); + const client = await mcp.connect(server, { + name: "resource-scope", + version: "1", + }); + const definition = (await client.listTools()).tools.find( + ({ name }) => name === "resource_fixture", + ); + if (definition === undefined) throw new Error("Missing resource fixture"); + const advertised = new Ajv2020({ strict: false }).compile( + definition.inputSchema, + ); + for (const value of [7, "seven"]) { + const args = { child: { value } }; + expect(advertised(args)).toBe(canonical(args)); + const result = await client.callTool({ + name: "resource_fixture", + arguments: args, + }); + expect(result.isError === true).toBe(!canonical(args)); } + expect(calls).toBe(1); }, ); diff --git a/tests/boundary/mcp/comparisonUnknownsMcp.test.ts b/tests/boundary/mcp/comparisonUnknownsMcp.test.ts index f872d2f154..4f47530415 100644 --- a/tests/boundary/mcp/comparisonUnknownsMcp.test.ts +++ b/tests/boundary/mcp/comparisonUnknownsMcp.test.ts @@ -1,5 +1,4 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { expect, it, onTestFinished } from "vitest"; +import { expect, it } from "vitest"; import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; @@ -9,23 +8,23 @@ import { type Evidence, } from "../../../src/domain/evidence.js"; import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; -import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; const connect = async () => { const session = createTestBinarySession(() => { throw new Error("Evidence comparison must not launch a provider"); }); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "comparison-unknowns", version: "1" }); - onTestFinished(async () => { - await Promise.allSettled([client.close(), server.close(), session.close()]); + return createApplicationMcpHarness({ + session, + clientIdentity: { name: "comparison-unknowns", version: "1" }, + serverOptions: { + environment: { + ...process.env, + REA_MCP_MAX_INPUT_ELEMENTS: process.env.REA_MCP_MAX_INPUT_ELEMENTS, + }, + }, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); - return { client, session }; }; const observeAgain = ( diff --git a/tests/boundary/mcp/deferredRuntime.test.ts b/tests/boundary/mcp/deferredRuntime.test.ts index dbdb207e75..61cf7122bd 100644 --- a/tests/boundary/mcp/deferredRuntime.test.ts +++ b/tests/boundary/mcp/deferredRuntime.test.ts @@ -11,6 +11,7 @@ import { } from "node:fs/promises"; import { resolve, join, dirname } from "node:path"; import { expect, it, onTestFinished } from "vitest"; +import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { GENERATED_MCP_TOOL_CATALOG } from "../../fixtures/mcpToolCatalog.js"; import { PROMPT_CONTRACTS } from "../../../src/contracts/promptContracts.js"; @@ -45,8 +46,10 @@ const connect = async ( { versionNegotiation: { mode } }, ); onTestFinished(async () => { - await client.close(); - await transport.close(); + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + ]); }); await client.connect(transport); return { client, stderr: () => stderr }; @@ -197,9 +200,10 @@ it.each([ client.callTool({ name: "list_unknowns", arguments: { limit: 1 } }), ]); expect(results.every((result) => result.isError !== true)).toBe(true); - expect(results[0]?.structuredContent).toEqual({ - result: { items: [], total: 0, offset: 0, next_offset: null }, - }); + expect(results.map((result) => result.structuredContent)).toEqual([ + { result: { items: [], total: 0, offset: 0, next_offset: null } }, + { result: { items: [], total: 0, offset: 0, next_offset: null } }, + ]); expect(await client.listTools()).toEqual(before); const invalid = await client.callTool({ name: "list_unknowns", diff --git a/tests/boundary/mcp/electronMcp.test.ts b/tests/boundary/mcp/electronMcp.test.ts index 8a45fff203..cca7595b1d 100644 --- a/tests/boundary/mcp/electronMcp.test.ts +++ b/tests/boundary/mcp/electronMcp.test.ts @@ -400,12 +400,12 @@ it("exposes endpoint-scoped Electron discovery and inspection as Evidence", asyn expect(inspected.isError).not.toBe(true); expect(inspected.structuredContent).toMatchObject({ normalized_result: { - target: { file_path: expect.stringMatching(/[\\/]index\.html$/u) }, + target: { file_path: join(root, "index.html") }, scripts: { items: [ expect.objectContaining({ frame_id: "frame-main", - file_path: expect.stringMatching(/[\\/]app\.js$/u), + file_path: join(root, "app.js"), cdp_hash: null, length: null, is_module: null, @@ -416,7 +416,7 @@ it("exposes endpoint-scoped Electron discovery and inspection as Evidence", asyn expect.objectContaining({ target_id: "electron-worker", opener_target_id: "electron-page", - file_path: expect.stringMatching(/[\\/]worker\.js$/u), + file_path: join(root, "worker.js"), }), ], }, @@ -684,7 +684,7 @@ it("exposes the target-free static JavaScript application workflow", async () => }) .parse(analyzed.structuredContent); expect(projected.normalized_result).toMatchObject({ - input_path: expect.stringMatching(/rea-electron-static-mcp-/u), + input_path: root, summary: { browser_windows: 3, context_bridge_apis: 2, diff --git a/tests/boundary/mcp/espImageMcp.test.ts b/tests/boundary/mcp/espImageMcp.test.ts index eb18906655..34cc035ab0 100644 --- a/tests/boundary/mcp/espImageMcp.test.ts +++ b/tests/boundary/mcp/espImageMcp.test.ts @@ -14,6 +14,7 @@ import { import { ESP8266_REFUSAL } from "../../../src/domain/firmware/espImage.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { artifactExtractionResultSchema } from "../../../src/domain/artifactGraph.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; // The compiled stdio fixture checks advertised schemas against their dialect and // validates actual tool responses, avoiding a second source server module graph. @@ -62,14 +63,31 @@ it.each(["full", "compact"] as const)( kind: "inventory", collection: "occurrences", logical_paths: ["segment-03.bin", "signature-block-0.bin"], - fields: ["format_metadata", "source_location"], + fields: ["logical_path", "format_metadata", "source_location"], }, }); expect(retained.isError, JSON.stringify(retained)).not.toBe(true); - const rows = JSON.stringify(retained.structuredContent); - expect(rows).toContain('"classification":"mapped"'); - expect(rows).toContain('"scheme":"ecdsa"'); - expect(rows).toContain('"signature_status":"unverified"'); + const view = analysisViewResultSchema.parse( + parseEvidence(retained.structuredContent).normalized_result, + ); + expect(view).toMatchObject({ + kind: "inventory", + items: expect.arrayContaining([ + expect.objectContaining({ + logical_path: "segment-03.bin", + format_metadata: expect.objectContaining({ + classification: "mapped", + }), + }), + expect.objectContaining({ + logical_path: "signature-block-0.bin", + format_metadata: expect.objectContaining({ + scheme: "ecdsa", + signature_status: "unverified", + }), + }), + ]), + }); const extracted = await call("extract_artifact", {}); expect(extracted.isError, JSON.stringify(extracted)).not.toBe(true); const result = artifactExtractionResultSchema.parse( diff --git a/tests/boundary/mcp/evidenceInputContractsMcp.test.ts b/tests/boundary/mcp/evidenceInputContractsMcp.test.ts index 99f00e6beb..26198d415a 100644 --- a/tests/boundary/mcp/evidenceInputContractsMcp.test.ts +++ b/tests/boundary/mcp/evidenceInputContractsMcp.test.ts @@ -1,4 +1,3 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { McpServer } from "@modelcontextprotocol/server"; import { Ajv2020 } from "ajv/dist/2020.js"; import { describe, expect, expectTypeOf, it } from "vitest"; @@ -9,6 +8,7 @@ import { toolContract } from "../../../src/contracts/toolContracts.js"; import { transformAdvertisedInputJsonSchema } from "../../../src/contracts/toolSchemaMetadata.js"; import { toolRegistrationOptions } from "../../../src/server/toolRegistrationOptions.js"; import { advertisedSchemaErrors } from "../../fixtures/mcpAdvertisedSchemaValidity.js"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; describe("Evidence input SDK boundaries", () => { it("borrows SDK Evidence inputs while leaving ordinary request constraints active", async () => { @@ -37,9 +37,9 @@ describe("Evidence input SDK boundaries", () => { expect(invalid.issues).toBeDefined(); }); - it.each(["full", "compact"] as const)( + mcpTest.for(["full", "compact"] as const)( "advertises valid reference inputs after actual SDK conversion (%s)", - async (profile) => { + async (profile, { mcp }) => { const contract = toolContract("build_call_path"); const registration = toolRegistrationOptions(contract); const inputSchema = @@ -53,44 +53,38 @@ describe("Evidence input SDK boundaries", () => { ) : registration.inputSchema; const server = new McpServer({ name: "evidence-contract", version: "1" }); - const client = new Client({ name: "evidence-contract", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); server.registerTool(contract.name, { inputSchema }, async () => ({ content: [], })); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const { tools } = await client.listTools(); - expect(advertisedSchemaErrors(tools)).toEqual([]); - const tool = tools.find(({ name }) => name === contract.name); - if (tool === undefined) - throw new Error("named contract not advertised"); - expect(tool.inputSchema.description).toContain( - '{"kind":"retained-evidence","evidence_id":"ev_..."}', - ); - expect(tool.inputSchema.description).toContain("parent_evidence_id"); - const validate = new Ajv2020({ - strict: false, - validateFormats: false, - }).compile(tool.inputSchema); - expect( - validate({ - functions: [ - { - kind: "retained-evidence", - evidence_id: `ev_${"a".repeat(64)}`, - }, - ], - start: { address: "0x1000" }, - goal: { address: "0x1001" }, - }), - JSON.stringify(validate.errors), - ).toBe(true); - } finally { - await Promise.allSettled([client.close(), server.close()]); - } + const client = await mcp.connect(server, { + name: "evidence-contract", + version: "1", + }); + const { tools } = await client.listTools(); + expect(advertisedSchemaErrors(tools)).toEqual([]); + const tool = tools.find(({ name }) => name === contract.name); + if (tool === undefined) throw new Error("named contract not advertised"); + expect(tool.inputSchema.description).toContain( + '{"kind":"retained-evidence","evidence_id":"ev_..."}', + ); + expect(tool.inputSchema.description).toContain("parent_evidence_id"); + const validate = new Ajv2020({ + strict: false, + validateFormats: false, + }).compile(tool.inputSchema); + expect( + validate({ + functions: [ + { + kind: "retained-evidence", + evidence_id: `ev_${"a".repeat(64)}`, + }, + ], + start: { address: "0x1000" }, + goal: { address: "0x1001" }, + }), + JSON.stringify(validate.errors), + ).toBe(true); }, ); }); diff --git a/tests/boundary/mcp/evmInterfaceMcp.test.ts b/tests/boundary/mcp/evmInterfaceMcp.test.ts index c463e59629..a63b93533c 100644 --- a/tests/boundary/mcp/evmInterfaceMcp.test.ts +++ b/tests/boundary/mcp/evmInterfaceMcp.test.ts @@ -2,7 +2,6 @@ import { advertisedMcpToolSchemas, assertRecordedMcpEvidence, } from "../../fixtures/advertisedMcpTool.js"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { createHash } from "node:crypto"; import { Ajv2020 } from "ajv/dist/2020.js"; import { expect, it, onTestFinished, vi } from "vitest"; @@ -23,6 +22,8 @@ import { EvidenceMcpServer } from "../../../src/server/EvidenceMcpServer.js"; import { createToolResultDelivery } from "../../../src/server/toolResult.js"; import { registerEvmTools } from "../../../src/server/registerEvmTools.js"; import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; +import { ownArtifactSession } from "../../support/mcp/artifactSession.js"; import { createDeferred, createTestBinarySession, @@ -47,110 +48,111 @@ const decodedMetadataFixture = () => { return value; }; -it("preserves shared EVM detail guidance and defaults through full SDK discovery", async () => { - const inspect = vi.fn(async () => { - throw new Error("Discovery must not invoke the provider"); - }); - const server = new EvidenceMcpServer( - { name: "evm-detail", version: "1" }, - {}, - undefined, - createToolResultDelivery(undefined), - ); - registerEvmTools(server, new EvmInterfaceService({ inspect }), silentLogger); - const client = new Client({ name: "evm-detail", version: "1" }); - const [ct, st] = InMemoryTransport.createLinkedPair(); - onTestFinished(async () => { - await client.close(); - await server.close(); - }); - await server.connect(st); - await client.connect(ct); - const { inputSchema } = await advertisedMcpToolSchemas( - client, - "inspect_evm_interface", - ); - expect(inputSchema).toHaveProperty(["properties", "detail"], { - type: "string", - enum: ["complete", "summary"], - default: "complete", - description: expect.stringContaining( - "summary retains that complete Evidence", - ), - }); - expect(inspect).not.toHaveBeenCalled(); -}); +mcpTest( + "preserves shared EVM detail guidance and defaults through full SDK discovery", + async ({ mcp }) => { + const inspect = vi.fn(async () => { + throw new Error("Discovery must not invoke the provider"); + }); + const server = new EvidenceMcpServer( + { name: "evm-detail", version: "1" }, + {}, + undefined, + createToolResultDelivery(undefined), + ); + registerEvmTools( + server, + new EvmInterfaceService({ inspect }), + silentLogger, + ); + const client = await mcp.connect(server, { + name: "evm-detail", + version: "1", + }); + const { inputSchema } = await advertisedMcpToolSchemas( + client, + "inspect_evm_interface", + ); + expect(inputSchema).toHaveProperty(["properties", "detail"], { + type: "string", + enum: ["complete", "summary"], + default: "complete", + description: expect.stringContaining( + "summary retains that complete Evidence", + ), + }); + expect(inspect).not.toHaveBeenCalled(); + }, +); -it("advertises exact valid schemas and records inline EVM interface evidence without a binary target", async () => { - const value = decodedMetadataFixture(); - const inspect = vi.fn(() => - Promise.resolve(ok(evmInterfaceExecution(value))), - ); - const service = new EvmInterfaceService({ - inspect, - }); - const session = createTestBinarySession(() => { - throw new Error("deep provider must not start"); - }); - const server = createServer( - { kind: "session", session }, - { evmInterface: service }, - ); - const client = new Client({ name: "evm-interface-contract", version: "1" }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); - }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); - const schemas = await advertisedMcpToolSchemas( - client, - "inspect_evm_interface", - ); - const { ajv, inputSchema } = schemas; - expect( - ajv.validate(inputSchema, { path: value.artifact.path, encoding: "hex" }), - ).toBe(true); - expect( - ajv.validate(inputSchema, { - path: value.artifact.path, - encoding: "hex", - approval: true, - }), - ).toBe(false); - expect(() => registerEvmTools(server, service, silentLogger)).toThrow( - /inspect_evm_interface.*already registered/, - ); - expect(inspect).not.toHaveBeenCalled(); - expect( - (await client.listTools()).tools.filter( - (tool) => tool.name === "inspect_evm_interface", - ), - ).toHaveLength(1); - const response = await client.callTool({ - name: "inspect_evm_interface", - arguments: { path: value.artifact.path, encoding: "hex" }, - }); - assertRecordedMcpEvidence( - response, - "inspect_evm_interface", - schemas, - session, - value, - "inferred", - ); - expect( - ( - await client.callTool({ - name: "inspect_evm_interface", - arguments: { path: "relative.hex", encoding: "hex" }, - }) - ).isError, - ).toBe(true); -}); +mcpTest( + "advertises exact valid schemas and records inline EVM interface evidence without a binary target", + async ({ mcp, onTestFinished }) => { + const value = decodedMetadataFixture(); + const inspect = vi.fn(() => + Promise.resolve(ok(evmInterfaceExecution(value))), + ); + const service = new EvmInterfaceService({ + inspect, + }); + const session = createTestBinarySession(() => { + throw new Error("deep provider must not start"); + }); + const server = createServer( + { kind: "session", session }, + { evmInterface: service }, + ); + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect(server, { + name: "evm-interface-contract", + version: "1", + }); + const schemas = await advertisedMcpToolSchemas( + client, + "inspect_evm_interface", + ); + const { ajv, inputSchema } = schemas; + expect( + ajv.validate(inputSchema, { path: value.artifact.path, encoding: "hex" }), + ).toBe(true); + expect( + ajv.validate(inputSchema, { + path: value.artifact.path, + encoding: "hex", + approval: true, + }), + ).toBe(false); + expect(() => registerEvmTools(server, service, silentLogger)).toThrow( + /inspect_evm_interface.*already registered/, + ); + expect(inspect).not.toHaveBeenCalled(); + expect( + (await client.listTools()).tools.filter( + (tool) => tool.name === "inspect_evm_interface", + ), + ).toHaveLength(1); + const response = await client.callTool({ + name: "inspect_evm_interface", + arguments: { path: value.artifact.path, encoding: "hex" }, + }); + assertRecordedMcpEvidence( + response, + "inspect_evm_interface", + schemas, + session, + value, + "inferred", + ); + expect( + ( + await client.callTool({ + name: "inspect_evm_interface", + arguments: { path: "relative.hex", encoding: "hex" }, + }) + ).isError, + ).toBe(true); + }, +); it("rejects an EVM provider response with a non-file subject before recording Evidence", async () => { const value = evmInterfaceFixture(); @@ -239,27 +241,29 @@ it.each(["unavailable", "incompatible-output"])( }, ); -it("retains the injected EVM owner and retries failed cleanup on server shutdown", async () => { - const close = vi - .fn() - .mockRejectedValueOnce(new Error("retained EVM worker")) - .mockResolvedValue(undefined); - const service = new EvmInterfaceService({ - close, - inspect: () => Promise.resolve(ok(evmInterfaceExecution())), - }); - const session = createTestBinarySession(() => { - throw new Error("deep provider must not start"); - }); - const server = createServer( - { kind: "session", session }, - { evmInterface: service }, - ); - const client = new Client({ name: "evm-cleanup-retry", version: "1" }); - const [a, b] = InMemoryTransport.createLinkedPair(); - try { - await server.connect(b); - await client.connect(a); +mcpTest( + "retains the injected EVM owner and retries failed cleanup on server shutdown", + async ({ mcp, onTestFinished }) => { + const close = vi + .fn() + .mockRejectedValueOnce(new Error("retained EVM worker")) + .mockResolvedValue(undefined); + const service = new EvmInterfaceService({ + close, + inspect: () => Promise.resolve(ok(evmInterfaceExecution())), + }); + const session = createTestBinarySession(() => { + throw new Error("deep provider must not start"); + }); + const server = createServer( + { kind: "session", session }, + { evmInterface: service }, + ); + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect(server, { + name: "evm-cleanup-retry", + version: "1", + }); const response = await client.callTool({ name: "inspect_evm_interface", arguments: { path: "/artifacts/source-owned.hex", encoding: "hex" }, @@ -268,12 +272,8 @@ it("retains the injected EVM owner and retries failed cleanup on server shutdown await expect(server.close()).rejects.toThrow("REA session cleanup failed"); await server.close(); expect(close).toHaveBeenCalledTimes(2); - } finally { - await client.close(); - await server.close(); - await session.close(); - } -}); + }, +); it("settles the admitted EVM operation after SDK cancellation and accepts a subsequent request", async () => { const entered = createDeferred(); @@ -527,23 +527,25 @@ it.each(["valid", "wrong-subject", "no-subject"])( }, ); -it("rejects summary before provider work when retention is unavailable and preserves complete delivery", async () => { - const value = evmInterfaceFixture(); - const inspect = vi.fn(() => - Promise.resolve(ok(evmInterfaceExecution(value))), - ); - const session = createTestBinarySession(() => { - throw new Error("deep provider must not start"); - }); - const server = createServer( - { kind: "fixed", analysis: session }, - { evmInterface: new EvmInterfaceService({ inspect }) }, - ); - const client = new Client({ name: "evm-fixed-delivery", version: "1" }); - const [a, b] = InMemoryTransport.createLinkedPair(); - try { - await server.connect(b); - await client.connect(a); +mcpTest( + "rejects summary before provider work when retention is unavailable and preserves complete delivery", + async ({ mcp, onTestFinished }) => { + const value = evmInterfaceFixture(); + const inspect = vi.fn(() => + Promise.resolve(ok(evmInterfaceExecution(value))), + ); + const session = createTestBinarySession(() => { + throw new Error("deep provider must not start"); + }); + const server = createServer( + { kind: "fixed", analysis: session }, + { evmInterface: new EvmInterfaceService({ inspect }) }, + ); + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect(server, { + name: "evm-fixed-delivery", + version: "1", + }); const summary = await client.callTool({ name: "inspect_evm_interface", arguments: { @@ -564,9 +566,5 @@ it("rejects summary before provider work when retention is unavailable and prese expect(parseEvidence(complete.structuredContent).normalized_result).toEqual( value, ); - } finally { - await client.close(); - await server.close(); - await session.close(); - } -}); + }, +); diff --git a/tests/boundary/mcp/firmwareContainersMcp.test.ts b/tests/boundary/mcp/firmwareContainersMcp.test.ts index 2d9a42dbb8..dcbd447de5 100644 --- a/tests/boundary/mcp/firmwareContainersMcp.test.ts +++ b/tests/boundary/mcp/firmwareContainersMcp.test.ts @@ -23,6 +23,7 @@ import { const APP = patternBytes(300, 9); const REGION = "family-0xe48bff56/0x10000000-0x1000012b.bin"; +const ABSOLUTE_REGION = "family-0xe48bff57/0x10ffff00-0x10ffffff.bin"; // The compiled stdio fixture checks advertised schemas against their dialect and // validates actual tool responses, avoiding a second source server module graph. @@ -93,9 +94,17 @@ it.each(["full", "compact"] as const)( parseEvidence(extracted.structuredContent).normalized_result, ); outputRoot = result.output_root; + expect(result.containment_verified).toBe(true); + expect(result.coverage.status).toBe("unknown"); expect( Buffer.compare(await readFile(join(result.output_root, REGION)), APP), ).toBe(0); + expect( + Buffer.compare( + await readFile(join(result.output_root, ABSOLUTE_REGION)), + Buffer.alloc(256, 0xef), + ), + ).toBe(0); await call("close_binary", {}); } finally { if (outputRoot !== undefined) diff --git a/tests/boundary/mcp/fitImageMcp.test.ts b/tests/boundary/mcp/fitImageMcp.test.ts index 10aac8249e..d0e31d33c3 100644 --- a/tests/boundary/mcp/fitImageMcp.test.ts +++ b/tests/boundary/mcp/fitImageMcp.test.ts @@ -11,6 +11,7 @@ import { } from "../../../src/artifacts/fit/FitImage.fixture.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { artifactExtractionResultSchema } from "../../../src/domain/artifactGraph.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; // The compiled stdio fixture checks advertised schemas against their dialect and // validates actual tool responses, avoiding a second source server module graph. @@ -48,11 +49,27 @@ it.each(["full", "compact"] as const)( kind: "inventory", collection: "occurrences", logical_paths: ["images/kernel.bin"], - fields: ["format_metadata"], + fields: ["logical_path", "format_metadata"], }, }); expect(retained.isError, JSON.stringify(retained)).not.toBe(true); - expect(JSON.stringify(retained.structuredContent)).toContain("fit-image"); + const view = analysisViewResultSchema.parse( + parseEvidence(retained.structuredContent).normalized_result, + ); + expect(view).toMatchObject({ + kind: "inventory", + items: [ + expect.objectContaining({ + logical_path: "images/kernel.bin", + format_metadata: expect.objectContaining({ + kind: "fit-image", + name: "kernel", + type: "kernel", + arch: "arm64", + }), + }), + ], + }); const extracted = await call("extract_artifact", {}); expect(extracted.isError, JSON.stringify(extracted)).not.toBe(true); const result = artifactExtractionResultSchema.parse( @@ -72,9 +89,13 @@ it.each(["full", "compact"] as const)( await writeFile(path, corrupted); await call("open_binary", { path }); const invalid = await call("inspect_artifact", {}); - expect(JSON.stringify(parseMcpToolError(invalid))).toContain( - "FIT image kernel hash-1 crc32 does not match", - ); + expect(parseMcpToolError(invalid).error).toMatchObject({ + category: "integrity_mismatch", + details: { + reason: "integrity", + detail: expect.stringContaining("kernel hash-1 crc32"), + }, + }); await call("close_binary", {}); } finally { if (outputRoot !== undefined) diff --git a/tests/boundary/mcp/fixedEvidenceConsumersMcp.test.ts b/tests/boundary/mcp/fixedEvidenceConsumersMcp.test.ts index f0b441209d..358a5d9193 100644 --- a/tests/boundary/mcp/fixedEvidenceConsumersMcp.test.ts +++ b/tests/boundary/mcp/fixedEvidenceConsumersMcp.test.ts @@ -282,10 +282,15 @@ it.each([ ...JAVASCRIPT_FEATURE_TRACE_EXAMPLE, application: ANDROID_APPLICATION_GRAPH_EXAMPLE.inventory_evidence[0], }, + issue: { path: ["application", "operation"], reason: "invalid_value" }, }, { name: "project_android_application_graph", input: { inventory_evidence: [application] }, + issue: { + path: ["inventory_evidence", 0, "operation"], + reason: "invalid_value", + }, }, { name: "reconcile_javascript_runtime", @@ -297,13 +302,23 @@ it.each([ }, ], }, + issue: { + path: ["static_layers", 0, "analysis", "normalized_result", "input_path"], + reason: "missing_argument", + }, }, ])( "rejects an authenticated source with the wrong role in $name", - async ({ name, input }) => { + async ({ name, input, issue }) => { const reply = await client.callTool({ name, arguments: input }); expect(parseMcpToolError(reply)).toMatchObject({ - error: { code: "invalid_request" }, + error: { + code: "invalid_request", + details: { + operation: name, + issues: expect.arrayContaining([expect.objectContaining(issue)]), + }, + }, }); await client.ping(); }, diff --git a/tests/boundary/mcp/flutterMcp.test.ts b/tests/boundary/mcp/flutterMcp.test.ts index f5336230e1..d76c022367 100644 --- a/tests/boundary/mcp/flutterMcp.test.ts +++ b/tests/boundary/mcp/flutterMcp.test.ts @@ -1,4 +1,3 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { expect, it, onTestFinished } from "vitest"; import { Ajv2020 } from "ajv/dist/2020.js"; @@ -7,7 +6,6 @@ import { createAnalysisExecution } from "../../../src/application/AnalysisProvid import { assertAdvertisedMcpContracts } from "./mcpContractHarness.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { ok } from "../../../src/domain/result.js"; -import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createFlutterBuildAnalysisProvider } from "../../../src/composition/flutter.js"; import { @@ -280,17 +278,11 @@ it("advertises an exact valid Flutter schema and records inline evidence through const session = createTestBinarySession(() => { throw new Error("deep provider must not start"); }); - const server = createServer( - { kind: "session", session }, - { flutterAnalysis: fakeProvider() }, - ); - onTestFinished(() => server.close()); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - const client = new Client({ name: "flutter-boundary", version: "0.0.0" }); - onTestFinished(() => client.close()); - await client.connect(clientTransport); + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "flutter-boundary", version: "0.0.0" }, + serverOptions: { flutterAnalysis: fakeProvider() }, + }); const { tools } = await client.listTools(); assertAdvertisedMcpContracts( diff --git a/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts b/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts index aada5039c0..127e81e551 100644 --- a/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts +++ b/tests/boundary/mcp/functionComparisonCoverageMcp.test.ts @@ -1,12 +1,11 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { expect, it } from "vitest"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { functionDossierSchema } from "../../../src/domain/native/functionDossier.js"; import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; -import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; const sourceWithCoverage = ( template: typeof FUNCTION_COMPARISON_EXAMPLE.left, @@ -115,119 +114,122 @@ it("keeps incomplete calls unknown through public comparison and verification wh const session = createTestBinarySession(() => { throw new Error("Function comparison must not launch a provider"); }); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "function-coverage-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const left = sourceWithCoverage( - FUNCTION_COMPARISON_EXAMPLE.left, - "coverage-left", - FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, - { status: "complete" }, - ); - const right = sourceWithCoverage( - FUNCTION_COMPARISON_EXAMPLE.right, - "coverage-right", - FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, - { status: "unknown", reason: "Call graph unavailable." }, - ); - expect(left.subject).not.toBeNull(); - expect(right.subject).not.toBeNull(); - expect(left.limitations).not.toContain( - "Artifact identity is unavailable for this observation.", - ); - expect(right.limitations).not.toContain( - "Artifact identity is unavailable for this observation.", - ); - const compared = await client.callTool({ - name: "compare_functions", - arguments: { left, right }, - }); - expect( - compared.isError, - JSON.stringify(compared.structuredContent), - ).not.toBe(true); - const comparison = parseEvidence(compared.structuredContent); - expect(session.evidenceById(left.evidence_id)?.subject).not.toBeNull(); - expect(session.evidenceById(right.evidence_id)?.subject).not.toBeNull(); - expect(comparison.normalized_result).toMatchObject({ - status: "unknown", - summary: { unchanged: 7, changed: 0, unknown: 1 }, - dimensions: expect.arrayContaining([ + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "function-coverage-test", version: "1" }, + serverOptions: { + environment: { + ...process.env, + REA_MCP_MAX_INPUT_ELEMENTS: process.env.REA_MCP_MAX_INPUT_ELEMENTS, + }, + }, + }); + const left = sourceWithCoverage( + FUNCTION_COMPARISON_EXAMPLE.left, + "coverage-left", + FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, + { status: "complete" }, + ); + const right = sourceWithCoverage( + FUNCTION_COMPARISON_EXAMPLE.right, + "coverage-right", + FUNCTION_COMPARISON_EXAMPLE.left.normalized_result, + { status: "unknown", reason: "Call graph unavailable." }, + ); + expect(left.subject).not.toBeNull(); + expect(right.subject).not.toBeNull(); + expect(left.limitations).not.toContain( + "Artifact identity is unavailable for this observation.", + ); + expect(right.limitations).not.toContain( + "Artifact identity is unavailable for this observation.", + ); + const compared = await client.callTool({ + name: "compare_functions", + arguments: { left, right }, + }); + expect(compared.isError, JSON.stringify(compared.structuredContent)).not.toBe( + true, + ); + const comparison = parseEvidence(compared.structuredContent); + expect(session.evidenceById(left.evidence_id)?.subject).toEqual( + expect.any(Object), + ); + expect(session.evidenceById(right.evidence_id)?.subject).toEqual( + expect.any(Object), + ); + expect(comparison.normalized_result).toMatchObject({ + status: "unknown", + summary: { unchanged: 7, changed: 0, unknown: 1 }, + dimensions: expect.arrayContaining([ + expect.objectContaining({ + dimension: "calls", + status: "unknown", + conclusion_kind: "unresolved_branch", + left_count: 0, + right_count: 0, + limitations: ["Right: Call graph unavailable."], + evidence_links: [left.evidence_id, right.evidence_id], + }), + ]), + }); + expect(comparison.evidence_links).toEqual( + expect.arrayContaining([left.evidence_id, right.evidence_id]), + ); + + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "function-comparison" }, + }); + expect(unknowns.isError).not.toBe(true); + expect(unknowns.structuredContent).toMatchObject({ + result: { + items: [ expect.objectContaining({ - dimension: "calls", + domain: "function-comparison", + status: "open", + supporting_evidence_ids: expect.arrayContaining([ + left.evidence_id, + right.evidence_id, + ]), + }), + ], + }, + }); + + const verification = await verify( + async (name, arguments_) => + client.callTool({ name, arguments: arguments_ }), + comparison, + ["calls", "overall", "pseudocode"], + ); + expect(verification.normalized_result).toMatchObject({ + status: "unknown", + summary: { total: 3, passed: 1, failed: 0, unknown: 2 }, + claims: { + items: expect.arrayContaining([ + expect.objectContaining({ + claim_id: "calls", status: "unknown", - conclusion_kind: "unresolved_branch", - left_count: 0, - right_count: 0, - limitations: ["Right: Call graph unavailable."], - evidence_links: [left.evidence_id, right.evidence_id], + observed_status: "unknown", + }), + expect.objectContaining({ + claim_id: "overall", + status: "unknown", + observed_status: "unknown", + }), + expect.objectContaining({ + claim_id: "pseudocode", + status: "pass", + observed_status: "unchanged", }), ]), - }); - expect(comparison.evidence_links).toEqual( - expect.arrayContaining([left.evidence_id, right.evidence_id]), - ); + }, + }); - const unknowns = await client.callTool({ - name: "list_unknowns", - arguments: { domain: "function-comparison" }, - }); - expect(unknowns.isError).not.toBe(true); - expect(unknowns.structuredContent).toMatchObject({ - result: { - items: [ - expect.objectContaining({ - domain: "function-comparison", - status: "open", - supporting_evidence_ids: expect.arrayContaining([ - left.evidence_id, - right.evidence_id, - ]), - }), - ], - }, - }); - - const verification = await verify( - async (name, arguments_) => - client.callTool({ name, arguments: arguments_ }), - comparison, - ["calls", "overall", "pseudocode"], - ); - expect(verification.normalized_result).toMatchObject({ - status: "unknown", - summary: { total: 3, passed: 1, failed: 0, unknown: 2 }, - claims: { - items: expect.arrayContaining([ - expect.objectContaining({ - claim_id: "calls", - status: "unknown", - observed_status: "unknown", - }), - expect.objectContaining({ - claim_id: "overall", - status: "unknown", - observed_status: "unknown", - }), - expect.objectContaining({ - claim_id: "pseudocode", - status: "pass", - observed_status: "unchanged", - }), - ]), - }, - }); - - await assertChangedDespiteUnknownCalls( - async (name, arguments_) => - client.callTool({ name, arguments: arguments_ }), - left, - ); - } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); - } + await assertChangedDespiteUnknownCalls( + async (name, arguments_) => + client.callTool({ name, arguments: arguments_ }), + left, + ); }); diff --git a/tests/boundary/mcp/functionComparisonMcp.test.ts b/tests/boundary/mcp/functionComparisonMcp.test.ts index ae08b1d7c9..100f29963f 100644 --- a/tests/boundary/mcp/functionComparisonMcp.test.ts +++ b/tests/boundary/mcp/functionComparisonMcp.test.ts @@ -1,6 +1,5 @@ import { functionSnapshotFixture } from "../../../src/domain/functionDossierEvidence.fixture.js"; import { ok as resultOk } from "../../../src/domain/result.js"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { describe, expect, it, onTestFinished } from "vitest"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -13,7 +12,6 @@ import { compareFunctions } from "../../../src/domain/functionComparison.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; -import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; import { reconstructionVerificationResultSchema } from "../../../src/domain/reconstructionVerificationSchemas.js"; import { functionDossierSchema } from "../../../src/domain/native/functionDossier.js"; @@ -140,120 +138,110 @@ it.each([ : { relationshipCoverage: left.relationship_coverage }), }, ); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "function-claims", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const compared = await client.callTool({ - name: "compare_functions", - arguments: { left, right }, - }); - expect(compared.isError).not.toBe(true); - const comparison = parseEvidence(compared.structuredContent); - expect(comparison.limitations).not.toContainEqual( - expect.stringContaining( - "without address or generated-name normalization", + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "function-claims", version: "1" }, + }); + const compared = await client.callTool({ + name: "compare_functions", + arguments: { left, right }, + }); + expect(compared.isError).not.toBe(true); + const comparison = parseEvidence(compared.structuredContent); + expect(comparison.limitations).not.toContainEqual( + expect.stringContaining( + "without address or generated-name normalization", + ), + ); + expect(comparison.normalized_result).toMatchObject({ + status: comparisonStatus, + dimensions: expect.arrayContaining( + ["pseudocode", "assembly"].map((dimension) => + expect.objectContaining({ + dimension, + limitations: expect.arrayContaining([ + expect.stringContaining( + "without address or generated-name normalization", + ), + ]), + }), ), - ); - expect(comparison.normalized_result).toMatchObject({ - status: comparisonStatus, - dimensions: expect.arrayContaining( - ["pseudocode", "assembly"].map((dimension) => - expect.objectContaining({ + ), + }); + expect(comparison.parameters).toEqual({ + left_evidence_id: left.evidence_id, + right_evidence_id: right.evidence_id, + }); + expect(comparison.evidence_links).toEqual( + expect.arrayContaining([left.evidence_id, right.evidence_id]), + ); + + const verified = await client.callTool({ + name: "verify_reconstruction", + arguments: { + specification: { + name: "Function pseudocode comparison", + claims: ["pseudocode", "assembly", "cfg", "overall"].map( + (dimension) => ({ + kind: "structural-function", + claim_id: dimension, + title: `${dimension} agrees`, + comparison_evidence_id: comparison.evidence_id, dimension, - limitations: expect.arrayContaining([ - expect.stringContaining( - "without address or generated-name normalization", - ), - ]), }), ), - ), - }); - expect(comparison.parameters).toEqual({ - left_evidence_id: left.evidence_id, - right_evidence_id: right.evidence_id, - }); - expect(comparison.evidence_links).toEqual( - expect.arrayContaining([left.evidence_id, right.evidence_id]), - ); - - const verified = await client.callTool({ - name: "verify_reconstruction", - arguments: { - specification: { - name: "Function pseudocode comparison", - claims: ["pseudocode", "assembly", "cfg", "overall"].map( - (dimension) => ({ - kind: "structural-function", - claim_id: dimension, - title: `${dimension} agrees`, - comparison_evidence_id: comparison.evidence_id, - dimension, - }), - ), - }, }, - }); - expect( - verified.isError, - JSON.stringify(verified.structuredContent), - ).not.toBe(true); - const verification = parseEvidence(verified.structuredContent); - expect(verification.normalized_result).toMatchObject({ - status: verificationStatus, - summary: { - total: 4, - passed: verificationStatus === "pass" ? 4 : 2, - failed: verificationStatus === "fail" ? 2 : 0, - unknown: 0, - }, - claims: { - items: expect.arrayContaining([ - expect.objectContaining({ - dimension: "pseudocode", - status: verificationStatus, - observed_status: comparisonStatus, - comparison_evidence_id: comparison.evidence_id, - left_evidence_ids: [left.evidence_id], - right_evidence_ids: [right.evidence_id], - limitations: expect.arrayContaining([ - expect.stringContaining( - "without address or generated-name normalization", - ), - ]), - }), - ]), - }, - }); - const claims = reconstructionVerificationResultSchema.parse( - verification.normalized_result, - ).claims.items; - for (const claim of claims) { - const caveats = claim.limitations.join(" "); - if (claim.dimension === "cfg") - expect(caveats).not.toMatch(/generated-name normalization/u); - else expect(caveats).toMatch(/generated-name normalization/u); - if (claim.dimension === "assembly" || claim.dimension === "overall") - expect(caveats).toMatch(/Assembly is opaque/u); - else expect(caveats).not.toMatch(/Assembly is opaque/u); - } - expect( - session - .exportEvidenceBundle() - .records.find(({ evidence_id: id }) => id === comparison.evidence_id), - ).toEqual(comparison); - } finally { - await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); - expect(providerLaunches).toBe(0); + }, + }); + expect( + verified.isError, + JSON.stringify(verified.structuredContent), + ).not.toBe(true); + const verification = parseEvidence(verified.structuredContent); + expect(verification.normalized_result).toMatchObject({ + status: verificationStatus, + summary: { + total: 4, + passed: verificationStatus === "pass" ? 4 : 2, + failed: verificationStatus === "fail" ? 2 : 0, + unknown: 0, + }, + claims: { + items: expect.arrayContaining([ + expect.objectContaining({ + dimension: "pseudocode", + status: verificationStatus, + observed_status: comparisonStatus, + comparison_evidence_id: comparison.evidence_id, + left_evidence_ids: [left.evidence_id], + right_evidence_ids: [right.evidence_id], + limitations: expect.arrayContaining([ + expect.stringContaining( + "without address or generated-name normalization", + ), + ]), + }), + ]), + }, + }); + const claims = reconstructionVerificationResultSchema.parse( + verification.normalized_result, + ).claims.items; + for (const claim of claims) { + const caveats = claim.limitations.join(" "); + if (claim.dimension === "cfg") + expect(caveats).not.toMatch(/generated-name normalization/u); + else expect(caveats).toMatch(/generated-name normalization/u); + if (claim.dimension === "assembly" || claim.dimension === "overall") + expect(caveats).toMatch(/Assembly is opaque/u); + else expect(caveats).not.toMatch(/Assembly is opaque/u); } + expect( + session + .exportEvidenceBundle() + .records.find(({ evidence_id: id }) => id === comparison.evidence_id), + ).toEqual(comparison); + expect(providerLaunches).toBe(0); }, ); @@ -284,54 +272,41 @@ describe("function comparison MCP integration", () => { expect(session.recordEvidence(FUNCTION_COMPARISON_EXAMPLE.right).ok).toBe( true, ); - const server = createServer({ kind: "session", session }); - const client = new Client({ - name: "function-comparison-test", - version: "1", + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "function-comparison-test", version: "1" }, + }); + const result = await client.callTool({ + name: "compare_functions", + arguments: { + left: FUNCTION_COMPARISON_EXAMPLE.left, + right: FUNCTION_COMPARISON_EXAMPLE.right, + }, + }); + expect(result.isError).not.toBe(true); + expect(result.structuredContent).toMatchObject({ + normalized_result: { status: expect.any(String) }, + evidence_id: expect.stringMatching(/^ev_[a-f0-9]{64}$/u), + }); + const unknowns = await client.callTool({ + name: "list_unknowns", + arguments: { domain: "function-comparison" }, + }); + expect(unknowns.structuredContent).toMatchObject({ + result: { + items: [ + expect.objectContaining({ + domain: "function-comparison", + recommended_probes: [ + expect.objectContaining({ + rationale: + "Capture complete dossiers for both functions under the same target context.", + }), + ], + }), + ], + }, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const result = await client.callTool({ - name: "compare_functions", - arguments: { - left: FUNCTION_COMPARISON_EXAMPLE.left, - right: FUNCTION_COMPARISON_EXAMPLE.right, - }, - }); - expect(result.isError).not.toBe(true); - expect(result.structuredContent).toMatchObject({ - normalized_result: { status: expect.any(String) }, - evidence_id: expect.stringMatching(/^ev_[a-f0-9]{64}$/u), - }); - const unknowns = await client.callTool({ - name: "list_unknowns", - arguments: { domain: "function-comparison" }, - }); - expect(unknowns.structuredContent).toMatchObject({ - result: { - items: [ - expect.objectContaining({ - domain: "function-comparison", - recommended_probes: [ - expect.objectContaining({ - rationale: - "Capture complete dossiers for both functions under the same target context.", - }), - ], - }), - ], - }, - }); - } finally { - await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); - } }); it("accepts inline Evidence without prior session registration", async () => { @@ -340,31 +315,21 @@ describe("function comparison MCP integration", () => { execute: () => Promise.resolve(observed(null)), close: () => Promise.resolve(resultOk(null)), })); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "function-link-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const result = await client.callTool({ - name: "compare_functions", - arguments: { - left: FUNCTION_COMPARISON_EXAMPLE.left, - right: FUNCTION_COMPARISON_EXAMPLE.right, - }, - }); - expect(result.isError).not.toBe(true); - expect(result.structuredContent).toMatchObject({ - normalized_result: { status: expect.any(String) }, - }); - } finally { - await Promise.allSettled([ - client.close(), - server.close(), - session.close(), - ]); - } + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "function-link-test", version: "1" }, + }); + const result = await client.callTool({ + name: "compare_functions", + arguments: { + left: FUNCTION_COMPARISON_EXAMPLE.left, + right: FUNCTION_COMPARISON_EXAMPLE.right, + }, + }); + expect(result.isError).not.toBe(true); + expect(result.structuredContent).toMatchObject({ + normalized_result: { status: expect.any(String) }, + }); }); }); diff --git a/tests/boundary/mcp/functionComparisonStringDecodingMcp.test.ts b/tests/boundary/mcp/functionComparisonStringDecodingMcp.test.ts index 25abaad765..651e2f834f 100644 --- a/tests/boundary/mcp/functionComparisonStringDecodingMcp.test.ts +++ b/tests/boundary/mcp/functionComparisonStringDecodingMcp.test.ts @@ -1,5 +1,4 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it } from "vitest"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; @@ -9,8 +8,8 @@ import { type FunctionComparisonResult as Comparison, } from "../../../src/domain/functionComparisonSchemas.js"; -import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; const template = FUNCTION_COMPARISON_EXAMPLE.left; const base = functionDossierSchema.parse(template.normalized_result); @@ -88,20 +87,22 @@ type Compare = ( right: ReturnType, ) => Promise; -const session = createTestBinarySession(() => { - throw new Error("Function comparison must not launch a provider"); -}); -const server = createServer({ kind: "session", session }); -const client = new Client({ - name: "function-string-decoding-test", - version: "1", -}); -const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); let compare: Compare; -beforeAll(async () => { - await server.connect(serverTransport); - await client.connect(clientTransport); +beforeEach(async () => { + const session = createTestBinarySession(() => { + throw new Error("Function comparison must not launch a provider"); + }); + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "function-string-decoding-test", version: "1" }, + serverOptions: { + environment: { + ...process.env, + REA_MCP_MAX_INPUT_ELEMENTS: process.env.REA_MCP_MAX_INPUT_ELEMENTS, + }, + }, + }); compare = async (left, right) => { const response = await client.callTool({ name: "compare_functions", @@ -117,10 +118,6 @@ beforeAll(async () => { }; }); -afterAll(async () => { - await Promise.allSettled([client.close(), server.close(), session.close()]); -}); - const stringsNames = (result: Comparison) => result.dimensions.find(({ dimension }) => dimension === "strings_names"); @@ -260,6 +257,8 @@ describe("compare_functions referenced string decoding", () => { status: "changed", left_count: 2, right_count: 2, + left_digest: expect.stringMatching(/^[a-f0-9]{64}$/u), + right_digest: expect.stringMatching(/^[a-f0-9]{64}$/u), }); expect(stringsNames(changed)?.left_digest).not.toBe( stringsNames(changed)?.right_digest, diff --git a/tests/boundary/mcp/gdbCancellationMcp.test.ts b/tests/boundary/mcp/gdbCancellationMcp.test.ts index 9281ad63aa..492ab15807 100644 --- a/tests/boundary/mcp/gdbCancellationMcp.test.ts +++ b/tests/boundary/mcp/gdbCancellationMcp.test.ts @@ -8,6 +8,7 @@ import { Ajv2020 } from "ajv/dist/2020.js"; import { expect, it } from "vitest"; import { z } from "zod"; +import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { AnalysisCapabilityUnavailableError, @@ -125,9 +126,14 @@ process.stdin.on("data", (chunk) => { expect(exitedDuringRetry).toBe(true); expect(manager.status(sessionId)).toBeUndefined(); } finally { - await Promise.allSettled([client.close(), server.close()]); - await manager.closeAll().catch(() => undefined); - await rm(directory, { recursive: true, force: true }); + await closeVerifierResources([ + () => client.close(), + () => server.close(), + async () => { + await manager.closeAll(); + await rm(directory, { recursive: true, force: true }); + }, + ]); } }, ); @@ -154,9 +160,31 @@ process.stdin.on("data", async (chunk) => { `, ); await chmod(provider, 0o700); + const fixtureOwners = new WeakMap< + object, + Parameters[0] + >(); + const stopResults: string[] = []; const manager = new GdbSessionManager({ platform: "linux", environment: { REA_GDB_COMMAND: provider }, + // This fake debugger never starts an inferior. Its held MI command is + // unsynchronized after cancellation, so the real manager cannot infer + // that fixture fact from the debugger protocol. + stopProcess: async (session, name) => { + let owner = fixtureOwners.get(session); + if (owner === undefined) { + owner = Object.create(session, { + detachInferior: { + value: async () => ({ status: "no-inferior" as const }), + }, + }) as Parameters[0]; + fixtureOwners.set(session, owner); + } + const stopped = await stopOwnedDebuggerProcess(owner, name); + stopResults.push(stopped.status); + return stopped; + }, }); const server = new McpServer({ name: "gdb-cancellation", version: "1" }); registerGdbTools( @@ -196,10 +224,17 @@ process.stdin.on("data", async (chunk) => { ); expect(status.state).toBe("closed"); expect(status).toHaveProperty("diagnostics_truncated"); + await manager.closeAll(); + expect(stopResults).toContain("stopped"); } finally { - await Promise.allSettled([client.close(), server.close()]); - await manager.closeAll().catch(() => undefined); - await rm(directory, { recursive: true, force: true }); + await closeVerifierResources([ + () => client.close(), + () => server.close(), + async () => { + await manager.closeAll(); + await rm(directory, { recursive: true, force: true }); + }, + ]); } }, ); @@ -279,9 +314,14 @@ process.stdin.on("data", (chunk) => { inferior_tty_verified: true, }); } finally { - await Promise.allSettled([client.close(), server.close()]); - await manager.closeAll().catch(() => undefined); - await rm(directory, { recursive: true, force: true }); + await closeVerifierResources([ + () => client.close(), + () => server.close(), + async () => { + await manager.closeAll(); + await rm(directory, { recursive: true, force: true }); + }, + ]); } }, ); @@ -344,9 +384,14 @@ process.stdin.on("data", (chunk) => { ); expect(status.mi_invalid_utf8).toBe(true); } finally { - await Promise.allSettled([client.close(), server.close()]); - await manager.closeAll().catch(() => undefined); - await rm(directory, { recursive: true, force: true }); + await closeVerifierResources([ + () => client.close(), + () => server.close(), + async () => { + await manager.closeAll(); + await rm(directory, { recursive: true, force: true }); + }, + ]); } }, ); @@ -458,7 +503,7 @@ it("records partial GDB Evidence from an MCP command error", async () => { }, }); } finally { - await Promise.allSettled([client.close(), server.close()]); + await closeVerifierResources([() => client.close(), () => server.close()]); } }); @@ -532,8 +577,14 @@ it("recovers only the owning debugger cleanup and reopens admitted analysis", as (await analysisSession.execute("address_name", { address: "0x1" })).ok, ).toBe(true); } finally { - await Promise.allSettled([client.close(), server.close()]); - await analysisSession.close(); + await closeVerifierResources([ + () => client.close(), + () => server.close(), + async () => { + const closed = await analysisSession.close(); + if (!closed.ok) throw closed.error; + }, + ]); } }); @@ -628,6 +679,6 @@ it("records partial GDB startup Evidence from an MCP start error", async () => { }, }); } finally { - await Promise.allSettled([client.close(), server.close()]); + await closeVerifierResources([() => client.close(), () => server.close()]); } }); diff --git a/tests/boundary/mcp/ghidraMcpHarness.ts b/tests/boundary/mcp/ghidraMcpHarness.ts index 8be0b1fdc7..5b6a5c5b3e 100644 --- a/tests/boundary/mcp/ghidraMcpHarness.ts +++ b/tests/boundary/mcp/ghidraMcpHarness.ts @@ -2,7 +2,7 @@ import { ok as javaProbeOk, ok } from "../../../src/domain/result.js"; import type { GhidraJavaObservation } from "../../../src/ghidra/GhidraJavaProbe.js"; import { expect } from "vitest"; import { parseEvidence } from "../../../src/domain/evidence.js"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { composeBinarySession } from "../../../src/application/binary/BinarySessionComposition.js"; import type { BinarySession } from "../../../src/application/binary/BinarySession.js"; import { AnalysisProviderRegistry } from "../../../src/application/binary/AnalysisProviderRegistry.js"; @@ -15,7 +15,6 @@ import type { GhidraProviderClientFactory } from "../../../src/ghidra/GhidraProv import { GHIDRA_SESSION_CAPABILITIES } from "../../../src/ghidra/GhidraSessionValues.js"; import type { GhidraSessionInfo } from "../../../src/ghidra/GhidraSessionValues.js"; import { silentLogger } from "../../../src/logger.js"; -import { createServer } from "../../../src/server/createServer.js"; import { ghidraFunctionClassification, ghidraFunctionDossier, @@ -33,6 +32,7 @@ export const connectGhidraMcp = async ( readonly pdb?: GhidraSessionInfo["pdb"]; readonly sessionTarget?: GhidraSessionInfo["target"]; readonly attestDosTargetSnapshot?: ReturnType["attestDosTargetSnapshot"]; + readonly close?: ReturnType["close"]; } = {}, ) => { const calls: GhidraOperation[] = []; @@ -52,7 +52,7 @@ export const connectGhidraMcp = async ( if (execute !== undefined) return execute(operation, input, options); return Promise.resolve(ok(ghidraFixtureResult(operation, input))); }, - close: () => Promise.resolve(ok(null)), + close: fixture.close ?? (() => Promise.resolve(ok(null))), ...(fixture.attestDosTargetSnapshot === undefined ? {} : { attestDosTargetSnapshot: fixture.attestDosTargetSnapshot }), @@ -69,20 +69,18 @@ export const connectGhidraMcp = async ( const session = composeBinarySession( new AnalysisProviderRegistry([provider]), ); - const server = createServer( - { kind: "session", session }, - { logger: silentLogger }, - ); - const mcp = new Client({ name, version: "1.0.0" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - const close = async () => { - await Promise.allSettled([mcp.close(), server.close()]); - await session.close(); - }; + const { client: mcp, close } = await createApplicationMcpHarness({ + session, + clientIdentity: { name, version: "1.0.0" }, + serverOptions: { + logger: silentLogger, + environment: { + ...process.env, + REA_MCP_MAX_INPUT_ELEMENTS: process.env.REA_MCP_MAX_INPUT_ELEMENTS, + }, + }, + }); try { - await server.connect(serverTransport); - await mcp.connect(clientTransport); const opened = await mcp.callTool({ name: "open_binary", arguments: { path: fixture.targetPath ?? process.execPath }, @@ -91,16 +89,19 @@ export const connectGhidraMcp = async ( throw new Error( `The Ghidra MCP harness could not open its target: ${JSON.stringify(opened)}`, ); + return { calls, mcp, session, close }; } catch (cause: unknown) { - await close(); + try { + await close(); + } catch (cleanup: unknown) { + throw new AggregateError( + [cause, cleanup], + "Ghidra MCP harness setup and cleanup failed", + { cause }, + ); + } throw cause; } - return { - calls, - mcp, - session, - close, - }; }; export const sessionEvidence = (session: BinarySession, value: unknown) => { diff --git a/tests/boundary/mcp/ghidraMcpHarnessLifecycle.test.ts b/tests/boundary/mcp/ghidraMcpHarnessLifecycle.test.ts new file mode 100644 index 0000000000..aaaf396a09 --- /dev/null +++ b/tests/boundary/mcp/ghidraMcpHarnessLifecycle.test.ts @@ -0,0 +1,126 @@ +import { Client } from "@modelcontextprotocol/client"; +import { expect, it, vi } from "vitest"; + +import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; +import { err, ok } from "../../../src/domain/result.js"; +import { connectGhidraMcp } from "./ghidraMcpHarness.js"; + +const failureLeaves = (reason: unknown): unknown[] => + reason instanceof AggregateError + ? reason.errors.flatMap(failureLeaves) + : [reason]; + +it("reports independent cleanup failures and retries only retained owners", async () => { + const sdkFailure = new Error("controlled client close failure"); + const providerFailure = new ProviderCleanupError("ghidra", ["fixture"], { + reason: "controlled provider close failure", + }); + let allowProviderClose = false; + const closeProvider = vi.fn(async () => { + if (!allowProviderClose) return err(providerFailure); + providerFailure.resolveCleanup(); + return ok(null); + }); + const harness = await connectGhidraMcp("ghidra-harness-close", undefined, { + close: closeProvider, + }); + expect( + ( + await harness.mcp.callTool({ + name: "address_name", + arguments: { address: "0x401000", document: "fixture" }, + }) + ).isError, + ).not.toBe(true); + const closeClient = vi + .spyOn(harness.mcp, "close") + .mockRejectedValueOnce(sdkFailure); + try { + const first = harness.close(); + expect(harness.close()).toBe(first); + const failure: unknown = await first.catch((cause: unknown) => cause); + expect(failureLeaves(failure)).toEqual( + expect.arrayContaining([ + sdkFailure, + expect.objectContaining({ cause: providerFailure }), + ]), + ); + expect(failureLeaves(failure)).toHaveLength(2); + expect(await harness.session.status()).toMatchObject({ + cleanup_pending: true, + }); + + const second: unknown = await harness + .close() + .catch((cause: unknown) => cause); + expect(failureLeaves(second)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ cause: providerFailure }), + ]), + ); + expect(closeClient).toHaveBeenCalledTimes(2); + expect(closeProvider).toHaveBeenCalledTimes(2); + + allowProviderClose = true; + await harness.close(); + expect(closeClient).toHaveBeenCalledTimes(2); + expect(closeProvider).toHaveBeenCalledTimes(3); + expect(providerFailure.cleanupIncomplete).toBe(false); + expect(await harness.session.status()).toMatchObject({ + cleanup_pending: false, + }); + await harness.close(); + expect(closeProvider).toHaveBeenCalledTimes(3); + } finally { + allowProviderClose = true; + await harness.close(); + closeClient.mockRestore(); + } +}); + +it("preserves startup and cleanup failures while retrying the retained endpoint", async () => { + const startup = new Error("controlled handshake failure"); + const cleanup = new Error("controlled client close failure"); + const connect = vi + .spyOn(Client.prototype, "connect") + .mockRejectedValueOnce(startup); + const close = vi + .spyOn(Client.prototype, "close") + .mockRejectedValueOnce(cleanup); + try { + const failure: unknown = await connectGhidraMcp( + "ghidra-harness-startup", + ).catch((cause: unknown) => cause); + expect(failureLeaves(failure)).toEqual( + expect.arrayContaining([startup, cleanup]), + ); + expect(failureLeaves(failure)).toHaveLength(2); + expect(close).toHaveBeenCalledTimes(2); + } finally { + connect.mockRestore(); + close.mockRestore(); + } +}); + +it("preserves an open failure when endpoint cleanup also fails", async () => { + const startup = new Error("controlled open failure"); + const cleanup = new Error("controlled client close failure"); + const callTool = vi + .spyOn(Client.prototype, "callTool") + .mockRejectedValueOnce(startup); + const close = vi + .spyOn(Client.prototype, "close") + .mockRejectedValueOnce(cleanup); + try { + const failure: unknown = await connectGhidraMcp( + "ghidra-harness-open", + ).catch((cause: unknown) => cause); + expect(failureLeaves(failure)).toEqual( + expect.arrayContaining([startup, cleanup]), + ); + expect(failureLeaves(failure)).toHaveLength(2); + } finally { + callTool.mockRestore(); + close.mockRestore(); + } +}); diff --git a/tests/boundary/mcp/goBinaryMcp.test.ts b/tests/boundary/mcp/goBinaryMcp.test.ts index 3c682480f9..77404a224a 100644 --- a/tests/boundary/mcp/goBinaryMcp.test.ts +++ b/tests/boundary/mcp/goBinaryMcp.test.ts @@ -4,6 +4,7 @@ import { Ajv2020 } from "ajv/dist/2020.js"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import { readAnalysisSnapshot } from "../../../src/application/binary/AnalysisSnapshotFiles.js"; import { GoBinaryService } from "../../../src/application/go/GoBinaryService.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; @@ -26,6 +27,7 @@ import { createGoBinaryPathFailureFixture, } from "../../fixtures/go/image.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { mcpTest } from "../../support/mcp/mcpFixture.js"; @@ -39,14 +41,17 @@ const goMcpTest = mcpTest.extend<{ const session = createTestBinarySession(() => { throw new Error("Go metadata must not start a deep provider"); }); - try { + await withVerifierCleanup(async () => { const client = await mcp.connect( createServer({ kind: "session", session }), ); await use({ client, session }); - } finally { - await session.close(); - } + }, [ + async () => { + const closed = await session.close(); + if (!closed.ok) throw closed.error; + }, + ]); }, }); @@ -74,7 +79,9 @@ mcpTest( return result; }, }); - try { + let inspection: ReturnType | undefined; + let closing: ReturnType | undefined; + await withVerifierCleanup(async () => { const client = await mcp.connect( createServer({ kind: "session", session }, { goBinary: service }), ); @@ -86,17 +93,21 @@ mcpTest( client.setNotificationHandler("notifications/progress", () => { closeStarted.resolve(); }); - const inspection = client.callTool({ + inspection = client.callTool({ name: "inspect_go_binary", arguments: { path }, }); - await entered.promise; - const closing = client.callTool({ + await awaitAcknowledgement(entered.promise, inspection, "Go inspection"); + closing = client.callTool({ name: "close_binary", arguments: { snapshot_path: snapshotPath }, _meta: { progressToken: "go-close-race" }, }); - await closeStarted.promise; + await awaitAcknowledgement( + closeStarted.promise, + closing, + "Go snapshot close", + ); release.resolve(); const inspected = await inspection; expect(inspected.isError, JSON.stringify(inspected.content)).not.toBe( @@ -117,10 +128,19 @@ mcpTest( if (!snapshot.ok) throw snapshot.error; expect(snapshot.value.evidence_bundle.records).toContainEqual(evidence); expect(session.evidenceById(evidence.evidence_id)).toBeUndefined(); - } finally { - release.resolve(); - await session.close(); - } + }, [ + () => release.resolve(), + async () => { + await Promise.allSettled([ + ...(inspection === undefined ? [] : [inspection]), + ...(closing === undefined ? [] : [closing]), + ]); + }, + async () => { + const closed = await session.close(); + if (!closed.ok) throw closed.error; + }, + ]); }, ); diff --git a/tests/boundary/mcp/godotPckMcp.test.ts b/tests/boundary/mcp/godotPckMcp.test.ts index 2a8eee83b0..4c47f381fd 100644 --- a/tests/boundary/mcp/godotPckMcp.test.ts +++ b/tests/boundary/mcp/godotPckMcp.test.ts @@ -1,7 +1,10 @@ import { writeFile, unlink } from "node:fs/promises"; +import { createHash } from "node:crypto"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { afterAll, expect, it } from "vitest"; +import type { Client } from "@modelcontextprotocol/client"; +import { afterAll, expect, onTestFinished } from "vitest"; +import { mcpTest as it } from "../../support/mcp/mcpFixture.js"; +import { ownArtifactSession } from "../../support/mcp/artifactSession.js"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; @@ -11,6 +14,8 @@ import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; import { createServer } from "../../../src/server/createServer.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { artifactInspectionResultSchema } from "../../../src/domain/artifactInspection.js"; +import { artifactIdForContent } from "../../../src/domain/artifactIdentity.js"; +import type { ArtifactOccurrence } from "../../../src/domain/artifactGraph.js"; import { artifactInventorySummaryViewResultSchema, analysisViewResultSchema, @@ -91,39 +96,63 @@ const assertInvalidNodeField = async ( }); }; +const expectedPckMembers = ( + members: readonly { path: string; data: string }[], + occurrences: readonly ArtifactOccurrence[], +) => { + const expected = members.map((member) => { + const sha256 = createHash("sha256").update(member.data).digest("hex"); + return { + logical_path: member.path.slice("res://".length), + source_path: member.path, + artifact_id: artifactIdForContent(sha256), + sha256, + size: Buffer.byteLength(member.data), + }; + }); + expect(occurrences).toHaveLength(expected.length); + expect(occurrences).toEqual( + expect.arrayContaining( + expected.map(({ logical_path, source_path, artifact_id }) => + expect.objectContaining({ logical_path, source_path, artifact_id }), + ), + ), + ); + return expected; +}; + const profiles = ["full", "compact"] as const; +const members = [ + { path: "res://a.txt", data: "a" }, + { path: "res://b.txt", data: "b" }, +]; const inventoryCases = profiles.flatMap((profile) => [ { profile, version: 1 as const }, { profile, version: 4 as const }, ]); -it.each(inventoryCases)("retains PCK ($profile/v$version)", async (input) => { - const { profile, version } = input; - const root = await createTestTempDirectory("rea-pck-mcp-"); - const path = join(root, "game.pck"); - const members = [ - { path: "res://a.txt", data: "a" }, - { path: "res://b.txt", data: "b" }, - ]; - await writeFile( - path, - version === 1 ? buildGodotPck1(members) : buildGodotPck(4, members), - ); - const session = createTestBinarySession(new ArtifactProvider({})); - const server = createServer( - { kind: "session", session }, - { - environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, - }, - ); - const client = new Client( - { name: "pck-contract-test", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); +it.for(inventoryCases)( + "retains PCK ($profile/v$version)", + async (input, { mcp }) => { + const { profile, version } = input; + const root = await createTestTempDirectory("rea-pck-mcp-"); + const path = join(root, "game.pck"); + await writeFile( + path, + version === 1 ? buildGodotPck1(members) : buildGodotPck(4, members), + ); + const session = createTestBinarySession(new ArtifactProvider({})); + ownArtifactSession(session, onTestFinished); + const server = createServer( + { kind: "session", session }, + { + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, + }, + ); + const client = await mcp.connect( + server, + { name: "pck-contract-test", version: "1" }, + schemaValidation, + ); const validators = await viewValidators(client); expect( (await client.callTool({ name: "open_binary", arguments: { path } })) @@ -165,9 +194,10 @@ it.each(inventoryCases)("retains PCK ($profile/v$version)", async (input) => { profile === "full" ? validators : undefined, ); const wanted = graph.occurrences.filter((row) => row.logical_path !== "."); + const expectedMembers = expectedPckMembers(members, wanted); const occurrences = await view({ collection: "occurrences", - logical_paths: wanted.map((row) => row.logical_path), + logical_paths: expectedMembers.map((member) => member.logical_path), fields: [ "logical_path", "source_path", @@ -189,9 +219,14 @@ it.each(inventoryCases)("retains PCK ($profile/v$version)", async (input) => { format_metadata: row.format_metadata, })), ); - const ids = wanted.flatMap((row) => - row.artifact_id === null ? [] : [row.artifact_id], + expect(occurrences.items).toEqual( + expect.arrayContaining( + expectedMembers.map(({ logical_path, source_path, artifact_id }) => + expect.objectContaining({ logical_path, source_path, artifact_id }), + ), + ), ); + const ids = expectedMembers.map((member) => member.artifact_id); const nodes = await view({ collection: "nodes", artifact_ids: ids, @@ -207,6 +242,16 @@ it.each(inventoryCases)("retains PCK ($profile/v$version)", async (input) => { size: row.size, })), ); + expect(nodes.items).toHaveLength(expectedMembers.length); + expect(nodes.items).toEqual( + expect.arrayContaining( + expectedMembers.map(({ artifact_id, sha256, size }) => ({ + artifact_id, + sha256, + size, + })), + ), + ); await unlink(path); const historical = await view({ collection: "occurrences", @@ -226,14 +271,12 @@ it.each(inventoryCases)("retains PCK ($profile/v$version)", async (input) => { parent_occurrence_id: row.parent_occurrence_id, })), ); - } finally { - await client.close(); - await server.close(); - await session.close(); - } -}); + }, +); -it("carries unknown encrypted-directory coverage through compact MCP delivery", async () => { +it("carries unknown encrypted-directory coverage through compact MCP delivery", async ({ + mcp, +}) => { const root = await createTestTempDirectory("rea-pck-encrypted-mcp-"); const path = join(root, "game.pck"); await writeFile( @@ -243,39 +286,40 @@ it("carries unknown encrypted-directory coverage through compact MCP delivery", }), ); const session = createTestBinarySession(new ArtifactProvider({})); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "pck-unknown-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - await client.callTool({ name: "open_binary", arguments: { path } }); - const reply = await client.callTool({ - name: "inspect_artifact", - arguments: { detail: "summary" }, - }); - expect(reply.isError, JSON.stringify(reply.structuredContent)).not.toBe( - true, - ); - const summary = artifactInventorySummaryViewResultSchema.parse( - parseEvidence(reply.structuredContent).normalized_result, - ); - expect(summary.summary.source_coverage).toEqual({ - scope: "container-entries", - status: "unknown", - }); - expect(summary.summary.root_format_metadata).toMatchObject({ - directory_status: "encrypted", - declared_file_count: 1, - }); - expect(summary.summary.available_counts.occurrences).toBe(1); - expect(summary.unknowns.join(" ")).toContain( - "missing matches do not prove source absence", - ); - } finally { - await client.close(); - await server.close(); - await session.close(); - } + ownArtifactSession(session, onTestFinished); + const server = createServer( + { kind: "session", session }, + { + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: "compact" }, + }, + ); + const client = await mcp.connect(server, { + name: "pck-unknown-test", + version: "1", + }); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + const reply = await client.callTool({ + name: "inspect_artifact", + arguments: { detail: "summary" }, + }); + expect(reply.isError, JSON.stringify(reply.structuredContent)).not.toBe(true); + const summary = artifactInventorySummaryViewResultSchema.parse( + parseEvidence(reply.structuredContent).normalized_result, + ); + expect(summary.summary.source_coverage).toEqual({ + scope: "container-entries", + status: "unknown", + }); + expect(summary.summary.root_format_metadata).toMatchObject({ + directory_status: "encrypted", + declared_file_count: 1, + }); + expect(summary.summary.available_counts.occurrences).toBe(1); + expect(summary.unknowns.join(" ")).toContain( + "missing matches do not prove source absence", + ); }); diff --git a/tests/boundary/mcp/hexRecordsMcp.test.ts b/tests/boundary/mcp/hexRecordsMcp.test.ts index 5d7ecdda39..5d144967e8 100644 --- a/tests/boundary/mcp/hexRecordsMcp.test.ts +++ b/tests/boundary/mcp/hexRecordsMcp.test.ts @@ -8,6 +8,7 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { SREC_S37 } from "../../../src/domain/firmware/hexRecords.fixture.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { artifactExtractionResultSchema } from "../../../src/domain/artifactGraph.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; // The compiled stdio fixture checks advertised schemas against their dialect and // validates actual tool responses, avoiding a second source server module graph. @@ -48,13 +49,22 @@ it.each(["full", "compact"] as const)( kind: "inventory", collection: "occurrences", logical_paths: ["0x08000000-0x08000003.bin"], - fields: ["format_metadata"], + fields: ["logical_path", "format_metadata"], }, }); expect(retained.isError, JSON.stringify(retained)).not.toBe(true); - expect(JSON.stringify(retained.structuredContent)).toContain( - "hex-record-region", + const view = analysisViewResultSchema.parse( + parseEvidence(retained.structuredContent).normalized_result, ); + expect(view).toMatchObject({ + kind: "inventory", + items: [ + { + logical_path: "0x08000000-0x08000003.bin", + format_metadata: { kind: "hex-record-region" }, + }, + ], + }); const extracted = await call("extract_artifact", {}); expect(extracted.isError, JSON.stringify(extracted)).not.toBe(true); const result = artifactExtractionResultSchema.parse( diff --git a/tests/boundary/mcp/investigationAnalysisMcp.test.ts b/tests/boundary/mcp/investigationAnalysisMcp.test.ts index 56c6ae4f9e..109ee90617 100644 --- a/tests/boundary/mcp/investigationAnalysisMcp.test.ts +++ b/tests/boundary/mcp/investigationAnalysisMcp.test.ts @@ -140,6 +140,7 @@ it("keeps call-path publication ahead of close_binary record draining", async () const closeReached = createDeferred(); const releasePreparation = createDeferred(); const send = serverTransport.send.bind(serverTransport); + let preparationToken: string | number | undefined; serverTransport.send = async (message, options) => { await send(message, options); if ( @@ -148,12 +149,16 @@ it("keeps call-path publication ahead of close_binary record draining", async () "params" in message && typeof message.params === "object" && message.params !== null && - "message" in message.params + "progressToken" in message.params && + (typeof message.params.progressToken === "string" || + typeof message.params.progressToken === "number") ) { - if (message.params.message === "prepare: Preparing build_call_path") { + if (preparationToken === undefined) { + // The first request cannot advance until this notification is released. + preparationToken = message.params.progressToken; preparationReached.resolve(); await releasePreparation.promise; - } else if (message.params.message === "close_binary: started") { + } else if (message.params.progressToken !== preparationToken) { closeReached.resolve(); } } diff --git a/tests/boundary/mcp/javascriptFactsViewMcp.test.ts b/tests/boundary/mcp/javascriptFactsViewMcp.test.ts index 5c46881f11..38b27c241e 100644 --- a/tests/boundary/mcp/javascriptFactsViewMcp.test.ts +++ b/tests/boundary/mcp/javascriptFactsViewMcp.test.ts @@ -364,11 +364,9 @@ it("uses exact evidence locations without normalizing display properties or drop expect(discovery.page.rows.map(({ evidence }) => evidence)).toEqual( all.page.rows.map(({ evidence }) => evidence), ); - expect( - all.page.rows.every( - ({ properties }) => JSON.stringify(properties) === '{"nullable":null}', - ), - ).toBe(true); + expect(all.page.rows.map(({ properties }) => properties)).toEqual( + all.page.rows.map(() => ({ nullable: null })), + ); expect(all.page.rows.map(({ evidence }) => evidence)).toEqual( node.observations.map(({ evidence }) => evidence), ); diff --git a/tests/boundary/mcp/javascriptSourceSelection.test.ts b/tests/boundary/mcp/javascriptSourceSelection.test.ts index 953f4bd98b..ef614f0ee9 100644 --- a/tests/boundary/mcp/javascriptSourceSelection.test.ts +++ b/tests/boundary/mcp/javascriptSourceSelection.test.ts @@ -7,6 +7,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { join, resolve } from "node:path"; import { expect, it } from "vitest"; import { z } from "zod"; +import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; @@ -76,8 +77,10 @@ it("resolves ESM whitespace specifiers through the public application tool", asy ]), ); } finally { - await client.close(); - await transport.close(); + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + ]); } }, 60_000); @@ -107,8 +110,10 @@ it("retains missing ASAR native-addon inventory through the public application t ), ).toBe(true); } finally { - await client.close(); - await transport.close(); + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + ]); } }, 60_000); @@ -234,8 +239,10 @@ it("advertises exact source selection and retains its scope through summary, Evi ).toBe(1); await client.ping(); } finally { - await client.close(); - await transport.close(); + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + ]); } }, 60_000); @@ -303,8 +310,10 @@ it("selects nested ASAR paths and distinguishes missing inventoried bytes from i }); await client.ping(); } finally { - await client.close(); - await transport.close(); + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + ]); } }, 60_000); @@ -344,8 +353,10 @@ it("accepts ASAR headers with long legal JSON whitespace through the public appl .digest("hex"), ); } finally { - await client.close(); - await transport.close(); + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + ]); } }, 60_000); @@ -379,7 +390,9 @@ it("does not parse ASAR-looking JSON beyond the declared JSON extent through MCP }); await client.ping(); } finally { - await client.close(); - await transport.close(); + await closeVerifierResources([ + () => client.close(), + () => transport.close(), + ]); } }, 60_000); diff --git a/tests/boundary/mcp/jebMcp.test.ts b/tests/boundary/mcp/jebMcp.test.ts index 3db3f65a62..8509fb7752 100644 --- a/tests/boundary/mcp/jebMcp.test.ts +++ b/tests/boundary/mcp/jebMcp.test.ts @@ -1,4 +1,3 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; import { afterAll, expect, it, onTestFinished } from "vitest"; import { z } from "zod"; @@ -11,8 +10,8 @@ import { createAnalysisExecution } from "../../../src/application/AnalysisProvid import { toolContract } from "../../../src/contracts/toolContracts.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { ok } from "../../../src/domain/result.js"; -import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { expandToolAvailability, @@ -50,23 +49,12 @@ it("discovers current JEB operation support and recovery through binary_session" const session = createTestBinarySession(() => { throw new Error("Status must not start a deep provider"); }); - const server = createServer( - { kind: "session", session }, - { environment: {}, jebAnalysis: provider }, - ); - const client = new Client( - { name: "jeb-availability", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: { environment: {}, jebAnalysis: provider }, + clientIdentity: { name: "jeb-availability", version: "1" }, + schemaValidator: schemaValidation, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const read = async () => { const result = await client.callTool({ name: "binary_session", @@ -150,23 +138,12 @@ it("advertises exact valid JEB schemas and records inline evidence through MCP", const session = createTestBinarySession(() => { throw new Error("deep provider must not start"); }); - const server = createServer( - { kind: "session", session }, - { jebAnalysis: provider }, - ); - const client = new Client( - { name: "jeb-contract", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: { jebAnalysis: provider }, + clientIdentity: { name: "jeb-contract", version: "1" }, + schemaValidator: schemaValidation, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const advertised = (await client.listTools()).tools.find( (tool) => tool.name === "inspect_jeb_client", ); @@ -241,23 +218,12 @@ it("preserves JEB provider failure reasons through the MCP boundary", async () = const session = createTestBinarySession(() => { throw new Error("deep provider must not start"); }); - const server = createServer( - { kind: "session", session }, - { jebAnalysis: unavailable }, - ); - const client = new Client( - { name: "jeb-failure", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: { jebAnalysis: unavailable }, + clientIdentity: { name: "jeb-failure", version: "1" }, + schemaValidator: schemaValidation, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const failed = await client.callTool({ name: "list_jeb_units", arguments: {}, @@ -274,23 +240,14 @@ it("keeps the MCP server usable when REA_JEB_MCP_URL is invalid", async () => { const session = createTestBinarySession(() => { throw new Error("deep provider must not start"); }); - const server = createServer( - { kind: "session", session }, - { environment: { REA_JEB_MCP_URL: "http://localhost:8425/mcp" } }, - ); - const client = new Client( - { name: "jeb-invalid-endpoint", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: { + environment: { REA_JEB_MCP_URL: "http://localhost:8425/mcp" }, + }, + clientIdentity: { name: "jeb-invalid-endpoint", version: "1" }, + schemaValidator: schemaValidation, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); expect((await client.listTools()).tools.length).toBeGreaterThan(0); const failed = await client.callTool({ name: "inspect_jeb_client", @@ -358,93 +315,80 @@ it.each( ? refusal : { success: true, gui_client: false }; }); + onTestFinished(() => fixture.close()); const provider = new JebProvider({ environment: { REA_JEB_MCP_URL: fixture.endpoint }, }); const session = createTestBinarySession(() => { throw new Error("JEB must not start a deep provider"); }); - const server = createServer( - { kind: "session", session }, - { jebAnalysis: provider, environment: {} }, - ); - const client = new Client( - { name: "jeb-attribution", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - try { - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); - const call = () => client.callTool({ name: operation, arguments: input }); - if (metadata === "unknown-version") { - for (let attempt = 0; attempt < 2; attempt++) { - const called = await call(); - expect(called.isError).not.toBe(true); - const text = called.content - .map((block) => (block.type === "text" ? block.text : "")) - .join(""); - const evidence = parseEvidence(JSON.parse(text)); - expect(evidence.provider.version).toBeNull(); - expect(evidence.normalized_result).toMatchObject({ - engine: { version: null, gui_client: false }, - }); - expect(evidence.raw_result).toEqual(jebPrimaryResponses[tool]); - } - expect(fixture.calls).toEqual([tool, "get_client_information", tool]); - } else { - const failed = await client.callTool({ - name: operation, - arguments: input, - }); - expect(failed.isError).toBe(true); - const text = failed.content + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: { jebAnalysis: provider, environment: {} }, + clientIdentity: { name: "jeb-attribution", version: "1" }, + schemaValidator: schemaValidation, + }); + const call = () => client.callTool({ name: operation, arguments: input }); + if (metadata === "unknown-version") { + for (let attempt = 0; attempt < 2; attempt++) { + const called = await call(); + expect(called.isError).not.toBe(true); + const text = called.content .map((block) => (block.type === "text" ? block.text : "")) .join(""); - expect(JSON.parse(text)).toMatchObject({ - error: { - code: "execution_failure", - message: refusal.message, - details: { - provider_id: "jeb", - operation, - diagnostics: { - endpoint: fixture.endpoint, - upstream_response: refusal, - }, - partial_observation: { - kind: "retained-evidence", - evidence_id: expect.any(String), - }, + const evidence = parseEvidence(JSON.parse(text)); + expect(evidence.provider.version).toBeNull(); + expect(evidence.normalized_result).toMatchObject({ + engine: { version: null, gui_client: false }, + }); + expect(evidence.raw_result).toEqual(jebPrimaryResponses[tool]); + } + expect(fixture.calls).toEqual([tool, "get_client_information", tool]); + } else { + const failed = await client.callTool({ + name: operation, + arguments: input, + }); + expect(failed.isError).toBe(true); + const text = failed.content + .map((block) => (block.type === "text" ? block.text : "")) + .join(""); + expect(JSON.parse(text)).toMatchObject({ + error: { + code: "execution_failure", + message: refusal.message, + details: { + provider_id: "jeb", + operation, + diagnostics: { + endpoint: fixture.endpoint, + upstream_response: refusal, + }, + partial_observation: { + kind: "retained-evidence", + evidence_id: expect.any(String), }, }, - }); - const partial = z - .object({ - error: z.object({ - details: z.object({ - partial_observation: z.object({ - kind: z.literal("retained-evidence"), - evidence_id: z.string(), - }), + }, + }); + const partial = z + .object({ + error: z.object({ + details: z.object({ + partial_observation: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), }), }), - }) - .parse(JSON.parse(text)).error.details.partial_observation; - expect(session.evidenceById(partial.evidence_id)).toMatchObject({ - evidence_id: partial.evidence_id, - operation, - raw_result: jebPrimaryResponses[tool], - }); - expect(fixture.calls).toEqual([tool, "get_client_information"]); - } - } finally { - await client.close(); - await server.close(); - await session.close(); - await provider.close(); - await fixture.close(); + }), + }) + .parse(JSON.parse(text)).error.details.partial_observation; + expect(session.evidenceById(partial.evidence_id)).toMatchObject({ + evidence_id: partial.evidence_id, + operation, + raw_result: jebPrimaryResponses[tool], + }); + expect(fixture.calls).toEqual([tool, "get_client_information"]); } }, ); diff --git a/tests/boundary/mcp/largeBytePattern.test.ts b/tests/boundary/mcp/largeBytePattern.test.ts index 236dd74f55..d1f2a3ac51 100644 --- a/tests/boundary/mcp/largeBytePattern.test.ts +++ b/tests/boundary/mcp/largeBytePattern.test.ts @@ -6,7 +6,7 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; const LARGE_PATTERN_BYTES = 4 * 1024 * 1024; -const inputValidationText = (result: CallToolResult): string => { +const inputValidationError = (result: CallToolResult) => { expect(result.isError).toBe(true); const diagnostic = parseMcpToolError(result).error; expect(diagnostic).toMatchObject({ @@ -18,7 +18,7 @@ const inputValidationText = (result: CallToolResult): string => { }); const text = JSON.stringify(diagnostic); expect(text).not.toMatch(/RangeError|Maximum call stack/i); - return text; + return diagnostic; }; it("validates large search_bytes inputs without breaking stdio MCP", async () => { @@ -48,18 +48,28 @@ it("validates large search_bytes inputs without breaking stdio MCP", async () => expect(parseMcpToolError(largeSearch).error.code).toBe("target_unavailable"); const malformedPattern = `${largePattern.slice(0, -2)}a?`; - const malformed = inputValidationText( + const malformed = inputValidationError( await call("search_bytes", { pattern: malformedPattern }), ); - expect(malformed).toMatch(/pattern/i); + expect(malformed.details).toMatchObject({ + issues: [{ path: ["pattern"], reason: "invalid_value" }], + }); - const oversizedMask = inputValidationText( + const oversizedMask = inputValidationError( await call("search_bytes", { pattern: "aa", mask: "ff".repeat(LARGE_PATTERN_BYTES), }), ); - expect(oversizedMask).toContain("Mask length must equal pattern length"); + expect(oversizedMask.details).toMatchObject({ + issues: [ + { + path: ["mask"], + reason: "invalid_value", + message: "Mask length must equal pattern length", + }, + ], + }); const final = await call("search_bytes", { pattern: "aa" }); expect(parseMcpToolError(final).error.code).toBe("target_unavailable"); diff --git a/tests/boundary/mcp/managedArtifactSummaryMcp.test.ts b/tests/boundary/mcp/managedArtifactSummaryMcp.test.ts index 788a17a3a2..68b57aed0f 100644 --- a/tests/boundary/mcp/managedArtifactSummaryMcp.test.ts +++ b/tests/boundary/mcp/managedArtifactSummaryMcp.test.ts @@ -1,6 +1,5 @@ import { writeFile, unlink } from "node:fs/promises"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; import { expect, it, onTestFinished } from "vitest"; import { composeBinarySession } from "../../../src/application/binary/BinarySessionComposition.js"; @@ -20,14 +19,14 @@ import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarnes import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { createTestMcp } from "../../support/mcp/mcpFixture.js"; const createHarness = async () => { const session = composeBinarySession(new AnalysisProviderRegistry([]), [ new ManagedStaticProvider(), ]); - const harness = await createApplicationMcpHarness({ session }); - onTestFinished(harness.close); - return harness; + return createApplicationMcpHarness({ session }); }; it("returns a first-call overview and pages the unchanged source after its file disappears", async () => { @@ -227,15 +226,20 @@ it("refuses selected delivery before file access when the server cannot retain E throw new Error("Deep provider must not start"); }); const server = createServer({ kind: "fixed", analysis }); - const client = new Client({ name: "managed-fixed-delivery", version: "1" }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await analysis.close(); + const mcp = createTestMcp(); + onTestFinished(() => + closeVerifierResources([ + () => mcp.close(), + async () => { + const result = await analysis.close(); + if (!result.ok) throw result.error; + }, + ]), + ); + const client = await mcp.connect(server, { + name: "managed-fixed-delivery", + version: "1", }); - const [a, b] = InMemoryTransport.createLinkedPair(); - await server.connect(b); - await client.connect(a); for (const name of ["inspect_managed_artifact", "inspect_managed_members"]) { const rejected = await client.callTool({ name, diff --git a/tests/boundary/mcp/managedMcp.test.ts b/tests/boundary/mcp/managedMcp.test.ts index 3d1c5ba545..4e45cf91fb 100644 --- a/tests/boundary/mcp/managedMcp.test.ts +++ b/tests/boundary/mcp/managedMcp.test.ts @@ -4,7 +4,7 @@ import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { Client } from "@modelcontextprotocol/client"; import type { CallToolResult } from "@modelcontextprotocol/server"; import { Ajv2020 } from "ajv/dist/2020.js"; import { expect, it } from "vitest"; @@ -18,7 +18,7 @@ import type { BinarySession } from "../../../src/application/binary/BinarySessio import { MANAGED_NATIVE_VERIFICATION_EXAMPLE } from "../../../src/contracts/managed/managedWorkflowExamples.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { ManagedStaticProvider } from "../../../src/dotnet/ManagedStaticProvider.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { buildManagedPeFixture } from "../../../src/dotnet/ManagedPe.fixture.js"; import { elf } from "../../../src/domain/binaryTarget.fixture.js"; import { managedMemberInspectionSchema } from "../../../src/domain/managed/managedArtifact.js"; @@ -82,9 +82,10 @@ it("runs every managed static inspection independently of an active native targe const session = composeBinarySession(new AnalysisProviderRegistry([]), [ new ManagedStaticProvider(), ]); - const server = createServer( - { kind: "session", session }, - { + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "managed-independent-mcp", version: "1.0.0" }, + serverOptions: { availabilityPolicy: () => ({ processCaptureEnabled: false, investigationInputRoots: 0, @@ -92,120 +93,106 @@ it("runs every managed static inspection independently of an active native targe electronObservationEnabled: false, }), }, - ); - const client = new Client({ - name: "managed-independent-mcp", - version: "1.0.0", }); - - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const managedMembersTool = (await client.listTools()).tools.find( - ({ name }) => name === "inspect_managed_members", - ); - expect(managedMembersTool?.inputSchema).toMatchObject({ - additionalProperties: false, - }); - const artifact = structured( - await client.callTool({ - name: "inspect_managed_artifact", - arguments: { path }, - }), - ); - expect(artifact).toMatchObject({ - operation: "inspect_managed_artifact", - provider: { id: "rea-dotnet-static" }, - subject: { local_path: path, format: "pe" }, - }); - const misspelledPath = await client.callTool({ - name: "inspect_managed_members", - arguments: { pth: otherPath }, - }); - expect(misspelledPath.isError).toBe(true); - const invalidSelection = await client.callTool({ + const managedMembersTool = (await client.listTools()).tools.find( + ({ name }) => name === "inspect_managed_members", + ); + expect(managedMembersTool?.inputSchema).toMatchObject({ + additionalProperties: false, + }); + const artifact = structured( + await client.callTool({ name: "inspect_managed_artifact", - arguments: { path: join(directory, "missing.dll") }, - }); - expect(invalidSelection.isError).toBe(true); - const nativePath = join(directory, "native.elf"); - await writeFile(nativePath, elf(2, 1, 0x3e)); - expect( - parseMcpToolError( - await client.callTool({ - name: "inspect_managed_artifact", - arguments: { path: nativePath }, - }), - ).error, - ).toMatchObject({ - code: "unsupported_target", - category: "unsupported_target", - details: { operation: "inspect_managed_artifact", path: nativePath }, - }); - const members = structured( - await client.callTool({ name: "inspect_managed_members", arguments: {} }), - ); - expect(members).toMatchObject({ - operation: "inspect_managed_members", - provider: { id: "rea-dotnet-static" }, - subject: { local_path: path, format: "pe" }, - }); - const otherArtifact = structured( + arguments: { path }, + }), + ); + expect(artifact).toMatchObject({ + operation: "inspect_managed_artifact", + provider: { id: "rea-dotnet-static" }, + subject: { local_path: path, format: "pe" }, + }); + const misspelledPath = await client.callTool({ + name: "inspect_managed_members", + arguments: { pth: otherPath }, + }); + expect(misspelledPath.isError).toBe(true); + const invalidSelection = await client.callTool({ + name: "inspect_managed_artifact", + arguments: { path: join(directory, "missing.dll") }, + }); + expect(invalidSelection.isError).toBe(true); + const nativePath = join(directory, "native.elf"); + await writeFile(nativePath, elf(2, 1, 0x3e)); + expect( + parseMcpToolError( await client.callTool({ name: "inspect_managed_artifact", - arguments: { path: otherPath }, + arguments: { path: nativePath }, }), - ); - expect(otherArtifact).toMatchObject({ - subject: { local_path: otherPath, format: "pe" }, - }); - const reusedOtherTarget = structured( - await client.callTool({ name: "inspect_managed_members", arguments: {} }), - ); - expect(reusedOtherTarget).toMatchObject({ - operation: "inspect_managed_members", - subject: { local_path: otherPath, format: "pe" }, - }); - await verifyManagedTypeSelection(client, path, otherPath); - const boundaries = structured( - await client.callTool({ - name: "inspect_managed_native_boundaries", - arguments: { path }, - }), - ); - expect(boundaries).toMatchObject({ - operation: "inspect_managed_native_boundaries", - provider: { id: "rea-dotnet-static" }, - subject: { local_path: path, format: "pe" }, - }); - const partial = structured( - await client.callTool({ - name: "inspect_managed_members", - arguments: { path: partialPath }, - }), - ); - expect(partial).toMatchObject({ - subject: { local_path: partialPath, format: "pe" }, - normalized_result: { - methods: [ - { - body: { - status: "malformed", - normalized_il_sha256: null, - issue: "Method body leaves file-backed PE section data", - }, + ).error, + ).toMatchObject({ + code: "unsupported_target", + category: "unsupported_target", + details: { operation: "inspect_managed_artifact", path: nativePath }, + }); + const members = structured( + await client.callTool({ name: "inspect_managed_members", arguments: {} }), + ); + expect(members).toMatchObject({ + operation: "inspect_managed_members", + provider: { id: "rea-dotnet-static" }, + subject: { local_path: path, format: "pe" }, + }); + const otherArtifact = structured( + await client.callTool({ + name: "inspect_managed_artifact", + arguments: { path: otherPath }, + }), + ); + expect(otherArtifact).toMatchObject({ + subject: { local_path: otherPath, format: "pe" }, + }); + const reusedOtherTarget = structured( + await client.callTool({ name: "inspect_managed_members", arguments: {} }), + ); + expect(reusedOtherTarget).toMatchObject({ + operation: "inspect_managed_members", + subject: { local_path: otherPath, format: "pe" }, + }); + await verifyManagedTypeSelection(client, path, otherPath); + const boundaries = structured( + await client.callTool({ + name: "inspect_managed_native_boundaries", + arguments: { path }, + }), + ); + expect(boundaries).toMatchObject({ + operation: "inspect_managed_native_boundaries", + provider: { id: "rea-dotnet-static" }, + subject: { local_path: path, format: "pe" }, + }); + const partial = structured( + await client.callTool({ + name: "inspect_managed_members", + arguments: { path: partialPath }, + }), + ); + expect(partial).toMatchObject({ + subject: { local_path: partialPath, format: "pe" }, + normalized_result: { + methods: [ + { + body: { + status: "malformed", + normalized_il_sha256: null, + issue: "Method body leaves file-backed PE section data", }, - ], - coverage: { state: "partial" }, - }, - }); - await assertFieldMemberRefToken(client, fieldTokenPath); - } finally { - await Promise.all([client.close(), server.close()]); - await session.close(); - } + }, + ], + coverage: { state: "partial" }, + }, + }); + await assertFieldMemberRefToken(client, fieldTokenPath); }, 30_000); it("compares observed call-target names through the managed MCP contract", async () => { @@ -228,69 +215,63 @@ it("compares observed call-target names through the managed MCP contract", async const session = composeBinarySession(new AnalysisProviderRegistry([]), [ new ManagedStaticProvider(), ]); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "managed-call-target-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const outputAssertion = await createOutputAssertion(client); - const inspect = async (path: string) => - inlineEvidence( - structured( - await client.callTool({ - name: "inspect_managed_members", - arguments: { path }, - }), - ), - ); - const left = await inspect(leftPath); - const right = await inspect(rightPath); - const same = await inspect(samePath); - for (const [evidence, name] of [ - [left, "Clear"], - [right, "WriteLine"], - ] as const) { - const inspection = managedMemberInspectionSchema.parse( - evidence.normalized_result, - ); - expect(inspection.methods[0]?.body.anchors).toMatchObject([ - { opcode: "call", operand_kind: "method" }, - { opcode: "ret", operand_kind: "none" }, - ]); - expect(inspection.call_edges[0]).toMatchObject({ - target_name: name, - target_declaring_type: - "System.Runtime.Versioning.TargetFrameworkAttribute", - }); - } - const compare = async (rightEvidence: Record) => + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "managed-call-target-test", version: "1" }, + }); + const outputAssertion = await createOutputAssertion(client); + const inspect = async (path: string) => + inlineEvidence( structured( await client.callTool({ - name: "compare_managed_members", - arguments: { left, right: rightEvidence }, + name: "inspect_managed_members", + arguments: { path }, }), - ); - const changed = inlineEvidence(await compare(right)); - outputAssertion.assert("compare_managed_members", changed); - const changedReport = managedMemberComparisonResultSchema.parse( - changed.normalized_result, + ), ); - expect(changedReport.methods).toContainEqual( - expect.objectContaining({ - status: "changed", - dimensions: expect.arrayContaining(["call-target"]), + const left = await inspect(leftPath); + const right = await inspect(rightPath); + const same = await inspect(samePath); + for (const [evidence, name] of [ + [left, "Clear"], + [right, "WriteLine"], + ] as const) { + const inspection = managedMemberInspectionSchema.parse( + evidence.normalized_result, + ); + expect(inspection.methods[0]?.body.anchors).toMatchObject([ + { opcode: "call", operand_kind: "method" }, + { opcode: "ret", operand_kind: "none" }, + ]); + expect(inspection.call_edges[0]).toMatchObject({ + target_name: name, + target_declaring_type: + "System.Runtime.Versioning.TargetFrameworkAttribute", + }); + } + const compare = async (rightEvidence: Record) => + structured( + await client.callTool({ + name: "compare_managed_members", + arguments: { left, right: rightEvidence }, }), ); - const unchanged = inlineEvidence(await compare(same)); - expect( - managedMemberComparisonResultSchema.parse(unchanged.normalized_result) - .methods, - ).toContainEqual(expect.objectContaining({ status: "unchanged" })); - } finally { - await Promise.allSettled([client.close(), server.close(), session.close()]); - } + const changed = inlineEvidence(await compare(right)); + outputAssertion.assert("compare_managed_members", changed); + const changedReport = managedMemberComparisonResultSchema.parse( + changed.normalized_result, + ); + expect(changedReport.methods).toContainEqual( + expect.objectContaining({ + status: "changed", + dimensions: expect.arrayContaining(["call-target"]), + }), + ); + const unchanged = inlineEvidence(await compare(same)); + expect( + managedMemberComparisonResultSchema.parse(unchanged.normalized_result) + .methods, + ).toContainEqual(expect.objectContaining({ status: "unchanged" })); }); it("opens a managed PE and executes the managed static provider through MCP", async () => { @@ -302,9 +283,10 @@ it("opens a managed PE and executes the managed static provider through MCP", as const session = composeBinarySession(new AnalysisProviderRegistry([]), [ new ManagedStaticProvider(), ]); - const server = createServer( - { kind: "session", session }, - { + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "managed-mcp-test", version: "1.0.0" }, + serverOptions: { availabilityPolicy: () => ({ processCaptureEnabled: false, investigationInputRoots: 0, @@ -312,30 +294,20 @@ it("opens a managed PE and executes the managed static provider through MCP", as electronObservationEnabled: false, }), }, + }); + const assertOutput = await createOutputAssertion(client); + await verifyManagedCatalogAndNativeWorkflow(client, session, assertOutput); + const members = await inspectManagedStaticWorkflow( + client, + path, + assertOutput, + ); + await verifyManagedComparisonAndReconstruction( + client, + members, + rightPath, + assertOutput, ); - const client = new Client({ name: "managed-mcp-test", version: "1.0.0" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const assertOutput = await createOutputAssertion(client); - await verifyManagedCatalogAndNativeWorkflow(client, session, assertOutput); - const members = await inspectManagedStaticWorkflow( - client, - path, - assertOutput, - ); - await verifyManagedComparisonAndReconstruction( - client, - members, - rightPath, - assertOutput, - ); - } finally { - await Promise.all([client.close(), server.close()]); - await session.close(); - } }, 30_000); const verifyManagedCatalogAndNativeWorkflow = async ( diff --git a/tests/boundary/mcp/mcpContractHarness.ts b/tests/boundary/mcp/mcpContractHarness.ts index 035beb79ec..0cee9ffef9 100644 --- a/tests/boundary/mcp/mcpContractHarness.ts +++ b/tests/boundary/mcp/mcpContractHarness.ts @@ -26,9 +26,12 @@ export const assertAdvertisedMcpContracts = ( const ajv = new Ajv2020({ strict: false, allErrors: true }); for (const name of operationNames) { const contract = toolContract(name); - const listed = tools.find((tool) => tool.name === name); - expect(listed, name).toBeTruthy(); - if (listed?.inputSchema === undefined) continue; + const matches = tools.filter((tool) => tool.name === name); + if (matches.length !== 1) + throw new Error(`${name} must have exactly one advertised definition`); + const listed = matches[0]; + if (listed?.inputSchema === undefined) + throw new Error(`${name} must advertise an input schema`); const validate = ajv.compile( JSON.parse(JSON.stringify(listed.inputSchema)), ); diff --git a/tests/boundary/mcp/mcpHttpOptions.test.ts b/tests/boundary/mcp/mcpHttpOptions.test.ts index a7845bd06f..b6b8b3b163 100644 --- a/tests/boundary/mcp/mcpHttpOptions.test.ts +++ b/tests/boundary/mcp/mcpHttpOptions.test.ts @@ -205,9 +205,20 @@ describe("MCP_HOST and MCP_PORT", () => { "MCP_HOST", "MCP_PORT", ]); - const projected = JSON.stringify(projectAnalysisError(result.error)); - expect(projected).toContain("MCP_PORT must be a decimal integer"); - expect(projected).not.toContain("65536"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + details: { + settings: expect.arrayContaining([ + expect.objectContaining({ + setting: "MCP_PORT", + constraint: expect.stringContaining( + "MCP_PORT must be a decimal integer", + ), + }), + ]), + }, + }); + expect(JSON.stringify(projected)).not.toContain("65536"); }); }); diff --git a/tests/boundary/mcp/mobileApplicationGraphMcp.test.ts b/tests/boundary/mcp/mobileApplicationGraphMcp.test.ts index 48bfa8452a..a9830916b9 100644 --- a/tests/boundary/mcp/mobileApplicationGraphMcp.test.ts +++ b/tests/boundary/mcp/mobileApplicationGraphMcp.test.ts @@ -2,14 +2,13 @@ import { projectCliAnalysisResult } from "../../../src/cli/projectCliAnalysisRes import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { TextReader, Uint8ArrayReader, Uint8ArrayWriter, ZipWriter, } from "@zip.js/zip.js"; -import { afterEach, expect, it } from "vitest"; +import { expect, it } from "vitest"; import { createDirectAnalysis } from "../../../src/composition/directAnalysis.js"; import { androidApplicationProjectionResultSchema } from "../../../src/domain/android/androidApplication.js"; @@ -19,28 +18,18 @@ import { fragmentInventoryEvidence } from "../../fixtures/artifactEvidence.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; const { runProviderAnalysis } = createDirectAnalysis({}); -const resources: Array<{ close(): Promise }> = []; -afterEach(async () => { - for (const resource of resources.splice(0)) await resource.close(); -}); - async function connect() { const session = createTestBinarySession(() => { throw new Error("mobile graph projection must not launch a provider"); }); - const server = createServer({ kind: "session", session }); - const client = new Client({ - name: "mobile-application-graph-test", - version: "1", + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "mobile-application-graph-test", version: "1" }, }); - resources.push(client, server, session); - const [a, b] = InMemoryTransport.createLinkedPair(); - await server.connect(b); - await client.connect(a); return client; } diff --git a/tests/boundary/mcp/nativeDispatchBoundTarget.test.ts b/tests/boundary/mcp/nativeDispatchBoundTarget.test.ts index 7f9dc78b73..519240269c 100644 --- a/tests/boundary/mcp/nativeDispatchBoundTarget.test.ts +++ b/tests/boundary/mcp/nativeDispatchBoundTarget.test.ts @@ -7,6 +7,7 @@ import { describe, expect, it } from "vitest"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { nativeDispatchMetadataResultSchema } from "../../../src/domain/native/objcSwiftMetadata.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { localToolsMcpEnvironment } from "../../fixtures/localToolsMcp.js"; import { objcRecordBudgetMacho } from "../../fixtures/objcRecordBudgetMacho.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -40,7 +41,7 @@ const inspectReplacements = async (replacements: readonly Replacement[]) => { command: process.execPath, args: [resolve("scripts/rea.mjs"), "mcp"], cwd: process.cwd(), - env, + env: localToolsMcpEnvironment(env), stderr: "pipe", }); let stderr = ""; @@ -65,7 +66,7 @@ const inspectReplacements = async (replacements: readonly Replacement[]) => { name: "open_binary", arguments: { path }, }); - expect(opened.isError).not.toBe(true); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); const control = await successfulRead(); for (const replacement of replacements) { await rm(path, { recursive: true, force: true }); diff --git a/tests/boundary/mcp/nativeInventoryViewsMcp.test.ts b/tests/boundary/mcp/nativeInventoryViewsMcp.test.ts index 5ec387ea9d..fee3a49d8a 100644 --- a/tests/boundary/mcp/nativeInventoryViewsMcp.test.ts +++ b/tests/boundary/mcp/nativeInventoryViewsMcp.test.ts @@ -130,6 +130,7 @@ it("shares complete/summary source identity and advertised contracts across all ); if (!Array.isArray(complete.normalized_result)) throw new Error("Expected complete row array"); + expect(complete.normalized_result.length).toBeGreaterThan(0); if (operation === "search_procedures") expect(complete.normalized_result).toEqual(searchProcedureRows); expect(complete.parameters).not.toHaveProperty("detail"); diff --git a/tests/boundary/mcp/nativeLoadImageViewsMcp.test.ts b/tests/boundary/mcp/nativeLoadImageViewsMcp.test.ts index 967564de28..9f2b733643 100644 --- a/tests/boundary/mcp/nativeLoadImageViewsMcp.test.ts +++ b/tests/boundary/mcp/nativeLoadImageViewsMcp.test.ts @@ -15,10 +15,11 @@ import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; it("preserves raw and DOL verification facts and selected DOL sections through the advertised SDK output", async () => { const { call } = await connectLocalToolsMcp(); - for (const report of [ - nativeRawLoadImageFixture(), - nativeDolLoadImageFixture(), - ]) { + for (const { report, format } of [ + { report: nativeRawLoadImageFixture(), format: "raw-binary" }, + { report: nativeDolLoadImageFixture(), format: "gamecube-dol" }, + ] as const) { + expect(report.format).toBe(format); const parent = createEvidence( { path: "/selected/fixture", @@ -55,11 +56,11 @@ it("preserves raw and DOL verification facts and selected DOL sections through t verification, counts: { checks: checks.length, - sections: report.format === "gamecube-dol" ? 2 : null, + sections: format === "gamecube-dol" ? 2 : null, }, }, }); - if (report.format !== "gamecube-dol") continue; + if (format !== "gamecube-dol") continue; const page = await call("inspect_analysis_view", { source, view: { diff --git a/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts b/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts index d087c9893b..131c13adb0 100644 --- a/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts +++ b/tests/boundary/mcp/nativeMachoSummaryMcp.test.ts @@ -161,7 +161,8 @@ mcpTest( throw new Error("No provider expected"); }); onTestFinished(async () => { - await session.close(); + const closed = await session.close(); + if (!closed.ok) throw closed.error; }); const client = await mcp.connect( createServer( @@ -327,7 +328,8 @@ mcpTest( throw new Error("Imported views must not start a provider"); }); onTestFinished(async () => { - await session.close(); + const closed = await session.close(); + if (!closed.ok) throw closed.error; }); const client = await mcp.connect( createServer( @@ -511,7 +513,8 @@ mcpTest( close: () => Promise.resolve(ok(null)), })); onTestFinished(async () => { - await session.close(); + const closed = await session.close(); + if (!closed.ok) throw closed.error; }); const opened = await session.open(target); if (!opened.ok) throw opened.error; diff --git a/tests/boundary/mcp/nodeSeaMcp.test.ts b/tests/boundary/mcp/nodeSeaMcp.test.ts index dd89771a9e..0de49b09a2 100644 --- a/tests/boundary/mcp/nodeSeaMcp.test.ts +++ b/tests/boundary/mcp/nodeSeaMcp.test.ts @@ -11,6 +11,7 @@ import { parseEvidence } from "../../../src/domain/evidence.js"; import { z } from "zod"; import { artifactExtractionResultSchema } from "../../../src/domain/artifactGraph.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; // The compiled stdio fixture checks advertised schemas against their dialect and // validates actual tool responses, avoiding a second source server module graph. @@ -58,13 +59,22 @@ it.each(["full", "compact"] as const)( kind: "inventory", collection: "occurrences", logical_paths: ["main.js"], - fields: ["format_metadata"], + fields: ["logical_path", "format_metadata"], }, }); expect(retained.isError, JSON.stringify(retained)).not.toBe(true); - expect(JSON.stringify(retained.structuredContent)).toContain( - "main-script", + const view = analysisViewResultSchema.parse( + parseEvidence(retained.structuredContent).normalized_result, ); + expect(view).toMatchObject({ + kind: "inventory", + items: [ + { + logical_path: "main.js", + format_metadata: { kind: "node-sea-entry", role: "main-script" }, + }, + ], + }); const extracted = await call("extract_artifact", {}); expect(extracted.isError, JSON.stringify(extracted)).not.toBe(true); const result = artifactExtractionResultSchema.parse( @@ -74,14 +84,19 @@ it.each(["full", "compact"] as const)( expect(await readFile(join(result.output_root, "data.txt"), "utf8")).toBe( "data", ); - await call("close_binary", {}); + expect((await call("close_binary", {})).isError).not.toBe(true); await writeFile(path, nodeSeaPe({ blob: Buffer.alloc(16) })); - await call("open_binary", { path }); + expect((await call("open_binary", { path })).isError).not.toBe(true); const invalid = await call("inspect_artifact", {}); - expect(JSON.stringify(parseMcpToolError(invalid))).toContain( - "Node SEA blob magic", - ); - await call("close_binary", {}); + expect(parseMcpToolError(invalid).error).toMatchObject({ + code: "artifact_operation_failed", + details: { + operation: "inspect_artifact", + reason: "format", + detail: expect.stringContaining("magic"), + }, + }); + expect((await call("close_binary", {})).isError).not.toBe(true); await writeFile( path, nodeSeaPe({ @@ -93,11 +108,17 @@ it.each(["full", "compact"] as const)( }), }), ); - await call("open_binary", { path }); + expect((await call("open_binary", { path })).isError).not.toBe(true); const overLimit = await call("inspect_artifact", {}); - expect(JSON.stringify(parseMcpToolError(overLimit))).toContain( - "Node SEA asset count exceeds", - ); + expect(parseMcpToolError(overLimit).error).toMatchObject({ + code: "truncated", + details: { + operation: "inspect_artifact", + reason: "limit", + boundary: "capacity", + truncated: true, + }, + }); } finally { if (outputRoot !== undefined) await rm(outputRoot, { recursive: true, force: true }); diff --git a/tests/boundary/mcp/officialArguments.test.ts b/tests/boundary/mcp/officialArguments.test.ts index 7f3caae7cd..61bbbc5711 100644 --- a/tests/boundary/mcp/officialArguments.test.ts +++ b/tests/boundary/mcp/officialArguments.test.ts @@ -15,6 +15,7 @@ import { parseEvidence } from "../../../src/domain/evidence.js"; import { ok } from "../../../src/domain/result.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createBinarySessionTargets, createTestBinarySession, @@ -34,6 +35,53 @@ const connect = async (analysis: AnalysisOperationPort) => { return client; }; +const invalidArguments = [ + [ + "list_procedures", + { document: null }, + { path: ["document"], reason: "invalid_type", expected: "string" }, + ], + [ + "procedure_pseudo_code", + { procedure: "main", document: 3 }, + { path: ["document"], reason: "invalid_type", expected: "string" }, + ], + [ + "search_strings", + { pattern: "entry", case_sensitive: null }, + { + path: ["case_sensitive"], + reason: "invalid_type", + expected: "boolean", + }, + ], + [ + "search_bytes", + { pattern: "??" }, + { path: ["pattern"], reason: "invalid_value" }, + ], + [ + "search_bytes", + { pattern: "48 8B", mask: "ff" }, + { path: ["mask"], reason: "invalid_value" }, + ], + [ + "search_bytes", + { pattern: "4?" }, + { path: ["pattern"], reason: "invalid_value" }, + ], + [ + "search_bytes", + { pattern: "48", limit: 1001 }, + { path: ["limit"], reason: "out_of_range", maximum: 1000 }, + ], + [ + "search_bytes", + { pattern: "48", cursor: null }, + { path: ["cursor"], reason: "invalid_type", expected: "string" }, + ], +] as const; + describe("official operation arguments", () => { it("dispatches equivalent CLI and MCP arguments while retaining omissions and defaults", async () => { const [path] = await createBinarySessionTargets(); @@ -109,25 +157,32 @@ describe("official operation arguments", () => { const client = await connect({ execute: (name) => { calls.push(name); - return Promise.resolve(observed(null)); + return Promise.resolve(observed([])); }, }); - for (const [name, parameters] of [ - ["list_procedures", { document: null }], - ["procedure_pseudo_code", { procedure: "main", document: 3 }], - ["search_strings", { pattern: "entry", case_sensitive: null }], - ["search_bytes", { pattern: "??" }], - ["search_bytes", { pattern: "48 8B", mask: "ff" }], - ["search_bytes", { pattern: "4?" }], - ["search_bytes", { pattern: "48", limit: 1001 }], - ["search_bytes", { pattern: "48", cursor: null }], - ] as const) { + for (const [name, parameters, issue] of invalidArguments) { const result = await client.callTool({ name, arguments: parameters }); - expect(result.isError).toBe(true); + expect(parseMcpToolError(result)).toMatchObject({ + error: { + code: "invalid_request", + category: "invalid_input", + retryable: false, + recovery: { kind: "inspect_input_contract", operation: name }, + details: { operation: name, issues: [issue] }, + }, + }); } expect(calls).toEqual([]); + const valid = await client.callTool({ + name: "list_procedures", + arguments: {}, + }); + expect(valid.isError).not.toBe(true); + expect(calls).toEqual(["list_procedures"]); }); +}); +describe("official operation arguments", () => { it("preserves existing annotations when updates omit them and keeps empty clearing text", async () => { const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); const annotations = { diff --git a/tests/boundary/mcp/officialTools.test.ts b/tests/boundary/mcp/officialTools.test.ts index fa0e997b97..f262192431 100644 --- a/tests/boundary/mcp/officialTools.test.ts +++ b/tests/boundary/mcp/officialTools.test.ts @@ -18,6 +18,7 @@ import { silentLogger } from "../../../src/logger.js"; import { composeBinarySession } from "../../../src/application/binary/BinarySessionComposition.js"; import { AnalysisProviderRegistry } from "../../../src/application/binary/AnalysisProviderRegistry.js"; import { createBinarySessionTargets } from "../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; interface Invocation { readonly name: string; @@ -34,41 +35,27 @@ it("returns typed Hopper byte-search unavailability without launching the provid const session = composeBinarySession( new AnalysisProviderRegistry([provider]), ); - const server = createServer( - { kind: "session", session }, - { logger: silentLogger }, - ); - const client = new Client({ - name: "hopper-byte-search-unavailable", - version: "1", + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: { logger: silentLogger }, + clientIdentity: { name: "hopper-byte-search-unavailable", version: "1" }, + }); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: target }, + }); + expect(opened.isError).not.toBe(true); + const result = await client.callTool({ + name: "search_bytes", + arguments: { pattern: "48 8b" }, + }); + expect(result.isError).toBe(true); + expect(parseMcpToolError(result)).toMatchObject({ + error: { + code: "capability_unavailable", + details: { provider_id: "hopper", operation: "search_bytes" }, + }, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path: target }, - }); - expect(opened.isError).not.toBe(true); - const result = await client.callTool({ - name: "search_bytes", - arguments: { pattern: "48 8b" }, - }); - expect(result.isError).toBe(true); - expect(parseMcpToolError(result)).toMatchObject({ - error: { - code: "capability_unavailable", - details: { provider_id: "hopper", operation: "search_bytes" }, - }, - }); - } finally { - await client.close(); - await server.close(); - const closed = await session.close(); - if (!closed.ok) throw closed.error; - } }); it("advertises strict masked-byte arguments and bounded pages", () => { diff --git a/tests/boundary/mcp/optionalProviderIsolation.test.ts b/tests/boundary/mcp/optionalProviderIsolation.test.ts index b129c6f9a0..967719a3da 100644 --- a/tests/boundary/mcp/optionalProviderIsolation.test.ts +++ b/tests/boundary/mcp/optionalProviderIsolation.test.ts @@ -2,7 +2,7 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { McpServer } from "@modelcontextprotocol/server"; -import { afterAll, afterEach, expect } from "vitest"; +import { afterAll, expect, it, onTestFinished, vi } from "vitest"; import { z } from "zod"; import type { BinarySessionPort } from "../../../src/application/binary/BinarySessionPort.js"; @@ -45,20 +45,14 @@ const failedAdapters = [ ], ] as const; -const resources: Array<() => Promise> = []; -afterEach(async () => { - for (const close of resources.splice(0).reverse()) await close(); -}); - -const boot = async (factories: OptionalObservationFactories) => { +const boot = async ( + factories: OptionalObservationFactories, + configureServer?: (server: McpServer) => void, +) => { const loaded = await loadOptionalObservationProviders(factories); let server: McpServer | undefined; let session: BinarySessionPort | undefined; - let shutdown: (() => void) | undefined; - let resolveClose: (() => void) | undefined; - const closed = new Promise((resolve) => { - resolveClose = resolve; - }); + let shutdown: (() => Promise) | undefined; const output: string[] = []; const exitCodes: number[] = []; expect( @@ -70,7 +64,7 @@ const boot = async (factories: OptionalObservationFactories) => { throw new Error("Expected a session-owned server source"); session = source.session; // A configured policy cannot hide an actual adapter loading failure. - return createServer(source, { + const created = createServer(source, { ...options, availabilityPolicy: () => ({ processCaptureEnabled: true, @@ -81,22 +75,20 @@ const boot = async (factories: OptionalObservationFactories) => { v8InspectorObservationEnabled: true, }), }); + configureServer?.(created); + return created; }, serve: (factory) => { const created = factory({ era: "legacy" }); if (!(created instanceof McpServer)) throw new Error("Expected the production synchronous MCP factory"); server = created; - return { - close: async () => { - await server?.close(); - resolveClose?.(); - }, - }; + return { close: () => Promise.resolve() }; }, - registerShutdown: (handler) => { - shutdown = handler; - return () => {}; + registerShutdown: () => () => {}, + captureShutdown: (close) => { + shutdown = close; + onTestFinished(close); }, writeStderr: (message) => output.push(message), setExitCode: (code) => exitCodes.push(code), @@ -105,14 +97,8 @@ const boot = async (factories: OptionalObservationFactories) => { expect(output).toEqual([]); if (server === undefined || session === undefined || shutdown === undefined) throw new Error("Runtime did not construct the MCP server and session"); - const close = shutdown; - resources.push(async () => { - close(); - await closed; - await new Promise((resolve) => setImmediate(resolve)); - expect(exitCodes).toEqual([]); - }); - return { server, session, loaded }; + expect(exitCodes).toEqual([]); + return { server, session, loaded, shutdown }; }; mcpTest.for(failedAdapters)( @@ -243,6 +229,15 @@ mcpTest.for(failedAdapters)( arguments: { unexpected: true }, }); expect(malformed.isError).toBe(true); + expect(parseMcpToolError(malformed)).toMatchObject({ + error: { + code: "invalid_request", + category: "invalid_input", + retryable: false, + recovery: { kind: "inspect_input_contract", operation }, + details: { operation }, + }, + }); expect(JSON.stringify(malformed)).not.toContain(failure); }, ); @@ -275,3 +270,28 @@ mcpTest( expect(second.loaded.browserObservation).toBeDefined(); }, ); + +it("reports a server close failure and retries the runtime owner", async () => { + const failure = new Error("controlled close failure"); + let denied = true; + let close: ReturnType | undefined; + const connected = await boot(optionalObservationFactories([]), (server) => { + const realClose = server.close.bind(server); + close = vi.spyOn(server, "close").mockImplementation(async () => { + if (denied) throw failure; + await realClose(); + }); + }); + try { + await expect(connected.shutdown()).rejects.toMatchObject({ + errors: [failure], + }); + expect(close).toHaveBeenCalledTimes(1); + denied = false; + await connected.shutdown(); + await connected.shutdown(); + expect(close).toHaveBeenCalledTimes(2); + } finally { + denied = false; + } +}); diff --git a/tests/boundary/mcp/outputValidatorReuse.test.ts b/tests/boundary/mcp/outputValidatorReuse.test.ts index b7f570066d..af47dd088c 100644 --- a/tests/boundary/mcp/outputValidatorReuse.test.ts +++ b/tests/boundary/mcp/outputValidatorReuse.test.ts @@ -4,6 +4,10 @@ import { McpServer } from "@modelcontextprotocol/server"; import { expect, it } from "vitest"; import { z } from "zod"; +import { + closeVerifierResources, + withVerifierCleanup, +} from "../../../scripts/lib/verifier-cleanup.mjs"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { toolOutputSchemaWithMetadata } from "../../../src/contracts/toolSchemaMetadata.js"; @@ -24,6 +28,16 @@ const connect = async (server: McpServer, client: Client): Promise => { await client.connect(clientTransport); }; +const expectState = async ( + client: Client, + name: string, + state: string, +): Promise => { + const result = await client.callTool({ name }); + expect(result.isError).not.toBe(true); + expect(result.structuredContent).toEqual({ state }); +}; + it("validates changed schemas after catalog refreshes and availability changes", async () => { const client = new Client({ name: "output-reuse-test", version: "1" }); const server = new McpServer({ name: "output-reuse-fixture", version: "1" }); @@ -39,27 +53,19 @@ it("validates changed schemas after catalog refreshes and availability changes", { inputSchema: z.strictObject({}), outputSchema: schema }, () => ({ content: [], structuredContent: { state: "ready" } }), ); - try { + await withVerifierCleanup(async () => { await connect(server, client); await client.listTools(); - expect( - (await client.callTool({ name: "selected" })).structuredContent, - ).toEqual({ - state: "ready", - }); + await expectState(client, "selected", "ready"); await client.listTools(); - await client.callTool({ name: "equivalent" }); + await expectState(client, "equivalent", "ready"); selected.update({ outputSchema: advertised(z.strictObject({ state: z.literal("updated") })), }); state = "updated"; await client.listTools(); - expect( - (await client.callTool({ name: "selected" })).structuredContent, - ).toEqual({ - state: "updated", - }); + await expectState(client, "selected", "updated"); selected.disable(); const unavailable = await client.listTools(); expect(unavailable.tools.some((tool) => tool.name === "selected")).toBe( @@ -68,19 +74,17 @@ it("validates changed schemas after catalog refreshes and availability changes", await expect(client.callTool({ name: "selected" })).rejects.toThrow( /not found|disabled/iu, ); - await client.callTool({ name: "equivalent" }); + await expectState(client, "equivalent", "ready"); selected.enable(); await client.listTools(); - await client.callTool({ name: "selected" }); - } finally { - await Promise.allSettled([client.close(), server.close()]); - } + await expectState(client, "selected", "updated"); + }, [() => client.close(), () => server.close()]); }); it("uses the new server's schema after reconnecting the client", async () => { const client = new Client({ name: "output-reconnect-test", version: "1" }); const servers: McpServer[] = []; - try { + await withVerifierCleanup(async () => { for (const state of ["first", "second"]) { const server = new McpServer({ name: `fixture-${state}`, version: "1" }); servers.push(server); @@ -94,18 +98,14 @@ it("uses the new server's schema after reconnecting the client", async () => { ); await connect(server, client); await client.listTools(); - expect( - (await client.callTool({ name: "selected" })).structuredContent, - ).toEqual({ state }); + await expectState(client, "selected", state); await client.close(); await server.close(); } - } finally { - await Promise.allSettled([ - client.close(), - ...servers.map((server) => server.close()), - ]); - } + }, [ + () => client.close(), + () => closeVerifierResources(servers.map((server) => () => server.close())), + ]); }); it("advertises stable output identity without interpreting example data as references", async () => { @@ -130,7 +130,7 @@ it("advertises stable output identity without interpreting example data as refer }, () => ({ content: [], structuredContent: value }), ); - try { + await withVerifierCleanup(async () => { await connect(server, client); const first = await client.listTools(); const second = await client.listTools(); @@ -150,7 +150,5 @@ it("advertises stable output identity without interpreting example data as refer ); expect(validate({ payload: 7 }).valid).toBe(false); expect(validate({ ...value, unexpected: true }).valid).toBe(false); - } finally { - await Promise.allSettled([client.close(), server.close()]); - } + }, [() => client.close(), () => server.close()]); }); diff --git a/tests/boundary/mcp/packetCaptureMcp.test.ts b/tests/boundary/mcp/packetCaptureMcp.test.ts index 9a80bf254c..9779f16bc7 100644 --- a/tests/boundary/mcp/packetCaptureMcp.test.ts +++ b/tests/boundary/mcp/packetCaptureMcp.test.ts @@ -155,7 +155,21 @@ it("rejects oversized selector expansion through the MCP boundary before invokin display_filter: "界".repeat(8192), }, }); - expect(response.isError).toBe(true); - expect(JSON.stringify(response)).toContain("32 KiB"); + expect(parseMcpToolError(response)).toMatchObject({ + error: { + code: "invalid_request", + category: "invalid_input", + details: { + operation: "inspect_packet_capture", + issues: [ + { + path: [], + reason: "invalid_value", + message: expect.stringContaining("32 KiB encoded argument budget"), + }, + ], + }, + }, + }); expect(calls).toBe(0); }); diff --git a/tests/boundary/mcp/peCertificatesMcp.test.ts b/tests/boundary/mcp/peCertificatesMcp.test.ts index adfed26934..99c6386391 100644 --- a/tests/boundary/mcp/peCertificatesMcp.test.ts +++ b/tests/boundary/mcp/peCertificatesMcp.test.ts @@ -1,6 +1,6 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { afterAll, expect } from "vitest"; +import { afterAll, expect, onTestFinished } from "vitest"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { createServer } from "../../../src/server/createServer.js"; @@ -21,104 +21,95 @@ const connect = async (mcp: TestMcp) => { const session = createTestBinarySession(() => { throw new Error("PE certificate inspection must not start a deep provider"); }); + onTestFinished(async () => { + const result = await session.close(); + if (!result.ok) throw result.error; + }); const client = await mcp.connect( createServer({ kind: "session", session }), undefined, schemaValidation, ); - return { client, session }; + return client; }; mcpTest( "advertised output schema accepts partial unsigned-attribute evidence", async ({ mcp }) => { - const { client, session } = await connect(mcp); - try { - const advertised = (await client.listTools()).tools.find( - (tool) => tool.name === "inspect_pe_certificates", - ); - if (advertised?.outputSchema === undefined) - throw new Error("PE certificates output schema is missing"); - const schema: Record = advertised.outputSchema; - const ajv = schemaValidation.assertions; - expect(ajv.validateSchema(schema)).toBe(true); - const root = await createTestTempDirectory("rea-pe-cert-schema-"); - const path = join(root, "timestamped.exe"); - await writeFile( - path, - peCertificateFixture( - false, - pkcs7SignedDataFixture(undefined, undefined, undefined, [ - [1, 3, 6, 1, 4, 1, 311, 3, 3, 1], - ]), - ), - ); - const result = await client.callTool({ - name: "inspect_pe_certificates", - arguments: { path }, - }); - expect(result.isError).not.toBe(true); - expect( - ajv.validate(schema, result.structuredContent), - JSON.stringify(ajv.errors), - ).toBe(true); - } finally { - await session.close(); - } + const client = await connect(mcp); + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_pe_certificates", + ); + if (advertised?.outputSchema === undefined) + throw new Error("PE certificates output schema is missing"); + const schema: Record = advertised.outputSchema; + const ajv = schemaValidation.assertions; + expect(ajv.validateSchema(schema)).toBe(true); + const root = await createTestTempDirectory("rea-pe-cert-schema-"); + const path = join(root, "timestamped.exe"); + await writeFile( + path, + peCertificateFixture( + false, + pkcs7SignedDataFixture(undefined, undefined, undefined, [ + [1, 3, 6, 1, 4, 1, 311, 3, 3, 1], + ]), + ), + ); + const result = await client.callTool({ + name: "inspect_pe_certificates", + arguments: { path }, + }); + expect(result.isError).not.toBe(true); + expect( + ajv.validate(schema, result.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); }, ); mcpTest( "advertises inspect_pe_certificates and reads PKCS#7 signer identity", async ({ mcp }) => { - const { client, session } = await connect(mcp); - try { - const advertised = (await client.listTools()).tools.find( - (tool) => tool.name === "inspect_pe_certificates", - ); - expect(advertised?.annotations).toMatchObject({ - readOnlyHint: false, - idempotentHint: true, - }); - const root = await createTestTempDirectory("rea-pe-cert-mcp-"); - const path = join(root, "signed.exe"); - await writeFile(path, peCertificateFixture(true)); - const response = await client.callTool({ - name: "inspect_pe_certificates", - arguments: { path }, - }); - expect(response.isError, JSON.stringify(response.content)).not.toBe(true); - const evidence = toolContract( - "inspect_pe_certificates", - ).outputSchema.parse(response.structuredContent); - expect(evidence.normalized_result.format).toBe("pe32-plus"); - expect( - evidence.normalized_result.entries[0]?.pkcs7?.certificates[0] - ?.serial_hex, - ).toBe("11223344"); - } finally { - await session.close(); - } + const client = await connect(mcp); + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_pe_certificates", + ); + expect(advertised?.annotations).toMatchObject({ + readOnlyHint: false, + idempotentHint: true, + }); + const root = await createTestTempDirectory("rea-pe-cert-mcp-"); + const path = join(root, "signed.exe"); + await writeFile(path, peCertificateFixture(true)); + const response = await client.callTool({ + name: "inspect_pe_certificates", + arguments: { path }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const evidence = toolContract("inspect_pe_certificates").outputSchema.parse( + response.structuredContent, + ); + expect(evidence.normalized_result.format).toBe("pe32-plus"); + expect( + evidence.normalized_result.entries[0]?.pkcs7?.certificates[0]?.serial_hex, + ).toBe("11223344"); }, ); mcpTest( "requires an absolute path for inspect_pe_certificates", async ({ mcp }) => { - const { client, session } = await connect(mcp); - try { - const response = await client.callTool({ - name: "inspect_pe_certificates", - arguments: { path: "signed.exe" }, - }); - expect(response.isError).toBe(true); - expect(parseMcpToolError(response).error).toMatchObject({ - code: "invalid_request", - details: { operation: "inspect_pe_certificates" }, - }); - } finally { - await session.close(); - } + const client = await connect(mcp); + const response = await client.callTool({ + name: "inspect_pe_certificates", + arguments: { path: "signed.exe" }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "invalid_request", + details: { operation: "inspect_pe_certificates" }, + }); }, ); @@ -128,27 +119,23 @@ mcpTest.for([ ])( "reports decoder capacity as a nonretryable resource failure for $count certificates", async ({ count, reported_limits }, { mcp }) => { - const { client, session } = await connect(mcp); - try { - const root = await createTestTempDirectory("rea-pe-cert-mcp-budget-"); - const path = join(root, "large.exe"); - await writeFile( - path, - peCertificateFixture(false, repeatedCertificateSignedData(count)), - ); - const response = await client.callTool({ - name: "inspect_pe_certificates", - arguments: { path }, - }); - expect(response.isError).toBe(true); - expect(parseMcpToolError(response).error).toMatchObject({ - code: "resource_constraint", - category: "resource_constraint", - retryable: false, - details: { operation: "inspect_pe_certificates", reported_limits }, - }); - } finally { - await session.close(); - } + const client = await connect(mcp); + const root = await createTestTempDirectory("rea-pe-cert-mcp-budget-"); + const path = join(root, "large.exe"); + await writeFile( + path, + peCertificateFixture(false, repeatedCertificateSignedData(count)), + ); + const response = await client.callTool({ + name: "inspect_pe_certificates", + arguments: { path }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "resource_constraint", + category: "resource_constraint", + retryable: false, + details: { operation: "inspect_pe_certificates", reported_limits }, + }); }, ); diff --git a/tests/boundary/mcp/progressMcp.test.ts b/tests/boundary/mcp/progressMcp.test.ts index 533ec45bec..33bedf5f01 100644 --- a/tests/boundary/mcp/progressMcp.test.ts +++ b/tests/boundary/mcp/progressMcp.test.ts @@ -2,6 +2,8 @@ import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { setTimeout as delay } from "node:timers/promises"; import { describe, expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; + import { createAnalysisExecution, type AnalysisOperationPort, @@ -11,6 +13,7 @@ import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js import { err, ok } from "../../../src/domain/result.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; describe("ordinary MCP progress and cancellation", () => { it("advances MCP progress when provider waiting does not advance completed work", async () => { @@ -37,7 +40,7 @@ describe("ordinary MCP progress and cancellation", () => { total?: number | undefined; message?: string | undefined; }[] = []; - try { + await withVerifierCleanup(async () => { await server.connect(serverTransport); await client.connect(clientTransport); const result = await client.callTool( @@ -60,9 +63,7 @@ describe("ordinary MCP progress and cancellation", () => { ); expect(update.total).toBeUndefined(); }); - } finally { - await Promise.allSettled([client.close(), server.close()]); - } + }, [() => client.close(), () => server.close()]); }); it("keeps analyze_function progress strictly increasing from start to completion", async () => { @@ -84,7 +85,7 @@ describe("ordinary MCP progress and cancellation", () => { const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); const progress: number[] = []; - try { + await withVerifierCleanup(async () => { await server.connect(serverTransport); await client.connect(clientTransport); const result = await client.callTool( @@ -100,31 +101,40 @@ describe("ordinary MCP progress and cancellation", () => { progress.forEach((value, index) => { expect(value).toBeGreaterThan(progress[index - 1] ?? 0); }); - } finally { - await Promise.allSettled([client.close(), server.close()]); - } + }, [() => client.close(), () => server.close()]); }); it("propagates client cancellation to the active provider request", async () => { let providerObservedCancellation = false; - let markProviderStarted: (() => void) | undefined; + let providerWasStarted = false; + let markProviderStarted!: () => void; + let markProviderFinished!: () => void; + let releaseProvider!: () => void; const providerStarted = new Promise((resolve) => { markProviderStarted = resolve; }); + const providerFinished = new Promise((resolve) => { + markProviderFinished = resolve; + }); + const providerGate = new Promise((resolve) => { + releaseProvider = resolve; + }); const analysis: AnalysisOperationPort = { execute: async (operation, _parameters, options) => { - markProviderStarted?.(); - await new Promise((resolve) => { - options?.signal?.addEventListener( - "abort", - () => { - providerObservedCancellation = true; - resolve(); - }, - { once: true }, - ); - }); - return err(new AnalysisCancelledError(operation)); + const onAbort = () => { + providerObservedCancellation = true; + releaseProvider(); + }; + options?.signal?.addEventListener("abort", onAbort, { once: true }); + providerWasStarted = true; + markProviderStarted(); + try { + await providerGate; + return err(new AnalysisCancelledError(operation)); + } finally { + options?.signal?.removeEventListener("abort", onAbort); + markProviderFinished(); + } }, }; const server = createServer({ kind: "fixed", analysis }); @@ -132,19 +142,24 @@ describe("ordinary MCP progress and cancellation", () => { const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); const controller = new AbortController(); - try { + await withVerifierCleanup(async () => { await server.connect(serverTransport); await client.connect(clientTransport); const request = client.callTool( { name: "current_address", arguments: {} }, { signal: controller.signal }, ); - await providerStarted; + await awaitAcknowledgement(providerStarted, request, "provider request"); controller.abort(); await expect(request).rejects.toThrow(/abort/iu); await expect.poll(() => providerObservedCancellation).toBe(true); - } finally { - await Promise.allSettled([client.close(), server.close()]); - } + }, [ + async () => { + releaseProvider(); + if (providerWasStarted) await providerFinished; + }, + () => client.close(), + () => server.close(), + ]); }); }); diff --git a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts index f0a9c3f48e..d079b4b4b5 100644 --- a/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts +++ b/tests/boundary/mcp/pyinstallerArtifactMcp.test.ts @@ -2,8 +2,10 @@ import { access, readFile, rm, writeFile } from "node:fs/promises"; import { createHash } from "node:crypto"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { afterAll, expect, it } from "vitest"; +import type { Client } from "@modelcontextprotocol/client"; +import { afterAll, expect } from "vitest"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; +import { ownArtifactSession } from "../../support/mcp/artifactSession.js"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; import { createArtifactBinarySession } from "../../../src/composition/binary.js"; @@ -199,159 +201,142 @@ const writeSurrogateNamespaceExecutable = async (): Promise => { return path; }; -it.each([false, true])( +mcpTest.for([false, true])( "inventories a PyInstaller executable through inspect_artifact (opaque: %s)", - async (opaque) => { + async (opaque, { mcp, onTestFinished }) => { const path = await writeExecutable(opaque); const session = await createArtifactBinarySession(process.env); - const server = createServer({ kind: "session", session }); - const client = new Client( + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect( + createServer({ kind: "session", session }), { name: "pyinstaller-mcp-test", version: "1" }, - { jsonSchemaValidator: schemaValidation }, + schemaValidation, ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path }, - }); - expect(opened.isError).not.toBe(true); - const inspected = await client.callTool({ - name: "inspect_artifact", - arguments: {}, - }); - expect(inspected.isError).not.toBe(true); - const inspection = artifactInspectionResultSchema.parse( - parseEvidence(inspected.structuredContent).normalized_result, - ); - expect(inspection.subject.root_format).toBe("pyinstaller"); - if (!opaque) expect(inspection.next_probes).toEqual([]); + + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError).not.toBe(true); + const inspected = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(inspected.isError).not.toBe(true); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(inspected.structuredContent).normalized_result, + ); + expect(inspection.subject.root_format).toBe("pyinstaller"); + if (!opaque) expect(inspection.next_probes).toEqual([]); + expect( + inspection.substeps[0]?.evidence.normalized_result.coverage.status, + ).toBe(opaque ? "unknown" : "complete"); + if (opaque) expect( - inspection.substeps[0]?.evidence.normalized_result.coverage.status, - ).toBe(opaque ? "unknown" : "complete"); - if (opaque) - expect( - inspection.substeps[0]?.evidence.normalized_result.occurrences.find( - (row) => row.logical_path === "PYZ.pyz", - ), - ).toMatchObject({ - encrypted: false, - hash_status: "verified", - artifact_id: expect.any(String), - format_metadata: { encryption_source: "carchive-marker" }, - }); - expect( - inspection.substeps[0]?.evidence.normalized_result.occurrences.map( - (occurrence) => occurrence.logical_path, + inspection.substeps[0]?.evidence.normalized_result.occurrences.find( + (row) => row.logical_path === "PYZ.pyz", ), - ).toEqual( - expect.arrayContaining([ - "hello", - ...(opaque ? [] : ["PYZ.pyz/app.pyc"]), - ]), - ); - if (!opaque) - expect( - inspection.substeps[0]?.evidence.normalized_result.occurrences.find( - (row) => row.logical_path === "PYZ.pyz/app.pyc", - ), - ).toMatchObject({ - source_path: "app", - format_metadata: { - container_name: "PYZ.pyz", - source_name: "app", - }, - }); - const parent = parseEvidence(inspected.structuredContent); - const compact = await client.callTool({ - name: "inspect_artifact", - arguments: { detail: "summary" }, + ).toMatchObject({ + encrypted: false, + hash_status: "verified", + artifact_id: expect.any(String), + format_metadata: { encryption_source: "carchive-marker" }, }); - expect(compact.isError).not.toBe(true); - const summary = parseEvidence( - compact.structuredContent, - ).normalized_result; - expect(summary).toMatchObject({ - kind: "summary", - summary: { - root_format_metadata: { - kind: "pyinstaller", - python_version_raw: 312, - external_dependency_count: 1, - }, - observed_entry_kind_counts: { metadata: 3 }, - source_coverage: { status: opaque ? "unknown" : "complete" }, + expect( + inspection.substeps[0]?.evidence.normalized_result.occurrences.map( + (occurrence) => occurrence.logical_path, + ), + ).toEqual( + expect.arrayContaining(["hello", ...(opaque ? [] : ["PYZ.pyz/app.pyc"])]), + ); + if (!opaque) + expect( + inspection.substeps[0]?.evidence.normalized_result.occurrences.find( + (row) => row.logical_path === "PYZ.pyz/app.pyc", + ), + ).toMatchObject({ + source_path: "app", + format_metadata: { + container_name: "PYZ.pyz", + source_name: "app", }, }); - await assertMetadataQueries( - client, - parent, - opaque ? "unknown" : "complete", - ); - if (opaque) { - const missing = await client.callTool({ - name: "inspect_analysis_view", - arguments: { - source: { - kind: "retained-evidence", - evidence_id: parent.evidence_id, - }, - view: { - kind: "inventory", - collection: "occurrences", - logical_paths: ["PYZ.pyz/app.pyc"], - entry_kinds: ["file"], - fields: [], - offset: 0, - limit: 1, - }, + const parent = parseEvidence(inspected.structuredContent); + const compact = await client.callTool({ + name: "inspect_artifact", + arguments: { detail: "summary" }, + }); + expect(compact.isError).not.toBe(true); + const summary = parseEvidence(compact.structuredContent).normalized_result; + expect(summary).toMatchObject({ + kind: "summary", + summary: { + root_format_metadata: { + kind: "pyinstaller", + python_version_raw: 312, + external_dependency_count: 1, + }, + observed_entry_kind_counts: { metadata: 3 }, + source_coverage: { status: opaque ? "unknown" : "complete" }, + }, + }); + await assertMetadataQueries( + client, + parent, + opaque ? "unknown" : "complete", + ); + if (opaque) { + const missing = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, }, - }); - expect(missing.isError).not.toBe(true); - const page = analysisViewResultSchema.parse( - parseEvidence(missing.structuredContent).normalized_result, - ); - expect(page).toMatchObject({ - kind: "inventory", - items: [], - source_coverage: { status: "unknown" }, - }); - expect(page.unknowns).toContain( - "Producer traversal coverage is unknown; missing matches do not prove source absence.", - ); - } - } finally { - await client.close(); - await server.close(); - await session.close(); + view: { + kind: "inventory", + collection: "occurrences", + logical_paths: ["PYZ.pyz/app.pyc"], + entry_kinds: ["file"], + fields: [], + offset: 0, + limit: 1, + }, + }, + }); + expect(missing.isError).not.toBe(true); + const page = analysisViewResultSchema.parse( + parseEvidence(missing.structuredContent).normalized_result, + ); + expect(page).toMatchObject({ + kind: "inventory", + items: [], + source_coverage: { status: "unknown" }, + }); + expect(page.unknowns).toContain( + "Producer traversal coverage is unknown; missing matches do not prove source absence.", + ); } }, ); -it("preserves legacy CArchive identities and modern bytecode derivation through inspect_artifact", async () => { - const fixture = cArchiveBytecodeExecutable(); - const directory = await createTestTempDirectory( - "rea-pyinstaller-mcp-bytecode-", - ); - const path = join(directory, "bytecode-layouts"); - await writeFile(path, fixture.archive); - const session = await createArtifactBinarySession(process.env); - const server = createServer({ kind: "session", session }); - const client = new Client( - { - name: "pyinstaller-bytecode-mcp-test", - version: "1", - }, - { jsonSchemaValidator: schemaValidation }, - ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); +mcpTest( + "preserves legacy CArchive identities and modern bytecode derivation through inspect_artifact", + async ({ mcp, onTestFinished }) => { + const fixture = cArchiveBytecodeExecutable(); + const directory = await createTestTempDirectory( + "rea-pyinstaller-mcp-bytecode-", + ); + const path = join(directory, "bytecode-layouts"); + await writeFile(path, fixture.archive); + const session = await createArtifactBinarySession(process.env); + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect( + createServer({ kind: "session", session }), + { name: "pyinstaller-bytecode-mcp-test", version: "1" }, + schemaValidation, + ); + const opened = await client.callTool({ name: "open_binary", arguments: { path }, @@ -390,29 +375,21 @@ it("preserves legacy CArchive identities and modern bytecode derivation through )?.relation, ).toBe(module.derived ? "derived-from" : "contains"); } - } finally { - await client.close(); - await server.close(); - await session.close(); - } -}); + }, +); + +mcpTest( + "preserves surrogatepass PYZ module names through inspect_artifact", + async ({ mcp, onTestFinished }) => { + const path = await writeSurrogateNamespaceExecutable(); + const session = await createArtifactBinarySession(process.env); + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect( + createServer({ kind: "session", session }), + { name: "pyinstaller-surrogate-mcp-test", version: "1" }, + schemaValidation, + ); -it("preserves surrogatepass PYZ module names through inspect_artifact", async () => { - const path = await writeSurrogateNamespaceExecutable(); - const session = await createArtifactBinarySession(process.env); - const server = createServer({ kind: "session", session }); - const client = new Client( - { - name: "pyinstaller-surrogate-mcp-test", - version: "1", - }, - { jsonSchemaValidator: schemaValidation }, - ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); const opened = await client.callTool({ name: "open_binary", arguments: { path }, @@ -444,14 +421,10 @@ it("preserves surrogatepass PYZ module names through inspect_artifact", async () entry_kind: "file", format_metadata: { source_name: "healthy" }, }); - } finally { - await client.close(); - await server.close(); - await session.close(); - } -}); + }, +); -it.each([ +mcpTest.for([ { name: "recognized opaque magic agrees with cookie", cookie: 309, @@ -486,7 +459,10 @@ it.each([ }, ])( "retains PYZ header evidence through inspect_artifact: $name", - async ({ cookie, magic, version, conflict, derived }) => { + async ( + { cookie, magic, version, conflict, derived }, + { mcp, onTestFinished }, + ) => { const payload = pyzArchive({ magic, modules: [], @@ -526,67 +502,56 @@ it.each([ }), ); const session = await createArtifactBinarySession(process.env); - const server = createServer({ kind: "session", session }); - const client = new Client( - { - name: "opaque-pyz-metadata-test", - version: "1", - }, - { jsonSchemaValidator: schemaValidation }, + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect( + createServer({ kind: "session", session }), + { name: "opaque-pyz-metadata-test", version: "1" }, + schemaValidation, ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); + + expect( + (await client.callTool({ name: "open_binary", arguments: { path } })) + .isError, + ).not.toBe(true); + const reply = await client.callTool({ + name: "inspect_artifact", + arguments: {}, + }); + expect(reply.isError).not.toBe(true); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(reply.structuredContent).normalized_result, + ); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + expect(inventory?.coverage.status).toBe("unknown"); + expect( + inventory?.occurrences.find((row) => row.logical_path === "PYZ.pyz"), + ).toMatchObject({ + entry_kind: "file", + hash_status: "verified", + format_metadata: { + pyz_status: "opaque", + pyz_sha256: createHash("sha256").update(payload).digest("hex"), + pyz_python_magic: magic.toString("hex"), + pyz_python_version: version, + pyz_entry_count: null, + }, + }); + expect( + inventory?.occurrences.find((row) => row.logical_path === "hello") + ?.format_metadata, + ).toMatchObject({ + bytecode_header: derived ? "derived" : "unknown", + bytecode_magic: derived ? magic.toString("hex") : null, + }); + if (conflict) expect( - (await client.callTool({ name: "open_binary", arguments: { path } })) - .isError, - ).not.toBe(true); - const reply = await client.callTool({ - name: "inspect_artifact", - arguments: {}, - }); - expect(reply.isError).not.toBe(true); - const inspection = artifactInspectionResultSchema.parse( - parseEvidence(reply.structuredContent).normalized_result, - ); - const inventory = inspection.substeps[0]?.evidence.normalized_result; - expect(inventory?.coverage.status).toBe("unknown"); - expect( - inventory?.occurrences.find((row) => row.logical_path === "PYZ.pyz"), - ).toMatchObject({ - entry_kind: "file", - hash_status: "verified", - format_metadata: { - pyz_status: "opaque", - pyz_sha256: createHash("sha256").update(payload).digest("hex"), - pyz_python_magic: magic.toString("hex"), - pyz_python_version: version, - pyz_entry_count: null, - }, - }); - expect( - inventory?.occurrences.find((row) => row.logical_path === "hello") + inventory?.occurrences.find((row) => row.logical_path === "other.pyz") ?.format_metadata, ).toMatchObject({ - bytecode_header: derived ? "derived" : "unknown", - bytecode_magic: derived ? magic.toString("hex") : null, + pyz_status: "opaque", + pyz_python_version: "3.13", + pyz_python_magic: "f30d0d0a", }); - if (conflict) - expect( - inventory?.occurrences.find((row) => row.logical_path === "other.pyz") - ?.format_metadata, - ).toMatchObject({ - pyz_status: "opaque", - pyz_python_version: "3.13", - pyz_python_magic: "f30d0d0a", - }); - } finally { - await client.close(); - await server.close(); - await session.close(); - } }, ); @@ -614,9 +579,9 @@ const PYZ_ROLE_FILES = [ }, ]; -it.each([false, true])( +mcpTest.for([false, true])( "preserves PYZ member roles through MCP inventory and extraction (encryption marker=%s)", - async (encrypted) => { + async (encrypted, { mcp, onTestFinished }) => { const files = encrypted ? [] : PYZ_ROLE_FILES; const directory = await createTestTempDirectory("rea-pyz-roles-mcp-"); const path = join(directory, "roles"); @@ -658,17 +623,14 @@ it.each([false, true])( }), ); const session = await createArtifactBinarySession(process.env); - const server = createServer({ kind: "session", session }); - const client = new Client( + ownArtifactSession(session, onTestFinished); + const client = await mcp.connect( + createServer({ kind: "session", session }), { name: "pyz-role-test", version: "1" }, - { jsonSchemaValidator: schemaValidation }, + schemaValidation, ); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); let outputRoot: string | undefined; try { - await server.connect(serverTransport); - await client.connect(clientTransport); expect( (await client.callTool({ name: "open_binary", arguments: { path } })) .isError, @@ -747,9 +709,6 @@ it.each([false, true])( ).rejects.toMatchObject({ code: "ENOENT" }); } } finally { - await client.close(); - await server.close(); - await session.close(); if (outputRoot !== undefined) await rm(outputRoot, { recursive: true, force: true }); } diff --git a/tests/boundary/mcp/pythonPycMcp.test.ts b/tests/boundary/mcp/pythonPycMcp.test.ts index ac727d0c6d..9fde999066 100644 --- a/tests/boundary/mcp/pythonPycMcp.test.ts +++ b/tests/boundary/mcp/pythonPycMcp.test.ts @@ -1,10 +1,9 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; -import { afterAll, expect } from "vitest"; +import { afterAll, expect, it } from "vitest"; import { pythonPycSchema } from "../../../src/domain/python/pythonPyc.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; -import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { @@ -13,7 +12,7 @@ import { pythonPycHeader, } from "../../../src/python/PythonPyc.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { mcpTest, type TestMcp } from "../../support/mcp/mcpFixture.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; const completePycEvidenceSchema = toolContract( "inspect_python_pyc", @@ -37,139 +36,37 @@ const marshalSingleItemTuple = (item: Buffer): Buffer => const schemaValidation = createMcpSchemaValidation(); afterAll(() => schemaValidation.clear()); -const connect = async (mcp: TestMcp) => { - const session = createTestBinarySession(() => { - throw new Error("Python pyc inspection must not start a deep provider"); +const connect = () => + createApplicationMcpHarness({ + session: createTestBinarySession(() => { + throw new Error("Python pyc inspection must not start a deep provider"); + }), + serverOptions: { + environment: { + ...process.env, + REA_MCP_MAX_INPUT_ELEMENTS: process.env.REA_MCP_MAX_INPUT_ELEMENTS, + }, + }, + schemaValidator: schemaValidation, }); - const client = await mcp.connect( - createServer({ kind: "session", session }), - undefined, - schemaValidation, - ); - return { client, session }; -}; -mcpTest( - "reads signed TYPE_INT64 constants and rejects a truncated payload", - async ({ mcp }) => { - const { client, session } = await connect(mcp); - try { - const root = await createTestTempDirectory("rea-python-pyc-mcp-int64-"); - for (const [name, value, bits] of [ - ["positive", 42n, 6], - ["negative", -1n, 1], - ["minimum", -(1n << 63n), 64], - ["maximum", (1n << 63n) - 1n, 63], - ] as const) { - const path = join(root, `${name}.pyc`); - await writeFile( - path, - Buffer.concat([ - pythonPycHeader(3531), - marshalCode("py311", { - constsObject: marshalSingleItemTuple(marshalInt64(value)), - }), - ]), - ); - const response = await client.callTool({ - name: "inspect_python_pyc", - arguments: { path }, - }); - expect(response.isError, JSON.stringify(response.content)).not.toBe( - true, - ); - const evidence = completePycEvidenceSchema.parse( - response.structuredContent, - ); - expect( - evidence.normalized_result.code_objects[0]?.consts, - ).toContainEqual({ kind: "int", bits, value: value.toString() }); - } - - const ordinaryPath = join(root, "ordinary-int.pyc"); - await writeFile( - ordinaryPath, - Buffer.concat([ - pythonPycHeader(3531), - marshalCode("py311", { - constsObject: marshalSingleItemTuple(marshalInt32(-42)), - }), - ]), - ); - const ordinary = await client.callTool({ - name: "inspect_python_pyc", - arguments: { path: ordinaryPath }, - }); - expect(ordinary.isError, JSON.stringify(ordinary.content)).not.toBe(true); - const ordinaryEvidence = completePycEvidenceSchema.parse( - ordinary.structuredContent, - ); - expect( - ordinaryEvidence.normalized_result.code_objects[0]?.consts, - ).toContainEqual({ kind: "int", bits: 6, value: "-42" }); - - const truncatedPath = join(root, "truncated.pyc"); - await writeFile( - truncatedPath, - Buffer.concat([ - pythonPycHeader(3531), - Buffer.from([0x49]), - Buffer.alloc(7), - ]), - ); - const truncated = await client.callTool({ - name: "inspect_python_pyc", - arguments: { path: truncatedPath }, - }); - expect(truncated.isError).toBe(true); - expect(parseMcpToolError(truncated).error).toMatchObject({ - code: "invalid_request", - details: { - operation: "inspect_python_pyc", - issues: [ - { - reason: "invalid_format", - message: expect.stringContaining("64-bit integer"), - }, - ], - }, - }); - } finally { - await session.close(); - } - }, -); - -mcpTest( - "advertises inspect_python_pyc and reads nested code objects", - async ({ mcp }) => { - const { client, session } = await connect(mcp); - try { - const advertised = (await client.listTools()).tools.find( - (tool) => tool.name === "inspect_python_pyc", - ); - expect(advertised?.annotations).toMatchObject({ - readOnlyHint: false, - idempotentHint: true, - }); - if (advertised?.outputSchema === undefined) - throw new Error("Missing inspect_python_pyc output schema"); - const outputSchema: Record = advertised.outputSchema; - const ajv = schemaValidation.assertions; - expect(ajv.validateSchema(outputSchema)).toBe(true); - const root = await createTestTempDirectory("rea-python-pyc-mcp-"); - const path = join(root, "sample.pyc"); - const nested = marshalCode("py311", { - name: "inner", - firstlineno: -7, - }); +it("reads signed TYPE_INT64 constants and rejects a truncated payload", async () => { + const { client, close } = await connect(); + try { + const root = await createTestTempDirectory("rea-python-pyc-mcp-int64-"); + for (const [name, value, bits] of [ + ["positive", 42n, 6], + ["negative", -1n, 1], + ["minimum", -(1n << 63n), 64], + ["maximum", (1n << 63n) - 1n, 63], + ] as const) { + const path = join(root, `${name}.pyc`); await writeFile( path, Buffer.concat([ pythonPycHeader(3531), marshalCode("py311", { - firstlineno: -1, - consts: [nested], + constsObject: marshalSingleItemTuple(marshalInt64(value)), }), ]), ); @@ -181,173 +78,260 @@ mcpTest( const evidence = completePycEvidenceSchema.parse( response.structuredContent, ); + expect(evidence.normalized_result.code_objects[0]?.consts).toContainEqual( + { kind: "int", bits, value: value.toString() }, + ); + } + + const ordinaryPath = join(root, "ordinary-int.pyc"); + await writeFile( + ordinaryPath, + Buffer.concat([ + pythonPycHeader(3531), + marshalCode("py311", { + constsObject: marshalSingleItemTuple(marshalInt32(-42)), + }), + ]), + ); + const ordinary = await client.callTool({ + name: "inspect_python_pyc", + arguments: { path: ordinaryPath }, + }); + expect(ordinary.isError, JSON.stringify(ordinary.content)).not.toBe(true); + const ordinaryEvidence = completePycEvidenceSchema.parse( + ordinary.structuredContent, + ); + expect( + ordinaryEvidence.normalized_result.code_objects[0]?.consts, + ).toContainEqual({ kind: "int", bits: 6, value: "-42" }); + + const truncatedPath = join(root, "truncated.pyc"); + await writeFile( + truncatedPath, + Buffer.concat([ + pythonPycHeader(3531), + Buffer.from([0x49]), + Buffer.alloc(7), + ]), + ); + const truncated = await client.callTool({ + name: "inspect_python_pyc", + arguments: { path: truncatedPath }, + }); + expect(truncated.isError).toBe(true); + expect(parseMcpToolError(truncated).error).toMatchObject({ + code: "invalid_request", + details: { + operation: "inspect_python_pyc", + issues: [ + { + reason: "invalid_format", + message: expect.stringContaining("64-bit integer"), + }, + ], + }, + }); + } finally { + await close(); + } +}); + +it("advertises inspect_python_pyc and reads nested code objects", async () => { + const { client, close } = await connect(); + try { + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_python_pyc", + ); + expect(advertised?.annotations).toMatchObject({ + readOnlyHint: false, + idempotentHint: true, + }); + if (advertised?.outputSchema === undefined) + throw new Error("Missing inspect_python_pyc output schema"); + const outputSchema: Record = advertised.outputSchema; + const ajv = schemaValidation.assertions; + expect(ajv.validateSchema(outputSchema)).toBe(true); + const root = await createTestTempDirectory("rea-python-pyc-mcp-"); + const path = join(root, "sample.pyc"); + const nested = marshalCode("py311", { + name: "inner", + firstlineno: -7, + }); + await writeFile( + path, + Buffer.concat([ + pythonPycHeader(3531), + marshalCode("py311", { + firstlineno: -1, + consts: [nested], + }), + ]), + ); + const response = await client.callTool({ + name: "inspect_python_pyc", + arguments: { path }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const evidence = completePycEvidenceSchema.parse( + response.structuredContent, + ); + expect( + ajv.validate(outputSchema, response.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + expect(evidence.normalized_result.header.python_version).toBe("3.12"); + expect(evidence.normalized_result.code_objects[0]?.firstlineno).toBe(-1); + expect(evidence.normalized_result.code_objects[1]?.name).toBe("inner"); + expect(evidence.normalized_result.code_objects[1]?.firstlineno).toBe(-7); + expect(evidence.normalized_result.code_objects[0]?.consts).toContainEqual({ + kind: "code", + name: "inner", + firstlineno: -7, + }); + } finally { + await close(); + } +}); + +it("requires an absolute path for inspect_python_pyc", async () => { + const { client, close } = await connect(); + try { + const response = await client.callTool({ + name: "inspect_python_pyc", + arguments: { path: "sample.pyc" }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "invalid_request", + details: { operation: "inspect_python_pyc" }, + }); + } finally { + await close(); + } +}); + +it("advertised pyc output schema preserves historical signed fields", async () => { + const { client, close } = await connect(); + try { + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_python_pyc", + ); + if (advertised?.outputSchema === undefined) + throw new Error("Missing inspect_python_pyc output schema"); + const outputSchema: Record = advertised.outputSchema; + const ajv = schemaValidation.assertions; + const root = await createTestTempDirectory("rea-python-pyc-mcp-flags-"); + for (const testCase of [ + { + name: "legacy-negative", + magic: 3394, + layout: "py37" as const, + fields: { stacksize: 0xffffffff }, + expected: { stacksize: -1, flags: 0 }, + }, + { + name: "alpha-3401-negative-flags", + magic: 3401, + layout: "py37" as const, + fields: { flags: 0x80000000 }, + expected: { stacksize: 1, flags: -0x80000000 }, + }, + { + name: "ambiguous-3410-negative-fields", + magic: 3410, + layout: "py38" as const, + fields: { stacksize: 0xffffffff, flags: 0x80000000 }, + expected: { stacksize: -1, flags: -0x80000000 }, + }, + { + name: "unknown-positive", + magic: 3531, + layout: "py311" as const, + fields: { flags: 0x40000000 }, + expected: { flags: 0x40000000 }, + }, + ]) { + const path = join(root, `${testCase.name}.pyc`); + await writeFile( + path, + Buffer.concat([ + pythonPycHeader(testCase.magic), + marshalCode(testCase.layout, testCase.fields), + ]), + ); + const response = await client.callTool({ + name: "inspect_python_pyc", + arguments: { path }, + }); + expect(response.isError, JSON.stringify(response.content)).not.toBe(true); + const evidence = completePycEvidenceSchema.parse( + response.structuredContent, + ); expect( ajv.validate(outputSchema, response.structuredContent), JSON.stringify(ajv.errors), ).toBe(true); - expect(evidence.normalized_result.header.python_version).toBe("3.12"); - expect(evidence.normalized_result.code_objects[0]?.firstlineno).toBe(-1); - expect(evidence.normalized_result.code_objects[1]?.name).toBe("inner"); - expect(evidence.normalized_result.code_objects[0]?.consts).toContainEqual( - { - kind: "code", - name: "inner", - firstlineno: -7, - }, + expect(evidence.normalized_result.code_objects[0]).toMatchObject( + testCase.expected, ); - } finally { - await session.close(); + if (testCase.magic === 3410) + expect(evidence.normalized_result.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("Magic 3410 is shared"), + ]), + ); } - }, -); - -mcpTest("requires an absolute path for inspect_python_pyc", async ({ mcp }) => { - const { client, session } = await connect(mcp); - try { - const response = await client.callTool({ - name: "inspect_python_pyc", - arguments: { path: "sample.pyc" }, - }); - expect(response.isError).toBe(true); - expect(parseMcpToolError(response).error).toMatchObject({ - code: "invalid_request", - details: { operation: "inspect_python_pyc" }, - }); } finally { - await session.close(); + await close(); } }); -mcpTest( - "advertised pyc output schema preserves historical signed fields", - async ({ mcp }) => { - const { client, session } = await connect(mcp); - try { - const advertised = (await client.listTools()).tools.find( - (tool) => tool.name === "inspect_python_pyc", +it("rejects negative code fields for magics after constructor validation was added", async () => { + const { client, close } = await connect(); + try { + const root = await createTestTempDirectory("rea-python-pyc-mcp-signed-"); + for (const { name, fields, message } of [ + { + name: "negative-stacksize", + fields: { stacksize: 0xffffffff }, + message: "co_stacksize is negative", + }, + { + name: "negative-flags", + fields: { flags: 0x80000000 }, + message: "co_flags is negative", + }, + ]) { + const path = join(root, `${name}.pyc`); + await writeFile( + path, + Buffer.concat([pythonPycHeader(3411), marshalCode("py38", fields)]), ); - if (advertised?.outputSchema === undefined) - throw new Error("Missing inspect_python_pyc output schema"); - const outputSchema: Record = advertised.outputSchema; - const ajv = schemaValidation.assertions; - const root = await createTestTempDirectory("rea-python-pyc-mcp-flags-"); - for (const testCase of [ - { - name: "legacy-negative", - magic: 3394, - layout: "py37" as const, - fields: { stacksize: 0xffffffff }, - expected: { stacksize: -1, flags: 0 }, + const response = await client.callTool({ + name: "inspect_python_pyc", + arguments: { path }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "invalid_request", + details: { + operation: "inspect_python_pyc", + issues: [ + { + reason: "invalid_format", + message: expect.stringContaining(message), + }, + ], }, - { - name: "alpha-3401-negative-flags", - magic: 3401, - layout: "py37" as const, - fields: { flags: 0x80000000 }, - expected: { stacksize: 1, flags: -0x80000000 }, - }, - { - name: "ambiguous-3410-negative-fields", - magic: 3410, - layout: "py38" as const, - fields: { stacksize: 0xffffffff, flags: 0x80000000 }, - expected: { stacksize: -1, flags: -0x80000000 }, - }, - { - name: "unknown-positive", - magic: 3531, - layout: "py311" as const, - fields: { flags: 0x40000000 }, - expected: { flags: 0x40000000 }, - }, - ]) { - const path = join(root, `${testCase.name}.pyc`); - await writeFile( - path, - Buffer.concat([ - pythonPycHeader(testCase.magic), - marshalCode(testCase.layout, testCase.fields), - ]), - ); - const response = await client.callTool({ - name: "inspect_python_pyc", - arguments: { path }, - }); - expect(response.isError, JSON.stringify(response.content)).not.toBe( - true, - ); - const evidence = completePycEvidenceSchema.parse( - response.structuredContent, - ); - expect( - ajv.validate(outputSchema, response.structuredContent), - JSON.stringify(ajv.errors), - ).toBe(true); - expect(evidence.normalized_result.code_objects[0]).toMatchObject( - testCase.expected, - ); - if (testCase.magic === 3410) - expect(evidence.normalized_result.limitations).toEqual( - expect.arrayContaining([ - expect.stringContaining("Magic 3410 is shared"), - ]), - ); - } - } finally { - await session.close(); + }); } - }, -); + } finally { + await close(); + } +}); -mcpTest( - "rejects negative code fields for magics after constructor validation was added", - async ({ mcp }) => { - const { client, session } = await connect(mcp); - try { - const root = await createTestTempDirectory("rea-python-pyc-mcp-signed-"); - for (const { name, fields, message } of [ - { - name: "negative-stacksize", - fields: { stacksize: 0xffffffff }, - message: "co_stacksize is negative", - }, - { - name: "negative-flags", - fields: { flags: 0x80000000 }, - message: "co_flags is negative", - }, - ]) { - const path = join(root, `${name}.pyc`); - await writeFile( - path, - Buffer.concat([pythonPycHeader(3411), marshalCode("py38", fields)]), - ); - const response = await client.callTool({ - name: "inspect_python_pyc", - arguments: { path }, - }); - expect(response.isError).toBe(true); - expect(parseMcpToolError(response).error).toMatchObject({ - code: "invalid_request", - details: { - operation: "inspect_python_pyc", - issues: [ - { - reason: "invalid_format", - message: expect.stringContaining(message), - }, - ], - }, - }); - } - } finally { - await session.close(); - } - }, -); - -mcpTest("reports an unknown pyc magic as unsupported", async ({ mcp }) => { - const { client, session } = await connect(mcp); +it("reports an unknown pyc magic as unsupported", async () => { + const { client, close } = await connect(); try { const root = await createTestTempDirectory("rea-python-pyc-mcp-magic-"); const path = join(root, "unknown.pyc"); @@ -364,6 +348,6 @@ mcpTest("reports an unknown pyc magic as unsupported", async ({ mcp }) => { details: { operation: "inspect_python_pyc" }, }); } finally { - await session.close(); + await close(); } }); diff --git a/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts b/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts index d4b59f73c3..d02abb509e 100644 --- a/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts +++ b/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts @@ -1,10 +1,9 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { afterAll, afterEach, expect, it } from "vitest"; +import { afterAll, expect, it } from "vitest"; import { Ajv2020 } from "ajv/dist/2020.js"; import type { AnySchema } from "ajv"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { JAVASCRIPT_FEATURE_TRACE_EXAMPLE, JAVASCRIPT_APPLICATION_VERSION_COMPARISON_EXAMPLE, @@ -14,12 +13,8 @@ import { JAVASCRIPT_EXPORT_SHAPE_COMPARISON_EXAMPLE } from "../../../src/contrac import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; -const resources: Array<{ close(): Promise }> = []; const schemaValidation = createMcpSchemaValidation(); afterAll(() => schemaValidation.clear()); -afterEach(async () => { - for (const resource of resources.splice(0)) await resource.close(); -}); const retained = (evidence_id: string) => ({ kind: "retained-evidence", evidence_id, @@ -32,18 +27,14 @@ async function connect(profile?: "full" | "compact") { const session = createTestBinarySession(() => { throw new Error("references must not launch a provider"); }); - const server = createServer( - { kind: "session", session }, - { environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile } }, - ); - const client = new Client( - { name: "retained-evidence-test", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - resources.push(client, server, session); - const [a, b] = InMemoryTransport.createLinkedPair(); - await server.connect(b); - await client.connect(a); + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "retained-evidence-test", version: "1" }, + schemaValidator: schemaValidation, + serverOptions: { + environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, + }, + }); return { client, session }; } diff --git a/tests/boundary/mcp/reverseEngineeringEnvironmentMcp.test.ts b/tests/boundary/mcp/reverseEngineeringEnvironmentMcp.test.ts index 73a4d6eb3b..ff47db600f 100644 --- a/tests/boundary/mcp/reverseEngineeringEnvironmentMcp.test.ts +++ b/tests/boundary/mcp/reverseEngineeringEnvironmentMcp.test.ts @@ -57,9 +57,9 @@ it("uses the captured server environment to resolve reverse-engineering provider configuredProvider: provider, }); const response = await executeRizinCommand(client, artifact); - expect(JSON.stringify(response.structuredContent)).toContain( - "captured-environment-provider", - ); + expect(response.structuredContent).toMatchObject({ + raw_result: { stdout: "captured-environment-provider" }, + }); }); it("prefers an explicit provider environment over the captured server environment", async () => { @@ -82,9 +82,9 @@ it("prefers an explicit provider environment over the captured server environmen }); const response = await executeRizinCommand(client, artifact); - expect(JSON.stringify(response.structuredContent)).toContain( - "explicit-override", - ); + expect(response.structuredContent).toMatchObject({ + raw_result: { stdout: "explicit-override" }, + }); expect(JSON.stringify(response.structuredContent)).not.toContain( '"captured"', ); @@ -112,9 +112,9 @@ it("snapshots an explicit provider environment when the server is created", asyn providerEnvironment.REA_RIZIN_COMMAND = configured; const response = await executeRizinCommand(client, artifact); - expect(JSON.stringify(response.structuredContent)).toContain( - "explicit-snapshot", - ); + expect(response.structuredContent).toMatchObject({ + raw_result: { stdout: "explicit-snapshot" }, + }); expect(JSON.stringify(response.structuredContent)).not.toContain( '"captured"', ); diff --git a/tests/boundary/mcp/reverseEngineeringPartialMcp.test.ts b/tests/boundary/mcp/reverseEngineeringPartialMcp.test.ts index 2fb8c4af2d..435b0e7526 100644 --- a/tests/boundary/mcp/reverseEngineeringPartialMcp.test.ts +++ b/tests/boundary/mcp/reverseEngineeringPartialMcp.test.ts @@ -302,6 +302,7 @@ it.each(["success", "timeout"])( expect((await session.open(target)).ok).toBe(true); expect(session.activeTarget()?.path).toBe(target); const originalDigest = session.activeTarget()?.sha256; + expect(originalDigest).toEqual(expect.any(String)); const { server, handler } = createReverseEngineeringMcpHarness(); const service = new ReverseEngineeringService({ @@ -370,6 +371,7 @@ it.each(["success", "timeout"])( expect(session.activeTarget()).toBeUndefined(); expect((await session.execute("address_name", {})).ok).toBe(false); expect((await session.open(target)).ok).toBe(true); + expect(session.activeTarget()?.sha256).toEqual(expect.any(String)); expect(session.activeTarget()?.sha256).not.toBe(originalDigest); await session.close(); await rm(dirname(target), { recursive: true, force: true }); diff --git a/tests/boundary/mcp/rizinDebuggerCancellationMcp.test.ts b/tests/boundary/mcp/rizinDebuggerCancellationMcp.test.ts index e1f623daf9..79ce8626c2 100644 --- a/tests/boundary/mcp/rizinDebuggerCancellationMcp.test.ts +++ b/tests/boundary/mcp/rizinDebuggerCancellationMcp.test.ts @@ -7,6 +7,7 @@ import { McpServer } from "@modelcontextprotocol/server"; import { expect, it } from "vitest"; import { z } from "zod"; +import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs"; import { AnalysisCapabilityUnavailableError } from "../../../src/domain/analysisErrorCore.js"; import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; import { evidenceSchema } from "../../../src/domain/evidence.js"; @@ -110,9 +111,14 @@ process.stdin.on("data", (chunk) => { expect(status.output_invalid_utf8).toBe(true); expect(status.recent_output).toContain("�"); } finally { - await Promise.allSettled([client.close(), server.close()]); - await manager.closeAll().catch(() => undefined); - await rm(directory, { recursive: true, force: true }); + await closeVerifierResources([ + () => client.close(), + () => server.close(), + async () => { + await manager.closeAll(); + await rm(directory, { recursive: true, force: true }); + }, + ]); } }, ); @@ -174,8 +180,14 @@ it("admits Rizin startup across a bound analysis session", async () => { expect(currentRunId).not.toBe(initialRunId); expect(analysisSession.activeTarget()?.path).toBe(target); } finally { - await Promise.allSettled([client.close(), server.close()]); - await analysisSession.close(); + await closeVerifierResources([ + () => client.close(), + () => server.close(), + async () => { + const closed = await analysisSession.close(); + if (!closed.ok) throw closed.error; + }, + ]); } }); @@ -244,9 +256,14 @@ process.stdin.on("data", async (chunk) => { expect(status.state).toBe("closed"); expect(status).toHaveProperty("diagnostics_truncated"); } finally { - await Promise.allSettled([client.close(), server.close()]); - await manager.closeAll().catch(() => undefined); - await rm(directory, { recursive: true, force: true }); + await closeVerifierResources([ + () => client.close(), + () => server.close(), + async () => { + await manager.closeAll(); + await rm(directory, { recursive: true, force: true }); + }, + ]); } }, ); diff --git a/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts b/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts index c97c153106..6686c95868 100644 --- a/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts +++ b/tests/boundary/mcp/sessionAvailabilityRecovery.test.ts @@ -118,6 +118,8 @@ it("restores MCP tool availability after cancelling a target switch", async () = if (!settled.ok) throw settled.error; const restored = await readStatus(); expect(restored).toMatchObject({ open: true, path: first }); + expect(before.analysis_run?.run_id).toEqual(expect.any(String)); + expect(restored.analysis_run?.run_id).toEqual(expect.any(String)); expect(restored.analysis_run?.run_id).not.toBe(before.analysis_run?.run_id); expect(clients).toEqual([{ calls: 1, closed: 1 }]); expect( diff --git a/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts b/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts index d43f830022..627f0b7a6d 100644 --- a/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts +++ b/tests/boundary/mcp/sourceToBundleViewsMcp.test.ts @@ -1,9 +1,8 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { afterAll, expect, it, onTestFinished } from "vitest"; +import { afterAll, expect, it } from "vitest"; import { createMcpSchemaValidation } from "../../fixtures/mcpSchemaValidation.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { sourceToBundleComparisonResultSchema } from "../../../src/domain/javascript/sourceToBundleComparisonSchemas.js"; import { analysisViewResultSchema } from "../../../src/domain/analysisView/analysisView.js"; @@ -215,24 +214,14 @@ const comparisonFixture = async (profile: "full" | "compact") => { const session = createTestBinarySession(() => { throw new Error("Views must not open a native provider"); }); - const server = createServer( - { kind: "session", session }, - { + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "source-view", version: "1" }, + schemaValidator: schemaValidation, + serverOptions: { environment: { ...process.env, REA_MCP_INPUT_SCHEMA_PROFILE: profile }, }, - ); - const client = new Client( - { name: "source-view", version: "1" }, - { jsonSchemaValidator: schemaValidation }, - ); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); }); - const [a, b] = InMemoryTransport.createLinkedPair(); - await server.connect(b); - await client.connect(a); expect(session.recordEvidence(application.value).ok).toBe(true); const inputs = { reference: reference.value, diff --git a/tests/boundary/mcp/streamableHttpMcp.test.ts b/tests/boundary/mcp/streamableHttpMcp.test.ts index 4abfc349a4..11150c8ce8 100644 --- a/tests/boundary/mcp/streamableHttpMcp.test.ts +++ b/tests/boundary/mcp/streamableHttpMcp.test.ts @@ -55,6 +55,16 @@ describe("rea mcp --http", () => { stdout += chunk.toString("utf8"); }); const first = await openSession(started.url, "streamable-http-boundary"); + expect(() => + assertAdvertisedMcpContracts( + ["binary_session"], + [first.toolDefinition, first.toolDefinition], + ), + ).toThrow(); + const { name } = first.toolDefinition; + expect(() => + assertAdvertisedMcpContracts(["binary_session"], [{ name }]), + ).toThrow(); await expectIdentity(first); await first.transport.terminateSession(); expect(first.transport.sessionId).toBeUndefined(); diff --git a/tests/boundary/mcp/webScriptSelectionMcp.test.ts b/tests/boundary/mcp/webScriptSelectionMcp.test.ts index 4a98c5b82b..1127ec2cfa 100644 --- a/tests/boundary/mcp/webScriptSelectionMcp.test.ts +++ b/tests/boundary/mcp/webScriptSelectionMcp.test.ts @@ -3,7 +3,6 @@ import { access, mkdir, readdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import { expect, it, onTestFinished } from "vitest"; @@ -16,8 +15,10 @@ import { readWithoutFifoWriter } from "../../fixtures/fifoInput.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { scriptScenarioFixture } from "../../fixtures/webScriptCapture.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createTestMcp } from "../../support/mcp/mcpFixture.js"; +import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { webScriptExportResultSchema } from "../../../src/domain/webScriptExport.js"; @@ -41,16 +42,17 @@ it.skipIf(process.platform === "win32")( recordEvidence: undefined, artifactResources: artifactResources, }); - const client = new Client({ name: "script-selection-test", version: "1" }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await artifactResources.close(); + const mcp = createTestMcp(); + onTestFinished(() => + closeVerifierResources([ + () => mcp.close(), + () => artifactResources.close(), + ]), + ); + const client = await mcp.connect(server, { + name: "script-selection-test", + version: "1", }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const outcome = await readWithoutFifoWriter(capturePath, () => client.callTool({ @@ -105,16 +107,17 @@ it("refuses an existing output directory as invalid output_directory input", asy recordEvidence: undefined, artifactResources, }); - const client = new Client({ name: "script-export-existing", version: "1" }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await artifactResources.close(); + const mcp = createTestMcp(); + onTestFinished(() => + closeVerifierResources([ + () => mcp.close(), + () => artifactResources.close(), + ]), + ); + const client = await mcp.connect(server, { + name: "script-export-existing", + version: "1", }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const result = await client.callTool({ name: "export_web_scripts", @@ -157,17 +160,10 @@ it("publishes script bytes and retains the returned Evidence without starting an const session = createTestBinarySession(() => { throw new Error("local script export must not start an analysis provider"); }); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "script-export-retention", version: "1" }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "script-export-retention", version: "1" }, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const response = await client.callTool({ name: "export_web_scripts", arguments: { capture_path: capturePath, output_directory: outputDirectory }, diff --git a/tests/boundary/mcp/workflowSnapshotReplayMcp.test.ts b/tests/boundary/mcp/workflowSnapshotReplayMcp.test.ts index 3739230002..1cad17a2aa 100644 --- a/tests/boundary/mcp/workflowSnapshotReplayMcp.test.ts +++ b/tests/boundary/mcp/workflowSnapshotReplayMcp.test.ts @@ -19,6 +19,7 @@ import { createTestBinarySession, } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; import { readAnalysisSnapshot } from "../../../src/application/binary/AnalysisSnapshotFiles.js"; import { ok } from "../../../src/domain/result.js"; @@ -75,33 +76,48 @@ describe("MCP external command snapshot retirement", () => { name: "execute_rizin_command", arguments: { path: targetPath, command: "iI" }, }); - const readyDeadline = Date.now() + 5_000; - while (Date.now() < readyDeadline) { - try { - await access(startedPath); - break; - } catch { - await new Promise((resolve) => setTimeout(resolve, 10)); - } - } - await expect(access(startedPath)).resolves.toBeUndefined(); - let closeSettled = false; - const closeRequest = mcp - .callTool({ - name: "close_binary", - arguments: { snapshot_path: snapshotPath }, - }) - .finally(() => { - closeSettled = true; + const closeStarted = createDeferred(); + let closeRequest: ReturnType | undefined; + try { + const readyDeadline = Date.now() + 5_000; + while (Date.now() < readyDeadline) { + try { + await access(startedPath); + break; + } catch { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + } + await expect(access(startedPath)).resolves.toBeUndefined(); + closeRequest = mcp + .callTool( + { + name: "close_binary", + arguments: { snapshot_path: snapshotPath }, + }, + { onprogress: () => closeStarted.resolve() }, + ) + .finally(() => { + closeSettled = true; + }); + // Close queues its transition before sending its first progress event. + await awaitAcknowledgement( + closeStarted.promise, + closeRequest, + "snapshot close progress", + ); + expect(closeSettled).toBe(false); + await expect(access(snapshotPath)).rejects.toMatchObject({ + code: "ENOENT", }); - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(closeSettled).toBe(false); - await expect(access(snapshotPath)).rejects.toMatchObject({ - code: "ENOENT", - }); - - await writeFile(releasePath, "release"); + } finally { + await writeFile(releasePath, "release"); + await Promise.allSettled([ + analysisRequest, + ...(closeRequest === undefined ? [] : [closeRequest]), + ]); + } const analyzed = await analysisRequest; expect(analyzed.isError, JSON.stringify(analyzed.content)).not.toBe(true); const evidence = toolContract("execute_rizin_command").outputSchema.parse( @@ -179,18 +195,38 @@ describe("MCP composed workflow target admission", () => { name: "binary_overview", arguments: {}, }); - await entered.promise; - const closeRequest = mcp.callTool({ - name: "close_binary", - arguments: { snapshot_path: snapshotPath }, - _meta: { progressToken: "close-race" }, - }); - await closeStarted.promise; - const openRequest = mcp.callTool({ - name: "open_binary", - arguments: { path: secondPath }, - }); - release.resolve(); + let closeRequest: ReturnType | undefined; + let openRequest: ReturnType | undefined; + try { + await awaitAcknowledgement( + entered.promise, + analysisRequest, + "workflow provider entry", + ); + closeRequest = mcp.callTool({ + name: "close_binary", + arguments: { snapshot_path: snapshotPath }, + _meta: { progressToken: "close-race" }, + }); + await awaitAcknowledgement( + closeStarted.promise, + closeRequest, + "workflow close progress", + ); + openRequest = mcp.callTool({ + name: "open_binary", + arguments: { path: secondPath }, + }); + } finally { + release.resolve(); + await Promise.allSettled([ + analysisRequest, + ...(closeRequest === undefined ? [] : [closeRequest]), + ...(openRequest === undefined ? [] : [openRequest]), + ]); + } + if (closeRequest === undefined || openRequest === undefined) + throw new Error("Expected queued close and open requests"); const analyzed = await analysisRequest; expect(analyzed.isError, JSON.stringify(analyzed.content)).not.toBe(true); diff --git a/tests/boundary/pcap/pcapProvider.test.ts b/tests/boundary/pcap/pcapProvider.test.ts index 21f6c83f34..d690e3e62d 100644 --- a/tests/boundary/pcap/pcapProvider.test.ts +++ b/tests/boundary/pcap/pcapProvider.test.ts @@ -3,6 +3,8 @@ import { watch } from "node:fs"; import { readFile, symlink, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; import { expect, onTestFinished, test, vi } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { PcapProvider } from "../../../src/pcap/PcapProvider.js"; import { @@ -784,31 +786,34 @@ fs.writeFileSync(${JSON.stringify(marker)}, 'ready'); setInterval(() => {}, 1000 onTestFinished(() => provider.close()); const controller = new AbortController(); let watcher: ReturnType | undefined; - let timer: ReturnType | undefined; - const ready = new Promise((resolve, reject) => { + const ready = new Promise((resolve) => { watcher = watch(root, (_event, name) => { if (name === "ready") resolve(); }); - timer = setTimeout(() => { - controller.abort(); - reject(new Error("tool did not start")); - }, 5000); }); - try { - const pending = provider.execute( - { operation: "inspect_pcap_capture", input: { path } }, - { signal: controller.signal }, + const pending = provider.execute( + { operation: "inspect_pcap_capture", input: { path } }, + { signal: controller.signal }, + ); + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + ready, + pending, + `${phase} tool startup`, + 5_000, ); - await ready; controller.abort(new Error("caller cancellation")); const result = await pending; if (result.ok) throw new Error("expected cancellation"); expect(projectAnalysisError(result.error).code).toBe("cancelled"); await provider.close(); - } finally { - watcher?.close(); - clearTimeout(timer); - } + }, [ + () => controller.abort(), + async () => { + await pending; + }, + () => watcher?.close(), + ]); }, ); @@ -826,20 +831,24 @@ it.each(["change", "cancel"] as const)( { operation: "inspect_pcap_capture", input: { path } }, { signal: controller.signal }, ); - try { - await held.firstRead; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(held.firstRead, pending, "snapshot read"); if (action === "cancel") controller.abort(); else await writeFile(path, "changed source bytes"); - } finally { held.release(); - } - const result = await pending; - if (result.ok) throw new Error("expected interrupted acquisition"); - expect(result.error._tag).toBe( - action === "cancel" - ? "AnalysisCancelledError" - : "AnalysisArtifactChangedError", - ); + const result = await pending; + if (result.ok) throw new Error("expected interrupted acquisition"); + expect(result.error._tag).toBe( + action === "cancel" + ? "AnalysisCancelledError" + : "AnalysisArtifactChangedError", + ); + }, [ + held.release, + async () => { + await pending; + }, + ]); }, ); @@ -855,8 +864,8 @@ it("admits one capture and drains it before close releases resources", async () input: { path }, }); let closing: Promise | undefined; - try { - await held.firstRead; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(held.firstRead, pending, "snapshot read"); const concurrent = await provider.execute({ operation: "inspect_pcap_capture", input: { path }, @@ -873,16 +882,23 @@ it("admits one capture and drains it before close releases resources", async () }); await Promise.resolve(); expect(closed).toBe(false); - } finally { held.release(); - } - expect((await pending).ok).toBe(true); - await closing; - const afterClose = await provider.execute({ - operation: "inspect_pcap_capture", - input: { path }, - }); - expect(afterClose.ok).toBe(false); + expect((await pending).ok).toBe(true); + await closing; + const afterClose = await provider.execute({ + operation: "inspect_pcap_capture", + input: { path }, + }); + expect(afterClose.ok).toBe(false); + }, [ + held.release, + async () => { + await pending; + }, + async () => { + await closing; + }, + ]); }); it.each([false, true])( diff --git a/tests/boundary/pcap/pcapSnapshotCancellation.test.ts b/tests/boundary/pcap/pcapSnapshotCancellation.test.ts index e96f975a91..4dd4e710e4 100644 --- a/tests/boundary/pcap/pcapSnapshotCancellation.test.ts +++ b/tests/boundary/pcap/pcapSnapshotCancellation.test.ts @@ -124,6 +124,7 @@ test("verify maps cancellation after final descriptor close exactly once", async test("close failure during final cancellation retains owner and resolves its marker on retry", async () => { const { path, contents } = await makeCapture(); const snapshot = await PcapSnapshot.create(); + onTestFinished(() => snapshot.close()); const identity = { bytes: contents.length, sha256: createHash("sha256").update(contents).digest("hex"), @@ -140,6 +141,7 @@ test("close failure during final cancellation retains owner and resolves its mar controller.abort(reason); throw closeFailure; }); + onTestFinished(() => closeSpy.mockRestore()); const error = await snapshot .verify(identity, path, controller.signal) @@ -185,6 +187,7 @@ test("retains an open descriptor after failed close and releases it on retry", a retained = this.handle; throw closeFailure; }); + onTestFinished(() => closeSpy.mockRestore()); const error = await snapshot.verify(identity, path).catch((e) => e); expect(error).toBeInstanceOf(ProviderCleanupError); diff --git a/tests/boundary/process/ProcessEvidence.test.ts b/tests/boundary/process/ProcessEvidence.test.ts index 3528dbe871..914106e351 100644 --- a/tests/boundary/process/ProcessEvidence.test.ts +++ b/tests/boundary/process/ProcessEvidence.test.ts @@ -127,11 +127,9 @@ it("does not treat captures with different redacted inputs as equivalent", () => it("preserves selected local environment and input values", async () => { const left = await captureEvidenceForEnvironmentSecret("secret-left"); const right = await captureEvidenceForEnvironmentSecret("secret-right"); - const serialized = JSON.stringify(left); const leftCapture = parseProcessCapture(left.normalized_result); const rightCapture = parseProcessCapture(right.normalized_result); - expect(serialized).toContain("secret-left"); expect(leftCapture.manifest.scenario).toMatchObject({ environment: { API_TOKEN: "secret-left" }, }); diff --git a/tests/boundary/process/androidAnalysisMcp.test.ts b/tests/boundary/process/androidAnalysisMcp.test.ts index 5394176179..e3caf81183 100644 --- a/tests/boundary/process/androidAnalysisMcp.test.ts +++ b/tests/boundary/process/androidAnalysisMcp.test.ts @@ -1,8 +1,8 @@ import { AndroidAnalysisService } from "../../../src/application/android/AndroidAnalysisService.js"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import { Ajv2020 } from "ajv/dist/2020.js"; -import { expect, it as test, onTestFinished } from "vitest"; +import { expect, it as test } from "vitest"; +import { closeVerifierResources } from "../../../scripts/lib/verifier-cleanup.mjs"; import { z } from "zod"; import { access } from "node:fs/promises"; import { ANDROID_TOOL_CONTRACTS } from "../../../src/contracts/android/androidToolContracts.js"; @@ -18,6 +18,7 @@ import type { AndroidAnalysisPort } from "../../../src/application/android/Andro import { silentLogger } from "../../../src/logger.js"; import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { createJadxProtocolFixture, @@ -107,9 +108,14 @@ it("keeps a replacement server usable after the SDK discards its discovery probe error: { _tag: "AnalysisCapabilityUnavailableError" }, }); } finally { - await replacement.close(); - await transport.closeAndroid(); - await session.close(); + await closeVerifierResources([ + () => replacement.close(), + () => transport.closeAndroid(), + async () => { + const closed = await session.close(); + if (!closed.ok) throw closed.error; + }, + ]); } }); @@ -118,23 +124,11 @@ it("publishes and executes all APK contracts with inline Evidence and no active const session = createTestBinarySession(() => { throw new Error("APK analysis must not acquire a deep binary provider"); }); - const server = createServer( - { kind: "session", session }, - { androidAnalysis: provider }, - ); - const client = new Client({ - name: "android-contract-regression", - version: "1", + const { client, close } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "android-contract-regression", version: "1" }, + serverOptions: { androidAnalysis: provider }, }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); - }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const advertised = await client.listTools(); const wireTools = new Map(advertised.tools.map((tool) => [tool.name, tool])); for (const contract of ANDROID_TOOL_CONTRACTS) { @@ -182,8 +176,7 @@ it("publishes and executes all APK contracts with inline Evidence and no active expect(evidence.raw_result).not.toBeNull(); } expect(launches).toHaveLength(1); - await client.close(); - await server.close(); + await close(); await verifyJadxFixtureCleanup(launches); }); @@ -241,23 +234,14 @@ test.each([ const session = createTestBinarySession(() => { throw new Error("Unexpected native provider acquisition"); }); - const server = createServer( - { kind: "session", session }, - { androidAnalysis: provider }, - ); - const client = new Client({ - name: "android-partial-observation-regression", - version: "1", + const { client, close } = await createApplicationMcpHarness({ + session, + clientIdentity: { + name: "android-partial-observation-regression", + version: "1", + }, + serverOptions: { androidAnalysis: provider }, }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); - }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const response = await client.callTool({ name: "inspect_android_package", @@ -284,8 +268,7 @@ test.each([ }); if (!containsAmbientEnvironment) expect(JSON.stringify(partial)).not.toContain("JAVA_HOME"); - await client.close(); - await server.close(); + await close(); await verifyJadxFixtureCleanup(launches); }, ); @@ -323,23 +306,11 @@ it("cleans an active engine when the MCP client disconnects", async () => { const session = createTestBinarySession(() => { throw new Error("Unexpected native provider acquisition"); }); - const server = createServer( - { kind: "session", session }, - { androidAnalysis: provider }, - ); - const client = new Client({ - name: "android-disconnect-regression", - version: "1", + const { client, close } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "android-disconnect-regression", version: "1" }, + serverOptions: { androidAnalysis: provider }, }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); - }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const pending = client .callTool({ name: "inspect_android_package", arguments: { path: apk } }) .catch(() => undefined); @@ -360,5 +331,6 @@ it("cleans an active engine when the MCP client disconnects", async () => { } }) .toBe(true); + await close(); await verifyJadxFixtureCleanup(launches); }); diff --git a/tests/boundary/process/androidSearchEvidenceBudget.test.ts b/tests/boundary/process/androidSearchEvidenceBudget.test.ts index eac53ea19f..4011e02c8d 100644 --- a/tests/boundary/process/androidSearchEvidenceBudget.test.ts +++ b/tests/boundary/process/androidSearchEvidenceBudget.test.ts @@ -108,7 +108,9 @@ test("delivers bounded search_android_classes Evidence while retaining the compl // The inline raw record keeps the pagination audit, not the class names. const singleRaw = JSON.stringify(single.raw_result); expect(singleRaw).not.toContain("fixture.Class"); - expect(singleRaw).toContain("items_omitted"); + const singlePages = listClassCalls(single.raw_result); + expect(singlePages).toHaveLength(1); + expect(responsePage(singlePages[0]!)).toMatchObject({ items_omitted: 1 }); expect(Buffer.byteLength(JSON.stringify(single))).toBeLessThan(32 * 1024); // Every matching producer row remains in the retained source record. const completeId = single.evidence_links[0]; @@ -121,7 +123,11 @@ test("delivers bounded search_android_classes Evidence while retaining the compl expect(complete).toBeDefined(); expect(complete?.operation).toBe("search_android_classes"); expect(rawInventoryNames(complete?.raw_result)).toBe(1); - expect(JSON.stringify(complete?.raw_result)).toContain("fixture.Target"); + expect( + listClassCalls(complete?.raw_result).flatMap( + (call) => responsePage(call).items, + ), + ).toEqual(["fixture.Target"]); expect(JSON.stringify(complete?.raw_result)).not.toContain("fixture.Class"); // Selection precedes paging: a match beyond the first unfiltered page is diff --git a/tests/boundary/process/jadxIntegration.test.ts b/tests/boundary/process/jadxIntegration.test.ts index ad89c8def1..3c35768497 100644 --- a/tests/boundary/process/jadxIntegration.test.ts +++ b/tests/boundary/process/jadxIntegration.test.ts @@ -441,6 +441,8 @@ it("retire sessions when APK bytes, engine bytes or JVM options change", async ( Buffer.concat([await readFile(apk), Buffer.from("new artifact bytes")]), ); const changed = await inspect(); + expect(first.subject?.digest.sha256).toEqual(expect.any(String)); + expect(changed.subject?.digest.sha256).toEqual(expect.any(String)); expect(changed.subject?.digest.sha256).not.toBe(first.subject?.digest.sha256); await verifyCleanup(launches.slice(0, 1)); await writeJadxJarInventory(jar, ["fixture/NewEngineMarker.class"]); diff --git a/tests/boundary/process/packageVerifierCommandSemantics.test.ts b/tests/boundary/process/packageVerifierCommandSemantics.test.ts index 2b58f6a829..226ac2c56a 100644 --- a/tests/boundary/process/packageVerifierCommandSemantics.test.ts +++ b/tests/boundary/process/packageVerifierCommandSemantics.test.ts @@ -7,9 +7,11 @@ import { expect, it } from "vitest"; import { z } from "zod"; import { createPackageVerifierCommands } from "../../../scripts/lib/verify-package-commands.mjs"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import { systemProcessOwnershipHost } from "../../../src/process/ProcessOwnershipObservation.js"; import { liveProcesses } from "../../../src/process/ProcessOwnershipProcessTree.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; const observed = (promise: Promise) => promise.then( @@ -417,11 +419,11 @@ it("rejects a new cleanup owner once workspace release has started", async () => events.push("caller-owner-released"); }, }; - const closing = fixture.commands.close(); + const closing = observed(fixture.commands.close()); let rejection: unknown; let retained; - try { - await releaseStarted; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(releaseStarted, closing, "workspace release"); try { fixture.commands.adopt(capability, path); } catch (cause: unknown) { @@ -432,15 +434,15 @@ it("rejects a new cleanup owner once workspace release has started", async () => callerResource: existsSync(path), callerCloseCalls: releases.length, }; - } finally { - try { - await capability.close(); - } finally { - unblockWorkspaceClose(); - await closing; - await fixture.commands.close(); - } - } + }, [ + () => capability.close(), + () => unblockWorkspaceClose(), + async () => { + const result = await closing; + if (!result.ok) throw result.cause; + }, + () => fixture.commands.close(), + ]); expect(rejection).toBeInstanceOf(Error); expect(retained).toEqual({ diff --git a/tests/boundary/process/processCaptureProgress.test.ts b/tests/boundary/process/processCaptureProgress.test.ts index 5cf1b2aa7c..79a898e9b0 100644 --- a/tests/boundary/process/processCaptureProgress.test.ts +++ b/tests/boundary/process/processCaptureProgress.test.ts @@ -2,7 +2,7 @@ import { spawn } from "node:child_process"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { Client } from "@modelcontextprotocol/client"; import { expect, onTestFinished } from "vitest"; import { z } from "zod"; @@ -11,7 +11,7 @@ import { toolContract } from "../../../src/contracts/toolContracts.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { parseProcessCapture } from "../../../src/domain/process/processCaptureParsing.js"; import { silentLogger } from "../../../src/logger.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { itWithCaptureCapability } from "./processCaptureCapability.js"; @@ -205,39 +205,27 @@ const connectProgressClient = async ( recordNotifications = false, ): Promise<{ readonly client: Client; - readonly notifications: string[]; - readonly cleanupProgress: Promise; + readonly notifications: Array<{ + readonly params: { readonly message?: string | undefined }; + }>; }> => { const session = createTestBinarySession(() => { throw new Error("Process capture must not launch a binary provider"); }); - const server = createServer( - { kind: "session", session }, - { logger: silentLogger }, - ); - const client = new Client({ - name: "process-capture-progress", - version: "1", - }); - const notifications: string[] = []; - let sawCleanup: (() => void) | undefined; - const cleanupProgress = new Promise((resolve) => { - sawCleanup = resolve; + const { client } = await createApplicationMcpHarness({ + session, + serverOptions: { logger: silentLogger }, + clientIdentity: { name: "process-capture-progress", version: "1" }, }); + const notifications: Array<{ + readonly params: { readonly message?: string | undefined }; + }> = []; if (recordNotifications) { client.setNotificationHandler("notifications/progress", (notification) => { - notifications.push(JSON.stringify(notification)); - if (notification.params.message?.startsWith("cleanup:")) sawCleanup?.(); + notifications.push(notification); }); } - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); - onTestFinished(async () => { - await Promise.allSettled([client.close(), server.close(), session.close()]); - }); - return { client, notifications, cleanupProgress }; + return { client, notifications }; }; itWithCaptureCapability( @@ -530,11 +518,15 @@ itWithCaptureCapability( const tokenEvidence = toolContract( "capture_process_scenario", ).outputSchema.parse(captured.structuredContent); - expect( - tokenEvidence.normalized_result.frames - .map((frame) => frame.data) - .join(""), - ).toContain("TOKEN_CHILD_STARTED"); + const output = tokenEvidence.normalized_result.frames + .map((frame) => frame.data) + .join(""); + expect(output).toContain("TOKEN_CHILD_STARTED"); + expect(output).toContain("TOKEN_CHILD_DONE"); + expect(tokenEvidence.normalized_result.exit).toMatchObject({ + reason: "exited", + code: 0, + }); }, 30_000, ); @@ -606,7 +598,9 @@ itWithCaptureCapability( await expect .poll( () => - cancelling.notifications.some((note) => note.includes("running:")), + cancelling.notifications.some((note) => + note.params.message?.startsWith("running:"), + ), { timeout: 4_000 }, ) .toBe(true); @@ -622,17 +616,16 @@ itWithCaptureCapability( controller.abort(); await completion; } - // Client cancellation ends its wait before the server has released its - // process tree. Wait for the actual terminal notification under the test's - // existing deadline instead of giving cleanup a separate polling deadline. - await cancelling.cleanupProgress; - expect(cancelling.notifications.join("\n")).toContain( - "disposition=cancelled", - ); - expect(cancelling.notifications.join("\n")).toContain( - "owned_process_group=cleaned", - ); - expect(cancelling.notifications.join("\n")).not.toMatch( + // Cancellation ends the client wait before the server releases its tree. + // Bound the notification wait so a missing cleanup event reaches teardown. + const cleanupMessage = () => + cancelling.notifications + .map((note) => note.params.message) + .find((message) => message?.startsWith("cleanup:")); + await expect.poll(cleanupMessage, { timeout: 15_000 }).toBeDefined(); + expect(cleanupMessage()).toContain("disposition=cancelled"); + expect(cleanupMessage()).toContain("owned_process_group=cleaned"); + expect(JSON.stringify(cancelling.notifications)).not.toMatch( /setInterval|CHILD/u, ); }, diff --git a/tests/boundary/process/processCli.test.ts b/tests/boundary/process/processCli.test.ts index b7765effe6..618c009914 100644 --- a/tests/boundary/process/processCli.test.ts +++ b/tests/boundary/process/processCli.test.ts @@ -15,6 +15,7 @@ import { compareProcessEvidenceFiles, } from "../../../src/application/process/ProcessCli.js"; import { createEvidence } from "../../../src/domain/evidence.js"; +import { parseProcessCaptureEvidence } from "../../../src/application/process/ProcessEvidence.js"; import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js"; import { INVESTIGATION_EXAMPLES } from "../../../src/contracts/investigationExamples.js"; import { observeProcessStartIdentity } from "../../../src/process/ProcessOwnershipObservation.js"; @@ -370,7 +371,12 @@ describe("process CLI evidence validation", () => { predicate_type: "rea.process-capture", operation: "capture_process_scenario", }); - expect(JSON.stringify(evidence)).toContain("minimal-capture"); + const parsed = parseProcessCaptureEvidence(evidence); + if (!parsed.ok) + throw new Error(`Invalid capture evidence: ${parsed.error.kind}`); + expect( + parsed.value.capture.frames.map(({ data }) => data).join(""), + ).toContain("minimal-capture"); }); it("rejects unrelated capture evidence", async () => { diff --git a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts index d3166d4bc1..5c0b49c87a 100644 --- a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts +++ b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts @@ -10,15 +10,14 @@ import { } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; -import { expect, it, onTestFinished } from "vitest"; +import { expect, it } from "vitest"; import { analysisCliErrorEnvelopeSchema } from "../../../src/contracts/errorSchemas.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { compareProcessCaptures } from "../../../src/domain/process/processComparison.js"; import { parseProcessCapture } from "../../../src/domain/process/processCaptureParsing.js"; import { processScenarioSchema } from "../../../src/domain/process/processScenario.js"; import { FINALIZED_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCapture.fixture.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; @@ -49,17 +48,10 @@ const callCaptureViaMcp = async (scenario: Record) => { const session = createTestBinarySession(() => { throw new Error("Process capture must not launch a binary provider"); }); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "filesystem-coverage", version: "1" }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "filesystem-coverage", version: "1" }, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); const result = await client.callTool({ name: "capture_process_scenario", arguments: scenario, diff --git a/tests/boundary/process/processFixtureLifecycle.test.ts b/tests/boundary/process/processFixtureLifecycle.test.ts new file mode 100644 index 0000000000..6e07a4c0e4 --- /dev/null +++ b/tests/boundary/process/processFixtureLifecycle.test.ts @@ -0,0 +1,251 @@ +import { EventEmitter, once } from "node:events"; +import { createServer, type Socket } from "node:net"; +import { setTimeout as delay } from "node:timers/promises"; +import { PassThrough } from "node:stream"; +import type { ChildProcessWithoutNullStreams } from "node:child_process"; +import { expect, it, vi } from "vitest"; + +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; +import { createTestProcessRegistry } from "../../support/process/processFixture.js"; + +const within = async ( + operation: Promise, + timeoutMs: number, + timeoutValue: string, +): Promise => { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + operation, + new Promise((resolve) => { + timer = setTimeout(() => resolve(timeoutValue), timeoutMs); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +}; + +it("bounds a timed-out run when the leader exits but output remains open", async () => { + vi.useFakeTimers(); + const child = Object.assign(new EventEmitter(), { + stdin: new PassThrough(), + stdout: new PassThrough(), + stderr: new PassThrough(), + pid: 123, + exitCode: null as number | null, + signalCode: null as NodeJS.Signals | null, + kill: vi.fn(() => false), + }); + const processes = createTestProcessRegistry( + () => child as unknown as ChildProcessWithoutNullStreams, + ); + const pending = processes.run("fixture", [], { timeoutMs: 100 }); + const outcome = pending.then( + () => "resolved", + (error: unknown) => error, + ); + try { + child.stdout.write("partial output"); + child.exitCode = 0; + child.emit("exit", 0, null); + await vi.advanceTimersByTimeAsync(2_101); + const result = await Promise.race([ + outcome, + Promise.resolve("still pending"), + ]); + expect(result).toBeInstanceOf(Error); + expect(child.kill).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + message: "Test process exceeded its 100ms deadline", + cause: { exitCode: 0, signal: null, stdout: "partial output" }, + }); + } finally { + child.emit("close", 0, null); + await outcome; + await processes.shutdown(); + vi.useRealTimers(); + } +}); + +it("retains an unconfirmed child for shutdown retry after the run deadline", async () => { + vi.useFakeTimers(); + const child = Object.assign(new EventEmitter(), { + stdin: new PassThrough(), + stdout: new PassThrough(), + stderr: new PassThrough(), + pid: 456, + exitCode: null as number | null, + signalCode: null as NodeJS.Signals | null, + kill: vi.fn(() => true), + }); + const processes = createTestProcessRegistry( + () => child as unknown as ChildProcessWithoutNullStreams, + ); + const pending = processes.run("fixture", [], { timeoutMs: 100 }); + const outcome = pending.then( + () => "resolved", + (error: unknown) => error, + ); + try { + await vi.advanceTimersByTimeAsync(2_101); + expect(await outcome).toMatchObject({ + message: "Test process exceeded its 100ms deadline", + cause: { exitCode: null, signal: null }, + }); + const firstShutdown = processes.shutdown().then( + () => "resolved", + (error: unknown) => error, + ); + await vi.advanceTimersByTimeAsync(2_001); + expect(await firstShutdown).toMatchObject({ + message: "Test child process shutdown failed", + errors: [ + expect.objectContaining({ + message: "test child process 456 did not exit after SIGKILL", + }), + expect.any(Error), + ], + }); + child.kill.mockImplementation(() => { + child.exitCode = 0; + child.emit("exit", 0, null); + return true; + }); + await expect(processes.shutdown()).resolves.toBeUndefined(); + expect(child.kill).toHaveBeenCalledTimes(5); + } finally { + child.exitCode = 0; + child.emit("exit", 0, null); + child.emit("close", 0, null); + await outcome; + await processes.shutdown(); + vi.useRealTimers(); + } +}); + +it("returns normal completion without a timeout and preserves child errors", async () => { + const child = Object.assign(new EventEmitter(), { + stdin: new PassThrough(), + stdout: new PassThrough(), + stderr: new PassThrough(), + pid: 789, + exitCode: null as number | null, + signalCode: null as NodeJS.Signals | null, + kill: vi.fn(() => true), + }); + const processes = createTestProcessRegistry( + () => child as unknown as ChildProcessWithoutNullStreams, + ); + const run = processes.run("fixture", []); + child.stdout.write("completed"); + child.exitCode = 0; + child.emit("exit", 0, null); + child.emit("close", 0, null); + await expect(run).resolves.toEqual({ + exitCode: 0, + signal: null, + stdout: "completed", + stderr: "", + }); + await processes.shutdown(); + + const failure = new Error("spawn failed"); + child.exitCode = null; + const failedRun = processes.run("fixture", []); + child.emit("error", failure); + child.emit("close", null, null); + await expect(failedRun).rejects.toBe(failure); + await processes.shutdown(); +}); + +const releaseDescendant = async (peer: Socket | undefined): Promise => { + if (peer === undefined) { + // The child has a finite self-expiry even if it never connects. + await delay(9_500); + return; + } + if (peer.destroyed) return; + const acknowledgement = once(peer, "data"); + const closed = once(peer, "close"); + peer.write("stop"); + const result = await within( + Promise.all([acknowledgement, closed]), + 10_000, + "still open", + ); + if (result === "still open") { + peer.destroy(); + throw new Error("descendant connection did not close"); + } + expect(String(result[0][0])).toBe("stopping"); +}; + +it("times out when a real descendant holds the exited leader's output open", async () => { + const server = createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("Expected TCP listener"); + let peer: Socket | undefined; + let descendantPid: number | undefined; + const connected = new Promise((resolve) => { + server.once("connection", (socket) => { + peer = socket; + socket.once("data", (data: Buffer) => { + descendantPid = Number(data.toString()); + resolve(); + }); + }); + }); + const descendant = ` + const net = require("node:net"); + const socket = net.connect(Number(process.argv[1]), "127.0.0.1", () => { + process.stdout.write("descendant output\\n"); + socket.write(String(process.pid)); + }); + socket.on("data", () => socket.write("stopping", () => process.exit(0))); + setTimeout(() => process.exit(0), 9000); + `; + const leader = ` + const { spawn } = require("node:child_process"); + const child = spawn(process.execPath, ["-e", process.argv[1], process.argv[2]], { + detached: true, stdio: ["ignore", "inherit", "inherit"], + }); + child.unref(); + `; + const processes = createTestProcessRegistry(); + const pending = processes.run( + process.execPath, + ["-e", leader, descendant, String(address.port)], + { timeoutMs: 3000 }, + ); + const outcome = pending.then( + () => "resolved", + (error: unknown) => error, + ); + try { + await awaitAcknowledgement(connected, outcome, "descendant connection"); + const result = await within(outcome, 7_000, "still pending"); + expect(result).toMatchObject({ + message: "Test process exceeded its 3000ms deadline", + cause: { + exitCode: 0, + signal: null, + stdout: expect.stringContaining("descendant output"), + }, + }); + expect(descendantPid).toBeGreaterThan(0); + } finally { + try { + await releaseDescendant(peer); + } finally { + await outcome; + await processes.shutdown(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + } +}, 17_000); diff --git a/tests/boundary/process/selectedExecutableCleanup.test.ts b/tests/boundary/process/selectedExecutableCleanup.test.ts index 6bc6d173bc..5e2ab21ae0 100644 --- a/tests/boundary/process/selectedExecutableCleanup.test.ts +++ b/tests/boundary/process/selectedExecutableCleanup.test.ts @@ -4,6 +4,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; const injected = vi.hoisted(() => ({ path: "", @@ -47,9 +49,11 @@ vi.mock("node:fs/promises", async (importOriginal) => { }; }); -import { ProcessCaptureResourceScope } from "../../../src/process/capture/ProcessCaptureLifecycle.js"; +import { + ProcessCaptureResourceScope, + observeSelectedExecutable, +} from "../../../src/process/capture/ProcessCaptureLifecycle.js"; import { ProcessCaptureReadFailure } from "../../../src/process/capture/ProcessCaptureRead.js"; -import { observeSelectedExecutable } from "../../../src/process/capture/ProcessCaptureLifecycle.js"; const roots: string[] = []; const createExecutable = async (content: string): Promise => { @@ -282,33 +286,45 @@ it.each([ observeSelectedExecutable(path, controller.signal, fileSystem, scope), ), ); - await entered; - controller.abort(reason); - release(); - const failure = await reading; - expect(failure.cause).toBe(primary); - expect(failure.outcome).toMatchObject({ kind: "failed", cause: primary }); - expect(failure.cancellation?.reason).toBe(reason); - expect(failure.userCategory).toBeUndefined(); - expect(failure.cleanupIncomplete).toBe(closeDenied); - expect(failure.cleanupCause).toBe(injected.closeFailure); - expect(failure.cleanupLeases).toHaveLength(closeDenied ? 1 : 0); - expect(injected.closed).toEqual([path]); + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, reading, "selected executable read"); + controller.abort(reason); + release(); + const failure = await reading; + expect(failure.cause).toBe(primary); + expect(failure.outcome).toMatchObject({ kind: "failed", cause: primary }); + expect(failure.cancellation?.reason).toBe(reason); + expect(failure.userCategory).toBeUndefined(); + expect(failure.cleanupIncomplete).toBe(closeDenied); + expect(failure.cleanupCause).toBe(injected.closeFailure); + expect(failure.cleanupLeases).toHaveLength(closeDenied ? 1 : 0); + expect(injected.closed).toEqual([path]); - injected.closeFailure = undefined; - await scope.retryCleanup(); - expect(failure.cleanupIncomplete).toBe(false); - expect(injected.handles[0]?.isOpen()).toBe(false); - await expect( - scope.run(() => - observeSelectedExecutable(path, undefined, undefined, scope), - ), - ).resolves.toMatchObject({ - sha256: createHash("sha256") - .update("selected executable read failure") - .digest("hex"), - }); - await scope.close(); + injected.closeFailure = undefined; + await scope.retryCleanup(); + expect(failure.cleanupIncomplete).toBe(false); + expect(injected.handles[0]?.isOpen()).toBe(false); + await expect( + scope.run(() => + observeSelectedExecutable(path, undefined, undefined, scope), + ), + ).resolves.toMatchObject({ + sha256: createHash("sha256") + .update("selected executable read failure") + .digest("hex"), + }); + await scope.close(); + }, [ + () => controller.abort(), + release, + async () => { + await reading; + }, + () => { + injected.closeFailure = undefined; + }, + () => scope.close(), + ]); }, ); diff --git a/tests/boundary/process/verifierTerminalOwner.test.ts b/tests/boundary/process/verifierTerminalOwner.test.ts index e7cf557850..928e8f92b2 100644 --- a/tests/boundary/process/verifierTerminalOwner.test.ts +++ b/tests/boundary/process/verifierTerminalOwner.test.ts @@ -11,6 +11,7 @@ import { type VerifierTerminalResult, } from "../../../scripts/lib/owned-verifier-terminal.mjs"; import { createPackageVerifierCommands } from "../../../scripts/lib/verify-package-commands.mjs"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import type { VerifierCleanupOwner } from "../../../scripts/lib/verifier-command-owner.mjs"; import { cleanupOwnedProcessGroup, @@ -20,6 +21,7 @@ import { import { systemProcessOwnershipHost } from "../../../src/process/ProcessOwnershipObservation.js"; import { liveProcesses } from "../../../src/process/ProcessOwnershipProcessTree.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; type Outcome = | { readonly ok: true; readonly value: Value } @@ -42,16 +44,7 @@ const deferred = () => { const waitForBoundary = ( boundary: Promise, running: Promise>, -) => - Promise.race([ - boundary, - running.then((outcome) => { - throw new Error( - "Terminal settled before the authored lifecycle boundary", - { cause: outcome }, - ); - }), - ]); +) => awaitAcknowledgement(boundary, running, "terminal lifecycle", 5_000); const terminalFixture = async () => { const parent = await createTestTempDirectory("rea-verifier-terminal-"); @@ -362,7 +355,7 @@ posix( let closeObservation: ReturnType | undefined; let closing: Promise> | undefined; let outcome: Outcome | undefined; - try { + await withVerifierCleanup(async () => { await waitForBoundary(cleanupEntered.promise, running); const owner = fixture.adopted[0]; if (owner === undefined) throw new Error("Terminal was not adopted"); @@ -370,14 +363,16 @@ posix( closing = observed(fixture.commands.close()); await waitForBoundary(closeObservation.entered, closing); allowCleanup.resolve(); - await closing; + expect(await closing).toEqual({ ok: true, value: undefined }); outcome = await running; - } finally { - allowCleanup.resolve(); - await fixture.commands.close(); - await running; - closeObservation?.restore(); - } + }, [ + () => allowCleanup.resolve(), + () => fixture.commands.close(), + async () => { + await running; + }, + () => closeObservation?.restore(), + ]); expect(outcome).toEqual({ ok: false, @@ -426,7 +421,7 @@ posix( let closeObservation: ReturnType | undefined; let closing: Promise> | undefined; let outcome: Outcome | undefined; - try { + await withVerifierCleanup(async () => { await waitForBoundary(loadEntered.promise, running); const owner = fixture.adopted[0]; if (owner === undefined) throw new Error("Terminal was not adopted"); @@ -434,14 +429,16 @@ posix( closing = observed(fixture.commands.close()); await waitForBoundary(closeObservation.entered, closing); allowLoad.resolve(); - await closing; + expect(await closing).toEqual({ ok: true, value: undefined }); outcome = await running; - } finally { - allowLoad.resolve(); - await fixture.commands.close(); - await running; - closeObservation?.restore(); - } + }, [ + () => allowLoad.resolve(), + () => fixture.commands.close(), + async () => { + await running; + }, + () => closeObservation?.restore(), + ]); expect(outcome).toEqual({ ok: false, diff --git a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts index 1e48af6df0..5ddd2026ed 100644 --- a/tests/boundary/providers/GdbSessionManagerCoverage.test.ts +++ b/tests/boundary/providers/GdbSessionManagerCoverage.test.ts @@ -991,7 +991,8 @@ it.each([ const manager = managerWithFakeSpawner(fake.spawnProcess); const started = await expectGdbSessionStarted(manager); const stdin = fake.children[0]?.stdin; - if (stdin === null || stdin === undefined) return; + if (stdin === null || stdin === undefined) + throw new Error("Expected GDB stdin"); const commandSent = new Promise((resolve) => stdin.once("data", () => resolve()), ); @@ -1039,7 +1040,8 @@ it("reports process exit rather than protocol failure when stdin closes afterwar const started = await expectGdbSessionStarted(manager); const child = fake.children[0]; const stdin = child?.stdin; - if (child === undefined || stdin === null || stdin === undefined) return; + if (child === undefined || stdin === null || stdin === undefined) + throw new Error("Expected GDB child and stdin"); const commandSent = new Promise((resolve) => stdin.once("data", () => resolve()), ); @@ -1101,9 +1103,11 @@ it("retains partial GDB command Evidence while an inherited writer remains", asy }); const started = await expectGdbSessionStarted(manager); const child = fake.children[0]; - if (child === undefined || child.stdin === null) return; + if (child === undefined || child.stdin === null) + throw new Error("Expected GDB child and stdin"); + const stdin = child.stdin; const commandSent = new Promise((resolve) => - child.stdin!.once("data", () => resolve()), + stdin.once("data", () => resolve()), ); const pending = manager.execute( started.value.session_id, @@ -1299,7 +1303,7 @@ it("rejects a GDB leader exit without a verified sink and retains inherited outp const manager = managerWithFakeSpawner(fake.spawnProcess); const started = await expectGdbSessionStarted(manager); const child = fake.children[0]; - if (child === undefined) return; + if (child === undefined) throw new Error("Expected GDB child"); child.exitLeader(7); const closed = await manager.close(started.value.session_id); @@ -1369,7 +1373,7 @@ it("retains the GDB session while an unknown descendant holds output after leade inferiorTty: tty, }); const child = fake.children[0]; - if (child === undefined) return; + if (child === undefined) throw new Error("Expected GDB child"); expect(manager.status(started.value.session_id)?.inferior_tty_verified).toBe( true, ); @@ -1402,7 +1406,7 @@ it("retries close after an exited GDB's inherited output streams settle", async const manager = managerWithFakeSpawner(fake.spawnProcess); const started = await expectGdbSessionStarted(manager); const child = fake.children[0]; - if (child === undefined) return; + if (child === undefined) throw new Error("Expected GDB child"); child.exitLeader(3); const firstClose = await manager.close(started.value.session_id); @@ -2077,8 +2081,8 @@ it("preserves the protocol failure when owned-process stop is confirmed", async }); const started = await expectGdbSessionStarted(manager); const stdin = fake.children[0]?.stdin; - expect(stdin).not.toBeNull(); - if (stdin === null || stdin === undefined) return; + if (stdin === null || stdin === undefined) + throw new Error("Expected GDB stdin"); stdin.write = (() => { throw new Error("injected MI stdin write failure"); }) as typeof stdin.write; @@ -2120,8 +2124,8 @@ it("fails pending commands when the child closes stdin unexpectedly", async () = const started = await expectGdbSessionStarted(manager); const stdin = fake.children[0]?.stdin; - expect(stdin).toBeDefined(); - if (stdin === null || stdin === undefined) return; + if (stdin === null || stdin === undefined) + throw new Error("Expected GDB stdin"); const inputClosed = new Promise((resolve) => stdin.once("close", resolve), ); @@ -2145,8 +2149,8 @@ it.each([ const manager = managerWithFakeSpawner(fake.spawnProcess); const started = await expectGdbSessionStarted(manager); const stdin = fake.children[0]?.stdin; - expect(stdin).toBeDefined(); - if (stdin === null || stdin === undefined) return; + if (stdin === null || stdin === undefined) + throw new Error("Expected GDB stdin"); const originalWrite = stdin.write.bind(stdin); Object.defineProperty(stdin, "write", { value: (...arguments_: unknown[]) => { @@ -3017,7 +3021,8 @@ it("routes protocol failure cleanup through detach verification without direct s const manager = managerWithFakeSpawner(fake.spawnProcess); const started = await expectGdbSessionStarted(manager); const child = fake.children[0]; - if (child?.stdin === null || child?.stdin === undefined) return; + if (child?.stdin === null || child?.stdin === undefined) + throw new Error("Expected GDB child and stdin"); Object.defineProperty(child.stdin, "write", { value: ( _chunk: string, @@ -3549,7 +3554,7 @@ process.stdin.on('data', (chunk) => { } }); -it("stops the GDB session when a complete MI record exceeds the protocol limit", async () => { +it("closes the GDB session when a complete MI record exceeds the protocol limit", async () => { const manager = await managerFor(` process.stdout.write('(gdb)\\n'); let input = ''; @@ -3571,9 +3576,20 @@ process.stdin.on('data', (chunk) => { const failed = await manager.execute(started.value.session_id, "bad-record"); - expect(failed.ok).toBe(false); - if (!failed.ok) - expect(failed.error._tag).toBe("AnalysisCapabilityUnavailableError"); + expect(failed).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCapabilityUnavailableError", + reason: expect.stringContaining("MI line exceeding the 16777216 byte"), + cleanupIncomplete: true, + cleanup: { resources: [started.value.session_id] }, + partialObservation: { + predicate_type: "rea.debugger.console-observation", + raw_result: { command: "bad-record", completion_status: "unknown" }, + }, + }, + }); + expect(manager.status(started.value.session_id)?.state).toBe("closed"); }); it("rejects a partial MI line that exceeds the protocol limit", async () => { @@ -3601,9 +3617,20 @@ process.stdin.on('data', (chunk) => { "partial-line", ); - expect(failed.ok).toBe(false); - if (!failed.ok) - expect(failed.error._tag).toBe("AnalysisCapabilityUnavailableError"); + expect(failed).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisCapabilityUnavailableError", + reason: expect.stringContaining("MI line exceeding the 16777216 byte"), + cleanupIncomplete: true, + cleanup: { resources: [started.value.session_id] }, + partialObservation: { + predicate_type: "rea.debugger.console-observation", + raw_result: { command: "partial-line", completion_status: "unknown" }, + }, + }, + }); + expect(manager.status(started.value.session_id)?.state).toBe("closed"); }); it("caps the number of MI records retained for a single command", async () => { @@ -3684,7 +3711,8 @@ it.each(["incomplete", "rejected"] as const)( }); const started = await expectGdbSessionStarted(manager); const stdin = fake.children[0]?.stdin; - if (stdin === null || stdin === undefined) return; + if (stdin === null || stdin === undefined) + throw new Error("Expected GDB stdin"); Object.defineProperty(stdin, "write", { value: ( _chunk: string, diff --git a/tests/boundary/providers/GdbSessionReviewRegressions.test.ts b/tests/boundary/providers/GdbSessionReviewRegressions.test.ts index 534963916e..0992920881 100644 --- a/tests/boundary/providers/GdbSessionReviewRegressions.test.ts +++ b/tests/boundary/providers/GdbSessionReviewRegressions.test.ts @@ -822,13 +822,16 @@ process.stdin.on("data", (chunk) => { expect(started.ok).toBe(true); if (!started.ok) return; const controller = new AbortController(); - setTimeout(() => controller.abort(), 50); - - const result = await manager.execute( + const pending = manager.execute( started.value.session_id, "shell sleep 600", controller.signal, ); + await expect + .poll(() => manager.status(started.value.session_id)?.recent_mi_records) + .toContain("@target before cancellation"); + controller.abort(); + const result = await pending; expect(result.ok).toBe(false); if (!result.ok) { diff --git a/tests/boundary/providers/RizinDebugSessionCoverage.fixtures.ts b/tests/boundary/providers/RizinDebugSessionCoverage.fixtures.ts index 4d11c60503..f9fcd91f50 100644 --- a/tests/boundary/providers/RizinDebugSessionCoverage.fixtures.ts +++ b/tests/boundary/providers/RizinDebugSessionCoverage.fixtures.ts @@ -43,7 +43,10 @@ export const incompleteStop: typeof stopOwnedDebuggerProcess = async ( return { status: "incomplete", reason: "injected stop uncertainty" }; }; -export const abortOnCheck = (targetCheck: number): AbortSignal => { +export const abortOnCheck = ( + targetCheck: number, + reason?: unknown, +): AbortSignal => { let checks = 0; let aborted = false; type Listener = Parameters[1]; @@ -68,7 +71,7 @@ export const abortOnCheck = (targetCheck: number): AbortSignal => { }, dispatchEvent: () => true, onabort: null, - reason: undefined, + reason, throwIfAborted: () => undefined, } as AbortSignal; }; diff --git a/tests/boundary/providers/RizinDebugSessionCoverage.part1.test.ts b/tests/boundary/providers/RizinDebugSessionCoverage.part1.test.ts index dcdfd279e0..b167e484e5 100644 --- a/tests/boundary/providers/RizinDebugSessionCoverage.part1.test.ts +++ b/tests/boundary/providers/RizinDebugSessionCoverage.part1.test.ts @@ -72,9 +72,11 @@ it.skipIf(process.platform === "win32")( ); it.skipIf(process.platform === "win32")( - "returns a startup timeout when Rizin exits before its initial frame", + "reports Rizin exiting before its initial frame", async () => { - const provider = await makeProvider("process.exit(0);"); + const provider = await makeProvider( + 'process.stderr.write("early startup diagnostic\\n", () => process.exit(0));', + ); const manager = createManager({ REA_RIZIN_COMMAND: provider, }); @@ -84,6 +86,15 @@ it.skipIf(process.platform === "win32")( inferior_tty: await rizinTestInferiorTty(), }); expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCapabilityUnavailableError", + providerId: "rizin", + operation: "start_rizin_debug_session", + capturedOutput: { stderr: "early startup diagnostic\n" }, + }); + expect(result.error.cause).toBeInstanceOf(Error); + expect((result.error.cause as Error).message).toContain("Rizin exited"); }, ); diff --git a/tests/boundary/providers/RizinDebugSessionCoverage.part2.test.ts b/tests/boundary/providers/RizinDebugSessionCoverage.part2.test.ts index 92bf472dcc..c38041e327 100644 --- a/tests/boundary/providers/RizinDebugSessionCoverage.part2.test.ts +++ b/tests/boundary/providers/RizinDebugSessionCoverage.part2.test.ts @@ -191,19 +191,3 @@ it.skipIf(process.platform === "win32")( expect(result.ok).toBe(false); }, ); - -it.skipIf(process.platform === "win32")( - "cancels while waiting for the previous command turn", - async () => { - const injected = createInjectedProcess(); - const { manager, started } = await startInjected(injected.spawnProcess); - expect(started.ok).toBe(true); - if (!started.ok) return; - const result = await manager.execute( - started.value.session_id, - "dr", - abortOnCheck(2), - ); - expect(result.ok).toBe(false); - }, -); diff --git a/tests/boundary/providers/RizinDebugSessionCoverage.part3.test.ts b/tests/boundary/providers/RizinDebugSessionCoverage.part3.test.ts index c65ffbfdf2..ad6c551def 100644 --- a/tests/boundary/providers/RizinDebugSessionCoverage.part3.test.ts +++ b/tests/boundary/providers/RizinDebugSessionCoverage.part3.test.ts @@ -1,4 +1,6 @@ import { afterEach, expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { AnalysisCancelledError, AnalysisInputError, @@ -147,23 +149,36 @@ it.skipIf(process.platform === "win32")( it.skipIf(process.platform === "win32")( "cancels a command after capturing its completion frame", async () => { - let signal: AbortSignal; + const reason = { stage: "completion frame captured" }; const injected = createInjectedProcess({ onWrite: (text, control) => { const marker = /!echo (.+)\n/u.exec(text)?.[1]; - if (marker !== undefined) control.output(Buffer.from(`${marker}\0`)); + if (marker !== undefined) + control.output(Buffer.from(`registers${marker}\0`)); }, }); - const { manager, started } = await startInjected(injected.spawnProcess); - expect(started.ok).toBe(true); - if (!started.ok) return; - signal = abortOnCheck(5); + const { manager, started } = await startInjected(injected.spawnProcess, { + stopProcess: async () => ({ status: "stopped" }), + }); + if (!started.ok) throw started.error; + const signal = abortOnCheck(5, reason); + const registration = vi.spyOn(signal, "addEventListener"); const result = await manager.execute( started.value.session_id, "dr", signal, ); expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(registration).toHaveBeenCalledTimes(1); + expect(result.error.capturedOutput?.stdout).toBe("registers"); + expect(result.error.partialObservation).toMatchObject({ + parameters: { session_id: started.value.session_id, command: "dr" }, + raw_result: { output: "registers", completion_status: "unknown" }, + }); + expect(manager.status(started.value.session_id)?.state).toBe("closed"); }, ); @@ -174,14 +189,29 @@ it.skipIf(process.platform === "win32")( const { manager, started } = await startInjected(injected.spawnProcess, { stopProcess: async () => ({ status: "stopped" }), }); - expect(started.ok).toBe(true); - if (!started.ok) return; + if (!started.ok) throw started.error; + const reason = { stage: "frame waiter installation" }; + const signal = abortOnCheck(5, reason); + const registration = vi.spyOn(signal, "addEventListener"); const result = await manager.execute( started.value.session_id, "wait", - abortOnCheck(5), + signal, ); expect(result.ok).toBe(false); + if (result.ok) return; + expect(registration).toHaveBeenCalledTimes(2); + expect(registration).toHaveBeenCalledWith("abort", expect.any(Function), { + once: true, + }); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(result.error.capturedOutput?.stdout).toBe(""); + expect(result.error.partialObservation).toMatchObject({ + parameters: { session_id: started.value.session_id, command: "wait" }, + raw_result: { output: "", completion_status: "unknown" }, + }); + expect(manager.status(started.value.session_id)?.state).toBe("closed"); }, ); @@ -489,26 +519,36 @@ it.skipIf(process.platform === "win32").each([ "noisy", controller.signal, ); - await entered; - controller.abort(reason); - releaseStop(); - const result = await pending; - expect(result.ok).toBe(false); - if (result.ok) return; - expect(result.error).toMatchObject({ - _tag: "AnalysisCancelledError", - cleanupIncomplete: incomplete, - }); - expect(result.error.cause).toBe(reason); - expect(result.error.capturedOutput?.stdout).toBe( - stage === "output" ? "beyond s" : stage === "exited" ? "partial" : "", - ); - expect(result.error.partialObservation).toBeDefined(); - if (incomplete) { - stopping = false; - expect((await manager.close(started.value.session_id)).ok).toBe(true); - expect(result.error.cleanupIncomplete).toBe(false); - } + await withVerifierCleanup(async () => { + await awaitAcknowledgement(entered, pending, "debugger final cleanup"); + controller.abort(reason); + releaseStop(); + const result = await pending; + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "AnalysisCancelledError", + cleanupIncomplete: incomplete, + }); + expect(result.error.cause).toBe(reason); + expect(result.error.capturedOutput?.stdout).toBe( + stage === "output" ? "beyond s" : stage === "exited" ? "partial" : "", + ); + expect(result.error.partialObservation).toBeDefined(); + if (incomplete) { + stopping = false; + expect((await manager.close(started.value.session_id)).ok).toBe(true); + expect(result.error.cleanupIncomplete).toBe(false); + } + }, [ + releaseStop, + async () => { + await pending; + }, + () => { + stopping = false; + }, + ]); }, ); diff --git a/tests/boundary/providers/RizinDebugSessionCoverage.part4.test.ts b/tests/boundary/providers/RizinDebugSessionCoverage.part4.test.ts index 0314129df8..7d4301d9b8 100644 --- a/tests/boundary/providers/RizinDebugSessionCoverage.part4.test.ts +++ b/tests/boundary/providers/RizinDebugSessionCoverage.part4.test.ts @@ -1,5 +1,8 @@ import { afterEach, expect, it } from "vitest"; +import { access, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { evidenceSchema } from "../../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { rizinTestInferiorTty } from "./RizinTestInferiorTty.js"; import { createRizinCoverageHarness, @@ -233,12 +236,15 @@ it.skipIf(process.platform === "win32")( "cancels a command after sending it and stops its owned session", async () => { const provider = await makeProvider( - 'process.stdout.write("\\0"); process.stdin.resume();', + 'process.stdout.write("\\0"); process.stdin.on("data", () => process.stdout.write("slow-started\\0"));', ); const { manager, sessionId } = await start(provider); const abort = new AbortController(); const execution = manager.execute(sessionId, "slow", abort.signal); - setTimeout(() => abort.abort(), 30); + await expect + .poll(() => manager.status(sessionId)?.recent_output) + .toContain("slow-started"); + abort.abort(); expect((await execution).ok).toBe(false); expect(manager.status(sessionId)?.state).toBe("closed"); }, @@ -312,14 +318,17 @@ it.skipIf(process.platform === "win32")( "preserves the cleanup uncertainty when cancellation cannot confirm process stop", async () => { const provider = await makeProvider( - 'process.stdout.write("\\0"); process.stdin.resume();', + 'process.stdout.write("\\0"); process.stdin.on("data", () => process.stdout.write("slow-started\\0"));', ); const { manager, sessionId } = await start(provider, { stopProcess: incompleteStop, }); const abort = new AbortController(); const execution = manager.execute(sessionId, "slow", abort.signal); - setTimeout(() => abort.abort(), 30); + await expect + .poll(() => manager.status(sessionId)?.recent_output) + .toContain("slow-started"); + abort.abort(); const result = await execution; expect(result.ok).toBe(false); if (!result.ok) { @@ -335,8 +344,14 @@ it.skipIf(process.platform === "win32")( it.skipIf(process.platform === "win32")( "cancels a queued command without sending it after the prior command completes", async () => { + const release = join( + await createTestTempDirectory("rea-rizin-queue-"), + "release", + ); + const unexpected = `${release}-second`; const provider = await makeProvider( `process.stdout.write("\\0"); +import * as fs from "node:fs"; let input = ""; process.stdin.setEncoding("utf8"); process.stdin.on("data", (chunk) => { @@ -345,19 +360,36 @@ process.stdin.on("data", (chunk) => { const end = input.indexOf("\\n"); if (end < 0) break; const line = input.slice(0, end); input = input.slice(end + 1); - if (line === "first") setTimeout(() => {}, 100); - else if (line.startsWith("!echo ")) setTimeout(() => process.stdout.write(line.slice(6) + "\\0"), 100); + if (line === "first") process.stdout.write("first-started\\0"); + else if (line === "second") fs.writeFileSync(${JSON.stringify(unexpected)}, "sent"); + else if (line.startsWith("!echo ")) { + const timer = setInterval(() => { + if (!fs.existsSync(${JSON.stringify(release)})) return; + clearInterval(timer); + process.stdout.write(line.slice(6) + "\\0"); + }, 10); + } } });`, ); const { manager, sessionId } = await start(provider, { - frameTimeoutMs: 300, + frameTimeoutMs: 5_000, }); const first = manager.execute(sessionId, "first"); + await expect + .poll(() => manager.status(sessionId)?.recent_output) + .toContain("first-started"); const abort = new AbortController(); const queued = manager.execute(sessionId, "second", abort.signal); - setTimeout(() => abort.abort(), 20); - expect((await queued).ok).toBe(false); + const reason = { stage: "queued command" }; + abort.abort(reason); + expect(await queued).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError", cause: reason }, + }); + await writeFile(release, "release"); expect((await first).ok).toBe(true); + await expect(access(unexpected)).rejects.toMatchObject({ code: "ENOENT" }); + expect(manager.status(sessionId)?.state).toBe("ready"); }, ); diff --git a/tests/boundary/providers/RizinDebugSessionReviewRegressions.test.ts b/tests/boundary/providers/RizinDebugSessionReviewRegressions.test.ts index 49f214fb89..92c07fcf84 100644 --- a/tests/boundary/providers/RizinDebugSessionReviewRegressions.test.ts +++ b/tests/boundary/providers/RizinDebugSessionReviewRegressions.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import { RizinDebugSessionManager } from "../../../src/rizin/RizinDebugSessionManager.js"; import { @@ -12,6 +13,8 @@ import { } from "../../../src/process/ProviderProcess.js"; import { evidenceSchema } from "../../../src/domain/evidence.js"; import { absoluteCallerPath } from "../../../src/filesystem/callerPath.js"; +import { createDeferred } from "../../fixtures/binarySession.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { createRizinTestManager, rizinTestInferiorTty, @@ -678,13 +681,16 @@ process.stdin.on("data", (chunk) => { expect(started.ok).toBe(true); if (!started.ok) return; const controller = new AbortController(); - setTimeout(() => controller.abort(), 50); - - const result = await manager.execute( + const pending = manager.execute( started.value.session_id, "slow", controller.signal, ); + await expect + .poll(() => manager.status(started.value.session_id)?.output_invalid_utf8) + .toBe(true); + controller.abort(); + const result = await pending; expect(result.ok).toBe(false); if (!result.ok) { @@ -748,17 +754,34 @@ it.skipIf(process.platform === "win32")( `#!/usr/bin/env node\nprocess.stdin.resume();\nsetInterval(() => {}, 1000);\n`, ); await chmod(provider, 0o700); - const manager = createRizinTestManager({ - REA_RIZIN_COMMAND: provider, - }); + const spawned = createDeferred(); + const manager = createRizinTestManager( + { REA_RIZIN_COMMAND: provider }, + { + spawnProcess: async (options) => { + const launched = await spawnOwnedProviderProcess(options); + spawned.resolve(); + return launched; + }, + }, + ); managers.push(manager); const controller = new AbortController(); - setTimeout(() => controller.abort(), 50); - - const started = await manager.start( + const starting = manager.start( { path: "/tmp/fixture.bin", inferior_tty: await rizinTestInferiorTty() }, controller.signal, ); + const started = await withVerifierCleanup(async () => { + await awaitAcknowledgement(spawned.promise, starting, "Rizin spawn"); + controller.abort(); + return await starting; + }, [ + () => controller.abort(), + async () => { + await starting; + }, + () => manager.closeAll(), + ]); expect(started.ok).toBe(false); if (!started.ok) expect(started.error._tag).toBe("AnalysisCancelledError"); diff --git a/tests/boundary/providers/ghidra/ghidraClient.test.ts b/tests/boundary/providers/ghidra/ghidraClient.test.ts index 104fbcccac..a07c00c5c4 100644 --- a/tests/boundary/providers/ghidra/ghidraClient.test.ts +++ b/tests/boundary/providers/ghidra/ghidraClient.test.ts @@ -21,6 +21,8 @@ import { PassThrough } from "node:stream"; import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; import pino from "pino"; +import { withVerifierCleanup } from "../../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../../fixtures/lifecycleAcknowledgement.js"; import type { Logger } from "pino"; import type { ProgressUpdate } from "../../../../src/application/ProgressReporter.js"; @@ -1153,8 +1155,12 @@ it("reports owner closure to live startup callers without cancelling their signa }); const controllers = [new AbortController(), new AbortController()]; const waits = controllers.map(({ signal }) => client.start(signal)); - try { - await pending; + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + pending, + Promise.race(waits), + "startup observation", + ); const closing = client.close(); release?.(); for (const wait of waits) @@ -1168,11 +1174,17 @@ it("reports owner closure to live startup callers without cancelling their signa await expect(closing).resolves.toEqual({ ok: true, value: null }); expect(controllers.every(({ signal }) => !signal.aborted)).toBe(true); expect(client.isReady()).toBe(false); - } finally { - release?.(); - await client.close(); - observe.mockRestore(); - } + }, [ + () => release?.(), + async () => { + const result = await client.close(); + if (!result.ok) throw result.error; + }, + async () => { + await Promise.allSettled(waits); + }, + () => observe.mockRestore(), + ]); }); it.skipIf(process.platform === "win32")( diff --git a/tests/boundary/providers/hopper/hopperInstructions.test.ts b/tests/boundary/providers/hopper/hopperInstructions.test.ts index db22c3b9f0..613039859f 100644 --- a/tests/boundary/providers/hopper/hopperInstructions.test.ts +++ b/tests/boundary/providers/hopper/hopperInstructions.test.ts @@ -98,8 +98,9 @@ describe("Hopper instruction block boundaries", () => { maxBuffer: 1_024 * 1_024, }, ); - expect( - z.object({ error: z.string() }).parse(JSON.parse(stdout)).error, - ).toBe("Instruction enumeration requires Hopper basic-block ownership"); + const { error } = z.object({ error: z.string() }).parse(JSON.parse(stdout)); + expect(error).toContain("Hopper"); + expect(error).toContain("basic-block"); + expect(error).toContain("ownership"); }); }); diff --git a/tests/boundary/providers/hopper/linuxPrivateDisplay.test.ts b/tests/boundary/providers/hopper/linuxPrivateDisplay.test.ts index 608c90d5f1..2722ed611c 100644 --- a/tests/boundary/providers/hopper/linuxPrivateDisplay.test.ts +++ b/tests/boundary/providers/hopper/linuxPrivateDisplay.test.ts @@ -196,7 +196,8 @@ describe("Hopper demo dialog timeout outcomes", () => { })}\n`, ); expect(parsed.ok).toBe(true); - if (!parsed.ok || parsed.value.status !== "error") return; + if (!parsed.ok || parsed.value.status !== "error") + throw new Error("Expected an error diagnostic for the invalid timeout"); const projected = projectAnalysisError( new HopperProcessError(82, { diagnostic: parsed.value }), ); diff --git a/tests/boundary/providers/kuna/kunaProjectCache.test.ts b/tests/boundary/providers/kuna/kunaProjectCache.test.ts index 0d1b9c92f3..3b949c8024 100644 --- a/tests/boundary/providers/kuna/kunaProjectCache.test.ts +++ b/tests/boundary/providers/kuna/kunaProjectCache.test.ts @@ -28,6 +28,7 @@ import { } from "../../../../src/kuna/KunaProjectCache.js"; import type { KunaExportStatus } from "../../../../src/kuna/KunaProjectExport.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../../fixtures/lifecycleAcknowledgement.js"; import { OwnedFileHandle } from "../../../../src/filesystem/OwnedFileHandle.js"; import { RegularFileCleanupFailure } from "../../../../src/filesystem/OwnedRegularFile.js"; import * as stableFileHash from "../../../../src/filesystem/StableFileHash.js"; @@ -300,6 +301,7 @@ it("publishes a verified export and serves later requests from the cache", async // Kuna listed and decompiled a staged copy from REA's own HOME, in reference mode. const [listing] = await callsOf(kuna, "functions"); const [exported] = await callsOf(kuna, "decompile-project"); + expect(exported?.argv[1]).toEqual(expect.any(String)); expect(listing?.argv).toEqual(["functions", exported?.argv[1], "--json"]); expect(exported?.argv.slice(2)).toEqual([ "-o", @@ -393,20 +395,26 @@ it.each(["success", "failure"] as const)( return closeOwner.call(this); }); + const firstLookup = warmCache + .lookup(built.value.key) + .catch((cause: unknown) => cause); try { - const firstLookup = warmCache - .lookup(built.value.key) - .catch((cause) => cause); - await initialCloseEntered.promise; + await awaitAcknowledgement( + initialCloseEntered.promise, + firstLookup, + "initial descriptor close", + ); initialClose.reject(initialCloseFailure); // The failed descriptor is retained before the first lookup binds its // originating error, so this lookup begins the retry in that window. - queueMicrotask(() => { - concurrentLookup = warmCache - .lookup(built.value.key) - .catch((cause: unknown) => cause); - }); - await retryEntered.promise; + concurrentLookup = Promise.resolve().then(() => + warmCache.lookup(built.value.key).catch((cause: unknown) => cause), + ); + await awaitAcknowledgement( + retryEntered.promise, + concurrentLookup, + "descriptor close retry", + ); const firstFailure = await firstLookup; expect(firstFailure).toBeInstanceOf(ProviderCleanupError); @@ -441,6 +449,12 @@ it.each(["success", "failure"] as const)( expect(retryResult.cleanupIncomplete).toBe(false); } } finally { + initialClose.resolve(); + retryGate.resolve(); + await Promise.allSettled([ + firstLookup, + ...(concurrentLookup === undefined ? [] : [concurrentLookup]), + ]); close.mockRestore(); expect(await warmCache.close()).toEqual({ ok: true, value: null }); await cache.close(); @@ -481,7 +495,11 @@ it("resolves a hash failure delivered after its retained owner was released", as .lookup(built.value.key) .catch((cause: unknown) => cause); try { - await failureRetained.promise; + await awaitAcknowledgement( + failureRetained.promise, + firstLookup, + "retained hash failure", + ); expect(warmCache.hasPendingCleanup).toBe(true); expect(await warmCache.lookup(built.value.key)).toMatchObject({ key: built.value.key, @@ -1127,6 +1145,8 @@ it("reports an abandoned build's unproven cleanup even after a retry replaced it { project: "hang" }, ); let exports = 0; + const cleanupStarted = deferred(); + const releaseCleanup = deferred(); const cache = new KunaProjectCache(cacheRoot, { maxBytes: 64 * 1024 ** 2, launcher: async (options) => { @@ -1137,7 +1157,8 @@ it("reports an abandoned build's unproven cleanup even after a retry replaced it return { ...spawned, cleanup: async () => { - await new Promise((resolve) => setTimeout(resolve, 500)); + cleanupStarted.resolve(); + await releaseCleanup.promise; await cleanupOwnedProcessGroup(spawned.ownership); return { cleaned: false, reason: "first export unverified" }; }, @@ -1151,25 +1172,46 @@ it("reports an abandoned build's unproven cleanup even after a retry replaced it ...input, signal: first.signal, }); - await expect.poll(() => callsOf(kuna, "decompile-project")).toHaveLength(1); - first.abort(); - await firstBuild; - - // A retry for the same key starts while the first is still stopping. const second = new AbortController(); - const secondBuild = cache.build({ - installation, - ...input, - signal: second.signal, - }); - await expect.poll(() => callsOf(kuna, "decompile-project")).toHaveLength(2); - second.abort(); - await secondBuild; + let secondBuild: ReturnType | undefined; + let stopping: ReturnType | undefined; + try { + await expect.poll(() => callsOf(kuna, "decompile-project")).toHaveLength(1); + first.abort(); + await firstBuild; + stopping = cache.settled(keyOf(installation, input)); + await awaitAcknowledgement( + cleanupStarted.promise, + stopping, + "abandoned export cleanup", + ); - expect(await cache.close()).toMatchObject({ - ok: false, - error: { diagnostics: { reason: "first export unverified" } }, - }); + // A retry for the same key starts while the first is still stopping. + secondBuild = cache.build({ + installation, + ...input, + signal: second.signal, + }); + await expect.poll(() => callsOf(kuna, "decompile-project")).toHaveLength(2); + second.abort(); + await secondBuild; + releaseCleanup.resolve(); + + expect(await cache.close()).toMatchObject({ + ok: false, + error: { diagnostics: { reason: "first export unverified" } }, + }); + } finally { + first.abort(); + second.abort(); + releaseCleanup.resolve(); + await Promise.allSettled([ + firstBuild, + ...(secondBuild === undefined ? [] : [secondBuild]), + ...(stopping === undefined ? [] : [stopping]), + ]); + await cache.close(); + } }); it("removes its staging when publication itself fails", async () => { diff --git a/tests/boundary/providers/kuna/kunaQueryWork.test.ts b/tests/boundary/providers/kuna/kunaQueryWork.test.ts index 8b3ee62957..218490d081 100644 --- a/tests/boundary/providers/kuna/kunaQueryWork.test.ts +++ b/tests/boundary/providers/kuna/kunaQueryWork.test.ts @@ -1,4 +1,6 @@ import { expect, it } from "vitest"; +import { withVerifierCleanup } from "../../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../../fixtures/lifecycleAcknowledgement.js"; import { KunaQueryWork } from "../../../../src/kuna/KunaQueryWork.js"; import { KunaRunSlots } from "../../../../src/kuna/KunaRunSlots.js"; @@ -36,11 +38,12 @@ it("does not materialize values for a cancelled queued regex query", async () => return [`${"a".repeat(40)}!`]; }, }); - try { - await ready; - const cancelled = new AbortController(); + const cancelled = new AbortController(); + let queued: ReturnType | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(ready, slow, "regex worker admission"); let materialized = false; - const queued = work.match({ + queued = work.match({ operation: "search_procedures", pattern: "value", caseSensitive: true, @@ -61,9 +64,17 @@ it("does not materialize values for a cancelled queued regex query", async () => ok: false, error: { _tag: "AnalysisCancelledError" }, }); - } finally { - lifetime.abort(); - await slow; - expect(await commands.close()).toEqual({ ok: true, value: null }); - } + }, [ + () => cancelled.abort(), + () => lifetime.abort(), + async () => { + await queued; + }, + async () => { + await slow; + }, + async () => { + expect(await commands.close()).toEqual({ ok: true, value: null }); + }, + ]); }); diff --git a/tests/boundary/providers/kuna/kunaSessionFixture.ts b/tests/boundary/providers/kuna/kunaSessionFixture.ts index 9cd64ef4ed..1264da3fe0 100644 --- a/tests/boundary/providers/kuna/kunaSessionFixture.ts +++ b/tests/boundary/providers/kuna/kunaSessionFixture.ts @@ -1,6 +1,9 @@ import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; + +import { onTestFinished } from "vitest"; + import type { AnalysisClient } from "../../../../src/application/AnalysisProvider.js"; import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; import type { JsonValue } from "../../../../src/domain/jsonValue.js"; @@ -54,13 +57,25 @@ export const session = async ( ...(singleFunctionRuns === undefined ? {} : { singleFunctionRuns }), }, ); + // Finished hooks run in reverse order: clients, shared cache, then root. + onTestFinished(async () => { + const closed = await provider.closeOwnedResources(); + if (!closed.ok) throw closed.error; + }); const profile = await withKunaResourceScope((resourceScope) => provider.resolveAnalysisProfile(target, { resourceScope }), ); if (!profile.ok || profile.value.profile === null) throw new Error("profile did not resolve"); const resolved = profile.value.profile; - const open = () => provider.createClient(target, resolved); + const open = () => { + const client = provider.createClient(target, resolved); + onTestFinished(async () => { + const closed = await client.close(); + if (!closed.ok) throw closed.error; + }); + return client; + }; return { root, kuna, target, cacheRoot, provider, profile: resolved, open }; }; diff --git a/tests/boundary/providers/kuna/kunaWorkspaceRollback.test.ts b/tests/boundary/providers/kuna/kunaWorkspaceRollback.test.ts index 038b652f80..c34f28e031 100644 --- a/tests/boundary/providers/kuna/kunaWorkspaceRollback.test.ts +++ b/tests/boundary/providers/kuna/kunaWorkspaceRollback.test.ts @@ -1,9 +1,11 @@ import { unlink, writeFile } from "node:fs/promises"; import { expect, it as test, vi } from "vitest"; +import { withVerifierCleanup } from "../../../../scripts/lib/verifier-cleanup.mjs"; import { KunaTargetCommands } from "../../../../src/kuna/KunaTargetCommands.js"; import { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeRoot.js"; import { ProviderCleanupError } from "../../../../src/domain/providerCleanupError.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; +import { awaitAcknowledgement } from "../../../fixtures/lifecycleAcknowledgement.js"; import { session } from "./kunaSessionFixture.js"; const it = test.skipIf(process.platform === "win32"); @@ -103,18 +105,28 @@ it.each(["clean", "refuse"] as const)( }); const pending = commands.prepare(); let settled = false; - void pending.then(() => { - settled = true; - }); + void pending.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); let closing: ReturnType | undefined; - try { - await started; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(started, pending, "workspace rollback"); lifetime.abort({ caller: "shutdown during rollback" }); closing = commands.close(); let closed = false; - void closing.then(() => { - closed = true; - }); + void closing.then( + () => { + closed = true; + }, + () => { + closed = true; + }, + ); await new Promise((resolve) => setImmediate(resolve)); expect(settled).toBe(false); expect(closed).toBe(false); @@ -131,14 +143,23 @@ it.each(["clean", "refuse"] as const)( } expect(result.error.cleanupIncomplete).toBe(false); expect(projectAnalysisError(result.error).code).toBe("artifact_changed"); - } finally { - holdFailures = false; - release(); - await Promise.all([pending, closing]); - rootClose?.mockRestore(); - roots.mockRestore(); - expect((await commands.close()).ok).toBe(true); - } + }, [ + () => { + holdFailures = false; + release(); + }, + async () => { + await pending; + }, + async () => { + await closing; + }, + () => rootClose?.mockRestore(), + () => roots.mockRestore(), + async () => { + expect((await commands.close()).ok).toBe(true); + }, + ]); }, ); diff --git a/tests/boundary/providers/native/lldbReadbackFiles.test.ts b/tests/boundary/providers/native/lldbReadbackFiles.test.ts index 2814a68b00..f37dbfc390 100644 --- a/tests/boundary/providers/native/lldbReadbackFiles.test.ts +++ b/tests/boundary/providers/native/lldbReadbackFiles.test.ts @@ -77,6 +77,7 @@ it("bounds PID readback while retaining newline-terminated producer fixtures", a expect( await files.readTargetPid(join(directory, "missing.pid")), ).toBeUndefined(); + expect(requestedLengths.length).toBeGreaterThan(0); expect(requestedLengths.every((length) => length <= 19)).toBe(true); expect(bytesRead).toBeLessThanOrEqual(19 * values.length); } finally { diff --git a/tests/boundary/providers/native/nativeMachoUuidlessInspection.test.ts b/tests/boundary/providers/native/nativeMachoUuidlessInspection.test.ts index 56aad4677a..87d2c21443 100644 --- a/tests/boundary/providers/native/nativeMachoUuidlessInspection.test.ts +++ b/tests/boundary/providers/native/nativeMachoUuidlessInspection.test.ts @@ -1,11 +1,9 @@ import { expect, it } from "vitest"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { machoImage } from "../../../../src/artifacts/apple/MachoImage.fixture.js"; import { inspectionImage } from "../../../../src/native/NativeMachoInspection.fixture.js"; -import { createServer } from "../../../../src/server/createServer.js"; import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; import { NativeCommandFailure, @@ -15,6 +13,7 @@ import { } from "../../../../src/native/CommandRunner.js"; import { err, ok } from "../../../../src/domain/result.js"; import { createTestBinarySession } from "../../../fixtures/binarySession.js"; +import { createApplicationMcpHarness } from "../../../fixtures/applicationMcpHarness.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { NativeFixtureRunner, @@ -229,7 +228,6 @@ it("keeps other dyld_info nonzero exits as command failures", async () => { it("does not treat a newline in the echoed operand path as dyld_info evidence", async () => { const target = await makeTarget("\nmissing LC_UUID load command\n"); - const path = target.path; const runner = new UuidlessDyldRunner("malformed load command table"); const result = await new NativeMacOSProvider({}, runner, "darwin") .createClient(target) @@ -295,41 +293,35 @@ it("delivers partial UUID-less inspection Evidence through production MCP", asyn const session = createTestBinarySession( new NativeMacOSProvider({}, new UuidlessDyldRunner(), "darwin"), ); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "macho-uuidless-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const opened = await client.callTool({ - name: "open_binary", - arguments: { path }, - }); - expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + const { client } = await createApplicationMcpHarness({ + session, + clientIdentity: { name: "macho-uuidless-test", version: "1" }, + }); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); - const inspected = await client.callTool({ - name: "inspect_macho", - arguments: {}, - }); - expect(inspected.isError, JSON.stringify(inspected.content)).not.toBe(true); - expect(inspected.structuredContent).toMatchObject({ - normalized_result: { - uuid: null, - imports: { items: [], total: null, exhaustive: false }, - exports: { - items: expect.arrayContaining([ - expect.objectContaining({ name: "_main", source: "nm" }), - ]), - total: null, - exhaustive: false, - }, - limitations: expect.arrayContaining([ - "dyld_info -imports reported 'missing LC_UUID load command'; import observations are unavailable.", + const inspected = await client.callTool({ + name: "inspect_macho", + arguments: {}, + }); + expect(inspected.isError, JSON.stringify(inspected.content)).not.toBe(true); + expect(inspected.structuredContent).toMatchObject({ + normalized_result: { + uuid: null, + imports: { items: [], total: null, exhaustive: false }, + exports: { + items: expect.arrayContaining([ + expect.objectContaining({ name: "_main", source: "nm" }), ]), + total: null, + exhaustive: false, }, - }); - } finally { - await Promise.all([client.close(), server.close(), session.close()]); - } + limitations: expect.arrayContaining([ + "dyld_info -imports reported 'missing LC_UUID load command'; import observations are unavailable.", + ]), + }, + }); }); diff --git a/tests/boundary/providers/native/nativeProvider.test.ts b/tests/boundary/providers/native/nativeProvider.test.ts index 6763c249f5..a19206cfd1 100644 --- a/tests/boundary/providers/native/nativeProvider.test.ts +++ b/tests/boundary/providers/native/nativeProvider.test.ts @@ -8,7 +8,9 @@ import { PassThrough } from "node:stream"; import { afterEach, beforeAll, describe, expect, it } from "vitest"; import { build } from "plist"; +import { withVerifierCleanup } from "../../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../../fixtures/lifecycleAcknowledgement.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; @@ -428,23 +430,69 @@ describe("native command failure capture and limits", () => { }); it("retains output written before cancellation and reaps the stalled command", async () => { - const runner = new XcrunCommandRunner({}, () => - Promise.resolve( - ok(nativeResolvedToolFixture(process.execPath, "a".repeat(64))), - ), + let outputReady!: () => void; + let handshakeFailed!: (cause: Error) => void; + const ready = new Promise((resolve, reject) => { + outputReady = resolve; + handshakeFailed = reject; + }); + const launcher: NativeCommandLauncher = async (options) => { + const launched = await spawnOwnedProviderProcess({ + ...options, + stdin: "pipe", + }); + // The awaiting runner installs its supervisor in a microtask before this + // callback. Hold child output until its capture listeners are in place. + setImmediate(() => { + const { stdin, stdout, stderr } = launched.process; + if (stdin == null || stdout === null || stderr === null) { + handshakeFailed(new Error("Missing command handshake streams")); + return; + } + let out = ""; + let err = ""; + const observe = () => { + if (out === "started" && err === "warning") outputReady(); + }; + stdout.on("data", (chunk: Buffer) => { + out += chunk.toString(); + observe(); + }); + stderr.on("data", (chunk: Buffer) => { + err += chunk.toString(); + observe(); + }); + stdin.end("go"); + }); + return launched; + }; + const runner = new XcrunCommandRunner( + {}, + () => + Promise.resolve( + ok(nativeResolvedToolFixture(process.execPath, "a".repeat(64))), + ), + launcher, ); const controller = new AbortController(); const pending = runner.run( "file", [ "-e", - 'process.stdout.write("started"); process.stderr.write("warning"); setInterval(() => {}, 1000)', + 'process.stdin.once("data", () => { process.stdout.write("started"); process.stderr.write("warning"); }); setInterval(() => {}, 1000)', ], { signal: controller.signal }, ); - setTimeout(() => controller.abort(), 100); - - const result = await pending; + const result = await withVerifierCleanup(async () => { + await awaitAcknowledgement(ready, pending, "command output", 10_000); + controller.abort(); + return await pending; + }, [ + () => controller.abort(), + async () => { + await pending; + }, + ]); expect(result.ok).toBe(false); if (result.ok) return; diff --git a/tests/boundary/providers/startupCloseWaiters.test.ts b/tests/boundary/providers/startupCloseWaiters.test.ts index bf54b7fee9..4d0ff6601e 100644 --- a/tests/boundary/providers/startupCloseWaiters.test.ts +++ b/tests/boundary/providers/startupCloseWaiters.test.ts @@ -3,11 +3,13 @@ import { readFileSync } from "node:fs"; import { access } from "node:fs/promises"; import { expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; import { GhidraClient } from "../../../src/ghidra/GhidraClient.js"; import { HopperClient } from "../../../src/hopper/HopperClient.js"; import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; import { createDeferred } from "../../fixtures/binarySession.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { GhidraFixtureLauncher, ghidraFixturePath, @@ -72,16 +74,20 @@ it.each(providers)( await originalClose.call(this); }); let closing: ReturnType | undefined; - try { + let retry: ReturnType | undefined; + const controller = new AbortController(); + await withVerifierCleanup(async () => { await expect(client.start()).resolves.toMatchObject({ ok: true }); closing = client.close(); - await entered.promise; - const controller = new AbortController(); - const retry = client.start(controller.signal); + await awaitAcknowledgement(entered.promise, closing, "provider close"); + retry = client.start(controller.signal); let settled: Awaited | undefined; - void retry.then((result) => { - settled = result; - }); + void retry.then( + (result) => { + settled = result; + }, + () => undefined, + ); controller.abort(); await vi.waitFor(() => expect(settled).toMatchObject({ ok: false, error: cancelled }), @@ -96,11 +102,19 @@ it.each(providers)( }); await expect(client.start()).resolves.toMatchObject({ ok: true }); expect(children).toHaveLength(2); - } finally { - gate.resolve(); - await closing; - await client.close(); - close.mockRestore(); - } + }, [ + () => gate.resolve(), + () => controller.abort(), + async () => { + await retry; + }, + async () => { + await closing; + }, + async () => { + await expect(client.close()).resolves.toMatchObject({ ok: true }); + }, + () => close.mockRestore(), + ]); }, ); diff --git a/tests/boundary/setup/linuxHopperPublisherVerification.test.ts b/tests/boundary/setup/linuxHopperPublisherVerification.test.ts index 180276a35e..29f00a0781 100644 --- a/tests/boundary/setup/linuxHopperPublisherVerification.test.ts +++ b/tests/boundary/setup/linuxHopperPublisherVerification.test.ts @@ -22,6 +22,8 @@ import { fileURLToPath } from "node:url"; import { createMessage, generateKey, readPrivateKey, sign } from "openpgp"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { LinuxHopperPublisher, @@ -1035,9 +1037,13 @@ portableTar( throw new Error("Fixture descriptor close temporarily unavailable"); return await originalClose.apply(this); }); - try { - const inspection = f.publisher.inspectLauncher(f.launcher); - await metadataCloseStarted; + const inspection = f.publisher.inspectLauncher(f.launcher); + await withVerifierCleanup(async () => { + await awaitAcknowledgement( + metadataCloseStarted, + inspection, + "metadata close", + ); const hashFailure = await f.publisher.hashLauncher(f.launcher).then( () => undefined, (cause: unknown) => cause, @@ -1057,11 +1063,14 @@ portableTar( }); await expect(f.publisher.close()).resolves.toBeUndefined(); expect(close).toHaveBeenCalledTimes(9); - } finally { - releaseMetadataClose(); - close.mockRestore(); - await f.publisher.close(); - } + }, [ + releaseMetadataClose, + async () => { + await inspection; + }, + () => close.mockRestore(), + () => f.publisher.close(), + ]); }, ); diff --git a/tests/boundary/wasm/interfaceComparison.test.ts b/tests/boundary/wasm/interfaceComparison.test.ts deleted file mode 100644 index b692d9b291..0000000000 --- a/tests/boundary/wasm/interfaceComparison.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { expect, it } from "vitest"; - -import { parseWabtWat } from "../../../src/wasm/WabtWat.js"; -import { createEvidence } from "../../../src/domain/evidence.js"; -import { wasmInterfaceComparisonSchema } from "../../../src/contracts/wasm/wasmComparisonContracts.js"; -import { compareWasmInterfaces } from "../../../src/application/wasm/WasmInterfaceComparisonService.js"; -import { wasmArtifactFixture } from "../../fixtures/wasm/artifact.js"; - -it("does not call colliding rendered debug identifiers unchanged after a second function signature changes", async () => { - const input = (parameter: string) => { - const report = wasmArtifactFixture(); - report.interface = parseWabtWat( - `(module (func $same (param i32)) (func $same (param ${parameter})) (export "first" (func $same)) (export "second" (func $same)))`, - ).interface; - return createEvidence( - { - path: report.artifact.path, - sha256: report.artifact.sha256, - format: "file", - }, - { id: "fixture", name: "typed fixture", version: "1" }, - { - operation: "inspect_wasm_artifact", - parameters: { path: report.artifact.path }, - result: report, - confidence: "observed", - }, - ); - }; - const result = await compareWasmInterfaces({ - left: input("i32"), - right: input("i64"), - scope: "both", - }); - if (!result.ok) throw result.error; - const report = wasmInterfaceComparisonSchema.parse( - result.value.evidence.normalized_result, - ); - expect(report.changes.map(({ state }) => state)).toEqual([ - "unknown", - "unknown", - ]); - expect(report.summary.unchanged).toBe(0); -}); diff --git a/tests/composition/analysis-sessions/binarySession.test.ts b/tests/composition/analysis-sessions/binarySession.test.ts index c1e56cac2f..18bcf0c1b0 100644 --- a/tests/composition/analysis-sessions/binarySession.test.ts +++ b/tests/composition/analysis-sessions/binarySession.test.ts @@ -218,6 +218,7 @@ it("drains session calls and retires analysis after an admitted external mutatio expect((await session.open(target)).ok).toBe(true); expect(session.activeTarget()?.path).toBe(target); const originalDigest = session.activeTarget()?.sha256; + expect(originalDigest).toEqual(expect.any(String)); const started = createDeferred(); const release = createDeferred(); @@ -248,6 +249,7 @@ it("drains session calls and retires analysis after an admitted external mutatio expect(clients).toHaveLength(0); expect((await session.open(target)).ok).toBe(true); expect((await session.execute("address_name", {})).ok).toBe(true); + expect(session.activeTarget()?.sha256).toEqual(expect.any(String)); expect(session.activeTarget()?.sha256).not.toBe(originalDigest); expect(clients.at(-1)?.targetSha256).toBe(session.activeTarget()?.sha256); expect(calls).toEqual(["address_name"]); @@ -259,6 +261,7 @@ it("leaves analysis unbound when a command returns before its writer finishes", const session = createTestBinarySession(createCacheProvider([])); expect((await session.open(target)).ok).toBe(true); const originalDigest = session.activeTarget()?.sha256; + expect(originalDigest).toEqual(expect.any(String)); const releaseWriter = createDeferred(); let writer: Promise | undefined; const result = await session.withAdmittedExternalMutation( @@ -278,6 +281,7 @@ it("leaves analysis unbound when a command returns before its writer finishes", await writer; expect((await session.execute("address_name", {})).ok).toBe(false); expect((await session.open(target)).ok).toBe(true); + expect(session.activeTarget()?.sha256).toEqual(expect.any(String)); expect(session.activeTarget()?.sha256).not.toBe(originalDigest); await session.close(); }); @@ -324,6 +328,7 @@ it("retires analysis when an external mutation uses a hard link", async () => { const session = createTestBinarySession(createCacheProvider([])); expect((await session.open(target)).ok).toBe(true); const before = session.activeTarget()?.sha256; + expect(before).toEqual(expect.any(String)); const result = await session.withAdmittedExternalMutation( alias, @@ -341,6 +346,7 @@ it("retires analysis when an external mutation uses a hard link", async () => { }); expect(session.activeTarget()).toBeUndefined(); expect((await session.open(target)).ok).toBe(true); + expect(session.activeTarget()?.sha256).toEqual(expect.any(String)); expect(session.activeTarget()?.sha256).not.toBe(before); await session.close(); }); diff --git a/tests/composition/analysis-sessions/investigationRecords.test.ts b/tests/composition/analysis-sessions/investigationRecords.test.ts index 61e757ce6d..1d6b0063d9 100644 --- a/tests/composition/analysis-sessions/investigationRecords.test.ts +++ b/tests/composition/analysis-sessions/investigationRecords.test.ts @@ -11,19 +11,24 @@ import { createTestProviderRouter, } from "../../fixtures/binarySession.js"; -const createSession = ( - records: InvestigationRecords = new InvestigationRecords(), -) => { - const session = new BinarySession( - createTestProviderRouter(createCacheProvider([])), - records, - ); +const ownSession = (session: BinarySession): BinarySession => { onTestFinished(async () => { - await session.close(); + const closed = await session.close(); + if (!closed.ok) throw closed.error; }); return session; }; +const createSession = ( + records: InvestigationRecords = new InvestigationRecords(), +) => + ownSession( + new BinarySession( + createTestProviderRouter(createCacheProvider([])), + records, + ), + ); + it("uses the composed record owner and preserves best-effort post-commit notifications", async () => { const records = new InvestigationRecords(); const session = createSession(records); @@ -65,10 +70,7 @@ it("uses the composed record owner and preserves best-effort post-commit notific it("notifies after snapshot and Evidence import both commit, and rejects mismatched targets atomically", async () => { const [first, second] = await createBinarySessionTargets(); - const source = createTestBinarySession(createCacheProvider([])); - onTestFinished(async () => { - await source.close(); - }); + const source = ownSession(createTestBinarySession(createCacheProvider([]))); expect((await source.open(first)).ok).toBe(true); expect( ( diff --git a/tests/composition/analysis-sessions/managedApplicationGraph.test.ts b/tests/composition/analysis-sessions/managedApplicationGraph.test.ts index 5749168fea..2bb30e5b7f 100644 --- a/tests/composition/analysis-sessions/managedApplicationGraph.test.ts +++ b/tests/composition/analysis-sessions/managedApplicationGraph.test.ts @@ -220,9 +220,11 @@ describe("managed application graph projection", () => { expect(observedEvidenceIds("managed-pinvoke-import")).toEqual([ boundaryEvidence.evidence_id, ]); - for (const edge of graph.edges.filter( + const containmentEdges = graph.edges.filter( ({ relation }) => relation === "contains", - )) + ); + expect(containmentEdges.length).toBeGreaterThan(0); + for (const edge of containmentEdges) expect(edge.evidence.evidence_ids).toEqual(allEvidenceIds); const method = graph.nodes.find( ({ kind, observations }) => diff --git a/tests/composition/analysis-sessions/providerBinding.test.ts b/tests/composition/analysis-sessions/providerBinding.test.ts index 0c18ac9762..afc2ef9b54 100644 --- a/tests/composition/analysis-sessions/providerBinding.test.ts +++ b/tests/composition/analysis-sessions/providerBinding.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, onTestFinished } from "vitest"; import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; import { createAnalysisExecution, @@ -128,33 +129,54 @@ describe("target-bound provider routing", () => { expect(beta.clients[0]?.closed).toBe(true); }); - it("preserves the active client when a provider switch is cancelled while draining", async () => { + it("preserves the active client when a provider switch is cancelled with a call in flight", async () => { const target = await databaseTarget(); const alpha = deepProvider("alpha"); const beta = deepProvider("beta"); const pending = deferred>(); + const callStarted = deferred(); + const profileStarted = deferred(); alpha.pending = pending.promise; + alpha.onPending = () => callStarted.resolve(); + beta.onProfile = () => profileStarted.resolve(); const session = selectableSession([alpha.provider, beta.provider]); - expect((await session.open(target, { providerId: "alpha" })).ok).toBe(true); - const running = session.execute("address_name", { address: "0x1" }); - await new Promise((resolve) => setImmediate(resolve)); - const controller = new AbortController(); - const switching = session.open(target, { + const reason = new Error("cancel provider switch"); + let running: Promise | undefined; + let switching: ReturnType | undefined; + onTestFinished(async () => { + controller.abort(reason); + pending.resolve(successfulExecution("alpha")); + await Promise.allSettled([ + ...(running === undefined ? [] : [running]), + ...(switching === undefined ? [] : [switching]), + ]); + expect(await session.close()).toEqual(ok(null)); + }); + expect((await session.open(target, { providerId: "alpha" })).ok).toBe(true); + running = session.execute("address_name", { address: "0x1" }); + await awaitAcknowledgement(callStarted.promise, running, "provider call"); + + switching = session.open(target, { providerId: "beta", signal: controller.signal, }); - while (beta.profileResolutions === 0) - await new Promise((resolve) => setImmediate(resolve)); - controller.abort(); + await awaitAcknowledgement( + profileStarted.promise, + switching, + "provider profile resolution", + ); + expect(alpha.clients[0]?.closed).toBe(false); + controller.abort(reason); pending.resolve(successfulExecution("alpha")); expect(await running).toMatchObject({ ok: true }); const cancelled = await switching; expect(cancelled).toEqual({ ok: false, - error: new AnalysisCancelledError("open_binary"), + error: new AnalysisCancelledError("open_binary", { cause: reason }), }); + if (!cancelled.ok) expect(cancelled.error.cause).toBe(reason); expect(alpha.clients[0]?.closed).toBe(false); expect(beta.clients).toHaveLength(0); expect(await bindingStatus(session)).toMatchObject({ @@ -165,7 +187,7 @@ describe("target-bound provider routing", () => { const retained = await session.execute("address_name", { address: "0x2" }); expect(retained.ok && retained.value.provider.id).toBe("alpha"); expect(alpha.clients).toHaveLength(1); - await session.close(); + expect(await session.close()).toEqual(ok(null)); expect(alpha.clients[0]?.closed).toBe(true); }); }); @@ -251,7 +273,8 @@ interface DeepProviderFixture { readonly clients: ClientState[]; available: boolean; fail: boolean; - profileResolutions: number; + onProfile: (() => void) | undefined; + onPending: (() => void) | undefined; pending: Promise> | undefined; } @@ -265,7 +288,8 @@ const deepProvider = (id: string): DeepProviderFixture => { clients: [], available: true, fail: false, - profileResolutions: 0, + onProfile: undefined, + onPending: undefined, pending: undefined, }; const provider: AnalysisProviderCandidate = { @@ -292,7 +316,7 @@ const deepProvider = (id: string): DeepProviderFixture => { diagnostics: {}, }), resolveAnalysisProfile: () => { - fixture.profileResolutions += 1; + fixture.onProfile?.(); return Promise.resolve( ok({ profile: createAnalysisProfile( @@ -314,7 +338,10 @@ const deepProvider = (id: string): DeepProviderFixture => { return Promise.resolve( err(new ProviderAdapterError(id, operation)), ); - if (fixture.pending !== undefined) return fixture.pending; + if (fixture.pending !== undefined) { + fixture.onPending?.(); + return fixture.pending; + } return Promise.resolve(successfulExecution(id, profile?.provider)); }, close: () => { diff --git a/tests/composition/browserScenarioStartupRedaction.test.ts b/tests/composition/browserScenarioStartupRedaction.test.ts index 8dc80b2127..a0fdef9afc 100644 --- a/tests/composition/browserScenarioStartupRedaction.test.ts +++ b/tests/composition/browserScenarioStartupRedaction.test.ts @@ -2,6 +2,8 @@ import { BrowserObservationError } from "../../src/domain/browserObservationErro import { PlaywrightScenarioSession } from "../../src/browser/PlaywrightScenarioSession.js"; import { chromium, type BrowserContext } from "playwright-core"; import { expect, it, vi } from "vitest"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../fixtures/lifecycleAcknowledgement.js"; import { captureBrowserScenario } from "../../src/application/BrowserScenarioCaptureService.js"; import { createBrowserScenarioProvider } from "../../src/composition/browserScenario.js"; import { browserScenarioSchema } from "../../src/domain/browserScenario.js"; @@ -68,11 +70,12 @@ it.each([ const closeProvider = provider.close.bind(provider); const cancellation = new AbortController(); const reason = { caller: "cancelled opening" }; - try { - const capturing = captureBrowserScenario(provider, scenario, { + let capturing: ReturnType | undefined; + await withVerifierCleanup(async () => { + capturing = captureBrowserScenario(provider, scenario, { signal: cancellation.signal, }); - await started; + await awaitAcknowledgement(started, capturing, "browser launch"); if (late) cancellation.abort(reason); const result = await capturing; if (result.ok) throw new Error("Expected startup failure"); @@ -113,12 +116,17 @@ it.each([ expect(result.error.cleanupIncomplete).toBe(false); expect(retry.cleanupIncomplete).toBe(false); expect(result.error.cleanupResources).toEqual([]); - } finally { - blocked = false; - finishOpening(); - await closeProvider(); - launch.mockRestore(); - } + }, [ + () => { + blocked = false; + finishOpening(); + }, + async () => { + await capturing; + }, + closeProvider, + () => launch.mockRestore(), + ]); }, ); diff --git a/tests/composition/directAnalysisCleanup.test.ts b/tests/composition/directAnalysisCleanup.test.ts index d18226ee9e..e1ea540fe3 100644 --- a/tests/composition/directAnalysisCleanup.test.ts +++ b/tests/composition/directAnalysisCleanup.test.ts @@ -656,7 +656,7 @@ it.each(["complete", "summary"] as const)( "retains completed Evidence when cancellation arrives during managed close before %s delivery", async (detail) => { const [path] = await createBinarySessionTargets(); - const files = await readdir(dirname(path)); + const files = (await readdir(dirname(path))).sort(); const original = parseEvidence( await runProviderAnalysis( factories().dependencies, @@ -702,7 +702,7 @@ it.each(["complete", "summary"] as const)( ), }); expect(session.activeTarget()).toBeUndefined(); - expect(await readdir(dirname(path))).toEqual(files); + expect((await readdir(dirname(path))).sort()).toEqual(files); }, ); diff --git a/tests/composition/reverseEngineering.test.ts b/tests/composition/reverseEngineering.test.ts index a1cf245597..cc5987bcb6 100644 --- a/tests/composition/reverseEngineering.test.ts +++ b/tests/composition/reverseEngineering.test.ts @@ -15,17 +15,22 @@ import { projectAnalysisError } from "../../src/domain/analysisErrorProjection.j import { AnalysisCancelledError } from "../../src/domain/analysisErrorCore.js"; import { REVERSE_ENGINEERING_TOOL_CONTRACTS } from "../../src/contracts/reverseEngineeringToolContracts.js"; -describe("reverse engineering command services", () => { - it("documents the shared combined capture limit for both command tools", () => { - for (const contract of REVERSE_ENGINEERING_TOOL_CONTRACTS) - expect(contract.description).toContain("shared 16 MiB combined limit"); - expect(REVERSE_ENGINEERING_TOOL_CONTRACTS[0]?.description).toContain( - "does not follow local debug links by default", - ); - for (const contract of REVERSE_ENGINEERING_TOOL_CONTRACTS) - expect(contract.description).toContain("command_timeout_ms"); - }); +it("documents the shared combined capture limit for both command tools", () => { + expect(REVERSE_ENGINEERING_TOOL_CONTRACTS.map(({ name }) => name)).toEqual( + expect.arrayContaining(["inspect_with_objdump", "execute_rizin_command"]), + ); + for (const contract of REVERSE_ENGINEERING_TOOL_CONTRACTS) + expect(contract.description).toContain("shared 16 MiB combined limit"); + expect( + REVERSE_ENGINEERING_TOOL_CONTRACTS.find( + ({ name }) => name === "inspect_with_objdump", + )?.description, + ).toContain("does not follow local debug links by default"); + for (const contract of REVERSE_ENGINEERING_TOOL_CONTRACTS) + expect(contract.description).toContain("command_timeout_ms"); +}); +describe("reverse engineering command services", () => { it("uses compatible local DWARF link flags and disables debuginfod through the environment", async () => { const directory = await mkdtemp(join(tmpdir(), "rea-reverse-tools-")); const path = join(directory, "fixture.bin"); diff --git a/tests/composition/reverseEngineeringCommandPort.test.ts b/tests/composition/reverseEngineeringCommandPort.test.ts index 95c4b47524..bd2d21dfc3 100644 --- a/tests/composition/reverseEngineeringCommandPort.test.ts +++ b/tests/composition/reverseEngineeringCommandPort.test.ts @@ -7,16 +7,10 @@ import { expect, it, vi } from "vitest"; import { ReverseEngineeringService } from "../../src/application/reverse/ReverseEngineeringService.js"; import { analysisErrorProjectionSchema } from "../../src/contracts/errorSchemas.js"; import { projectAnalysisError } from "../../src/domain/analysisErrorProjection.js"; -import { REVERSE_ENGINEERING_TOOL_CONTRACTS } from "../../src/contracts/reverseEngineeringToolContracts.js"; import { execFileOutput } from "../../src/process/ExecFileOutput.js"; type ExecFileOutputOptions = NonNullable[2]>; -it("documents the shared combined capture limit for both command tools", () => { - for (const contract of REVERSE_ENGINEERING_TOOL_CONTRACTS) - expect(contract.description).toContain("shared 16 MiB combined limit"); -}); - it("uses provider profiles supplied through the application port", async () => { const directory = await mkdtemp(join(tmpdir(), "rea-reverse-port-")); const path = join(directory, "fixture.bin"); diff --git a/tests/composition/reverseEngineeringEdgeCases.test.ts b/tests/composition/reverseEngineeringEdgeCases.test.ts index 226284cc37..b268c90b29 100644 --- a/tests/composition/reverseEngineeringEdgeCases.test.ts +++ b/tests/composition/reverseEngineeringEdgeCases.test.ts @@ -23,6 +23,8 @@ import { } from "../../src/process/OwnedCommand.js"; import type { ProviderProcessSupervisor } from "../../src/process/ProviderProcess.js"; import { REVERSE_ENGINEERING_TOOL_CONTRACTS } from "../../src/contracts/reverseEngineeringToolContracts.js"; +import { awaitAcknowledgement } from "../fixtures/lifecycleAcknowledgement.js"; +import { withVerifierCleanup } from "../../scripts/lib/verifier-cleanup.mjs"; registerSharedReverseEngineeringEdgeCaseTests(); registerOwnedRizinCommandEvidenceTests({ sweepTokenOwnedProcesses: true }); @@ -117,15 +119,13 @@ it.each(["cancellation", "deadline"] as const)( run: async () => ({ stdout: "", stderr: "" }), }); - try { - if (interruption === "deadline") vi.useFakeTimers(); - const resultPromise = service.inspectWithObjdump( - { path, operation: "file_headers" }, - { signal: controller.signal }, - ); - await statStarted; + const resultPromise = service.inspectWithObjdump( + { path, operation: "file_headers", command_timeout_ms: 1_000 }, + { signal: controller.signal }, + ); + await withVerifierCleanup(async () => { + await awaitAcknowledgement(statStarted, resultPromise, "stat request"); if (interruption === "cancellation") controller.abort(); - else await vi.advanceTimersByTimeAsync(120_000); await expect(resultPromise).resolves.toMatchObject({ ok: false, error: { @@ -137,10 +137,18 @@ it.each(["cancellation", "deadline"] as const)( }); rejectStat?.(new Error("late stat failure")); await Promise.resolve(); - } finally { - vi.useRealTimers(); - await rm(directory, { recursive: true, force: true }); - } + }, [ + () => { + controller.abort(); + rejectStat?.(new Error("stat fixture cleanup")); + }, + async () => { + await resultPromise; + }, + async () => { + await rm(directory, { recursive: true, force: true }); + }, + ]); }, ); @@ -378,11 +386,17 @@ it("rejects work after reaching the concurrent provider-operation limit", async }, }); - try { - const admitted = Array.from({ length: 8 }, () => - service.executeRizinCommand({ path, command: "iI" }), + const admitted = Array.from({ length: 8 }, () => + service.executeRizinCommand({ path, command: "iI" }), + ); + await withVerifierCleanup(async () => { + const rejection = service.executeRizinCommand({ path, command: "iI" }); + await awaitAcknowledgement( + rejection.then(() => {}), + Promise.all(admitted), + "capacity response", ); - const rejected = await service.executeRizinCommand({ path, command: "iI" }); + const rejected = await rejection; expect(rejected.ok).toBe(false); if (!rejected.ok) { @@ -393,9 +407,6 @@ it("rejects work after reaching the concurrent provider-operation limit", async max_concurrent_operations: 8, active_operations: 8, }); - expect(rejected.error.message).toContain( - "Concurrent reverse-engineering operation capacity reached (8 operations).", - ); } expect(providerCalls).toBe(0); @@ -403,10 +414,15 @@ it("rejects work after reaching the concurrent provider-operation limit", async const results = await Promise.all(admitted); expect(results.every((result) => result.ok)).toBe(true); expect(providerCalls).toBe(8); - } finally { - releaseProviders?.(); - await rm(directory, { recursive: true, force: true }); - } + }, [ + () => releaseProviders?.(), + async () => { + await Promise.allSettled(admitted); + }, + async () => { + await rm(directory, { recursive: true, force: true }); + }, + ]); }); it("cancels when an AbortSignal becomes aborted during listener registration", async () => { @@ -443,7 +459,10 @@ it("cancels when an AbortSignal becomes aborted during listener registration", a it("drains an active provider operation before service shutdown resolves", async () => { const { directory, path } = await createArtifact(); let signal: AbortSignal | undefined; - let releaseRun: (() => void) | undefined; + let releaseRun!: () => void; + const runGate = new Promise((resolve) => { + releaseRun = resolve; + }); let runStarted: (() => void) | undefined; const started = new Promise((resolve) => { runStarted = resolve; @@ -452,30 +471,39 @@ it("drains an active provider operation before service shutdown resolves", async run: async (_command, _args, options) => { signal = options?.signal; runStarted?.(); - await new Promise((resolve) => { - releaseRun = resolve; - }); + await runGate; return { stdout: "finished", stderr: "" }; }, }); - try { - const operation = service.executeRizinCommand({ path, command: "iI" }); - await started; + const operation = service.executeRizinCommand({ path, command: "iI" }); + let closing: Promise | undefined; + await withVerifierCleanup(async () => { + await awaitAcknowledgement(started, operation, "provider command"); let closed = false; - const closing = service.close().then(() => { + closing = service.close().then(() => { closed = true; }); expect(signal?.aborted).toBe(true); expect(closed).toBe(false); - releaseRun?.(); + releaseRun(); await closing; expect(closed).toBe(true); - await operation; - } finally { - releaseRun?.(); - await rm(directory, { recursive: true, force: true }); - } + await expect(operation).resolves.toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + }, [ + releaseRun, + async () => { + await operation; + }, + async () => { + await closing; + }, + () => service.close(), + () => rm(directory, { recursive: true, force: true }), + ]); }); it("relays caller cancellation to the admitted provider signal", async () => { @@ -486,27 +514,26 @@ it("relays caller cancellation to the admitted provider signal", async () => { const started = new Promise((resolve) => { announceStarted = resolve; }); - let releaseRun: (() => void) | undefined; + let releaseRun!: () => void; + const runGate = new Promise((resolve) => { + releaseRun = resolve; + }); let providerSignal: AbortSignal | undefined; const service = new CoreReverseEngineeringService({ commands: createReverseEngineeringCommandPort(process.env), run: async (_command, _args, options) => { providerSignal = options?.signal; announceStarted?.(); - await new Promise((resolve) => { - releaseRun = resolve; - }); + await runGate; return { stdout: "captured", stderr: "" }; }, }); - let operation: ReturnType | undefined; - - try { - operation = service.inspectWithObjdump( - { path, operation: "file_headers" }, - { signal: controller.signal }, - ); - await started; + const operation = service.inspectWithObjdump( + { path, operation: "file_headers" }, + { signal: controller.signal }, + ); + await withVerifierCleanup(async () => { + await awaitAcknowledgement(started, operation, "provider command"); controller.abort(); expect(addAbortListener).toHaveBeenCalledWith( "abort", @@ -514,16 +541,20 @@ it("relays caller cancellation to the admitted provider signal", async () => { { once: true }, ); expect(providerSignal?.aborted).toBe(true); - releaseRun?.(); + releaseRun(); await expect(operation).resolves.toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" }, }); - } finally { - releaseRun?.(); - if (operation !== undefined) await operation.catch(() => undefined); - await rm(directory, { recursive: true, force: true }); - } + }, [ + () => controller.abort(), + releaseRun, + async () => { + await operation; + }, + () => service.close(), + () => rm(directory, { recursive: true, force: true }), + ]); }); it("passes an AbortSignal-like caller value through without relaying it", async () => { diff --git a/tests/conformance/ida/providerContracts.test.ts b/tests/conformance/ida/providerContracts.test.ts index 465dba37e6..ae2599c5ea 100644 --- a/tests/conformance/ida/providerContracts.test.ts +++ b/tests/conformance/ida/providerContracts.test.ts @@ -48,6 +48,11 @@ describe.each(["attached", "headless"] as const)( const operations = IDA_OPERATIONS.filter( (operation) => mode === "headless" || !HEADLESS_ONLY.has(operation), ); + it("includes the baseline operations needed by this contract matrix", () => { + expect(operations).toContain("list_procedures"); + if (mode === "headless") + expect(operations).toContain("resolve_containing_procedure"); + }); it.each(operations)( "declares %s UI effects matching the upstream tools it reaches", async (operation) => { diff --git a/tests/evaluation/providerRequiredCases.test.ts b/tests/evaluation/providerRequiredCases.test.ts index 6f86f6e837..7fb3181175 100644 --- a/tests/evaluation/providerRequiredCases.test.ts +++ b/tests/evaluation/providerRequiredCases.test.ts @@ -49,6 +49,9 @@ const report = (lane: "packet" | "signal") => ({ "tests/boundary/captures/captureDescriptorCleanup.test.ts", cleanupCases, ), + testFile("tests/acceptance/pcapVerifier.test.ts", [ + "verifies complete and unresolved HTTP coverage with real Wireshark tools", + ]), ] : [ testFile( @@ -100,7 +103,7 @@ it.each(["packet", "signal"] as const)( assertionResults: [{ fullName: "host-specific", status: "pending" }], }); expect(run(lane, result)).toContain("cases passed"); - for (const resultIndex of [0, 1]) { + for (const resultIndex of report(lane).testResults.keys()) { const skipped = report(lane); skipped.testResults[resultIndex]!.assertionResults[0]!.status = "pending"; expect(() => run(lane, skipped)).toThrow( @@ -208,6 +211,8 @@ it.each([ expect(step?.run).toContain('--outputFile.json="$report"'); expect(step?.run).toContain(limits); expect(step?.run).toContain(cleanup); + if (lane === "packet") + expect(step?.run).toContain("tests/acceptance/pcapVerifier.test.ts"); expect(step?.run).toContain( `node scripts/ci/require-provider-tests.mjs ${lane} "$report"`, ); diff --git a/tests/fixtures/advertisedMcpTool.ts b/tests/fixtures/advertisedMcpTool.ts index 1e8b484a41..559c6817d6 100644 --- a/tests/fixtures/advertisedMcpTool.ts +++ b/tests/fixtures/advertisedMcpTool.ts @@ -23,9 +23,12 @@ export const advertisedMcpToolSchemas = async ( validateFormats: false, }), ) => { - const advertised = (await client.listTools()).tools.find( + const matches = (await client.listTools()).tools.filter( (tool) => tool.name === name, ); + if (matches.length > 1) + throw new Error(`${name} has multiple advertised definitions`); + const advertised = matches[0]; if (advertised?.outputSchema === undefined) throw new Error(`${name} must publish both schemas`); const inputSchema: Record = advertised.inputSchema; diff --git a/tests/fixtures/cutter_bridge_protocol_test.py b/tests/fixtures/cutter_bridge_protocol_test.py index 1f8bf76e10..fd605f7cdd 100644 --- a/tests/fixtures/cutter_bridge_protocol_test.py +++ b/tests/fixtures/cutter_bridge_protocol_test.py @@ -42,7 +42,10 @@ spec.loader.exec_module(bridge) def _frame(connection): data = bytearray() while b"\n" not in data: - data.extend(connection.recv(4096)) + chunk = connection.recv(4096) + if not chunk: + raise EOFError("connection closed before a complete test frame") + data.extend(chunk) if len(data) > bridge.MAX_FRAME_BYTES: raise AssertionError("oversized test frame") line, remainder = bytes(data).split(b"\n", 1) @@ -477,7 +480,7 @@ class ProtocolProducerTest(unittest.TestCase): rejected.settimeout(1) try: observed = rejected.recv(16) - except OSError: + except (ConnectionResetError, ConnectionAbortedError): observed = b"" self.assertEqual(observed, b"") finally: @@ -514,7 +517,7 @@ class ProtocolProducerTest(unittest.TestCase): rejected.settimeout(1) try: observed = rejected.recv(16) - except OSError: + except (ConnectionResetError, ConnectionAbortedError): observed = b"" self.assertEqual(observed, b"") finally: diff --git a/tests/fixtures/jebMcp.ts b/tests/fixtures/jebMcp.ts index da43ab6c1a..7ddf25a5ea 100644 --- a/tests/fixtures/jebMcp.ts +++ b/tests/fixtures/jebMcp.ts @@ -6,7 +6,13 @@ import { JEB_ENGINE_TOOLS } from "../../src/jeb/JebMcpConnection.js"; const messageSchema = z.object({ id: z.union([z.string(), z.number()]).optional(), method: z.string(), - params: z.object({ name: z.string().optional() }).passthrough().optional(), + params: z + .object({ + name: z.string().optional(), + arguments: z.unknown().optional(), + }) + .passthrough() + .optional(), }); /** Real HTTP MCP transport; close drains admitted catalog/tool callbacks, preserving handled 500 replies. */ @@ -14,7 +20,7 @@ export const startJebMcpFixture = async ( respond: (tool: string) => JsonValue | Promise, catalogReply?: JsonValue | (() => JsonValue | Promise), ) => { - const calls: string[] = []; + const requests: { name: string; arguments: unknown }[] = []; const { server, close } = createVerifierHttpServer( async (request, response) => { if (request.method !== "POST") { @@ -50,7 +56,10 @@ export const startJebMcpFixture = async ( message.method === "tools/call" && message.params?.name !== undefined ) { - calls.push(message.params.name); + requests.push({ + name: message.params.name, + arguments: message.params.arguments, + }); const payload = await respond(message.params.name); result = { structuredContent: payload, @@ -72,7 +81,10 @@ export const startJebMcpFixture = async ( throw new Error("Expected TCP fixture listener"); return { endpoint: `http://127.0.0.1:${address.port}/mcp`, - calls, + requests, + get calls() { + return requests.map(({ name }) => name); + }, close, }; }; diff --git a/tests/fixtures/lifecycleAcknowledgement.ts b/tests/fixtures/lifecycleAcknowledgement.ts index 0789185325..f821ded973 100644 --- a/tests/fixtures/lifecycleAcknowledgement.ts +++ b/tests/fixtures/lifecycleAcknowledgement.ts @@ -3,6 +3,7 @@ export const awaitAcknowledgement = async ( acknowledgement: Promise, operation: Promise, boundary: string, + timeoutMs = 2_000, ): Promise => { let deadline: ReturnType | undefined; try { @@ -14,7 +15,7 @@ export const awaitAcknowledgement = async ( ({ kind: "settled", status: "rejected", cause }) as const, ), new Promise<{ readonly kind: "deadline" }>((resolve) => { - deadline = setTimeout(() => resolve({ kind: "deadline" }), 2_000); + deadline = setTimeout(() => resolve({ kind: "deadline" }), timeoutMs); }), ]); if (observed.kind !== "acknowledged") diff --git a/tests/fixtures/localToolsMcp.ts b/tests/fixtures/localToolsMcp.ts index e2de6357d2..bc2a598b8d 100644 --- a/tests/fixtures/localToolsMcp.ts +++ b/tests/fixtures/localToolsMcp.ts @@ -37,6 +37,17 @@ const compileValidator = (schema: unknown): ValidateFunction => { return validate; }; +/** Default to native tools while preserving explicit caller configuration. */ +export const localToolsMcpEnvironment = ( + environment: Readonly> = {}, +): Record => ({ + PATH: process.env.PATH ?? "/usr/bin:/bin", + REA_LOG_LEVEL: "silent", + HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper", + GHIDRA_INSTALL_DIR: process.execPath, + ...environment, +}); + /** Start the built production server without configuring deep binary providers. */ export async function connectLocalToolsMcp( environment: Readonly> = {}, @@ -51,12 +62,7 @@ export async function connectLocalToolsMcp( fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)), "mcp", ], - env: { - PATH: process.env.PATH ?? "/usr/bin:/bin", - REA_LOG_LEVEL: "silent", - HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper", - ...environment, - }, + env: localToolsMcpEnvironment(environment), stderr: "pipe", }); const client = new Client( diff --git a/tests/fixtures/pcap/offlineHttpCapture.ts b/tests/fixtures/pcap/offlineHttpCapture.ts index cf3c8b421d..8abcf1c84d 100644 --- a/tests/fixtures/pcap/offlineHttpCapture.ts +++ b/tests/fixtures/pcap/offlineHttpCapture.ts @@ -1,9 +1,13 @@ /** Owned offline Ethernet/IPv4/TCP fixture; no traffic or external input. */ -export const offlineHttpCapture = (uri: string, packets = 1): Buffer => { +export const offlineHttpCapture = ( + uri: string, + packets = 1, + host: string | null = "example.test", +): Buffer => { if (!Number.isSafeInteger(packets) || packets < 1 || packets > 40_000) throw new RangeError("Offline test captures support 1..40000 packets"); const payload = Buffer.from( - `GET ${uri} HTTP/1.1\r\nHost: example.test\r\nConnection: close\r\n\r\n`, + `GET ${uri} HTTP/${host === null ? "1.0" : "1.1"}\r\n${host === null ? "" : `Host: ${host}\r\n`}Connection: close\r\n\r\n`, ); const checksum = (bytes: Buffer): number => { let total = 0; diff --git a/tests/process-global/bundleComparisonFailures.test.ts b/tests/process-global/bundleComparisonFailures.test.ts index 722bc47112..2542508220 100644 --- a/tests/process-global/bundleComparisonFailures.test.ts +++ b/tests/process-global/bundleComparisonFailures.test.ts @@ -84,6 +84,7 @@ it.each(["missing", "invalid", "cleanup"])( expect(result.error.cleanupResources).toContain(rightPath); if (leftFailure === "cleanup") expect(result.error.cleanupResources).toContain(leftPath); + expect(owners.size).toBe(leftFailure === "cleanup" ? 2 : 1); expect([...owners].every((owner) => owner.handle.fd >= 0)).toBe(true); // Retry the exact leases carried through the combined failure, rather than @@ -428,6 +429,7 @@ it.each([ expect(JSON.parse(stdout)).toMatchObject({ code: "cancelled" }); expect(process.exitCode).toBe(130); expect(process.listenerCount("SIGINT")).toBe(before); + expect(owners.size).toBeGreaterThan(0); expect([...owners].every((owner) => owner.handle.fd < 0)).toBe(true); expect(await readdir(directory)).toEqual(["bundle.json"]); } finally { @@ -552,7 +554,7 @@ it.each(["before-commit", "after-commit", "cleanup-retry"])( }); const loaded = await readEvidenceBundle(output); expect(loaded).toEqual({ ok: true, value: bundle }); - expect(await readdir(directory)).toEqual([ + expect((await readdir(directory)).sort()).toEqual([ "bundle.json", "output.json", ]); diff --git a/tests/process-global/configurationWriteDiagnostics.test.ts b/tests/process-global/configurationWriteDiagnostics.test.ts index a97ca36a8a..3c97292c11 100644 --- a/tests/process-global/configurationWriteDiagnostics.test.ts +++ b/tests/process-global/configurationWriteDiagnostics.test.ts @@ -95,7 +95,7 @@ it.each([ }); expect(await readFile(path, "utf8")).toBe(original); expect(await readFile(`${path}.rea.backup`, "utf8")).toBe(original); - expect(await readdir(root)).toEqual([ + expect((await readdir(root)).sort()).toEqual([ "config.json", "config.json.rea.backup", ]); diff --git a/tests/process-global/cutterDescriptorLifetime.test.ts b/tests/process-global/cutterDescriptorLifetime.test.ts index 3e4e87d0f0..463f699abc 100644 --- a/tests/process-global/cutterDescriptorLifetime.test.ts +++ b/tests/process-global/cutterDescriptorLifetime.test.ts @@ -38,6 +38,7 @@ import { AnalysisCancelledError } from "../../src/domain/analysisErrorCore.js"; import { AnalysisError } from "../../src/domain/analysisErrorBase.js"; import { projectAnalysisError } from "../../src/domain/analysisErrorProjection.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../fixtures/lifecycleAcknowledgement.js"; import { cliTest } from "../support/cli/cliFixture.js"; beforeEach(() => { @@ -164,19 +165,28 @@ it("shutdown waits for descriptor finalization before retrying retained cleanup" await gate; }; const client = new CutterBridgeClient({ REA_CUTTER_BRIDGE_DIR: root }); - const reading = client.listDescriptors().catch((cause: unknown) => cause); - await closing; - let settled = false; - const shutdown = client.close().then(() => { - settled = true; - }); - await new Promise((resolve) => setImmediate(resolve)); - expect(settled).toBe(false); - fault.blocked.clear(); - release(); - await reading; - await shutdown; - expect(fault.handles.every((handle) => handle.fd === -1)).toBe(true); + const reading = client.listDescriptors(); + let shutdown: Promise | undefined; + try { + await awaitAcknowledgement(closing, reading, "descriptor finalization"); + let settled = false; + shutdown = client.close().then(() => { + settled = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + expect(settled).toBe(false); + fault.blocked.clear(); + release(); + expect(await reading).toHaveLength(1); + await shutdown; + expect(fault.handles.every((handle) => handle.fd === -1)).toBe(true); + } finally { + release(); + fault.onClose = undefined; + fault.blocked.clear(); + await Promise.allSettled([reading, ...(shutdown ? [shutdown] : [])]); + await client.close(); + } }); it.each([null, "primitive abort", new Error("selected abort")])( @@ -236,7 +246,11 @@ it("does not assign a peer's later cancellation to a descriptor that already fai .listSessions({ signal: controller.signal }) .catch((cause: unknown) => cause); try { - await waiting; + await awaitAcknowledgement( + waiting, + reading, + "peer descriptor finalization", + ); await new Promise((resolve) => setImmediate(resolve)); controller.abort(reason); release(); diff --git a/tests/process-global/darwinCompilerCleanup.test.ts b/tests/process-global/darwinCompilerCleanup.test.ts index 3cc9abd6eb..651b1eb03b 100644 --- a/tests/process-global/darwinCompilerCleanup.test.ts +++ b/tests/process-global/darwinCompilerCleanup.test.ts @@ -96,6 +96,34 @@ setInterval(() => {}, 1000); return { reader, readyPath: join(directory, "ready") }; }; +// Preserve an early operation outcome instead of reporting only a missing file. +const waitForCompilerReady = async ( + readyPath: string, + pending: Promise, +): Promise => { + let settled: { readonly value: unknown } | undefined; + void pending.then( + (value) => { + settled = { value }; + }, + (value: unknown) => { + settled = { value }; + }, + ); + const outcome = await vi.waitFor( + async () => { + if (settled !== undefined) return settled; + expect(await readFile(readyPath, "utf8")).toBe("ready"); + return undefined; + }, + { timeout: 10_000 }, + ); + if (outcome !== undefined) + throw new Error("Compiler operation settled before its ready marker", { + cause: outcome.value, + }); +}; + it .skipIf(process.platform === "win32") .each(["ordinary", "cancel during cleanup"] as const)( @@ -177,23 +205,26 @@ it const preparation = reader .prepare(controller.signal) .catch((cause: unknown) => cause); - await vi.waitFor(async () => - expect(await readFile(readyPath, "utf8")).toBe("ready"), - ); - const root = [...fault.roots][0]; - if (root === undefined) throw new Error("Expected helper root"); - controller.abort(reason); - const failure: unknown = await preparation; - expect(failure).toBeInstanceOf(DarwinProcessOwnershipInspectionError); - if (!(failure instanceof DarwinProcessOwnershipInspectionError)) - throw new Error("Expected retained cleanup failure"); - expect(failure.cause).toBe(reason); - expect(failure.cancellation?.reason).toBe(reason); - expect(failure.cleanupFailure?.cleanupIncomplete).toBe(true); - fault.blocked = false; - await reader.close(); - expect(failure.cleanupFailure?.cleanupIncomplete).toBe(false); - await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" }); + try { + await waitForCompilerReady(readyPath, preparation); + const root = [...fault.roots][0]; + if (root === undefined) throw new Error("Expected helper root"); + controller.abort(reason); + const failure: unknown = await preparation; + expect(failure).toBeInstanceOf(DarwinProcessOwnershipInspectionError); + if (!(failure instanceof DarwinProcessOwnershipInspectionError)) + throw new Error("Expected retained cleanup failure"); + expect(failure.cause).toBe(reason); + expect(failure.cancellation?.reason).toBe(reason); + expect(failure.cleanupFailure?.cleanupIncomplete).toBe(true); + fault.blocked = false; + await reader.close(); + expect(failure.cleanupFailure?.cleanupIncomplete).toBe(false); + await expect(access(root)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + controller.abort(reason); + await preparation; + } }, ); @@ -365,9 +396,7 @@ it { signal: controller.signal }, ); try { - await vi.waitFor(async () => - expect(await readFile(readyPath, "utf8")).toBe("ready"), - ); + await waitForCompilerReady(readyPath, pending); controller.abort(reason); const result = await pending; if (result.ok) throw new Error("Expected cancelled preparation"); @@ -384,8 +413,11 @@ it expect(result.error.cleanupIncomplete).toBe(false); } finally { controller.abort(reason); - await pending; - await client.close(); + try { + await pending; + } finally { + await client.close(); + } } }, ); diff --git a/tests/process-global/derivedOperationFailures.test.ts b/tests/process-global/derivedOperationFailures.test.ts index ef0489e0cb..3312d4ffd1 100644 --- a/tests/process-global/derivedOperationFailures.test.ts +++ b/tests/process-global/derivedOperationFailures.test.ts @@ -686,6 +686,7 @@ it.each(["missing", "invalid", "cleanup"])( }); if (leftFailure === "cleanup") expect(result.error.cleanupResources).toContain(leftPath); + expect(owners.size).toBe(leftFailure === "cleanup" ? 2 : 1); expect([...owners].every((owner) => owner.handle.fd >= 0)).toBe(true); close.mockRestore(); diff --git a/tests/process-global/ghidraEndpointLifetime.test.ts b/tests/process-global/ghidraEndpointLifetime.test.ts index 320fdac18e..8ea97201bd 100644 --- a/tests/process-global/ghidraEndpointLifetime.test.ts +++ b/tests/process-global/ghidraEndpointLifetime.test.ts @@ -142,7 +142,8 @@ it.skipIf(process.platform === "win32").each(["valid", "invalid"] as const)( const failure = result.error; if (kind === "invalid") expect(failure.message).toBe("Ghidra TCP endpoint is invalid"); - expect(fault.handles[0]?.fd).not.toBe(-1); + expect(fault.handles).toHaveLength(1); + expect(fault.handles[0]?.fd).toBeGreaterThanOrEqual(0); expect(resources.hasPending).toBe(true); fault.blocked = false; await resources.retryCleanup(); diff --git a/tests/process-global/ghidraImportSnapshotLifetime.test.ts b/tests/process-global/ghidraImportSnapshotLifetime.test.ts index effa887bcd..6ea987d69b 100644 --- a/tests/process-global/ghidraImportSnapshotLifetime.test.ts +++ b/tests/process-global/ghidraImportSnapshotLifetime.test.ts @@ -383,6 +383,8 @@ it("copies every byte across chunk boundaries through the destination owner", as }); expect(Buffer.compare(await readFile(snapshot.path), bytes)).toBe(0); expect(snapshot.sha256).toBe(sha256); + expect(fault.handles.length).toBeGreaterThan(0); + expect(fault.destinations.length).toBeGreaterThan(0); expect( [...fault.handles, ...fault.destinations].every( (handle) => handle.fd === -1, diff --git a/tests/process-global/processObservationCommandFinalization.test.ts b/tests/process-global/processObservationCommandFinalization.test.ts index 878c058497..8b58f0b16c 100644 --- a/tests/process-global/processObservationCommandFinalization.test.ts +++ b/tests/process-global/processObservationCommandFinalization.test.ts @@ -29,19 +29,29 @@ const waitForFile = async (path: string, timeoutMs = 4_000): Promise => { }; const waitForCommandOrFailure = async ( - readyPath: string, + ready: Promise, pending: Promise, ): Promise => { const outcome = await Promise.race([ - waitForFile(readyPath).then(() => undefined), + ready.then(() => ({ ready: true as const })), pending.then( - () => new Error("The command completed before its ready marker"), - (cause: unknown) => cause, + () => ({ ready: false as const, cause: undefined }), + (cause: unknown) => ({ ready: false as const, cause }), ), ]); - if (outcome !== undefined) throw outcome; + if (!outcome.ready) + throw new Error("The command completed before its ready marker", { + cause: outcome.cause, + }); }; +it("rejects an early command failure even when its reason is undefined", async () => { + const ready = new Promise(() => {}); + await expect( + waitForCommandOrFailure(ready, Promise.reject(undefined)), + ).rejects.toBeInstanceOf(Error); +}); + const delayedCommandSource = ( readyPath: string, donePath: string, @@ -92,7 +102,7 @@ const expectOwnedCommandToFinishOnAbort = async ( pending = run(controller.signal, directory); void pending.catch(() => {}); - await waitForCommandOrFailure(command.readyPath, pending); + await waitForCommandOrFailure(waitForFile(command.readyPath), pending); controller.abort(reason); const failure = await pending.then( () => undefined, @@ -251,7 +261,7 @@ const expectDarwinReaderCommandToFinishOnAbort = async ( ? reader.read([processEntry], controller.signal) : reader.identities([processEntry], controller.signal); void pending.catch(() => {}); - await waitForCommandOrFailure(command.readyPath, pending); + await waitForCommandOrFailure(waitForFile(command.readyPath), pending); controller.abort(reason); const failure = await pending.then( () => undefined, diff --git a/tests/process-global/setupConfigurationConflicts.test.ts b/tests/process-global/setupConfigurationConflicts.test.ts index f480181362..defb160b61 100644 --- a/tests/process-global/setupConfigurationConflicts.test.ts +++ b/tests/process-global/setupConfigurationConflicts.test.ts @@ -113,7 +113,7 @@ it.each(["before", "after"] as const)( ).toMatchObject({ status: "failed", reason: "conflict" }); expect(Buffer.compare(await readFile(path), changed)).toBe(0); expect(await readFile(`${path}.rea.backup`, "utf8")).toBe(original); - expect(await readdir(root)).toEqual([ + expect((await readdir(root)).sort()).toEqual([ "config.json", "config.json.rea.backup", ]); @@ -139,7 +139,7 @@ it("preserves an unrelated concurrent edit when a matching recovery backup alrea expect(result).toMatchObject({ status: "failed", reason: "conflict" }); expect(await readFile(path, "utf8")).toBe(changed); expect(await readFile(`${path}.rea.backup`, "utf8")).toBe(original); - expect(await readdir(root)).toEqual([ + expect((await readdir(root)).sort()).toEqual([ "config.json", "config.json.rea.backup", ]); diff --git a/tests/support/analysis-efficiency/test_codex_context.py b/tests/support/analysis-efficiency/test_codex_context.py index 5db8af2075..8bd6d8c8a5 100644 --- a/tests/support/analysis-efficiency/test_codex_context.py +++ b/tests/support/analysis-efficiency/test_codex_context.py @@ -23,7 +23,7 @@ class ManagedCoverageTests(unittest.TestCase): original = {**expected, 'module_ref_count': 0, 'pinvoke_import_count': 2} scope, checks = project(original, expected, {'module_ref_count': 0, 'pinvoke_import_count': 2}) self.assertEqual(scope, expected) - self.assertTrue(all(checks.values())) + self.assertEqual(checks, {'module_ref_count': True, 'pinvoke_import_count': True}) self.assertEqual(original['pinvoke_import_count'], 2) for count in [3, '2', True]: scope, checks = project({**expected, 'pinvoke_import_count': count}, expected, {'pinvoke_import_count': 2}) diff --git a/tests/support/mcp/artifactSession.ts b/tests/support/mcp/artifactSession.ts index ddd8f46118..7fa6b05c0f 100644 --- a/tests/support/mcp/artifactSession.ts +++ b/tests/support/mcp/artifactSession.ts @@ -5,6 +5,17 @@ import { createArtifactBinarySession } from "../../../src/composition/binary.js" import { createServer } from "../../../src/server/createServer.js"; import type { TestMcp } from "./mcpFixture.js"; +/** Register session teardown while retaining its owner for a failed-close retry. */ +export const ownArtifactSession = ( + session: Awaited>, + onTestFinished: (handler: OnTestFinishedHandler) => void, +): void => { + onTestFinished(async () => { + const result = await session.close(); + if (!result.ok) throw result.error; + }); +}; + const openSessionTarget = async ( mcp: TestMcp, onTestFinished: (handler: OnTestFinishedHandler) => void, @@ -12,10 +23,7 @@ const openSessionTarget = async ( path: string, ): Promise => { const session = await createArtifactBinarySession(process.env); - onTestFinished(async () => { - const result = await session.close(); - if (!result.ok) throw result.error; - }); + ownArtifactSession(session, onTestFinished); const client = await mcp.connect(createServer({ kind: "session", session })); const opened = await client.callTool({ name: operation, diff --git a/tests/support/process/processFixture.ts b/tests/support/process/processFixture.ts index 8e6c25a79b..496ab20ec4 100644 --- a/tests/support/process/processFixture.ts +++ b/tests/support/process/processFixture.ts @@ -71,7 +71,13 @@ const stopChild = async ( }; /** Create a child-process registry for a fixture composition root. */ -export const createTestProcessRegistry = (): TestProcesses => { +export const createTestProcessRegistry = ( + spawnChild: ( + command: string, + arguments_: readonly string[], + options: SpawnOptionsWithoutStdio, + ) => ChildProcessWithoutNullStreams = nodeSpawn, +): TestProcesses => { const children = new Set(); const spawn = ( @@ -79,7 +85,7 @@ export const createTestProcessRegistry = (): TestProcesses => { arguments_: readonly string[], options: SpawnOptionsWithoutStdio = {}, ): ChildProcessWithoutNullStreams => { - const child = nodeSpawn(command, arguments_, { + const child = spawnChild(command, arguments_, { ...options, stdio: ["pipe", "pipe", "pipe"], }); @@ -103,20 +109,44 @@ export const createTestProcessRegistry = (): TestProcesses => { let timedOut = false; let killTimeout: NodeJS.Timeout | undefined; + let closeTimeout: NodeJS.Timeout | undefined; + let rejectUnclosed: ((error: Error) => void) | undefined; + const captured = (): ProcessResult => ({ + exitCode: child.exitCode, + signal: child.signalCode, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }); + const timeoutFailure = (): Error => + new Error(`Test process exceeded its ${timeoutMs}ms deadline`, { + cause: captured(), + }); const timeout = timeoutMs === undefined ? undefined : setTimeout(() => { timedOut = true; - child.kill("SIGTERM"); + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGTERM"); + } killTimeout = setTimeout(() => { - child.kill("SIGKILL"); + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + closeTimeout = setTimeout(() => { + // An inherited output pipe may stay open after the leader exits. + rejectUnclosed?.(timeoutFailure()); + child.stdout.destroy(); + child.stderr.destroy(); + }, SHUTDOWN_GRACE_MS); + closeTimeout.unref(); }, SHUTDOWN_GRACE_MS); killTimeout.unref(); }, timeoutMs); timeout?.unref(); const result = await new Promise((resolve, reject) => { + rejectUnclosed = reject; child.once("error", reject); child.once("close", (exitCode, signal) => resolve({ @@ -129,6 +159,7 @@ export const createTestProcessRegistry = (): TestProcesses => { }).finally(() => { if (timeout !== undefined) clearTimeout(timeout); if (killTimeout !== undefined) clearTimeout(killTimeout); + if (closeTimeout !== undefined) clearTimeout(closeTimeout); }); if (timedOut) { throw new Error(`Test process exceeded its ${timeoutMs}ms deadline`, { diff --git a/tests/support/publishedDirectoryCloseFailure.ts b/tests/support/publishedDirectoryCloseFailure.ts new file mode 100644 index 0000000000..3da2a4aaa4 --- /dev/null +++ b/tests/support/publishedDirectoryCloseFailure.ts @@ -0,0 +1,61 @@ +import { stat } from "node:fs/promises"; +import { dirname } from "node:path"; +import { vi } from "vitest"; + +import { SafeOutputTree } from "../../src/artifacts/SafeOutputTree.js"; +import { OwnedFileHandle } from "../../src/filesystem/OwnedFileHandle.js"; + +/** Fail closing the parent directory after publication until retry is allowed. */ +export const injectPublishedDirectoryCloseFailure = async ( + output: string, + persistent = false, +) => { + const parent = await stat(dirname(output)); + const owners: OwnedFileHandle[] = []; + let committing = false; + let injected = false; + let denied = true; + const originalClose = OwnedFileHandle.prototype.close; + const close = vi + .spyOn(OwnedFileHandle.prototype, "close") + .mockImplementation(async function (this: OwnedFileHandle) { + if (committing && this.handle.fd >= 0) { + const identity = await this.handle.stat(); + if (identity.dev === parent.dev && identity.ino === parent.ino) { + owners.push(this); + if (owners.length === 1) { + const handle = this.handle; + const closeHandle = handle.close.bind(handle); + handle.close = async () => { + if (denied) { + injected = true; + if (!persistent) denied = false; + throw new Error("injected published directory close failure"); + } + handle.close = closeHandle; + await closeHandle(); + }; + } + } + } + await originalClose.call(this); + }); + const originalCommit = SafeOutputTree.prototype.commit; + const commit = vi + .spyOn(SafeOutputTree.prototype, "commit") + .mockImplementation(function (this: SafeOutputTree, signal) { + committing = true; + return originalCommit.call(this, signal); + }); + return { + owners, + wasInjected: () => injected, + allowCleanup: () => { + denied = false; + }, + restore: () => { + commit.mockRestore(); + close.mockRestore(); + }, + }; +};