diff --git a/src/application/javascript/JavaScriptRuntimeReconciliationService.test.ts b/src/application/javascript/JavaScriptRuntimeReconciliationService.test.ts index 0ef9b06b81..74dea98147 100644 --- a/src/application/javascript/JavaScriptRuntimeReconciliationService.test.ts +++ b/src/application/javascript/JavaScriptRuntimeReconciliationService.test.ts @@ -1,5 +1,6 @@ import { unknownTarget } from "../../domain/targetIdentity.js"; import { expect, it } from "vitest"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import { createEvidence, parseEvidence, @@ -31,6 +32,41 @@ const reconcile = (observations: readonly Evidence[]) => { }; }; +it("stops resolving retained reconciliation sources when the caller cancels mid-request", () => { + const sources = [ + JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE.static_layers[0]!.analysis, + electronRuntimeEvidence("/synthetic", SOURCE), + ]; + const controller = new AbortController(); + const reason = { request: "cancel runtime reconciliation" }; + const looked: string[] = []; + const result = reconcileJavaScriptRuntimeEvidence( + { + static_layers: [ + { + analysis: { + kind: "retained-evidence", + evidence_id: sources[0]!.evidence_id, + }, + }, + ], + runtime_observations: [ + { kind: "retained-evidence", evidence_id: sources[1]!.evidence_id }, + ], + }, + (id) => { + looked.push(id); + controller.abort(reason); + return sources.find(({ evidence_id }) => evidence_id === id); + }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(looked).toHaveLength(1); +}); + it("keeps equal runtime payloads at distinct selected endpoints separate through service graph Evidence", () => { const first = electronRuntimeEvidence("/synthetic", SOURCE); const second = createEvidence( diff --git a/src/application/v8CodeCache/V8CodeCacheService.test.ts b/src/application/v8CodeCache/V8CodeCacheService.test.ts index f63e58bbd8..7b347fe023 100644 --- a/src/application/v8CodeCache/V8CodeCacheService.test.ts +++ b/src/application/v8CodeCache/V8CodeCacheService.test.ts @@ -1,7 +1,10 @@ import { createHash } from "node:crypto"; import { expect, it } from "vitest"; -import { AnalysisOutputError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, +} from "../../domain/analysisErrorCore.js"; import { parseEvidence } from "../../domain/evidence.js"; import { ok } from "../../domain/result.js"; import { @@ -35,6 +38,26 @@ const inspect = (report: V8CodeCache) => inspect: async () => ok(report), }).inspect({ path: report.artifact.path }); +it("retains a completed V8 code cache observation when the caller cancels as the provider completes", async () => { + const report = reportFor(8); + const controller = new AbortController(); + const reason = { request: "late V8 code cache cancellation" }; + const result = await new V8CodeCacheService({ + identity: { id: "cache-fixture", name: "Cache fixture", version: "1" }, + close: async () => {}, + inspect: async () => { + controller.abort(reason); + return ok(report); + }, + }).inspect({ path: report.artifact.path }, { signal: controller.signal }); + if (result.ok) throw new Error("Expected late cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + const evidence = parseEvidence(result.error.partialObservation); + expect(evidence.operation).toBe("inspect_v8_code_cache"); + expect(evidence.normalized_result).toEqual(report); +}); + it.each([0, 9])( "rejects complete coverage claiming %s scanned bytes for an eight-byte payload", async (scanned_bytes) => { diff --git a/src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts b/src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts index 99c188f678..b34141753d 100644 --- a/src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts +++ b/src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts @@ -2,18 +2,27 @@ import { createHash } from "node:crypto"; import { access, mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { describe, expect, it, vi } from "vitest"; +import { describe, expect, it, onTestFinished } from "vitest"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; -import * as processOutput from "../../process/ExecFileOutput.js"; +import { spawnFailure } from "../../../tests/support/process/spawnFailure.js"; +import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +import { spawnOwnedProviderProcess } from "../../process/ProviderProcess.js"; +import { cleanupOwnedProcessGroup } from "../../process/ProcessOwnership.js"; +import { ArtifactResourceScope } from "../../resources/ArtifactResourceScope.js"; import { encodeNibArchiveFixture } from "./NibArchive.fixture.js"; import type { analyzeInterfaceBuilderBundle } from "./InterfaceBuilderAnalysis.js"; import { ArtifactReaderFailure } from "../../resources/ArtifactReader.js"; -import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisError, + primaryAnalysisError, +} from "../../domain/analysisErrorBase.js"; +import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { analyzeAppleAssetCatalogs, collectInterfaceBuilderResourceKeys, + type AssetUtilLauncher, } from "./AppleAssetCatalogAnalysis.js"; import { waitFor } from "../../../tests/support/testWait.js"; import { @@ -21,6 +30,72 @@ import { invariantViolation, } from "../../domain/internalInvariant.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; + +/** + * Run a stand-in Node script as assetutil under the real owned-process + * launcher. The script reads its own arguments first, then `--info `. + */ +const standInAssetUtil = ( + script: string, + ...scriptArguments: string[] +): AssetUtilLauncher & { readonly pids: number[] } => { + const pids: number[] = []; + const launcher: AssetUtilLauncher = async (options) => { + const spawned = await spawnOwnedProviderProcess({ + ...options, + command: process.execPath, + arguments: ["-e", script, "--", ...scriptArguments, ...options.arguments], + expectedCommand: process.execPath, + }); + if (spawned.process.pid !== undefined) pids.push(spawned.process.pid); + return spawned; + }; + return Object.assign(launcher, { pids }); +}; + +const alive = (pid: number): boolean => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +/** A completed assetutil run the owned command reports as failed. */ +const assetUtilCommandFailure = ( + reason: OwnedCommandFailure["reason"], + output: { + readonly stdout?: string; + readonly stderr: string; + readonly exitCode: number | null; + readonly signal: NodeJS.Signals | null; + readonly truncated?: boolean; + readonly stopRequested?: boolean; + readonly collectionFailure?: boolean; + }, +): OwnedCommandFailure => { + const stream = (text: string) => ({ + text, + bytes: Buffer.byteLength(text), + observedBytes: Buffer.byteLength(text), + }); + return new OwnedCommandFailure( + reason, + "Command failed; inspect its exit status and diagnostics.", + { + stdout: stream(output.stdout ?? ""), + stderr: stream(output.stderr), + exitCode: output.exitCode, + signal: output.signal, + ...(output.truncated === true ? { diagnosticTruncated: true } : {}), + ...(output.stopRequested === true ? { stopRequested: true } : {}), + }, + null, + { collectionFailure: output.collectionFailure === true }, + ); +}; describe("Apple asset catalog traversal coverage", () => { it.for([false, true])( @@ -433,16 +508,7 @@ describe("Apple asset catalog and Interface Builder concurrency", () => { const ready = join(work, "ready"); const exited = join(work, "exited"); const script = `const fs=require("node:fs");process.on("SIGTERM",()=>{setTimeout(()=>{fs.writeFileSync(process.argv[2],"exited");process.exit(0)},500)});fs.writeFileSync(process.argv[1],"ready");setInterval(()=>{},1000);`; - const realExecFileOutput = processOutput.execFileOutput; - const spy = vi - .spyOn(processOutput, "execFileOutput") - .mockImplementation((_command, _args, options) => - realExecFileOutput( - process.execPath, - ["-e", script, ready, exited], - options, - ), - ); + const launcher = standInAssetUtil(script, ready, exited); const controller = new AbortController(); let settled: Promise | undefined; try { @@ -452,21 +518,247 @@ describe("Apple asset catalog and Interface Builder concurrency", () => { targetSha256: "a".repeat(64), signal: controller.signal, analyzeInterfaceBuilder: async () => emptyGraph, + assetUtilLauncher: launcher, platform: "darwin", }).catch((cause: unknown) => cause); await waitFor(() => expect(access(ready)).resolves.toBeUndefined()); controller.abort("stop asset inspection"); - expect(await settled).toMatchObject({ reason: "cancelled" }); - await expect(access(exited)).resolves.toBeUndefined(); + expect(await settled).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: "stop asset inspection", + }); + expect(launcher.pids).toHaveLength(1); + expect(launcher.pids.filter(alive)).toEqual([]); } finally { controller.abort("stop asset inspection"); await settled; - spy.mockRestore(); } }, ); }); +it.skipIf(process.platform === "win32")( + "reports an assetutil that cannot be launched as unavailable", + async () => { + const bundle = await createTestTempDirectory("rea-assetutil-launch-"); + await mkdir(join(bundle, "Contents", "Resources"), { recursive: true }); + await writeFile( + join(bundle, "Contents", "Resources", "Assets.car"), + "catalog", + ); + const missing = join( + await createTestTempDirectory("rea-assetutil-missing-"), + "assetutil", + ); + let launches = 0; + await expect( + analyzeAppleAssetCatalogs({ + environment: {}, + bundlePath: bundle, + targetSha256: "a".repeat(64), + analyzeInterfaceBuilder: async () => + ({ + graph: { nodes: [], truncated: false, coverage: [] }, + }) as unknown as Awaited< + ReturnType + >, + assetUtilLauncher: (options) => { + launches++; + return spawnOwnedProviderProcess({ ...options, command: missing }); + }, + platform: "darwin", + }), + ).rejects.toMatchObject({ + _tag: "AnalysisCapabilityUnavailableError", + reason: "macOS assetutil is unavailable", + cause: { code: "ENOENT" }, + diagnostics: { catalog_path: "Contents/Resources/Assets.car" }, + }); + expect(launches).toBe(1); + }, +); + +it.each([ + ["EAGAIN", "processes"], + ["EMFILE", "file-descriptors"], +])( + "reports a host that lacks capacity to launch assetutil (%s) as a retryable constraint", + async (code, resource) => { + const bundle = await createTestTempDirectory("rea-assetutil-capacity-"); + await mkdir(join(bundle, "Contents", "Resources"), { recursive: true }); + await writeFile( + join(bundle, "Contents", "Resources", "Assets.car"), + "catalog", + ); + await expect( + analyzeAppleAssetCatalogs({ + environment: {}, + bundlePath: bundle, + targetSha256: "a".repeat(64), + analyzeInterfaceBuilder: async () => + ({ + graph: { nodes: [], truncated: false, coverage: [] }, + }) as unknown as Awaited< + ReturnType + >, + assetUtilLauncher: async () => { + throw spawnFailure(code, "/usr/bin/assetutil"); + }, + platform: "darwin", + }), + ).rejects.toMatchObject({ + _tag: "AnalysisResourceConstraintError", + retryable: true, + reportedLimits: { spawn_error: code, exhausted_resource: resource }, + diagnostics: { + catalog_path: "Contents/Resources/Assets.car", + boundary: "host-capacity", + system_code: code, + }, + }); + }, +); + +describe("assetutil process cleanup", () => { + /** + * A stand-in assetutil that completes, while its process-group cleanup + * proof is withheld until `allowCleanup`. + */ + const withheldCleanup = async (onRefusedCleanup?: () => void) => { + const bundle = await createTestTempDirectory("rea-assetutil-cleanup-"); + await writeFile(join(bundle, "Assets.car"), "catalog"); + let mayClean = false; + let launches = 0; + const standIn = standInAssetUtil('process.stdout.write("[]");'); + const resources = new ArtifactResourceScope(); + onTestFinished(async () => { + mayClean = true; + await resources.close(); + }); + const inspect = (signal?: AbortSignal) => + analyzeAppleAssetCatalogs({ + ...(signal === undefined ? {} : { signal }), + environment: {}, + bundlePath: bundle, + targetSha256: "a".repeat(64), + resources, + analyzeInterfaceBuilder: async () => + ({ + graph: { nodes: [], truncated: false, coverage: [] }, + }) as unknown as Awaited< + ReturnType + >, + assetUtilLauncher: async (options) => { + launches++; + const spawned = await standIn(options); + return { + ...spawned, + cleanup: async () => { + if (mayClean) return cleanupOwnedProcessGroup(spawned.ownership); + onRefusedCleanup?.(); + return { + cleaned: false, + reason: "assetutil process owner remains unverified", + }; + }, + }; + }, + platform: "darwin", + }); + return { + inspect, + resources, + launches: () => launches, + allowCleanup: () => { + mayClean = true; + }, + }; + }; + + it.skipIf(process.platform === "win32")( + "retains an unproven assetutil process cleanup for a later retry", + async () => { + const f = await withheldCleanup(); + const failure = await f.inspect().catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisError)) throw failure; + // The run itself completed; only its process cleanup is unproven. + expect(failure).toBeInstanceOf(ProviderCleanupError); + expect(failure.cleanupIncomplete).toBe(true); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "cleanup_incomplete", + details: { + operation: "inspect_asset_catalog", + // The completed run's output is kept with its cleanup report. + captured_output: { stdout: "[]", exit_code: 0 }, + resources: expect.arrayContaining([ + expect.stringMatching(/^process-group:\d+$/u), + ]), + diagnostics: { + reason: "assetutil process owner remains unverified", + }, + }, + }); + expect(failure.cleanupResources).toContainEqual( + expect.stringMatching(/^process-group:\d+$/u), + ); + expect(f.resources.hasPending).toBe(true); + expect(f.resources.cleanupResources).toEqual(failure.cleanupResources); + }, + ); + + it.skipIf(process.platform === "win32")( + "keeps a cancellation during refused assetutil cleanup through the retry", + async () => { + const controller = new AbortController(); + const reason = { caller: "stop during assetutil cleanup" }; + const f = await withheldCleanup(() => controller.abort(reason)); + const failure = await f + .inspect(controller.signal) + .catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisError)) throw failure; + // The caller's cancellation is the outcome; the unproven cleanup is + // retained beside it rather than replacing it. + const cancelled = primaryAnalysisError(failure); + expect(cancelled).toBeInstanceOf(AnalysisCancelledError); + expect(cancelled.cause).toBe(reason); + expect(projectAnalysisError(failure)).toMatchObject({ + category: "cancelled", + details: { cleanup: "incomplete" }, + }); + expect(failure.cleanupIncomplete).toBe(true); + expect(f.resources.hasPending).toBe(true); + f.allowCleanup(); + await f.resources.close(); + expect(failure.cleanupIncomplete).toBe(false); + expect(f.resources.hasPending).toBe(false); + expect(primaryAnalysisError(failure)).toBe(cancelled); + }, + ); + + it.skipIf(process.platform === "win32")( + "releases a retained assetutil cleanup before the next inspection launches", + async () => { + const f = await withheldCleanup(); + const first = await f.inspect().catch((cause: unknown) => cause); + if (!(first instanceof AnalysisError)) throw first; + expect(first.cleanupIncomplete).toBe(true); + // While the retained owner still cannot be proven stopped, no new + // assetutil process is admitted. + await expect(f.inspect()).rejects.toMatchObject({ + reason: "unavailable", + cleanup: { resources: first.cleanupResources }, + }); + expect(f.launches()).toBe(1); + f.allowCleanup(); + const retried = await f.inspect(); + expect(retried.catalogs).toHaveLength(1); + expect(f.launches()).toBe(2); + expect(first.cleanupIncomplete).toBe(false); + expect(f.resources.hasPending).toBe(false); + }, + ); +}); + it("joins the Interface Builder cleanup failure to a catalog defect", async () => { const bundle = await createTestTempDirectory("rea-asset-catalog-defect-"); await mkdir(join(bundle, "Contents", "Resources"), { recursive: true }); @@ -523,20 +815,18 @@ it.each(["not json", "{}", "[null]"])( }, ); it.each([ - ["ETIMEDOUT", "AnalysisTimeoutError"], - ["ENOENT", "AnalysisCapabilityUnavailableError"], - ["ERR_CHILD_PROCESS_STDIO_MAXBUFFER", "AnalysisResourceConstraintError"], - [1, "AnalysisOutputError"], + ["timeout", "AnalysisTimeoutError"], + ["output-limit", "AnalysisResourceConstraintError"], + ["process", "AnalysisOutputError"], ] as const)( "preserves assetutil command failure %s and diagnostics", - async (code, tag) => { + async (reason, tag) => { const bundle = await createTestTempDirectory("rea-asset-command-"); await writeFile(join(bundle, "Assets.car"), "inert catalog"); - const cause = Object.assign(new Error("native refusal"), { - code, - stdout: "", + const cause = assetUtilCommandFailure(reason, { stderr: "specific catalog diagnostic", - signal: code === "ETIMEDOUT" ? "SIGTERM" : null, + exitCode: reason === "process" ? 1 : null, + signal: reason === "timeout" ? "SIGKILL" : null, }); await expect( analyzeAppleAssetCatalogs({ @@ -552,8 +842,8 @@ it.each([ cause, diagnostics: { catalog_path: "Assets.car" }, capturedOutput: { stderr: "specific catalog diagnostic" }, - ...(code === "ETIMEDOUT" ? { timeoutMs: 60000 } : {}), - ...(code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER" + ...(reason === "timeout" ? { timeoutMs: 60000 } : {}), + ...(reason === "output-limit" ? { fixedAllowance: { limit: 64 * 1024 * 1024, unit: "bytes" } } : {}), }); @@ -582,10 +872,10 @@ it.each(["reported carutil", "tab-indented carutil", "observed assetutil"])( ? `assetutil Error: Unable to read file at '${path}'\n` + `carutil: error:\n${variant === "tab-indented carutil" ? "\t" : " "}${path} is not a valid AssetCatalog\n` : `assetutil: file error error exiting: 'can't read from: '${path}' Undefined error: 0' invalid file\n`); - const cause = Object.assign(new Error("assetutil exited 1"), { - code: 1, + const cause = assetUtilCommandFailure("process", { stdout: "partial metadata", stderr, + exitCode: 1, signal: null, }); const error = await analyzeAppleAssetCatalogs({ @@ -641,12 +931,11 @@ it.each([ : scenario.startsWith("different file-error") ? `assetutil: file error error exiting: 'can't read from: '${path}' Undefined error: ${scenario === "different file-error code" ? "13" : "0"}' ${scenario === "different file-error text" ? "access denied" : "invalid file"}\n` : `carutil: error:\n${scenario.startsWith("tab-indented") ? "\t" : " "}${rejectedPath} is not a valid AssetCatalog\n`; - const cause = Object.assign(new Error("assetutil failed"), { - code: 1, - stdout: "", + const cause = assetUtilCommandFailure("process", { stderr, + exitCode: scenario === "signal" ? null : 1, signal: scenario === "signal" ? "SIGTERM" : null, - outputTruncated: scenario === "truncated capture", + truncated: scenario === "truncated capture", collectionFailure: scenario === "failed capture", }); await expect( @@ -768,17 +1057,8 @@ it.skipIf(process.platform === "win32")( targetSha256: "a".repeat(64), environment: {}, signal: controller.signal, - runAssetUtil: async (_path, signal) => - ( - await processOutput.execFileOutput( - process.execPath, - ["-e", program], - { - signal, - stopSignal: "SIGTERM", - }, - ) - ).stdout, + assetUtilLauncher: standInAssetUtil(program), + platform: "darwin", }) .catch((cause: unknown) => cause) .finally(() => { diff --git a/src/artifacts/apple/AppleAssetCatalogAnalysis.ts b/src/artifacts/apple/AppleAssetCatalogAnalysis.ts index 4766d40222..790794c495 100644 --- a/src/artifacts/apple/AppleAssetCatalogAnalysis.ts +++ b/src/artifacts/apple/AppleAssetCatalogAnalysis.ts @@ -5,13 +5,11 @@ import { } from "../../domain/analysisErrorBase.js"; import { AnalysisOutputError, - AnalysisTimeoutError, AnalysisCapabilityUnavailableError, - AnalysisResourceConstraintError, OwnerCancellation, } from "../../domain/analysisErrorCore.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { lstat } from "node:fs/promises"; import { streamChunkToBuffer, @@ -41,16 +39,38 @@ import { type AppleAssetAnalysisCoverage, } from "../../domain/apple/appleAssetCatalog.js"; import { - ExecFileOutputCancelledError, - execFileOutput, - execFileOutputFailure, - execFileFailureCapturedOutput, -} from "../../process/ExecFileOutput.js"; + prepareOwnedCommand, + runOwnedCommand, +} from "../../process/OwnedCommand.js"; +import type { + OwnedProviderProcessSpawnOptions, + SpawnedOwnedProviderProcess, +} from "../../process/ProviderProcess.js"; +import { providerOutcomeOf } from "../../process/ProviderOutcome.js"; +import { describeLaunchFailure } from "../../process/launchFailure.js"; +import { + classifyProviderOutcome, + type ProviderOutcomeContext, +} from "../../process/classifyProviderOutcome.js"; +import { capturedProviderOutput } from "../../process/capturedProviderOutput.js"; +import { + processInspectionCancellationError, + processInspectionFailure, + retainArtifactCommandCleanup, + retainProcessInspectionCleanup, +} from "../../providers/OwnedCommandCleanup.js"; const ASSETUTIL = "/usr/bin/assetutil"; +const ASSETUTIL_TIMEOUT_MS = 60_000; +const OPERATION = "inspect_asset_catalog"; const MAX_CATALOG_BYTES = 64 * 1024 * 1024; const MAX_TOTAL_METADATA_BYTES = 64 * 1024 * 1024; +/** Launches assetutil under owned supervision; tests substitute a stand-in process. */ +export type AssetUtilLauncher = ( + options: OwnedProviderProcessSpawnOptions, +) => Promise; + /** Inspect compiled asset catalog metadata without extracting rendition bytes. */ export const analyzeAppleAssetCatalogs = async (input: { readonly bundlePath: string; @@ -60,10 +80,12 @@ export const analyzeAppleAssetCatalogs = async (input: { readonly page?: unknown; readonly signal?: AbortSignal; readonly resources?: ArtifactResourceScope; + /** Produce one catalog's metadata without a process; tests of parsing only. */ readonly runAssetUtil?: ( path: string, signal?: AbortSignal, ) => Promise; + readonly assetUtilLauncher?: AssetUtilLauncher; readonly analyzeInterfaceBuilder?: typeof analyzeInterfaceBuilderBundle; /** The host whose assetutil runs; tests select macOS on another host. */ readonly platform?: NodeJS.Platform; @@ -212,6 +234,7 @@ const collectAssetCatalogs = async (input: { path: string, signal?: AbortSignal, ) => Promise; + readonly assetUtilLauncher?: AssetUtilLauncher; }) => { const reader = new DirectoryArtifactReader(input.bundlePath); const catalogs: { @@ -256,7 +279,14 @@ const collectAssetCatalogs = async (input: { entry, runAssetUtil: input.runAssetUtil ?? - ((path, signal) => runAssetUtil(path, input.environment, signal)), + ((path, signal) => + runAssetUtil( + path, + input.environment, + signal, + input.assetUtilLauncher, + )), + resources: input.resources, ...(input.signal === undefined ? {} : { signal: input.signal }), }); totalMetadataBytes += inspected.metadataBytes; @@ -325,6 +355,7 @@ const inspectAssetCatalogEntry = async (input: { path: string, signal?: AbortSignal, ) => Promise; + readonly resources: ArtifactResourceScope; readonly signal?: AbortSignal; }) => { const { entry, reader, signal } = input; @@ -343,7 +374,7 @@ const inspectAssetCatalogEntry = async (input: { try { output = await input.runAssetUtil(entry.adapterKey, signal); } catch (cause) { - throw assetUtilFailure(cause, entry); + throw assetUtilFailure(cause, entry, input.resources); } const after = await lstat(entry.adapterKey, { bigint: true }); if ( @@ -426,103 +457,156 @@ export const collectInterfaceBuilderResourceKeys = ( ]); }); +/** + * Run assetutil under owned supervision and return its complete output. The + * macOS ownership helper is prepared first, under its own compile allowance, + * so a cold helper build does not spend assetutil's deadline. Each stream + * keeps its own 64 MiB allowance. + */ const runAssetUtil = async ( path: string, environment: Readonly, signal?: AbortSignal, -): Promise => - ( - await execFileOutput(ASSETUTIL, ["--info", path], { - env: environment, - timeout: 60_000, - maxBuffer: MAX_CATALOG_BYTES, - stopSignal: "SIGTERM", + launcher?: AssetUtilLauncher, +): Promise => { + await prepareOwnedCommand(signal); + const snapshot = await runOwnedCommand( + { + command: ASSETUTIL, + arguments: ["--info", path], + runId: `rea-assetutil-${randomUUID()}`, + hostEnvironment: { ...environment }, + }, + { + timeoutMs: ASSETUTIL_TIMEOUT_MS, + diagnosticBytes: { stdout: MAX_CATALOG_BYTES, stderr: MAX_CATALOG_BYTES }, + }, + { ...(signal === undefined ? {} : { signal }), - }) - ).stdout; + ...(launcher === undefined ? {} : { launcher }), + }, + ); + return snapshot.stdout.text; +}; -/** Preserve the command owner's failure and captured output before artifact adaptation. */ -const assetUtilFailure = (cause: unknown, entry: ArtifactEntry): unknown => { +/** + * Classify one assetutil run through the shared command outcome owner, and + * transfer any unresolved process or helper cleanup to the caller's scope, + * which retries it before the next inspection and on close. + */ +const assetUtilFailure = ( + cause: unknown, + entry: ArtifactEntry, + resources: ArtifactResourceScope, +): unknown => { if (cause instanceof AnalysisError) return cause; - if (cause instanceof ExecFileOutputCancelledError) - return new ArtifactReaderFailure( - "cancelled", - "Asset catalog inspection cancelled", - { cause: cause.cause }, + const outcome = providerOutcomeOf(cause); + // A command that exited 0 failed only because its cleanup is unproven. + if ( + outcome?.kind === "exited" && + outcome.exitCode === 0 && + outcome.signal === null + ) { + // The completed run's output stays with the cleanup report that replaces it. + const [cleanup, ...more] = retainArtifactCommandCleanup( + cause, + resources, + OPERATION, + outcome.output === undefined + ? undefined + : { capturedOutput: capturedProviderOutput(outcome.output.snapshot) }, ); - const output = execFileOutputFailure(cause); - const code = - cause instanceof Error && "code" in cause ? cause.code : undefined; - const capturedOutput: AnalysisCapturedOutput | undefined = - output === undefined ? undefined : execFileFailureCapturedOutput(output); - const options = { - cause, - capturedOutput, - diagnostics: { catalog_path: entry.path }, - }; - if (code === "ETIMEDOUT") - return new AnalysisTimeoutError("inspect_asset_catalog", 60_000, options); - if (code === "ENOENT") - return new AnalysisCapabilityUnavailableError( - "artifact", - "inspect_asset_catalog", - "macOS assetutil is unavailable", - options, - ); - if (code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER") - return new AnalysisResourceConstraintError( - "inspect_asset_catalog", - "transport", - "assetutil output exceeds the 64 MiB capture allowance", - { maximum_output_bytes: MAX_CATALOG_BYTES }, + if (cleanup !== undefined) + return more.reduce( + (primary, failure) => + analysisErrorWithCleanupDependencies(primary, failure, OPERATION), + cleanup, + ); + } + const primary = + processInspectionCancellationError(cause, OPERATION) ?? + (outcome !== undefined + ? classifyProviderOutcome(outcome, assetUtilOutcomeContext(entry)) + : processInspectionFailure(cause) !== undefined + ? new ProviderAdapterError(ARTIFACT_GRAPH_PROVIDER.id, OPERATION, { + cause, + diagnostics: { catalog_path: entry.path }, + }) + : undefined); + if (primary === undefined) return cause; + return retainProcessInspectionCleanup(cause, primary, resources, OPERATION); +}; + +/** What an assetutil outcome means; owner-observed facts are shared. */ +const assetUtilOutcomeContext = ( + entry: ArtifactEntry, +): ProviderOutcomeContext => ({ + providerId: ARTIFACT_GRAPH_PROVIDER.id, + operation: OPERATION, + capturedOutput: capturedProviderOutput, + details: { diagnostics: { catalog_path: entry.path } }, + timeout: { ms: ASSETUTIL_TIMEOUT_MS }, + outputLimit: { + reason: "assetutil output exceeds the 64 MiB capture allowance", + limits: { maximum_output_bytes: MAX_CATALOG_BYTES }, + recovery: fixedAllowanceOptions( + "The assetutil capture allowance", + "capture", { - ...options, - ...fixedAllowanceOptions("The assetutil capture allowance", "capture", { - limit: MAX_CATALOG_BYTES, - unit: "bytes", - }), + limit: MAX_CATALOG_BYTES, + unit: "bytes", }, - ); - if (output !== undefined) { + ), + }, + launchUnavailable: (failure, details) => + new AnalysisCapabilityUnavailableError( + "artifact", + OPERATION, + failure.kind === "missing" + ? "macOS assetutil is unavailable" + : `macOS assetutil could not be launched (${describeLaunchFailure(failure)})`, + details, + ), + dialect: (outcome, details) => { // assetutil's explicit input rejection is distinct from unusable tool // output. Match its complete diagnostic for this exact selected path; // other text, interrupted runs and incomplete captures establish no cause. + const stderr = outcome.output?.snapshot.stderr.text ?? ""; const rejectsCatalog = [ `carutil: error:\n ${entry.adapterKey} is not a valid AssetCatalog`, `carutil: error:\n\t${entry.adapterKey} is not a valid AssetCatalog`, `assetutil: file error error exiting: 'can't read from: '${entry.adapterKey}' Undefined error: 0' invalid file`, ].some((rejection) => { - const offset = output.stderr.indexOf(rejection); + const offset = stderr.indexOf(rejection); const end = offset + rejection.length; return ( offset >= 0 && - (offset === 0 || output.stderr[offset - 1] === "\n") && - (end === output.stderr.length || output.stderr[end] === "\n") + (offset === 0 || stderr[offset - 1] === "\n") && + (end === stderr.length || stderr[end] === "\n") ); }); if ( - output.code === 1 && - output.signal === null && - !output.killed && - !output.outputTruncated && - output.collectionFailure !== true && + outcome.kind === "exited" && + outcome.exitCode === 1 && + outcome.signal === null && + !outcome.stopRequested && + outcome.output?.truncated === false && rejectsCatalog ) return new ArtifactOperationError( - "inspect_asset_catalog", + OPERATION, "format", undefined, `assetutil rejected ${entry.path} as an invalid asset catalog`, - options, + details, ); return new AnalysisOutputError( - "inspect_asset_catalog", + OPERATION, `assetutil did not produce usable catalog metadata for ${entry.path}`, - options, + details, ); - } - return cause; -}; + }, +}); const digestBounded = async ( reader: DirectoryArtifactReader, diff --git a/src/browser/modules/NativeModuleResolver.test.ts b/src/browser/modules/NativeModuleResolver.test.ts index e479cf6b2d..b1de363488 100644 --- a/src/browser/modules/NativeModuleResolver.test.ts +++ b/src/browser/modules/NativeModuleResolver.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { NativeModuleResolver, type ModuleResolverLauncher, @@ -7,52 +7,60 @@ import { webModuleResolutionObservationSchema } from "../../domain/webModuleTrac import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { completedModuleResolverLauncher } from "../../../tests/fixtures/webModuleTrace.js"; +import { + controlledCleanup, + phaseGate, +} from "../../../tests/support/lifecycleHarness.js"; const input = { importerUrl: "https://app.test/main.js", importMap: null, specifiers: ["./dep.js"], }; -describe("native module resolver cleanup ownership", () => { - it.each([false, true])( - "awaits owned cleanup when acquisition is cancelled (cleanup fails: %s)", - async (cleanupFails) => { - const controller = new AbortController(); - let closed = false; - const provider = new NativeModuleResolver( - { REA_BROWSER_EXECUTABLE: "/browser" }, - () => { - controller.abort(); - return Promise.resolve({ - version: () => "fixture", - newContext: () => { - throw new Error("must not acquire context"); - }, - close: () => { - closed = true; - if (cleanupFails) - return Promise.reject(new Error("fixture cleanup refusal")); - return Promise.resolve(); - }, - }); +/** The browser connection completes after the caller cancelled acquisition. */ +const resolveWithLateBrowserConnection = async ( + cleanupFails: boolean, +): Promise => { + const controller = new AbortController(); + let closed = false; + const provider = new NativeModuleResolver( + { REA_BROWSER_EXECUTABLE: "/browser" }, + () => { + controller.abort(); + return Promise.resolve({ + version: () => "fixture", + newContext: () => { + throw new Error("must not acquire context"); + }, + close: () => { + closed = true; + if (cleanupFails) + return Promise.reject(new Error("fixture cleanup refusal")); + return Promise.resolve(); }, - ); - const result = await provider.resolve(input, { - signal: controller.signal, }); - expect(closed).toBe(true); - if (result.ok) throw new Error("expected cancelled result"); - const projected = projectAnalysisError(result.error); - expect(projected.code).toBe( - cleanupFails ? "cleanup_incomplete" : "cancelled", - ); - if (cleanupFails) { - expect(projected.details).toMatchObject({ - diagnostics: { primary_error: { code: "cancelled" } }, - }); - } }, ); + const result = await provider.resolve(input, { + signal: controller.signal, + }); + expect(closed).toBe(true); + if (result.ok) throw new Error("expected cancelled result"); + const projected = projectAnalysisError(result.error); + expect(projected.code).toBe( + cleanupFails ? "cleanup_incomplete" : "cancelled", + ); + if (cleanupFails) { + expect(projected.details).toMatchObject({ + diagnostics: { primary_error: { code: "cancelled" } }, + }); + } +}; +describe("native module resolver cleanup ownership", () => { + it("closes a browser whose connection completes after cancellation", () => + resolveWithLateBrowserConnection(false)); + it("reports failed cleanup of a browser whose connection completes after cancellation", () => + resolveWithLateBrowserConnection(true)); it("returns cleanup_incomplete when owned cleanup fails after an engine error", async () => { let launches = 0; let closes = 0; @@ -107,6 +115,58 @@ describe("native module resolver cleanup ownership", () => { }); }); +describe("native module resolver cleanup retry", () => { + it("retries retained browser cleanup before launching the next resolution", async () => { + let launches = 0; + const cleanup = controlledCleanup({ + refused: true, + failure: new Error("fixture owned cleanup refusal"), + }); + const retriedClose = phaseGate(); + const completed = completedModuleResolverLauncher(input, async () => {}); + const provider = new NativeModuleResolver( + { REA_BROWSER_EXECUTABLE: "/browser" }, + async (...args) => { + launches += 1; + if (launches > 1) return completed(...args); + return { + version: () => "fixture", + newContext: () => + Promise.reject(new Error("fixture context failure")), + close: () => cleanup.close(() => retriedClose.hold(launches)), + }; + }, + ); + let retrying: ReturnType | undefined; + // Release the held close and join the gated request even when an + // assertion fails while it is held. + onTestFinished(async () => { + cleanup.allow(); + retriedClose.release(); + await retrying?.catch(() => undefined); + await provider.close(); + }); + const first = await provider.resolve(input); + if (first.ok) throw new Error("expected retained browser cleanup"); + expect(first.error.cleanupIncomplete).toBe(true); + cleanup.allow(); + retrying = provider.resolve(input); + // The retained close is still running: no replacement browser yet. + expect( + await retriedClose.reached(retrying, "retained browser close retry"), + ).toBe(1); + await new Promise((resolve) => setImmediate(resolve)); + expect(launches).toBe(1); + retriedClose.release(); + const retried = await retrying; + if (!retried.ok) throw retried.error; + expect(cleanup.attempts).toBe(2); + expect(launches).toBe(2); + expect(first.error.cleanupIncomplete).toBe(false); + expect(first.error.cleanupResources).toEqual([]); + }); +}); + describe("native module resolver shutdown admission", () => { it("awaits acquisition and cleanup while refusing queued work during shutdown", async () => { let acquired: () => void = () => { diff --git a/src/hopper/HopperCleanup.ts b/src/hopper/HopperCleanup.ts index 90335ce9e2..ec8ff93ac0 100644 --- a/src/hopper/HopperCleanup.ts +++ b/src/hopper/HopperCleanup.ts @@ -25,7 +25,7 @@ import { import type { HopperRequestActivity } from "./HopperRequestQueue.js"; import { parseHopperShutdownReply } from "./HopperReplyContracts.js"; import { errorMessage } from "../kernel/errorMessage.js"; -import { retainProcessInspectionOwners } from "../providers/OwnedCommandCleanup.js"; +import { retainArtifactCommandCleanup } from "../providers/OwnedCommandCleanup.js"; import type { ArtifactResourceScope } from "../resources/index.js"; interface CleanupIssue { @@ -194,11 +194,8 @@ export const confirmExternalApplicationExit = async ( } catch (cause: unknown) { // Exit stays unverified; descriptors the read left open stay owned. if ( - retainProcessInspectionOwners( - cause, - resources.inspections, - "close_binary", - ).length === 0 + retainArtifactCommandCleanup(cause, resources.inspections, "close_binary") + .length === 0 ) throw cause; return "unverified"; diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index 05fd05be63..aee03001ba 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -75,7 +75,7 @@ import { ProviderProcessSupervisor, } from "../process/ProviderProcess.js"; import { observeProcessStartIdentity } from "../process/ProcessOwnershipObservation.js"; -import { retainProcessInspectionOwners } from "../providers/OwnedCommandCleanup.js"; +import { retainArtifactCommandCleanup } from "../providers/OwnedCommandCleanup.js"; import { ArtifactResourceScope } from "../resources/index.js"; import type { BridgeLaunch, BridgeLauncher } from "./BridgeLauncher.js"; import { hopperOperationStage } from "./HopperOperationStage.js"; @@ -707,7 +707,7 @@ export class HopperClient< } catch (cause: unknown) { // The identity stays unavailable; descriptors the read left open stay // owned until session cleanup releases them. - retainProcessInspectionOwners( + retainArtifactCommandCleanup( cause, this.#resources.inspections, "open_binary", diff --git a/src/inspector/V8InspectorProvider.handshake.test.ts b/src/inspector/V8InspectorProvider.handshake.test.ts new file mode 100644 index 0000000000..011110ed38 --- /dev/null +++ b/src/inspector/V8InspectorProvider.handshake.test.ts @@ -0,0 +1,154 @@ +import { createServer } from "node:http"; +import type { Duplex } from "node:stream"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { expect, it, onTestFinished } from "vitest"; +import { WebSocketServer } from "ws"; +import { closeVerifierResources } from "../../scripts/lib/verifier-cleanup.mjs"; +import { awaitAcknowledgement } from "../../tests/fixtures/lifecycleAcknowledgement.js"; +import { phaseGate } from "../../tests/support/lifecycleHarness.js"; + +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { observeJavaScriptRuntime } from "../application/javascript/JavaScriptRuntimeObservationService.js"; +import { observeJavaScriptRuntimeInputSchema } from "../domain/javascript/javascriptRuntimeObservation.js"; +import { + V8InspectorAdmission, + V8InspectorProvider, +} from "./V8InspectorProvider.js"; + +/** The Inspector commands a zero-length observation sends; others are refused. */ +const MODELED_INSPECTOR_COMMANDS = new Set([ + "Runtime.enable", + "Debugger.enable", +]); + +it("cancels a held inspector WebSocket handshake and admits the next observation", async () => { + const root = await mkdtemp(join(tmpdir(), "rea-inspector-handshake-")); + const scriptPath = join(root, "target.js"); + await writeFile(scriptPath, "globalThis.fixture = true;\n"); + let endpoint = ""; + const server = createServer((request, response) => { + response.setHeader("content-type", "application/json"); + if (request.url === "/json/version") + response.end( + JSON.stringify({ + Browser: "Node.js/v24.18.0", + "Protocol-Version": "1.3", + "V8-Version": "fixture", + }), + ); + else if (request.url === "/json/list") + response.end( + JSON.stringify([ + { + id: "fixture", + type: "node", + url: pathToFileURL(scriptPath).href, + attached: false, + webSocketDebuggerUrl: endpoint.replace("http:", "ws:") + "/fixture", + }, + ]), + ); + else response.writeHead(404).end(); + }); + const sockets = new WebSocketServer({ noServer: true }); + sockets.on("connection", (socket) => { + socket.on("message", (raw) => { + const command = JSON.parse(raw.toString()) as { + id: number; + method: string; + }; + // Only the commands this fixture models succeed; any other Inspector + // call is refused, so it cannot pass as a successful observation. + socket.send( + JSON.stringify( + MODELED_INSPECTOR_COMMANDS.has(command.method) + ? { id: command.id, result: {} } + : { + id: command.id, + error: { + code: -32601, + message: `Unmodeled Inspector command: ${command.method}`, + }, + }, + ), + ); + }); + }); + // The first upgrade is held unanswered; later upgrades complete normally. + const handshake = phaseGate(); + let upgrades = 0; + server.on("upgrade", (request, socket, head) => { + upgrades += 1; + if (upgrades === 1) { + // Flowing, so the client's close is observed. + socket.resume(); + void handshake.hold(socket); + return; + } + sockets.handleUpgrade(request, socket, head, (upgraded) => + sockets.emit("connection", upgraded, request), + ); + }); + const admission = new V8InspectorAdmission(); + const provider = () => new V8InspectorProvider(undefined, admission); + let held: Duplex | undefined; + onTestFinished(async () => { + await closeVerifierResources([ + () => held?.destroy(), + () => handshake.release(), + () => provider().close(), + () => { + for (const socket of sockets.clients) socket.terminate(); + }, + () => new Promise((resolve) => sockets.close(() => resolve())), + async () => { + if (server.listening) + await new Promise((resolve) => server.close(() => resolve())); + }, + () => rm(root, { recursive: true, force: true }), + ]); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("Inspector fixture did not bind a TCP port"); + endpoint = `http://127.0.0.1:${String(address.port)}`; + const input = observeJavaScriptRuntimeInputSchema.parse({ + inspector_endpoint: endpoint, + target_id: "fixture", + observation_ms: 0, + }); + + const controller = new AbortController(); + const reason = new Error("cancelled during the inspector handshake"); + const observing = observeJavaScriptRuntime(provider(), input, { + signal: controller.signal, + }); + held = await handshake.reached(observing, "inspector WebSocket handshake"); + const heldSocket = held; + // The HTTP server allows half-open sockets, so the client's close arrives + // as end-of-stream (or as a reset). + const heldClosed = new Promise((resolve) => { + heldSocket.once("end", resolve); + heldSocket.once("close", () => resolve()); + }); + controller.abort(reason); + const result = await observing; + if (result.ok) throw new Error("Expected handshake cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(result.error.cleanupIncomplete).toBe(false); + await awaitAcknowledgement( + heldClosed, + new Promise(() => undefined), + "held handshake socket close", + ); + + const next = await observeJavaScriptRuntime(provider(), input); + if (!next.ok) throw next.error; + expect(upgrades).toBe(2); +}); diff --git a/src/javascript/analysis/JavaScriptAnalysisWorker.startup.test.ts b/src/javascript/analysis/JavaScriptAnalysisWorker.startup.test.ts new file mode 100644 index 0000000000..0c6ba5c14b --- /dev/null +++ b/src/javascript/analysis/JavaScriptAnalysisWorker.startup.test.ts @@ -0,0 +1,102 @@ +import { createHash } from "node:crypto"; +import { expect, it, onTestFinished, vi } from "vitest"; +import { JavaScriptAnalysisWorker } from "./JavaScriptAnalysisWorker.js"; +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import { + AnalysisOutputError, + AnalysisTimeoutError, +} from "../../domain/analysisErrorCore.js"; +import { primaryAnalysisError } from "../../domain/analysisErrorBase.js"; +import { spawnOwnedProviderProcess } from "../../process/ProviderProcess.js"; +import { phaseArrival } from "../../../tests/support/lifecycleHarness.js"; + +const text = "export const observed = 1;"; +const file: JavaScriptArtifactFile = { + path: "app.mjs", + sha256: createHash("sha256").update(text).digest("hex"), + container_sha256: "1".repeat(64), + bytes: Buffer.byteLength(text), + inventory_artifact_id: "fixture-source", + kind: "javascript", + unpacked: false, + text: { included: true, value: text }, +}; +const budget = { nodes: 100, unknowns: 100 }; + +const alive = (pid: number): boolean => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +/** Launch a stand-in worker script under the real owned-process launcher. */ +const standInWorker = (script: string) => { + const pids: number[] = []; + const spawned = phaseArrival(); + const launcher: typeof spawnOwnedProviderProcess = async (options) => { + const owned = await spawnOwnedProviderProcess({ + ...options, + arguments: ["-e", script], + }); + if (owned.process.pid !== undefined) pids.push(owned.process.pid); + spawned.arrive(); + return owned; + }; + return { launcher, pids, spawned }; +}; + +it("refuses a worker that emits facts before its ready frame and stops it", async () => { + const fixture = standInWorker( + `process.stdout.write(JSON.stringify({ kind: "phase", id: "00000000-0000-4000-8000-000000000000", phase: "parse" }) + "\\n"); setInterval(() => {}, 1000);`, + ); + const worker = new JavaScriptAnalysisWorker( + { heapMb: 128, timeoutMs: 30_000 }, + fixture.launcher, + ); + onTestFinished(() => worker.close()); + + const refused = await worker.analyze(file, budget); + if (refused.ok) throw new Error("Expected a handshake refusal"); + const primary = primaryAnalysisError(refused.error.error); + expect(primary).toBeInstanceOf(AnalysisOutputError); + expect(refused.error.error.cleanupIncomplete).toBe(false); + expect(worker.canReuse()).toBe(false); + // The refusal itself stops the child; close() is only teardown. + expect(fixture.pids).toHaveLength(1); + expect(fixture.pids.filter(alive)).toEqual([]); +}); + +it("times out a worker whose ready frame never arrives and stops it", async () => { + const fixture = standInWorker("setInterval(() => {}, 1000);"); + const timeoutMs = 500; + const worker = new JavaScriptAnalysisWorker( + { heapMb: 128, timeoutMs }, + fixture.launcher, + ); + onTestFinished(() => worker.close()); + + // The startup budget also covers the launch. Hold the deadline clock until + // the stand-in is running, so a slow launch cannot end the test before the + // missing ready frame is what the deadline interrupts. + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] }); + onTestFinished(() => { + vi.useRealTimers(); + }); + const analysis = worker.analyze(file, budget); + await fixture.spawned.reached(analysis, "stand-in worker launched"); + vi.advanceTimersByTime(timeoutMs); + vi.useRealTimers(); + + const refused = await analysis; + if (refused.ok) throw new Error("Expected a startup deadline"); + expect(primaryAnalysisError(refused.error.error)).toBeInstanceOf( + AnalysisTimeoutError, + ); + expect(refused.error.error.cleanupIncomplete).toBe(false); + // The refusal itself stops the child; close() is only teardown. + expect(fixture.pids).toHaveLength(1); + expect(fixture.pids.filter(alive)).toEqual([]); +}); diff --git a/src/process/OwnedCommand.test.ts b/src/process/OwnedCommand.test.ts index 413cdb77c3..273521f293 100644 --- a/src/process/OwnedCommand.test.ts +++ b/src/process/OwnedCommand.test.ts @@ -512,3 +512,83 @@ it("reports a deadline that elapses during final cleanup as a timeout", async () vi.useRealTimers(); } }); + +it.each(["cancelled", "timeout"] as const)( + "reports a %s interruption during refused final cleanup with its retry owner", + async (interruption) => { + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "performance"] }); + const controller = new AbortController(); + const reason = { request: "abort during refused final cleanup" }; + const fixture = ownedInput(null); + let denied = true; + const cleanupStarted = createDeferred(); + const releaseCleanup = createDeferred(); + fixture.cleanup.mockImplementation(async () => { + if (!denied) return { cleaned: true, signaled: false }; + cleanupStarted.resolve(); + await releaseCleanup.promise; + return { cleaned: false, reason: "process group still present" }; + }); + let pending: Promise | undefined; + try { + pending = runOwnedCommand( + spawn, + { timeoutMs: 1_000, diagnosticBytes: 1024 }, + { + signal: controller.signal, + launcher: async () => { + setImmediate(() => { + fixture.child.stdout.write("completed output"); + fixture.child.complete(); + }); + return { + process: fixture.child, + ownership: fixture.ownership, + cleanup: fixture.cleanup, + }; + }, + }, + ).catch((cause: unknown) => cause); + let started = false; + void cleanupStarted.promise.then(() => (started = true)); + while (!started) await vi.advanceTimersByTimeAsync(10); + expect(performance.now()).toBeLessThan(1_000); + if (interruption === "cancelled") controller.abort(reason); + else await vi.advanceTimersByTimeAsync(1_000); + releaseCleanup.resolve(); + const failure = await pending; + if (!(failure instanceof OwnedCommandFailure)) throw failure; + expect(failure).toMatchObject({ + reason: interruption, + cleanupFailure: "process group still present", + snapshot: { exitCode: 0, stdout: { text: "completed output" } }, + }); + expect(failure.cancellation?.reason).toBe( + interruption === "cancelled" ? reason : undefined, + ); + expect(failure.cleanupOwner?.launch.ownership).toBe(fixture.ownership); + denied = false; + await expect(failure.cleanupOwner?.stop()).resolves.toEqual({ + status: "verified-cleanup", + }); + expect(fixture.cleanup).toHaveBeenCalledTimes(2); + } finally { + denied = false; + releaseCleanup.resolve(); + // Join the command even when an assertion failed before it settled; + // its deadline bounds the fake time this can take. + if (pending !== undefined) { + let settled = false; + const joined = pending.finally(() => { + settled = true; + }); + for (let step = 0; !settled && step < 200; step += 1) + await vi.advanceTimersByTimeAsync(10); + const outcome = await joined; + if (outcome instanceof OwnedCommandFailure) + await outcome.cleanupOwner?.stop(); + } + vi.useRealTimers(); + } + }, +); diff --git a/src/process/OwnedCommand.ts b/src/process/OwnedCommand.ts index 636c4cbb57..76b09a693b 100644 --- a/src/process/OwnedCommand.ts +++ b/src/process/OwnedCommand.ts @@ -501,10 +501,26 @@ const collectOwnedCommand = async ( (callerSignal?.aborted ? { reason: callerSignal.reason } : undefined); if (stopped.status === "incomplete" && retainCleanup !== undefined) retainCleanup(supervisor, stopped); - else if (stopped.status === "incomplete") + else if (stopped.status === "incomplete") { + // A completed run interrupted during final cleanup is reported as that + // interruption, as when the cleanup succeeds (below); only the retry + // owner differs. + const interruption = + failure !== undefined + ? undefined + : callerSignal?.aborted + ? ("cancelled" as const) + : deadline.interruption === "timeout" + ? ("timeout" as const) + : undefined; throw new OwnedCommandFailure( - failure?.reason ?? "process", - failure?.message ?? "Command cleanup failed.", + failure?.reason ?? interruption ?? "process", + failure?.message ?? + (interruption === "cancelled" + ? "Command cancelled during final cleanup." + : interruption === "timeout" + ? "Command deadline elapsed during final cleanup." + : "Command cleanup failed."), snapshot, stopped.reason, { @@ -525,6 +541,7 @@ const collectOwnedCommand = async ( ownedCommandResources(launched.ownership, spawn.cwd), supervisor, ); + } if (failure !== undefined) throw new OwnedCommandFailure( failure.reason, diff --git a/src/providers/OwnedCommandCleanup.ts b/src/providers/OwnedCommandCleanup.ts index 65e69ee380..c36e48831a 100644 --- a/src/providers/OwnedCommandCleanup.ts +++ b/src/providers/OwnedCommandCleanup.ts @@ -16,7 +16,10 @@ import { } from "../resources/index.js"; import { analysisErrorWithCleanupDependencies } from "../domain/analysisErrorCleanup.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; -import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import type { + AnalysisCapturedOutput, + AnalysisError, +} from "../domain/analysisErrorBase.js"; import type { RetainedResourceScope } from "../ports/AnalysisProvider.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { DarwinProcessOwnershipInspectionError } from "../process/DarwinProcessRunTokenReader.js"; @@ -29,6 +32,11 @@ export const retainOwnedCommandCleanup = ( resourceOwner: RetainedResourceScope, providerId: string, operation: string, + /** + * Output of a run that completed, for a caller whose command cleanup error + * is the primary report; a secondary cleanup leaves output to its primary. + */ + completed?: { readonly capturedOutput: AnalysisCapturedOutput }, ): readonly AnalysisError[] => { const commandFailure = cause instanceof OwnedCommandFailure ? cause : undefined; @@ -90,7 +98,13 @@ export const retainOwnedCommandCleanup = ( process_cleanup: jsonValueSchema.parse(result), }), }, - { cause, operation }, + { + cause, + operation, + ...(completed === undefined + ? {} + : { capturedOutput: completed.capturedOutput }), + }, ); const cleanup = failureForStop( commandFailure.cleanupFailure ?? @@ -190,14 +204,15 @@ export const processInspectionCancellationError = ( }; /** - * Retain explicit inspection owners in an artifact scope, for a reader that - * reports the inspection as unavailable instead of failing. Returns the - * cleanup failures that stay open until the scope releases them. + * Transfer helper and command cleanup owners to an artifact scope, which + * retries them before its next operation and on close. Returns the unresolved + * cleanup failures, each bound to its retained owner. */ -export const retainProcessInspectionOwners = ( +export const retainArtifactCommandCleanup = ( cause: unknown, resources: ArtifactResourceScope, operation: string, + completed?: Parameters[4], ): readonly AnalysisError[] => retainOwnedCommandCleanup( cause, @@ -212,6 +227,7 @@ export const retainProcessInspectionOwners = ( }, "process-ownership", operation, + completed, ); /** Bind explicit inspection owners to an artifact scope while preserving the caller's primary. */ @@ -221,7 +237,7 @@ export const retainProcessInspectionCleanup = ( resources: ArtifactResourceScope, operation: string, ): AnalysisError => { - const failures = retainProcessInspectionOwners(cause, resources, operation); + const failures = retainArtifactCommandCleanup(cause, resources, operation); for (const failure of failures) primary = analysisErrorWithCleanupDependencies(primary, failure, operation); return bindProcessInspectionCancellations(cause, primary, operation); diff --git a/tests/boundary/browser/cdpElectronProvider.test.ts b/tests/boundary/browser/cdpElectronProvider.test.ts index 5e6fdec21e..f702460a9a 100644 --- a/tests/boundary/browser/cdpElectronProvider.test.ts +++ b/tests/boundary/browser/cdpElectronProvider.test.ts @@ -3,10 +3,13 @@ import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { createPackage } from "@electron/asar"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; +import { phaseGate } from "../../support/lifecycleHarness.js"; +import { CdpConnection } from "../../../src/browser/CdpConnection.js"; import { CdpElectronProvider } from "../../../src/browser/CdpElectronProvider.js"; import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; import { @@ -564,6 +567,135 @@ describe("CdpElectronProvider target access", () => { }); }); +describe("CdpElectronProvider transport lifecycle", () => { + const inspectInput = (endpoint: string) => + inspectElectronPageInputSchema.parse({ + cdp_endpoint: endpoint, + target_id: "electron-page", + observation_ms: 0, + }); + const connectionsClosed = (browser: FakeCdpBrowser): Promise => + new Promise((resolve) => { + const poll = (): void => { + if (browser.activeConnectionCount() === 0) resolve(); + else setTimeout(poll, 5); + }; + poll(); + }); + + it("cancels a held Electron CDP handshake and admits the next inspection", async () => { + const root = await electronFixture(); + const handshake = phaseGate(); + const browser = await startFakeCdpBrowser({ + electronFileUrl: pathToFileURL(join(root, "index.html")).href, + handshakeGate: handshake, + }); + browsers.push(browser); + const provider = new CdpElectronProvider(); + const controller = new AbortController(); + const reason = new Error("cancelled during the Electron handshake"); + const inspecting = provider.inspectPage(inspectInput(browser.endpoint), { + signal: controller.signal, + }); + // Release and join the held request and close the provider even when a + // barrier or assertion below fails. + onTestFinished(async () => { + handshake.release(); + await inspecting; + await provider.close(); + }); + await handshake.reached(inspecting, "Electron CDP WebSocket handshake"); + controller.abort(reason); + const cancelled = await inspecting; + if (cancelled.ok) throw new Error("Expected handshake cancellation"); + expect(cancelled.error).toBeInstanceOf(AnalysisCancelledError); + expect(cancelled.error.cause).toBe(reason); + expect(cancelled.error.cleanupIncomplete).toBe(false); + expect(browser.openedConnectionCount()).toBe(0); + + handshake.release(); + const inspected = await provider.inspectPage( + inspectInput(browser.endpoint), + ); + if (!inspected.ok) throw inspected.error; + await provider.close(); + await awaitAcknowledgement( + connectionsClosed(browser), + new Promise(() => undefined), + "Electron transports closed", + ); + }); + + it("retains a failed Electron transport close and retries it before the next inspection", async () => { + const root = await electronFixture(); + const browser = await startFakeCdpBrowser({ + electronFileUrl: pathToFileURL(join(root, "index.html")).href, + }); + browsers.push(browser); + const provider = new CdpElectronProvider(); + const originalClose = CdpConnection.prototype.close; + let denyClose = true; + let closes = 0; + // The first close after denial ends is the retained transport's retry. + const retriedClose = phaseGate(); + let retryHeld = false; + // Each transport connect, observed when it is requested. + const connectSpy = vi.spyOn(CdpConnection, "connect"); + let retrying: ReturnType | undefined; + const closeSpy = vi + .spyOn(CdpConnection.prototype, "close") + .mockImplementation(async function (this: CdpConnection) { + closes += 1; + if (denyClose) + throw new Error("controlled Electron transport close failure"); + if (!retryHeld) { + retryHeld = true; + await retriedClose.hold(connectSpy.mock.calls.length); + } + return originalClose.call(this); + }); + try { + const first = await provider.inspectPage(inspectInput(browser.endpoint)); + if (first.ok) throw new Error("Expected incomplete transport cleanup"); + expect(first.error.cleanupIncomplete).toBe(true); + expect(first.error.partialObservation).toMatchObject({ + kind: "electron-page-observation", + inspection: { target: { file_path: join(root, "index.html") } }, + }); + const failedCloses = closes; + + const connectsBeforeRetry = connectSpy.mock.calls.length; + + denyClose = false; + retrying = provider.inspectPage(inspectInput(browser.endpoint)); + // The retained close is still running: no replacement connect yet. + expect( + await retriedClose.reached(retrying, "retained transport close retry"), + ).toBe(connectsBeforeRetry); + await new Promise((resolve) => setImmediate(resolve)); + expect(connectSpy).toHaveBeenCalledTimes(connectsBeforeRetry); + retriedClose.release(); + const retried = await retrying; + if (!retried.ok) throw retried.error; + expect(closes).toBeGreaterThan(failedCloses + 1); + expect(first.error.cleanupIncomplete).toBe(false); + await provider.close(); + await awaitAcknowledgement( + connectionsClosed(browser), + new Promise(() => undefined), + "Electron transports closed", + ); + } finally { + denyClose = false; + retriedClose.release(); + await retrying?.catch(() => undefined); + closeSpy.mockRestore(); + connectSpy.mockRestore(); + await provider.close().catch(() => undefined); + } + }); +}); + describe("CdpElectronProvider target selection", () => { it("lists and inspects every canonical local file target", async () => { const root = await electronFixture(); diff --git a/tests/boundary/filesystem/assetCatalogFinalization.test.ts b/tests/boundary/filesystem/assetCatalogFinalization.test.ts index 405e90fe7e..37d7046ebb 100644 --- a/tests/boundary/filesystem/assetCatalogFinalization.test.ts +++ b/tests/boundary/filesystem/assetCatalogFinalization.test.ts @@ -4,7 +4,7 @@ import { expect, it, vi } from "vitest"; import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; import { ArtifactReaderFailure } from "../../../src/resources/ArtifactReader.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; -import * as processOutput from "../../../src/process/ExecFileOutput.js"; +import * as ownedCommand from "../../../src/process/OwnedCommand.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { DirectoryArtifactReader } from "../../../src/resources/index.js"; @@ -62,8 +62,13 @@ it.skipIf(process.platform !== "darwin").each([ } await originalClose.call(this); }); + const stream = (text: string) => ({ + text, + bytes: text.length, + observedBytes: text.length, + }); const utility = vi - .spyOn(processOutput, "execFileOutput") + .spyOn(ownedCommand, "runOwnedCommand") .mockImplementationOnce(async () => { if (primary === "integrity") await writeFile( @@ -71,13 +76,22 @@ it.skipIf(process.platform !== "darwin").each([ "changed catalog bytes with a different length", ); if (primary === "timeout") - throw Object.assign(new Error("controlled assetutil deadline"), { - code: "ETIMEDOUT", - stdout: "", - stderr: "assetutil timeout detail", - signal: "SIGTERM", - }); - return { stdout: "[]", stderr: "", stdoutBytes: 2, stderrBytes: 0 }; + throw new ownedCommand.OwnedCommandFailure( + "timeout", + "Command deadline elapsed.", + { + stdout: stream(""), + stderr: stream("assetutil timeout detail"), + exitCode: null, + signal: "SIGKILL", + }, + ); + return { + stdout: stream("[]"), + stderr: stream(""), + exitCode: 0, + signal: null, + }; }); const client = new ArtifactProvider( {}, @@ -206,3 +220,63 @@ it.each(["decode_interface_builder", "inspect_keyed_archive"] as const)( } }, ); + +it.skipIf(process.platform !== "darwin")( + "prepares process ownership before the assetutil deadline and bounds each stream separately", + async () => { + const bundle = await createTestTempDirectory("rea-asset-limits-"); + const resources = join(bundle, "Contents", "Resources"); + await mkdir(resources, { recursive: true }); + await writeFile(join(resources, "Assets.car"), "inert catalog bytes"); + const stages: string[] = []; + const prepare = vi + .spyOn(ownedCommand, "prepareOwnedCommand") + .mockImplementationOnce(async () => { + stages.push("prepare"); + }); + const run = vi + .spyOn(ownedCommand, "runOwnedCommand") + .mockImplementationOnce(async (_spawn, limits) => { + stages.push("run"); + // A cold helper build must not spend the command deadline, and + // stderr diagnostics must not shrink the stdout allowance. + expect(limits).toMatchObject({ + timeoutMs: 60_000, + diagnosticBytes: { + stdout: 64 * 1024 * 1024, + stderr: 64 * 1024 * 1024, + }, + }); + const stream = (text: string) => ({ + text, + bytes: text.length, + observedBytes: text.length, + }); + return { + stdout: stream("[]"), + stderr: stream(""), + exitCode: 0, + signal: null, + }; + }); + const client = new ArtifactProvider( + {}, + "darwin", + ).createBundleResourceClient({ + callerPath: bundle, + canonicalPath: bundle, + infoPlistPath: join(bundle, "Contents", "Info.plist"), + layout: "contents", + executableSha256: null, + }); + try { + const result = await client.execute("inspect_asset_catalog", {}); + if (!result.ok) throw result.error; + expect(stages).toEqual(["prepare", "run"]); + } finally { + await client.close(); + run.mockRestore(); + prepare.mockRestore(); + } + }, +); diff --git a/tests/boundary/filesystem/harmony/application.test.ts b/tests/boundary/filesystem/harmony/application.test.ts index 1ffdbca190..d93117dfc1 100644 --- a/tests/boundary/filesystem/harmony/application.test.ts +++ b/tests/boundary/filesystem/harmony/application.test.ts @@ -28,6 +28,7 @@ import { } from "../../../../src/domain/harmony/harmonyApplication.js"; import { artifactInventoryResultSchema } from "../../../../src/domain/artifactGraph.js"; import { createEvidence } from "../../../../src/domain/evidence.js"; +import { AnalysisCancelledError } from "../../../../src/domain/analysisErrorCore.js"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; import { ZIP_PACKAGE_FORMATS, @@ -396,6 +397,38 @@ describe("HarmonyOS packaging boundaries", () => { ]); }); + it("stops resolving retained inventory pages when the caller cancels mid-request", async () => { + const path = await writeApplicationZip("rea-harmony-cancel-", "entry.hap", [ + { path: "module.json", content: '{"module":{"name":"entry"}}' }, + { path: "resources.index", content: "resource index" }, + ]); + const pages = fragmentInventoryEvidence( + await inventoryApplicationPackage(path), + 2, + ); + const controller = new AbortController(); + const reason = { request: "cancel Harmony projection" }; + const looked: string[] = []; + const result = projectHarmonyApplicationEvidence( + { + inventory_evidence: pages.map(({ evidence_id }) => ({ + kind: "retained-evidence", + evidence_id, + })), + }, + (id) => { + looked.push(id); + controller.abort(reason); + return pages.find(({ evidence_id }) => evidence_id === id); + }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(looked).toEqual([pages[0]?.evidence_id]); + }); + it("rejects non-HarmonyOS inventory Evidence", async () => { const inventory = await createNonApplicationZipInventory( "rea-harmony-invalid-", diff --git a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts index 845a6f068a..c68c122dce 100644 --- a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts +++ b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts @@ -21,7 +21,11 @@ import { createEvidence, evidenceSubjectInput, } from "../../../src/domain/evidence.js"; -import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisInputError, +} from "../../../src/domain/analysisErrorCore.js"; +import { JAVASCRIPT_APPLICATION_VERSION_COMPARISON_EXAMPLE } from "../../../src/contracts/javascript/javascriptApplicationWorkflowExamples.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { @@ -230,6 +234,38 @@ describe("JavaScript application comparison workflows", () => { }); }); +describe("JavaScript application workflow cancellation", () => { + it("stops resolving retained version-comparison sources when the caller cancels mid-request", () => { + const { left, right } = JAVASCRIPT_APPLICATION_VERSION_COMPARISON_EXAMPLE; + const sources = [left, right]; + const controller = new AbortController(); + const reason = { request: "cancel version comparison" }; + const looked: string[] = []; + const result = compareApplicationVersionsEvidence( + { + left: { + kind: "retained-evidence", + evidence_id: left.evidence_id, + }, + right: { + kind: "retained-evidence", + evidence_id: right.evidence_id, + }, + }, + (id) => { + looked.push(id); + controller.abort(reason); + return sources.find(({ evidence_id }) => evidence_id === id); + }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(looked).toEqual([left.evidence_id]); + }); +}); + describe("historical native comparison projections", () => { it("keeps historical native member omission distinct through authenticated comparison", async () => { const root = await createTestTempDirectory( diff --git a/tests/boundary/javascript/javascriptAnalysisWorkerLaunch.test.ts b/tests/boundary/javascript/javascriptAnalysisWorkerLaunch.test.ts new file mode 100644 index 0000000000..1316f0d002 --- /dev/null +++ b/tests/boundary/javascript/javascriptAnalysisWorkerLaunch.test.ts @@ -0,0 +1,47 @@ +import { createHash } from "node:crypto"; +import { expect, it, onTestFinished } from "vitest"; +import { JavaScriptAnalysisWorker } from "../../../src/javascript/analysis/JavaScriptAnalysisWorker.js"; +import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; +import { primaryAnalysisError } from "../../../src/domain/analysisErrorBase.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; + +const text = "export const observed = 1;"; +const file: JavaScriptArtifactFile = { + path: "app.mjs", + sha256: createHash("sha256").update(text).digest("hex"), + container_sha256: "1".repeat(64), + bytes: Buffer.byteLength(text), + inventory_artifact_id: "fixture-source", + kind: "javascript", + unpacked: false, + text: { included: true, value: text }, +}; +const budget = { nodes: 100, unknowns: 100 }; + +// The relaunch runs the built worker entry, so this lives in a built project. +it("reports a JavaScript worker that fails to launch and launches a fresh worker on the next analysis", async () => { + let launches = 0; + const launchFailure = Object.assign(new Error("spawn EAGAIN"), { + code: "EAGAIN", + }); + const worker = new JavaScriptAnalysisWorker( + { heapMb: 128, timeoutMs: 30_000 }, + async (options) => { + launches += 1; + if (launches === 1) throw launchFailure; + return spawnOwnedProviderProcess(options); + }, + ); + onTestFinished(() => worker.close()); + + const refused = await worker.analyze(file, budget); + if (refused.ok) throw new Error("Expected a worker launch failure"); + const primary = primaryAnalysisError(refused.error.error); + expect(primary.userCategory).not.toBe("cancelled"); + expect(primary.cause).toBe(launchFailure); + expect(refused.error.error.cleanupIncomplete).toBe(false); + + const analyzed = await worker.analyze(file, budget); + if (!analyzed.ok) throw analyzed.error.error; + expect(launches).toBe(2); +}); diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index 548be74082..ea921be9bf 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -81,6 +81,50 @@ nativeTest.each([ }, ); +it("reports a SQLite worker that fails to launch and removes its snapshot before the next inspection", async () => { + const { database } = await createFixture(); + const roots: string[] = []; + let launches = 0; + const provider = new SqliteDatabaseProvider( + process.env, + (spawn) => { + launches += 1; + if (launches === 1) + throw Object.assign(new Error("spawn E2BIG"), { code: "E2BIG" }); + return spawnOwnedProviderProcess({ + ...spawn, + arguments: spawn.arguments.map((argument, index) => + index === 1 ? fileURLToPath(workerUrl) : argument, + ), + }); + }, + async () => { + const root = await PrivateRuntimeRoot.create({ + prefix: "rea-sqlite-launch-", + }); + roots.push(root.path); + return root; + }, + ); + onTestFinished(() => provider.close()); + + const refused = await provider.inspect({ path: database.path }); + if (refused.ok) throw new Error("Expected a worker launch failure"); + expect(refused.error.diagnostics).toMatchObject({ + phase: "worker", + reason: "spawn E2BIG", + }); + expect(refused.error.cleanupIncomplete).toBe(false); + expect(refused.error.capturedOutput).toBeUndefined(); + const [refusedRoot] = roots; + if (refusedRoot === undefined) throw new Error("Missing private root"); + await expectPrivateRootRemoved(refusedRoot); + + const inspected = await provider.inspect({ path: database.path }); + if (!inspected.ok) throw inspected.error; + expect(launches).toBe(2); +}); + it("enforces a selected worker budget and completes the corrected same-database retry", async () => { const { database } = await createFixture(); const original = await readFile(database.path); diff --git a/tests/composition/analysis-sessions/managedApplicationGraph.test.ts b/tests/composition/analysis-sessions/managedApplicationGraph.test.ts index fd48d0df18..4c4e06a3fd 100644 --- a/tests/composition/analysis-sessions/managedApplicationGraph.test.ts +++ b/tests/composition/analysis-sessions/managedApplicationGraph.test.ts @@ -1,5 +1,6 @@ import { unknownTarget } from "../../../src/domain/targetIdentity.js"; import { describe, expect, it } from "vitest"; +import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; import { projectManagedApplicationGraphEvidence } from "../../../src/application/managed/ManagedApplicationGraphService.js"; import { @@ -144,6 +145,38 @@ describe("managed application graph MethodDef scope", () => { }); }); +describe("managed application graph cancellation", () => { + it("stops resolving retained managed sources when the caller cancels mid-request", () => { + const members = MANAGED_APPLICATION_GRAPH_EXAMPLE.managed_members; + const controller = new AbortController(); + const reason = { request: "cancel managed graph projection" }; + const looked: string[] = []; + const result = projectManagedApplicationGraphEvidence( + { + managed_members: { + kind: "retained-evidence", + evidence_id: members.evidence_id, + }, + // Never read: cancellation is observed before this lookup. + managed_native_boundaries: { + kind: "retained-evidence", + evidence_id: `ev_${"b".repeat(64)}`, + }, + }, + (id) => { + looked.push(id); + controller.abort(reason); + return id === members.evidence_id ? members : undefined; + }, + { signal: controller.signal }, + ); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cause).toBe(reason); + expect(looked).toEqual([members.evidence_id]); + }); +}); + describe("managed application graph projection", () => { it("projects managed metadata and native declarations into authenticated graph Evidence", () => { const { artifactEvidence, memberEvidence, boundaryEvidence } = diff --git a/tests/fixtures/fakeCdpBrowser.ts b/tests/fixtures/fakeCdpBrowser.ts index be92d04762..5265ba1d40 100644 --- a/tests/fixtures/fakeCdpBrowser.ts +++ b/tests/fixtures/fakeCdpBrowser.ts @@ -4,6 +4,8 @@ import { type ServerResponse, } from "node:http"; +import type { Duplex } from "node:stream"; + import { WebSocket, WebSocketServer } from "ws"; import { emitEvents } from "./fakeCdpBrowserEvents.js"; @@ -28,6 +30,8 @@ export const startFakeCdpBrowser = async ( referer: string | undefined; }[] = []; const sockets = new Set(); + // Upgrades a handshake gate holds; close() destroys them. + const heldUpgrades = new Set(); let openedConnections = 0; let frameTreeReads = 0; let port = 0; @@ -51,6 +55,21 @@ export const startFakeCdpBrowser = async ( webSockets.emit("connection", webSocket, request), ); }; + if (options.handshakeGate !== undefined) { + // Flowing, so a client that abandons the handshake is observed. + socket.resume(); + socket.once("end", () => socket.destroy()); + heldUpgrades.add(socket); + socket.once("close", () => heldUpgrades.delete(socket)); + void options.handshakeGate.hold(request.url ?? "").then( + () => { + heldUpgrades.delete(socket); + accept(); + }, + () => socket.destroy(), + ); + return; + } if (options.handshakeDelayMs === undefined) { accept(); return; @@ -132,6 +151,7 @@ export const startFakeCdpBrowser = async ( }, httpRequests, async close() { + for (const socket of heldUpgrades) socket.destroy(); for (const socket of sockets) socket.terminate(); await new Promise((resolve) => webSockets.close(() => resolve())); await new Promise((resolve, reject) => diff --git a/tests/fixtures/fakeCdpBrowserTypes.ts b/tests/fixtures/fakeCdpBrowserTypes.ts index 8e87831d61..42390df482 100644 --- a/tests/fixtures/fakeCdpBrowserTypes.ts +++ b/tests/fixtures/fakeCdpBrowserTypes.ts @@ -1,6 +1,7 @@ import type { JsonValue } from "../../src/domain/jsonValue.js"; import type { CdpEvent } from "../../src/browser/CdpConnection.js"; import type { ServerResponse } from "node:http"; +import type { PhaseGate } from "../support/lifecycleHarness.js"; export interface FakeCdpCommand { readonly id: number; readonly method: string; @@ -53,6 +54,11 @@ export interface FakeOptions { readonly discoveryDelayMs?: number; /** Delay WebSocket upgrade acceptance so handshake and attach deadlines stay distinct. */ readonly handshakeDelayMs?: number; + /** + * Hold each WebSocket upgrade (by request URL) until the test releases the + * gate. A held socket the client abandons is destroyed, not accepted. + */ + readonly handshakeGate?: PhaseGate; /** Delay selected wire replies so one caller's discovery deadline can expire alongside an active peer. */ readonly commandDelayMs?: (command: FakeCdpCommand) => number | undefined; readonly malformedDiscovery?: boolean; diff --git a/tests/process-global/assetCatalogCancellationReason.test.ts b/tests/process-global/assetCatalogCancellationReason.test.ts index 951a21ecc3..ed741271f3 100644 --- a/tests/process-global/assetCatalogCancellationReason.test.ts +++ b/tests/process-global/assetCatalogCancellationReason.test.ts @@ -9,21 +9,26 @@ const state = vi.hoisted(() => ({ reason: undefined as unknown, })); -vi.mock("../../src/process/ExecFileOutput.js", async (importOriginal) => { +vi.mock("../../src/process/OwnedCommand.js", async (importOriginal) => { const actual = - await importOriginal< - typeof import("../../src/process/ExecFileOutput.js") - >(); + await importOriginal(); return { ...actual, - execFileOutput: vi.fn(async () => { + runOwnedCommand: vi.fn(async () => { state.controller?.abort(state.reason); - throw new actual.ExecFileOutputCancelledError(state.reason); + throw new actual.OwnedCommandFailure( + "cancelled", + "Command cancelled.", + null, + null, + { cancellation: { reason: state.reason } }, + ); }), }; }); -import { execFileOutput } from "../../src/process/ExecFileOutput.js"; +import { runOwnedCommand } from "../../src/process/OwnedCommand.js"; +import { AnalysisCancelledError } from "../../src/domain/analysisErrorCore.js"; import { ArtifactReaderFailure } from "../../src/resources/ArtifactReader.js"; import { analyzeAppleAssetCatalogs } from "../../src/artifacts/apple/AppleAssetCatalogAnalysis.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; @@ -55,7 +60,7 @@ const analyze = (bundle: string, signal: AbortSignal) => }); it("retains the caller reason when assetutil observes cancellation", async () => { - vi.mocked(execFileOutput).mockClear(); + vi.mocked(runOwnedCommand).mockClear(); const bundle = await createBundle(); const reason = { requested: "stop asset catalog" }; const controller = new AbortController(); @@ -65,19 +70,18 @@ it("retains the caller reason when assetutil observes cancellation", async () => const failure: unknown = await analyze(bundle, controller.signal).catch( (cause: unknown) => cause, ); - expect(failure).toBeInstanceOf(ArtifactReaderFailure); - if (!(failure instanceof ArtifactReaderFailure)) + expect(failure).toBeInstanceOf(AnalysisCancelledError); + if (!(failure instanceof AnalysisCancelledError)) throw new Error("Expected asset catalog cancellation"); - expect(execFileOutput).toHaveBeenCalledTimes(1); - expect(failure.reason).toBe("cancelled"); - expect(failure.cancellation?.reason).toBe(reason); + expect(runOwnedCommand).toHaveBeenCalledTimes(1); + expect(failure.cause).toBe(reason); } finally { await rm(bundle, { recursive: true, force: true }); } }); it("retains the caller reason when the catalog digest is cancelled", async () => { - vi.mocked(execFileOutput).mockClear(); + vi.mocked(runOwnedCommand).mockClear(); const bundle = await createBundle(); const reason = { requested: "stop during catalog digest" }; const controller = new AbortController(); @@ -104,7 +108,7 @@ it("retains the caller reason when the catalog digest is cancelled", async () => expect(readEntered).toBe(true); expect(controller.signal.aborted).toBe(true); expect(controller.signal.reason).toBe(reason); - expect(execFileOutput).not.toHaveBeenCalled(); + expect(runOwnedCommand).not.toHaveBeenCalled(); expect(open).toHaveBeenCalledTimes(1); expect(failure).toBeInstanceOf(ArtifactReaderFailure); if (!(failure instanceof ArtifactReaderFailure)) diff --git a/tests/support/providerLifecycleCoverage.ts b/tests/support/providerLifecycleCoverage.ts index 29399af1b6..8e04699c97 100644 --- a/tests/support/providerLifecycleCoverage.ts +++ b/tests/support/providerLifecycleCoverage.ts @@ -187,8 +187,11 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< }, }, "aioquic-qlog-json": historicalCapture( - gap( - "Successful qlog decoding is covered at the parser; no regression completes it through the decoder child.", + covered( + test( + "tests/acceptance/tools/qlogCapture.test.ts", + "runs built CLI and real stdio SDK with exact evidence parity, selection, retained references and strict schemas", + ), ), ), "apk-signature": inProcess( @@ -821,8 +824,11 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< }, }, "mitmproxy-native-tnetstring": historicalCapture( - gap( - "The decoder tests bind mitmproxy configuration but no regression completes a mitmproxy capture.", + covered( + test( + "tests/boundary/browser/historicalCaptureWideNumbers.test.ts", + "retains every exclusion in a wide native-number reply through the owned decoder", + ), ), ), "model-artifact": inProcess( @@ -1032,8 +1038,11 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "rea-apple-application": { ownership: "process", stages: { - launch: gap( - "No regression for an assetutil or Interface Builder tool launch failure.", + launch: covered( + test( + "src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts", + "reports an assetutil that cannot be launched as unavailable", + ), ), handshake: ONE_SHOT, operation: covered( @@ -1048,8 +1057,18 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "settles a cancelled assetutil run only after its process exits", ), ), - cleanup: gap("No regression for failed assetutil process cleanup."), - retry: gap("No regression for assetutil cleanup retry."), + cleanup: covered( + test( + "src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts", + "retains an unproven assetutil process cleanup for a later retry", + ), + ), + retry: covered( + test( + "src/artifacts/apple/AppleAssetCatalogAnalysis.test.ts", + "releases a retained assetutil cleanup before the next inspection launches", + ), + ), }, }, "rea-apple-ips": inProcess( @@ -1174,8 +1193,11 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< ownership: "connection", stages: { launch: ATTACH_ONLY, - handshake: gap( - "No Electron-provider regression for a late, absent or failed CDP handshake.", + handshake: covered( + test( + "tests/boundary/browser/cdpElectronProvider.test.ts", + "cancels a held Electron CDP handshake and admits the next inspection", + ), ), operation: covered( test( @@ -1193,10 +1215,18 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "classifies cancellation that arrives during final Electron detach", ), ), - cleanup: gap( - "No regression retains a failed Electron inspection cleanup; the cleanup tests cover cancellation during a successful close.", + cleanup: covered( + test( + "tests/boundary/browser/cdpElectronProvider.test.ts", + "retains a failed Electron transport close and retries it before the next inspection", + ), + ), + retry: covered( + test( + "tests/boundary/browser/cdpElectronProvider.test.ts", + "retains a failed Electron transport close and retries it before the next inspection", + ), ), - retry: gap("No regression for Electron inspection cleanup retry."), }, }, "rea-devicetree": inProcess( @@ -1234,7 +1264,12 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "retains partial native export coverage after an observed match", ), ), - gap("No cancellation regression for the managed workflow provider."), + covered( + test( + "tests/composition/analysis-sessions/managedApplicationGraph.test.ts", + "stops resolving retained managed sources when the caller cancels mid-request", + ), + ), ), "rea-elf-notes": inProcess( covered( @@ -1270,16 +1305,31 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "preserves Harmony archive formats through the public CLI/MCP lifecycle", ), ), - gap("No cancellation regression for the Harmony application provider."), + covered( + test( + "tests/boundary/filesystem/harmony/application.test.ts", + "stops resolving retained inventory pages when the caller cancels mid-request", + ), + ), ), "rea-javascript-application": { ownership: "process", stages: { - launch: gap( - "No regression for a JavaScript analysis worker process that fails to launch.", + launch: covered( + test( + "tests/boundary/javascript/javascriptAnalysisWorkerLaunch.test.ts", + "reports a JavaScript worker that fails to launch and launches a fresh worker on the next analysis", + ), ), - handshake: gap( - "No regression for a late or invalid worker admission frame from the provider side.", + handshake: covered( + test( + "src/javascript/analysis/JavaScriptAnalysisWorker.startup.test.ts", + "refuses a worker that emits facts before its ready frame and stops it", + ), + test( + "src/javascript/analysis/JavaScriptAnalysisWorker.startup.test.ts", + "times out a worker whose ready frame never arrives and stops it", + ), ), operation: covered( test( @@ -1314,7 +1364,12 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "traces a contextBridge API through IPC into linked native evidence", ), ), - gap("No cancellation regression for JavaScript application workflows."), + covered( + test( + "tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts", + "stops resolving retained version-comparison sources when the caller cancels mid-request", + ), + ), ), "rea-javascript-runtime-reconciliation": inProcess( covered( @@ -1323,7 +1378,12 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "matches renderer, frame, script bytes, and worker without claiming execution", ), ), - gap("No cancellation regression for JavaScript runtime reconciliation."), + covered( + test( + "src/application/javascript/JavaScriptRuntimeReconciliationService.test.ts", + "stops resolving retained reconciliation sources when the caller cancels mid-request", + ), + ), ), "rea-kernel-build-config": inProcess( covered( @@ -1377,8 +1437,11 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< ), ), "rea-pe-certificates": inProcess( - gap( - "Successful certificate tables are covered at the parser and through generated CLI titles, not by a literal-titled provider regression.", + covered( + test( + "tests/acceptance/peCertificatesCli.test.ts", + "reports PKCS#7 signer identity and RDN indices through CLI and MCP", + ), ), typedFailures("tests/boundary/filesystem/peCertificates.test.ts"), ), @@ -1524,14 +1587,22 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "reports the observed header and payload through the public CLI", ), ), - gap("No cancellation regression for the V8 code cache provider."), + covered( + test( + "src/application/v8CodeCache/V8CodeCacheService.test.ts", + "retains a completed V8 code cache observation when the caller cancels as the provider completes", + ), + ), ), "rea-v8-inspector": { ownership: "connection", stages: { launch: ATTACH_ONLY, - handshake: gap( - "No regression for a late, absent or failed inspector WebSocket handshake.", + handshake: covered( + test( + "src/inspector/V8InspectorProvider.handshake.test.ts", + "cancels a held inspector WebSocket handshake and admits the next observation", + ), ), operation: covered( test( @@ -1568,8 +1639,15 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "retains the selected executable and actual startup diagnostic", ), ), - handshake: gap( - "No regression for a late or failed browser-engine connection after Chromium launches.", + handshake: covered( + test( + "src/browser/modules/NativeModuleResolver.test.ts", + "closes a browser whose connection completes after cancellation", + ), + test( + "src/browser/modules/NativeModuleResolver.test.ts", + "reports failed cleanup of a browser whose connection completes after cancellation", + ), ), operation: covered( test( @@ -1593,8 +1671,11 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< "returns cleanup_incomplete when owned cleanup fails after an engine error", ), ), - retry: gap( - "The resolver retries retained browser cleanup before later work, but no regression names that retry.", + retry: covered( + test( + "src/browser/modules/NativeModuleResolver.test.ts", + "retries retained browser cleanup before launching the next resolution", + ), ), }, }, @@ -1797,8 +1878,11 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< sqlite: { ownership: "process", stages: { - launch: gap( - "No regression for a SQLite worker process that fails to launch.", + launch: covered( + test( + "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts", + "reports a SQLite worker that fails to launch and removes its snapshot before the next inspection", + ), ), handshake: notApplicable( "The SQLite worker writes one reply per request; there is no handshake.", @@ -1979,25 +2063,4 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< * Every stage marked as a gap above, as `providerId:stage`. This list may only * shrink: close a gap by naming its regression tests and removing the entry. */ -export const PROVIDER_LIFECYCLE_GAPS: readonly string[] = [ - "aioquic-qlog-json:operation", - "mitmproxy-native-tnetstring:operation", - "rea-apple-application:cleanup", - "rea-apple-application:launch", - "rea-apple-application:retry", - "rea-cdp-electron:cleanup", - "rea-cdp-electron:handshake", - "rea-cdp-electron:retry", - "rea-dotnet-workflows:cancel", - "rea-harmony-application:cancel", - "rea-javascript-application-workflows:cancel", - "rea-javascript-application:handshake", - "rea-javascript-application:launch", - "rea-javascript-runtime-reconciliation:cancel", - "rea-pe-certificates:operation", - "rea-v8-code-cache:cancel", - "rea-v8-inspector:handshake", - "rea-web-module-trace:handshake", - "rea-web-module-trace:retry", - "sqlite:launch", -]; +export const PROVIDER_LIFECYCLE_GAPS: readonly string[] = [];