diff --git a/src/native/LldbCallTracer.ts b/src/native/LldbCallTracer.ts index 46fa3258ae..bf1dcc5b83 100644 --- a/src/native/LldbCallTracer.ts +++ b/src/native/LldbCallTracer.ts @@ -1,6 +1,6 @@ import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { randomUUID } from "node:crypto"; -import { open, readFile, writeFile } from "node:fs/promises"; +import { writeFile } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -41,7 +41,11 @@ import { } from "../process/ProviderDeadline.js"; import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; import { resolveXcrunTool, type ResolvedTool } from "./CommandRunner.js"; -import { projectNativeCalls } from "./NativeCallObservation.js"; +import { projectPartialNativeCalls } from "./NativeCallObservation.js"; +import { LldbReadbackFiles } from "./LldbReadbackFiles.js"; +import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; +import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js"; +import { resolvedAnalysisError } from "../domain/analysisErrorBase.js"; import { readLldbObservationJournal, readBoundedPrefix, @@ -212,12 +216,24 @@ export class LldbCallTracer implements NativeCallTracer { ); const root = await PrivateRuntimeRoot.create({ prefix: "rea-lldb-" }); const paths = lldbRuntimePaths(root.path); + const owner: RetainedLldbResources = { + root, + rootLease: new AnalysisCleanupLease(() => root.close(), [root.path]), + files: new LldbReadbackFiles(), + supervisor: undefined, + pidPath: paths.pid, + targetIdentity: undefined, + targetPid: undefined, + processStopped: true, + targetStopped: true, + }; let result: Result; try { - result = await this.#run(lldb.value, root, request, signal); + result = await this.#run(lldb.value, owner, request, signal); } catch (cause: unknown) { const partialObservation = await partialNativeObservation({ request, + files: owner.files, pidPath: paths.pid, stdoutPath: paths.stdoutCapture, stderrPath: paths.stderrCapture, @@ -225,71 +241,32 @@ export class LldbCallTracer implements NativeCallTracer { version: null, reason: "tracer-failure", }); - const retained = this.#pendingCleanup; - const resources = - retained === undefined ? [] : await lldbCleanupResources(retained); + if ( + !owner.processStopped || + !owner.targetStopped || + owner.files.hasPending + ) + this.#pendingCleanup = owner; result = err( new ProviderAdapterError(PROVIDER, OPERATION, { cause, - diagnostics: { - reason: cause instanceof Error ? cause.message : String(cause), - }, + diagnostics: { reason: errorReason(cause) }, partialObservation, - ...(retained === undefined - ? {} - : { + ...(this.#pendingCleanup === owner + ? { cleanup: { reason: "LLDB trace failed while owned resources remain", - resources, + resources: lldbCleanupResources(owner), }, - }), + } + : {}), }), ); } - if (this.#pendingCleanup?.root === root) return result; - try { - await root.close(); - } catch (cause: unknown) { - this.#pendingCleanup = { - root, - supervisor: undefined, - pidPath: paths.pid, - targetIdentity: undefined, - processStopped: true, - targetStopped: true, - }; - const partialObservation = result.ok - ? partialObservationFromTrace(request, result.value) - : result.error.partialObservation; - return err( - new ProviderCleanupError( - PROVIDER, - [root.path], - { - reason: cause instanceof Error ? cause.message : String(cause), - ...(result.ok - ? {} - : { - execution_failure: projectAnalysisError(result.error).code, - primary_failure: result.error.message, - }), - }, - { - operation: OPERATION, - cause: result.ok - ? cause - : new AggregateError( - [result.error, cause], - "LLDB trace and runtime cleanup both failed", - ), - ...(partialObservation === undefined ? {} : { partialObservation }), - }, - ), - ); - } - if (result.ok && signal?.aborted === true) - return err( + if (result.ok && signal?.aborted) + result = err( new AnalysisCancelledError(OPERATION, { + cause: signal.reason, partialObservation: partialObservationFromTrace( request, result.value, @@ -297,7 +274,82 @@ export class LldbCallTracer implements NativeCallTracer { ), }), ); - return result; + if (owner.files.hasPending) this.#pendingCleanup = owner; + if (this.#pendingCleanup === owner) { + if (!result.ok && result.error.cleanupIncomplete) { + if (owner.files.hasPending) { + const files = lldbCleanupFailure( + owner, + "LLDB readback descriptor cleanup remains incomplete", + owner.files.cleanupCause, + ); + result.error.retainSecondaryFailure(files, { + readback_cleanup: projectAnalysisError(files), + }); + } + return err(bindLldbCleanup(result.error, owner)); + } + const cleanup = lldbCleanupFailure( + owner, + "LLDB readback descriptor cleanup remains incomplete", + owner.files.cleanupCause, + result.ok + ? partialObservationFromTrace(request, result.value) + : undefined, + ); + return result.ok + ? err(cleanup) + : err( + analysisErrorWithCleanupFailure( + result.error, + cleanup, + OPERATION, + ).bindCleanupDependencies([result.error, cleanup]), + ); + } + let rootCleanup: Result; + try { + await releaseLldbRuntime(owner); + rootCleanup = ok(undefined); + } catch (cause: unknown) { + rootCleanup = err(cause); + this.#pendingCleanup = owner; + } + if (result.ok && signal?.aborted) + result = err( + new AnalysisCancelledError(OPERATION, { + cause: signal.reason, + partialObservation: partialObservationFromTrace( + request, + result.value, + "cancelled", + ), + }), + ); + if (!rootCleanup.ok) { + const cleanup = lldbCleanupFailure( + owner, + errorReason(rootCleanup.error), + rootCleanup.error, + result.ok + ? partialObservationFromTrace(request, result.value) + : undefined, + ); + if (!result.ok && result.error.cleanupIncomplete) + bindLldbCleanup(result.error, owner); + return result.ok + ? err(cleanup) + : err( + analysisErrorWithCleanupFailure( + result.error, + cleanup, + OPERATION, + ).bindCleanupDependencies([result.error, cleanup]), + ); + } + return result.ok + ? result + : err(resolvedAnalysisError(bindLldbCleanup(result.error, owner))); } /** Retry retained LLDB cleanup without abandoning its process or runtime owner. */ @@ -322,53 +374,69 @@ export class LldbCallTracer implements NativeCallTracer { async #retryPendingCleanup(): Promise { const owner = this.#pendingCleanup; if (owner === undefined) return undefined; - let firstFailure: string | undefined; + const failures: unknown[] = []; + // Files are independent children: release them even while process cleanup fails. + try { + await owner.files.retryCleanup(); + } catch (cause: unknown) { + failures.push(cause); + } if (!owner.processStopped) { - const stopped = await owner.supervisor?.stop(); - if (stopped?.status === "incomplete") firstFailure = stopped.reason; + if (owner.supervisor === undefined) + failures.push( + new Error("LLDB process cleanup ownership was unavailable"), + ); else { - owner.supervisor?.dispose(); - owner.processStopped = true; + try { + const stopped = await owner.supervisor.stop(); + if (stopped.status === "incomplete") + failures.push(new Error(stopped.reason)); + else { + owner.supervisor.dispose(); + owner.processStopped = true; + } + } catch (cause: unknown) { + failures.push(cause); + } } } - if (firstFailure === undefined && !owner.targetStopped) { - const targetTermination = await ensureTerminated( - owner.pidPath, - owner.targetIdentity, - true, - ); - if (targetTermination !== "terminated") - firstFailure = targetTerminationReason(targetTermination); - else owner.targetStopped = true; - } - if (firstFailure === undefined) { + if (owner.processStopped && !owner.targetStopped) { + owner.targetPid = await owner.files.readTargetPid(owner.pidPath); try { - await owner.root.close(); + const targetTermination = await ensureTerminated( + owner.targetPid, + owner.targetIdentity, + ); + if (targetTermination !== "terminated") + failures.push(new Error(targetTerminationReason(targetTermination))); + else owner.targetStopped = true; + } catch (cause: unknown) { + failures.push(cause); + } + } + if (failures.length === 0) { + try { + await releaseLldbRuntime(owner); this.#pendingCleanup = undefined; return undefined; } catch (cause: unknown) { - firstFailure = cause instanceof Error ? cause.message : String(cause); + failures.push(cause); } } - const resources = await lldbCleanupResources(owner); - return new ProviderCleanupError( - PROVIDER, - resources, - { - reason: firstFailure ?? "LLDB cleanup remains incomplete", - resources, - }, - { operation: OPERATION }, + return lldbCleanupFailure( + owner, + failures.map(errorReason).join("; "), + new AggregateError(failures, "LLDB cleanup remains incomplete"), ); } async #run( lldb: ResolvedTool, - root: PrivateRuntimeRoot, + retained: RetainedLldbResources, request: Parameters[0], signal?: AbortSignal, ): Promise> { - const directory = root.path; + const directory = retained.root.path; const paths = lldbRuntimePaths(directory); const { input } = request; await writeFile( @@ -401,6 +469,9 @@ export class LldbCallTracer implements NativeCallTracer { }), { mode: 0o600 }, ); + retained.processStopped = false; + retained.targetStopped = false; + this.#pendingCleanup = retained; const exited = await this.launch( lldb.path, [ @@ -417,21 +488,18 @@ export class LldbCallTracer implements NativeCallTracer { pidPath: paths.pid, identityAckPath: paths.identityAck, environment: this.environment, + files: retained.files, ...(signal === undefined ? {} : { signal }), }, ); - const retained: RetainedLldbResources = { - root, - supervisor: exited.cleanupOwner, - pidPath: paths.pid, - targetIdentity: exited.targetIdentity, - processStopped: exited.cleanupOwner === undefined, - targetStopped: !exited.targetLaunched, - }; - this.#pendingCleanup = retained; + retained.supervisor = exited.cleanupOwner; + retained.targetIdentity = exited.targetIdentity; + retained.processStopped = + exited.cleanupOwner === undefined && exited.cleanupFailure === undefined; + retained.targetStopped = !exited.targetLaunched; const output = exited.kind === "exited" - ? await readTracerOutput(paths.result) + ? await readTracerOutput(paths.result, retained.files) : undefined; // An explicit bridge observation can prove no process was created or that // the created process exited. A missing PID alone remains ambiguous. @@ -439,19 +507,24 @@ export class LldbCallTracer implements NativeCallTracer { output !== undefined && output.status !== "traced" && output.target_process_exit_observed === true; + retained.targetPid = await retained.files.readTargetPid(paths.pid); let targetTermination: TargetTermination = exited.targetLaunched && !bridgeObservedExit - ? await ensureTerminated(paths.pid, exited.targetIdentity, true) + ? await ensureTerminated(retained.targetPid, exited.targetIdentity) : "terminated"; if ( output !== undefined && output.status !== "traced" && output.target_process_created === false && - (await readTargetPid(paths.pid)) === undefined + retained.targetPid === undefined ) targetTermination = "terminated"; retained.targetStopped = targetTermination === "terminated"; - if (retained.processStopped && retained.targetStopped) + if ( + retained.processStopped && + retained.targetStopped && + !retained.files.hasPending + ) this.#pendingCleanup = undefined; const terminated = targetTermination === "terminated"; const cleanupDetails = [ @@ -463,7 +536,7 @@ export class LldbCallTracer implements NativeCallTracer { ? undefined : { reason: cleanupDetails.join("; "), - resources: await lldbCleanupResources(retained), + resources: lldbCleanupResources(retained), }; const partialReason = exited.kind === "cancelled" @@ -478,6 +551,7 @@ export class LldbCallTracer implements NativeCallTracer { ): Promise => partialNativeObservation({ request, + files: retained.files, pidPath: paths.pid, stdoutPath: paths.stdoutCapture, stderrPath: paths.stderrCapture, @@ -488,6 +562,7 @@ export class LldbCallTracer implements NativeCallTracer { if (exited.kind === "cancelled") return err( new AnalysisCancelledError(OPERATION, { + cause: signal?.reason, ...(lifecycleCleanup === undefined ? {} : { cleanup: lifecycleCleanup }), @@ -515,7 +590,7 @@ export class LldbCallTracer implements NativeCallTracer { return err( new ProviderCleanupError( PROVIDER, - await lldbCleanupResources(retained), + lldbCleanupResources(retained), { reason: "LLDB process-group cleanup could not be verified", cleanup_failure: exited.cleanupFailure, @@ -533,7 +608,7 @@ export class LldbCallTracer implements NativeCallTracer { return err( new ProviderCleanupError( PROVIDER, - await lldbCleanupResources(retained), + lldbCleanupResources(retained), { reason: "LLDB target process termination could not be verified", ...(lifecycleCleanup === undefined @@ -568,22 +643,49 @@ export class LldbCallTracer implements NativeCallTracer { output.target_output.stdout_bytes, output.target_output.stdout_truncated, output.target_output.stdout_complete, + retained.files, ); const stderr = await capturedOutput( paths.stderrCapture, output.target_output.stderr_bytes, output.target_output.stderr_truncated, output.target_output.stderr_complete, + retained.files, ); if (!stdout.ok || !stderr.ok) { - const partial = await getPartialObservation("capture-failure"); return err( new ProviderAdapterError(PROVIDER, OPERATION, { - partialObservation: partial, + cause: new AggregateError( + [ + ...(stdout.ok ? [] : [stdout.error]), + ...(stderr.ok ? [] : [stderr.error]), + ], + "LLDB capture readback failed", + ), + partialObservation: projectPartialNativeCalls( + { + path: request.executable, + sha256: request.expectedSha256, + architecture: request.architecture, + }, + request.input, + { + debugger: { ...lldb, version: output.version }, + run: output, + stdout: stdout.ok ? stdout.value : null, + stderr: stderr.ok ? stderr.value : null, + terminated, + }, + "capture-failure", + ), diagnostics: { reason: "LLDB target output capture could not be read", - ...(stdout.ok ? {} : { stdout_capture_error: stdout.error }), - ...(stderr.ok ? {} : { stderr_capture_error: stderr.error }), + ...(stdout.ok + ? {} + : { stdout_capture_error: errorReason(stdout.error) }), + ...(stderr.ok + ? {} + : { stderr_capture_error: errorReason(stderr.error) }), ...(lifecycleCleanup === undefined ? {} : { target_cleanup: lifecycleCleanup }), @@ -670,6 +772,7 @@ const tracerFailure = ( const partialNativeObservation = async (options: { readonly request: Parameters[0]; + readonly files: LldbReadbackFiles; readonly pidPath: string; readonly stdoutPath: string; readonly stderrPath: string; @@ -678,10 +781,10 @@ const partialNativeObservation = async (options: { readonly reason: NativeCallPartialObservation["coverage"]["reason"]; }): Promise => { const [journal, pid, stdout, stderr] = await Promise.all([ - readLldbObservationJournal(options.observationPath), - readTargetPid(options.pidPath), - readPartialCapture(options.stdoutPath), - readPartialCapture(options.stderrPath), + readLldbObservationJournal(options.observationPath, options.files), + options.files.readTargetPid(options.pidPath), + readPartialCapture(options.stdoutPath, options.files), + readPartialCapture(options.stderrPath, options.files), ]); return projectLldbPartialObservation({ target: { @@ -693,14 +796,28 @@ const partialNativeObservation = async (options: { working_directory: options.request.input.working_directory ?? null, }, pid, - stdout: stdout?.capture ?? null, - stderr: stderr?.capture ?? null, + stdout: stdout.ok ? stdout.value.capture : null, + stderr: stderr.ok ? stderr.value.capture : null, version: options.version, - journal: journal.parsed, - journalLimitations: journal.limitations, + journal: journal.ok + ? journal.value.parsed + : { events: [], otherStops: [], limitations: [] }, + journalLimitations: journal.ok + ? journal.value.limitations + : [ + `The observation journal could not be read: ${errorReason(journal.error)}`, + ], limitations: [ - ...(stdout?.limitations ?? []), - ...(stderr?.limitations ?? []), + ...(stdout.ok + ? stdout.value.limitations + : [ + `The target stdout capture could not be read: ${errorReason(stdout.error)}`, + ]), + ...(stderr.ok + ? stderr.value.limitations + : [ + `The target stderr capture could not be read: ${errorReason(stderr.error)}`, + ]), ], reason: options.reason, }); @@ -711,7 +828,7 @@ const partialObservationFromTrace = ( trace: NativeCallTrace, reason: NativeCallPartialObservation["coverage"]["reason"] = "cleanup-failure", ): NativeCallPartialObservation => { - const observation = projectNativeCalls( + return projectPartialNativeCalls( { path: request.executable, sha256: request.expectedSha256, @@ -719,12 +836,8 @@ const partialObservationFromTrace = ( }, request.input, trace, + reason, ); - return { - kind: "native-call-observation", - ...observation, - coverage: { ...observation.coverage, status: "partial", reason }, - }; }; const lldbRuntimePaths = (directory: string) => ({ @@ -741,69 +854,94 @@ const lldbRuntimePaths = (directory: string) => ({ const readTracerOutput = async ( path: string, + files: LldbReadbackFiles, ): Promise | undefined> => { - let text: string; - try { - text = await readFile(path, "utf8"); - } catch { - return undefined; - } - const parsed = safeParseJson(text); + const text = await files.read(path, (file) => file.readFile("utf8")); + if (!text.ok) return undefined; + const parsed = safeParseJson(text.value); if (!parsed.ok) return undefined; const output = tracerOutputSchema.safeParse(parsed.value); return output.success ? output.data : undefined; }; -const capturedOutput = async ( +const capturedOutput = ( path: string, observedBytes: number, truncated: boolean, complete: boolean, -): Promise> => { - let handle: Awaited> | undefined; - try { - handle = await open(path, "r"); + files: LldbReadbackFiles, +): Promise> => + files.read(path, async (handle) => { const { size } = await handle.stat(); - const captureHandle = handle; const { buffer, bytesRead } = await readBoundedPrefix( Math.min(size, MAX_OUTPUT_BYTES), async (target, offset, length, position) => - (await captureHandle.read(target, offset, length, position)).bytesRead, + (await handle.read(target, offset, length, position)).bytesRead, ); - const captured = { + return { text: buffer.subarray(0, bytesRead).toString("utf8"), bytes: observedBytes, truncated: truncated || size > MAX_OUTPUT_BYTES, complete: complete && bytesRead >= Math.min(observedBytes, MAX_OUTPUT_BYTES), - } satisfies CapturedOutput; - await handle.close(); - handle = undefined; - return ok(captured); - } catch (cause: unknown) { - const failure = cause instanceof Error ? cause.message : String(cause); - if (handle !== undefined) { - try { - await handle.close(); - } catch (closeCause: unknown) { - return err( - `${failure}; closing capture file failed: ${closeCause instanceof Error ? closeCause.message : String(closeCause)}`, - ); - } - } - return err(failure); - } -}; + }; + }); interface RetainedLldbResources { readonly root: PrivateRuntimeRoot; - readonly supervisor: ProviderProcessSupervisor | undefined; + readonly rootLease: AnalysisCleanupLease; + readonly files: LldbReadbackFiles; readonly pidPath: string; - readonly targetIdentity: string | undefined; + supervisor: ProviderProcessSupervisor | undefined; + targetIdentity: string | undefined; + targetPid: number | undefined; processStopped: boolean; targetStopped: boolean; } +const errorReason = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + +/** Root release certifies the hierarchy only after every child confirms completion. */ +const releaseLldbRuntime = async ( + owner: RetainedLldbResources, +): Promise => { + if (!owner.processStopped || !owner.targetStopped || owner.files.hasPending) + throw new Error("LLDB runtime still owns unresolved child resources"); + await owner.rootLease.release(); +}; + +const bindLldbCleanup = ( + failure: Failure, + owner: RetainedLldbResources, +): Failure => { + if (failure.cleanupIncomplete) { + owner.rootLease.bindCleanupError(failure); + failure.bindCleanupResourceOwner({ + get cleanupResources() { + return lldbCleanupResources(owner); + }, + }); + } + return failure; +}; + +const lldbCleanupFailure = ( + owner: RetainedLldbResources, + reason: string, + cause: unknown, + partialObservation?: NativeCallPartialObservation, +): ProviderCleanupError => + bindLldbCleanup( + new ProviderCleanupError( + PROVIDER, + lldbCleanupResources(owner), + { reason }, + { operation: OPERATION, cause, partialObservation }, + ), + owner, + ); + type LldbOutcome = | { readonly kind: "exited"; @@ -821,16 +959,13 @@ type LldbExit = LldbOutcome & { readonly cleanupOwner?: ProviderProcessSupervisor; }; -const lldbCleanupResources = async ( - owner: RetainedLldbResources, -): Promise => [ +const lldbCleanupResources = (owner: RetainedLldbResources): string[] => [ ...(owner.processStopped ? [] : ["lldb-process-group"]), ...(owner.targetStopped ? [] - : [ - `native-target:${String((await readTargetPid(owner.pidPath)) ?? "unknown")}`, - ]), - owner.root.path, + : [`native-target:${String(owner.targetPid ?? "unknown")}`]), + ...owner.files.cleanupResources, + ...(owner.rootLease.released ? [] : [owner.root.path]), ]; /** Supervise LLDB as an owned process group while observing its target PID identity. */ @@ -843,6 +978,7 @@ const runLldb = async ( readonly pidPath: string; readonly identityAckPath: string; readonly environment: Readonly; + readonly files: LldbReadbackFiles; }, ): Promise => { if (options.signal?.aborted === true) @@ -894,7 +1030,7 @@ const runLldb = async ( ); let identityAckWritten = false; while (!(await supervisor.waitForOutputClose(20))) { - const pid = await readTargetPid(options.pidPath); + const pid = await options.files.readTargetPid(options.pidPath); if (pid !== undefined && !identityAckWritten) { let observation: ProcessIdentityObservation | undefined; try { @@ -946,21 +1082,6 @@ const runLldb = async ( } }; -/** - * Confirm target exit, and signal a survivor only when its OS start identity - * still matches the one captured while it was stopped at entry. - */ -const readTargetPid = async (pidPath: string): Promise => { - try { - const contents = (await readFile(pidPath, "utf8")).trim(); - if (!/^[1-9]\d*$/u.test(contents)) return undefined; - const pid = Number(contents); - return Number.isSafeInteger(pid) ? pid : undefined; - } catch { - return undefined; - } -}; - type TargetTermination = | "terminated" | "identity-unavailable" @@ -969,14 +1090,10 @@ type TargetTermination = /** Only signal a surviving target when its captured OS start identity still matches. */ const ensureTerminated = async ( - pidPath: string, + pid: number | undefined, identity: string | undefined, - missingPidIsUnknown: boolean, ): Promise => { - const pid = await readTargetPid(pidPath); - if (pid === undefined) - return missingPidIsUnknown ? "identity-unavailable" : "terminated"; - if (!Number.isSafeInteger(pid) || pid <= 0) return "identity-unavailable"; + if (pid === undefined) return "identity-unavailable"; let signalDisposition: Exclude | undefined; for (let attempt = 0; attempt < 20; attempt++) { if (!alive(pid)) return "terminated"; diff --git a/src/native/LldbReadbackFiles.ts b/src/native/LldbReadbackFiles.ts new file mode 100644 index 0000000000..3ebd4a9bbb --- /dev/null +++ b/src/native/LldbReadbackFiles.ts @@ -0,0 +1,97 @@ +import type { FileHandle } from "node:fs/promises"; +import { err, ok, type Result } from "../domain/result.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { + ArtifactFileCleanupFailure, + withArtifactRegularFile, +} from "../artifacts/ArtifactFileOperation.js"; +import type { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js"; + +/** One trace's readback descriptors, separate from the facts already read. */ +export class LldbReadbackFiles { + readonly #resources = new ArtifactResourceScope(); + readonly #targetPids = new Map(); + readonly #failed = new Map< + AnalysisCleanupLease, + { readonly path: string; readonly cause: ArtifactFileCleanupFailure } + >(); + + get hasPending(): boolean { + return this.#resources.hasPending; + } + + get cleanupResources(): readonly string[] { + return [ + ...new Set( + [...this.#failed] + .filter(([lease]) => !lease.released) + .map(([, failure]) => failure.path), + ), + ]; + } + + get cleanupCause(): unknown { + const failures = [...this.#failed] + .filter(([lease]) => !lease.released) + .map(([, failure]) => failure.cause); + return failures.length === 0 + ? undefined + : new AggregateError( + failures, + "LLDB readback descriptors remain open or unconfirmed", + ); + } + + /** Preserve a completed value even if final close transfers its owner here. */ + async read( + path: string, + read: (handle: FileHandle) => Promise, + ): Promise> { + const blocked = [...this.#failed].find( + ([lease, failure]) => !lease.released && failure.path === path, + ); + if (blocked !== undefined) return err(blocked[1].cause); + let completed: { readonly value: Value } | undefined; + try { + return ok( + await withArtifactRegularFile( + path, + this.#resources, + undefined, + async (handle) => { + const value = await read(handle); + completed = { value }; + return value; + }, + ), + ); + } catch (cause: unknown) { + if (cause instanceof ArtifactFileCleanupFailure) + this.#failed.set(cause.cleanupLease, { path, cause }); + return completed === undefined ? err(cause) : ok(completed.value); + } + } + + /** The producer publishes one target PID per runtime; retain its parsed fact. */ + async readTargetPid(path: string): Promise { + const known = this.#targetPids.get(path); + if (known !== undefined) return known; + const contents = await this.read(path, async (file) => + (await file.readFile("utf8")).trim(), + ); + if (!contents.ok || !/^[1-9]\d*$/u.test(contents.value)) return undefined; + const pid = Number(contents.value); + if (!Number.isSafeInteger(pid)) return undefined; + this.#targetPids.set(path, pid); + return pid; + } + + async retryCleanup(): Promise { + try { + await this.#resources.retryCleanup(); + } finally { + for (const lease of this.#failed.keys()) + if (lease.released) this.#failed.delete(lease); + } + } +} diff --git a/src/native/LldbRetainedObservations.ts b/src/native/LldbRetainedObservations.ts index 61d31ce015..7253607bb3 100644 --- a/src/native/LldbRetainedObservations.ts +++ b/src/native/LldbRetainedObservations.ts @@ -1,4 +1,6 @@ -import { open } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; +import type { LldbReadbackFiles } from "./LldbReadbackFiles.js"; +import type { Result } from "../domain/result.js"; import { nativeCallEventSchema } from "../domain/native/nativeCallObservation.js"; import type { NativeCallPartialObservation } from "../domain/native/nativeCallPartialObservation.js"; @@ -153,10 +155,7 @@ export const readBoundedPrefix = async ( return { buffer, bytesRead }; }; -const readPrefix = ( - handle: Awaited>, - requestedBytes: number, -) => +const readPrefix = (handle: FileHandle, requestedBytes: number) => readBoundedPrefix( requestedBytes, async (buffer, offset, length, position) => @@ -164,54 +163,42 @@ const readPrefix = ( ); /** Read at most the bridge's journal bound, checking size before allocation. */ -export const readLldbObservationJournal = async ( +export const readLldbObservationJournal = ( path: string, -): Promise<{ - readonly parsed: ParsedLldbJournal; - readonly limitations: string[]; -}> => { - let handle: Awaited> | undefined; - try { - handle = await open(path, "r"); + files: LldbReadbackFiles, +): Promise< + Result< + { readonly parsed: ParsedLldbJournal; readonly limitations: string[] }, + unknown + > +> => + files.read(path, async (handle) => { const { size } = await handle.stat(); const length = Math.min(size, LLDB_OBSERVATION_JOURNAL_MAX_BYTES); const { buffer, bytesRead } = await readPrefix(handle, length); - const parsed = parseLldbObservationJournal(buffer.subarray(0, bytesRead)); - const limitations = [ - ...(size > LLDB_OBSERVATION_JOURNAL_MAX_BYTES - ? [ - "The observation journal exceeded 8 MiB; only its bounded prefix was read.", - ] - : []), - ...(bytesRead < length - ? [ - "The observation journal became shorter while being read; the available prefix was parsed.", - ] - : []), - ]; return { - parsed, - limitations, - }; - } catch (cause: unknown) { - return { - parsed: { events: [], otherStops: [], limitations: [] }, + parsed: parseLldbObservationJournal(buffer.subarray(0, bytesRead)), limitations: [ - `The observation journal could not be read: ${cause instanceof Error ? cause.message : String(cause)}`, + ...(size > LLDB_OBSERVATION_JOURNAL_MAX_BYTES + ? [ + "The observation journal exceeded 8 MiB; only its bounded prefix was read.", + ] + : []), + ...(bytesRead < length + ? [ + "The observation journal became shorter while being read; the available prefix was parsed.", + ] + : []), ], }; - } finally { - await handle?.close().catch(() => undefined); - } -}; + }); /** Retain an available output prefix without claiming an unreported total. */ -export const readPartialCapture = async ( +export const readPartialCapture = ( path: string, -): Promise => { - let handle: Awaited> | undefined; - try { - handle = await open(path, "r"); + files: LldbReadbackFiles, +): Promise> => + files.read(path, async (handle) => { const { size } = await handle.stat(); const length = Math.min(size, MAX_CAPTURE_BYTES); const { buffer, bytesRead } = await readPrefix(handle, length); @@ -229,9 +216,4 @@ export const readPartialCapture = async ( ] : [], }; - } catch { - return null; - } finally { - await handle?.close().catch(() => undefined); - } -}; + }); diff --git a/src/native/NativeCallObservation.ts b/src/native/NativeCallObservation.ts index 6b7ec75863..141c707755 100644 --- a/src/native/NativeCallObservation.ts +++ b/src/native/NativeCallObservation.ts @@ -22,6 +22,10 @@ import { } from "./NativeCallTarget.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { err, ok, type Result } from "../domain/result.js"; +import { + nativeCallPartialObservationSchema, + type NativeCallPartialObservation, +} from "../domain/native/nativeCallPartialObservation.js"; import type { NativeCallTrace, NativeCallTracer } from "./LldbCallTracer.js"; const OPERATION = "observe_native_calls"; @@ -74,27 +78,19 @@ export const observeNativeCalls = async ( signal, ); if (!traced.ok) return traced; - const observation = projectNativeCalls( - target, - parsed.data, - traced.value, - ); return signal?.aborted ? err( new AnalysisCancelledError(OPERATION, { cause: signal.reason, - partialObservation: { - ...observation, - kind: "native-call-observation", - coverage: { - ...observation.coverage, - status: "partial", - reason: "cancelled", - }, - }, + partialObservation: projectPartialNativeCalls( + target, + parsed.data, + traced.value, + "cancelled", + ), }), ) - : ok(observation); + : ok(projectNativeCalls(target, parsed.data, traced.value)); } catch (cause: unknown) { // Unknown tracer/parser failures carry no file cleanup authority. return err( @@ -112,13 +108,13 @@ export const observeNativeCalls = async ( }; /** Join the request, the tracer's run and REA's own checks into the result contract. */ -export const projectNativeCalls = ( +const nativeCallFacts = ( target: Pick & { readonly architecture: string; }, input: NativeCallObservationInput, - trace: NativeCallTrace, -): NativeCallObservationResult => { + trace: NativeCallReadback, +) => { const { run } = trace; const unresolved = run.breakpoints .filter(({ location_count: count }) => count === 0) @@ -133,7 +129,7 @@ export const projectNativeCalls = ( run.outcome === "duration-elapsed" || run.outcome === "stop-limit" || unresolved.length > 0; - return nativeCallObservationResultSchema.parse({ + return { target: { path: target.path, sha256: target.sha256, @@ -227,10 +223,13 @@ export const projectNativeCalls = ( "Resolved breakpoint locations exceeded the aggregate 8 MiB metadata budget; omitted locations are counted in location_count. This metadata limit does not change the process outcome.", ] : []), - ...(trace.stdout.truncated || trace.stderr.truncated + ...(trace.stdout === null || trace.stderr === null + ? ["One or more target output captures were unavailable."] + : []), + ...(trace.stdout?.truncated === true || trace.stderr?.truncated === true ? ["Target output beyond 1 MiB per stream is counted but not kept."] : []), - ...(!trace.stdout.complete || !trace.stderr.complete + ...(trace.stdout?.complete === false || trace.stderr?.complete === false ? [ "Target output draining did not complete; reported byte counts are observed lower bounds, not stream totals.", ] @@ -241,5 +240,39 @@ export const projectNativeCalls = ( `REA could not confirm that process ${run.pid} exited; check for it before relying on host state.`, ]), ], + }; +}; + +/** Completed tracer metadata with independently available output captures. */ +type NativeCallReadback = Omit & { + readonly stdout: NativeCallTrace["stdout"] | null; + readonly stderr: NativeCallTrace["stderr"] | null; +}; + +export const projectNativeCalls = ( + target: Pick & { + readonly architecture: string; + }, + input: NativeCallObservationInput, + trace: NativeCallTrace, +): NativeCallObservationResult => + nativeCallObservationResultSchema.parse( + nativeCallFacts(target, input, trace), + ); + +/** Retain already parsed run facts when capture or finalization remains incomplete. */ +export const projectPartialNativeCalls = ( + target: Pick & { + readonly architecture: string; + }, + input: NativeCallObservationInput, + trace: NativeCallReadback, + reason: NativeCallPartialObservation["coverage"]["reason"], +): NativeCallPartialObservation => { + const facts = nativeCallFacts(target, input, trace); + return nativeCallPartialObservationSchema.parse({ + ...facts, + kind: "native-call-observation", + coverage: { ...facts.coverage, status: "partial", reason }, }); }; diff --git a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts index abc71ecb34..1489045307 100644 --- a/tests/boundary/providers/native/lldbCallTracerFailures.test.ts +++ b/tests/boundary/providers/native/lldbCallTracerFailures.test.ts @@ -1,4 +1,12 @@ -import { access, readFile, rm, writeFile } from "node:fs/promises"; +import { + access, + readFile, + rm, + writeFile, + open, + stat, + type FileHandle, +} from "node:fs/promises"; import { dirname } from "node:path"; import { describe, expect, it, onTestFinished, vi } from "vitest"; @@ -6,6 +14,8 @@ import { z } from "zod"; import { AnalysisCancelledError } from "../../../../src/domain/analysisErrorCore.js"; import { EvidenceIntegrityError } from "../../../../src/domain/evidenceErrors.js"; +import { ArtifactFileCleanupFailure } from "../../../../src/artifacts/ArtifactFileOperation.js"; +import { resolvedAnalysisError } from "../../../../src/domain/analysisErrorBase.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; import { nativeCallObservationInputSchema } from "../../../../src/domain/native/nativeCallObservation.js"; import { nativeCallPartialObservationSchema } from "../../../../src/domain/native/nativeCallPartialObservation.js"; @@ -86,6 +96,7 @@ const configSchema = z.object({ observation_path: z.string(), stdout_capture_path: z.string(), stderr_capture_path: z.string(), + pid_path: z.string(), }); const configFromArguments = async (arguments_: readonly string[]) => { @@ -145,7 +156,15 @@ const fixtureTracer = ( if (configPath === undefined) throw new Error("fake LLDB launcher was not called"); const root = dirname(configPath); - await expect(access(root)).rejects.toThrow(); + if (outcome === "cancelled") await expect(access(root)).rejects.toThrow(); + else { + // The simulated launcher reported incomplete cleanup without returning + // its process owner. That absence cannot establish completion. + await expect(access(root)).resolves.toBeUndefined(); + onTestFinished(async () => { + await rm(root, { recursive: true, force: true }); + }); + } }, }; }; @@ -547,3 +566,303 @@ describe("LLDB retains ambiguous target ownership", () => { } }); }); + +it("retains journal descriptor cleanup after a cancelled run instead of removing its runtime", async () => { + let release: (() => Promise) | undefined; + let restoreRead: (() => void) | undefined; + let captured: FileHandle | undefined; + let allowClose = false; + let configPath: string | undefined; + let attempts = 0; + const tracer = new LldbCallTracer( + {}, + async (_executable, args) => { + const resolved = await configFromArguments(args); + configPath = resolved.configPath; + await writeFile( + resolved.config.observation_path, + JSON.stringify({ + kind: "other-stop", + reason: "controlled journal fact", + }) + "\n", + ); + const probe = await open(resolved.config.observation_path, "r"); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + const original = prototype.read; + const read = vi + .spyOn(prototype, "read") + .mockImplementationOnce(async function ( + this: FileHandle, + ...parameters: Parameters + ) { + captured = this; + release = this.close.bind(this); + this.close = async () => { + attempts += 1; + if (!allowClose) + throw new Error("controlled journal descriptor close failure"); + await release!(); + }; + return Reflect.apply(original, this, parameters); + }); + restoreRead = () => read.mockRestore(); + return { + kind: "cancelled", + targetLaunched: false, + targetIdentity: undefined, + cleanupFailure: undefined, + }; + }, + async () => ok(fakeTool), + ); + onTestFinished(async () => { + restoreRead?.(); + allowClose = true; + await tracer.close(); + await release?.(); + }); + const result = await tracer.trace(request); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected cancelled controlled run"); + const partial = nativeCallPartialObservationSchema.parse( + result.error.partialObservation, + ); + expect(partial.process.other_stops).toEqual(["controlled journal fact"]); + expect(attempts).toBeGreaterThan(0); + if (configPath === undefined) + throw new Error("Expected owned runtime config path"); + const rootRetained = await access(dirname(configPath)).then( + () => true, + () => false, + ); + expect({ + cleanupIncomplete: result.error.cleanupIncomplete, + rootRetained, + descriptorStillOpen: captured !== undefined && captured.fd >= 0, + }).toEqual({ + cleanupIncomplete: true, + rootRetained: true, + descriptorStillOpen: true, + }); + expect((await tracer.close()).ok).toBe(false); + allowClose = true; + expect((await tracer.close()).ok).toBe(true); + expect(result.error.cleanupIncomplete).toBe(false); + await expect(access(dirname(configPath))).rejects.toMatchObject({ + code: "ENOENT", + }); +}); + +/** Write the complete producer facts used by readback ownership controls. */ +const writeReadbackFixture = async ( + config: z.infer, + stdout: string, + stderr: string, +) => { + await writeFile( + config.observation_path, + JSON.stringify({ kind: "event", event: EVENT }) + "\n", + ); + await writeFile(config.stdout_capture_path, stdout); + await writeFile(config.stderr_capture_path, stderr); + await writeFile(config.pid_path, "12345\n"); + await writeFile( + config.result_path, + JSON.stringify({ + ...successfulTracerOutput, + target_output: { + ...successfulTracerOutput.target_output, + stdout_bytes: Buffer.byteLength(stdout), + stderr_bytes: Buffer.byteLength(stderr), + }, + }), + ); +}; + +const readbackPath = ( + config: z.infer, + selected: "journal" | "stdout" | "stderr" | "result" | "pid", +): string => + ({ + journal: config.observation_path, + stdout: config.stdout_capture_path, + stderr: config.stderr_capture_path, + result: config.result_path, + pid: config.pid_path, + })[selected]; + +const readbackCases = [ + ...(["journal", "stdout", "stderr", "result", "pid"] as const).map( + (selected) => ({ + selected, + readFails: false, + rootFails: false, + cancels: false, + }), + ), + { + selected: "stdout" as const, + readFails: true, + rootFails: false, + cancels: false, + }, + { + selected: "stdout" as const, + readFails: false, + rootFails: true, + cancels: false, + }, + { + selected: "stdout" as const, + readFails: false, + rootFails: false, + cancels: true, + }, +]; +for (const { selected, readFails, rootFails, cancels } of readbackCases) + it(`retains ${selected} readback cleanup and sibling facts (read fails: ${readFails}, root fails: ${rootFails})`, async () => { + const controller = new AbortController(); + const reason = { caller: "readback-final-close" }; + let allowClose = false; + let launches = 0; + let runtime: string | undefined; + let captured: FileHandle | undefined; + let originalClose: (() => Promise) | undefined; + let restore: (() => void) | undefined; + let restoreRoot: (() => void) | undefined; + const physicalFailure = new Error("controlled capture read failure"); + const tracer = new LldbCallTracer( + {}, + async (_executable, args) => { + launches++; + const { config } = await configFromArguments(args); + runtime = dirname(config.result_path); + await writeReadbackFixture(config, "stdout fact", "stderr fact"); + const selectedPath = readbackPath(config, selected); + const selectedStat = await stat(selectedPath); + const probe = await open(selectedPath, "r"); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + const arm = async (file: FileHandle) => { + if (captured !== undefined) return; + const metadata = await file.stat(); + if ( + metadata.ino !== selectedStat.ino || + metadata.dev !== selectedStat.dev + ) + return; + captured = file; + originalClose = file.close.bind(file); + file.close = async () => { + if (!allowClose) { + if (cancels) controller.abort(reason); + throw new Error("controlled selected descriptor cleanup failure"); + } + await originalClose!(); + }; + }; + const readMethod = prototype.read; + const readFileMethod = prototype.readFile; + const read = vi + .spyOn(prototype, "read") + .mockImplementation(async function ( + this: FileHandle, + ...parameters: Parameters + ) { + await arm(this); + if (readFails && captured === this) throw physicalFailure; + return Reflect.apply(readMethod, this, parameters); + }); + const readFile = vi + .spyOn(prototype, "readFile") + .mockImplementation(async function ( + this: FileHandle, + ...parameters: Parameters + ) { + await arm(this); + return Reflect.apply(readFileMethod, this, parameters); + }); + restore = () => { + read.mockRestore(); + readFile.mockRestore(); + }; + return selected === "journal" + ? { + kind: "cancelled", + targetLaunched: false, + targetIdentity: undefined, + cleanupFailure: undefined, + } + : { + kind: "exited", + targetLaunched: false, + targetIdentity: undefined, + cleanupFailure: undefined, + exitCode: 0, + output: "", + }; + }, + async () => ok(fakeTool), + ); + onTestFinished(async () => { + restore?.(); + restoreRoot?.(); + allowClose = true; + await tracer.close(); + await originalClose?.(); + }); + const result = await tracer.trace(request, controller.signal); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected retained readback cleanup"); + expect(result.error.cleanupIncomplete).toBe(true); + const partial = nativeCallPartialObservationSchema.parse( + result.error.partialObservation, + ); + expect(partial.events).toEqual([EVENT]); + if (selected === "pid") expect(partial.process.pid).toBe(12345); + if (readFails) { + expect(partial.process.stdout).toBeNull(); + const primary = result.error.cleanupPrimary; + if (!(primary?.cause instanceof AggregateError)) + throw new Error("Expected retained capture read failure"); + const frame = primary.cause.errors[0]; + if (!(frame instanceof ArtifactFileCleanupFailure)) + throw new Error("Expected exact file cleanup outcome"); + expect(frame.fileCleanupFailure.outcome).toEqual({ + kind: "failed", + cause: physicalFailure, + }); + } else expect(partial.process.stdout?.text).toBe("stdout fact"); + expect(partial.process.stderr?.text).toBe("stderr fact"); + expect(captured?.fd).toBeGreaterThanOrEqual(0); + if (runtime === undefined) throw new Error("Missing owned runtime"); + await expect(access(runtime)).resolves.toBeUndefined(); + const replacement = await tracer.trace(request); + expect(replacement.ok).toBe(false); + expect(launches).toBe(1); + allowClose = true; + if (rootFails) { + const close = vi + .spyOn(PrivateRuntimeRoot.prototype, "close") + .mockRejectedValueOnce(new Error("controlled parent cleanup failure")); + restoreRoot = () => close.mockRestore(); + const rootRetry = await tracer.close(); + expect(rootRetry.ok).toBe(false); + expect(captured?.fd).toBe(-1); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cleanupResources).toEqual([runtime]); + expect((await tracer.close()).ok).toBe(true); + if (!rootRetry.ok) expect(rootRetry.error.cleanupIncomplete).toBe(false); + } else expect((await tracer.close()).ok).toBe(true); + expect(result.error.cleanupIncomplete).toBe(false); + if (cancels) + expect(resolvedAnalysisError(result.error)).toMatchObject({ + _tag: "AnalysisCancelledError", + cause: reason, + }); + expect(result.error.cleanupResources).toEqual([]); + if (!replacement.ok) + expect(replacement.error.cleanupIncomplete).toBe(false); + await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); + });