diff --git a/src/artifacts/SafeOutputTree.ts b/src/artifacts/SafeOutputTree.ts index 8d32ac5dcb..a20afb7f9d 100644 --- a/src/artifacts/SafeOutputTree.ts +++ b/src/artifacts/SafeOutputTree.ts @@ -5,11 +5,11 @@ import { open, readdir, realpath, - rmdir, type FileHandle, } from "node:fs/promises"; import { basename, dirname, isAbsolute, join, resolve } from "node:path"; import type { Readable } from "node:stream"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import { streamChunkToBuffer } from "./StreamBytes.js"; import { @@ -18,13 +18,14 @@ import { normalizeArtifactPath, } from "./ArtifactPaths.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import { SafeOutputTreeCreationFailure } from "./SafeOutputTreeCreationFailure.js"; +import { hashOutputFile } from "./SafeOutputTreeReadback.js"; import { removeOwnedTree } from "./SafeOutputTreeCleanup.js"; import { assertHandleIdentity, assertFilePathIdentity, assertPathIdentity, isAbsent, - pathHasIdentity, readDirectoryIdentity, readFileIdentity, replacedDirectory, @@ -57,9 +58,12 @@ export type SafeOutputCleanup = export class SafeOutputTree { readonly #registry = new ArtifactPathRegistry(); readonly #outputRoot: string; - readonly #rootIdentity: DirectoryIdentity; + #rootIdentity: DirectoryIdentity | undefined; readonly #nestedDirectories = new Map(); + readonly #unknownDirectories = new Set(); readonly #ownedFiles = new Map(); + readonly #openFiles = new Map(); + readonly #openHandles = new Set(); #published = false; #cleanup: SafeOutputCleanup = { status: "not-required", @@ -67,11 +71,9 @@ export class SafeOutputTree { private constructor( outputRoot: string, - rootIdentity: DirectoryIdentity, private readonly platform: NodeJS.Platform, ) { this.#outputRoot = outputRoot; - this.#rootIdentity = rootIdentity; } /** @@ -113,57 +115,52 @@ export class SafeOutputTree { ); throw cause; }); - let rootIdentity: DirectoryIdentity | undefined; + const tree = new SafeOutputTree(canonicalOutput, platform); try { - rootIdentity = await readDirectoryIdentity(canonicalOutput); + const rootIdentity = await readDirectoryIdentity(canonicalOutput); + tree.#rootIdentity = rootIdentity; if (platform !== "win32") { - const stagingHandle = await open( - canonicalOutput, - constants.O_RDONLY | constants.O_DIRECTORY, + const stagingHandle = tree.#trackHandle( + await open( + canonicalOutput, + constants.O_RDONLY | constants.O_DIRECTORY, + ), ); + let setup: + | { readonly kind: "ready" } + | { + readonly kind: "failed"; + readonly cause: unknown; + } = { kind: "ready" }; try { await assertHandleIdentity( - stagingHandle, + stagingHandle.handle, rootIdentity, canonicalOutput, ); - await stagingHandle.chmod(0o700); - } finally { - await stagingHandle.close(); + await stagingHandle.handle.chmod(0o700); + } catch (cause: unknown) { + setup = { kind: "failed", cause }; } + try { + await tree.#closeHandle(stagingHandle); + } catch (closeCause: unknown) { + if (setup.kind === "failed") + throw ArtifactReaderFailure.withCleanup( + setup.cause, + ArtifactReaderFailure.cleanupObservation( + closeCause, + canonicalOutput, + ), + ); + throw closeCause; + } + if (setup.kind === "failed") throw setup.cause; } await assertPathIdentity(canonicalOutput, rootIdentity); - return new SafeOutputTree(canonicalOutput, rootIdentity, platform); + return tree; } catch (cause: unknown) { - let removalFailure: unknown; - let ownedRootAtPath = false; - try { - ownedRootAtPath = - rootIdentity !== undefined && - (await pathHasIdentity(canonicalOutput, rootIdentity)); - if (ownedRootAtPath) await rmdir(canonicalOutput); - else - removalFailure = new Error( - "Extraction output root identity changed during setup", - ); - } catch (cleanupCause: unknown) { - removalFailure = cleanupCause; - } - let absent = false; - try { - absent = ownedRootAtPath && (await isAbsent(canonicalOutput)); - } catch (cleanupCause: unknown) { - removalFailure ??= cleanupCause; - } - if (!absent) - throw ArtifactReaderFailure.withCleanup(cause, { - reason: - removalFailure instanceof Error - ? removalFailure.message - : "Extraction output root ownership could not be verified after setup failure", - resources: [canonicalOutput], - }); - throw cause; + throw new SafeOutputTreeCreationFailure(cause, tree); } } @@ -171,6 +168,11 @@ export class SafeOutputTree { return this.#outputRoot; } + /** The output was durably committed, even if descriptor cleanup later failed. */ + get published(): boolean { + return this.#published; + } + get cleanup(): SafeOutputCleanup { return structuredClone(this.#cleanup); } @@ -211,58 +213,62 @@ export class SafeOutputTree { { cause }, ); }); - const fileIdentity = await readFileIdentity(handle, destination).catch( - async (cause: unknown) => { - await handle.close().catch(() => undefined); - throw cause; - }, + const ownedHandle = this.#trackHandle(handle); + this.#openFiles.set(destination, ownedHandle); + const fileIdentity = await readFileIdentity( + ownedHandle.handle, + destination, ); + await assertFilePathIdentity(destination, fileIdentity); this.#ownedFiles.set(destination, fileIdentity); const hash = createHash("sha256"); let bytes = 0; - try { - await this.#assertLineage(lineage); - for await (const raw of source) { - abortIfNeeded(signal); - const chunk = streamChunkToBuffer(raw); - if (chunk.length > expected.bytes - bytes) - throw new ArtifactReaderFailure( - "integrity", - `Extracted content exceeds the inventoried size: ${path}`, - ); - bytes += chunk.length; - hash.update(chunk); - await writeAll(handle, chunk); - } - if (bytes !== expected.bytes) + await this.#assertLineage(lineage); + for await (const raw of source) { + abortIfNeeded(signal); + const chunk = streamChunkToBuffer(raw); + if (chunk.length > expected.bytes - bytes) throw new ArtifactReaderFailure( "integrity", - `Extracted content size disagrees with inventory: ${path}`, + `Extracted content exceeds the inventoried size: ${path}`, ); - const sha256 = hash.digest("hex"); - if (sha256 !== expected.sha256) - throw new ArtifactReaderFailure( - "integrity", - `Extracted content disagrees with inventory: ${path}`, - ); - await this.#assertLineage(lineage); - await assertFilePathIdentity(destination, fileIdentity); - await handle.sync(); - await handle.close(); - const readback = await hashFile(destination, bytes, signal); - await this.#assertLineage(lineage); - await assertFilePathIdentity(destination, fileIdentity); - if (readback.sha256 !== sha256 || readback.bytes !== bytes) - throw new ArtifactReaderFailure( - "integrity", - `Durable readback verification failed: ${path}`, - ); - return { relativePath: path, sha256, bytesWritten: bytes }; - } catch (cause: unknown) { - // best-effort cleanup: file-handle close must not mask the write failure. - await handle.close().catch(() => undefined); - throw cause; + bytes += chunk.length; + hash.update(chunk); + await writeAll(ownedHandle.handle, chunk); } + if (bytes !== expected.bytes) + throw new ArtifactReaderFailure( + "integrity", + `Extracted content size disagrees with inventory: ${path}`, + ); + const sha256 = hash.digest("hex"); + if (sha256 !== expected.sha256) + throw new ArtifactReaderFailure( + "integrity", + `Extracted content disagrees with inventory: ${path}`, + ); + await this.#assertLineage(lineage); + await assertFilePathIdentity(destination, fileIdentity); + await ownedHandle.handle.sync(); + await this.#closeFileHandle(destination, ownedHandle); + const readbackHandle = this.#trackHandle( + await open(destination, constants.O_RDONLY | constants.O_NOFOLLOW), + ); + const readback = await hashOutputFile( + readbackHandle, + destination, + fileIdentity, + { maximum: bytes, signal }, + ); + await this.#closeHandle(readbackHandle); + await this.#assertLineage(lineage); + await assertFilePathIdentity(destination, fileIdentity); + if (readback.sha256 !== sha256 || readback.bytes !== bytes) + throw new ArtifactReaderFailure( + "integrity", + `Durable readback verification failed: ${path}`, + ); + return { relativePath: path, sha256, bytesWritten: bytes }; } catch (cause: unknown) { try { source.destroy(); @@ -282,34 +288,80 @@ export class SafeOutputTree { this.#published = true; return; } - const parent = await open( - dirname(this.#outputRoot), - constants.O_RDONLY | constants.O_DIRECTORY, - ); - let output: FileHandle | undefined; - try { - output = await open( - this.#outputRoot, + const parent = this.#trackHandle( + await open( + dirname(this.#outputRoot), constants.O_RDONLY | constants.O_DIRECTORY, + ), + ); + let output: OwnedFileHandle | undefined; + let sync: + | { readonly kind: "synced" } + | { + readonly kind: "failed"; + readonly cause: unknown; + } = { kind: "synced" }; + try { + output = this.#trackHandle( + await open( + this.#outputRoot, + constants.O_RDONLY | constants.O_DIRECTORY, + ), ); - await assertHandleIdentity(output, this.#rootIdentity, this.#outputRoot); - await output.sync(); - await parent.sync(); + await assertHandleIdentity( + output.handle, + this.#requireRootIdentity(), + this.#outputRoot, + ); + await output.handle.sync(); + await parent.handle.sync(); this.#published = true; } catch (cause: unknown) { + sync = { kind: "failed", cause }; + } + const closeResults = await Promise.allSettled( + [parent, output] + .filter(isDefined) + .map((handle) => this.#closeHandle(handle)), + ); + const closeFailures = closeResults + .filter(isRejected) + .map(({ reason }) => reason); + if (closeFailures.length > 0) { + const closeFailure = new AggregateError( + closeFailures, + "Extraction output directory handles could not be closed", + ); + const cleanup = ArtifactReaderFailure.cleanupObservation( + closeFailure, + this.#outputRoot, + ); + if (sync.kind === "failed") + throw ArtifactReaderFailure.withCleanup( + new ArtifactReaderFailure( + "path", + "Could not durably sync extraction output", + { cause: sync.cause }, + ), + cleanup, + ); + throw ArtifactReaderFailure.withCleanup(closeFailure, cleanup); + } + if (sync.kind === "failed") throw new ArtifactReaderFailure( "path", "Could not durably sync extraction output", - { cause }, + { cause: sync.cause }, ); - } finally { - await Promise.allSettled([parent.close(), output?.close()]); - } } /** Remove only this operation's unsealed tree and verify absence. */ async rollback(): Promise { - if (this.#published || this.#cleanup.status === "complete") + if (this.#published) { + await this.#closePendingHandles(); + return structuredClone(this.#cleanup); + } + if (this.#cleanup.status === "complete") return structuredClone(this.#cleanup); this.#cleanup = { status: "incomplete", @@ -317,10 +369,12 @@ export class SafeOutputTree { }; let removalFailure: unknown; try { + await this.#admitPendingFiles(); + await this.#closePendingHandles(); await this.#assertOwnedDirectories(); await removeOwnedTree({ outputRoot: this.#outputRoot, - rootIdentity: this.#rootIdentity, + rootIdentity: this.#requireRootIdentity(), directories: this.#nestedDirectories, files: this.#ownedFiles, }); @@ -362,10 +416,10 @@ export class SafeOutputTree { let current = this.#outputRoot; let logicalParent = ""; const lineage: DirectoryLineageEntry[] = [ - { path: this.#outputRoot, identity: this.#rootIdentity }, + { path: this.#outputRoot, identity: this.#requireRootIdentity() }, ]; if (parts.length === 0) - await assertPathIdentity(this.#outputRoot, this.#rootIdentity); + await assertPathIdentity(this.#outputRoot, this.#requireRootIdentity()); for (const part of parts) { const parent = lineage.at(-1); if (parent === undefined) throw replacedDirectory(this.#outputRoot); @@ -384,8 +438,13 @@ export class SafeOutputTree { if (identity === undefined) throw replacedDirectory(current); await assertPathIdentity(current, identity); } else { - const identity = await readDirectoryIdentity(current); - this.#nestedDirectories.set(current, identity); + try { + const identity = await readDirectoryIdentity(current); + this.#nestedDirectories.set(current, identity); + } catch (cause: unknown) { + this.#unknownDirectories.add(current); + throw cause; + } } await assertPathIdentity(parent.path, parent.identity); const identity = this.#nestedDirectories.get(current); @@ -405,9 +464,24 @@ export class SafeOutputTree { } async #assertOwnedDirectories(): Promise { - await assertPathIdentity(this.#outputRoot, this.#rootIdentity); + const rootIdentity = this.#requireRootIdentity(); + await assertPathIdentity(this.#outputRoot, rootIdentity); for (const [path, identity] of this.#nestedDirectories) await assertPathIdentity(path, identity); + if (this.#unknownDirectories.size > 0) + throw new ArtifactReaderFailure( + "path", + `Extraction directory ownership is unknown: ${[...this.#unknownDirectories].join(", ")}`, + ); + } + + #requireRootIdentity(): DirectoryIdentity { + if (this.#rootIdentity === undefined) + throw new ArtifactReaderFailure( + "path", + `Extraction output root identity is unknown: ${this.#outputRoot}`, + ); + return this.#rootIdentity; } #assertWritable(): void { @@ -422,39 +496,59 @@ export class SafeOutputTree { "Extraction tree cleanup has already started", ); } + + #trackHandle(handle: FileHandle): OwnedFileHandle { + const owner = new OwnedFileHandle(handle); + this.#openHandles.add(owner); + return owner; + } + + async #closeHandle(owner: OwnedFileHandle): Promise { + await owner.close(); + this.#openHandles.delete(owner); + } + + async #closePendingHandles(): Promise { + const failures: unknown[] = []; + for (const owner of this.#openHandles) { + try { + await this.#closeHandle(owner); + } catch (cause: unknown) { + failures.push(cause); + } + } + if (failures.length > 0) + throw new AggregateError(failures, "Output tree handles remain open"); + } + + async #closeFileHandle(path: string, owner: OwnedFileHandle): Promise { + await this.#closeHandle(owner); + this.#openFiles.delete(path); + } + + async #admitPendingFiles(): Promise { + for (const [path, owner] of this.#openFiles) { + let identity = this.#ownedFiles.get(path); + if (identity === undefined) { + identity = await readFileIdentity(owner.handle, path); + await assertFilePathIdentity(path, identity); + this.#ownedFiles.set(path, identity); + } + } + } } +const isDefined = (value: T | undefined): value is T => value !== undefined; + +const isRejected = ( + result: PromiseSettledResult, +): result is PromiseRejectedResult => result.status === "rejected"; + type DirectoryLineageEntry = { readonly path: string; readonly identity: DirectoryIdentity; }; -const hashFile = async ( - path: string, - maximum: number, - signal?: AbortSignal, -): Promise<{ readonly sha256: string; readonly bytes: number }> => { - const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); - const hash = createHash("sha256"); - let bytes = 0; - try { - for await (const raw of handle.createReadStream({ autoClose: false })) { - abortIfNeeded(signal); - const chunk = streamChunkToBuffer(raw); - bytes += chunk.length; - if (bytes > maximum) - throw new ArtifactReaderFailure( - "integrity", - "Readback exceeded the bytes written", - ); - hash.update(chunk); - } - } finally { - await handle.close(); - } - return { sha256: hash.digest("hex"), bytes }; -}; - const writeAll = async ( handle: Awaited>, chunk: Buffer, @@ -475,6 +569,14 @@ const writeAll = async ( } }; +const abortIfNeeded = (signal?: AbortSignal): void => { + if (signal?.aborted === true) + throw new ArtifactReaderFailure( + "cancelled", + "Artifact extraction cancelled", + ); +}; + const isAlreadyExists = (cause: unknown): boolean => cause instanceof Error && "code" in cause && cause.code === "EEXIST"; @@ -494,11 +596,3 @@ const throwIfDestinationCaseCollision = async ( cause === undefined ? undefined : { cause }, ); }; - -const abortIfNeeded = (signal?: AbortSignal): void => { - if (signal?.aborted === true) - throw new ArtifactReaderFailure( - "cancelled", - "Artifact extraction cancelled", - ); -}; diff --git a/src/artifacts/SafeOutputTreeCreationFailure.ts b/src/artifacts/SafeOutputTreeCreationFailure.ts new file mode 100644 index 0000000000..d8f132e352 --- /dev/null +++ b/src/artifacts/SafeOutputTreeCreationFailure.ts @@ -0,0 +1,31 @@ +import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import type { SafeOutputTree } from "./SafeOutputTree.js"; + +/** Setup failed after this tree was acquired; callers still own this exact tree. */ +export class SafeOutputTreeCreationFailure extends ArtifactReaderFailure { + readonly tree: SafeOutputTree; + + constructor(cause: unknown, tree: SafeOutputTree) { + const primary = + cause instanceof ArtifactReaderFailure + ? cause + : new ArtifactReaderFailure("io", errorMessage(cause), { cause }); + super( + primary.reason, + primary.message, + { + cause: primary, + ...(primary.cleanup === undefined ? {} : { cleanup: primary.cleanup }), + ...(primary.partialObservation === undefined + ? {} + : { partialObservation: primary.partialObservation }), + }, + primary.details, + ); + this.name = "SafeOutputTreeCreationFailure"; + this.tree = tree; + } +} + +const errorMessage = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); diff --git a/src/artifacts/SafeOutputTreeIdentity.ts b/src/artifacts/SafeOutputTreeIdentity.ts index d93cda4236..038a95ba22 100644 --- a/src/artifacts/SafeOutputTreeIdentity.ts +++ b/src/artifacts/SafeOutputTreeIdentity.ts @@ -71,20 +71,6 @@ export const assertFilePathIdentity = async ( ); }; -export const pathHasIdentity = async ( - path: string, - expected: DirectoryIdentity, -): Promise => { - try { - await assertPathIdentity(path, expected); - return true; - } catch (cause: unknown) { - if (isNotFound(cause) || cause instanceof ArtifactReaderFailure) - return false; - throw cause; - } -}; - export const isAbsent = async (path: string): Promise => lstat(path).then( () => false, diff --git a/src/artifacts/SafeOutputTreeReadback.ts b/src/artifacts/SafeOutputTreeReadback.ts new file mode 100644 index 0000000000..3cee16f488 --- /dev/null +++ b/src/artifacts/SafeOutputTreeReadback.ts @@ -0,0 +1,62 @@ +import { createHash } from "node:crypto"; +import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js"; +import { streamChunkToBuffer } from "./StreamBytes.js"; + +import { ArtifactReaderFailure } from "./ArtifactReader.js"; +import { + assertFilePathIdentity, + readFileIdentity, + type FileIdentity, +} from "./SafeOutputTreeIdentity.js"; + +/** Hash one retained readback descriptor and confirm it still names the output file. */ +export const hashOutputFile = async ( + owner: OwnedFileHandle, + path: string, + identity: FileIdentity, + options: { + readonly maximum: number; + readonly signal?: AbortSignal | undefined; + }, +): Promise<{ readonly sha256: string; readonly bytes: number }> => { + const admittedIdentity = await readFileIdentity(owner.handle, path); + await assertFilePathIdentity(path, identity); + if ( + admittedIdentity.dev !== identity.dev || + admittedIdentity.ino !== identity.ino + ) + throw new ArtifactReaderFailure( + "path", + `Extraction file identity changed: ${path}`, + ); + const hash = createHash("sha256"); + let bytes = 0; + try { + for await (const raw of readFileHandleChunks(owner.handle, options)) { + if (options.signal?.aborted === true) + throw new ArtifactReaderFailure( + "cancelled", + "Artifact extraction cancelled", + ); + const chunk = streamChunkToBuffer(raw); + bytes += chunk.length; + if (bytes > options.maximum) + throw new ArtifactReaderFailure( + "integrity", + "Readback exceeded the bytes written", + ); + hash.update(chunk); + } + } catch (cause: unknown) { + if (options.signal?.aborted === true) + throw new ArtifactReaderFailure( + "cancelled", + "Artifact extraction cancelled", + { cause }, + ); + throw cause; + } + await assertFilePathIdentity(path, identity); + return { sha256: hash.digest("hex"), bytes }; +}; diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 79f384eef7..f38f601c17 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -15,6 +15,7 @@ import { import { abortIfNeeded } from "../ArtifactHash.js"; import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { SafeOutputTree } from "../SafeOutputTree.js"; +import { SafeOutputTreeCreationFailure } from "../SafeOutputTreeCreationFailure.js"; import { ZipArtifactReader } from "../ZipArtifactReader.js"; import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js"; import { @@ -220,9 +221,15 @@ const materializeSelection = async ({ }; let localReaderOwner: ArtifactResourceOwner | undefined = readerOwner; let output: SafeOutputTree | undefined; + let completed: ArtifactExtractionResult | undefined; const extracted: ExtractedOccurrence[] = []; try { - output = await SafeOutputTree.create(input.outputRoot); + try { + output = await SafeOutputTree.create(input.outputRoot); + } catch (cause: unknown) { + if (cause instanceof SafeOutputTreeCreationFailure) output = cause.tree; + throw cause; + } await writeSelectedEntries({ reader, output, byPath, extracted, signal }); localReaderOwner = undefined; const closeAttempt = await input.resourceScope.release(readerOwner); @@ -237,14 +244,9 @@ const materializeSelection = async ({ extracted.sort((left, right) => compareUnicodeCodePoints(left.relative_path, right.relative_path), ); - const result = createExtractionResult( - input, - inventory, - selected, - extracted, - ); + completed = createExtractionResult(input, inventory, selected, extracted); await output.commit(); - return result; + return completed; } catch (cause: unknown) { const cleanupFailures = await releaseExtractionOwners( input.resourceScope, @@ -257,13 +259,20 @@ const materializeSelection = async ({ ({ cause: cleanupCause, resource }) => ArtifactReaderFailure.cleanupObservation(cleanupCause, resource), ); - throw ArtifactReaderFailure.withCleanup(cause, { - reason: observations.map(({ reason }) => reason).join("; "), - resources: [ - ...new Set(observations.flatMap(({ resources }) => resources)), - ], - }); + throw ArtifactReaderFailure.withCleanup( + cause, + { + reason: observations.map(({ reason }) => reason).join("; "), + resources: [ + ...new Set(observations.flatMap(({ resources }) => resources)), + ], + }, + output?.published === true && completed !== undefined + ? { kind: "artifact-extraction", extraction: completed } + : undefined, + ); } + if (output?.published === true && completed !== undefined) return completed; throw cause; } }; diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index afe271b8e6..71dad30c3a 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -94,7 +94,12 @@ import type { FileOffsetPartialObservation } from "./native/fileOffsetPartialObs import type { NativeCallPartialObservation } from "./native/nativeCallPartialObservation.js"; import type { ElectronActivePartialObservation } from "./javascript/electronActiveObservation.js"; import type { JavaScriptRuntimeObservation } from "./javascript/javascriptRuntimeObservation.js"; -import type { ArtifactInventoryPartialObservation } from "./artifactPartialObservation.js"; +import type { + ArtifactInventoryPartialObservation, + ArtifactExtractionPartialObservation, +} from "./artifactPartialObservation.js"; +import type { WebScriptExportPartialObservation } from "./webScriptExport.js"; +import type { JavaScriptRecoveryPartialObservation } from "./javascript/javascriptRecovery.js"; import type { BrowserScenarioPartialObservation } from "./browserScenarioCapture.js"; import type { Evidence } from "./evidence.js"; import type { AndroidPartialObservation } from "./android/androidPartialObservation.js"; @@ -114,6 +119,9 @@ export type AnalysisPartialObservation = | BrowserScenarioPartialObservation | JavaScriptRuntimeObservation | ArtifactInventoryPartialObservation + | ArtifactExtractionPartialObservation + | WebScriptExportPartialObservation + | JavaScriptRecoveryPartialObservation | Evidence | AndroidPartialObservation | FirmwarePartialObservation diff --git a/src/domain/artifactPartialObservation.ts b/src/domain/artifactPartialObservation.ts index d813788c01..10baa50f02 100644 --- a/src/domain/artifactPartialObservation.ts +++ b/src/domain/artifactPartialObservation.ts @@ -1,7 +1,14 @@ import type { ArtifactInventorySnapshot } from "./artifactInventorySnapshot.js"; +import type { ArtifactExtractionResult } from "./artifactGraph.js"; /** A complete inventory retained when releasing its provider resources fails. */ export interface ArtifactInventoryPartialObservation { readonly kind: "artifact-inventory"; readonly inventory: ArtifactInventorySnapshot; } + +/** Durable extracted files retained when their descriptor cleanup is unresolved. */ +export interface ArtifactExtractionPartialObservation { + readonly kind: "artifact-extraction"; + readonly extraction: ArtifactExtractionResult; +} diff --git a/src/domain/javascript/javascriptRecovery.ts b/src/domain/javascript/javascriptRecovery.ts index f5cf2b8052..f12d9ed4dc 100644 --- a/src/domain/javascript/javascriptRecovery.ts +++ b/src/domain/javascript/javascriptRecovery.ts @@ -106,3 +106,9 @@ export type JavaScriptRecoveryInput = z.output< export type JavaScriptRecoveryResult = z.output< typeof javascriptRecoveryResultSchema >; + +/** Completed publication retained when cleanup fails after durable commit. */ +export interface JavaScriptRecoveryPartialObservation { + readonly kind: "javascript-recovery"; + readonly result: JavaScriptRecoveryResult; +} diff --git a/src/domain/webScriptExport.ts b/src/domain/webScriptExport.ts index 7e3dc36b8b..c52b08a296 100644 --- a/src/domain/webScriptExport.ts +++ b/src/domain/webScriptExport.ts @@ -104,6 +104,12 @@ export type WebScriptExportResult = z.output< typeof webScriptExportResultSchema >; +/** Completed publication retained when cleanup fails after durable commit. */ +export interface WebScriptExportPartialObservation { + readonly kind: "web-script-export"; + readonly result: WebScriptExportResult; +} + /** Adapter input to the provider-neutral path planner and publisher. */ export interface CapturedWebScript { readonly source: ExportedWebScript["source"]; diff --git a/tests/boundary/filesystem/safeOutputTree.test.ts b/tests/boundary/filesystem/safeOutputTree.test.ts index c9b015b6ea..0f99765222 100644 --- a/tests/boundary/filesystem/safeOutputTree.test.ts +++ b/tests/boundary/filesystem/safeOutputTree.test.ts @@ -3,6 +3,7 @@ import { access, chmod, mkdir, + open, readFile, readdir, readlink, @@ -14,11 +15,12 @@ import { import { join } from "node:path"; import { Readable } from "node:stream"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js"; +import { SafeOutputTreeCreationFailure } from "../../../src/artifacts/SafeOutputTreeCreationFailure.js"; import { ArtifactReaderFailure } from "../../../src/artifacts/ArtifactReader.js"; describe("safe artifact output cleanup", () => { @@ -48,6 +50,109 @@ describe("safe artifact output cleanup", () => { ); }); +describe("safe artifact output file identity admission", () => { + it.skipIf(process.platform === "win32")( + "retains an O_EXCL file handle until its identity can be retried", + async () => { + const parent = await createTestTempDirectory("rea-safe-output-admit-"); + const output = join(parent, "published"); + const destination = join(output, "payload.txt"); + const displaced = join(parent, "owned-payload.txt"); + const outside = join(parent, "outside.txt"); + await writeFile(outside, "outside bytes"); + const prototypeHandle = await open(join(parent, "handle"), "w"); + const handlePrototype = Object.getPrototypeOf(prototypeHandle) as Pick< + typeof prototypeHandle, + "stat" + >; + await prototypeHandle.close(); + await rm(join(parent, "handle")); + const tree = await SafeOutputTree.create(output); + const originalStat = handlePrototype.stat; + const stat = vi.spyOn(handlePrototype, "stat"); + stat + .mockImplementationOnce(function ( + this: typeof prototypeHandle, + ...args + ) { + return originalStat.apply(this, args); + }) + .mockRejectedValueOnce(new Error("injected file identity failure")); + + await expect( + tree.write("payload.txt", Readable.from(Buffer.from("owned")), { + sha256: createHash("sha256").update("owned").digest("hex"), + bytes: Buffer.byteLength("owned"), + }), + ).rejects.toThrow("injected file identity failure"); + stat.mockRestore(); + await rename(destination, displaced); + await symlink(outside, destination, "file"); + + const failure = await tree.rollback().catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + expect(await readlink(destination)).toBe(outside); + expect(await readFile(outside, "utf8")).toBe("outside bytes"); + + await rm(destination); + await rename(displaced, destination); + expect(await tree.rollback()).toEqual({ + status: "complete", + residualPaths: [], + }); + expect(await readdir(parent)).toEqual(["outside.txt"]); + }, + ); +}); + +describe("safe artifact output setup ownership", () => { + it.skipIf(process.platform === "win32")( + "returns its exact tree owner when setup fails after root identity capture", + async () => { + const parent = await createTestTempDirectory("rea-safe-output-setup-"); + const output = join(parent, "published"); + const blocker = join(output, "unowned.txt"); + const prototypeHandle = await open(join(parent, "handle"), "w"); + const handlePrototype = Object.getPrototypeOf(prototypeHandle) as Pick< + typeof prototypeHandle, + "chmod" + >; + await prototypeHandle.close(); + await rm(join(parent, "handle")); + const chmodSpy = vi + .spyOn(handlePrototype, "chmod") + .mockImplementationOnce(async function (this: typeof prototypeHandle) { + await writeFile(blocker, "leave until cleanup retry"); + throw new Error("injected setup failure"); + }); + + const failure = await SafeOutputTree.create(output).catch( + (cause: unknown) => cause, + ); + chmodSpy.mockRestore(); + expect(failure).toBeInstanceOf(SafeOutputTreeCreationFailure); + if (!(failure instanceof SafeOutputTreeCreationFailure)) + throw new Error("expected setup failure to retain its tree owner"); + expect(failure.cause).toMatchObject({ + message: "injected setup failure", + }); + expect(failure.tree.outputRoot).toBe(output); + + const cleanupFailure = await failure.tree + .rollback() + .catch((cause: unknown) => cause); + expect(cleanupFailure).toBeInstanceOf(ArtifactReaderFailure); + expect(await readFile(blocker, "utf8")).toBe("leave until cleanup retry"); + await rm(blocker); + expect(await failure.tree.rollback()).toEqual({ + status: "complete", + residualPaths: [], + }); + expect(await readdir(parent)).toEqual([]); + }, + ); +}); + describe("safe artifact output tree identity", () => { it.skipIf(process.platform === "win32")( "refuses a replaced root and retries cleanup when the owned root is restored", diff --git a/tests/boundary/filesystem/safeOutputTreeCloseFailures.test.ts b/tests/boundary/filesystem/safeOutputTreeCloseFailures.test.ts new file mode 100644 index 0000000000..a58d123126 --- /dev/null +++ b/tests/boundary/filesystem/safeOutputTreeCloseFailures.test.ts @@ -0,0 +1,195 @@ +import { createHash } from "node:crypto"; +import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Readable } from "node:stream"; + +import { describe, expect, it, onTestFinished, vi } from "vitest"; + +const injectedFilesystemFailures = vi.hoisted(() => ({ + closeNext: false, + blockCloseRetries: false, + chmodNext: false, + rootIdentityPath: undefined as string | undefined, +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const filesystem = await importOriginal(); + return { + ...filesystem, + lstat: async (...args: Parameters) => { + if (args[0] === injectedFilesystemFailures.rootIdentityPath) { + injectedFilesystemFailures.rootIdentityPath = undefined; + throw new Error("injected root identity failure"); + } + return filesystem.lstat(...args); + }, + open: async (...args: Parameters) => { + const handle = await filesystem.open(...args); + if (injectedFilesystemFailures.chmodNext) { + injectedFilesystemFailures.chmodNext = false; + handle.chmod = async () => { + throw new Error("primary setup chmod failure"); + }; + } + if (injectedFilesystemFailures.closeNext) { + injectedFilesystemFailures.closeNext = false; + const close = handle.close.bind(handle); + let attempted = false; + handle.close = async () => { + if (!attempted || injectedFilesystemFailures.blockCloseRetries) { + attempted = true; + throw new Error("injected close failure"); + } + await close(); + }; + } + return handle; + }, + }; +}); + +import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; +import { SafeOutputTree } from "../../../src/artifacts/SafeOutputTree.js"; +import { SafeOutputTreeCreationFailure } from "../../../src/artifacts/SafeOutputTreeCreationFailure.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +describe("safe artifact output close failures", () => { + it.skipIf(process.platform === "win32")( + "retains a failed commit handle and does not remove published output", + async () => { + const parent = await createTestTempDirectory("rea-safe-output-commit-"); + const output = join(parent, "published"); + const tree = await SafeOutputTree.create(output); + const bytes = Buffer.from("durable output"); + await tree.write("file.txt", Readable.from(bytes), { + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.byteLength, + }); + injectedFilesystemFailures.closeNext = true; + + await expect(tree.commit()).rejects.toMatchObject({ + cleanup: { resources: [output] }, + }); + expect(await readFile(join(output, "file.txt"), "utf8")).toBe( + "durable output", + ); + expect(await tree.rollback()).toEqual({ status: "not-required" }); + expect(await readFile(join(output, "file.txt"), "utf8")).toBe( + "durable output", + ); + await rm(parent, { recursive: true, force: true }); + }, + ); + + it.skipIf(process.platform === "win32")( + "preserves setup failure when closing the staging handle also fails", + async () => { + const parent = await createTestTempDirectory( + "rea-safe-output-setup-close-", + ); + const output = join(parent, "published"); + injectedFilesystemFailures.chmodNext = true; + injectedFilesystemFailures.closeNext = true; + const failure = await SafeOutputTree.create(output).catch( + (cause: unknown) => cause, + ); + expect(failure).toMatchObject({ + name: "SafeOutputTreeCreationFailure", + message: "primary setup chmod failure", + cleanup: { reason: "injected close failure", resources: [output] }, + }); + if (!(failure instanceof SafeOutputTreeCreationFailure)) + throw new Error("Expected setup failure to retain the tree owner"); + expect(await failure.tree.rollback()).toMatchObject({ + status: "complete", + }); + await expect(access(output)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + + it("retains an unknown root owner without inferring identity on retry", async () => { + const parent = await createTestTempDirectory("rea-safe-output-unknown-"); + const output = join(parent, "published"); + injectedFilesystemFailures.rootIdentityPath = output; + + const failure = await SafeOutputTree.create(output).catch( + (cause: unknown) => cause, + ); + expect(failure).toMatchObject({ + name: "SafeOutputTreeCreationFailure", + cause: { message: "injected root identity failure" }, + }); + if (!(failure instanceof SafeOutputTreeCreationFailure)) + throw new Error("expected setup error to retain its tree owner"); + await access(output); + + for (let attempt = 0; attempt < 2; attempt += 1) { + const cleanup = await failure.tree + .rollback() + .catch((cause: unknown) => cause); + expect(cleanup).toMatchObject({ + cleanup: { resources: [output] }, + message: expect.stringContaining("identity is unknown"), + }); + await access(output); + } + }); +}); + +it.skipIf(process.platform === "win32").each([false, true])( + "preserves extracted output when commit close fails (persistent: %s)", + async (persistent) => { + const root = await createTestTempDirectory("rea-extraction-commit-close-"); + const source = join(root, "source"); + const output = join(root, "published"); + await mkdir(source); + await writeFile(join(source, "payload.txt"), "committed bytes"); + const client = new ArtifactProvider(process.env).createClient({ + path: source, + sourcePath: source, + sha256: "0".repeat(64), + kind: "archive", + format: "asar", + }); + onTestFinished(async () => { + injectedFilesystemFailures.blockCloseRetries = false; + await client.close(); + vi.restoreAllMocks(); + }); + const commit = SafeOutputTree.prototype.commit; + vi.spyOn(SafeOutputTree.prototype, "commit").mockImplementation( + async function (this: SafeOutputTree) { + injectedFilesystemFailures.closeNext = true; + injectedFilesystemFailures.blockCloseRetries = persistent; + await commit.call(this); + }, + ); + + const result = await client.execute("extract_artifact", { + output_root: output, + }); + const extraction = { + output_root: output, + artifacts: [expect.objectContaining({ relative_path: "payload.txt" })], + }; + expect(result).toMatchObject( + persistent + ? { + ok: false, + error: { + cleanupIncomplete: true, + partialObservation: { kind: "artifact-extraction", extraction }, + }, + } + : { ok: true, value: { result: extraction } }, + ); + expect(await readFile(join(output, "payload.txt"), "utf8")).toBe( + "committed bytes", + ); + injectedFilesystemFailures.blockCloseRetries = false; + expect(await client.close()).toMatchObject({ ok: true }); + expect(await readFile(join(output, "payload.txt"), "utf8")).toBe( + "committed bytes", + ); + }, +); diff --git a/tests/boundary/providers/native/artifactProvider.cleanupRetry.test.ts b/tests/boundary/providers/native/artifactProvider.cleanupRetry.test.ts index fd8bd052e8..ca20ee7ea6 100644 --- a/tests/boundary/providers/native/artifactProvider.cleanupRetry.test.ts +++ b/tests/boundary/providers/native/artifactProvider.cleanupRetry.test.ts @@ -1,4 +1,4 @@ -import { access, mkdir, readFile, writeFile } from "node:fs/promises"; +import { access, mkdir, open, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { createPackage } from "@electron/asar"; @@ -167,6 +167,47 @@ it("retains a failed rollback owner and preserves the materialization failure", await expect(access(output)).rejects.toThrow(); }); +it("retains a tree whose setup failed after root identity was captured", async () => { + const { root, archive } = await createArchive(); + const output = join(root, "output"); + const blocker = join(output, "unowned.txt"); + const client = createClient(archive); + const prototypeHandle = await open(join(root, "prototype-handle"), "w"); + const handlePrototype = Object.getPrototypeOf(prototypeHandle) as Pick< + typeof prototypeHandle, + "chmod" + >; + await prototypeHandle.close(); + await rm(join(root, "prototype-handle")); + onTestFinished(async () => { + vi.restoreAllMocks(); + await rm(blocker, { force: true }); + await client.close(); + }); + vi.spyOn(handlePrototype, "chmod").mockImplementationOnce(async function ( + this: typeof prototypeHandle, + ) { + await writeFile(blocker, "keep until owner retry"); + throw new Error("injected output setup failure"); + }); + + const extraction = await client.execute("extract_artifact", { + output_root: output, + }); + expect(extraction).toMatchObject({ + ok: false, + error: { + _tag: "ArtifactOperationError", + detail: expect.stringContaining("injected output setup failure"), + cleanup: { resources: [output] }, + }, + }); + expect(await readFile(blocker, "utf8")).toBe("keep until owner retry"); + await rm(blocker); + expect(await client.close()).toMatchObject({ ok: true }); + await expect(access(output)).rejects.toThrow(); +}); + it("waits for admitted extraction before close and rejects later work", async () => { const { root, archive } = await createArchive(); const output = join(root, "output");