diff --git a/src/artifacts/DotnetBundleArtifactReader.ts b/src/artifacts/DotnetBundleArtifactReader.ts index 30aa45a704..a3af87e5c2 100644 --- a/src/artifacts/DotnetBundleArtifactReader.ts +++ b/src/artifacts/DotnetBundleArtifactReader.ts @@ -151,12 +151,65 @@ export class DotnetBundleArtifactReader implements ArtifactReader { selected?.entry, "Selected .NET bundle entry differs from its observed manifest", ); + let inputStream: Readable | undefined; + let decodedStream: Readable | undefined; + let finalizing = false; + const onFailure = (cause: unknown): void => { + stream.destroy( + this.#memberFailure(selected.file, decodedStream?.errored ?? cause), + ); + }; + const observeSource = (input: Readable, decoded: Readable): void => { + inputStream = input; + decodedStream = decoded; + input.on("error", onFailure); + if (decoded !== input) decoded.on("error", onFailure); + // The descriptor owner observes abort first and retains in-flight I/O + // failures, even while this iterator is suspended at a yield. + signal?.removeEventListener("abort", onAbort); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); + }; const stream = Readable.from( - this.#readMember(source, selected.file, signal), + this.#readMember( + source, + selected.file, + { + source: observeSource, + failure: onFailure, + finalizing: () => { + finalizing = true; + }, + }, + signal, + ), { objectMode: false }, ); + const onAbort = (): void => { + if (stream.destroyed || finalizing) return; + if (inputStream !== undefined && !inputStream.destroyed) return; + onFailure( + inputStream?.errored ?? + new ArtifactReaderFailure( + "cancelled", + ".NET bundle read cancelled", + { + cause: signal?.reason, + }, + ), + ); + }; + stream.on("error", () => undefined); this.#streams.add(stream); - stream.once("close", () => this.#streams.delete(stream)); + stream.once("close", () => { + this.#streams.delete(stream); + signal?.removeEventListener("abort", onAbort); + inputStream?.removeListener("error", onFailure); + if (decodedStream !== inputStream) + decodedStream?.removeListener("error", onFailure); + }); + signal?.addEventListener("abort", onAbort, { once: true }); + if (signal?.aborted) onAbort(); return stream; }); } @@ -192,6 +245,11 @@ export class DotnetBundleArtifactReader implements ArtifactReader { async *#readMember( source: StableRegularFileDescriptor, file: DotnetBundleFile, + lifecycle: { + source(input: Readable, decoded: Readable): void; + failure(cause: unknown): void; + finalizing(): void; + }, signal?: AbortSignal, ): AsyncGenerator { const storedBytes = file.compressedSize || file.size; @@ -205,6 +263,7 @@ export class DotnetBundleArtifactReader implements ArtifactReader { }); const inflater = file.compressedSize === 0 ? undefined : createInflateRaw(); const decoded = inflater ?? input; + lifecycle.source(input, decoded); const pumping = inflater === undefined ? undefined : pipeline(input, inflater); // Observe early pipeline rejections while the output iterator is still reading. @@ -212,16 +271,25 @@ export class DotnetBundleArtifactReader implements ArtifactReader { let bytes = 0; try { for await (const raw of decoded) { - abortIfNeeded(signal); - const chunk: Buffer = raw; - bytes += chunk.length; - if (bytes > file.size) - throw new ArtifactReaderFailure( - "limit", - `.NET bundle entry expands beyond its declared size: ${file.path}`, - ); - yield chunk; + try { + abortIfNeeded(signal); + const chunk: Buffer = raw; + bytes += chunk.length; + if (bytes > file.size) + throw new ArtifactReaderFailure( + "limit", + `.NET bundle entry expands beyond its declared size: ${file.path}`, + ); + yield chunk; + } catch (cause) { + // Iterator cleanup can emit AbortError. Retain the failure that + // caused this body to exit, including the caller's cancellation. + lifecycle.failure(cause); + throw cause; + } } + // Final source validation owns its in-flight I/O before late cancellation. + lifecycle.finalizing(); await pumping; if ( bytes !== file.size || @@ -232,22 +300,32 @@ export class DotnetBundleArtifactReader implements ArtifactReader { `.NET bundle member length or compressed extent does not match its declaration: ${file.path}`, ); await this.#source.readSelectedPath(signal); + abortIfNeeded(signal); } catch (cause) { - if (cause instanceof ArtifactReaderFailure) throw cause; - const malformedCompression = - inflater !== undefined && - cause instanceof Error && - "code" in cause && - ["Z_DATA_ERROR", "Z_BUF_ERROR"].includes(String(cause.code)); - throw new ArtifactReaderFailure( - malformedCompression ? "format" : "io", - `Could not read .NET bundle member: ${file.path}`, - { cause }, - ); + const failure = this.#memberFailure(file, cause); + lifecycle.failure(failure); + throw failure; } finally { input.destroy(); decoded.destroy(); await pumping?.catch(() => undefined); } } + + #memberFailure( + file: DotnetBundleFile, + cause: unknown, + ): ArtifactReaderFailure { + if (cause instanceof ArtifactReaderFailure) return cause; + const malformedCompression = + file.compressedSize !== 0 && + cause instanceof Error && + "code" in cause && + ["Z_DATA_ERROR", "Z_BUF_ERROR"].includes(String(cause.code)); + return new ArtifactReaderFailure( + malformedCompression ? "format" : "io", + `Could not read .NET bundle member: ${file.path}`, + { cause }, + ); + } } diff --git a/tests/boundary/filesystem/dotnetBundleCancellation.test.ts b/tests/boundary/filesystem/dotnetBundleCancellation.test.ts new file mode 100644 index 0000000000..2264a3880b --- /dev/null +++ b/tests/boundary/filesystem/dotnetBundleCancellation.test.ts @@ -0,0 +1,295 @@ +import { once } from "node:events"; +import { open, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { finished } from "node:stream/promises"; +import { randomBytes } from "node:crypto"; +import { createInflateRaw } from "node:zlib"; +import { expect, it, vi } from "vitest"; +import { DotnetBundleArtifactReader } from "../../../src/artifacts/DotnetBundleArtifactReader.js"; +import { dotnetBundleFixture } from "../../../src/artifacts/dotnet/DotnetBundle.fixture.js"; +import { createDeferred } from "../../fixtures/binarySession.js"; +import { awaitAcknowledgement } from "../../fixtures/lifecycleAcknowledgement.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +vi.mock("node:zlib", async (importOriginal) => { + const original = await importOriginal(); + return { ...original, createInflateRaw: vi.fn(original.createInflateRaw) }; +}); + +const cases = [false, true].flatMap((compressed) => + (["unstarted", "backpressured"] as const).flatMap((phase) => + (["null", "object"] as const).map((reasonKind) => ({ + compressed, + phase, + reasonKind, + })), + ), +); + +it("retains cancellation before decoder cleanup while compressed input is pending", async () => { + const root = await createTestTempDirectory("rea-dotnet-decoder-cancel-"); + const path = join(root, "host.exe"); + await writeFile( + path, + dotnetBundleFixture([ + { path: "member", bytes: randomBytes(256 * 1024), compressed: true }, + ]).bytes, + ); + const handle = await open(path, "r"); + const reader = new DotnetBundleArtifactReader(path, { + handle, + initial: await handle.stat(), + }); + const started = createDeferred(); + const pending = createDeferred(); + let restore = () => {}; + let retire = async () => {}; + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + expect(entries[0]?.compressedSize).toBeGreaterThan(64 * 1024); + const controller = new AbortController(); + const output = await reader.open(entries[0]!, controller.signal); + const settled = finished(output).catch((cause: unknown) => cause); + retire = async () => { + output.destroy(); + await settled; + }; + const originalRead = handle.read.bind(handle); + const spy = vi + .spyOn(handle, "read") + .mockImplementationOnce(originalRead) + .mockImplementationOnce(async (...args) => { + started.resolve(); + await pending.promise; + return originalRead(...args); + }); + restore = () => spy.mockRestore(); + const ready = once(output, "readable"); + output.read(0); + await awaitAcknowledgement( + ready.then(() => undefined), + settled, + "member readable", + ); + await awaitAcknowledgement( + started.promise, + settled, + "descriptor read started", + ); + // The factory is a pass-through spy: observe genuine native decoder bytes. + const created = vi.mocked(createInflateRaw).mock.results.at(-1); + if (created?.type !== "return") throw new Error("Missing native decoder"); + const decoder = created.value; + while (decoder.readableLength === 0) + await awaitAcknowledgement( + once(decoder, "readable").then(() => undefined), + settled, + "decoder readable", + ); + expect(decoder.destroyed).toBe(false); + const closed = new Promise((resolve) => + decoder.once("close", () => resolve()), + ); + const decoderSettled = finished(decoder).catch((cause: unknown) => cause); + const reason = { stop: "compressed prefetch" }; + controller.abort(reason); + output.resume(); + await awaitAcknowledgement(closed, decoderSettled, "decoder closed"); + pending.resolve(); + const failure = await settled; + expect(failure).toMatchObject({ reason: "cancelled", cause: reason }); + expect((failure as Error).cause).toBe(reason); + } finally { + pending.resolve(); + await retire(); + await reader.close(); + restore(); + await handle.close(); + } +}); + +it.each([false, true])( + "settles final descriptor validation before cancellation (fails: %s)", + async (fails) => { + const root = await createTestTempDirectory("rea-dotnet-final-validation-"); + const path = join(root, "host.exe"); + await writeFile(path, dotnetBundleFixture().bytes); + const handle = await open(path, "r"); + const reader = new DotnetBundleArtifactReader(path, { + handle, + initial: await handle.stat(), + }); + const started = createDeferred(); + const pending = createDeferred(); + const physical = Object.assign(new Error("final descriptor stat failure"), { + code: "EIO", + }); + let restore = () => {}; + let retire = async () => {}; + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const controller = new AbortController(); + const output = await reader.open(entries[0]!, controller.signal); + const settled = finished(output).catch((cause: unknown) => cause); + retire = async () => { + output.destroy(); + await settled; + }; + const originalStat = handle.stat.bind(handle); + const spy = vi + .spyOn(handle, "stat") + .mockImplementationOnce(async (...args) => { + started.resolve(); + await pending.promise; + if (fails) throw physical; + return originalStat(...args); + }); + restore = () => spy.mockRestore(); + output.resume(); + await awaitAcknowledgement( + started.promise, + settled, + "final validation started", + ); + const reason = { stop: "final validation" }; + controller.abort(reason); + pending.resolve(); + const failure = await settled; + expect(failure).toMatchObject({ reason: fails ? "io" : "cancelled" }); + expect((failure as Error).cause).toBe(fails ? physical : reason); + } finally { + pending.resolve(); + await retire(); + await reader.close(); + restore(); + await handle.close(); + } + }, +); + +it.each([false, true])( + "preserves a prefetched descriptor failure when a suspended consumer cancels (compressed: %s)", + async (compressed) => { + const root = await createTestTempDirectory("rea-dotnet-prefetch-cancel-"); + const path = join(root, "host.exe"); + await writeFile( + path, + dotnetBundleFixture([ + { + path: "member", + bytes: compressed + ? randomBytes(256 * 1024) + : Buffer.alloc(128 * 1024, 65), + compressed, + }, + ]).bytes, + ); + const handle = await open(path, "r"); + const reader = new DotnetBundleArtifactReader(path, { + handle, + initial: await handle.stat(), + }); + const started = createDeferred(); + const pending = createDeferred(); + const physical = Object.assign(new Error("prefetched descriptor failure"), { + code: "EIO", + }); + let restore = () => {}; + let retire = async () => {}; + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const controller = new AbortController(); + const output = await reader.open(entries[0]!, controller.signal); + const settled = finished(output).catch((cause: unknown) => cause); + retire = async () => { + output.destroy(); + await settled; + }; + const originalRead = handle.read.bind(handle); + const spy = vi + .spyOn(handle, "read") + .mockImplementationOnce(originalRead) + .mockImplementationOnce(async () => { + started.resolve(); + await pending.promise; + throw physical; + }); + restore = () => spy.mockRestore(); + const ready = once(output, "readable"); + output.read(0); + await awaitAcknowledgement( + ready.then(() => undefined), + settled, + "member readable", + ); + await awaitAcknowledgement( + started.promise, + settled, + "descriptor read started", + ); + expect(output.readableLength).toBeGreaterThan(0); + pending.resolve(); + controller.abort({ stop: "prefetch pending" }); + output.resume(); + const failure = await settled; + expect(failure).toMatchObject({ reason: "io" }); + expect((failure as Error).cause).toBe(physical); + } finally { + pending.resolve(); + await retire(); + await reader.close(); + restore(); + await handle.close(); + } + }, +); + +it.each(cases)( + "retires a $phase .NET member on cancellation (compressed: $compressed, reason: $reasonKind)", + async ({ compressed, phase, reasonKind }) => { + const root = await createTestTempDirectory("rea-dotnet-idle-cancel-"); + const path = join(root, "host.exe"); + await writeFile( + path, + dotnetBundleFixture([ + // One physical read leaves the outer iterator suspended, with no + // outstanding prefetch. The separate EIO case covers that state. + { path: "member", bytes: Buffer.alloc(1024, 65), compressed }, + ]).bytes, + ); + const reader = new DotnetBundleArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + const controller = new AbortController(); + const output = await reader.open(entries[0]!, controller.signal); + const settled = finished(output).catch((cause: unknown) => cause); + const errors: Error[] = []; + output.on("error", (error) => errors.push(error)); + if (phase === "backpressured") { + const ready = once(output, "readable"); + output.read(0); + await awaitAcknowledgement( + ready.then(() => undefined), + settled, + "idle member readable", + ); + expect(output.readableLength).toBeGreaterThan(0); + } + const reason = reasonKind === "null" ? null : { stop: "idle member" }; + controller.abort(reason); + // Cancellation must not require a consumer to resume the iterator first. + expect(output.destroyed).toBe(true); + expect(await settled).toMatchObject({ + reason: "cancelled", + }); + expect(errors[0]?.cause).toBe(reason); + } finally { + await reader.close(); + } + await reader.close(); + }, +);