mirror of
https://github.com/morluto/rea.git
synced 2026-11-19 21:23:45 +00:00
fix(zip): publish EOF after extraction finalization (#12164)
* fix(zip): publish EOF after extraction finalization Adapt the ZIP extraction changes from #8431 to current main. * test(zip): bound producer-settlement acknowledgements --------- Co-authored-by: hongthana <thanawat.rn@gmail.com>
This commit is contained in:
1 parent
b95a757c33
commit
ca3f35dcbb
2 files changed
+216
-26
No files matched your search
@@ -16,8 +16,12 @@ import { constants, zstdCompressSync } from "node:zlib";
|
||||
import { appendFile, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { Duplex } from "node:stream";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
import { finished } from "node:stream/promises";
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
|
||||
import {
|
||||
ERR_INVALID_COMPRESSED_DATA,
|
||||
TextReader,
|
||||
Uint8ArrayReader,
|
||||
Uint8ArrayWriter,
|
||||
@@ -77,8 +81,8 @@ const enumerate = async (reader: ZipArtifactReader): Promise<number> => {
|
||||
return files;
|
||||
};
|
||||
|
||||
/** Hold real zip.js finalization after its streaming work settles. */
|
||||
const holdZipProducerSettlement = () => {
|
||||
/** Hold getData settlement after real zip.js streaming and sink closure. */
|
||||
const holdZipProducerSettlement = (failure?: Error) => {
|
||||
const reached = createDeferred<void>();
|
||||
const release = createDeferred<void>();
|
||||
const original = ZipReader.prototype.getEntriesGenerator;
|
||||
@@ -89,12 +93,14 @@ const holdZipProducerSettlement = () => {
|
||||
if (!entry.directory) {
|
||||
const getData = entry.getData.bind(entry);
|
||||
entry.getData = async (writer, dataOptions) => {
|
||||
try {
|
||||
return await getData(writer, dataOptions);
|
||||
} finally {
|
||||
reached.resolve();
|
||||
await release.promise;
|
||||
}
|
||||
const result = await getData(writer, dataOptions).finally(
|
||||
async () => {
|
||||
reached.resolve();
|
||||
await release.promise;
|
||||
},
|
||||
);
|
||||
if (failure !== undefined) throw failure;
|
||||
return result;
|
||||
};
|
||||
}
|
||||
yield entry;
|
||||
@@ -675,6 +681,177 @@ it.each([null, { caller: "opaque ZIP interruption" }])(
|
||||
},
|
||||
);
|
||||
|
||||
describe("ZipArtifactReader extraction finalization", () => {
|
||||
it.each([Buffer.alloc(0), Buffer.from("ZIP finalization bytes: café")])(
|
||||
"publishes exact bytes and EOF only after producer success: %j",
|
||||
async (content) => {
|
||||
const directory = await createTestTempDirectory("rea-zip-finalization-");
|
||||
const path = join(directory, "member.zip");
|
||||
const writer = new ZipWriter(new Uint8ArrayWriter());
|
||||
await writer.add("member.bin", new Uint8ArrayReader(content));
|
||||
await writeFile(path, await writer.close());
|
||||
const held = holdZipProducerSettlement();
|
||||
const reader = new ZipArtifactReader(path, "zip");
|
||||
let consuming: Promise<Buffer> | undefined;
|
||||
try {
|
||||
const entries: ArtifactEntry[] = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
const stream = await reader.open(entries[0]!);
|
||||
consuming = buffer(stream);
|
||||
let settled = false;
|
||||
void consuming.then(
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
() => {
|
||||
settled = true;
|
||||
},
|
||||
);
|
||||
await awaitAcknowledgement(
|
||||
held.reached,
|
||||
consuming,
|
||||
"successful ZIP producer settlement",
|
||||
);
|
||||
await setImmediate();
|
||||
expect(settled).toBe(false);
|
||||
expect(stream.readableEnded).toBe(false);
|
||||
held.release();
|
||||
await expect(consuming).resolves.toEqual(content);
|
||||
expect(stream.readableEnded).toBe(true);
|
||||
} finally {
|
||||
held.release();
|
||||
await reader.close();
|
||||
await consuming?.catch(() => undefined);
|
||||
held.enumeration.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("reports a producer finalization failure after sink closure without success", async () => {
|
||||
const directory = await createTestTempDirectory(
|
||||
"rea-zip-finalization-error-",
|
||||
);
|
||||
const path = join(directory, "empty.zip");
|
||||
const writer = new ZipWriter(new Uint8ArrayWriter());
|
||||
await writer.add("empty.bin", new Uint8ArrayReader(new Uint8Array()));
|
||||
await writeFile(path, await writer.close());
|
||||
const failure = new Error(ERR_INVALID_COMPRESSED_DATA);
|
||||
const held = holdZipProducerSettlement(failure);
|
||||
const reader = new ZipArtifactReader(path, "zip");
|
||||
let consuming: Promise<Buffer> | undefined;
|
||||
try {
|
||||
const entries: ArtifactEntry[] = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
const stream = await reader.open(entries[0]!);
|
||||
consuming = buffer(stream);
|
||||
await awaitAcknowledgement(
|
||||
held.reached,
|
||||
consuming,
|
||||
"ZIP producer error settlement",
|
||||
);
|
||||
await setImmediate();
|
||||
held.release();
|
||||
await expect(consuming).rejects.toMatchObject({
|
||||
name: "ArtifactReaderFailure",
|
||||
reason: "format",
|
||||
cause: failure,
|
||||
});
|
||||
expect(stream.readableEnded).toBe(false);
|
||||
} finally {
|
||||
held.release();
|
||||
await reader.close();
|
||||
await consuming?.catch(() => undefined);
|
||||
held.enumeration.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("ZipArtifactReader finalization ownership", () => {
|
||||
it("rejects source mutation during held finalization before publishing EOF", async () => {
|
||||
const directory = await createTestTempDirectory(
|
||||
"rea-zip-finalization-source-",
|
||||
);
|
||||
const path = join(directory, "member.zip");
|
||||
await writeOrderedZip(path, ["member.txt"]);
|
||||
const held = holdZipProducerSettlement();
|
||||
const reader = new ZipArtifactReader(path, "zip");
|
||||
let consuming: Promise<Buffer> | undefined;
|
||||
try {
|
||||
const entries: ArtifactEntry[] = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
const stream = await reader.open(entries[0]!);
|
||||
consuming = buffer(stream);
|
||||
await awaitAcknowledgement(
|
||||
held.reached,
|
||||
consuming,
|
||||
"ZIP source verification",
|
||||
);
|
||||
await appendFile(path, "changed after sink closure");
|
||||
held.release();
|
||||
await expect(consuming).rejects.toMatchObject({
|
||||
name: "ArtifactReaderFailure",
|
||||
reason: "integrity",
|
||||
});
|
||||
expect(stream.readableEnded).toBe(false);
|
||||
} finally {
|
||||
held.release();
|
||||
await reader.close();
|
||||
await consuming?.catch(() => undefined);
|
||||
held.enumeration.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves cancellation during held finalization and joins the producer before source close", async () => {
|
||||
const directory = await createTestTempDirectory(
|
||||
"rea-zip-finalization-cancel-",
|
||||
);
|
||||
const path = join(directory, "member.zip");
|
||||
await writeOrderedZip(path, ["member.txt"]);
|
||||
const held = holdZipProducerSettlement();
|
||||
const fileClose = vi.spyOn(OwnedFileHandle.prototype, "close");
|
||||
const reader = new ZipArtifactReader(path, "zip");
|
||||
const caller = new AbortController();
|
||||
let consuming: Promise<Buffer> | undefined;
|
||||
try {
|
||||
const entries: ArtifactEntry[] = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
const stream = await reader.open(entries[0]!, caller.signal);
|
||||
consuming = buffer(stream);
|
||||
await awaitAcknowledgement(
|
||||
held.reached,
|
||||
consuming,
|
||||
"ZIP finalization cancellation",
|
||||
);
|
||||
await setImmediate();
|
||||
const reason = { caller: "cancelled during ZIP finalization" };
|
||||
caller.abort(reason);
|
||||
const error: unknown = await consuming.catch((cause: unknown) => cause);
|
||||
expect(error).toBeInstanceOf(ArtifactReaderFailure);
|
||||
if (!(error instanceof ArtifactReaderFailure))
|
||||
throw new Error("Expected ZIP finalization cancellation");
|
||||
expect(error.reason).toBe("cancelled");
|
||||
expect(error.cancellation?.reason).toBe(reason);
|
||||
expect(error.cause).toBe(reason);
|
||||
let closed = false;
|
||||
const closing = reader.close().then(() => {
|
||||
closed = true;
|
||||
});
|
||||
await setImmediate();
|
||||
expect(closed).toBe(false);
|
||||
expect(fileClose).not.toHaveBeenCalled();
|
||||
held.release();
|
||||
await closing;
|
||||
expect(fileClose).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
held.release();
|
||||
await reader.close();
|
||||
await consuming?.catch(() => undefined);
|
||||
held.enumeration.mockRestore();
|
||||
fileClose.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("ZipArtifactReader producer ownership", () => {
|
||||
it.each(["unread", "abandoned", "cancelled"] as const)(
|
||||
"joins late extraction settlement for an %s stream before closing its source",
|
||||
@@ -716,7 +893,11 @@ describe("ZipArtifactReader producer ownership", () => {
|
||||
void secondClose.then(() => {
|
||||
secondClosed = true;
|
||||
});
|
||||
await held.reached;
|
||||
await awaitAcknowledgement(
|
||||
held.reached,
|
||||
closing,
|
||||
"ZIP extraction retirement",
|
||||
);
|
||||
expect(stream.destroyed).toBe(true);
|
||||
expect(fileClose).not.toHaveBeenCalled();
|
||||
let closed = false;
|
||||
@@ -751,10 +932,18 @@ describe("ZipArtifactReader producer ownership", () => {
|
||||
const entries: ArtifactEntry[] = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
const stream = await reader.open(entries[0]!);
|
||||
await held.reached;
|
||||
const writableFinished = finished(stream, {
|
||||
readable: false,
|
||||
cleanup: true,
|
||||
});
|
||||
await awaitAcknowledgement(
|
||||
held.reached,
|
||||
writableFinished,
|
||||
"buffered ZIP producer settlement",
|
||||
);
|
||||
held.release();
|
||||
// getData has finished, but this tiny member remains buffered and unread.
|
||||
await Promise.resolve();
|
||||
await writableFinished;
|
||||
const closeFailure = new Error("ZIP descriptor close denied");
|
||||
const close = vi
|
||||
.spyOn(OwnedFileHandle.prototype, "close")
|
||||
@@ -886,8 +1075,12 @@ it("joins a late ZIP producer and closes the source despite an enumeration clean
|
||||
try {
|
||||
const first = await traversal.next();
|
||||
if (first.done) throw new Error("Expected ZIP member");
|
||||
await reader.open(first.value);
|
||||
await held.reached;
|
||||
const stream = await reader.open(first.value);
|
||||
await awaitAcknowledgement(
|
||||
held.reached,
|
||||
finished(stream, { readable: false, cleanup: true }),
|
||||
"ZIP producer before mixed cleanup",
|
||||
);
|
||||
const closing = reader.close();
|
||||
const failureCheck = expect(closing).rejects.toBe(failure);
|
||||
await Promise.resolve();
|
||||
|
||||
@@ -569,11 +569,8 @@ const extractStream = (
|
||||
if (!output.write(Buffer.from(chunk)))
|
||||
await once(output, "drain", { signal: controller.signal });
|
||||
},
|
||||
close: async () => {
|
||||
await verifySource();
|
||||
assertArtifactReadActive(signal);
|
||||
output.end();
|
||||
},
|
||||
// Sink closure can precede getData's final validation or rejection.
|
||||
// The producer's successful settlement owns consumer-visible EOF below.
|
||||
abort: (reason) => {
|
||||
output.destroy(toError(typedZipFailure(reason, entry.compressionMethod)));
|
||||
},
|
||||
@@ -587,14 +584,14 @@ const extractStream = (
|
||||
onprogress: () => assertArtifactReadActive(signal),
|
||||
}),
|
||||
)
|
||||
.then(
|
||||
() => undefined,
|
||||
(cause: unknown) => {
|
||||
output.destroy(
|
||||
toError(typedZipFailure(cause, entry.compressionMethod)),
|
||||
);
|
||||
},
|
||||
);
|
||||
.then(async () => {
|
||||
await verifySource();
|
||||
assertArtifactReadActive(signal);
|
||||
output.end();
|
||||
})
|
||||
.catch((cause: unknown) => {
|
||||
output.destroy(toError(typedZipFailure(cause, entry.compressionMethod)));
|
||||
});
|
||||
return { output, completion };
|
||||
};
|
||||
|
||||
|
||||
Reference in new issue
Block a user