From 6d66f3a8ee77e9f91e7aaa9bb8b16349bd07e22d Mon Sep 17 00:00:00 2001 From: ShuaixinHuang <1836656514@qq.com> Date: Thu, 19 Nov 2026 03:59:57 +0800 Subject: [PATCH] fix(artifacts): preserve PyInstaller retirement and map identity (#11830) Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com> --- src/artifacts/ReleaseExposureReader.test.ts | 108 +++++++++- src/artifacts/ReleaseExposureReader.ts | 12 +- .../pyinstaller/PyInstallerBoundaries.test.ts | 195 +++++++++++++++++- .../pyinstaller/PyInstallerMemberStream.ts | 27 ++- tests/acceptance/releaseExposureCli.test.ts | 77 ++++++- 5 files changed, 401 insertions(+), 18 deletions(-) diff --git a/src/artifacts/ReleaseExposureReader.test.ts b/src/artifacts/ReleaseExposureReader.test.ts index f0331a1738..0ce79dc177 100644 --- a/src/artifacts/ReleaseExposureReader.test.ts +++ b/src/artifacts/ReleaseExposureReader.test.ts @@ -5022,9 +5022,19 @@ describe("ReleaseExposureReader packaged map identity", () => { ); }); -it.for(["a.js", "z.js"])( - "keeps a distinct normalized linker-map occurrence when %s declares the other", - async (script, context) => { +it.for( + ["map", "script"].flatMap((role) => + ["a", "z"].flatMap((order) => + [1, 2].map((declarations) => ({ + role, + owner: `${order}.${role === "map" ? "js" : "html"}`, + declarations, + })), + ), + ), +)( + "keeps a distinct normalized linker-map occurrence for $declarations $role declarations in $owner", + async ({ role, owner, declarations }, context) => { const root = await createTestTempDirectory("rea-map-identities-"); const first = "café.map"; const second = "cafe\u0301.map"; @@ -5034,25 +5044,101 @@ it.for(["a.js", "z.js"])( context.skip(); return; } - await writeFile(join(root, script), `//# sourceMappingURL=${first}\n`); + const declaration = + role === "map" + ? `//# ${"sourceMappingURL"}=${first}\n` + : ``; + await writeFile(join(root, owner), declaration.repeat(declarations)); const scanned = await inspect(root); expect(scanned.debug_companions.paths).toEqual([join(root, second)]); }, ); -it("keeps a distinct normalized linker map beside a page's script target", async (context) => { - const root = await createTestTempDirectory("rea-script-map-identities-"); - const first = "café.map"; - const second = "cafe\u0301.map"; - await writeFile(join(root, first), "const value = 1;\n"); +it.for( + ["map", "script"].flatMap((firstRole) => + [false, true].map((otherSpelling) => ({ firstRole, otherSpelling })), + ), +)( + "retracts exact occurrences across callers, $firstRole first, other spelling: $otherSpelling", + async ({ firstRole, otherSpelling }, context) => { + const root = await createTestTempDirectory("rea-shared-map-identity-"); + const first = "café.map"; + const second = "cafe\u0301.map"; + await writeFile(join(root, first), "linker map\n"); + await writeFile(join(root, second), "linker map\n"); + if ((await readdir(root)).length !== 2) { + context.skip(); + return; + } + await writeFile( + join(root, firstRole === "map" ? "z.js" : "zz.js"), + `//# ${"sourceMappingURL"}=${firstRole === "map" || !otherSpelling ? first : second}\n`, + ); + await writeFile( + join(root, firstRole === "script" ? "z.html" : "zz.html"), + ``, + ); + const scanned = await inspect(root); + expect(scanned.debug_companions.paths).toEqual( + otherSpelling ? [] : [join(root, second)], + ); + }, +); + +it.for([false, true])( + "retracts an exact guess produced after an early script declaration (NFD script: %s)", + async (nfdScript, context) => { + const root = await createTestTempDirectory("rea-late-exact-map-guess-"); + const first = "café.map"; + const second = "cafe\u0301.map"; + await writeFile(join(root, first), "linker map\n"); + await writeFile(join(root, second), "linker map\n"); + if ((await readdir(root)).length !== 2) { + context.skip(); + return; + } + const scriptTarget = nfdScript ? second : first; + const otherTarget = nfdScript ? first : second; + await writeFile( + join(root, "a.html"), + ``, + ); + await writeFile( + join(root, "b.js"), + `//# ${"sourceMappingURL"}=${otherTarget}\n`, + ); + // The other spelling's map declaration routes both candidates through the + // map reader. That can create an exact guess after the HTML was admitted. + await writeFile( + join(root, "z.js"), + `//# ${"sourceMappingURL"}=${scriptTarget}\n`, + ); + const scanned = await inspect(root); + expect(scanned.debug_companions).toEqual({ paths: [], skipped: 0 }); + expect(debugCompanionsStatus(scanned)).not.toBe("exposed"); + expect(scanned.source_maps.skipped).toBeGreaterThan(0); + }, +); + +it("keeps a missing exact declaration unresolved when later guesses share its lookup key", async (context) => { + const root = await createTestTempDirectory("rea-missing-exact-map-guess-"); + const missing = "cấ.map"; + const first = "câ\u0301.map"; + const second = "ca\u0302\u0301.map"; + await writeFile(join(root, first), "linker map\n"); await writeFile(join(root, second), "linker map\n"); if ((await readdir(root)).length !== 2) { context.skip(); return; } - await writeFile(join(root, "a.html"), ``); + expect(first.normalize("NFC")).toBe(missing); + expect(second.normalize("NFC")).toBe(missing); + await writeFile(join(root, "a.html"), ``); + await writeFile(join(root, "b.js"), `//# ${"sourceMappingURL"}=${first}\n`); + await writeFile(join(root, "z.js"), `//# ${"sourceMappingURL"}=${missing}\n`); const scanned = await inspect(root); - expect(scanned.debug_companions.paths).toEqual([join(root, second)]); + expect(scanned.debug_companions).toEqual({ paths: [], skipped: 1 }); + expect(debugCompanionsStatus(scanned)).toBe("unknown"); }); it("keeps a distinct normalized sibling linker map beside a declared one", async (context) => { diff --git a/src/artifacts/ReleaseExposureReader.ts b/src/artifacts/ReleaseExposureReader.ts index a5fefea550..dc1945c23f 100644 --- a/src/artifacts/ReleaseExposureReader.ts +++ b/src/artifacts/ReleaseExposureReader.ts @@ -1999,8 +1999,8 @@ const noteScriptTarget = (state: ScanState, location: string): void => { const target = memberLocationKey(state, location); const spellings = state.scriptTargets.get(target); const spelling = anchoredLocation(state, location); - if (spellings?.has(spelling)) return; retractLinkerMapGuesses(state, location); + if (spellings?.has(spelling)) return; state.scriptTargetBytes += Buffer.byteLength(spelling); if (state.scriptTargetBytes > MAX_RELEASE_EXPOSURE_SCRIPT_TARGET_BYTES) { state.skippedMaps += 1; @@ -2114,7 +2114,6 @@ const noteMapTarget = (state: ScanState, location: string): void => { const target = memberLocationKey(state, location); const spellings = state.mapTargets.get(target); const spelling = anchoredLocation(state, location); - if (spellings?.has(spelling)) return; // A late declaration can identify an exact occurrence already examined as // a linker-map candidate. Claim that identity before final target coverage // considers other spellings in the normalized bucket. @@ -2123,6 +2122,7 @@ const noteMapTarget = (state: ScanState, location: string): void => { // Retract once per exact declaration; repeating it must not consume peers // that only share its normalized lookup key. state.skippedMaps += retractLinkerMapGuesses(state, location); + if (spellings?.has(spelling)) return; state.mapTargetBytes += Buffer.byteLength(spelling); if (state.mapTargetBytes > MAX_RELEASE_EXPOSURE_MAP_TARGET_BYTES) { state.skippedMaps += 1; @@ -2178,6 +2178,14 @@ const retractLinkerMapGuesses = ( if (paths.size === 0) state.linkerMapGuesses.delete(key); return 1; } + // A read can add an exact guess after an early declaration. Retract that + // guess first; an admitted missing spelling cannot identify a peer source. + if ( + state.examinedMembers.get(key)?.has(spelling) === true && + (state.mapTargets.get(key)?.has(spelling) === true || + state.scriptTargets.get(key)?.has(spelling) === true) + ) + return 0; // Without an exact source spelling, normalization cannot prove which // occurrence the declaration means. Keep companion coverage unresolved. state.skippedCompanions += paths.size; diff --git a/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts index a12f7dd1dd..87ffbe2025 100644 --- a/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerBoundaries.test.ts @@ -1,11 +1,14 @@ import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js"; import { awaitAcknowledgement } from "../../../tests/fixtures/lifecycleAcknowledgement.js"; import { dirname, join } from "node:path"; +import { getEventListeners, once } from "node:events"; +import { Readable } from "node:stream"; +import { finished } from "node:stream/promises"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { createHash, randomBytes } from "node:crypto"; import { rename, unlink, writeFile, type FileHandle } from "node:fs/promises"; import { buffer } from "node:stream/consumers"; -import { deflateSync } from "node:zlib"; +import { deflateSync, Inflate } from "node:zlib"; import { setImmediate } from "node:timers/promises"; import { expect, it, vi } from "vitest"; import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js"; @@ -16,6 +19,7 @@ import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { marshalCode } from "../../python/PythonPyc.fixture.js"; import * as artifactPaths from "../../resources/ArtifactPaths.js"; import { PyInstallerPyzStore } from "./PyInstallerPyzStore.js"; +import { pyInstallerMemberStream } from "./PyInstallerMemberStream.js"; import { ArtifactReaderFailure, type ArtifactEntry, @@ -50,6 +54,195 @@ const deferred = () => { return { promise, resolve }; }; +it.each( + [false, true].flatMap((compressed) => + [ + { stop: "caller-null", reason: null }, + { stop: "caller-object", reason: { stage: "buffered member" } }, + { stop: "reader-close", reason: undefined }, + ].map((test) => ({ compressed, ...test })), + ), +)( + "retires completed buffered members ($stop, compressed: $compressed)", + async ({ compressed, stop, reason }) => { + const payload = Buffer.from("buffered"); + await withPyInstallerReader( + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + members: [ + { name: "data", typeCode: "x", payload, compress: compressed }, + ], + }), + async (reader) => { + const controller = new AbortController(); + const entry = (await members(reader))[0]!; + const stream = await reader.open(entry, controller.signal); + const retired = finished(stream).catch((cause: unknown) => cause); + try { + await awaitAcknowledgement( + once(stream, "finish").then(() => undefined), + retired, + "PyInstaller writable finish", + ); + // The pipeline's completion callbacks have settled; no consumer has read. + await setImmediate(); + expect(stream.readableLength).toBe(payload.length); + expect(stream.closed).toBe(false); + if (stop === "reader-close") await reader.close(); + else controller.abort(reason); + await setImmediate(); + expect(stream.destroyed).toBe(true); + expect(stream.closed).toBe(true); + expect(stream.errored).toBeInstanceOf(ArtifactReaderFailure); + expect(stream.errored).toMatchObject({ reason: "cancelled" }); + if (stop !== "reader-close") + expect(stream.errored?.cause).toBe(reason); + } finally { + stream.destroy(); + await retired; + } + }, + ); + }, +); + +it.each( + [false, true].flatMap((compressed) => + (["consume", "cancel"] as const).map((stop) => ({ compressed, stop })), + ), +)( + "keeps completion owned until readable retirement ($stop, compressed: $compressed)", + async ({ compressed, stop }) => { + const payload = Buffer.from("buffered"); + const stored = compressed ? deflateSync(payload) : payload; + const controller = new AbortController(); + const member = pyInstallerMemberStream( + Readable.from([stored], { objectMode: false }), + { + sourceBytes: stored.length, + compressed, + header: undefined, + expectedBytes: payload.length, + maxBytes: payload.length, + signal: controller.signal, + }, + ); + let settled = false; + void member.completion.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + try { + await awaitAcknowledgement( + once(member.output, "finish").then(() => undefined), + member.completion, + "member writable finish", + ); + await setImmediate(); + expect(settled).toBe(false); + expect(getEventListeners(controller.signal, "abort")).toHaveLength(1); + if (stop === "consume") { + expect((await buffer(member.output)).equals(payload)).toBe(true); + await member.completion; + } else { + const reason = { stage: "after producer completion" }; + controller.abort(reason); + await expect(member.completion).rejects.toMatchObject({ + reason: "cancelled", + cause: reason, + }); + expect(member.output.errored?.cause).toBe(reason); + } + expect(settled).toBe(true); + expect(getEventListeners(controller.signal, "abort")).toEqual([]); + } finally { + member.output.destroy(); + await member.completion.catch(() => undefined); + } + }, +); + +it.each([false, true])( + "forwards a queued source failure before cancellation (compressed: %s)", + async (compressed) => { + const source = new Readable({ read() {} }); + const controller = new AbortController(); + const io = Object.assign(new Error("queued descriptor EIO"), { + code: "EIO", + }); + const member = pyInstallerMemberStream(source, { + sourceBytes: 8, + compressed, + header: undefined, + expectedBytes: 8, + maxBytes: 8, + signal: controller.signal, + }); + const consumed = buffer(member.output); + void consumed.catch(() => undefined); + source.destroy(io); + // destroyed/errored are already established; the error event has not fired. + controller.abort({ stage: "before error forwarding" }); + try { + await expect(consumed).rejects.toMatchObject({ reason: "io", cause: io }); + await expect(member.completion).rejects.toMatchObject({ + reason: "io", + cause: io, + }); + } finally { + member.output.destroy(); + await member.completion.catch(() => undefined); + } + }, +); + +it("forwards an established native inflate failure before cancellation", async () => { + const controller = new AbortController(); + const nativeDestroy = Inflate.prototype.destroy; + let compressionFailure: Error | undefined; + const destroy = vi + .spyOn(Inflate.prototype, "destroy") + .mockImplementation(function (this: Inflate, error) { + const result = nativeDestroy.call(this, error); + if (error !== undefined && !controller.signal.aborted) { + compressionFailure = error; + // Native zlib has failed; its error event has not forwarded the cause. + controller.abort({ stage: "inflate error forwarding" }); + } + return result; + }); + const encoded = Buffer.from("invalid zlib"); + const member = pyInstallerMemberStream(Readable.from([encoded]), { + sourceBytes: encoded.length, + compressed: true, + header: undefined, + expectedBytes: 8, + maxBytes: 8, + signal: controller.signal, + }); + try { + await expect(buffer(member.output)).rejects.toMatchObject({ + reason: "format", + cause: { code: "Z_DATA_ERROR" }, + }); + expect(compressionFailure).toBeDefined(); + expect(member.output.errored?.cause).toBe(compressionFailure); + await expect(member.completion).rejects.toMatchObject({ + reason: "format", + cause: compressionFailure, + }); + } finally { + member.output.destroy(); + await member.completion.catch(() => undefined); + destroy.mockRestore(); + } +}); + it.each(["success", "io", "caller cancellation", "reader close"] as const)( "joins the pending descriptor read after a module prefix stop (%s)", async (outcome) => { diff --git a/src/artifacts/pyinstaller/PyInstallerMemberStream.ts b/src/artifacts/pyinstaller/PyInstallerMemberStream.ts index dab142dcee..c7426d1afa 100644 --- a/src/artifacts/pyinstaller/PyInstallerMemberStream.ts +++ b/src/artifacts/pyinstaller/PyInstallerMemberStream.ts @@ -129,7 +129,14 @@ export const pyInstallerMemberStream = ( output.on("error", () => undefined); const onAbort = (): void => { // Verification and private-file cleanup own their accepted native I/O. - if (completing) return; + if (completing || output.destroyed) return; + // A producer can already own an error before its error event forwards it. + // Do not replace that established failure with a downstream cancellation. + if ( + source.errored !== null || + (inflate !== undefined && inflate.errored !== null) + ) + return; // The descriptor source owns cancellation until its native read settles. // Retiring it downstream earlier would hide a native I/O rejection. if (!source.readableEnded && !source.destroyed) return; @@ -142,6 +149,12 @@ export const pyInstallerMemberStream = ( ); }; options.signal.addEventListener("abort", onAbort, { once: true }); + const retired = new Promise((resolve) => { + output.once("close", () => { + options.signal.removeEventListener("abort", onAbort); + resolve(); + }); + }); const onSourceFailure = (cause: unknown): void => { failure ??= cause instanceof ArtifactReaderFailure @@ -166,7 +179,7 @@ export const pyInstallerMemberStream = ( ); output.destroy(failure); }); - const completion = ( + const production = ( inflate === undefined ? pipeline(source, output) : pipeline(source, inflate, output) @@ -177,9 +190,17 @@ export const pyInstallerMemberStream = ( throw failure ?? cause; }) .finally(() => { - options.signal.removeEventListener("abort", onAbort); source.removeListener("error", onSourceFailure); }); + // Production can finish with unread output. Both its cancellation listener + // and its owning reader operation survive until that readable side retires. + // Await both settlements even on failure before the reader operation settles. + const completion = Promise.allSettled([production, retired]).then( + ([produced]) => { + if (produced.status === "rejected") throw produced.reason; + if (output.errored !== null) throw output.errored; + }, + ); // Preserve rejection for the owning scope without an early unhandled promise. void completion.catch(() => undefined); if (options.signal.aborted) onAbort(); diff --git a/tests/acceptance/releaseExposureCli.test.ts b/tests/acceptance/releaseExposureCli.test.ts index e63469238c..61a33b70a8 100644 --- a/tests/acceptance/releaseExposureCli.test.ts +++ b/tests/acceptance/releaseExposureCli.test.ts @@ -1,4 +1,4 @@ -import { writeFile } from "node:fs/promises"; +import { readdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; @@ -107,3 +107,78 @@ cliTest( expect(report.summary.by_status.not_assessed).toBe(3); }, ); + +cliTest.for(["map", "script"])( + "retains the distinct linker-map source across declaration owners, %s first", + async (firstRole, { cli, skip }) => { + const root = await createTestTempDirectory("rea-release-map-identity-cli-"); + const declared = "café.map"; + const companion = "cafe\u0301.map"; + await writeFile(join(root, declared), "linker map\n"); + await writeFile(join(root, companion), "linker map\n"); + if ((await readdir(root)).length !== 2) { + skip(); + return; + } + await writeFile( + join(root, firstRole === "map" ? "z.js" : "zz.js"), + `//# ${"sourceMappingURL"}=${declared}\n`, + ); + await writeFile( + join(root, firstRole === "script" ? "z.html" : "zz.html"), + ``, + ); + const result = await cli.run({ + arguments: ["audit-release-exposure", root, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const report = outputSchema.parse(result.json).normalized_result; + expect( + report.checks.find(({ id }) => id === "native.debug_companions"), + ).toMatchObject({ + status: "exposed", + confidence: "inferred", + observation: { + counts: { companions: 1 }, + samples: [join(root, companion)], + }, + }); + }, +); + +cliTest( + "retracts a script target guessed after its early declaration", + async ({ cli, skip }) => { + const root = await createTestTempDirectory("rea-release-late-guess-cli-"); + const declared = "café.map"; + const other = "cafe\u0301.map"; + await writeFile(join(root, declared), "linker map\n"); + await writeFile(join(root, other), "linker map\n"); + if ((await readdir(root)).length !== 2) { + skip(); + return; + } + await writeFile( + join(root, "a.html"), + ``, + ); + await writeFile(join(root, "b.js"), `//# ${"sourceMappingURL"}=${other}\n`); + await writeFile( + join(root, "z.js"), + `//# ${"sourceMappingURL"}=${declared}\n`, + ); + const result = await cli.run({ + arguments: ["audit-release-exposure", root, "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + const report = outputSchema.parse(result.json).normalized_result; + const check = report.checks.find( + ({ id }) => id === "native.debug_companions", + ); + expect(check?.status).not.toBe("exposed"); + expect(check?.observation).toMatchObject({ + counts: { companions: 0 }, + samples: [], + }); + }, +);