fix(artifacts): preserve PyInstaller retirement and map identity (#11830)

Co-authored-by: ShuaixinHuang <197200901+ShuaixinHuang@users.noreply.github.com>
This commit is contained in:
ShuaixinHuangandShuaixinHuang authored and GitHub committed 2026-11-18 14:59:57 -05:00
1 parent 0db5f93df6
commit 6d66f3a8ee
5 files changed
+401 -18

No files matched your search

+97 -11
View File
@@ -5022,9 +5022,19 @@ describe("ReleaseExposureReader packaged map identity", () => {
);
});
it.for(["a.js", "z.js"])(
"keeps a distinct normalized linker-map occurrence when %s declares the other",
async (script, context) => {
it.for(
["map", "script"].flatMap((role) =>
["a", "z"].flatMap((order) =>
[1, 2].map((declarations) => ({
role,
owner: `${order}.${role === "map" ? "js" : "html"}`,
declarations,
})),
),
),
)(
"keeps a distinct normalized linker-map occurrence for $declarations $role declarations in $owner",
async ({ role, owner, declarations }, context) => {
const root = await createTestTempDirectory("rea-map-identities-");
const first = "café.map";
const second = "cafe\u0301.map";
@@ -5034,25 +5044,101 @@ it.for(["a.js", "z.js"])(
context.skip();
return;
}
await writeFile(join(root, script), `//# sourceMappingURL=${first}\n`);
const declaration =
role === "map"
? `//# ${"sourceMappingURL"}=${first}\n`
: `<script src="${first}"></script>`;
await writeFile(join(root, owner), declaration.repeat(declarations));
const scanned = await inspect(root);
expect(scanned.debug_companions.paths).toEqual([join(root, second)]);
},
);
it("keeps a distinct normalized linker map beside a page's script target", async (context) => {
const root = await createTestTempDirectory("rea-script-map-identities-");
const first = "café.map";
const second = "cafe\u0301.map";
await writeFile(join(root, first), "const value = 1;\n");
it.for(
["map", "script"].flatMap((firstRole) =>
[false, true].map((otherSpelling) => ({ firstRole, otherSpelling })),
),
)(
"retracts exact occurrences across callers, $firstRole first, other spelling: $otherSpelling",
async ({ firstRole, otherSpelling }, context) => {
const root = await createTestTempDirectory("rea-shared-map-identity-");
const first = "café.map";
const second = "cafe\u0301.map";
await writeFile(join(root, first), "linker map\n");
await writeFile(join(root, second), "linker map\n");
if ((await readdir(root)).length !== 2) {
context.skip();
return;
}
await writeFile(
join(root, firstRole === "map" ? "z.js" : "zz.js"),
`//# ${"sourceMappingURL"}=${firstRole === "map" || !otherSpelling ? first : second}\n`,
);
await writeFile(
join(root, firstRole === "script" ? "z.html" : "zz.html"),
`<script src="${firstRole === "script" || !otherSpelling ? first : second}"></script>`,
);
const scanned = await inspect(root);
expect(scanned.debug_companions.paths).toEqual(
otherSpelling ? [] : [join(root, second)],
);
},
);
it.for([false, true])(
"retracts an exact guess produced after an early script declaration (NFD script: %s)",
async (nfdScript, context) => {
const root = await createTestTempDirectory("rea-late-exact-map-guess-");
const first = "café.map";
const second = "cafe\u0301.map";
await writeFile(join(root, first), "linker map\n");
await writeFile(join(root, second), "linker map\n");
if ((await readdir(root)).length !== 2) {
context.skip();
return;
}
const scriptTarget = nfdScript ? second : first;
const otherTarget = nfdScript ? first : second;
await writeFile(
join(root, "a.html"),
`<script src="${scriptTarget}"></script>`,
);
await writeFile(
join(root, "b.js"),
`//# ${"sourceMappingURL"}=${otherTarget}\n`,
);
// The other spelling's map declaration routes both candidates through the
// map reader. That can create an exact guess after the HTML was admitted.
await writeFile(
join(root, "z.js"),
`//# ${"sourceMappingURL"}=${scriptTarget}\n`,
);
const scanned = await inspect(root);
expect(scanned.debug_companions).toEqual({ paths: [], skipped: 0 });
expect(debugCompanionsStatus(scanned)).not.toBe("exposed");
expect(scanned.source_maps.skipped).toBeGreaterThan(0);
},
);
it("keeps a missing exact declaration unresolved when later guesses share its lookup key", async (context) => {
const root = await createTestTempDirectory("rea-missing-exact-map-guess-");
const missing = "cấ.map";
const first = "câ\u0301.map";
const second = "ca\u0302\u0301.map";
await writeFile(join(root, first), "linker map\n");
await writeFile(join(root, second), "linker map\n");
if ((await readdir(root)).length !== 2) {
context.skip();
return;
}
await writeFile(join(root, "a.html"), `<script src="${first}"></script>`);
expect(first.normalize("NFC")).toBe(missing);
expect(second.normalize("NFC")).toBe(missing);
await writeFile(join(root, "a.html"), `<script src="${missing}"></script>`);
await writeFile(join(root, "b.js"), `//# ${"sourceMappingURL"}=${first}\n`);
await writeFile(join(root, "z.js"), `//# ${"sourceMappingURL"}=${missing}\n`);
const scanned = await inspect(root);
expect(scanned.debug_companions.paths).toEqual([join(root, second)]);
expect(scanned.debug_companions).toEqual({ paths: [], skipped: 1 });
expect(debugCompanionsStatus(scanned)).toBe("unknown");
});
it("keeps a distinct normalized sibling linker map beside a declared one", async (context) => {
+10 -2
View File
@@ -1999,8 +1999,8 @@ const noteScriptTarget = (state: ScanState, location: string): void => {
const target = memberLocationKey(state, location);
const spellings = state.scriptTargets.get(target);
const spelling = anchoredLocation(state, location);
if (spellings?.has(spelling)) return;
retractLinkerMapGuesses(state, location);
if (spellings?.has(spelling)) return;
state.scriptTargetBytes += Buffer.byteLength(spelling);
if (state.scriptTargetBytes > MAX_RELEASE_EXPOSURE_SCRIPT_TARGET_BYTES) {
state.skippedMaps += 1;
@@ -2114,7 +2114,6 @@ const noteMapTarget = (state: ScanState, location: string): void => {
const target = memberLocationKey(state, location);
const spellings = state.mapTargets.get(target);
const spelling = anchoredLocation(state, location);
if (spellings?.has(spelling)) return;
// A late declaration can identify an exact occurrence already examined as
// a linker-map candidate. Claim that identity before final target coverage
// considers other spellings in the normalized bucket.
@@ -2123,6 +2122,7 @@ const noteMapTarget = (state: ScanState, location: string): void => {
// Retract once per exact declaration; repeating it must not consume peers
// that only share its normalized lookup key.
state.skippedMaps += retractLinkerMapGuesses(state, location);
if (spellings?.has(spelling)) return;
state.mapTargetBytes += Buffer.byteLength(spelling);
if (state.mapTargetBytes > MAX_RELEASE_EXPOSURE_MAP_TARGET_BYTES) {
state.skippedMaps += 1;
@@ -2178,6 +2178,14 @@ const retractLinkerMapGuesses = (
if (paths.size === 0) state.linkerMapGuesses.delete(key);
return 1;
}
// A read can add an exact guess after an early declaration. Retract that
// guess first; an admitted missing spelling cannot identify a peer source.
if (
state.examinedMembers.get(key)?.has(spelling) === true &&
(state.mapTargets.get(key)?.has(spelling) === true ||
state.scriptTargets.get(key)?.has(spelling) === true)
)
return 0;
// Without an exact source spelling, normalization cannot prove which
// occurrence the declaration means. Keep companion coverage unresolved.
state.skippedCompanions += paths.size;
@@ -1,11 +1,14 @@
import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js";
import { awaitAcknowledgement } from "../../../tests/fixtures/lifecycleAcknowledgement.js";
import { dirname, join } from "node:path";
import { getEventListeners, once } from "node:events";
import { Readable } from "node:stream";
import { finished } from "node:stream/promises";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { createHash, randomBytes } from "node:crypto";
import { rename, unlink, writeFile, type FileHandle } from "node:fs/promises";
import { buffer } from "node:stream/consumers";
import { deflateSync } from "node:zlib";
import { deflateSync, Inflate } from "node:zlib";
import { setImmediate } from "node:timers/promises";
import { expect, it, vi } from "vitest";
import { PyInstallerArtifactReader } from "../PyInstallerArtifactReader.js";
@@ -16,6 +19,7 @@ import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { marshalCode } from "../../python/PythonPyc.fixture.js";
import * as artifactPaths from "../../resources/ArtifactPaths.js";
import { PyInstallerPyzStore } from "./PyInstallerPyzStore.js";
import { pyInstallerMemberStream } from "./PyInstallerMemberStream.js";
import {
ArtifactReaderFailure,
type ArtifactEntry,
@@ -50,6 +54,195 @@ const deferred = () => {
return { promise, resolve };
};
it.each(
[false, true].flatMap((compressed) =>
[
{ stop: "caller-null", reason: null },
{ stop: "caller-object", reason: { stage: "buffered member" } },
{ stop: "reader-close", reason: undefined },
].map((test) => ({ compressed, ...test })),
),
)(
"retires completed buffered members ($stop, compressed: $compressed)",
async ({ compressed, stop, reason }) => {
const payload = Buffer.from("buffered");
await withPyInstallerReader(
cArchiveExecutable({
layout: "2.1",
pythonVersion: 309,
members: [
{ name: "data", typeCode: "x", payload, compress: compressed },
],
}),
async (reader) => {
const controller = new AbortController();
const entry = (await members(reader))[0]!;
const stream = await reader.open(entry, controller.signal);
const retired = finished(stream).catch((cause: unknown) => cause);
try {
await awaitAcknowledgement(
once(stream, "finish").then(() => undefined),
retired,
"PyInstaller writable finish",
);
// The pipeline's completion callbacks have settled; no consumer has read.
await setImmediate();
expect(stream.readableLength).toBe(payload.length);
expect(stream.closed).toBe(false);
if (stop === "reader-close") await reader.close();
else controller.abort(reason);
await setImmediate();
expect(stream.destroyed).toBe(true);
expect(stream.closed).toBe(true);
expect(stream.errored).toBeInstanceOf(ArtifactReaderFailure);
expect(stream.errored).toMatchObject({ reason: "cancelled" });
if (stop !== "reader-close")
expect(stream.errored?.cause).toBe(reason);
} finally {
stream.destroy();
await retired;
}
},
);
},
);
it.each(
[false, true].flatMap((compressed) =>
(["consume", "cancel"] as const).map((stop) => ({ compressed, stop })),
),
)(
"keeps completion owned until readable retirement ($stop, compressed: $compressed)",
async ({ compressed, stop }) => {
const payload = Buffer.from("buffered");
const stored = compressed ? deflateSync(payload) : payload;
const controller = new AbortController();
const member = pyInstallerMemberStream(
Readable.from([stored], { objectMode: false }),
{
sourceBytes: stored.length,
compressed,
header: undefined,
expectedBytes: payload.length,
maxBytes: payload.length,
signal: controller.signal,
},
);
let settled = false;
void member.completion.then(
() => {
settled = true;
},
() => {
settled = true;
},
);
try {
await awaitAcknowledgement(
once(member.output, "finish").then(() => undefined),
member.completion,
"member writable finish",
);
await setImmediate();
expect(settled).toBe(false);
expect(getEventListeners(controller.signal, "abort")).toHaveLength(1);
if (stop === "consume") {
expect((await buffer(member.output)).equals(payload)).toBe(true);
await member.completion;
} else {
const reason = { stage: "after producer completion" };
controller.abort(reason);
await expect(member.completion).rejects.toMatchObject({
reason: "cancelled",
cause: reason,
});
expect(member.output.errored?.cause).toBe(reason);
}
expect(settled).toBe(true);
expect(getEventListeners(controller.signal, "abort")).toEqual([]);
} finally {
member.output.destroy();
await member.completion.catch(() => undefined);
}
},
);
it.each([false, true])(
"forwards a queued source failure before cancellation (compressed: %s)",
async (compressed) => {
const source = new Readable({ read() {} });
const controller = new AbortController();
const io = Object.assign(new Error("queued descriptor EIO"), {
code: "EIO",
});
const member = pyInstallerMemberStream(source, {
sourceBytes: 8,
compressed,
header: undefined,
expectedBytes: 8,
maxBytes: 8,
signal: controller.signal,
});
const consumed = buffer(member.output);
void consumed.catch(() => undefined);
source.destroy(io);
// destroyed/errored are already established; the error event has not fired.
controller.abort({ stage: "before error forwarding" });
try {
await expect(consumed).rejects.toMatchObject({ reason: "io", cause: io });
await expect(member.completion).rejects.toMatchObject({
reason: "io",
cause: io,
});
} finally {
member.output.destroy();
await member.completion.catch(() => undefined);
}
},
);
it("forwards an established native inflate failure before cancellation", async () => {
const controller = new AbortController();
const nativeDestroy = Inflate.prototype.destroy;
let compressionFailure: Error | undefined;
const destroy = vi
.spyOn(Inflate.prototype, "destroy")
.mockImplementation(function (this: Inflate, error) {
const result = nativeDestroy.call(this, error);
if (error !== undefined && !controller.signal.aborted) {
compressionFailure = error;
// Native zlib has failed; its error event has not forwarded the cause.
controller.abort({ stage: "inflate error forwarding" });
}
return result;
});
const encoded = Buffer.from("invalid zlib");
const member = pyInstallerMemberStream(Readable.from([encoded]), {
sourceBytes: encoded.length,
compressed: true,
header: undefined,
expectedBytes: 8,
maxBytes: 8,
signal: controller.signal,
});
try {
await expect(buffer(member.output)).rejects.toMatchObject({
reason: "format",
cause: { code: "Z_DATA_ERROR" },
});
expect(compressionFailure).toBeDefined();
expect(member.output.errored?.cause).toBe(compressionFailure);
await expect(member.completion).rejects.toMatchObject({
reason: "format",
cause: compressionFailure,
});
} finally {
member.output.destroy();
await member.completion.catch(() => undefined);
destroy.mockRestore();
}
});
it.each(["success", "io", "caller cancellation", "reader close"] as const)(
"joins the pending descriptor read after a module prefix stop (%s)",
async (outcome) => {
@@ -129,7 +129,14 @@ export const pyInstallerMemberStream = (
output.on("error", () => undefined);
const onAbort = (): void => {
// Verification and private-file cleanup own their accepted native I/O.
if (completing) return;
if (completing || output.destroyed) return;
// A producer can already own an error before its error event forwards it.
// Do not replace that established failure with a downstream cancellation.
if (
source.errored !== null ||
(inflate !== undefined && inflate.errored !== null)
)
return;
// The descriptor source owns cancellation until its native read settles.
// Retiring it downstream earlier would hide a native I/O rejection.
if (!source.readableEnded && !source.destroyed) return;
@@ -142,6 +149,12 @@ export const pyInstallerMemberStream = (
);
};
options.signal.addEventListener("abort", onAbort, { once: true });
const retired = new Promise<void>((resolve) => {
output.once("close", () => {
options.signal.removeEventListener("abort", onAbort);
resolve();
});
});
const onSourceFailure = (cause: unknown): void => {
failure ??=
cause instanceof ArtifactReaderFailure
@@ -166,7 +179,7 @@ export const pyInstallerMemberStream = (
);
output.destroy(failure);
});
const completion = (
const production = (
inflate === undefined
? pipeline(source, output)
: pipeline(source, inflate, output)
@@ -177,9 +190,17 @@ export const pyInstallerMemberStream = (
throw failure ?? cause;
})
.finally(() => {
options.signal.removeEventListener("abort", onAbort);
source.removeListener("error", onSourceFailure);
});
// Production can finish with unread output. Both its cancellation listener
// and its owning reader operation survive until that readable side retires.
// Await both settlements even on failure before the reader operation settles.
const completion = Promise.allSettled([production, retired]).then(
([produced]) => {
if (produced.status === "rejected") throw produced.reason;
if (output.errored !== null) throw output.errored;
},
);
// Preserve rejection for the owning scope without an early unhandled promise.
void completion.catch(() => undefined);
if (options.signal.aborted) onAbort();
+76 -1
View File
@@ -1,4 +1,4 @@
import { writeFile } from "node:fs/promises";
import { readdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
@@ -107,3 +107,78 @@ cliTest(
expect(report.summary.by_status.not_assessed).toBe(3);
},
);
cliTest.for(["map", "script"])(
"retains the distinct linker-map source across declaration owners, %s first",
async (firstRole, { cli, skip }) => {
const root = await createTestTempDirectory("rea-release-map-identity-cli-");
const declared = "café.map";
const companion = "cafe\u0301.map";
await writeFile(join(root, declared), "linker map\n");
await writeFile(join(root, companion), "linker map\n");
if ((await readdir(root)).length !== 2) {
skip();
return;
}
await writeFile(
join(root, firstRole === "map" ? "z.js" : "zz.js"),
`//# ${"sourceMappingURL"}=${declared}\n`,
);
await writeFile(
join(root, firstRole === "script" ? "z.html" : "zz.html"),
`<script src="${declared}"></script>`,
);
const result = await cli.run({
arguments: ["audit-release-exposure", root, "--json"],
});
expect(result.exitCode, result.stderr).toBe(0);
const report = outputSchema.parse(result.json).normalized_result;
expect(
report.checks.find(({ id }) => id === "native.debug_companions"),
).toMatchObject({
status: "exposed",
confidence: "inferred",
observation: {
counts: { companions: 1 },
samples: [join(root, companion)],
},
});
},
);
cliTest(
"retracts a script target guessed after its early declaration",
async ({ cli, skip }) => {
const root = await createTestTempDirectory("rea-release-late-guess-cli-");
const declared = "café.map";
const other = "cafe\u0301.map";
await writeFile(join(root, declared), "linker map\n");
await writeFile(join(root, other), "linker map\n");
if ((await readdir(root)).length !== 2) {
skip();
return;
}
await writeFile(
join(root, "a.html"),
`<script src="${declared}"></script>`,
);
await writeFile(join(root, "b.js"), `//# ${"sourceMappingURL"}=${other}\n`);
await writeFile(
join(root, "z.js"),
`//# ${"sourceMappingURL"}=${declared}\n`,
);
const result = await cli.run({
arguments: ["audit-release-exposure", root, "--json"],
});
expect(result.exitCode, result.stderr).toBe(0);
const report = outputSchema.parse(result.json).normalized_result;
const check = report.checks.find(
({ id }) => id === "native.debug_companions",
);
expect(check?.status).not.toBe("exposed");
expect(check?.observation).toMatchObject({
counts: { companions: 0 },
samples: [],
});
},
);