mirror of
https://github.com/morluto/rea.git
synced 2026-10-26 17:35:11 +00:00
morluto
45a411e8f8
Project PYZ namespace packages as metadata so their query paths do not claim output directories. This lets independent CArchive files retain the same logical path and remain inspectable and extractable. Fixes #6539
1214 lines
36 KiB
TypeScript
1214 lines
36 KiB
TypeScript
import { buffer } from "node:stream/consumers";
|
|
import { readFile } from "node:fs/promises";
|
|
import { dirname, join } from "node:path";
|
|
import { deflateSync } from "node:zlib";
|
|
import { expect, it, vi } from "vitest";
|
|
import { withPyInstallerReader } from "../../../tests/fixtures/pyinstallerReader.js";
|
|
import {
|
|
scanCanonicalArtifactInventory,
|
|
inventoryArtifact,
|
|
materializeArtifactInventory,
|
|
} from "../../../tests/fixtures/artifactInventory.js";
|
|
import {
|
|
cArchiveExecutable,
|
|
pyzArchive,
|
|
PYTHON_39_MAGIC,
|
|
} from "./CArchive.fixture.js";
|
|
import { readPyzArchive } from "./PyzArchive.js";
|
|
import { createEvidence } from "../../domain/evidence.js";
|
|
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
|
import { parseArtifactInventoryEvidence } from "../../domain/artifactInventoryEvidence.js";
|
|
import {
|
|
ArtifactReaderFailure,
|
|
type ArtifactEntry,
|
|
} from "../ArtifactReader.js";
|
|
|
|
it("keeps compressed source locations unknown when stored and decoded lengths coincide", async () => {
|
|
const payload = Buffer.alloc(11, 0x61);
|
|
expect(deflateSync(payload).length).toBe(payload.length);
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [{ name: "data", typeCode: "x", payload, compress: true }],
|
|
}),
|
|
async (_reader, _carrier, path) => {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
expect(
|
|
inventory.occurrences.find((row) => row.logical_path === "data"),
|
|
).toMatchObject({
|
|
hash_status: "verified",
|
|
source_location: null,
|
|
format_metadata: { compression_flag: 1 },
|
|
});
|
|
},
|
|
);
|
|
});
|
|
|
|
it("bounds an admitted drive-prefixed CArchive path before diagnostic serialization", async () => {
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "C:" + "x".repeat(10 * 1024 * 1024),
|
|
typeCode: "x",
|
|
payload: Buffer.from("x"),
|
|
compress: false,
|
|
},
|
|
],
|
|
}),
|
|
async (reader) => {
|
|
const failure = await reader
|
|
.entries()
|
|
[Symbol.asyncIterator]()
|
|
.next()
|
|
.then(
|
|
() => undefined,
|
|
(cause: unknown) => cause,
|
|
);
|
|
expect(failure).toMatchObject({ reason: "path" });
|
|
expect((failure as Error).message.length).toBeLessThan(1024);
|
|
},
|
|
);
|
|
});
|
|
|
|
it.each(["magic", "marshal"])(
|
|
"reuses the private bytes of an opaque outer-compressed PYZ (%s)",
|
|
async (kind) => {
|
|
const payload = pyzArchive({
|
|
magic: kind === "magic" ? Buffer.alloc(4) : PYTHON_39_MAGIC,
|
|
modules: [],
|
|
...(kind === "marshal" ? { toc: Buffer.from([0x4e]) } : {}),
|
|
});
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [{ name: "PYZ.pyz", typeCode: "z", payload, compress: true }],
|
|
}),
|
|
async (reader, carrier) => {
|
|
const entries: ArtifactEntry[] = [];
|
|
for await (const entry of reader.entries()) entries.push(entry);
|
|
const entry = entries[0]!;
|
|
expect(entry.kind).toBe("file");
|
|
const read = vi
|
|
.spyOn(carrier, "read")
|
|
.mockRejectedValue(
|
|
new Error("opaque image must not be reinflated from carrier"),
|
|
);
|
|
try {
|
|
for (let run = 0; run < 2; run++)
|
|
expect(await buffer(await reader.open(entry))).toEqual(payload);
|
|
} finally {
|
|
read.mockRestore();
|
|
}
|
|
},
|
|
);
|
|
},
|
|
);
|
|
|
|
it.each([true, false])(
|
|
"authenticates mixed-kind duplicate pages in any merge order (option first: %s)",
|
|
async (optionFirst) => {
|
|
const logical = `__pyinstaller__/runtime-options/${optionFirst ? 0 : 1}`;
|
|
const option = {
|
|
name: "pyi-option",
|
|
typeCode: "o",
|
|
payload: Buffer.alloc(0),
|
|
compress: false,
|
|
};
|
|
const file = {
|
|
name: logical,
|
|
typeCode: "x",
|
|
payload: Buffer.from("first"),
|
|
compress: false,
|
|
};
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
...(optionFirst ? [option, file] : [file, option]),
|
|
{
|
|
...file,
|
|
name: logical.replaceAll("/", "\\"),
|
|
payload: Buffer.from("second"),
|
|
},
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
const inventory = await inventoryArtifact(path);
|
|
const group = inventory.occurrences.filter(
|
|
(row) => row.logical_path === logical,
|
|
);
|
|
expect(group).toHaveLength(3);
|
|
const pages = group.map((row, index) =>
|
|
createEvidence(
|
|
{
|
|
path,
|
|
sha256: inventory.manifest.root_sha256,
|
|
format: "pyinstaller",
|
|
},
|
|
{
|
|
id: "rea-artifact-graph",
|
|
name: "artifact inventory",
|
|
version: "1",
|
|
},
|
|
{
|
|
operation: "inventory_artifact",
|
|
parameters: { page: index },
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
result: jsonValueSchema.parse({
|
|
...inventory,
|
|
nodes: index === 0 ? inventory.nodes : [],
|
|
edges: index === 0 ? inventory.edges : [],
|
|
occurrences:
|
|
index === 0
|
|
? [
|
|
...inventory.occurrences.filter(
|
|
(item) => item.logical_path !== logical,
|
|
),
|
|
row,
|
|
]
|
|
: [row],
|
|
}),
|
|
},
|
|
),
|
|
);
|
|
for (const order of [
|
|
[0, 1, 2],
|
|
[0, 2, 1],
|
|
[1, 0, 2],
|
|
[1, 2, 0],
|
|
[2, 0, 1],
|
|
[2, 1, 0],
|
|
]) {
|
|
const merged = parseArtifactInventoryEvidence(
|
|
order.map((index) => pages[index]),
|
|
).inventory;
|
|
expect(merged.complete).toBe(true);
|
|
expect(merged.manifest).toEqual(inventory.manifest);
|
|
expect(merged.occurrences).toEqual(inventory.occurrences);
|
|
}
|
|
},
|
|
);
|
|
},
|
|
);
|
|
|
|
it.each(["duplicate", "size", "offset", "overlap"])(
|
|
"bounds %s diagnostics for large nondeclaration names",
|
|
async (mode) => {
|
|
const name = "member".repeat(200_000);
|
|
const first = {
|
|
name: mode === "overlap" ? name + "-a" : name,
|
|
typeCode: "x",
|
|
payload: Buffer.from("body"),
|
|
compress: false,
|
|
...(mode === "size" ? { declaredUncompressedSize: 5 } : {}),
|
|
};
|
|
const bytes = cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members:
|
|
mode === "duplicate" || mode === "overlap"
|
|
? [
|
|
first,
|
|
{ ...first, name: mode === "duplicate" ? name : name + "-b" },
|
|
]
|
|
: [first],
|
|
});
|
|
const toc = 64 + bytes.readUInt32BE(bytes.length - 88 + 12);
|
|
if (mode === "offset")
|
|
bytes.writeUInt32BE(
|
|
bytes.readUInt32BE(bytes.length - 88 + 12) + 1,
|
|
toc + 4,
|
|
);
|
|
if (mode === "overlap")
|
|
bytes.writeUInt32BE(0, toc + bytes.readUInt32BE(toc) + 4);
|
|
await withPyInstallerReader(bytes, async (_reader, _handle, path) => {
|
|
const failure: unknown = await scanCanonicalArtifactInventory(path).then(
|
|
() => undefined,
|
|
(cause: unknown) => cause,
|
|
);
|
|
expect(failure).toBeInstanceOf(ArtifactReaderFailure);
|
|
if (!(failure instanceof ArtifactReaderFailure))
|
|
throw new Error("Expected parser failure");
|
|
expect(failure.reason).toBe(mode === "offset" ? "format" : "integrity");
|
|
expect(failure.message.length).toBeLessThan(128);
|
|
expect(failure.message).toContain(
|
|
`table index ${mode === "duplicate" || mode === "overlap" ? 1 : 0}`,
|
|
);
|
|
});
|
|
},
|
|
);
|
|
|
|
it.each(["empty", "span"])(
|
|
"bounds PYZ %s diagnostics for a maximum-size marshal name",
|
|
async (mode) => {
|
|
const image = pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [
|
|
{ name: "n".repeat(65_536), typeCode: 0, body: Buffer.from("body") },
|
|
],
|
|
});
|
|
image.writeInt32LE(
|
|
mode === "empty" ? 0 : image.length + 1,
|
|
image.length - 4,
|
|
);
|
|
await withPyInstallerReader(image, async (_reader, handle) => {
|
|
const parsed = await readPyzArchive(handle, 0, image.length);
|
|
const message = !parsed.ok
|
|
? parsed.error.message
|
|
: parsed.value.kind === "opaque"
|
|
? parsed.value.limitation
|
|
: undefined;
|
|
expect(message).toBeDefined();
|
|
expect(message?.length).toBeLessThan(256);
|
|
expect(message).toContain("table index 0");
|
|
});
|
|
},
|
|
);
|
|
|
|
it("keeps malformed-declaration diagnostics bounded while identifying the TOC row", async () => {
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "option".repeat(200_000),
|
|
typeCode: "o",
|
|
payload: Buffer.from("unexpected"),
|
|
compress: false,
|
|
},
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
const failure: unknown = await scanCanonicalArtifactInventory(path).then(
|
|
() => undefined,
|
|
(cause: unknown) => cause,
|
|
);
|
|
expect(failure).toBeInstanceOf(ArtifactReaderFailure);
|
|
if (!(failure instanceof ArtifactReaderFailure))
|
|
throw new Error("Expected a parser failure");
|
|
expect(failure.reason).toBe("integrity");
|
|
expect(failure.message.length).toBeLessThan(128);
|
|
expect(failure.message).toContain("table index 0");
|
|
},
|
|
);
|
|
});
|
|
|
|
for (const typeCode of ["o", "d"] as const) {
|
|
it.each(["stored", "compressed", "flag-only"])(
|
|
`rejects local payload/compression on ${typeCode} declarations (%s)`,
|
|
async (mode) => {
|
|
const bytes = cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: typeCode === "d" ? "../owner:member" : "pyi-option",
|
|
typeCode,
|
|
payload:
|
|
mode === "flag-only"
|
|
? Buffer.alloc(0)
|
|
: Buffer.from("local payload"),
|
|
compress: mode === "compressed",
|
|
},
|
|
],
|
|
});
|
|
if (mode === "flag-only") {
|
|
const archiveStart = 64;
|
|
const toc = bytes.readUInt32BE(bytes.length - 88 + 12);
|
|
bytes[archiveStart + toc + 16] = 1;
|
|
}
|
|
await withPyInstallerReader(bytes, async (_reader, _handle, path) => {
|
|
await expect(
|
|
scanCanonicalArtifactInventory(path),
|
|
).rejects.toMatchObject({
|
|
reason: "integrity",
|
|
message: expect.stringContaining("declaration"),
|
|
});
|
|
});
|
|
},
|
|
);
|
|
it(`does not claim a destination case collision for a ${typeCode} query path`, async () => {
|
|
const logical = `__pyinstaller__/${typeCode === "o" ? "runtime-options" : "dependencies"}/0`;
|
|
const realPath = logical.toUpperCase();
|
|
const payload = Buffer.from("real file");
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: typeCode === "d" ? "../owner:member" : "pyi-option",
|
|
typeCode,
|
|
payload: Buffer.alloc(0),
|
|
compress: false,
|
|
},
|
|
{ name: realPath, typeCode: "x", payload, compress: false },
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
expect(
|
|
[
|
|
...inventory.limitations,
|
|
...inventory.occurrences.flatMap((row) => row.limitations),
|
|
].some((note) => note.includes("case-insensitive destination")),
|
|
).toBe(false);
|
|
const output = join(dirname(path), "extracted");
|
|
await materializeArtifactInventory(
|
|
{
|
|
inputPath: path,
|
|
inputFormat: "elf",
|
|
outputRoot: output,
|
|
environment: process.env,
|
|
integrityPolicy: "fail",
|
|
},
|
|
path,
|
|
inventory,
|
|
);
|
|
expect(await readFile(join(output, realPath))).toEqual(payload);
|
|
},
|
|
);
|
|
});
|
|
}
|
|
|
|
it("keeps genuine destination collisions on files without annotating same-path metadata", async () => {
|
|
const logical = "__pyinstaller__/runtime-options/0";
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "pyi-option",
|
|
typeCode: "o",
|
|
payload: Buffer.alloc(0),
|
|
compress: false,
|
|
},
|
|
...[logical, logical.toUpperCase()].map((name) => ({
|
|
name,
|
|
typeCode: "x",
|
|
payload: Buffer.from(name),
|
|
compress: false,
|
|
})),
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
expect(
|
|
inventory.limitations.some((note) =>
|
|
note.includes("case-insensitive destination"),
|
|
),
|
|
).toBe(true);
|
|
const metadata = inventory.occurrences.find(
|
|
(row) => row.entry_kind === "metadata",
|
|
);
|
|
expect(
|
|
metadata?.limitations.some((note) =>
|
|
note.includes("case-insensitive destination"),
|
|
),
|
|
).toBe(false);
|
|
expect(
|
|
inventory.occurrences
|
|
.filter(
|
|
(row) => row.entry_kind === "file" && row.logical_path !== ".",
|
|
)
|
|
.every((row) =>
|
|
row.limitations.some((note) =>
|
|
note.includes("case-insensitive destination"),
|
|
),
|
|
),
|
|
).toBe(true);
|
|
},
|
|
);
|
|
});
|
|
|
|
it("anchors declarations to their source container rather than a same-named directory", async () => {
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "../owner:member",
|
|
typeCode: "d",
|
|
payload: Buffer.alloc(0),
|
|
compress: false,
|
|
},
|
|
{
|
|
name: "__pyinstaller__",
|
|
typeCode: "z",
|
|
compress: false,
|
|
payload: pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [
|
|
{ name: "dependencies", typeCode: 3, body: Buffer.alloc(0) },
|
|
],
|
|
}),
|
|
},
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
const root = inventory.occurrences.find(
|
|
(row) => row.logical_path === ".",
|
|
);
|
|
const directory = inventory.occurrences.find(
|
|
(row) => row.logical_path === "__pyinstaller__/dependencies",
|
|
);
|
|
const metadata = inventory.occurrences.find(
|
|
(row) => row.entry_kind === "metadata",
|
|
);
|
|
expect(directory).toBeDefined();
|
|
expect(root).toBeDefined();
|
|
expect(metadata).toBeDefined();
|
|
expect(metadata?.parent_occurrence_id).toBe(root?.occurrence_id);
|
|
expect(metadata?.parent_occurrence_id).not.toBe(directory?.occurrence_id);
|
|
},
|
|
);
|
|
});
|
|
|
|
it("resolves a unique source-map file despite a metadata query-path alias", async () => {
|
|
const logical = "__pyinstaller__/runtime-options/0";
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "pyi-option",
|
|
typeCode: "o",
|
|
payload: Buffer.alloc(0),
|
|
compress: false,
|
|
},
|
|
{
|
|
name: logical,
|
|
typeCode: "x",
|
|
payload: Buffer.from("let value = 1;"),
|
|
compress: false,
|
|
},
|
|
{
|
|
name: logical + ".map",
|
|
typeCode: "x",
|
|
payload: Buffer.from(
|
|
JSON.stringify({
|
|
version: 3,
|
|
sources: ["source.js"],
|
|
names: [],
|
|
mappings: "AAAA",
|
|
}),
|
|
),
|
|
compress: false,
|
|
},
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
const source = inventory.occurrences.find(
|
|
(row) => row.logical_path === logical && row.entry_kind === "file",
|
|
);
|
|
const map = inventory.occurrences.find(
|
|
(row) => row.logical_path === logical + ".map",
|
|
);
|
|
expect(source?.artifact_id).toBeTruthy();
|
|
expect(map).toBeDefined();
|
|
expect(
|
|
inventory.edges.find(
|
|
(edge) => edge.occurrence_id === map?.occurrence_id,
|
|
),
|
|
).toMatchObject({
|
|
relation: "maps-source",
|
|
parent_artifact_id: source?.artifact_id,
|
|
});
|
|
expect(
|
|
map?.limitations.some((note) => note.includes("multiple occurrences")),
|
|
).toBe(false);
|
|
},
|
|
);
|
|
});
|
|
|
|
it.each(["x", "s", "empty-module"])(
|
|
"does not treat a same-named %s as a crypto-key module",
|
|
async (kind) => {
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "pyimod00_crypto_key",
|
|
typeCode: kind === "empty-module" ? "m" : kind,
|
|
payload:
|
|
kind === "empty-module"
|
|
? Buffer.alloc(0)
|
|
: Buffer.from("ordinary content"),
|
|
compress: false,
|
|
},
|
|
{
|
|
name: "PYZ.pyz",
|
|
typeCode: "z",
|
|
compress: false,
|
|
payload: pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [
|
|
{
|
|
name: "app",
|
|
typeCode: 0,
|
|
body: Buffer.from("readable module"),
|
|
},
|
|
],
|
|
}),
|
|
},
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
expect(
|
|
inventory.occurrences.find((row) => row.logical_path === ".")
|
|
?.format_metadata,
|
|
).toMatchObject({ encryption_marker_present: false });
|
|
expect(
|
|
inventory.occurrences.find(
|
|
(row) => row.logical_path === "PYZ.pyz/app.pyc",
|
|
),
|
|
).toMatchObject({
|
|
encrypted: false,
|
|
hash_status: "verified",
|
|
format_metadata: { encryption_source: null },
|
|
});
|
|
const output = join(dirname(path), "extracted");
|
|
await materializeArtifactInventory(
|
|
{
|
|
inputPath: path,
|
|
inputFormat: "elf",
|
|
outputRoot: output,
|
|
environment: process.env,
|
|
integrityPolicy: "fail",
|
|
},
|
|
path,
|
|
inventory,
|
|
);
|
|
expect(
|
|
(await readFile(join(output, "PYZ.pyz/app.pyc"))).subarray(16),
|
|
).toEqual(Buffer.from("readable module"));
|
|
},
|
|
);
|
|
},
|
|
);
|
|
|
|
it.each([
|
|
"module-in-toc",
|
|
"module-crossing-toc",
|
|
"namespace-in-toc",
|
|
"namespace-past-image",
|
|
])("rejects PYZ positions outside the payload region (%s)", async (kind) => {
|
|
const image = pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [
|
|
{
|
|
name: "app",
|
|
typeCode: kind.startsWith("namespace") ? 3 : 0,
|
|
body: Buffer.from("payload"),
|
|
},
|
|
],
|
|
});
|
|
const toc = image.readUInt32BE(8);
|
|
if (kind === "module-crossing-toc")
|
|
image.writeInt32LE(toc - 12 + 1, image.length - 4);
|
|
else
|
|
image.writeInt32LE(
|
|
kind === "namespace-past-image" ? image.length + 1 : toc + 1,
|
|
image.length - 9,
|
|
);
|
|
await withPyInstallerReader(image, async (_reader, handle) => {
|
|
await expect(
|
|
readPyzArchive(handle, 0, image.length),
|
|
).resolves.toMatchObject({ ok: false, error: { reason: "integrity" } });
|
|
});
|
|
});
|
|
|
|
it("retains legacy namespace payload extents without materializing them", async () => {
|
|
const image = pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [
|
|
{
|
|
name: "ns",
|
|
typeCode: 3,
|
|
legacyNamespacePayload: true,
|
|
body: Buffer.from("legacy namespace code object"),
|
|
},
|
|
],
|
|
});
|
|
const length = image.readInt32LE(image.length - 4);
|
|
expect(length).toBeGreaterThan(0);
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{ name: "PYZ.pyz", typeCode: "z", compress: false, payload: image },
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
expect(
|
|
inventory.occurrences.find((row) => row.logical_path === "PYZ.pyz/ns"),
|
|
).toMatchObject({
|
|
entry_kind: "metadata",
|
|
declared_size: 0,
|
|
compressed_size: length,
|
|
format_metadata: {
|
|
relative_offset: 12,
|
|
type_code: 3,
|
|
compression_flag: 1,
|
|
},
|
|
limitations: expect.arrayContaining([
|
|
expect.stringContaining("not materialized"),
|
|
]),
|
|
});
|
|
const result = await materializeArtifactInventory(
|
|
{
|
|
inputPath: path,
|
|
inputFormat: "elf",
|
|
outputRoot: join(dirname(path), "out"),
|
|
environment: process.env,
|
|
integrityPolicy: "fail",
|
|
},
|
|
path,
|
|
inventory,
|
|
);
|
|
expect(result.artifacts).toEqual([]);
|
|
},
|
|
);
|
|
});
|
|
|
|
it.each(["magic", "marshal"])(
|
|
"retains unknown traversal coverage for an opaque PYZ %s",
|
|
async (kind) => {
|
|
const known = pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [{ name: "known", typeCode: 0, body: Buffer.from("known") }],
|
|
});
|
|
const opaque = pyzArchive({
|
|
magic: kind === "magic" ? Buffer.alloc(4) : PYTHON_39_MAGIC,
|
|
modules: [],
|
|
...(kind === "marshal" ? { toc: Buffer.from([0x4e]) } : {}),
|
|
});
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{ name: "known.pyz", typeCode: "z", payload: known, compress: false },
|
|
{
|
|
name: "opaque.pyz",
|
|
typeCode: "z",
|
|
payload: opaque,
|
|
compress: false,
|
|
},
|
|
],
|
|
}),
|
|
async (reader, _root, path) => {
|
|
for (let run = 0; run < 2; run++) {
|
|
const observed: unknown[] = [];
|
|
const listed: ArtifactEntry[] = [];
|
|
for await (const entry of reader.entries(undefined, {
|
|
onCoverage: (coverage) => observed.push(coverage),
|
|
}))
|
|
listed.push(entry);
|
|
expect(listed).toHaveLength(3);
|
|
expect(observed).toEqual([
|
|
expect.objectContaining({ status: "unknown" }),
|
|
]);
|
|
}
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
expect(inventory.coverage).toMatchObject({ status: "unknown" });
|
|
expect(inventory.limitations.join(" ")).toContain("opaque");
|
|
expect(
|
|
inventory.occurrences.find(
|
|
(row) => row.logical_path === "known.pyz/known.pyc",
|
|
),
|
|
).toMatchObject({ hash_status: "verified" });
|
|
expect(
|
|
inventory.occurrences.find((row) => row.logical_path === "opaque.pyz")
|
|
?.format_metadata,
|
|
).toMatchObject({ pyz_status: "opaque" });
|
|
},
|
|
);
|
|
},
|
|
);
|
|
|
|
it.each([false, true])(
|
|
"derives namespace encryption from stored payload and the marker (%s)",
|
|
async (marker) => {
|
|
const image = pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [
|
|
{
|
|
name: "legacy",
|
|
typeCode: 3,
|
|
legacyNamespacePayload: true,
|
|
body: Buffer.from("declared namespace payload"),
|
|
},
|
|
{ name: "empty", typeCode: 3, body: Buffer.alloc(0) },
|
|
],
|
|
});
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
...(marker
|
|
? [
|
|
{
|
|
name: "pyimod00_crypto_key",
|
|
typeCode: "m",
|
|
payload: Buffer.from("key module"),
|
|
compress: false,
|
|
},
|
|
]
|
|
: []),
|
|
{ name: "PYZ.pyz", typeCode: "z", payload: image, compress: false },
|
|
],
|
|
}),
|
|
async (_reader, _root, path) => {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
expect(
|
|
inventory.occurrences.find(
|
|
(row) => row.logical_path === "PYZ.pyz/legacy",
|
|
),
|
|
).toMatchObject({
|
|
entry_kind: "metadata",
|
|
encrypted: marker,
|
|
compressed_size: expect.any(Number),
|
|
format_metadata: {
|
|
encryption_source: marker ? "carchive-marker" : null,
|
|
},
|
|
});
|
|
expect(
|
|
inventory.occurrences.find(
|
|
(row) => row.logical_path === "PYZ.pyz/empty",
|
|
),
|
|
).toMatchObject({
|
|
entry_kind: "metadata",
|
|
encrypted: false,
|
|
compressed_size: 0,
|
|
format_metadata: { encryption_source: null },
|
|
});
|
|
},
|
|
);
|
|
},
|
|
);
|
|
|
|
it.each([
|
|
["sequence", Buffer.from("5ba1860100", "hex")],
|
|
["string", Buffer.from("7501000100", "hex")],
|
|
[
|
|
"depth",
|
|
Buffer.concat([
|
|
...Array.from({ length: 18 }, () => Buffer.from([0x29, 1])),
|
|
Buffer.from("6900000000", "hex"),
|
|
]),
|
|
],
|
|
[
|
|
"nodes",
|
|
Buffer.concat([
|
|
Buffer.from("5ba0860100", "hex"),
|
|
Buffer.alloc(100_000 * 5).fill(Buffer.from("6900000000", "hex")),
|
|
]),
|
|
],
|
|
])(
|
|
"preserves the marshal %s resource failure through inventory",
|
|
async (_guard, toc) => {
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "PYZ.pyz",
|
|
typeCode: "z",
|
|
compress: false,
|
|
payload: pyzArchive({ magic: PYTHON_39_MAGIC, modules: [], toc }),
|
|
},
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
await expect(
|
|
scanCanonicalArtifactInventory(path),
|
|
).rejects.toMatchObject({
|
|
reason: "limit",
|
|
message: expect.stringContaining("PYZ marshal"),
|
|
});
|
|
},
|
|
);
|
|
},
|
|
);
|
|
|
|
it.each([1, 2])(
|
|
"rejects namespace data extending into the table (length %s)",
|
|
async (length) => {
|
|
const image = pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [{ name: "ns", typeCode: 3, body: Buffer.alloc(0) }],
|
|
});
|
|
// The fixture's last two marshal integers are the namespace offset/length.
|
|
image.writeInt32LE(12, image.length - 9);
|
|
image.writeInt32LE(length, image.length - 4);
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{ name: "PYZ.pyz", typeCode: "z", compress: false, payload: image },
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
await expect(
|
|
scanCanonicalArtifactInventory(path),
|
|
).rejects.toMatchObject({
|
|
reason: "integrity",
|
|
message: expect.stringContaining("payload region"),
|
|
});
|
|
},
|
|
);
|
|
},
|
|
);
|
|
|
|
it("retains the producer's nonzero namespace offset without inventing data", async () => {
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "PYZ.pyz",
|
|
typeCode: "z",
|
|
compress: false,
|
|
payload: pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [{ name: "ns", typeCode: 3, body: Buffer.alloc(0) }],
|
|
}),
|
|
},
|
|
],
|
|
}),
|
|
async (_reader, _handle, path) => {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
expect(
|
|
inventory.occurrences.find((row) => row.logical_path === "PYZ.pyz/ns"),
|
|
).toMatchObject({
|
|
entry_kind: "metadata",
|
|
declared_size: 0,
|
|
compressed_size: 0,
|
|
format_metadata: {
|
|
relative_offset: 12,
|
|
type_code: 3,
|
|
compression_flag: 0,
|
|
},
|
|
});
|
|
},
|
|
);
|
|
});
|
|
|
|
it("retains multipackage references as distinct unresolved declarations", async () => {
|
|
const names = [
|
|
"../owner/app:assets/x:y",
|
|
"../owner/app:assets/x:y",
|
|
"missing-colon",
|
|
":empty-owner",
|
|
];
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: names.map((name) => ({
|
|
name,
|
|
typeCode: "d",
|
|
payload: Buffer.alloc(0),
|
|
compress: false,
|
|
})),
|
|
}),
|
|
async (reader) => {
|
|
const entries: ArtifactEntry[] = [];
|
|
for await (const entry of reader.entries()) entries.push(entry);
|
|
expect(reader.formatMetadata()).toMatchObject({
|
|
external_dependency_count: names.length,
|
|
runtime_option_count: 0,
|
|
});
|
|
expect(new Set(entries.map((entry) => entry.adapterKey)).size).toBe(
|
|
names.length,
|
|
);
|
|
for (const [index, entry] of entries.entries()) {
|
|
expect(entry.kind).toBe("metadata");
|
|
expect(entry.path).toBe(
|
|
`__pyinstaller__/dependencies/${String(index)}`,
|
|
);
|
|
expect(entry.formatMetadata).toMatchObject({
|
|
source_name: names[index],
|
|
type_code: "d",
|
|
dependency_status: index < 2 ? "unresolved" : "invalid-reference",
|
|
dependency_archive: index < 2 ? "../owner/app" : null,
|
|
dependency_member: index < 2 ? "assets/x:y" : null,
|
|
});
|
|
await expect(reader.open(entry)).rejects.toMatchObject({
|
|
reason: "format",
|
|
});
|
|
}
|
|
},
|
|
);
|
|
});
|
|
|
|
for (const container of ["carchive", "outer-pyz", "inner-pyz"] as const) {
|
|
it.each(["junk", "second-stream"])(
|
|
`rejects unused compressed input in ${container} (%s)`,
|
|
async (suffixKind) => {
|
|
const suffix =
|
|
suffixKind === "junk"
|
|
? Buffer.from("trailing bytes")
|
|
: deflateSync(Buffer.from("another stream"));
|
|
const body = Buffer.from("first stream");
|
|
const image = pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [
|
|
{
|
|
name: "app",
|
|
typeCode: 0,
|
|
body,
|
|
...(container === "inner-pyz" ? { compressedSuffix: suffix } : {}),
|
|
},
|
|
],
|
|
});
|
|
const bytes = cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: container === "carchive" ? "data" : "PYZ.pyz",
|
|
typeCode: container === "carchive" ? "x" : "z",
|
|
payload: container === "carchive" ? body : image,
|
|
compress: container !== "inner-pyz",
|
|
...(container === "inner-pyz" ? {} : { compressedSuffix: suffix }),
|
|
},
|
|
],
|
|
});
|
|
await withPyInstallerReader(bytes, async (reader) => {
|
|
const consume = async () => {
|
|
for await (const entry of reader.entries())
|
|
if (entry.kind === "file") await buffer(await reader.open(entry));
|
|
};
|
|
await expect(consume()).rejects.toMatchObject({
|
|
reason: "integrity",
|
|
message: expect.stringContaining("complete declared source span"),
|
|
});
|
|
});
|
|
},
|
|
);
|
|
}
|
|
|
|
it("fills successful short reads through actual classification, TOCs and selected module hashing", async () => {
|
|
const bytes = cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "PYZ.pyz",
|
|
typeCode: "z",
|
|
compress: false,
|
|
payload: pyzArchive({
|
|
magic: PYTHON_39_MAGIC,
|
|
modules: [
|
|
{ name: "app", typeCode: 0, body: Buffer.from("module body") },
|
|
],
|
|
}),
|
|
},
|
|
],
|
|
});
|
|
await withPyInstallerReader(bytes, async (_reader, handle, path) => {
|
|
const read = handle.read;
|
|
let shortReads = 0;
|
|
const spy = vi
|
|
.spyOn(Object.getPrototypeOf(handle), "read")
|
|
.mockImplementation(function (this: unknown, ...args: unknown[]) {
|
|
if (typeof args[2] === "number" && args[2] > 7) {
|
|
args[2] = 7;
|
|
shortReads += 1;
|
|
}
|
|
return Reflect.apply(read, this, args);
|
|
});
|
|
try {
|
|
const inventory = await scanCanonicalArtifactInventory(path);
|
|
expect(inventory.manifest.root_format).toBe("pyinstaller");
|
|
expect(
|
|
inventory.occurrences.find(
|
|
(row) => row.logical_path === "PYZ.pyz/app.pyc",
|
|
),
|
|
).toMatchObject({
|
|
hash_status: "verified",
|
|
format_metadata: { source_name: "app" },
|
|
});
|
|
expect(shortReads).toBeGreaterThan(20);
|
|
} finally {
|
|
spy.mockRestore();
|
|
}
|
|
});
|
|
});
|
|
|
|
it("distinguishes actual EOF after a short PYZ header read", async () => {
|
|
const bytes = pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] });
|
|
await withPyInstallerReader(bytes, async (_reader, handle) => {
|
|
const read = handle.read.bind(handle);
|
|
let calls = 0;
|
|
const spy = vi
|
|
.spyOn(handle, "read")
|
|
.mockImplementation((...args: unknown[]) => {
|
|
calls += 1;
|
|
if (calls > 1)
|
|
return Promise.resolve({ bytesRead: 0, buffer: args[0] });
|
|
args[2] = 3;
|
|
return Reflect.apply(read, handle, args);
|
|
});
|
|
try {
|
|
await expect(
|
|
readPyzArchive(handle, 0, bytes.length),
|
|
).resolves.toMatchObject({
|
|
ok: false,
|
|
error: { reason: "format", message: expect.stringContaining("header") },
|
|
});
|
|
expect(calls).toBe(2);
|
|
} finally {
|
|
spy.mockRestore();
|
|
}
|
|
});
|
|
});
|
|
|
|
it("retains native I/O failure after a short read and simultaneous cancellation", async () => {
|
|
const bytes = pyzArchive({ magic: PYTHON_39_MAGIC, modules: [] });
|
|
await withPyInstallerReader(bytes, async (_reader, handle) => {
|
|
const read = handle.read.bind(handle),
|
|
caller = new AbortController();
|
|
const failure = Object.assign(new Error("late native EIO"), {
|
|
code: "EIO",
|
|
});
|
|
let calls = 0;
|
|
const spy = vi
|
|
.spyOn(handle, "read")
|
|
.mockImplementation((...args: unknown[]) => {
|
|
calls += 1;
|
|
if (calls > 1) {
|
|
caller.abort(new Error("caller leaves"));
|
|
return Promise.reject(failure);
|
|
}
|
|
args[2] = 3;
|
|
return Reflect.apply(read, handle, args);
|
|
});
|
|
try {
|
|
await expect(
|
|
readPyzArchive(handle, 0, bytes.length, caller.signal),
|
|
).rejects.toMatchObject({ reason: "io", cause: failure });
|
|
} finally {
|
|
spy.mockRestore();
|
|
}
|
|
});
|
|
});
|
|
|
|
it("rejects path-component expansion before normalization or consumer trie allocation", async () => {
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: [
|
|
{
|
|
name: "a/".repeat(100_000) + "x",
|
|
typeCode: "x",
|
|
payload: Buffer.alloc(0),
|
|
compress: false,
|
|
},
|
|
],
|
|
}),
|
|
async (reader) => {
|
|
const list = async () => {
|
|
for await (const entry of reader.entries()) void entry;
|
|
};
|
|
await expect(list()).rejects.toMatchObject({
|
|
reason: "limit",
|
|
boundary: "memory",
|
|
message: expect.stringContaining("namespace budget"),
|
|
});
|
|
},
|
|
);
|
|
});
|
|
|
|
it.each([true, false])(
|
|
"extracts a real file sharing a metadata query path (option first: %s)",
|
|
async (optionFirst) => {
|
|
const path = `__pyinstaller__/runtime-options/${optionFirst ? "0" : "1"}`;
|
|
const option = {
|
|
name: "pyi-option value",
|
|
typeCode: "o",
|
|
payload: Buffer.alloc(0),
|
|
compress: false,
|
|
};
|
|
const file = {
|
|
name: path,
|
|
typeCode: "x",
|
|
payload: Buffer.from("real file payload"),
|
|
compress: false,
|
|
};
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: optionFirst ? [option, file] : [file, option],
|
|
}),
|
|
async (_reader, _handle, source) => {
|
|
const inventory = await scanCanonicalArtifactInventory(source);
|
|
const output = join(dirname(source), "extracted");
|
|
await materializeArtifactInventory(
|
|
{
|
|
inputPath: source,
|
|
inputFormat: "elf",
|
|
outputRoot: output,
|
|
environment: process.env,
|
|
integrityPolicy: "fail",
|
|
},
|
|
source,
|
|
inventory,
|
|
);
|
|
expect(await readFile(join(output, path))).toEqual(file.payload);
|
|
},
|
|
);
|
|
},
|
|
);
|
|
|
|
it("reserves possible case-annotation expansion along deep sibling paths", async () => {
|
|
const names = ["a/".repeat(4000) + "x", "A/".repeat(4000) + "X"];
|
|
await withPyInstallerReader(
|
|
cArchiveExecutable({
|
|
layout: "2.1",
|
|
pythonVersion: 309,
|
|
members: names.map((name) => ({
|
|
name,
|
|
typeCode: "x",
|
|
payload: Buffer.alloc(0),
|
|
compress: false,
|
|
})),
|
|
}),
|
|
async (reader) => {
|
|
const list = async () => {
|
|
for await (const entry of reader.entries()) void entry;
|
|
};
|
|
await expect(list()).rejects.toMatchObject({
|
|
reason: "limit",
|
|
boundary: "memory",
|
|
});
|
|
},
|
|
);
|
|
});
|