Files
reverse-engineering-anything/src/artifacts/pyinstaller/CArchiveLayout.ts
T
morluto 5bcf71c660 fix(pyinstaller): retain colliding archive observations
Keep source-distinct CArchive rows and decoded PYZ children in inventory. Refuse only when extraction reaches a destination conflict.

Fixes #6674
2026-10-23 21:28:10 +08:00

380 lines
11 KiB
TypeScript

import type { FileHandle } from "node:fs/promises";
import { readPyInstallerBytes } from "./PyInstallerBytes.js";
import { err, ok, type Result } from "../../domain/result.js";
import {
pyInstallerFailure,
type PyInstallerFailure,
} from "./PyInstallerFailure.js";
/** Magic at the start of a CArchive cookie. */
export const PYINSTALLER_COOKIE_MAGIC = Buffer.from("MEI\x0c\x0b\x0a\x0b\x0e");
/** PyInstaller 2.1+ cookie size, including the 64-byte library name. */
export const PYINSTALLER_COOKIE_V21_BYTES = 88;
/** PyInstaller 2.0 cookie size, without the library name. */
export const PYINSTALLER_COOKIE_V20_BYTES = 24;
/**
* How far before EOF the cookie may sit. macOS code signatures are appended
* after the cookie, and a large binary's signature hashes are larger than the
* bootloader's own 8 KiB scan chunk, so detection uses one bounded tail.
*/
export const PYINSTALLER_COOKIE_SEARCH_BYTES = 16 * 1024 * 1024;
const MAX_TOC_BYTES = 32 * 1024 * 1024;
const MAX_TOC_ENTRIES = 100_000;
const TOC_HEADER_BYTES = 18;
/** Observed CArchive cookie. Offsets are relative to the archive start. */
export interface PyInstallerCookie {
readonly layout: "2.0" | "2.1";
readonly archiveStart: number;
readonly packageLength: number;
readonly tocOffset: number;
readonly tocLength: number;
readonly pythonVersion: number;
readonly pythonLibrary: string | null;
readonly cookieOffset: number;
}
/** One CArchive table entry. */
export interface CArchiveEntry {
readonly index: number;
readonly tocRecordOffset: number;
readonly tocRecordLength: number;
readonly name: string;
readonly typeCode: string;
readonly offset: number;
readonly compressedSize: number;
readonly uncompressedSize: number;
readonly compressionFlag: number;
}
/** Runtime options and external dependencies are declarations, never local blobs. */
export const isCArchiveDeclaration = (
entry: CArchiveEntry,
): entry is CArchiveEntry & { readonly typeCode: "o" | "d" } =>
entry.typeCode === "o" || entry.typeCode === "d";
/** True when a native executable has a structurally valid CArchive cookie. */
export const containsPyInstallerArchive = async (
handle: FileHandle,
fileSize: number,
signal?: AbortSignal,
): Promise<boolean> => {
const found = await findPyInstallerCookie(handle, fileSize, signal);
return found?.ok === true;
};
/** Locate a structurally valid cookie. Invalid magic copies are skipped. */
export const findPyInstallerCookie = async (
handle: FileHandle,
fileSize: number,
signal?: AbortSignal,
): Promise<Result<PyInstallerCookie, PyInstallerFailure> | undefined> => {
if (
!Number.isSafeInteger(fileSize) ||
fileSize < PYINSTALLER_COOKIE_V20_BYTES
)
return undefined;
const tailSize = Math.min(fileSize, PYINSTALLER_COOKIE_SEARCH_BYTES);
const read = await readPyInstallerBytes(handle, {
position: fileSize - tailSize,
length: tailSize,
label: "CArchive tail",
signal,
});
if (!read.ok) return read;
const observed = read.value;
let failure: PyInstallerFailure | undefined;
let from = observed.length;
while (from >= PYINSTALLER_COOKIE_MAGIC.length) {
const relative = observed
.subarray(0, from)
.lastIndexOf(PYINSTALLER_COOKIE_MAGIC);
if (relative < 0) break;
const cookieOffset = fileSize - tailSize + relative;
const parsed = parseCookieAt(observed, relative, cookieOffset, fileSize);
if (parsed.ok) return parsed;
failure = parsed.error;
from = relative;
}
return failure === undefined ? undefined : err(failure);
};
/** Read and parse the TOC that a validated cookie names. */
export const readCArchiveToc = async (
handle: FileHandle,
cookie: PyInstallerCookie,
signal?: AbortSignal,
): Promise<Result<readonly CArchiveEntry[], PyInstallerFailure>> => {
if (cookie.tocLength > MAX_TOC_BYTES)
return err(
pyInstallerFailure(
"limit",
"CArchive table of contents exceeds its bound",
),
);
const position = cookie.archiveStart + cookie.tocOffset;
const read = await readPyInstallerBytes(handle, {
position,
length: cookie.tocLength,
label: "CArchive table of contents",
signal,
});
if (!read.ok) return read;
const parsed = parseCArchiveToc(read.value);
if (!parsed.ok) return parsed;
const valid = validateCArchiveMembers(parsed.value, cookie.tocOffset);
return valid.ok ? parsed : valid;
};
/** Parse TOC bytes. Each entry is a big-endian header plus a NUL-padded name. */
export const parseCArchiveToc = (
bytes: Buffer,
): Result<readonly CArchiveEntry[], PyInstallerFailure> => {
const entries: CArchiveEntry[] = [];
let offset = 0;
while (offset < bytes.length) {
if (entries.length >= MAX_TOC_ENTRIES)
return err(
pyInstallerFailure(
"limit",
"CArchive table of contents exceeds its entry bound",
),
);
if (offset + TOC_HEADER_BYTES > bytes.length)
return err(
pyInstallerFailure(
"format",
"CArchive table of contents entry is truncated",
),
);
const entryLength = bytes.readUInt32BE(offset);
if (entryLength < TOC_HEADER_BYTES || offset + entryLength > bytes.length)
return err(
pyInstallerFailure(
"format",
"CArchive table of contents entry extends outside the table",
),
);
const nameBytes = bytes.subarray(
offset + TOC_HEADER_BYTES,
offset + entryLength,
);
const name = tocName(nameBytes);
if (!name.ok) return name;
const compressionFlag = bytes[offset + 16] ?? 0;
if (compressionFlag !== 0 && compressionFlag !== 1)
return err(
pyInstallerFailure(
"format",
`CArchive compression flag ${String(compressionFlag)} is not stored or zlib`,
),
);
const typeCode = bytes[offset + 17] ?? 0;
if (typeCode < 0x20 || typeCode > 0x7e)
return err(
pyInstallerFailure(
"format",
"CArchive type code is not a printable character",
),
);
entries.push({
index: entries.length,
tocRecordOffset: offset,
tocRecordLength: entryLength,
name: name.value,
typeCode: String.fromCharCode(typeCode),
offset: bytes.readUInt32BE(offset + 4),
compressedSize: bytes.readUInt32BE(offset + 8),
uncompressedSize: bytes.readUInt32BE(offset + 12),
compressionFlag,
});
offset += entryLength;
}
return ok(entries);
};
const parseCookieAt = (
tail: Buffer,
relative: number,
cookieOffset: number,
fileSize: number,
): Result<PyInstallerCookie, PyInstallerFailure> => {
const modern = parseCookie(tail, relative, cookieOffset, fileSize, "2.1");
if (modern.ok) return modern;
const legacy = parseCookie(tail, relative, cookieOffset, fileSize, "2.0");
return legacy.ok ? legacy : modern;
};
const parseCookie = (
tail: Buffer,
relative: number,
cookieOffset: number,
fileSize: number,
layout: "2.0" | "2.1",
): Result<PyInstallerCookie, PyInstallerFailure> => {
const cookieBytes =
layout === "2.1"
? PYINSTALLER_COOKIE_V21_BYTES
: PYINSTALLER_COOKIE_V20_BYTES;
if (relative + cookieBytes > tail.length)
return err(
pyInstallerFailure(
"format",
"CArchive cookie extends past the end of the file",
),
);
const view = tail.subarray(relative, relative + cookieBytes);
const packageLength = view.readUInt32BE(8);
const tocOffset = view.readUInt32BE(12);
const tocLength = view.readUInt32BE(16);
const pythonVersion = view.readUInt32BE(20);
if (
!safeSum(tocOffset, tocLength, cookieBytes) ||
packageLength !== tocOffset + tocLength + cookieBytes
)
return err(
pyInstallerFailure(
"format",
"CArchive cookie does not describe a table of contents inside the package",
),
);
const archiveStart = cookieOffset + cookieBytes - packageLength;
if (archiveStart < 0 || archiveStart + packageLength > fileSize)
return err(
pyInstallerFailure("format", "CArchive cookie range is outside the file"),
);
const library =
layout === "2.1" ? libraryName(view.subarray(24, 88)) : ok(null);
if (!library.ok) return library;
return ok({
layout,
archiveStart,
packageLength,
tocOffset,
tocLength,
pythonVersion,
pythonLibrary: library.value,
cookieOffset,
});
};
const libraryName = (
bytes: Buffer,
): Result<string | null, PyInstallerFailure> => {
const end = bytes.indexOf(0);
const text = end < 0 ? bytes : bytes.subarray(0, end);
if (end >= 0 && bytes.subarray(end).some((byte) => byte !== 0))
return err(
pyInstallerFailure(
"format",
"CArchive Python library name is not NUL-padded",
),
);
if (text.length === 0) return ok(null);
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
);
} catch {
return err(
pyInstallerFailure(
"format",
"CArchive Python library name is not valid UTF-8",
),
);
}
};
const tocName = (bytes: Buffer): Result<string, PyInstallerFailure> => {
const end = bytes.indexOf(0);
const text = end < 0 ? bytes : bytes.subarray(0, end);
if (text.length === 0)
return err(pyInstallerFailure("format", "CArchive entry name is empty"));
try {
return ok(
new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(text),
);
} catch {
return err(
pyInstallerFailure("format", "CArchive entry name is not valid UTF-8"),
);
}
};
const safeSum = (left: number, middle: number, right: number): boolean =>
Number.isSafeInteger(left) &&
Number.isSafeInteger(middle) &&
Number.isSafeInteger(right) &&
left >= 0 &&
middle >= 0 &&
right >= 0 &&
left + middle + right <= Number.MAX_SAFE_INTEGER;
/** Source directory semantics are validated once before any consumer projects paths. */
const validateCArchiveMembers = (
entries: readonly CArchiveEntry[],
tocOffset: number,
): Result<null, PyInstallerFailure> => {
for (const entry of entries) {
if (isCArchiveDeclaration(entry)) {
if (
entry.compressedSize !== 0 ||
entry.uncompressedSize !== 0 ||
entry.compressionFlag !== 0
)
return err(
pyInstallerFailure(
"integrity",
`CArchive declaration at table index ${String(entry.index)} has local payload or compression`,
),
);
}
if (
entry.compressionFlag === 0 &&
entry.compressedSize !== entry.uncompressedSize
)
return err(
pyInstallerFailure(
"integrity",
`Stored CArchive member size disagrees with its table entry at table index ${String(entry.index)}`,
),
);
if (entry.offset > tocOffset)
return err(
pyInstallerFailure(
"format",
`CArchive member offset is outside the archive at table index ${String(entry.index)}`,
),
);
}
const ranges = entries
.filter((entry) => entry.compressedSize > 0)
.map((entry) => ({
index: entry.index,
start: entry.offset,
end: entry.offset + entry.compressedSize,
}))
.sort((left, right) => left.start - right.start);
let occupied = 0;
for (const range of ranges) {
if (
!Number.isSafeInteger(range.end) ||
range.end > tocOffset ||
range.start < occupied
)
return err(
pyInstallerFailure(
"integrity",
`CArchive member extends outside the archive or overlaps another member at table index ${String(range.index)}`,
),
);
occupied = range.end;
}
return ok(null);
};