fix(artifacts): retain raw archive keys across entry projections

Publish Godot res:// declarations and PyInstaller CArchive/PYZ keys through
shared occurrence source_path while preserving safe logical extraction paths.
Consolidate PyInstaller file, directory and namespace projection through one
owner, and account for the additional retained source spelling before allocation.

Cover producer inventory, Windows separator conversion, PYZ module/data keys,
CLI extraction bytes and retained MCP inventory views. Format fixtures are
synthetic; real Godot/PyInstaller producer lanes were not run.
This commit is contained in:
morluto committed 2026-10-25 03:30:22 +08:00
1 parent 03ed0c63d9
commit 7fe5e5ff5e
11 files changed
+182 -53

No files matched your search

+3
View File
@@ -7,6 +7,9 @@ releases have not been verified. Format 1 is unsupported.
Inspection retains the original header and directory facts, including engine
version, flags, declared file count, source paths, offsets and declared MD5.
For a member declared as `res://scenes/main.gd`, the occurrence's
`source_path` retains that exact declaration while `logical_path` is
`scenes/main.gd`, the relative path used for lookup and extraction.
Observed plaintext bytes receive content-addressed SHA-256 identities. MD5 is a
container integrity claim, separate from the observed SHA-256. A mismatch fails
by default; `--integrity-policy record-and-continue` retains the observed bytes
+6 -2
View File
@@ -25,8 +25,12 @@ metadata retains the original name, table index, type, compression flag,
relative offset and known directory record extent. Logical paths follow the
carrier's path dialect: backslashes separate components for PE carriers, while
they remain literal filename characters for POSIX carriers. The source name
always retains its stored spelling. If the carrier dialect is unconfirmed, REA
preserves backslashes verbatim instead of inferring POSIX semantics.
always retains its stored spelling in occurrence `source_path` and
`format_metadata.source_name`. For PYZ rows, that source is the exact module or
data key in the PYZ table, not a reconstructed filesystem name;
`format_metadata.container_name` identifies the containing PYZ image. If the
carrier dialect is unconfirmed, REA preserves backslashes verbatim instead of
inferring POSIX semantics.
`relative_offset` is relative to the CArchive start for CArchive rows, or to
the decoded PYZ image for PYZ rows. A CArchive directory record starts at
`archive_start + toc_offset + directory_record_offset`; its size is
@@ -72,6 +72,9 @@ describe("GodotPckArtifactReader", () => {
"project.binary",
"data/é.json",
]);
expect(entries.map(({ sourcePath }) => sourcePath)).toEqual(
members.map(({ path }) => path),
);
expect(reader.formatMetadata()).toMatchObject({
embedded,
engine_major: 4,
+5 -3
View File
@@ -190,8 +190,9 @@ export class GodotPckArtifactReader implements ArtifactReader {
assertArtifactReadActive(signal);
const pathBytes = await cursor.uint32();
// Check worst-case JSON escaping before decoding or allocating the path.
// Its source bytes, decoded string and retained metadata share one budget.
if (pathBytes * 12 > MAX_DIRECTORY_METADATA_BYTES - retainedBytes)
// The logical path, raw source path, and metadata source path can each
// need six JSON bytes per input byte before retained facts are charged.
if (pathBytes * 18 > MAX_DIRECTORY_METADATA_BYTES - retainedBytes)
throw metadataLimit();
const rawPath = await cursor.bytes(pathBytes);
const offset = await cursor.uint64();
@@ -226,6 +227,7 @@ export class GodotPckArtifactReader implements ArtifactReader {
path: sourcePath.startsWith("res://")
? sourcePath.slice(6)
: sourcePath,
sourcePath,
kind: "file" as const,
declaredSize: Number(size),
compressedSize: null,
@@ -386,7 +388,7 @@ class PckCursor {
async bytes(length: number): Promise<Buffer> {
if (length > this.end - this.position)
throw formatFailure("directory entry exceeds the pack");
if (length > MAX_DIRECTORY_METADATA_BYTES / 12) throw metadataLimit();
if (length > MAX_DIRECTORY_METADATA_BYTES / 18) throw metadataLimit();
if (length > DIRECTORY_CHUNK_BYTES) {
// A large valid virtual path is bounded by retained representation, not
// the host filesystem's PATH_MAX. Only this path receives a larger read.
@@ -541,6 +541,7 @@ export class PyInstallerArtifactReader implements ArtifactReader {
this.#reserveMetadata(
{
path: entry.path,
source_path: entry.sourcePath ?? null,
adapter_key: entry.adapterKey,
format_metadata: entry.formatMetadata ?? null,
limitations: [...entry.limitations],
+12 -18
View File
@@ -66,7 +66,7 @@ export const projectStoredMember = (
): PyInstallerMember => {
if (isCArchiveDeclaration(entry))
return {
entry: fileEntry({
entry: projectEntry({
path: `__pyinstaller__/${entry.typeCode === "o" ? "runtime-options" : "dependencies"}/${String(entry.index)}`,
kind: "metadata",
declaredSize: entry.uncompressedSize,
@@ -93,7 +93,7 @@ export const projectStoredMember = (
if (bytecode.limitation !== undefined) limitations.push(bytecode.limitation);
const declaredSize = entry.uncompressedSize;
return {
entry: fileEntry({
entry: projectEntry({
path,
kind: symlink ? "symlink" : "file",
declaredSize,
@@ -126,7 +126,7 @@ export const projectPyzContainer = (
const limitations: string[] = [];
if (archive.kind === "opaque")
return {
entry: fileEntry({
entry: projectEntry({
path,
declaredSize: entry.uncompressedSize,
compressedSize: entry.compressedSize,
@@ -167,16 +167,12 @@ export const projectPyzContainer = (
},
};
return {
entry: {
entry: projectEntry({
path,
kind: "directory",
declaredSize: entry.uncompressedSize,
compressedSize: entry.compressedSize,
executable: false,
encrypted: false,
byteOffset,
declaredIntegrity: null,
unpacked: false,
limitations: [
...limitations,
`PyInstaller PYZ sha256 ${archive.sha256}.`,
@@ -191,7 +187,7 @@ export const projectPyzContainer = (
archive,
encrypted,
),
},
}),
read: undefined,
};
};
@@ -218,16 +214,13 @@ export const projectPyzMember = (
const encrypted = pyzEntryEncrypted(item, encryptionMarker);
if (item.namespace)
return {
entry: {
path: path,
entry: projectEntry({
path,
kind: "metadata",
declaredSize: 0,
compressedSize: item.length,
executable: false,
encrypted,
byteOffset: null,
declaredIntegrity: null,
unpacked: false,
limitations: [
"Namespace package has no importable bytecode; its declared payload is not materialized.",
...(encrypted
@@ -243,7 +236,7 @@ export const projectPyzMember = (
undefined,
encryptionMarker,
),
},
}),
read: undefined,
};
const header =
@@ -261,7 +254,7 @@ export const projectPyzMember = (
"PYZ tables declare compressed lengths; the uncompressed size is unknown until decoding.",
);
return {
entry: fileEntry({
entry: projectEntry({
path: path,
declaredSize: null,
compressedSize: item.length,
@@ -321,7 +314,7 @@ const pyzMemberPath = (
return `${container}/${dotted}.pyc`;
};
const fileEntry = (input: {
const projectEntry = (input: {
readonly path: string;
readonly declaredSize: number | null;
readonly compressedSize: number;
@@ -331,9 +324,10 @@ const fileEntry = (input: {
readonly formatMetadata: PyInstallerEntryMetadata;
readonly executable?: boolean;
readonly encrypted?: boolean;
readonly kind?: "file" | "symlink" | "metadata";
readonly kind?: "file" | "directory" | "symlink" | "metadata";
}): ArtifactEntry => ({
path: input.path,
sourcePath: input.formatMetadata.source_name,
kind: input.kind ?? "file",
declaredSize: input.declaredSize,
compressedSize: input.compressedSize,
@@ -218,8 +218,13 @@ it("inventories a 2.1 archive, nested PYZ modules, and derived bytecode", async
expect(
snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "PYZ.pyz",
)?.entry_kind,
).toBe("directory");
),
).toMatchObject({ entry_kind: "directory", source_path: "PYZ.pyz" });
expect(
snapshot.occurrences.find(
(occurrence) => occurrence.logical_path === "PYZ.pyz/ns",
),
).toMatchObject({ entry_kind: "metadata", source_path: "ns" });
const scriptEdge = snapshot.edges.find(
(edge) => edge.occurrence_id === hello?.occurrence_id,
);
@@ -254,6 +259,7 @@ it("decodes an interned Unicode module name from the PYZ table", async () => {
);
expect(module).toMatchObject({
entry_kind: "file",
source_path: "café",
artifact_id: expect.any(String),
format_metadata: {
kind: "pyinstaller-entry",
@@ -927,6 +933,8 @@ it("inventories and extracts a Windows member name under its canonical path", as
);
expect(member?.format_metadata).toMatchObject({ source_name: storedName });
expect(resource?.format_metadata).toMatchObject({ source_name: pyzName });
expect(member?.source_path).toBe(storedName);
expect(resource?.source_path).toBe(pyzName);
const directory = await createTestTempDirectory("rea-pyinstaller-out-");
const output = join(directory, "extracted");
await materializeArtifactInventory(
+1
View File
@@ -67,6 +67,7 @@ cliTest.for([2, 3, 4] as const)(
source_path: member.path,
flags: 0,
});
expect(row?.source_path).toBe(member.path);
expect(row?.hash_status).toBe("verified");
}
const extracted = await cli.run({
@@ -17,6 +17,7 @@ import { artifactInspectionResultSchema } from "../../src/domain/artifactInspect
import { artifactExtractionResultSchema } from "../../src/domain/artifactGraph.js";
import { parseEvidence } from "../../src/domain/evidence.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
import { cliTest } from "../support/cli/cliFixture.js";
const execute = promisify(execFile);
const cli = fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url));
@@ -192,6 +193,7 @@ it("lists CArchive members and the Python version through inspect-artifact", asy
),
).toMatchObject({
artifact_id: null,
source_path: "pyi-contents-directory _internal",
format_metadata: {
source_name: "pyi-contents-directory _internal",
type_code: "o",
@@ -217,6 +219,92 @@ it("lists CArchive members and the Python version through inspect-artifact", asy
});
});
cliTest(
"preserves Windows PyInstaller source names",
async ({ cli: driver }) => {
const directory = await createTestTempDirectory(
"rea-pyinstaller-windows-names-",
);
const path = join(directory, "windows.exe");
const storedName = String.raw`pkg\data.bin`;
const dataKey = String.raw`res\icon.bin`;
const stored = Buffer.from("stored bytes");
const icon = Buffer.from("icon bytes");
const module = Buffer.from("module bytes");
await writeFile(
path,
cArchiveExecutable({
carrier: "pe",
layout: "2.1",
pythonVersion: 309,
members: [
{ name: storedName, typeCode: "x", payload: stored, compress: false },
{
name: "PYZ.pyz",
typeCode: "z",
payload: pyzArchive({
magic: PYTHON_39_MAGIC,
modules: [
{ name: dataKey, typeCode: 2, body: icon },
{ name: "pkg.mod", typeCode: 0, body: module },
],
}),
compress: false,
},
],
}),
);
const inspected = await driver.run({
arguments: ["inspect-artifact", path, "--json"],
});
expect(inspected.exitCode, inspected.stdout + inspected.stderr).toBe(0);
const inspection = artifactInspectionResultSchema.parse(
parseEvidence(inspected.json).normalized_result,
);
const rows = inspection.substeps[0]?.evidence.normalized_result.occurrences;
expect(
rows?.find((row) => row.logical_path === "pkg/data.bin"),
).toMatchObject({
source_path: storedName,
format_metadata: { source_name: storedName },
});
expect(
rows?.find((row) => row.logical_path === "PYZ.pyz/res/icon.bin"),
).toMatchObject({
source_path: dataKey,
format_metadata: { container_name: "PYZ.pyz", source_name: dataKey },
});
expect(
rows?.find((row) => row.logical_path === "PYZ.pyz/pkg/mod.pyc"),
).toMatchObject({
source_path: "pkg.mod",
format_metadata: { container_name: "PYZ.pyz", source_name: "pkg.mod" },
});
const extracted = await driver.run({
arguments: ["extract-artifact", path, "--json"],
});
expect(extracted.exitCode, extracted.stdout + extracted.stderr).toBe(0);
const result = artifactExtractionResultSchema.parse(
parseEvidence(extracted.json).normalized_result,
);
try {
expect(
await readFile(join(result.output_root, "pkg", "data.bin")),
).toEqual(stored);
expect(
await readFile(join(result.output_root, "PYZ.pyz", "res", "icon.bin")),
).toEqual(icon);
expect(
(
await readFile(join(result.output_root, "PYZ.pyz", "pkg", "mod.pyc"))
).subarray(16),
).toEqual(module);
} finally {
await rm(result.output_root, { recursive: true, force: true });
}
},
);
it("extracts legacy and modern CArchive modules with one pyc header through the CLI", async () => {
const fixture = cArchiveBytecodeExecutable();
const directory = await createTestTempDirectory("rea-pyinstaller-bytecode-");
+41 -28
View File
@@ -38,6 +38,37 @@ const viewValidators = async (client: Client) => {
return { validateInput, validateOutput };
};
const retainedInventoryView =
(
client: Client,
evidenceId: string,
validators: Awaited<ReturnType<typeof viewValidators>>,
) =>
async (query: Record<string, unknown>) => {
const arguments_ = {
source: { kind: "retained-evidence", evidence_id: evidenceId },
view: { kind: "inventory", ...query },
};
expect(
validators.validateInput(arguments_),
JSON.stringify(validators.validateInput.errors),
).toBe(true);
const reply = await client.callTool({
name: "inspect_analysis_view",
arguments: arguments_,
});
expect(reply.isError, JSON.stringify(reply.structuredContent)).not.toBe(
true,
);
expect(
validators.validateOutput(reply.structuredContent),
JSON.stringify(validators.validateOutput.errors),
).toBe(true);
const evidence = parseEvidence(reply.structuredContent);
expect(evidence.evidence_links).toContain(evidenceId);
return analysisViewResultSchema.parse(evidence.normalized_result);
};
const assertInvalidNodeField = async (
client: Client,
evidenceId: string,
@@ -82,7 +113,7 @@ it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => {
try {
await server.connect(serverTransport);
await client.connect(clientTransport);
const { validateInput, validateOutput } = await viewValidators(client);
const validators = await viewValidators(client);
expect(
(await client.callTool({ name: "open_binary", arguments: { path } }))
.isError,
@@ -116,41 +147,22 @@ it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => {
session.evidenceForAnalysis(nested.evidence_id)?.normalized_result,
).toBe(nested.normalized_result);
const graph = nested.normalized_result;
const source = {
kind: "retained-evidence",
evidence_id: parent.evidence_id,
};
const view = async (query: Record<string, unknown>) => {
const arguments_ = { source, view: { kind: "inventory", ...query } };
expect(
validateInput(arguments_),
JSON.stringify(validateInput.errors),
).toBe(true);
const reply = await client.callTool({
name: "inspect_analysis_view",
arguments: arguments_,
});
expect(reply.isError, JSON.stringify(reply.structuredContent)).not.toBe(
true,
);
expect(
validateOutput(reply.structuredContent),
JSON.stringify(validateOutput.errors),
).toBe(true);
const evidence = parseEvidence(reply.structuredContent);
expect(evidence.evidence_links).toContain(parent.evidence_id);
return analysisViewResultSchema.parse(evidence.normalized_result);
};
const view = retainedInventoryView(client, parent.evidence_id, validators);
await assertInvalidNodeField(
client,
parent.evidence_id,
profile === "full" ? validateInput : undefined,
profile === "full" ? validators.validateInput : undefined,
);
const wanted = graph.occurrences.filter((row) => row.logical_path !== ".");
const occurrences = await view({
collection: "occurrences",
logical_paths: wanted.map((row) => row.logical_path),
fields: ["logical_path", "declared_integrity", "format_metadata"],
fields: [
"logical_path",
"source_path",
"declared_integrity",
"format_metadata",
],
});
expect(occurrences.kind).toBe("inventory");
if (occurrences.kind !== "inventory")
@@ -161,6 +173,7 @@ it.each(profiles)("reuses retained PCK inventory (%s)", async (profile) => {
artifact_id: row.artifact_id,
parent_occurrence_id: row.parent_occurrence_id,
logical_path: row.logical_path,
source_path: row.source_path,
declared_integrity: row.declared_integrity,
format_metadata: row.format_metadata,
})),
@@ -245,6 +245,18 @@ it.each([false, true])(
...(opaque ? [] : ["PYZ.pyz/app.pyc"]),
]),
);
if (!opaque)
expect(
inspection.substeps[0]?.evidence.normalized_result.occurrences.find(
(row) => row.logical_path === "PYZ.pyz/app.pyc",
),
).toMatchObject({
source_path: "app",
format_metadata: {
container_name: "PYZ.pyz",
source_name: "app",
},
});
const parent = parseEvidence(inspected.structuredContent);
const compact = await client.callTool({
name: "inspect_artifact",