From bd4f5e9c197ba942ed026c9fed6ff3fae663901d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:54:28 +0800 Subject: [PATCH] fix(parsers): preserve format evidence and caller input identity Retain bounded FAT and offset-tagged PE observations through archive hashing, classification, and content nodes. Preserve PyInstaller member paths according to the observed carrier dialect and share APK v1 signing-name semantics. Read SQLite's implicit sqlite_schema table without inventing a stored schema declaration. Distinguish unsupported ELF numbering and decoded-name admission from malformed data and fixed scan allowances. Preserve Sigrok basenames and classify matching explicit input-format refusals at their owning boundary. Validation: 554 distinct tests passed, 3 skipped across serial focused runs; build, typecheck/lint, docs, formatting, and discovery (2141 files) passed. Sigrok command-boundary simulators do not establish real-tool decoding. Small repeated public PyInstaller CLI runs had equivalent inventory facts; warm median elapsed 0.5431s before / 0.5426s after, with no speed claim. Fixes #6733 Fixes #6741 Fixes #6722 Fixes #6732 Fixes #6736 Fixes #6734 Fixes #6764 Fixes #6766 Refs #6739 --- docs/android-analysis.md | 4 + docs/elf-notes.md | 8 +- docs/python-packaged-applications.md | 7 +- docs/signal-recordings.md | 12 +- docs/sqlite-databases.md | 7 + src/application/BinaryTargetResolver.ts | 25 +- .../sqlite/SqliteDatabaseService.test.ts | 84 +++++++ .../sqlite/SqliteDatabaseService.ts | 25 +- src/artifacts/ArtifactClassificationProbe.ts | 66 ++++++ src/artifacts/PyInstallerArtifactReader.ts | 65 +++++- .../ArtifactGraphConstruction.test.ts | 37 +++ .../inventory/ArtifactGraphConstruction.ts | 65 +++--- src/artifacts/inventory/classify.ts | 82 ++++++- src/artifacts/inventory/scanReader.ts | 8 +- src/artifacts/pyinstaller/CArchive.fixture.ts | 9 +- src/artifacts/pyinstaller/CArchiveMembers.ts | 32 ++- .../PyInstallerArtifactReader.test.ts | 89 +++++++- .../PyInstallerFormatBoundaries.test.ts | 1 + src/domain/android/androidApplication.ts | 5 +- src/domain/apkSignature/apkV1MetadataName.ts | 35 +++ src/domain/apkSignature/apkV1Signature.ts | 58 ++--- src/domain/binaryTarget.ts | 9 +- src/domain/dosMz.test.ts | 9 +- src/domain/dosMz.ts | 19 +- src/domain/sqlite/sqliteDatabase.ts | 5 + src/ghidra/GhidraDosSnapshot.ts | 12 +- src/ghidra/GhidraLoadImageValues.ts | 8 +- src/native/elf/ElfNoteParser.test.ts | 63 ++++- src/native/elf/ElfNoteParser.ts | 28 ++- src/native/elf/ElfNotes.fixture.ts | 40 ++++ src/native/elf/ElfNotesProvider.test.ts | 60 +++++ src/native/elf/ElfNotesProvider.ts | 48 +++- src/signals/SigrokRecordingProvider.ts | 9 +- src/signals/sigrokSelectorRejection.ts | 17 +- src/sqlite/SqliteDatabaseInspection.ts | 20 ++ .../androidRuntimeComponentHints.test.ts | 76 +++++++ tests/acceptance/elfNotesCli.test.ts | 22 +- tests/acceptance/pyinstallerArchive.test.ts | 44 ++++ tests/acceptance/sqliteDatabaseCli.test.ts | 62 +++++ .../filesystem/android/application.test.ts | 54 +++++ .../filesystem/artifactFatContentNode.test.ts | 79 +++++++ .../filesystem/artifactMzInventory.test.ts | 30 ++- tests/boundary/mcp/elfNotesMcp.test.ts | 34 ++- .../signals/recordingProvider.test.ts | 215 ++++++++++++++---- .../sqlite/sqliteDatabaseInspection.test.ts | 31 +++ tests/fixtures/artifactClassification.ts | 3 +- tests/fixtures/godotPck.ts | 11 +- 47 files changed, 1523 insertions(+), 209 deletions(-) create mode 100644 src/domain/apkSignature/apkV1MetadataName.ts create mode 100644 tests/boundary/filesystem/artifactFatContentNode.test.ts diff --git a/docs/android-analysis.md b/docs/android-analysis.md index daa588ab72..ced3cbe34f 100644 --- a/docs/android-analysis.md +++ b/docs/android-analysis.md @@ -256,6 +256,10 @@ such as `libreactnativejni.so`, `libreactnative.so`, `libhermes.so`, `libhermes_executor.so`, `libflutter.so`, and `libunity.so`; unrelated names such as `libcommunity.so` and `libreactivity.so` do not identify those families. +The signing list retains the `META-INF/MANIFEST.MF`, `.SF`, `.RSA`, `.DSA`, and +`.EC` paths selected by Android's v1 verifier, including nested paths. These are +inventory metadata candidates, not proof that a v1 signature verifies or that +the APK is signed; use `inspect-apk-signature` for signature analysis. The bridge basis is inferred from the native path and repeated for each managed component, so it does not identify a specific JNI binding. Bridge hypotheses form a DEX/JVM-class by native-library Cartesian product. The projection computes the product and exact serialized diff --git a/docs/elf-notes.md b/docs/elf-notes.md index 9d3ab30f18..6a464fce14 100644 --- a/docs/elf-notes.md +++ b/docs/elf-notes.md @@ -46,13 +46,17 @@ The selected file is limited to 64 MiB by default (512 MiB maximum). At most names, section names, and GNU descriptor text share a scan budget based on the source bytes plus any decoded section-name table, capped at `max_file_bytes`. The raw source and decoded table must also fit together -within that ceiling; repeated table ranges count again. Each descriptor range is hashed once, +within that ceiling. If admission fails, the error reports the required +`max_file_bytes`; raising it can admit the decoded table up to the operation's +512 MiB maximum. Once admitted, raising that setting does not increase the +scan budget for the same source and decoded names. Repeated table ranges count again. Each descriptor range is hashed once, and each GNU version's trailing NUL run is scanned once; neither walk may exceed that byte ceiling. Malformed ELF headers, truncated header tables, declared note tables that extend past the file, and truncated note entries fail without partial success. ELF extended numbering (`PN_XNUM`, including a section count stored when `e_shnum` is 0) is -unsupported. +unsupported and returns an unsupported-target outcome rather than invalid-input +recovery. This workflow does not execute the image or verify GNU build-id uniqueness against package catalogs. diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md index cde4eb98c1..ac3f394174 100644 --- a/docs/python-packaged-applications.md +++ b/docs/python-packaged-applications.md @@ -12,8 +12,11 @@ The root occurrence's `format_metadata` records the CArchive layout (2.0 or 2.1), archive and cookie offsets, table extent, entry count, raw Python version, normalized major/minor when known, library name and encryption marker. Member metadata retains the original name, table index, type, compression flag, -relative offset and known directory record extent. Logical paths use archive -path rules; the source name retains its spelling, including backslashes. +relative offset and known directory record extent. Logical paths follow the +carrier's path dialect: backslashes separate components for PE carriers, while +they remain literal filename characters for POSIX carriers. The source name +always retains its stored spelling. If the carrier dialect is unconfirmed, REA +preserves backslashes verbatim instead of inferring POSIX semantics. `relative_offset` is relative to the CArchive start for CArchive rows, or to the decoded PYZ image for PYZ rows. A CArchive directory record starts at `archive_start + toc_offset + directory_record_offset`; its size is diff --git a/docs/signal-recordings.md b/docs/signal-recordings.md index b9fce01d47..e89745b7fd 100644 --- a/docs/signal-recordings.md +++ b/docs/signal-recordings.md @@ -38,12 +38,16 @@ missing or unknown selected channel target, REA returns `invalid_request` for successfully. The diagnostic and any partial stdout remain in `captured_output`; unrelated producer warnings do not change request status. Omitting `input_format` requests sigrok's autodetection. REA uses an immutable -private copy whose filename retains the selected recording's final suffix, so -filename-based format detection receives that hint without exposing the +private copy whose filename retains the selected recording's final suffix, +including a dotfile basename such as `.csv`. Filename-based format detection +receives that hint without exposing the caller's directory to sigrok. Omitting `annotations` retains every annotation class. Both omitted selections are recorded as null. -See the [upstream manual](https://sigrok.org/wiki/Sigrok-cli) for formats, -decoder stacking and annotation selectors. No arbitrary command flags are accepted. +If sigrok reports an explicitly selected input format as unknown, REA returns +`invalid_request` for `input_format`, retains the producer diagnostic, and +advises correcting the format or omitting it for autodetection. See the +[upstream manual](https://sigrok.org/wiki/Sigrok-cli) for formats, decoder +stacking and annotation selectors. No arbitrary command flags are accepted. Output defaults to 50 literal LF-separated lines. Use `line_offset` and `line_limit` for later pages, or MCP `line_limit: null` / CLI `--all_lines` diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md index a6b67b0aa9..157ccf3007 100644 --- a/docs/sqlite-databases.md +++ b/docs/sqlite-databases.md @@ -54,6 +54,13 @@ reported. If a virtual table cannot be resolved, possible backing tables have ordinary tables remain selectable. Schema inventory does not execute views, triggers or arbitrary SQL. +The exact table name `sqlite_schema` is also a supported read-only row source. +SQLite's [`table_list`](https://www.sqlite.org/pragma.html#pragma_table_list) +and `table_xinfo` pragmas establish its row-source type and columns. The +[schema catalog](https://www.sqlite.org/schematab.html) has no self-entry for +`sqlite_schema`, so REA does not invent a declaration, SQL statement or root +page for it in `schema.tables`. + Rows are optional. `table` selects one exact ordinary-table name. `columns` (CLI repeated `--column`) selects exact visible column names in the requested order; omission selects all visible columns, including generated columns. diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index 907b0098f8..c1c361715e 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -574,7 +574,9 @@ const readExecutableMetadata = async ( throwIfTargetResolutionCancelled(signal); if (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) { let evidence = parseMzHeaderEvidence(bytes, fileSize); - let signatureBytes: Buffer | undefined; + let signatureEvidence: + | { readonly offset: number; readonly bytes: Buffer } + | undefined; if (evidence.windowsHeader.state === "signature-needed") { const { offset } = evidence.windowsHeader; const signature = Buffer.alloc(Math.min(4, fileSize - offset)); @@ -585,8 +587,11 @@ const readExecutableMetadata = async ( offset, ); throwIfTargetResolutionCancelled(signal); - signatureBytes = signature.subarray(0, observedSignature.bytesRead); - evidence = parseMzHeaderEvidence(bytes, fileSize, signatureBytes); + signatureEvidence = { + offset, + bytes: signature.subarray(0, observedSignature.bytesRead), + }; + evidence = parseMzHeaderEvidence(bytes, fileSize, signatureEvidence); } const { windowsHeader } = evidence; if (windowsHeader.state === "declared") { @@ -618,7 +623,7 @@ const readExecutableMetadata = async ( evidence = parseMzHeaderEvidence( table.subarray(0, observed.bytesRead), fileSize, - signatureBytes, + signatureEvidence, ); if (evidence.windowsHeader.state === "candidate") return err( @@ -628,6 +633,7 @@ const readExecutableMetadata = async ( table.subarray(0, observed.bytesRead), hostArchitecture, fileSize, + signatureEvidence, ); } if (windowsHeader.state === "candidate") @@ -636,7 +642,7 @@ const readExecutableMetadata = async ( bytes, hostArchitecture, fileSize, - signatureBytes, + signatureEvidence, ); } const fatEvidence = await readFatSliceHeaderEvidence( @@ -797,11 +803,10 @@ const readPeMetadata = async ( throwIfTargetResolutionCancelled(signal); if (fileHeaderRead.bytesRead !== fileHeader.length) return err("invalid or truncated PE header"); - const windowsHeader = observeMzWindowsHeader( - mzPrefix, - fileSize, - fileHeader.subarray(0, 4), - ); + const windowsHeader = observeMzWindowsHeader(mzPrefix, fileSize, { + offset, + bytes: fileHeader.subarray(0, 4), + }); if (windowsHeader.state !== "declared") return err("invalid Windows new-header signature in MZ image"); if (["NE", "LE", "LX"].includes(windowsHeader.signature)) diff --git a/src/application/sqlite/SqliteDatabaseService.test.ts b/src/application/sqlite/SqliteDatabaseService.test.ts index 7732cacaeb..c8792bab65 100644 --- a/src/application/sqlite/SqliteDatabaseService.test.ts +++ b/src/application/sqlite/SqliteDatabaseService.test.ts @@ -122,6 +122,90 @@ it("retains the actual default without inventing an explicit caller selection", expect(result.value.parameters).not.toHaveProperty("worker_timeout_ms"); }); +it("accepts sqlite_schema rows without requiring a fabricated declaration", async () => { + const report = { + ...fixture(), + schema: { + ...fixture().schema, + scope: "selected_table", + tables: [], + }, + rows: { + table: "sqlite_schema", + columns: ["name", "type"], + values: [ + [ + { + type: "text", + value: "records", + bytes_hex: "7265636f726473", + encoding: "UTF-8", + }, + { + type: "text", + value: "table", + bytes_hex: "7461626c65", + encoding: "UTF-8", + }, + ], + ], + row_limit: 20, + returned_rows: 1, + truncated: false, + order: "unspecified", + }, + }; + const service = new SqliteDatabaseService({ + inspect: async () => ok(execution(report)), + }); + const result = await service.inspect({ + path: "/selected.db", + table: "sqlite_schema", + columns: ["name", "type"], + schema_scope: "selected_table", + }); + expect(result.ok).toBe(true); + + const foreign = { + ...report, + schema: { + ...report.schema, + tables: [ + { + name: "records", + root_page: 2, + kind: "table", + without_rowid: false, + strict: false, + column_count: 0, + columns_completeness: "complete", + columns: [], + }, + ], + }, + }; + const foreignService = new SqliteDatabaseService({ + inspect: async () => ok(execution(foreign)), + }); + expect( + await foreignService.inspect({ + path: "/selected.db", + table: "sqlite_schema", + columns: ["name", "type"], + schema_scope: "selected_table", + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + + expect( + await service.inspect({ + path: "/selected.db", + table: "missing", + columns: ["name", "type"], + schema_scope: "selected_table", + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); +}); + it("parses historical report omissions as unknown without supplying a default", () => { const { worker_timeout_ms: _default, ...historical } = fixture(); const parsed = sqliteDatabaseSchema.parse(historical); diff --git a/src/application/sqlite/SqliteDatabaseService.ts b/src/application/sqlite/SqliteDatabaseService.ts index 024a92c946..187f008d1a 100644 --- a/src/application/sqlite/SqliteDatabaseService.ts +++ b/src/application/sqlite/SqliteDatabaseService.ts @@ -17,6 +17,7 @@ import { type SqliteDatabase, SQLITE_ROW_LIMIT_DEFAULT, DEFAULT_SQLITE_WORKER_TIMEOUT_MS, + sqliteSchemaHasObjectDeclaration, sqliteSchemaScope, sqliteInspectionParameters, } from "../../domain/sqlite/sqliteDatabase.js"; @@ -26,6 +27,21 @@ import { jsonValueSchema } from "../../domain/jsonValue.js"; const OPERATION = "inspect_sqlite_database"; +const matchesSelectedSchema = ( + schema: SqliteDatabase["schema"], + table: string, +): boolean => + sqliteSchemaHasObjectDeclaration(table) + ? schema.tables.length === 1 && + schema.tables[0]?.name === table && + schema.indexes.every((index) => index.table === table) && + schema.triggers.every((trigger) => trigger.table === table) && + schema.views.length === 0 + : schema.tables.length === 0 && + schema.indexes.length === 0 && + schema.views.length === 0 && + schema.triggers.length === 0; + const matchesSelectedDatabase = ( report: SqliteDatabase, input: InspectSqliteDatabaseInput, @@ -41,13 +57,8 @@ const matchesSelectedDatabase = ( (report.wal === null || report.wal.path === `${input.path}-wal`) && report.schema.scope === sqliteSchemaScope(input) && (report.schema.scope !== "selected_table" || - (report.schema.tables.length === 1 && - report.schema.tables[0]?.name === input.table && - report.schema.indexes.every((index) => index.table === input.table) && - report.schema.triggers.every( - (trigger) => trigger.table === input.table, - ) && - report.schema.views.length === 0)) && + (input.table !== undefined && + matchesSelectedSchema(report.schema, input.table))) && (input.table === undefined ? report.rows === null : report.rows?.table === input.table && diff --git a/src/artifacts/ArtifactClassificationProbe.ts b/src/artifacts/ArtifactClassificationProbe.ts index d6e4e9c788..65626988e7 100644 --- a/src/artifacts/ArtifactClassificationProbe.ts +++ b/src/artifacts/ArtifactClassificationProbe.ts @@ -15,6 +15,10 @@ import { type MachoHeaderPrefixFacts, type FatSliceHeaderEvidence, } from "../domain/apple/machoContainer.js"; +import { + observeMzWindowsHeader, + type MzWindowsHeaderSignatureEvidence, +} from "../domain/dosMz.js"; /** Initial bytes required by the other bounded format classifiers. */ export const ARTIFACT_CLASSIFICATION_PREFIX_BYTES = 8_192; @@ -24,6 +28,7 @@ export interface ArtifactClassificationEvidence { readonly prefix: Buffer; readonly asarHeader: FinalAsarHeaderProbeObservation; readonly fatSliceHeaders?: FatSliceHeaderEvidence; + readonly mzWindowsHeaderSignature?: MzWindowsHeaderSignatureEvidence; } const observeFatSliceHeaders = (input: { @@ -74,6 +79,7 @@ const finishClassificationEvidence = ( archiveBytes: number, finishAsar: (archiveBytes: number) => FinalAsarHeaderProbeObservation, sliceHeadersByOffset: Map, + mzWindowsHeaderSignature: MzWindowsHeaderSignatureEvidence | undefined, ): ArtifactClassificationEvidence => { const finalHeader = readFatContainerHeader(prefix, archiveBytes); if (finalHeader.state === "parsed") { @@ -102,6 +108,9 @@ const finishClassificationEvidence = ( prefix, asarHeader: finishAsar(archiveBytes), ...(fatSliceHeaders === undefined ? {} : { fatSliceHeaders }), + ...(mzWindowsHeaderSignature === undefined + ? {} + : { mzWindowsHeaderSignature }), }; }; @@ -120,6 +129,37 @@ export const createArtifactClassificationProbe = ( number, { readonly bytes: Buffer; count: number } >(); + let mzWindowsHeaderSignature: MzWindowsHeaderSignatureEvidence | undefined; + let mzWindowsHeaderOffset: number | undefined; + let mzHeaderChecked = false; + const observeSignatureWindow = ( + offset: number, + sourceOffset: number, + sourceBytes: Uint8Array, + ): void => { + if (!Number.isSafeInteger(offset) || offset < 0) return; + const start = Math.max(offset, sourceOffset); + const end = Math.min(offset + 4, sourceOffset + sourceBytes.length); + if (start >= end) return; + const window = Buffer.alloc(4); + const previous = + mzWindowsHeaderSignature?.offset === offset + ? mzWindowsHeaderSignature.bytes + : undefined; + previous?.copy(window); + let observedLength = previous?.length ?? 0; + if (start < end && start <= offset + observedLength) { + Buffer.from( + sourceBytes.subarray(start - sourceOffset, end - sourceOffset), + ).copy(window, start - offset); + observedLength = Math.max(observedLength, end - offset); + } + if (observedLength === 0) return; + mzWindowsHeaderSignature = { + offset, + bytes: window.subarray(0, observedLength), + }; + }; const retain = (bytes: Uint8Array): void => { let offset = 0; while (offset < bytes.length) { @@ -193,6 +233,31 @@ export const createArtifactClassificationProbe = ( streamOffset, bytes, }); + if (!mzHeaderChecked && retainedBytes >= 64) { + mzHeaderChecked = true; + const prefix = retainedPrefix(); + if (prefix[0] === 0x4d && prefix[1] === 0x5a) { + const windowsHeader = observeMzWindowsHeader( + prefix, + Number.MAX_SAFE_INTEGER, + ); + if (windowsHeader.state === "signature-needed") { + mzWindowsHeaderOffset = windowsHeader.offset; + if (windowsHeader.offset < prefix.length) { + const inline = prefix.subarray( + windowsHeader.offset, + Math.min(windowsHeader.offset + 4, prefix.length), + ); + mzWindowsHeaderSignature = { + offset: windowsHeader.offset, + bytes: inline, + }; + } + } + } + } + if (mzWindowsHeaderOffset !== undefined) + observeSignatureWindow(mzWindowsHeaderOffset, streamOffset, bytes); observedBytes += bytes.length; }, finish(archiveBytes: number): ArtifactClassificationEvidence { @@ -201,6 +266,7 @@ export const createArtifactClassificationProbe = ( archiveBytes, (bytes) => asar.finish(bytes), fatSliceHeadersByOffset, + mzWindowsHeaderSignature, ); }, }; diff --git a/src/artifacts/PyInstallerArtifactReader.ts b/src/artifacts/PyInstallerArtifactReader.ts index 9f79d5f513..b065139af5 100644 --- a/src/artifacts/PyInstallerArtifactReader.ts +++ b/src/artifacts/PyInstallerArtifactReader.ts @@ -31,6 +31,12 @@ import { abortIfNeeded, hashReadable } from "./ArtifactHash.js"; import { readArtifactFileChunks } from "./readArtifactFileChunks.js"; import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; import { artifactIntegrityDetails } from "../domain/artifactIntegrity.js"; +import { parseMzHeaderEvidence } from "../domain/dosMz.js"; +import { + readFatContainerHeader, + readMachoHeaderPrefixFacts, +} from "../domain/apple/machoContainer.js"; +import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "./ArtifactClassificationProbe.js"; import { PyInstallerPyzStore } from "./pyinstaller/PyInstallerPyzStore.js"; import { pyInstallerMemberStream } from "./pyinstaller/PyInstallerMemberStream.js"; import { @@ -47,10 +53,12 @@ import { sealMember, type PyInstallerMember, type PyzImage, + type CarrierPathDialect, } from "./pyinstaller/CArchiveMembers.js"; import { readPyzArchive } from "./pyinstaller/PyzArchive.js"; import type { PyInstallerFailure } from "./pyinstaller/PyInstallerFailure.js"; import { pyInstallerFileAdmissionFailure } from "./pyinstaller/PyInstallerFileAdmission.js"; +import { readPyInstallerBytes } from "./pyinstaller/PyInstallerBytes.js"; const MAX_PARSED_FACT_BYTES = 32 * 1024 * 1024; const MAX_RETAINED_ENTRY_BYTES = 64 * 1024 * 1024; @@ -59,6 +67,45 @@ const RECORD_OWNERSHIP_RESERVATION = 256; // Repeated prefixes are charged again rather than underestimating retained work. const PATH_COMPONENT_RESERVATION = 1024; +const carrierPathDialect = async ( + handle: FileHandle, + fileSize: number, + signal?: AbortSignal, +): Promise => { + const prefixRead = await readPyInstallerBytes(handle, { + position: 0, + length: Math.min(fileSize, ARTIFACT_CLASSIFICATION_PREFIX_BYTES), + label: "carrier header", + signal, + }); + if (!prefixRead.ok) return "unknown"; + const prefix = prefixRead.value; + let windowsHeader = parseMzHeaderEvidence(prefix, fileSize).windowsHeader; + if (windowsHeader.state === "signature-needed") { + const signatureRead = await readPyInstallerBytes(handle, { + position: windowsHeader.offset, + length: 4, + label: "PE signature", + signal, + }); + if (!signatureRead.ok) return "unknown"; + windowsHeader = parseMzHeaderEvidence(prefix, fileSize, { + offset: windowsHeader.offset, + bytes: signatureRead.value, + }).windowsHeader; + } + if (windowsHeader.state === "declared") + return windowsHeader.signature === "PE" ? "windows" : "unknown"; + if (prefix.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46]))) + return "posix"; + const thinMachO = readMachoHeaderPrefixFacts(prefix); + if (thinMachO.identity !== null) return "posix"; + if (readFatContainerHeader(prefix, fileSize).state === "parsed") + return "posix"; + // A missing carrier proof does not establish POSIX separator semantics. + return "unknown"; +}; + interface LoadedArchive { readonly members: readonly PyInstallerMember[]; readonly traversal: ArtifactReaderTraversal; @@ -301,6 +348,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { const handle = await this.#openHandle(signal); await this.#verifySource(); const size = (await handle.stat()).size; + const pathDialect = await carrierPathDialect(handle, size, signal); const cookie = await this.#cookie(handle, size, signal); this.#budget = artifactExpansionBudgetBytes(cookie.packageLength); const toc = await this.#toc(handle, cookie, signal); @@ -320,6 +368,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { images, magic, encrypted, + pathDialect, signal, ); for (const member of members) @@ -478,6 +527,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { images: ReadonlyMap, magic: Buffer | undefined, encrypted: boolean, + pathDialect: CarrierPathDialect, signal: AbortSignal | undefined, ): Promise { const members: PyInstallerMember[] = []; @@ -503,10 +553,20 @@ export class PyInstallerArtifactReader implements ArtifactReader { abortIfNeeded(signal); const image = entry.typeCode === "z" ? images.get(entry) : undefined; if (image === undefined) { - append(projectStoredMember(entry, cookie.archiveStart, magic)); + append( + projectStoredMember(entry, cookie.archiveStart, magic, pathDialect), + ); continue; } - append(projectPyzContainer(entry, cookie.archiveStart, image, encrypted)); + append( + projectPyzContainer( + entry, + cookie.archiveStart, + image, + encrypted, + pathDialect, + ), + ); if (image.archive.kind !== "entries") continue; for (const item of image.archive.entries) { if (members.length % 256 === 0) await setImmediate(); @@ -517,6 +577,7 @@ export class PyInstallerArtifactReader implements ArtifactReader { item, encrypted, image.archive.pythonMagic, + pathDialect, item.namespace || encrypted ? undefined : { diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts index a3ac4243de..01c68d579b 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -15,10 +15,35 @@ import { } from "./ArtifactGraphConstruction.js"; import { createArtifactClassificationProbe } from "../ArtifactClassificationProbe.js"; import { artifactIdForContent } from "../../domain/artifactIdentity.js"; +import { pe } from "../../domain/binaryTarget.fixture.js"; import type { ArtifactEntry } from "../ArtifactReader.js"; import { classifyArtifactBytesAtPath } from "../../../tests/fixtures/artifactClassification.js"; +it("joins only observed PE signature bytes across stream chunks", () => { + const bytes = pe(0x8664, 8190); + const probe = createArtifactClassificationProbe(); + probe.write(bytes.subarray(0, 8191)); + probe.write(bytes.subarray(8191, 8192)); + const partial = probe.finish(bytes.length); + expect(partial.mzWindowsHeaderSignature).toEqual({ + offset: 8190, + bytes: Buffer.from("PE"), + }); + expect( + classifyArtifactContent("native.exe", partial, bytes.length).format, + ).toBe("unknown"); + probe.write(bytes.subarray(8192)); + const complete = probe.finish(bytes.length); + expect(complete.mzWindowsHeaderSignature).toEqual({ + offset: 8190, + bytes: Buffer.from([0x50, 0x45, 0, 0]), + }); + expect( + classifyArtifactContent("native.exe", complete, bytes.length).format, + ).toBe("pe"); +}); + it("keeps an ambiguous DOS MZ inventory candidate unknown", () => { const bytes = Buffer.alloc(128); bytes.write("MZ", 0, "ascii"); @@ -358,6 +383,18 @@ describe("bounded FAT inventory classification", () => { }); }); +it("keeps byte content format separate from path-specific package role", () => { + const probe = createArtifactClassificationProbe(); + probe.write(Buffer.from("PK\x03\x04", "binary")); + expect( + classifyArtifactContent("application.apk", probe.finish(4), 4), + ).toEqual({ + kind: "container", + format: "apk", + contentFormat: "zip", + }); +}); + describe("FAT prefix observation coverage", () => { it("does not invent an observation for a header beyond the retained prefix", () => { const bytes = Buffer.alloc(8_224); diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index 4e6c3c5b5b..f369f8a018 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -189,12 +189,12 @@ export const materializeDirectoryNodes = ( }; export const createRootNode = (input: { - readonly digest: { - readonly sha256: string; - readonly bytes: number; - readonly prefix: Buffer; - readonly fatSliceHeaders?: ArtifactClassificationEvidence["fatSliceHeaders"]; - } | null; + readonly digest: + | ({ + readonly sha256: string; + readonly bytes: number; + } & ArtifactClassificationEvidence) + | null; readonly occurrences: readonly MutableOccurrence[]; }): ArtifactNode => createArtifactNode({ @@ -218,11 +218,7 @@ export const createRootNode = (input: { format: input.digest === null ? "directory" - : classifyArtifactBytes( - input.digest.prefix, - input.digest.bytes, - input.digest.fatSliceHeaders, - ), + : classifyArtifactBytes(input.digest, input.digest.bytes), contentState: "materialized", }); @@ -510,28 +506,31 @@ export const classifyArtifactContent = ( ): { readonly kind: ArtifactOccurrence["artifact_kind"]; readonly format: ArtifactOccurrence["artifact_format"]; + readonly contentFormat: ArtifactNode["format"]; } => { - const { prefix, asarHeader, fatSliceHeaders } = evidence; - const format = classifyArtifactBytes(prefix, fileSize, fatSliceHeaders); + const { prefix, asarHeader } = evidence; + const contentFormat = classifyArtifactBytes(evidence, fileSize); const byPath = classifyArtifactPath(path); + let role: ReturnType; if ( - format === "plist" && + contentFormat === "plist" && !hasBinaryPlistSignature(prefix) && (["javascript", "source-map", "entitlements"].includes(byPath.kind) || path.toLowerCase().endsWith(".asar")) ) - return byPath; - if ( - format === "file" && + role = byPath; + else if ( + contentFormat === "file" && path.toLowerCase().endsWith(".asar") && asarHeader.state !== "rejected" ) { if (asarHeader.state === "limit") throw asarHeaderAdmissionFailure(asarHeader.admission); if (asarHeader.state === "matched") - return { kind: "container", format: "asar" }; - } - return artifactRoleForFormat(path, format); + role = { kind: "container", format: "asar" }; + else role = artifactRoleForFormat(path, contentFormat); + } else role = artifactRoleForFormat(path, contentFormat); + return { ...role, contentFormat }; }; const artifactRoleForFormat = ( @@ -558,15 +557,15 @@ const artifactRoleForFormat = ( /** Classify storage bytes independently of path names, roles, and permissions. */ export const classifyArtifactBytes = ( - prefix: Buffer, - fileSize = prefix.length, - fatSliceHeaders?: ArtifactClassificationEvidence["fatSliceHeaders"], + evidence: ArtifactClassificationEvidence, + fileSize = evidence.prefix.length, ): ArtifactNode["format"] => { + const { prefix, mzWindowsHeaderSignature } = evidence; if (hasZipSignature(prefix)) return "zip"; if (hasGodotPckHeader(prefix)) return "godot-pck"; if (hasBinaryPlistSignature(prefix)) return "plist"; if (hasXmlPlistSignature(prefix)) return "plist"; - const fatFormat = classifyFatArtifact(prefix, fileSize, fatSliceHeaders); + const fatFormat = classifyFatArtifact(evidence, fileSize); if (fatFormat !== null) return fatFormat; if (prefix.length >= 4) { const magic = prefix.readUInt32BE(0); @@ -582,8 +581,12 @@ export const classifyArtifactBytes = ( prefix.length < Math.min(fileSize, 64) ) return "unknown"; - const evidence = parseMzHeaderEvidence(prefix, fileSize); - const { windowsHeader } = evidence; + const mzEvidence = parseMzHeaderEvidence( + prefix, + fileSize, + mzWindowsHeaderSignature, + ); + const { windowsHeader } = mzEvidence; if (windowsHeader.state === "signature-needed") return "unknown"; if (windowsHeader.state === "candidate") return "unknown"; if (windowsHeader.state === "declared") { @@ -591,22 +594,24 @@ export const classifyArtifactBytes = ( if ( windowsHeader.signature === "PE" && windowsOffset >= 64 && - windowsOffset <= prefix.length - 4 + (windowsOffset <= prefix.length - 4 || + (mzWindowsHeaderSignature?.offset === windowsOffset && + mzWindowsHeaderSignature.bytes.length >= 4)) ) return "pe"; return "unknown"; } - if (evidence.dos.ok) return "dos-mz"; + if (mzEvidence.dos.ok) return "dos-mz"; return "unknown"; } return "file"; }; const classifyFatArtifact = ( - prefix: Buffer, + evidence: ArtifactClassificationEvidence, fileSize: number, - fatSliceHeaders?: ArtifactClassificationEvidence["fatSliceHeaders"], ): ArtifactNode["format"] | null => { + const { prefix, fatSliceHeaders } = evidence; const fatHeader = fatSliceHeaders?.header ?? readFatContainerHeader(prefix, fileSize); if (fatHeader.state === "needs-data" || fatHeader.state === "malformed") diff --git a/src/artifacts/inventory/classify.ts b/src/artifacts/inventory/classify.ts index 1b10028d03..5c42b0c6db 100644 --- a/src/artifacts/inventory/classify.ts +++ b/src/artifacts/inventory/classify.ts @@ -26,8 +26,12 @@ import { openRootArtifact, } from "./hashStableRootArtifact.js"; import type { ArtifactInventoryOptions } from "./types.js"; -import type { FatSliceHeaderEvidence } from "../../domain/apple/machoContainer.js"; import { readMachoHeaderPrefixFacts } from "../../domain/apple/machoContainer.js"; +import { + observeMzWindowsHeader, + type MzWindowsHeaderSignatureEvidence, +} from "../../domain/dosMz.js"; +import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js"; interface RootClassification { readonly format: ArtifactOccurrence["artifact_format"]; @@ -186,17 +190,20 @@ const classifyRootFormat = async ( return "dmg"; if (await locateGodotPck(handle, archiveBytes, signal, evidence.prefix)) return "godot-pck"; - const fatSliceHeaders = await readRootFatSliceHeaders( + const fatEvidence = await readRootFatSliceHeaders(handle, evidence, signal); + const mzWindowsHeaderSignature = await readRootMzWindowsHeaderSignature( handle, - evidence.fatSliceHeaders, + fatEvidence, + archiveBytes, signal, ); + const classificationEvidence = + mzWindowsHeaderSignature === undefined + ? fatEvidence + : { ...fatEvidence, mzWindowsHeaderSignature }; const format = classifyArtifactContent( path, - { - ...evidence, - ...(fatSliceHeaders === undefined ? {} : { fatSliceHeaders }), - }, + classificationEvidence, archiveBytes, ).format; // Only bytes no other format claims may lead a located ZIP, so an @@ -228,10 +235,11 @@ const classifyRootFormat = async ( /** Read exact FAT header windows missed by the bounded prefix probe. */ const readRootFatSliceHeaders = async ( handle: FileHandle, - existing: FatSliceHeaderEvidence | undefined, + evidence: ArtifactClassificationEvidence, signal?: AbortSignal, -): Promise => { - if (existing === undefined) return undefined; +): Promise => { + const existing = evidence.fatSliceHeaders; + if (existing === undefined) return evidence; const { header } = existing; const sliceHeadersByOffset = new Map(existing.sliceHeadersByOffset); for (let index = 0; index < header.slices.length; index += 1) { @@ -265,7 +273,59 @@ const readRootFatSliceHeaders = async ( readMachoHeaderPrefixFacts(thinHeader.subarray(0, bytesRead)), ); } - return { header, sliceHeadersByOffset }; + return { ...evidence, fatSliceHeaders: { header, sliceHeadersByOffset } }; +}; + +/** Read only the missing signature bytes requested by the bounded MZ parser. */ +const readRootMzWindowsHeaderSignature = async ( + handle: FileHandle, + evidence: ArtifactClassificationEvidence, + fileSize: number, + signal?: AbortSignal, +): Promise => { + const observation = observeMzWindowsHeader(evidence.prefix, fileSize); + if (observation.state !== "signature-needed") + return evidence.mzWindowsHeaderSignature; + const { offset } = observation; + if (!Number.isSafeInteger(offset) || offset < 0 || offset >= fileSize) + return evidence.mzWindowsHeaderSignature; + + const window = Buffer.alloc(4); + const existing = + evidence.mzWindowsHeaderSignature?.offset === offset + ? evidence.mzWindowsHeaderSignature.bytes + : undefined; + const inline = + offset < evidence.prefix.length + ? evidence.prefix.subarray( + offset, + Math.min(offset + 4, evidence.prefix.length), + ) + : undefined; + const known = + existing !== undefined && + (inline === undefined || existing.length > inline.length) + ? existing + : inline; + known?.copy(window); + let available = known?.length ?? 0; + const requiredLength = (): number => + available >= 2 && window[0] === 0x50 && window[1] === 0x45 ? 4 : 2; + while (available < requiredLength() && offset + available < fileSize) { + abortIfNeeded(signal); + const length = Math.min( + requiredLength() - available, + fileSize - offset - available, + ); + const chunk = Buffer.alloc(length); + const read = await handle.read(chunk, 0, length, offset + available); + abortIfNeeded(signal); + if (read.bytesRead === 0) break; + chunk.copy(window, available, 0, read.bytesRead); + available += read.bytesRead; + } + if (available === 0) return undefined; + return { offset, bytes: window.subarray(0, available) }; }; const hasKolyTrailer = async ( diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index bc4a71f905..9102f092ef 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -344,13 +344,17 @@ const digestArtifactEntry = async ( } const classified = entry.kind === "slice" - ? ({ kind: "universal-slice", format: "mach-o" } as const) + ? { + kind: "universal-slice" as const, + format: "mach-o" as const, + contentFormat: classifyArtifactBytes(digest, digest.bytes), + } : classifyArtifactContent(logicalPath, digest, digest.bytes); return { node: createArtifactNode({ sha256: digest.sha256, size: digest.bytes, - format: classifyArtifactBytes(digest.prefix, digest.bytes), + format: classified.contentFormat, contentState: "embedded", }), mismatched, diff --git a/src/artifacts/pyinstaller/CArchive.fixture.ts b/src/artifacts/pyinstaller/CArchive.fixture.ts index de4691ac18..73623db4a8 100644 --- a/src/artifacts/pyinstaller/CArchive.fixture.ts +++ b/src/artifacts/pyinstaller/CArchive.fixture.ts @@ -1,4 +1,5 @@ import { deflateSync } from "node:zlib"; +import { pe } from "../../domain/binaryTarget.fixture.js"; import { PYINSTALLER_COOKIE_MAGIC } from "./CArchiveLayout.js"; @@ -28,8 +29,10 @@ export interface CArchiveFixtureMember { readonly declaredUncompressedSize?: number; } -/** Build a tiny ELF with an appended CArchive, optionally followed by a trailer. */ +/** Build a tiny native carrier with an appended CArchive and optional trailer. */ export const cArchiveExecutable = (input: { + readonly carrier?: "elf" | "pe"; + readonly peHeaderOffset?: number; readonly layout: "2.0" | "2.1"; readonly pythonVersion: number; readonly pythonLibrary?: string; @@ -70,7 +73,9 @@ export const cArchiveExecutable = (input: { tocLength: toc.length, }); return Buffer.concat([ - ELF_PREFIX, + input.carrier === "pe" + ? pe(0x8664, input.peHeaderOffset ?? 64) + : ELF_PREFIX, ...payloads, toc, cookie, diff --git a/src/artifacts/pyinstaller/CArchiveMembers.ts b/src/artifacts/pyinstaller/CArchiveMembers.ts index 6e0669133a..b8cbd0aee9 100644 --- a/src/artifacts/pyinstaller/CArchiveMembers.ts +++ b/src/artifacts/pyinstaller/CArchiveMembers.ts @@ -15,6 +15,8 @@ import type { PyInstallerEntryMetadata } from "../../domain/artifactFormatMetada const CRYPTO_KEY_NAME = "pyimod00_crypto_key"; +export type CarrierPathDialect = "posix" | "windows" | "unknown"; + /** Parsed facts and the descriptor identity used to locate a retained image. */ export interface PyzImage { readonly archive: PyzArchive; @@ -59,6 +61,7 @@ export const projectStoredMember = ( entry: CArchiveEntry, archiveStart: number, pythonMagic: Buffer | undefined, + pathDialect: CarrierPathDialect, ): PyInstallerMember => { if (isCArchiveDeclaration(entry)) return { @@ -83,7 +86,7 @@ export const projectStoredMember = ( pythonMagic !== undefined && isBytecodeType(entry.typeCode) ? reconstructedPycHeader(pythonMagic) : undefined; - const path = canonicalMemberPath(entry.name); + const path = canonicalMemberPath(entry.name, pathDialect); const symlink = entry.typeCode === "n"; const limitations: string[] = []; if (symlink) limitations.push("PyInstaller symlink target was not read."); @@ -116,9 +119,10 @@ export const projectPyzContainer = ( archiveStart: number, image: PyzImage, encrypted: boolean, + pathDialect: CarrierPathDialect, ): PyInstallerMember => { const { archive } = image; - const path = canonicalMemberPath(entry.name); + const path = canonicalMemberPath(entry.name, pathDialect); const byteOffset = sourceByteOffset(entry, archiveStart); const limitations: string[] = []; if (archive.kind === "opaque") @@ -198,6 +202,7 @@ export const projectPyzMember = ( item: PyzEntry, encryptionMarker: boolean, pythonMagic: Buffer | undefined, + pathDialect: CarrierPathDialect, range: | { readonly handle: FileHandle; @@ -206,7 +211,11 @@ export const projectPyzMember = ( } | undefined, ): PyInstallerMember => { - const path = pyzMemberPath(canonicalMemberPath(container.name), item); + const path = pyzMemberPath( + canonicalMemberPath(container.name, pathDialect), + item, + pathDialect, + ); const encrypted = pyzEntryEncrypted(item, encryptionMarker); if (item.namespace) return { @@ -282,14 +291,17 @@ export const projectPyzMember = ( }; /** - * CArchive and PYZ data names treat `\` as a separator. The logical path is + * CArchive and PYZ data names use the carrier's path dialect. The logical path is * canonical; the stored spelling is kept when that conversion changes it. * `source_location` is an offset plus declared size, so a compressed member * publishes a file span only when that size is the physical extent. Otherwise * the span stays unknown, as ZIP does for compressed entries. */ -const canonicalMemberPath = (storedName: string): string => - storedName.replaceAll("\\", "/"); +const canonicalMemberPath = ( + storedName: string, + pathDialect: CarrierPathDialect, +): string => + pathDialect === "windows" ? storedName.replaceAll("\\", "/") : storedName; const sourceByteOffset = ( entry: CArchiveEntry, @@ -297,9 +309,13 @@ const sourceByteOffset = ( ): number | null => entry.compressionFlag === 0 ? archiveStart + entry.offset : null; -const pyzMemberPath = (container: string, item: PyzEntry): string => { +const pyzMemberPath = ( + container: string, + item: PyzEntry, + pathDialect: CarrierPathDialect, +): string => { if (item.typeCode === 2) - return `${container}/${canonicalMemberPath(item.name)}`; + return `${container}/${canonicalMemberPath(item.name, pathDialect)}`; const dotted = item.name.replaceAll(".", "/"); if (item.namespace) return `${container}/${dotted}`; if (item.package) return `${container}/${dotted}/__init__.pyc`; diff --git a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts index 1dc999dee7..6658045c1f 100644 --- a/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerArtifactReader.test.ts @@ -849,6 +849,7 @@ it("inventories and extracts a Windows member name under its canonical path", as const path = await writeArchive( "windows", cArchiveExecutable({ + carrier: "pe", layout: "2.1", pythonVersion: 309, pythonLibrary: "python39.dll", @@ -880,7 +881,7 @@ it("inventories and extracts a Windows member name under its canonical path", as await materializeArtifactInventory( { inputPath: path, - inputFormat: "elf", + inputFormat: "pe", outputRoot: output, environment: process.env, integrityPolicy: "fail", @@ -894,6 +895,92 @@ it("inventories and extracts a Windows member name under its canonical path", as ); }); +it("preserves distinct POSIX backslash and slash member paths", async () => { + const backslashName = String.raw`a\b.txt`; + const slashName = "a/b.txt"; + const backslashPayload = Buffer.from("literal-backslash"); + const slashPayload = Buffer.from("nested-slash"); + const path = await writeArchive( + "posix-paths", + cArchiveExecutable({ + layout: "2.1", + pythonVersion: 309, + pythonLibrary: "python39.dll", + members: [ + { + name: backslashName, + typeCode: "x", + payload: backslashPayload, + compress: false, + }, + { + name: slashName, + typeCode: "x", + payload: slashPayload, + compress: false, + }, + ], + }), + ); + const snapshot = await scanCanonicalArtifactInventory(path); + const backslashEntry = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === backslashName, + ); + const slashEntry = snapshot.occurrences.find( + (occurrence) => occurrence.logical_path === slashName, + ); + expect(backslashEntry?.format_metadata).toMatchObject({ + source_name: backslashName, + }); + expect(slashEntry?.format_metadata).toMatchObject({ source_name: slashName }); + expect(snapshot.coverage.status).toBe("complete"); + if (process.platform === "win32") return; + + const directory = await createTestTempDirectory("rea-pyinstaller-posix-out-"); + const output = join(directory, "extracted"); + await materializeArtifactInventory( + { + inputPath: path, + inputFormat: "elf", + outputRoot: output, + environment: process.env, + integrityPolicy: "fail", + }, + path, + snapshot, + ); + expect(await readFile(join(output, backslashName))).toEqual(backslashPayload); + expect(await readFile(join(output, "a", "b.txt"))).toEqual(slashPayload); +}); + +it("reads a PE signature beyond the carrier prefix before projecting paths", async () => { + const path = await writeArchive( + "distant-pe", + cArchiveExecutable({ + carrier: "pe", + peHeaderOffset: 8190, + layout: "2.1", + pythonVersion: 309, + members: [ + { + name: String.raw`pkg\data.bin`, + typeCode: "x", + payload: Buffer.from("windows-path"), + compress: false, + }, + ], + }), + ); + const reader = new PyInstallerArtifactReader(path); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + expect(entries.map((entry) => entry.path)).toContain("pkg/data.bin"); + } finally { + await reader.close(); + } +}); + it("rejects a stored member whose declared size disagrees with its bytes", async () => { const path = await writeArchive( "stored-size", diff --git a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts index 72087f42a9..80e2c53bb8 100644 --- a/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts +++ b/src/artifacts/pyinstaller/PyInstallerFormatBoundaries.test.ts @@ -127,6 +127,7 @@ it.each([true, false])( }; await withPyInstallerReader( cArchiveExecutable({ + carrier: "pe", layout: "2.1", pythonVersion: 309, members: [ diff --git a/src/domain/android/androidApplication.ts b/src/domain/android/androidApplication.ts index 4dd94f0fb7..134f3bb04b 100644 --- a/src/domain/android/androidApplication.ts +++ b/src/domain/android/androidApplication.ts @@ -24,6 +24,7 @@ import { } from "../mobileApplicationGraphProjection.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { err, ok, type Result } from "../result.js"; +import { isV1SignatureMetadataName } from "../apkSignature/apkV1MetadataName.js"; /** Deterministic, execution-free Android application inventory projection. */ export const androidApplicationProjectionResultSchema = z.strictObject({ @@ -173,9 +174,7 @@ const classify = (all: readonly Component[]) => ({ format === "elf" || /(?:^|\/)lib\/[^/]+\/[^/]+\.so$/iu.test(path), ), javascript: all.filter(({ format }) => format === "javascript-bundle"), - signing: all.filter(({ path }) => - /^META-INF\/[^/]+\.(?:MF|RSA|DSA|EC|SF)$/iu.test(path), - ), + signing: all.filter(({ path }) => isV1SignatureMetadataName(path)), }); const runtimeFamilies = ( diff --git a/src/domain/apkSignature/apkV1MetadataName.ts b/src/domain/apkSignature/apkV1MetadataName.ts new file mode 100644 index 0000000000..9328755839 --- /dev/null +++ b/src/domain/apkSignature/apkV1MetadataName.ts @@ -0,0 +1,35 @@ +/** Exact path-name rules used to select APK v1 signing metadata. */ +export const APK_V1_META_INF_PREFIX = "META-INF/"; +export const APK_V1_MANIFEST_PATH = `${APK_V1_META_INF_PREFIX}MANIFEST.MF`; + +const META_INF_PREFIX_BYTES = Uint8Array.from( + [...APK_V1_META_INF_PREFIX].map((character) => character.charCodeAt(0)), +); +const BLOCK_SUFFIXES = [".RSA", ".DSA", ".EC"] as const; + +export const isV1SignatureFileName = (name: string): boolean => + name.endsWith(".SF"); + +export const isV1SignatureBlockName = (name: string): boolean => + BLOCK_SUFFIXES.some((suffix) => name.endsWith(suffix)); + +export const v1SignatureFileNameForBlock = (block: string): string => + `${block.slice(0, block.lastIndexOf("."))}.SF`; + +export const isV1SignatureMetadataName = (name: string): boolean => + name.startsWith(APK_V1_META_INF_PREFIX) && + (name === APK_V1_MANIFEST_PATH || + isV1SignatureFileName(name) || + isV1SignatureBlockName(name)); + +/** Test the raw central-directory name before decoding noncandidate ZIP paths. */ +export const startsWithV1MetaInfPrefix = ( + bytes: Uint8Array, + start: number, + length: number, +): boolean => { + if (length < META_INF_PREFIX_BYTES.length) return false; + for (let index = 0; index < META_INF_PREFIX_BYTES.length; index += 1) + if (bytes[start + index] !== META_INF_PREFIX_BYTES[index]) return false; + return true; +}; diff --git a/src/domain/apkSignature/apkV1Signature.ts b/src/domain/apkSignature/apkV1Signature.ts index ff1a8c292f..554ee06211 100644 --- a/src/domain/apkSignature/apkV1Signature.ts +++ b/src/domain/apkSignature/apkV1Signature.ts @@ -25,6 +25,15 @@ import { sameZipName, type StoredZipName, } from "./zipEntryName.js"; +import { + APK_V1_MANIFEST_PATH, + APK_V1_META_INF_PREFIX, + isV1SignatureBlockName, + isV1SignatureFileName, + isV1SignatureMetadataName, + startsWithV1MetaInfPrefix, + v1SignatureFileNameForBlock, +} from "./apkV1MetadataName.js"; const CD_SIGNATURE = 0x02014b50; const CD_FIXED_SIZE = 46; @@ -56,37 +65,6 @@ export const MAX_V1_SIGNERS = 32; /** Duplicated central-directory names decoded for the report; the rest are counted. */ export const MAX_DUPLICATE_NAME_SAMPLE = 5; -const META_INF = "META-INF/"; -const MANIFEST = "META-INF/MANIFEST.MF"; -const META_INF_BYTES = Uint8Array.from( - [...META_INF].map((c) => c.charCodeAt(0)), -); - -/** True when the raw name starts with `META-INF/` (apksig's selection). */ -const startsWithMetaInf = ( - bytes: Uint8Array, - start: number, - length: number, -): boolean => { - if (length < META_INF_BYTES.length) return false; - for (let i = 0; i < META_INF_BYTES.length; i += 1) - if (bytes[start + i] !== META_INF_BYTES[i]) return false; - return true; -}; - -const BLOCK_SUFFIXES = [".RSA", ".DSA", ".EC"] as const; -const isSignatureFile = (name: string): boolean => name.endsWith(".SF"); -const isSignatureBlock = (name: string): boolean => - BLOCK_SUFFIXES.some((suffix) => name.endsWith(suffix)); - -/** The `.SF` file apksig pairs with a signature block. */ -const signatureFileFor = (block: string): string => - `${block.slice(0, block.lastIndexOf("."))}.SF`; - -/** The one owner of which `META-INF/` names take part in v1 signing. */ -const isV1SignatureName = (name: string): boolean => - name === MANIFEST || isSignatureFile(name) || isSignatureBlock(name); - /** A signing-relevant central-directory entry and where its data is stored. */ export interface ApkV1Entry { readonly name: string; @@ -331,10 +309,10 @@ export const centralDirectoryV1SignatureEntries = ( }); starts[i] = nameStart; lengths[i] = nameLength; - if (startsWithMetaInf(bytes, nameStart, nameLength)) { + if (startsWithV1MetaInfPrefix(bytes, nameStart, nameLength)) { const rawName = bytes.subarray(nameStart, nameStart + nameLength); const name = decodeZipName(rawName); - if (isV1SignatureName(name)) { + if (isV1SignatureMetadataName(name)) { if (entries.length >= MAX_V1_SIGNATURE_ENTRIES) return err({ kind: "budget-exceeded", @@ -373,7 +351,9 @@ export const centralDirectoryV1SignatureEntries = ( export const detectV1Signature = ( entries: readonly ApkV1Entry[], ): ApkV1Signature => { - const selected = entries.filter((entry) => entry.name.startsWith(META_INF)); + const selected = entries.filter((entry) => + entry.name.startsWith(APK_V1_META_INF_PREFIX), + ); // Case-sensitive throughout, as apksig matches JAR signature entries. const seen = new Set(); const duplicates = new Set(); @@ -382,24 +362,24 @@ export const detectV1Signature = ( else seen.add(name); const signatureFiles = selected .map((entry) => entry.name) - .filter(isSignatureFile); - const blocks = selected.filter((entry) => isSignatureBlock(entry.name)); + .filter(isV1SignatureFileName); + const blocks = selected.filter((entry) => isV1SignatureBlockName(entry.name)); const sfNames = new Set(signatureFiles); const paired = new Set(); const signers = blocks.flatMap((block) => { - const signatureFile = signatureFileFor(block.name); + const signatureFile = v1SignatureFileNameForBlock(block.name); if (!sfNames.has(signatureFile) || paired.has(block.name)) return []; paired.add(block.name); // apksig reads every signer's manifest, so a duplicate one affects them all. const duplicate = - [block.name, signatureFile, MANIFEST].find((name) => + [block.name, signatureFile, APK_V1_MANIFEST_PATH].find((name) => duplicates.has(name), ) ?? null; return [{ signatureFile, block, duplicate }]; }); return { present: signers.length > 0, - hasManifest: selected.some((entry) => entry.name === MANIFEST), + hasManifest: selected.some((entry) => entry.name === APK_V1_MANIFEST_PATH), signatureFiles, blockFiles: blocks.map((block) => block.name), signers, diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index 17982521be..b9ff4c9a20 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -10,7 +10,10 @@ import type { } from "./binaryTargetTypes.js"; import { err, ok, type Result } from "./result.js"; import { readElfFixedHeader } from "./elfFixedHeader.js"; -import { parseMzHeaderEvidence } from "./dosMz.js"; +import { + parseMzHeaderEvidence, + type MzWindowsHeaderSignatureEvidence, +} from "./dosMz.js"; import { decodePeMachine, peOptionalHeaderMagic, @@ -106,7 +109,7 @@ export const parseExecutableHeader = ( bytes: Buffer, hostArchitecture: NodeJS.Architecture, fileSize = bytes.length, - mzWindowsHeaderSignature?: Buffer, + mzWindowsHeaderSignature?: MzWindowsHeaderSignatureEvidence, fatEvidence?: FatSliceHeaderEvidence, ): Result => { if ( @@ -436,7 +439,7 @@ const parseElf = ( const parseMz = ( bytes: Buffer, fileSize: number, - signatureAtOffset?: Buffer, + signatureAtOffset?: MzWindowsHeaderSignatureEvidence, ): Result => { const evidence = parseMzHeaderEvidence(bytes, fileSize, signatureAtOffset); const { windowsHeader } = evidence; diff --git a/src/domain/dosMz.test.ts b/src/domain/dosMz.test.ts index 1ef964b36d..571643add5 100644 --- a/src/domain/dosMz.test.ts +++ b/src/domain/dosMz.test.ts @@ -264,11 +264,10 @@ describe("MZ Windows header observations", () => { offset, }); expect( - observeMzWindowsHeader( - prefix, - offset + 512, - Buffer.from("PE\0\0", "binary"), - ), + observeMzWindowsHeader(prefix, offset + 512, { + offset, + bytes: Buffer.from("PE\0\0", "binary"), + }), ).toEqual({ state: "declared", offset, signature: "PE" }); }); diff --git a/src/domain/dosMz.ts b/src/domain/dosMz.ts index b7735b0368..03a5be0e3d 100644 --- a/src/domain/dosMz.ts +++ b/src/domain/dosMz.ts @@ -39,6 +39,12 @@ export type MzWindowsHeaderObservation = readonly signature: "PE" | "NE" | "LE" | "LX"; }; +/** Exact signature bytes observed at the DOS header's declared offset. */ +export interface MzWindowsHeaderSignatureEvidence { + readonly offset: number; + readonly bytes: Buffer; +} + /** * Observe the Windows new-header pointer without mistaking short DOS module * bytes for e_lfanew. A nonzero pointer in a complete DOS header is only a @@ -51,7 +57,7 @@ export type MzWindowsHeaderObservation = export const observeMzWindowsHeader = ( bytes: Buffer, fileSize = bytes.length, - signatureAtOffset?: Buffer, + signatureEvidence?: MzWindowsHeaderSignatureEvidence, ): MzWindowsHeaderObservation => { if (bytes.length < 64) return fileSize >= 64 ? { state: "header-needed" } : { state: "absent" }; @@ -71,7 +77,10 @@ export const observeMzWindowsHeader = ( offset < bytes.length ? bytes.subarray(offset, Math.min(offset + 4, bytes.length)) : undefined; - const available = signatureAtOffset ?? inlineSignature; + const available = + signatureEvidence?.offset === offset + ? signatureEvidence.bytes + : inlineSignature; const signature = windowsHeaderSignature(available); if (signature !== "other" && signature !== "unobserved") return { state: "declared", offset, signature }; @@ -85,7 +94,7 @@ export const observeMzWindowsHeader = ( available.toString("latin1", 0, 2) === "PE" && available.length < 4 && remaining >= 4); - if (needsMore && signatureAtOffset === undefined) + if (needsMore && signatureEvidence?.offset !== offset) return { state: "signature-needed", offset }; return { state: "candidate", offset, signature }; } @@ -177,10 +186,10 @@ export interface MzHeaderEvidence { export const parseMzHeaderEvidence = ( bytes: Buffer, fileSize = bytes.length, - signatureAtOffset?: Buffer, + signatureEvidence?: MzWindowsHeaderSignatureEvidence, ): MzHeaderEvidence => ({ dos: parseDosMzHeader(bytes, fileSize), - windowsHeader: observeMzWindowsHeader(bytes, fileSize, signatureAtOffset), + windowsHeader: observeMzWindowsHeader(bytes, fileSize, signatureEvidence), }); const validateRelocations = ( diff --git a/src/domain/sqlite/sqliteDatabase.ts b/src/domain/sqlite/sqliteDatabase.ts index 88051d408a..3f939dff20 100644 --- a/src/domain/sqlite/sqliteDatabase.ts +++ b/src/domain/sqlite/sqliteDatabase.ts @@ -9,6 +9,11 @@ export const SQLITE_PROVIDER_IDENTITY = { version: null, } as const; +/** SQLite exposes this table as a row source but omits its own catalog entry. */ +export const SQLITE_SCHEMA_TABLE_NAME = "sqlite_schema"; +export const sqliteSchemaHasObjectDeclaration = (name: string): boolean => + name !== SQLITE_SCHEMA_TABLE_NAME; + /** Default ordinary-table sample size when the caller selects a table. */ export const SQLITE_ROW_LIMIT_DEFAULT = 20; /** Execution budget, bounded by the runtime's signed 32-bit timer range. */ diff --git a/src/ghidra/GhidraDosSnapshot.ts b/src/ghidra/GhidraDosSnapshot.ts index 603cfd6a30..2c4af0211d 100644 --- a/src/ghidra/GhidraDosSnapshot.ts +++ b/src/ghidra/GhidraDosSnapshot.ts @@ -58,15 +58,17 @@ export const attestGhidraDosSnapshot = async ( } if (captured !== captureBytes) throw new RegularFileChangedError(path); const windowsHeader = observeMzWindowsHeader(prefix, fileBytes); - let signature: Buffer | undefined; + let signature: + | { readonly offset: number; readonly bytes: Buffer } + | undefined; if (windowsHeader.state === "signature-needed") { const size = Math.min(4, fileBytes - windowsHeader.offset); - signature = Buffer.allocUnsafe(size); + const signatureBytes = Buffer.allocUnsafe(size); let read = 0; while (read < size) { if (signal?.aborted) throw cancelled(signal.reason); const { bytesRead } = await handle.read( - signature, + signatureBytes, read, size - read, windowsHeader.offset + read, @@ -74,6 +76,10 @@ export const attestGhidraDosSnapshot = async ( if (bytesRead === 0) throw new RegularFileChangedError(path); read += bytesRead; } + signature = { + offset: windowsHeader.offset, + bytes: signatureBytes, + }; } const prepared = prepareGhidraDosLoadImage(prefix, fileBytes, signature); if (!prepared.ok) return prepared; diff --git a/src/ghidra/GhidraLoadImageValues.ts b/src/ghidra/GhidraLoadImageValues.ts index 56d6447100..4970ebb31f 100644 --- a/src/ghidra/GhidraLoadImageValues.ts +++ b/src/ghidra/GhidraLoadImageValues.ts @@ -5,7 +5,11 @@ import { type NativeLoadImageCheck, type NativeLoadImageObservation, } from "../domain/native/nativeLoadImage.js"; -import { parseMzHeaderEvidence, type DosMzHeader } from "../domain/dosMz.js"; +import { + parseMzHeaderEvidence, + type DosMzHeader, + type MzWindowsHeaderSignatureEvidence, +} from "../domain/dosMz.js"; import type { JsonValue } from "../domain/jsonValue.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { validateDosComLength } from "../domain/dosCom.js"; @@ -55,7 +59,7 @@ export type GhidraDosImage = { export const prepareGhidraDosLoadImage = ( bytes: Buffer, fileBytes: number, - signatureAtOffset?: Buffer, + signatureAtOffset?: MzWindowsHeaderSignatureEvidence, ): Result => { const evidence = parseMzHeaderEvidence(bytes, fileBytes, signatureAtOffset); if (evidence.windowsHeader.state === "candidate") diff --git a/src/native/elf/ElfNoteParser.test.ts b/src/native/elf/ElfNoteParser.test.ts index 8f104bb110..044a8feafb 100644 --- a/src/native/elf/ElfNoteParser.test.ts +++ b/src/native/elf/ElfNoteParser.test.ts @@ -3,16 +3,21 @@ import { readFileSync } from "node:fs"; import { expect, it } from "vitest"; import { inspectElfNotesInputSchema } from "../../domain/native/elfNotes.js"; import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { BinaryFormatFailure } from "../metadata/BinaryContainer.js"; import { compressedElfNoteFixture, elfNote, elfNoteFixture, elfNoteTablePastEnd, + elfExtendedSectionCountFixture, gnuAbiTagNote, gnuBuildIdNote, gnuPropertyNote, } from "./ElfNotes.fixture.js"; -import { parseElfNotes } from "./ElfNoteParser.js"; +import { + ElfNoteSectionNameBudgetFailure, + parseElfNotes, +} from "./ElfNoteParser.js"; const defaults = inspectElfNotesInputSchema.parse({ path: "/artifacts/app", @@ -133,14 +138,51 @@ it("keeps note facts but marks section names unavailable for unknown compression ); }); -it("bounds compressed section-name expansion with the source bytes", () => { +it("tags valid extended section numbering as unsupported", () => { + let failure: unknown; + try { + parse(elfExtendedSectionCountFixture()); + } catch (cause) { + failure = cause; + } + expect(failure).toBeInstanceOf(BinaryFormatFailure); + expect(failure).toMatchObject({ + kind: "unsupported", + message: "ELF extended numbering is unsupported.", + }); +}); + +it("admits compressed section names when max_file_bytes includes expansion", () => { const bytes = compressedElfNoteFixture(); + const sectionOffset = Number(bytes.readBigUInt64LE(40)); + const nameHeader = sectionOffset + bytes.readUInt16LE(62) * 64; + const namesOffset = Number(bytes.readBigUInt64LE(nameHeader + 24)); + const decodedNamesBytes = Number(bytes.readBigUInt64LE(namesOffset + 8)); + const minimumMaxFileBytes = bytes.length + decodedNamesBytes; const input = inspectElfNotesInputSchema.parse({ path: defaults.path, max_file_bytes: bytes.length + 1, }); - expect(() => parse(bytes, input)).toThrow(ArtifactReaderFailure); - expect(() => parse(bytes, input)).toThrow(/memory budget/u); + let failure: unknown; + try { + parse(bytes, input); + } catch (cause) { + failure = cause; + } + expect(failure).toBeInstanceOf(ElfNoteSectionNameBudgetFailure); + expect(failure).toMatchObject({ + minimumMaxFileBytes: BigInt(minimumMaxFileBytes), + selectedMaxFileBytes: bytes.length + 1, + }); + + const admitted = parse( + bytes, + inspectElfNotesInputSchema.parse({ + path: defaults.path, + max_file_bytes: minimumMaxFileBytes, + }), + ); + expect(admitted.tables[0]?.name).toBe(".note.gnu.build-id"); }); it("reports independent PT_NOTE and SHT_NOTE declarations of the same bytes", () => { @@ -327,7 +369,18 @@ const unterminatedNoteSections = ( it("bounds repeated scans of an unterminated section name", () => { expect(parse(unterminatedNoteSections(1, 64)).tables[0]?.name).toBeNull(); - expect(() => parse(unterminatedNoteSections(3, 400))).toThrow(/byte budget/u); + let failure: unknown; + try { + parse(unterminatedNoteSections(3, 400)); + } catch (cause) { + failure = cause; + } + expect(failure).toBeInstanceOf(ArtifactReaderFailure); + expect(failure).toMatchObject({ reason: "limit", boundary: "memory" }); + expect(failure).toHaveProperty( + "message", + expect.stringMatching(/byte budget/u), + ); }); it("rejects note text that repeats beyond the shared byte budget", () => { diff --git a/src/native/elf/ElfNoteParser.ts b/src/native/elf/ElfNoteParser.ts index e5277bc9c6..8ec5d38f3d 100644 --- a/src/native/elf/ElfNoteParser.ts +++ b/src/native/elf/ElfNoteParser.ts @@ -13,6 +13,7 @@ import { elfObjectTypeName, gnuNoteTypeName, } from "../../domain/native/elfNotes.js"; +import { BinaryFormatFailure } from "../metadata/BinaryContainer.js"; import { decodeElfSectionNameTable } from "../metadata/ElfSectionNameTable.js"; const PT_NOTE = 4; @@ -29,6 +30,21 @@ interface TextBudget { readonly scans: Map; } +/** Caller can raise max_file_bytes to admit the decoded section-name table. */ +export class ElfNoteSectionNameBudgetFailure extends Error { + readonly _tag = "ElfNoteSectionNameBudgetFailure"; + + constructor( + readonly minimumMaxFileBytes: bigint, + readonly selectedMaxFileBytes: number, + ) { + super( + `ELF source and decoded section names require ${String(minimumMaxFileBytes)} bytes, above the selected max_file_bytes (${String(selectedMaxFileBytes)}).`, + ); + this.name = "ElfNoteSectionNameBudgetFailure"; + } +} + const fail = (message: string): never => { throw new ArtifactReaderFailure("format", message); }; @@ -304,7 +320,10 @@ export const parseElfNotes = ( shstrndx === 0xffff || (shoff !== 0 && shnum === 0) ) - return fail("ELF extended numbering is unsupported."); + throw new BinaryFormatFailure( + "unsupported", + "ELF extended numbering is unsupported.", + ); if (phnum !== 0) { if (phentsize < (wide ? 56 : 32)) return fail("ELF program header width is invalid."); @@ -344,10 +363,9 @@ export const parseElfNotes = ( ); if (decoded.state === "malformed") return fail(decoded.message); if (decoded.state === "limit") - throw new ArtifactReaderFailure( - "limit", - `ELF source and compressed section names exceed the ${String(input.max_file_bytes)}-byte memory budget.`, - { boundary: "memory" }, + throw new ElfNoteSectionNameBudgetFailure( + BigInt(bytes.length) + decoded.requiredBytes, + input.max_file_bytes, ); if (decoded.state === "unsupported") { shstrtab = null; diff --git a/src/native/elf/ElfNotes.fixture.ts b/src/native/elf/ElfNotes.fixture.ts index 48e08cdf3c..e404473cb9 100644 --- a/src/native/elf/ElfNotes.fixture.ts +++ b/src/native/elf/ElfNotes.fixture.ts @@ -209,6 +209,46 @@ export const elfNoteFixture = (options: ElfNoteImageOptions = {}): Buffer => { return bytes; }; +/** Valid ELF64 section count stored in section header zero per the gABI. */ +export const elfExtendedSectionCountFixture = (): Buffer => { + const count = 0xff00; + const sectionOffset = 0x200; + const noteOffset = 0x100; + const namesOffset = 0x180; + const note = gnuBuildIdNote("12345678"); + const names = Buffer.from("\0.note.gnu.build-id\0.shstrtab\0", "latin1"); + const bytes = Buffer.alloc(sectionOffset + count * 64); + bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1, 1]); + bytes.writeUInt16LE(1, 16); + bytes.writeUInt16LE(62, 18); + bytes.writeUInt32LE(1, 20); + bytes.writeBigUInt64LE(BigInt(sectionOffset), 40); + bytes.writeUInt16LE(64, 52); + bytes.writeUInt16LE(56, 54); + bytes.writeUInt16LE(64, 58); + bytes.writeUInt16LE(0, 60); + bytes.writeUInt16LE(2, 62); + bytes.writeBigUInt64LE(BigInt(count), sectionOffset + 32); + + const noteSection = sectionOffset + 64; + bytes.writeUInt32LE(1, noteSection); + bytes.writeUInt32LE(7, noteSection + 4); + bytes.writeBigUInt64LE(BigInt(noteOffset), noteSection + 24); + bytes.writeBigUInt64LE(BigInt(note.length), noteSection + 32); + bytes.writeBigUInt64LE(4n, noteSection + 48); + + const namesSection = sectionOffset + 2 * 64; + bytes.writeUInt32LE(20, namesSection); + bytes.writeUInt32LE(3, namesSection + 4); + bytes.writeBigUInt64LE(BigInt(namesOffset), namesSection + 24); + bytes.writeBigUInt64LE(BigInt(names.length), namesSection + 32); + bytes.writeBigUInt64LE(1n, namesSection + 48); + + note.copy(bytes, noteOffset); + names.copy(bytes, namesOffset); + return bytes; +}; + /** ELF note fixture with a zlib-compressed section-name string table. */ export const compressedElfNoteFixture = ( bits: 32 | 64 = 64, diff --git a/src/native/elf/ElfNotesProvider.test.ts b/src/native/elf/ElfNotesProvider.test.ts index 22a733bc88..182e596f41 100644 --- a/src/native/elf/ElfNotesProvider.test.ts +++ b/src/native/elf/ElfNotesProvider.test.ts @@ -5,13 +5,17 @@ import { ElfNotesService } from "../../application/binaryDiagnostics/ElfNotesSer import { AnalysisInputError, AnalysisResourceConstraintError, + AnalysisUnsupportedTargetError, } from "../../domain/analysisErrorCore.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { elfNotesSchema } from "../../domain/native/elfNotes.js"; import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; import { + compressedElfNoteFixture, elfNote, elfNoteFixture, elfNoteTablePastEnd, + elfExtendedSectionCountFixture, } from "./ElfNotes.fixture.js"; import { ElfNotesProvider } from "./ElfNotesProvider.js"; @@ -48,6 +52,25 @@ describe("ELF note provider failures", () => { }); }); + it("reports valid extended section numbering as an unsupported target", async () => { + const root = await createTestTempDirectory( + "rea-elf-note-extended-numbering-", + ); + const path = join(root, "extended-count.elf"); + await writeFile(path, elfExtendedSectionCountFixture()); + const result = await inspect(path); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("Expected an unsupported target"); + expect(result.error).toBeInstanceOf(AnalysisUnsupportedTargetError); + expect(result.error.message).toContain( + "ELF extended numbering is unsupported", + ); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "unsupported_target", + message: expect.stringContaining("ELF extended numbering is unsupported"), + }); + }); + it("names the note text budget instead of host memory", async () => { const root = await createTestTempDirectory("rea-elf-note-text-budget-"); const path = join(root, "program"); @@ -88,4 +111,41 @@ describe("ELF note provider failures", () => { expect(projected.remediation.action).toContain("max_file_bytes"); expect(projected.remediation.action).not.toMatch(/available host memory/u); }); + + it("reports and admits compressed names through max_file_bytes", async () => { + const bytes = compressedElfNoteFixture(); + const sectionOffset = Number(bytes.readBigUInt64LE(40)); + const nameHeader = sectionOffset + bytes.readUInt16LE(62) * 64; + const namesOffset = Number(bytes.readBigUInt64LE(nameHeader + 24)); + const decodedNamesBytes = Number(bytes.readBigUInt64LE(namesOffset + 8)); + const minimumMaxFileBytes = bytes.length + decodedNamesBytes; + const root = await createTestTempDirectory("rea-elf-note-name-budget-"); + const path = join(root, "compressed-names.elf"); + await writeFile(path, bytes); + + const refused = await inspect(path, { max_file_bytes: bytes.length + 1 }); + expect(refused.ok).toBe(false); + if (refused.ok) throw new Error("Expected section-name admission refusal"); + expect(refused.error).toBeInstanceOf(AnalysisResourceConstraintError); + if (!(refused.error instanceof AnalysisResourceConstraintError)) + throw refused.error; + expect(refused.error).toMatchObject({ + resource: "memory", + reportedLimits: { + max_file_bytes: bytes.length + 1, + minimum_required_bytes: String(minimumMaxFileBytes), + }, + retryable: false, + }); + expect(projectAnalysisError(refused.error).remediation.action).toContain( + "max_file_bytes", + ); + + const admitted = await inspect(path, { + max_file_bytes: minimumMaxFileBytes, + }); + if (!admitted.ok) throw admitted.error; + const report = elfNotesSchema.parse(admitted.value.normalized_result); + expect(report.tables[0]?.name).toBe(".note.gnu.build-id"); + }); }); diff --git a/src/native/elf/ElfNotesProvider.ts b/src/native/elf/ElfNotesProvider.ts index 8f6bf46170..c27f8afb1c 100644 --- a/src/native/elf/ElfNotesProvider.ts +++ b/src/native/elf/ElfNotesProvider.ts @@ -16,6 +16,7 @@ import { AnalysisCancelledError, AnalysisInputError, AnalysisResourceConstraintError, + AnalysisUnsupportedTargetError, } from "../../domain/analysisErrorCore.js"; import { fixedAllowanceOptions } from "../../domain/fixedAllowanceRemediation.js"; import { @@ -26,7 +27,11 @@ import { import { err, ok } from "../../domain/result.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { ELF_NOTES_PROVIDER } from "../../application/InvestigationProviders.js"; -import { parseElfNotes } from "./ElfNoteParser.js"; +import { BinaryFormatFailure } from "../metadata/BinaryContainer.js"; +import { + ElfNoteSectionNameBudgetFailure, + parseElfNotes, +} from "./ElfNoteParser.js"; const OPERATION = "inspect_elf_notes"; @@ -84,7 +89,7 @@ export class ElfNotesProvider implements ElfNotesPort { } const NOTE_TEXT_BUDGET = - "The ELF note text, hash, scan, and decompression budget bounded by max_file_bytes"; + "The ELF note text, hash, and scan budget fixed by the admitted image length plus decoded section names"; const inputBudgetRemediation = ( boundary: "file-size" | "capacity", @@ -98,10 +103,47 @@ const inputBudgetRemediation = ( ? `The image exceeds the ${String(MAX_INSPECT_ELF_NOTE_ENTRIES)}-entry note-table ceiling this operation accepts; select a smaller artifact.` : `Raise max_entries above ${String(input.max_entries)} or select a smaller artifact; the limit is the caller-selected note entry budget.`; +const sectionNameBudgetRemediation = ( + cause: ElfNoteSectionNameBudgetFailure, +): string => + cause.minimumMaxFileBytes <= BigInt(MAX_INSPECT_ELF_NOTE_FILE_BYTES) + ? `Raise max_file_bytes to at least ${String(cause.minimumMaxFileBytes)} bytes to admit the source and decoded ELF section names.` + : `The source and decoded ELF section names require ${String(cause.minimumMaxFileBytes)} bytes, above this operation's ${String(MAX_INSPECT_ELF_NOTE_FILE_BYTES)}-byte maximum; select an image with a smaller section-name table.`; + const elfNotesFailurePrimary = ( cause: unknown, input: InspectElfNotesInput, ) => { + if (cause instanceof ElfNoteSectionNameBudgetFailure) + return new AnalysisResourceConstraintError( + OPERATION, + "memory", + cause.message, + { + max_file_bytes: cause.selectedMaxFileBytes, + minimum_required_bytes: cause.minimumMaxFileBytes.toString(), + }, + { + cause, + retryable: false, + remediationAction: sectionNameBudgetRemediation(cause), + }, + ); + if (cause instanceof BinaryFormatFailure) + return cause.kind === "unsupported" + ? new AnalysisUnsupportedTargetError( + OPERATION, + input.path, + cause.message, + { cause }, + ) + : new AnalysisInputError(OPERATION, { cause }, [ + { + path: ["path"], + reason: "invalid_format", + message: cause.message, + }, + ]); if (cause instanceof ArtifactReaderFailure && cause.reason === "integrity") return new AnalysisArtifactChangedError( OPERATION, @@ -119,7 +161,7 @@ const elfNotesFailurePrimary = ( if (cause.reason === "limit") { // Caller-selected read and entry budgets name the bound to raise. // The note text, hash, and scan ceiling equals the admitted image - // length, so host memory cannot raise it. + // length plus decoded section names, so host memory cannot raise it. const noteTextBudget = cause.boundary === "memory"; return new AnalysisResourceConstraintError( OPERATION, diff --git a/src/signals/SigrokRecordingProvider.ts b/src/signals/SigrokRecordingProvider.ts index 5e8364d3d3..f722fb09ca 100644 --- a/src/signals/SigrokRecordingProvider.ts +++ b/src/signals/SigrokRecordingProvider.ts @@ -20,7 +20,7 @@ import { import { randomUUID } from "node:crypto"; import { constants } from "node:fs"; import { access, realpath, writeFile } from "node:fs/promises"; -import { extname, isAbsolute, join } from "node:path"; +import { basename, isAbsolute, join } from "node:path"; import { createAnalysisExecution, type AnalysisExecution, @@ -177,10 +177,9 @@ export class SigrokRecordingProvider implements SignalRecordingPort { phase = "workspace"; root = await this.createRuntime(); this.#resources.retain(root, undefined); - // Keep the selected file's suffix as private parser metadata. sigrok's - // autodetection receives a filename, not just the copied bytes. - const snapshotName = - "path" in input ? `recording${extname(input.path)}` : "recording"; + // The private root is fresh; reuse the basename so its parser-relevant + // suffix is preserved without growing a valid source name. + const snapshotName = "path" in input ? basename(input.path) : "recording"; const snapshotPath = join(root.path, snapshotName); const execute = async ( args: readonly string[], diff --git a/src/signals/sigrokSelectorRejection.ts b/src/signals/sigrokSelectorRejection.ts index 850f6cbb22..3b54b9f50c 100644 --- a/src/signals/sigrokSelectorRejection.ts +++ b/src/signals/sigrokSelectorRejection.ts @@ -8,11 +8,14 @@ export type SigrokSelectorRejection = AnalysisInputIssue & { /** Caller selectors that a sigrok command loads, keyed by their input field. */ export type SigrokSelection = - | Pick + | Pick< + DecodeSignalRecordingInput, + "decoders" | "annotations" | "input_format" + > | Pick; const rejectedSelector = ( - field: "decoder" | "decoders" | "annotations", + field: "decoder" | "decoders" | "annotations" | "input_format", producerMessage: string, recovery: string, ): SigrokSelectorRejection => ({ @@ -27,6 +30,8 @@ const HELP_RECOVERY = "Omit decoder to list installed decoder IDs, optionally filtered by query, then request help for one of those IDs."; const CHANNEL_RECOVERY = "Set each decoder channel target to a channel present in the recording; inspect_signal_decoders shows the selected decoder's channel IDs."; +const INPUT_FORMAT_RECOVERY = + "Use a supported sigrok input format name, or omit input_format to let sigrok-cli autodetect the recording."; /** * sigrok-cli loads selected decoders before decoding or showing help, so an @@ -94,6 +99,14 @@ export const parseSigrokSelectorRejection = ( ? undefined : rejectedSelector("decoder", refusal, HELP_RECOVERY); } + const inputFormat = selection.input_format?.split(":")[0] ?? ""; + const inputFormatRefusal = `Error: unknown input module '${inputFormat}'.`; + if (inputFormat !== "" && lines.has(`cli: ${inputFormatRefusal}`)) + return rejectedSelector( + "input_format", + inputFormatRefusal, + INPUT_FORMAT_RECOVERY, + ); // sigrok-cli stacks decoders with "," and appends options with ":". for (const entry of selection.decoders.split(",")) { const refusal = unknownDecoderRefusal(lines, entry.split(":")[0] ?? ""); diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index 9c71dfd45f..586a09130e 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { jsonParts } from "../domain/jsonSerialization.js"; import { SQLITE_ROW_LIMIT_DEFAULT, + sqliteSchemaHasObjectDeclaration, sqliteSchemaScope, type InspectSqliteDatabaseInput, type SqliteDatabase, @@ -312,6 +313,25 @@ const readSchema = ( selectedName === undefined || selected === undefined ? undefined : shadowParentOf(selectedName, selected); + // sqlite_schema is a queryable ordinary table in PRAGMA inventory, but has + // no self-row in its stored declaration table. Keep these row-source facts + // separate from schema.tables rather than manufacturing a declaration. + const selectedRowSource = + input.table === undefined ? undefined : metadata.get(input.table); + if ( + input.table !== undefined && + !sqliteSchemaHasObjectDeclaration(input.table) && + selectedRowSource?.type === "table" + ) + selectedTable = { + name: selectedRowSource.name, + kind: selectedRowSource.type, + columns: readColumns(db, selectedRowSource.name, { + budget: new OutputBudget(), + encoding, + selectedColumns: input.columns, + }), + }; // NOCASE folds only ASCII letters, as SQLite does when it resolves the // table named by an index or trigger. Fold only for an ordinary table whose // rows can be selected; a selection refused later keeps the exact filter, diff --git a/tests/acceptance/applications/androidRuntimeComponentHints.test.ts b/tests/acceptance/applications/androidRuntimeComponentHints.test.ts index 7ff556e5b6..e00def7096 100644 --- a/tests/acceptance/applications/androidRuntimeComponentHints.test.ts +++ b/tests/acceptance/applications/androidRuntimeComponentHints.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -61,3 +62,78 @@ cliTest( } }, ); + +cliTest( + "projects nested APK v1 filename candidates as inventory metadata through CLI and MCP", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-android-signing-candidates-", + ); + const environment = { HOME: root, USERPROFILE: root }; + const { call } = await connectLocalToolsMcp(environment); + const entries = [ + "META-INF/CERT.SF", + "META-INF/CERT.EC", + "META-INF/keys/NESTED.SF", + "META-INF/keys/NESTED.EC", + "META-INF/keys/readme.txt", + "assets/not-signing.SF", + ]; + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("AndroidManifest.xml", new TextReader("fixture")); + for (const entry of entries) + await writer.add(entry, new TextReader(`inventory:${entry}`)); + const path = join(root, "signing-candidates.apk"); + await writeFile(path, await writer.close()); + + const inspected = await cli.run({ + arguments: ["inspect-artifact", path, "--json"], + environment, + }); + expect(inspected.exitCode, inspected.stderr).toBe(0); + const input = { inventory_evidence: [parseEvidence(inspected.json)] }; + const projected = await cli.run({ + arguments: [ + "project-android-application-graph", + JSON.stringify(input), + "--json", + ], + environment, + }); + expect(projected.exitCode, projected.stderr).toBe(0); + const cliEvidence = parseEvidence(projected.json); + const cliGraph = androidApplicationProjectionResultSchema.parse( + cliEvidence.normalized_result, + ); + const mcp = await call("project_android_application_graph", input); + expect(mcp.isError).not.toBe(true); + const mcpEvidence = parseEvidence(mcp.structuredContent); + const mcpGraph = androidApplicationProjectionResultSchema.parse( + mcpEvidence.normalized_result, + ); + + const expectedPaths = [ + "META-INF/CERT.EC", + "META-INF/CERT.SF", + "META-INF/keys/NESTED.EC", + "META-INF/keys/NESTED.SF", + ]; + expect(cliGraph.components.signing.map(({ path }) => path).sort()).toEqual( + expectedPaths, + ); + expect(mcpGraph.components.signing).toEqual(cliGraph.components.signing); + for (const component of cliGraph.components.signing) + expect(component.sha256).toBe( + createHash("sha256") + .update(`inventory:${component.path}`) + .digest("hex"), + ); + expect(JSON.stringify(cliGraph.components.signing)).not.toContain( + "readme.txt", + ); + expect(JSON.stringify(cliGraph.components.signing)).not.toContain( + "not-signing.SF", + ); + expect(mcpEvidence).toEqual(cliEvidence); + }, +); diff --git a/tests/acceptance/elfNotesCli.test.ts b/tests/acceptance/elfNotesCli.test.ts index 48aeab98c1..8d9b734d5e 100644 --- a/tests/acceptance/elfNotesCli.test.ts +++ b/tests/acceptance/elfNotesCli.test.ts @@ -1,9 +1,10 @@ -import { writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; import { toolContract } from "../../src/contracts/toolContracts.js"; import { compressedElfNoteFixture, + elfExtendedSectionCountFixture, elfNoteFixture, } from "../../src/native/elf/ElfNotes.fixture.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; @@ -83,3 +84,22 @@ cliTest("reports a truncated ELF as invalid input", async ({ cli }) => { details: { operation: "inspect_elf_notes" }, }); }); + +cliTest( + "reports valid extended numbering as unsupported without mutation", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-elf-notes-extended-cli-"); + const path = join(root, "extended-count.elf"); + await writeFile(path, elfExtendedSectionCountFixture()); + const before = await readFile(path); + const result = await cli.run({ + arguments: ["inspect-elf-notes", path, "--json"], + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "unsupported_target", + message: expect.stringContaining("ELF extended numbering is unsupported"), + }); + expect(await readFile(path)).toEqual(before); + }, +); diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts index 393ff64a05..bf074dcdff 100644 --- a/tests/acceptance/pyinstallerArchive.test.ts +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -216,6 +216,50 @@ it("lists CArchive members and the Python version through inspect-artifact", asy }); }); +it("classifies a PyInstaller PE carrier when its signature straddles the probe prefix", async () => { + const directory = await createTestTempDirectory("rea-pyinstaller-far-pe-"); + const path = join(directory, "hello.exe"); + await writeFile( + path, + cArchiveExecutable({ + carrier: "pe", + peHeaderOffset: 8_190, + layout: "2.1", + pythonVersion: 314, + pythonLibrary: "libpython3.14.dylib", + members: [ + { + name: "healthy.txt", + typeCode: "x", + payload: Buffer.from("healthy sibling"), + compress: false, + }, + ], + }), + ); + const result = await execute(process.execPath, [ + cli, + "inspect-artifact", + path, + "--json", + ]); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(JSON.parse(result.stdout)).normalized_result, + ); + expect(inspection.subject.root_format).toBe("pyinstaller"); + const inventory = inspection.substeps[0]?.evidence.normalized_result; + expect(inventory?.manifest.root_format).toBe("pyinstaller"); + expect(inventory?.coverage.status).toBe("complete"); + expect( + inventory?.occurrences.find( + ({ logical_path }) => logical_path === "healthy.txt", + ), + ).toMatchObject({ + hash_status: "verified", + format_metadata: { source_name: "healthy.txt" }, + }); +}); + it("extracts CArchive members through extract-artifact", async () => { const path = await onefile(); const result = await execute(process.execPath, [ diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index 5ce76ea826..5323b2f549 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -891,3 +891,65 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( expect(await readFile(path)).toEqual(before); }, ); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "reads sqlite_schema rows through CLI and MCP in every schema scope", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-schema-table-"); + const path = join(root, "selected.db"); + const writer = new DatabaseSync(path); + try { + writer.exec("CREATE TABLE records(id INTEGER)"); + } finally { + writer.close(); + } + const original = await readFile(path); + const { call } = await connectLocalToolsMcp(); + for (const schema_scope of [ + "inventory", + "selected_table", + "database", + ] as const) { + const result = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "sqlite_schema", + "--column", + "name", + "--column", + "type", + "--schema-scope", + schema_scope, + "--json", + ], + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(0); + expect(result.json).toMatchObject({ + normalized_result: { + schema: { scope: schema_scope }, + rows: { + table: "sqlite_schema", + columns: ["name", "type"], + values: [ + [ + { type: "text", value: "records" }, + { type: "text", value: "table" }, + ], + ], + }, + }, + }); + const response = await call("inspect_sqlite_database", { + path, + table: "sqlite_schema", + columns: ["name", "type"], + schema_scope, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toEqual(result.json); + } + expect(await readFile(path)).toEqual(original); + }, +); diff --git a/tests/boundary/filesystem/android/application.test.ts b/tests/boundary/filesystem/android/application.test.ts index db405705f7..713e30aa55 100644 --- a/tests/boundary/filesystem/android/application.test.ts +++ b/tests/boundary/filesystem/android/application.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -175,6 +176,59 @@ describe("Android application projection", () => { }); }); +it("projects APK v1 metadata candidates at any META-INF depth", async () => { + const root = await createTestTempDirectory("rea-android-v1-metadata-"); + const path = join(root, "NestedSigning.apk"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + const names = [ + "META-INF/CERT.SF", + "META-INF/CERT.EC", + "META-INF/keys/CERT.SF", + "META-INF/keys/CERT.EC", + "META-INF/keys/readme.txt", + "META-INF/keys/MANIFEST.MF", + "assets/not-signing.SF", + "meta-inf/lower.SF", + ]; + for (const entry of names) + await writer.add(entry, new TextReader(`content:${entry}`)); + await writeFile(path, await writer.close()); + + const inventory = parseEvidence( + await runProviderAnalysis(path, "inventory_artifact", {}), + ); + const projection = androidApplicationProjectionResultSchema.parse( + requireSuccessfulProjection( + projectAndroidApplicationEvidence({ inventory_evidence: [inventory] }), + ).normalized_result, + ); + + expect(projection.components.signing.map(({ path }) => path).sort()).toEqual([ + "META-INF/CERT.EC", + "META-INF/CERT.SF", + "META-INF/keys/CERT.EC", + "META-INF/keys/CERT.SF", + ]); + for (const name of [ + "META-INF/CERT.SF", + "META-INF/CERT.EC", + "META-INF/keys/CERT.SF", + "META-INF/keys/CERT.EC", + ]) { + const component = projection.components.signing.find( + ({ path }) => path === name, + ); + expect(component).toMatchObject({ + path: name, + artifact_id: expect.stringMatching(/^art_/u), + sha256: createHash("sha256").update(`content:${name}`).digest("hex"), + }); + } + expect(projection.limitations).toContain( + "Manifest, resource, signing, and bytecode semantics require a dedicated Android provider; this projection reports exact inventory paths and hashes only.", + ); +}); + describe("APK family without a root manifest", () => { const missingManifest = "No root AndroidManifest.xml occurrence was observed. The APK family comes from the .apk suffix of a ZIP archive, so these bytes may not be an APK."; diff --git a/tests/boundary/filesystem/artifactFatContentNode.test.ts b/tests/boundary/filesystem/artifactFatContentNode.test.ts new file mode 100644 index 0000000000..f49b7b1870 --- /dev/null +++ b/tests/boundary/filesystem/artifactFatContentNode.test.ts @@ -0,0 +1,79 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect, it } from "vitest"; + +import { inventoryArtifact } from "../../fixtures/artifactInventory.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { buildGodotPck } from "../../fixtures/godotPck.js"; + +const fatMachO = (sliceOffset: number): Buffer => { + const thin = Buffer.alloc(48); + thin.writeUInt32LE(0xfeedfacf, 0); + thin.writeUInt32LE(0x0100000c, 4); + thin.writeUInt32LE(0, 8); + thin.writeUInt32LE(2, 12); + + const bytes = Buffer.alloc(sliceOffset + thin.length); + bytes.writeUInt32BE(0xcafebabe, 0); + bytes.writeUInt32BE(1, 4); + bytes.writeUInt32BE(0x0100000c, 8); + bytes.writeUInt32BE(0, 12); + bytes.writeUInt32BE(sliceOffset, 16); + bytes.writeUInt32BE(thin.length, 20); + bytes.writeUInt32BE(Math.log2(sliceOffset), 24); + thin.copy(bytes, sliceOffset); + return bytes; +}; + +const zipWithMember = async (bytes: Buffer): Promise => { + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("native.fat", new Uint8ArrayReader(bytes), { level: 0 }); + return Buffer.from(await writer.close()); +}; + +for (const sliceOffset of [4_096, 16_384]) { + for (const container of ["zip", "pck"] as const) { + it(`retains a FAT slice header at offset ${sliceOffset} in ${container} member nodes`, async () => { + const root = await createTestTempDirectory("rea-fat-member-node-"); + const bytes = fatMachO(sliceOffset); + const directPath = join(root, "direct.fat"); + await writeFile(directPath, bytes); + const direct = await inventoryArtifact(directPath); + const directNode = direct.nodes.find( + ({ artifact_id }) => artifact_id === direct.manifest.root_artifact_id, + ); + expect(directNode?.format).toBe("mach-o-universal"); + + const containerPath = join(root, `fixture.${container}`); + await writeFile( + containerPath, + container === "zip" + ? await zipWithMember(bytes) + : buildGodotPck(3, [{ path: "native.fat", data: bytes }]), + ); + const inventory = await inventoryArtifact(containerPath); + const occurrence = inventory.occurrences.find( + ({ logical_path }) => logical_path === "native.fat", + ); + expect(occurrence).toMatchObject({ + artifact_format: "mach-o-universal", + hash_status: "verified", + artifact_id: direct.manifest.root_artifact_id, + }); + const node = inventory.nodes.find( + ({ artifact_id }) => artifact_id === occurrence?.artifact_id, + ); + expect(node).toMatchObject({ + artifact_id: direct.manifest.root_artifact_id, + sha256: direct.manifest.root_sha256, + format: "mach-o-universal", + }); + expect(inventory.coverage).toEqual({ + scope: "container-entries", + status: "complete", + }); + }); + } +} diff --git a/tests/boundary/filesystem/artifactMzInventory.test.ts b/tests/boundary/filesystem/artifactMzInventory.test.ts index 675069f748..18cdcd2e17 100644 --- a/tests/boundary/filesystem/artifactMzInventory.test.ts +++ b/tests/boundary/filesystem/artifactMzInventory.test.ts @@ -1,11 +1,15 @@ import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { describe, expect, it } from "vitest"; import { classifyArtifactBytesAtPath } from "../../fixtures/artifactClassification.js"; import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../../../src/artifacts/ArtifactClassificationProbe.js"; -import { scanCanonicalArtifactInventory } from "../../fixtures/artifactInventory.js"; +import { + inventoryArtifact, + scanCanonicalArtifactInventory, +} from "../../fixtures/artifactInventory.js"; import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -82,6 +86,30 @@ describe("MZ artifact classification", () => { expect.arrayContaining(["dos-mz", "pe"]), ); }); + + it("retains a PE signature straddling the bounded prefix in ZIP member evidence", async () => { + const root = await createTestTempDirectory("rea-mz-far-pe-"); + const bytes = peFixture(ARTIFACT_CLASSIFICATION_PREFIX_BYTES - 2); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("native.exe", new Uint8ArrayReader(bytes), { level: 0 }); + const path = join(root, "fixture.zip"); + await writeFile(path, Buffer.from(await writer.close())); + + const inventory = await inventoryArtifact(path); + const occurrence = inventory.occurrences.find( + ({ logical_path }) => logical_path === "native.exe", + ); + expect(occurrence).toMatchObject({ + artifact_format: "pe", + hash_status: "verified", + }); + expect( + inventory.nodes.find( + ({ artifact_id }) => artifact_id === occurrence?.artifact_id, + ), + ).toMatchObject({ format: "pe", size: bytes.length }); + expect(inventory.coverage.status).toBe("complete"); + }); }); const dosFixture = (): Buffer => { diff --git a/tests/boundary/mcp/elfNotesMcp.test.ts b/tests/boundary/mcp/elfNotesMcp.test.ts index 7bea4702cc..1156308215 100644 --- a/tests/boundary/mcp/elfNotesMcp.test.ts +++ b/tests/boundary/mcp/elfNotesMcp.test.ts @@ -1,11 +1,14 @@ -import { writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; -import { elfNoteFixture } from "../../../src/native/elf/ElfNotes.fixture.js"; +import { + elfExtendedSectionCountFixture, + elfNoteFixture, +} from "../../../src/native/elf/ElfNotes.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { mcpTest, type TestMcp } from "../../support/mcp/mcpFixture.js"; @@ -68,3 +71,30 @@ mcpTest("requires an absolute path for inspect_elf_notes", async ({ mcp }) => { await session.close(); } }); + +mcpTest( + "reports valid extended numbering as unsupported without mutation", + async ({ mcp }) => { + const { client, session } = await connect(mcp); + try { + const root = await createTestTempDirectory("rea-elf-notes-extended-mcp-"); + const path = join(root, "extended-count.elf"); + await writeFile(path, elfExtendedSectionCountFixture()); + const before = await readFile(path); + const response = await client.callTool({ + name: "inspect_elf_notes", + arguments: { path }, + }); + expect(response.isError).toBe(true); + expect(parseMcpToolError(response).error).toMatchObject({ + code: "unsupported_target", + message: expect.stringContaining( + "ELF extended numbering is unsupported", + ), + }); + expect(await readFile(path)).toEqual(before); + } finally { + await session.close(); + } + }, +); diff --git a/tests/boundary/signals/recordingProvider.test.ts b/tests/boundary/signals/recordingProvider.test.ts index 759ea475e7..4c1659f26d 100644 --- a/tests/boundary/signals/recordingProvider.test.ts +++ b/tests/boundary/signals/recordingProvider.test.ts @@ -7,7 +7,7 @@ import { projectAnalysisError } from "../../../src/domain/analysisErrorProjectio import { OwnedCommandFailure } from "../../../src/process/OwnedCommand.js"; import { it, expect, onTestFinished } from "vitest"; import { writeFile, readdir, chmod, symlink } from "node:fs/promises"; -import { extname, join } from "node:path"; +import { basename, join } from "node:path"; import { readStableArtifact } from "../../../src/artifacts/readStableArtifact.js"; import { SigrokRecordingProvider } from "../../../src/signals/SigrokRecordingProvider.js"; import { SignalRecordingService } from "../../../src/application/signals/SignalRecordingService.js"; @@ -26,22 +26,11 @@ import { Cli } from "incur"; import { registerSignalCommands } from "../../../src/cli/signalCommands.js"; import { silentLogger } from "../../../src/logger.js"; -// A command-boundary simulator: real process ownership/filesystem, not real sigrok coverage. -const fixture = async (mode = "ok", failCleanup = false) => { - const directory = await createTestTempDirectory("rea-signal-test-"); - const path = join( - directory, - mode === "csv-autodetect" ? "capture.csv" : "recorded bytes.vcd", - ); - const snapshotName = `recording${extname(path)}`; - await writeFile( - path, - mode === "csv-autodetect" ? "CLK,MOSI\n0,0\n1,0\n" : "recording bytes", - ); - const script = join(directory, "producer.mjs"); - await writeFile( - script, - `import {readFileSync, writeFileSync, chmodSync, truncateSync} from 'node:fs'; +const producerSource = (mode: string, snapshotName: string): string => + `import {readFileSync, writeFileSync, chmodSync, truncateSync} from 'node:fs'; +const inputFileArgument = process.argv.find((argument) => argument.startsWith('--input-file=')); +const snapshotName = inputFileArgument?.slice('--input-file='.length); +if (inputFileArgument !== undefined && inputFileArgument !== ${JSON.stringify(`--input-file=${snapshotName}`)}) throw Error('wrong snapshot name'); if (process.argv.includes('--version')) { if (${JSON.stringify(mode)} === 'invalid-version-stdout') process.stdout.write(Buffer.concat([Buffer.from('sigrok-cli 0.7.2\\nlibsigrokdecode 0.5.3\\n'), Buffer.from([0xff])])); else console.log(${JSON.stringify(mode)} === 'bad-version' ? 'sigrok-cli 0.7.1' : 'sigrok-cli 0.7.2\\nlibsigrokdecode 0.5.3'); @@ -50,6 +39,7 @@ if (process.argv.includes('--version')) { } else if (${JSON.stringify(mode)} === 'bad-version') console.log('unexpected version fallback'); else if (${JSON.stringify(mode)} === 'nonzero') { console.error('invalid decoder'); process.exitCode = 1; } +else if (${JSON.stringify(mode)} === 'unknown-input-format') { if (!process.argv.includes('--input-format=missingformat')) console.error('fixture: selected input format was not forwarded'); console.error("cli: Error: unknown input module 'missingformat'."); process.exitCode = 1; } else if (${JSON.stringify(mode)} === 'unknown-decoder') { console.error("srd: ModuleNotFoundError: Failed to load decoder nosuchdecoder: import by name failed: No module named 'nosuchdecoder'\\nFailed to load protocol decoder nosuchdecoder."); process.exitCode = 1; } else if (${JSON.stringify(mode)} === 'broken-decoder') { console.error("srd: ModuleNotFoundError: Failed to load decoder uart: import by name failed: No module named 'serial'\\nFailed to load protocol decoder uart."); process.exitCode = 1; } else if (${JSON.stringify(mode)} === 'unknown-annotation') { console.error("Annotation 'nosuchclass' not found for protocol decoder 'uart'."); process.exitCode = 1; } @@ -62,18 +52,38 @@ else if (${JSON.stringify(mode)} === 'invalid-stderr') { process.stdout.write('1 else if (${JSON.stringify(mode)} === 'literal-replacement') process.stdout.write('10-20 uart-1: �\\n'); else if (${JSON.stringify(mode)} === 'invalid-catalog-stdout' && process.argv.includes('--list-supported')) process.stdout.write(Buffer.concat([Buffer.from('Supported protocol decoders:\\n uart '), Buffer.from([0xff])])); else if (${JSON.stringify(mode)} === 'invalid-help-stdout' && process.argv.includes('--show')) process.stdout.write(Buffer.concat([Buffer.from('ID: uart\\nOptions:\\n'), Buffer.from([0xfe])])); -else if (${JSON.stringify(mode)} === 'malformed-copy') { chmodSync(${JSON.stringify(snapshotName)}, 0o600); writeFileSync(${JSON.stringify(snapshotName)}, 'changed'); } -else if (${JSON.stringify(mode)} === 'oversized-copy') { chmodSync(${JSON.stringify(snapshotName)}, 0o600); truncateSync(${JSON.stringify(snapshotName)}, 32 * 1024 * 1024 + 1); } +else if (${JSON.stringify(mode)} === 'malformed-copy') { chmodSync(snapshotName, 0o600); writeFileSync(snapshotName, 'changed'); } +else if (${JSON.stringify(mode)} === 'oversized-copy') { chmodSync(snapshotName, 0o600); truncateSync(snapshotName, 32 * 1024 * 1024 + 1); } else if (${JSON.stringify(mode)} === 'line-budget') process.stdout.write('ID: uart' + '\\n'.repeat(65536)); else if (${JSON.stringify(mode)} === 'overflow') process.stdout.write('x'.repeat(5 * 1024 * 1024)); else if (${JSON.stringify(mode)} === 'wait') { console.log('ready'); setInterval(() => {}, 1000); } else if (${JSON.stringify(mode)} === 'many') process.stdout.write(Array.from({length:90},(_,i)=>'line '+(i+1)+'\\r\\n').join('')); else if (process.argv.includes('--list-supported')) { console.log('Supported hardware drivers:\\n ignored Device\\nSupported protocol decoders:\\n uart Universal Asynchronous Receiver/Transmitter\\n i2c Inter-Integrated Circuit\\n'); console.error('catalog warning'); } else if (process.argv.includes('--show')) console.log('ID: uart\\nOptions:\\n- baudrate: Baud rate (default 115200)\\n- parity: Parity'); -else if (${JSON.stringify(mode)} === 'csv-autodetect') { if (!process.argv.includes('--input-file=recording.csv') || readFileSync(${JSON.stringify(snapshotName)}, 'utf8') !== 'CLK,MOSI\\n0,0\\n1,0\\n') throw Error('CSV suffix or copy bytes lost'); console.log('10-20 spi-1: 1'); } -else { if(readFileSync(${JSON.stringify(snapshotName)}, 'utf8') !== 'recording bytes') throw Error('wrong copy'); console.log('10-20 uart-1: 42'); console.error('producer warning'); } -`, +else if (${JSON.stringify(mode)} === 'csv-autodetect') { if (!process.argv.includes(${JSON.stringify(`--input-file=${snapshotName}`)}) || readFileSync(snapshotName, 'utf8') !== 'CLK,MOSI\\n0,0\\n1,0\\n') throw Error('CSV suffix or copy bytes lost'); console.log('10-20 spi-1: 1'); } +else if (${JSON.stringify(mode)} === 'long-basename') { if (!process.argv.includes('--input-format=vcd') || readFileSync(snapshotName, 'utf8') !== 'recording bytes') throw Error('long source basename or explicit format was lost'); console.log('10-20 uart-1: 42'); } +else { if(readFileSync(snapshotName, 'utf8') !== 'recording bytes') throw Error('wrong copy'); console.log('10-20 uart-1: 42'); console.error('producer warning'); } +`; + +// A command-boundary simulator: real process ownership/filesystem, not real sigrok coverage. +const fixture = async ( + mode = "ok", + failCleanup = false, + recordingName?: string, +) => { + const directory = await createTestTempDirectory("rea-signal-test-"); + const path = join( + directory, + recordingName ?? + (mode === "csv-autodetect" ? "capture.csv" : "recorded bytes.vcd"), ); + const snapshotName = basename(path); + await writeFile( + path, + mode === "csv-autodetect" ? "CLK,MOSI\n0,0\n1,0\n" : "recording bytes", + ); + const script = join(directory, "producer.mjs"); + await writeFile(script, producerSource(mode, snapshotName)); await chmod(script, 0o700); const configuredCommand = mode === "symlink" ? join(directory, "configured-tool") : script; @@ -205,7 +215,7 @@ it("pages literal output without changing decoder arguments or normalizing text" expect(report.output_window.returned_lines).toBe(count); expect(report.tool.arguments).toEqual([ "--dont-scan", - "--input-file=recording.vcd", + `--input-file=${basename(f.path)}`, "--protocol-decoders=uart", "--protocol-decoder-samplenum", ]); @@ -248,7 +258,7 @@ it("decodes an immutable copy, preserves options and warnings, and releases its expect(report.selection.input_format).toBeNull(); expect(report.tool.arguments).toEqual([ "--dont-scan", - "--input-file=recording.vcd", + "--input-file=recorded bytes.vcd", "--protocol-decoders=uart:rx=RX:baudrate=9600", "--protocol-decoder-samplenum", "--protocol-decoder-annotations=uart=rx-data", @@ -259,24 +269,49 @@ it("decodes an immutable copy, preserves options and warnings, and releases its ]); }); -it("preserves a CSV filename hint through the private autodetection copy", async () => { - const f = await fixture("csv-autodetect"); +it.each(["capture.csv", ".capture.csv", ".csv"])( + "preserves the CSV filename hint from %s through the private copy", + async (recordingName) => { + const f = await fixture("csv-autodetect", false, recordingName); + const result = await f.service.decode({ + path: f.path, + decoders: "spi:clk=CLK:mosi=MOSI", + }); + if (!result.ok) throw result.error; + const report = signalRecordingSchema.parse(result.value.normalized_result); + expect(report.selection).toMatchObject({ + path: f.path, + input_format: null, + }); + expect(report.tool.arguments).toEqual([ + "--dont-scan", + `--input-file=${recordingName}`, + "--protocol-decoders=spi:clk=CLK:mosi=MOSI", + "--protocol-decoder-samplenum", + ]); + }, +); + +it("stages a long source basename without expanding it before decoding", async () => { + const recordingName = `a.${"x".repeat(246)}`; + expect(Buffer.byteLength(recordingName)).toBe(248); + const f = await fixture("long-basename", false, recordingName); const result = await f.service.decode({ path: f.path, - decoders: "spi:clk=CLK:mosi=MOSI", + decoders: "uart", + input_format: "vcd", }); if (!result.ok) throw result.error; + const report = signalRecordingSchema.parse(result.value.normalized_result); - expect(report.selection).toMatchObject({ - path: f.path, - input_format: null, - }); - expect(report.tool.arguments).toEqual([ - "--dont-scan", - "--input-file=recording.csv", - "--protocol-decoders=spi:clk=CLK:mosi=MOSI", - "--protocol-decoder-samplenum", - ]); + expect(report.artifact.path).toBe(f.path); + expect(report.selection.input_format).toBe("vcd"); + expect(report.annotations_text).toBe("10-20 uart-1: 42\n"); + expect(report.tool.arguments).toContain(`--input-file=${recordingName}`); + expect(f.calls()).toBe(2); + expect( + (await readdir(f.directory)).filter((name) => name.startsWith("decoder-")), + ).toEqual([]); }); it.each([ @@ -311,23 +346,32 @@ it.each([ { decoders: "uart:rx=RX,nosuchdecoder" }, "decoders", "Failed to load protocol decoder nosuchdecoder.", + "inspect_signal_decoders", ], [ "unknown-annotation", { decoders: "uart", annotations: "uart=rx-data:nosuchclass" }, "annotations", "Annotation 'nosuchclass' not found for protocol decoder 'uart'.", + "inspect_signal_decoders", + ], + [ + "unknown-input-format", + { decoders: "uart", input_format: "missingformat" }, + "input_format", + "Error: unknown input module 'missingformat'.", + "omit input_format", ], ] as const)( "reports sigrok's %s refusal as invalid caller input", - async (mode, selection, field, refusal) => { + async (mode, selection, field, refusal, recovery) => { const f = await fixture(mode); const result = await f.service.decode({ path: f.path, ...selection }); if (result.ok) throw new Error("Expected selector refusal"); expect(result.error._tag).toBe("AnalysisInputError"); expect(projectAnalysisError(result.error)).toMatchObject({ code: "invalid_request", - message: expect.stringContaining("inspect_signal_decoders"), + message: expect.stringContaining(recovery), details: { issues: [ { @@ -351,6 +395,95 @@ it.each([ }, ); +it("reports an unknown selected input format through CLI and MCP", async () => { + const f = await fixture("unknown-input-format"); + const refusal = "Error: unknown input module 'missingformat'."; + const issueMessage = + `sigrok-cli rejected the input_format selector: ${refusal} ` + + "Use a supported sigrok input format name, or omit input_format to let sigrok-cli autodetect the recording."; + const harness = await createApplicationMcpHarness({ + serverOptions: { environment: {}, signalRecording: f.service }, + }); + onTestFinished(harness.close); + const { error } = parseMcpToolError( + await harness.client.callTool({ + name: "decode_signal_recording", + arguments: { + path: f.path, + decoders: "uart", + input_format: "missingformat", + }, + }), + ); + expect(error).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + message: expect.stringContaining(issueMessage), + details: { + issues: [ + { + path: ["input_format"], + reason: "invalid_value", + message: expect.stringContaining(issueMessage), + }, + ], + captured_output: { + exit_code: 1, + stderr: expect.stringContaining(refusal), + }, + }, + }); + expect(JSON.stringify(error)).not.toContain( + "fixture: selected input format was not forwarded", + ); + + const cli = Cli.create("signal-input-format-refusal-test", { sync: false }); + registerSignalCommands(cli, silentLogger, {}, f.service); + let stdout = ""; + await cli.serve( + [ + "decode-signal-recording", + f.path, + "--decoders", + "uart", + "--input_format", + "missingformat", + "--format", + "json", + ], + { + stdout: (text) => { + stdout += text; + }, + exit: () => {}, + }, + ); + expect(JSON.parse(stdout)).toMatchObject({ + code: "invalid_request", + message: expect.stringContaining(issueMessage), + details: { + issues: [ + { + path: ["input_format"], + reason: "invalid_value", + message: expect.stringContaining(issueMessage), + }, + ], + captured_output: { + exit_code: 1, + stderr: expect.stringContaining(refusal), + }, + }, + }); + expect(stdout).not.toContain( + "fixture: selected input format was not forwarded", + ); + expect(f.calls()).toBe(4); + expect( + (await readdir(f.directory)).filter((name) => name.startsWith("decoder-")), + ).toEqual([]); +}); + it("projects sigrok's zero-exit missing-channel refusal through CLI and MCP with partial output", async () => { const f = await fixture("unknown-channel"); const refusal = 'No channel with name "NONEXISTENT" found.'; @@ -620,6 +753,8 @@ it.each([ ["unknown-decoder", { decoder: "uart" }], ["unknown-decoder", {}], ["unknown-annotation", { decoders: "uart", annotations: "uart=rx-data" }], + ["unknown-input-format", { decoders: "uart", input_format: null }], + ["unknown-input-format", { decoders: "uart", input_format: "vcd" }], ] as const)( "keeps %s outside caller selectors %j as a provider failure", async (mode, selection) => { @@ -813,7 +948,7 @@ it.each(["oversized-copy", "oversized-tool"])( path: mode === "oversized-tool" ? join(f.directory, "producer.mjs") - : expect.stringMatching(/decoder-.*[/\\]recording\.vcd$/u), + : expect.stringMatching(/decoder-.*[/\\]recorded bytes\.vcd$/u), }); expect( (await readdir(f.directory)).filter((name) => diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts index a22df47761..db19bdf80f 100644 --- a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -282,6 +282,37 @@ it("keeps unrelated catalog names outside the selected-table reply budget", asyn ).toThrowError(expect.objectContaining({ reason: "output-limit" })); }); +it.each(["inventory", "selected_table", "database"] as const)( + "reads sqlite_schema rows using PRAGMA row-source facts in %s scope", + async (schema_scope) => { + const path = await createDatabase( + "CREATE TABLE records(id INTEGER); INSERT INTO records VALUES (1);", + ); + const result = inspectSqliteDatabaseSnapshot(path, { + path, + table: "sqlite_schema", + columns: ["name", "type"], + schema_scope, + }); + + expect(result.rows).toMatchObject({ + table: "sqlite_schema", + columns: ["name", "type"], + values: [ + [ + { type: "text", value: "records" }, + { type: "text", value: "table" }, + ], + ], + }); + expect(result.schema.tables).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "sqlite_schema" }), + ]), + ); + }, +); + it("rejects malformed schema bytes instead of returning replacement text", async () => { const path = await createDatabase( "CREATE TABLE records(value TEXT /*schema-marker*/)", diff --git a/tests/fixtures/artifactClassification.ts b/tests/fixtures/artifactClassification.ts index f5e6617c69..abeea2d7ff 100644 --- a/tests/fixtures/artifactClassification.ts +++ b/tests/fixtures/artifactClassification.ts @@ -9,9 +9,10 @@ export const classifyArtifactBytesAtPath = ( ) => { const probe = createAsarHeaderSignatureProbe(); probe.write(bytes); - return classifyArtifactContent( + const { kind, format } = classifyArtifactContent( path, { prefix: bytes, asarHeader: probe.finish(fileSize) }, fileSize, ); + return { kind, format }; }; diff --git a/tests/fixtures/godotPck.ts b/tests/fixtures/godotPck.ts index 38dd2c5f0b..de6b60f14c 100644 --- a/tests/fixtures/godotPck.ts +++ b/tests/fixtures/godotPck.ts @@ -2,7 +2,7 @@ import { createHash } from "node:crypto"; export interface GodotPckMember { readonly path: string; - readonly data: string; + readonly data: string | Uint8Array; readonly flags?: number; readonly md5?: Buffer; readonly size?: bigint; @@ -35,10 +35,12 @@ export const buildGodotPck = ( ): Buffer => { const data: Buffer[] = []; const offsets: number[] = []; + const memberBytes: Buffer[] = []; let dataBytes = 0; for (const member of members) { offsets.push(dataBytes); const bytes = pad4(Buffer.from(member.data)); + memberBytes.push(Buffer.from(member.data)); data.push(bytes); dataBytes += bytes.length; } @@ -50,8 +52,11 @@ export const buildGodotPck = ( pckU32(path.length), path, pckU64(member.offset ?? BigInt(offsets[index] ?? 0)), - pckU64(member.size ?? BigInt(Buffer.byteLength(member.data))), - member.md5 ?? createHash("md5").update(member.data).digest(), + pckU64(member.size ?? BigInt(memberBytes[index]?.length ?? 0)), + member.md5 ?? + createHash("md5") + .update(memberBytes[index] ?? "") + .digest(), pckU32(member.flags ?? 0), ]; }),