From ae8fb57ef0789f90404f6c3d3e08ac31cdd9a013 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:49:34 -0500 Subject: [PATCH] fix(managed): preserve duplicate GUID stream ambiguity (#12217) * fix(managed): preserve duplicate GUID stream ambiguity * fix(managed): preserve ambiguous metadata stream declarations (#12226) Use one stream declaration owner for heap and table diagnostics. Keep user strings and row-layout uncertainty separate from independent PE/CLI observations. * test(managed): separate heap entries from cache behavior * fix(managed): preserve duplicate blob heap ambiguity (#12258) Resolve #Blob in the shared stream layout. Preserve nil references and independent facts without emitting unsupported keys, signatures or hashes. --- docs/guides/managed/managed-code-analysis.md | 23 +- .../managed/managedMemberInspectorCore.ts | 6 +- .../managed/managedMetadataHeaps.test.ts | 117 ++++-- src/domain/managed/managedMetadataHeaps.ts | 72 ++-- .../managed/managedMetadataInventory.ts | 40 +- .../managed/managedMetadataInventoryRows.ts | 92 ++-- src/domain/managed/managedMetadataLayout.ts | 183 ++++++-- src/domain/managed/managedReaderFailure.ts | 6 +- src/dotnet/ManagedFieldConstants.ts | 11 +- src/dotnet/ManagedHeapStreamAmbiguity.test.ts | 397 ++++++++++++++++++ src/dotnet/ManagedMemberInspector.test.ts | 20 +- src/dotnet/ManagedMemberRows.ts | 29 +- src/dotnet/ManagedMethodBodyReader.ts | 14 +- src/dotnet/ManagedMethodSpecs.ts | 9 +- src/dotnet/ManagedNativeBoundaryInspector.ts | 2 +- src/dotnet/ManagedPe.fixture.ts | 45 +- src/dotnet/ManagedTableStreamConflict.test.ts | 121 ++++++ src/dotnet/ManagedTypeNames.ts | 1 - src/dotnet/ManagedUserStrings.ts | 8 +- tests/acceptance/managedCliOutput.test.ts | 37 ++ .../filesystem/desktopApplication.test.ts | 28 +- tests/fixtures/managedSelectedScope.ts | 8 +- 22 files changed, 1012 insertions(+), 257 deletions(-) create mode 100644 src/dotnet/ManagedHeapStreamAmbiguity.test.ts create mode 100644 src/dotnet/ManagedTableStreamConflict.test.ts diff --git a/docs/guides/managed/managed-code-analysis.md b/docs/guides/managed/managed-code-analysis.md index 62fe4d518e..bb125b4f58 100644 --- a/docs/guides/managed/managed-code-analysis.md +++ b/docs/guides/managed/managed-code-analysis.md @@ -90,7 +90,21 @@ coverage/issues and limitations. No assembly is loaded or executed. ECMA-335 permits empty metadata streams to be omitted. An absent `#Blob` stream therefore does not prevent module or assembly identity from being read when all blob indexes are zero; a nonzero reference to that missing heap is -reported as a missing-stream issue. +reported as a missing-stream issue. Repeated `#Blob` declarations for the same +file range keep the blob facts and disclose the duplicate. Distinct ranges +leave nonzero blob indices unresolved. The result keeps module identity, row +counts, and other independent observations. A row whose current result requires +a decoded blob is omitted with partial coverage. A separate blob-backed fact +such as a method's local-variable signature remains unknown when other facts +about that method are available. An out-of-root stream declaration is malformed. + +If the metadata root declares both compressed (`#~`) and uncompressed (`#-`) +tables, REA reports both declarations and leaves the row layout unresolved. +The metadata version and PE/CLI evidence remain available. Member inventories, +module identity, and native imports that depend on the table layout are +unavailable. This applies even when the two declarations point to the same +bytes, because the table kinds differ. A declaration outside the metadata +root is still malformed. Use `normalized_result.parent_evidence_id` to page original rows: @@ -433,8 +447,11 @@ length prefix against the width II.23.2 assigns. A difference is reported with its byte offset as a user-string issue while preserving the readable literal. Toolchain conventions can differ; this conformance observation does not establish runtime rejection. A missing `#US` stream is one stream-level issue; -a malformed item is reported against its token. User strings carry their own -coverage: any of these makes `user_strings.state` partial and lists the issue +a malformed item is reported against its token. Repeated `#US` declarations +with the same file range keep the literals and disclose the duplicate. Distinct +ranges leave the literals unresolved and preserve the `ldstr` token anchors. +A declaration outside the metadata root still makes the metadata malformed. +User strings carry their own coverage: any of these makes `user_strings.state` partial and lists the issue in `user_strings.issues`, while the member `coverage` and `metadata.status` are unaffected, and readable literals remain. The state is also partial when an inspected method's CIL was not fully decoded (a malformed exception section diff --git a/src/domain/managed/managedMemberInspectorCore.ts b/src/domain/managed/managedMemberInspectorCore.ts index 574e778999..d2c69c84b2 100644 --- a/src/domain/managed/managedMemberInspectorCore.ts +++ b/src/domain/managed/managedMemberInspectorCore.ts @@ -33,6 +33,7 @@ import { type MetadataTokenFact, metadataToken, readMetadataString, + metadataBlobOffset, } from "./managedMetadataHeaps.js"; import { memberListTable } from "./managedMemberPointers.js"; import type { @@ -1098,10 +1099,7 @@ export const methodGenericArity = ( blob: Buffer, ): number | null => { const budget = managedDecodeBudget(layout); - budget.admitTransient( - blob.length * 256, - layout.blob?.offset ?? layout.rootOffset, - ); + budget.admitTransient(blob.length * 256, metadataBlobOffset(layout)); try { return signatureGenericArity(decodeSignature(blob)); } finally { diff --git a/src/domain/managed/managedMetadataHeaps.test.ts b/src/domain/managed/managedMetadataHeaps.test.ts index 58f19858c7..4777d11534 100644 --- a/src/domain/managed/managedMetadataHeaps.test.ts +++ b/src/domain/managed/managedMetadataHeaps.test.ts @@ -11,7 +11,7 @@ import { type ManagedMetadataLayout, } from "./managedMetadataLayout.js"; -describe("managed metadata heaps", () => { +describe("managed metadata heap entries", () => { it("admits a metadata root that omits its empty #Blob stream", () => { const bytes = Buffer.alloc(108); bytes.writeUInt32LE(0x424a_5342, 0); @@ -67,11 +67,11 @@ describe("managed metadata heaps", () => { expect( Buffer.compare( - readMetadataBlob(Buffer.alloc(0), layout, 0, 0), + readMetadataBlob(Buffer.alloc(0), layout, 0), Buffer.alloc(0), ), ).toBe(0); - expect(() => readMetadataBlob(Buffer.alloc(0), layout, 1, 0)).toThrowError( + expect(() => readMetadataBlob(Buffer.alloc(0), layout, 1)).toThrowError( expect.objectContaining({ issue: expect.objectContaining({ code: "missing-stream", @@ -91,8 +91,16 @@ describe("managed metadata heaps", () => { version: "", streamNames: ["#Blob"], strings: { name: "#Strings", offset: 0, size: 0 }, - guid: { name: "#GUID", offset: 0, size: 0 }, - blob: { name: "#Blob", offset: 0, size: bytes.length }, + guid: { + kind: "available", + stream: { name: "#GUID", offset: 0, size: 0 }, + issue: null, + }, + blob: { + kind: "available", + stream: { name: "#Blob", offset: 0, size: bytes.length }, + issue: null, + }, userStrings: null, tables: new Map(), rowCounts: [], @@ -104,9 +112,28 @@ describe("managed metadata heaps", () => { codedIndexSize: () => 2, }; - expect( - readMetadataBlob(bytes, layout, 1, layout.blob?.size ?? 0), - ).toHaveLength(itemLength); + expect(readMetadataBlob(bytes, layout, 1)).toHaveLength(itemLength); + }); + + it("requires a compressed blob length prefix to fit inside #Blob", () => { + const bytes = Buffer.from([0, 0x80, 0, 0]); + const layout = { + blob: { + kind: "available", + stream: { name: "#Blob", offset: 0, size: 2 }, + issue: null, + }, + } as unknown as ManagedMetadataLayout; + expect(() => readMetadataBlob(bytes, layout, 1)).toThrow( + expect.objectContaining({ + issue: expect.objectContaining({ + code: "invalid-blob", + scope: "metadata.#Blob", + detail: "Metadata blob length prefix leaves the #Blob byte range", + offset: 1, + }), + }), + ); }); it.each([[[0x80]], [[0xc0, 0x00, 0x00]]])( @@ -115,7 +142,11 @@ describe("managed metadata heaps", () => { // #US ends at the end of the artifact, inside a multi-byte length prefix. const bytes = Buffer.from([0x00, ...prefix]); const layout = { - userStrings: { name: "#US", offset: 0, size: bytes.length }, + userStrings: { + kind: "available", + stream: { name: "#US", offset: 0, size: bytes.length }, + issue: null, + }, } as unknown as ManagedMetadataLayout; expect(() => @@ -130,36 +161,40 @@ describe("managed metadata heaps", () => { ); }, ); - - it("does not re-charge the decode budget on metadata string cache hits", () => { - const strLen = 10 * 1024 * 1024; - const strBytes = Buffer.alloc(strLen + 2, 0x61); - strBytes[0] = 0x00; - strBytes[strLen + 1] = 0x00; - const layout = { - rootOffset: 0, - size: strBytes.length, - version: "", - streamNames: ["#Strings"], - strings: { name: "#Strings", offset: 0, size: strBytes.length }, - guid: { name: "#GUID", offset: 0, size: 0 }, - blob: { name: "#Blob", offset: 0, size: 0 }, - userStrings: null, - tables: new Map(), - rowCounts: [], - stringIndexSize: 2, - guidIndexSize: 2, - blobIndexSize: 2, - table: () => undefined, - tableIndexSize: () => 2, - codedIndexSize: () => 2, - } as unknown as ManagedMetadataLayout; - - const first = readMetadataString(strBytes, layout, 1, layout.strings.size); - expect(first).toHaveLength(strLen); - - expect(() => - readMetadataString(strBytes, layout, 1, layout.strings.size), - ).not.toThrow(); - }); +}); + +it("does not re-charge the decode budget on metadata string cache hits", () => { + const strLen = 10 * 1024 * 1024; + const strBytes = Buffer.alloc(strLen + 2, 0x61); + strBytes[0] = 0x00; + strBytes[strLen + 1] = 0x00; + const layout = { + rootOffset: 0, + size: strBytes.length, + version: "", + streamNames: ["#Strings"], + strings: { name: "#Strings", offset: 0, size: strBytes.length }, + guid: { + kind: "available", + stream: { name: "#GUID", offset: 0, size: 0 }, + issue: null, + }, + blob: null, + userStrings: null, + tables: new Map(), + rowCounts: [], + stringIndexSize: 2, + guidIndexSize: 2, + blobIndexSize: 2, + table: () => undefined, + tableIndexSize: () => 2, + codedIndexSize: () => 2, + } as unknown as ManagedMetadataLayout; + + const first = readMetadataString(strBytes, layout, 1, layout.strings.size); + expect(first).toHaveLength(strLen); + + expect(() => + readMetadataString(strBytes, layout, 1, layout.strings.size), + ).not.toThrow(); }); diff --git a/src/domain/managed/managedMetadataHeaps.ts b/src/domain/managed/managedMetadataHeaps.ts index a76473cd1a..c2b7d3d16c 100644 --- a/src/domain/managed/managedMetadataHeaps.ts +++ b/src/domain/managed/managedMetadataHeaps.ts @@ -9,7 +9,11 @@ import { metadataRowOffset, type ManagedMetadataLayout, } from "./managedMetadataLayout.js"; -import { managedFailure } from "./managedReaderFailure.js"; +import { + managedFailure, + ManagedHeapAmbiguityFailure, + ManagedReaderFailure, +} from "./managedReaderFailure.js"; /** Cursor for checked fixed-width metadata row decoding. */ export class MetadataRowCursor { @@ -101,8 +105,16 @@ export const readManagedCompressedUnsigned = ( // Issue code and label for the heap item a compressed length prefixes. const PREFIXED_ITEMS = { - blob: { code: "invalid-blob", label: "Metadata blob" }, - userString: { code: "invalid-string", label: "User string" }, + blob: { + code: "invalid-blob", + label: "Metadata blob", + range: "the #Blob byte range", + }, + userString: { + code: "invalid-string", + label: "User string", + range: "the artifact byte range", + }, } as const; const compressedLength = ( @@ -116,14 +128,14 @@ const compressedLength = ( /** Whether ECMA-335 II.23.2 assigns this width to the value. */ readonly minimal: boolean; } => { - const { code, label } = PREFIXED_ITEMS[item]; + const { code, label, range } = PREFIXED_ITEMS[item]; const decoded = readManagedCompressedUnsigned(bytes, offset); if (!decoded.ok) throw managedFailure( code, scope, decoded.error.kind === "truncated" - ? `${label} length prefix leaves the artifact byte range` + ? `${label} length prefix leaves ${range}` : `${label} uses a reserved compressed-length prefix`, offset >= 0 ? offset : null, ); @@ -183,12 +195,17 @@ export const readMetadataString = ( } }; +/** File coordinate used when reporting a budget for decoded blob facts. */ +export const metadataBlobOffset = (layout: ManagedMetadataLayout): number => + layout.blob?.kind === "available" + ? layout.blob.stream.offset + : layout.rootOffset; + /** Read one #Blob item while validating its encoded extent against the heap. */ export const readMetadataBlob = ( bytes: Buffer, layout: ManagedMetadataLayout, index: number, - heapExtent: number, ): Buffer => { if (index === 0) return Buffer.alloc(0); if (index < 0) @@ -198,14 +215,17 @@ export const readMetadataBlob = ( "Blob heap index is negative", null, ); - const heap = layout.blob; - if (heap === null) + const declaration = layout.blob; + if (declaration === null) throw managedFailure( "missing-stream", "metadata.#Blob", `Blob index ${String(index)} references an absent #Blob stream`, null, ); + if (declaration.kind === "ambiguous") + throw new ManagedHeapAmbiguityFailure(declaration.issue); + const heap = declaration.stream; if (index >= heap.size) throw managedFailure( "invalid-heap-index", @@ -214,16 +234,15 @@ export const readMetadataBlob = ( heap.offset + Math.max(index, 0), ); const start = heap.offset + index; - const decoded = compressedLength(bytes, start, "metadata.#Blob", "blob"); - if (decoded.length > heapExtent) - throw managedFailure( - "invalid-blob", - "metadata.#Blob", - "Blob length exceeds the admitted heap extent", - start, - ); + const heapEnd = heap.offset + heap.size; + const decoded = compressedLength( + bytes.subarray(0, heapEnd), + start, + "metadata.#Blob", + "blob", + ); const content = start + decoded.prefix; - if (content > heap.offset + heap.size - decoded.length) + if (content > heapEnd - decoded.length) throw managedFailure( "invalid-blob", "metadata.#Blob", @@ -250,14 +269,17 @@ export const readMetadataUserString = ( readonly issues: readonly ManagedParseIssue[]; } => { const { token, row: index } = reference; - const heap = layout.userStrings; + const declaration = layout.userStrings; const scope = `metadata.#US:${token}`; - if (heap === null) + if (declaration === null) throw managedFailure( "missing-stream", scope, "ldstr references #US, but the metadata has no #US stream", ); + if (declaration.kind === "ambiguous") + throw new ManagedReaderFailure(declaration.issue); + const heap = declaration.stream; if (index >= heap.size) throw managedFailure( "invalid-heap-index", @@ -351,22 +373,24 @@ export const readMetadataUserString = ( }; }; -/** Read one conventional GUID value from the one-based #GUID heap. */ +/** A zero index denotes absence; undefined denotes an ambiguous heap. */ export const readMetadataGuid = ( bytes: Buffer, layout: ManagedMetadataLayout, index: number, -): string | null => { +): string | null | undefined => { if (index === 0) return null; + if (layout.guid.kind === "ambiguous") return undefined; + const { offset: heapOffset, size } = layout.guid.stream; const relative = (index - 1) * 16; - if (relative < 0 || relative > layout.guid.size - 16) + if (relative < 0 || relative > size - 16) throw managedFailure( "invalid-guid", "metadata.#GUID", "GUID heap index is outside #GUID", - layout.guid.offset + Math.max(relative, 0), + heapOffset + Math.max(relative, 0), ); - const offset = layout.guid.offset + relative; + const offset = heapOffset + relative; const part1 = bytes.readUInt32LE(offset).toString(16).padStart(8, "0"); const part2 = bytes .readUInt16LE(offset + 4) diff --git a/src/domain/managed/managedMetadataInventory.ts b/src/domain/managed/managedMetadataInventory.ts index ec1f7ddecc..7fc86edc61 100644 --- a/src/domain/managed/managedMetadataInventory.ts +++ b/src/domain/managed/managedMetadataInventory.ts @@ -78,9 +78,6 @@ const readRows = ( return items; }; -const heapExtent = (layout: ManagedMetadataLayout): number => - Math.max(layout.strings.size, layout.blob?.size ?? 0); - const validateIdentityTableCounts = ( layout: ManagedMetadataLayout, issues: ManagedParseIssue[], @@ -109,11 +106,13 @@ export const readManagedModuleIdentity = ( bytes: Buffer, layout: ManagedMetadataLayout, ): Pick => { - const issues: ManagedParseIssue[] = []; + const issues: ManagedParseIssue[] = + layout.guid.issue === null ? [] : [layout.guid.issue]; + if (layout.blob?.issue) issues.push(layout.blob.issue); validateIdentityTableCounts(layout, issues); const module = readManagedValue( - () => readModule(bytes, layout, heapExtent(layout)), + () => readModule(bytes, layout, layout.strings.size), issues, ) ?? null; return { module, issues }; @@ -127,16 +126,17 @@ const readReferences = ( readonly references: readonly AssemblyReference[]; readonly referenceNames: readonly string[]; } => { - const references = readRows(layout.table(35), (row) => - readManagedValue( - () => readAssemblyReference(bytes, layout, row, heapExtent(layout)), + const referenceNames = new Set(); + const references = readRows(layout.table(35), (row) => { + const observed = readManagedValue( + () => readAssemblyReference(bytes, layout, row, layout.strings.size), issues, - ), - ); - return { - references, - referenceNames: [...new Set(references.map(({ name }) => name))].sort(), - }; + ); + if (observed === undefined) return undefined; + referenceNames.add(observed.name); + return observed.reference ?? undefined; + }); + return { references, referenceNames: [...referenceNames].sort() }; }; const readResources = ({ @@ -177,15 +177,7 @@ const readAttributes = ( const attributes = readRows(layout.table(12), (row) => readManagedValue( () => - readCustomAttribute( - bytes, - layout, - row, - heapExtent(layout), - issues, - memberLists, - typeNames, - ), + readCustomAttribute(bytes, layout, row, issues, memberLists, typeNames), issues, ), ); @@ -219,7 +211,7 @@ export const readManagedMetadataInventory = ( const { module } = identity; const assembly = readManagedValue( - () => readAssembly(bytes, layout, heapExtent(layout)), + () => readAssembly(bytes, layout, layout.strings.size), issues, ) ?? null; const { references, referenceNames } = readReferences(bytes, layout, issues); diff --git a/src/domain/managed/managedMetadataInventoryRows.ts b/src/domain/managed/managedMetadataInventoryRows.ts index f541d1d09d..1fe215386c 100644 --- a/src/domain/managed/managedMetadataInventoryRows.ts +++ b/src/domain/managed/managedMetadataInventoryRows.ts @@ -24,10 +24,15 @@ import { type ManagedCodedIndex, metadataToken, readMetadataBlob, + metadataBlobOffset, readMetadataGuid, readMetadataString, } from "./managedMetadataHeaps.js"; -import { managedFailure, readManagedValue } from "./managedReaderFailure.js"; +import { + managedFailure, + ManagedHeapAmbiguityFailure, + readManagedValue, +} from "./managedReaderFailure.js"; import type { ManagedTypeReferenceLookup } from "./managedTypeReferences.js"; import type { ManagedResourceDirectory } from "./managedMetadataInventory.js"; import type { ManagedType, parseTypes } from "./managedMemberInspectorCore.js"; @@ -95,9 +100,9 @@ export const readModule = ( return { name, generation, - mvid, - enc_id: encId, - enc_base_id: encBaseId, + mvid: mvid ?? null, + enc_id: encId ?? null, + enc_base_id: encBaseId ?? null, token: metadataToken(0, 1), row_offset: table.offset, }; @@ -120,7 +125,6 @@ export const readAssembly = ( bytes, layout, cursor.readIndex(layout.blobIndexSize), - heapExtent, ); const name = readMetadataString( bytes, @@ -189,41 +193,48 @@ export const readAssemblyReference = ( layout: ManagedMetadataLayout, row: number, heapExtent: number, -): AssemblyReference => { +): { readonly name: string; readonly reference: AssemblyReference | null } => { const { cursor, version, flags, keyIndex, nameIndex } = assemblyReferenceHeader(bytes, layout, row); - const keyOrToken = readMetadataBlob(bytes, layout, keyIndex, heapExtent); const name = readMetadataString(bytes, layout, nameIndex, heapExtent); - const culture = readMetadataString( - bytes, - layout, - cursor.readIndex(layout.stringIndexSize), - heapExtent, - ); - const hashValue = readMetadataBlob( - bytes, - layout, - cursor.readIndex(layout.blobIndexSize), - heapExtent, - ); - return { - name, - version, - culture: culture.length === 0 ? null : culture, - flags, - public_key_or_token: publicKeyIdentity( - keyOrToken, - (flags & 1) === 0 ? "public-key-token" : "public-key", + try { + const keyOrToken = readMetadataBlob(bytes, layout, keyIndex); + const culture = readMetadataString( + bytes, layout, - ), - hash_value_sha256: - hashValue.length === 0 - ? null - : managedDecodeBudget(layout).digest(hashValue, "sha256"), - hash_value_length: hashValue.length, - token: metadataToken(35, row), - row_offset: metadataRowOffset(layout, 35, row), - }; + cursor.readIndex(layout.stringIndexSize), + heapExtent, + ); + const hashValue = readMetadataBlob( + bytes, + layout, + cursor.readIndex(layout.blobIndexSize), + ); + const reference = { + name, + version, + culture: culture.length === 0 ? null : culture, + flags, + public_key_or_token: publicKeyIdentity( + keyOrToken, + (flags & 1) === 0 ? "public-key-token" : "public-key", + layout, + ), + hash_value_sha256: + hashValue.length === 0 + ? null + : managedDecodeBudget(layout).digest(hashValue, "sha256"), + hash_value_length: hashValue.length, + token: metadataToken(35, row), + row_offset: metadataRowOffset(layout, 35, row), + }; + return { name, reference }; + } catch (cause: unknown) { + // The name comes from #Strings and is independent of key and hash bytes. + if (cause instanceof ManagedHeapAmbiguityFailure) + return { name, reference: null }; + throw cause; + } }; export type ManagedMetadataTypeNameLookup = ( @@ -424,10 +435,7 @@ const decodeFixedString = ( const { value: length, width } = decoded.value; const start = 2 + width; if (start > blob.length - length) return null; - managedDecodeBudget(layout).reserveString( - length, - layout.blob?.offset ?? layout.rootOffset, - ); + managedDecodeBudget(layout).reserveString(length, metadataBlobOffset(layout)); try { return decodeStrictUtf8(blob.subarray(start, start + length), "preserve"); } catch (cause: unknown) { @@ -441,7 +449,6 @@ export const readCustomAttribute = ( bytes: Buffer, layout: ManagedMetadataLayout, row: number, - heapExtent: number, issues: ManagedParseIssue[], memberLists: TypeMemberListFacts, typeNames: ManagedMetadataTypeNameLookup, @@ -458,7 +465,6 @@ export const readCustomAttribute = ( bytes, layout, cursor.readIndex(layout.blobIndexSize), - heapExtent, ); const parent = decodeMetadataCodedIndex( parentRaw, @@ -536,7 +542,7 @@ export const readResource = ({ bytes, layout, cursor.readIndex(layout.stringIndexSize), - Math.max(layout.strings.size, layout.blob?.size ?? 0), + layout.strings.size, ); const implementationOffset = cursor.offset; const implementationRaw = cursor.readIndex( diff --git a/src/domain/managed/managedMetadataLayout.ts b/src/domain/managed/managedMetadataLayout.ts index 2809566be9..671793c612 100644 --- a/src/domain/managed/managedMetadataLayout.ts +++ b/src/domain/managed/managedMetadataLayout.ts @@ -1,5 +1,7 @@ import { isUtf8 } from "node:buffer"; +import type { ManagedParseIssue } from "./managedArtifact.js"; + import { attachManagedDecodeBudget, ManagedDecodeBudget, @@ -67,6 +69,20 @@ interface MetadataStream { readonly size: number; } +interface StreamDeclaration extends MetadataStream { + readonly headerOffset: number; +} + +type MetadataHeap = + | { + readonly kind: "available"; + readonly stream: MetadataStream; + readonly issue: ManagedParseIssue | null; + } + | { readonly kind: "ambiguous"; readonly issue: ManagedParseIssue }; + +type TrackedHeapName = "#GUID" | "#US" | "#Blob"; + export interface MetadataTableLayout { readonly index: number; readonly name: string; @@ -81,11 +97,11 @@ export interface ManagedMetadataLayout { readonly version: string; readonly streamNames: readonly string[]; readonly strings: MetadataStream; - readonly guid: MetadataStream; + readonly guid: MetadataHeap; /** Empty blob heaps may omit the stream; index zero remains the empty blob. */ - readonly blob: MetadataStream | null; + readonly blob: MetadataHeap | null; /** Optional: assemblies without `ldstr` literals may omit #US. */ - readonly userStrings: MetadataStream | null; + readonly userStrings: MetadataHeap | null; readonly tables: ReadonlyMap; readonly rowCounts: readonly number[]; readonly stringIndexSize: 2 | 4; @@ -178,17 +194,6 @@ const readStreamName = ( }; }; -const streamByName = ( - streams: ReadonlyMap, - names: readonly string[], -): MetadataStream | undefined => { - for (const name of names) { - const stream = streams.get(name); - if (stream !== undefined) return stream; - } - return undefined; -}; - const requireStream = ( streams: ReadonlyMap, name: string, @@ -220,9 +225,10 @@ const tableSize = (index: number, rows: readonly number[]): 2 | 4 => (rows[index] ?? 0) < 0x1_0000 ? 2 : 4; interface ParsedStreams { - readonly streams: ReadonlyMap; + readonly streams: ReadonlyMap; + readonly streamNames: readonly string[]; + readonly heapDeclarations: Record; readonly version: string; - readonly minimalDeltaOffset: number | null; } const readMetadataStreams = ( @@ -263,19 +269,29 @@ const readMetadataStreams = ( rootOffset + cursor + 2, ); cursor += 4; - const streams = new Map(); - let minimalDeltaOffset: number | null = null; + const streams = new Map(); + const streamNames = new Set(); + const heapDeclarations: ParsedStreams["heapDeclarations"] = { + "#GUID": [], + "#US": [], + "#Blob": [], + }; for (let index = 0; index < streamCount; index += 1) { requireRange(bytes, rootOffset, cursor, 8, "metadata.stream-header"); + const headerOffset = rootOffset + cursor; const relativeOffset = bytes.readUInt32LE(cursor); const streamSize = bytes.readUInt32LE(cursor + 4); const named = readStreamName(bytes, rootOffset, cursor + 8); - if (streams.has(named.name)) + const trackedHeap = + named.name === "#GUID" || named.name === "#US" || named.name === "#Blob" + ? named.name + : null; + if (trackedHeap === null && streamNames.has(named.name)) throw managedFailure( "invalid-stream", "metadata.stream-header", `Duplicate metadata stream ${named.name}`, - rootOffset + cursor, + headerOffset, ); requireRange( bytes, @@ -284,15 +300,66 @@ const readMetadataStreams = ( streamSize, `metadata.${named.name}`, ); - streams.set(named.name, { + const stream: StreamDeclaration = { name: named.name, offset: rootOffset + relativeOffset, size: streamSize, - }); - if (named.name === "#JTD") minimalDeltaOffset = rootOffset + cursor; + headerOffset, + }; + streamNames.add(named.name); + if (trackedHeap !== null) { + heapDeclarations[trackedHeap].push(stream); + } else { + streams.set(named.name, stream); + } cursor = named.next; } - return { streams, version, minimalDeltaOffset }; + return { + streams, + streamNames: [...streamNames], + heapDeclarations, + version, + }; +}; + +const describeStreamCoordinates = ({ + headerOffset, + offset, + size, +}: StreamDeclaration): string => + `header ${headerOffset} (offset ${offset}, size ${size})`; + +const resolveHeap = ( + name: TrackedHeapName, + declarations: readonly StreamDeclaration[], +): MetadataHeap | null => { + const [first, second] = declarations; + if (first === undefined) return null; + if (second === undefined) + return { kind: "available", stream: first, issue: null }; + + // An exact alias preserves source identity. Distinct byte ranges do not + // establish which heap supplies an identity or value for a given index. + const sameRange = declarations.every( + ({ offset, size }) => offset === first.offset && size === first.size, + ); + const coordinates = declarations.map(describeStreamCoordinates).join(", "); + const conflict = { + "#GUID": "Nonzero GUID indices are unresolved.", + "#US": "User-string literals are unresolved.", + "#Blob": "Nonzero blob indices are unresolved.", + }[name]; + const issue: ManagedParseIssue = { + code: "invalid-stream", + scope: `metadata.${name}`, + offset: second.headerOffset, + detail: sameRange + ? `Duplicate ${name} declarations describe the same byte range (${coordinates}).` + : `Duplicate ${name} declarations describe distinct byte ranges (${coordinates}). ${conflict}`, + }; + return sameRange + ? { kind: "available", stream: first, issue } + : { kind: "ambiguous", issue }; }; interface HeapSizes { @@ -308,17 +375,38 @@ interface ParsedTablesHeader { readonly valid: bigint; } -const readTablesHeader = ( - bytes: Buffer, - streams: ReadonlyMap, -): ParsedTablesHeader => { - const tablesStream = streamByName(streams, ["#~", "#-"]); - if (tablesStream === undefined) +const selectTableStream = ({ + streams, + streamNames, + version, +}: ParsedStreams): MetadataStream => { + const compressed = streams.get("#~"); + const uncompressed = streams.get("#-"); + if (compressed !== undefined && uncompressed !== undefined) + throw new ManagedReaderFailure( + { + code: "unsupported-metadata", + scope: "metadata.tables", + detail: `Conflicting table stream declarations: #~ at ${describeStreamCoordinates(compressed)} and #- at ${describeStreamCoordinates(uncompressed)}. Metadata row layout is unknown.`, + offset: Math.max(compressed.headerOffset, uncompressed.headerOffset), + }, + { metadataHeader: { version, streamNames } }, + ); + + const stream = compressed ?? uncompressed; + if (stream === undefined) throw managedFailure( "missing-stream", "metadata.tables", "Required metadata table stream #~ or #- is absent", ); + return stream; +}; + +const readTablesHeader = ( + bytes: Buffer, + tablesStream: MetadataStream, +): ParsedTablesHeader => { const tableBytes = bytes.subarray( tablesStream.offset, tablesStream.offset + tablesStream.size, @@ -460,28 +548,27 @@ export const readManagedMetadataLayout = ( ); const bytes = artifact.subarray(rootOffset, rootOffset + size); const budget = new ManagedDecodeBudget(maxRepresentationBytes); - const { streams, version, minimalDeltaOffset } = readMetadataStreams( - bytes, - rootOffset, - budget, - ); + const parsed = readMetadataStreams(bytes, rootOffset, budget); + const { streams, streamNames, heapDeclarations, version } = parsed; + const minimalDelta = streams.get("#JTD"); + // Runtime v9.0.0 MetadataReader recognizes #JTD before sizing references. // Minimal deltas also use baseline coordinates and cumulative heap handles; // widening local rows alone cannot establish standalone member identities. - if (minimalDeltaOffset !== null) + if (minimalDelta !== undefined) throw new ManagedReaderFailure( { code: "unsupported-metadata", scope: "metadata.#JTD", detail: "Minimal Edit-and-Continue metadata (#JTD) requires its baseline and prior generations; standalone metadata identities and inventories are unknown.", - offset: minimalDeltaOffset, + offset: minimalDelta.headerOffset, }, - { metadataHeader: { version, streamNames: [...streams.keys()] } }, + { metadataHeader: { version, streamNames } }, ); const { tableBytes, tablesOffset, heaps, valid } = readTablesHeader( artifact, - streams, + selectTableStream(parsed), ); const rowCounts = readRowCounts(tableBytes, tablesOffset, valid); const tables = buildMetadataTables({ @@ -491,15 +578,23 @@ export const readManagedMetadataLayout = ( valid, tablesOffset, }); + const strings = requireStream(streams, "#Strings"); + const guid = resolveHeap("#GUID", heapDeclarations["#GUID"]); + if (guid === null) + throw managedFailure( + "missing-stream", + "metadata.#GUID", + "Required metadata stream #GUID is absent", + ); const layout: ManagedMetadataLayout = { rootOffset, size, version, - streamNames: [...streams.keys()].sort(), - strings: requireStream(streams, "#Strings"), - guid: requireStream(streams, "#GUID"), - blob: streams.get("#Blob") ?? null, - userStrings: streams.get("#US") ?? null, + streamNames: [...streamNames].sort(), + strings, + guid, + blob: resolveHeap("#Blob", heapDeclarations["#Blob"]), + userStrings: resolveHeap("#US", heapDeclarations["#US"]), tables, rowCounts, stringIndexSize: heaps.string, diff --git a/src/domain/managed/managedReaderFailure.ts b/src/domain/managed/managedReaderFailure.ts index 12d2e46549..91c1b67a9e 100644 --- a/src/domain/managed/managedReaderFailure.ts +++ b/src/domain/managed/managedReaderFailure.ts @@ -22,6 +22,9 @@ export class ManagedReaderFailure extends Error { } } +/** Refuse a heap-dependent row whose shared stream issue is reported by layout inventory. */ +export class ManagedHeapAmbiguityFailure extends ManagedReaderFailure {} + /** A resource or unsupported layout leaves metadata unknown, not malformed. */ export const managedMetadataFailureStatus = ( issue: ManagedParseIssue, @@ -70,7 +73,8 @@ export const readManagedValue = ( managedMetadataFailureStatus(cause.issue) === "partial" ) throw cause; - issues.push(cause.issue); + if (!(cause instanceof ManagedHeapAmbiguityFailure)) + issues.push(cause.issue); return undefined; } }; diff --git a/src/dotnet/ManagedFieldConstants.ts b/src/dotnet/ManagedFieldConstants.ts index 6eb7d9140f..59f99f78a6 100644 --- a/src/dotnet/ManagedFieldConstants.ts +++ b/src/dotnet/ManagedFieldConstants.ts @@ -16,6 +16,7 @@ import { } from "../domain/managed/managedMetadataHeaps.js"; import { managedFailure, + ManagedHeapAmbiguityFailure, ManagedReaderFailure, } from "../domain/managed/managedReaderFailure.js"; import { @@ -113,12 +114,7 @@ export const createManagedFieldConstantReader = ( "Constant row padding must be zero.", offset, ); - const value = readMetadataBlob( - bytes, - layout, - first.blob_index, - layout.blob?.size ?? 0, - ); + const value = readMetadataBlob(bytes, layout, first.blob_index); budget.admitTransient(value.length * 8 + 1024, offset); result.raw_hex = value.toString("hex"); result.element_type = managedConstantType(first.type_code); @@ -135,7 +131,8 @@ export const createManagedFieldConstantReader = ( cause.issue.code === "resource-limit" ) throw cause; - issues.push(cause.issue); + if (!(cause instanceof ManagedHeapAmbiguityFailure)) + issues.push(cause.issue); result.limitation = cause.issue.detail; } finally { budget.releaseTransient(); diff --git a/src/dotnet/ManagedHeapStreamAmbiguity.test.ts b/src/dotnet/ManagedHeapStreamAmbiguity.test.ts new file mode 100644 index 0000000000..c582058842 --- /dev/null +++ b/src/dotnet/ManagedHeapStreamAmbiguity.test.ts @@ -0,0 +1,397 @@ +import { expect, it } from "vitest"; + +import { readManagedMetadataLayout } from "../domain/managed/managedMetadataLayout.js"; +import { readManagedPeLayout } from "../domain/managed/managedPeReader.js"; +import { inspectManagedArtifactBytes } from "./ManagedArtifactInspector.js"; +import { inspectManagedMembersBytes } from "./ManagedMemberInspector.js"; +import { inspectManagedNativeBoundariesBytes } from "./ManagedNativeBoundaryInspector.js"; +import { + buildManagedPeFixture, + fixtureUserStringHeap, + managedPeFixtureTarget, +} from "./ManagedPe.fixture.js"; + +const FIRST_MVID = "11111111-1111-1111-1111-111111111111"; +const SECOND_MVID = "22222222-2222-2222-2222-222222222222"; + +type Variant = "forward" | "reverse" | "alias" | "outside-root"; + +const duplicateHeap = ( + bytes: Buffer, + name: "#GUID" | "#US" | "#Blob", + alternateHeap: Buffer, + variant: Variant, +) => { + const pe = readManagedPeLayout(bytes); + if (pe.cli === null) throw new Error("Fixture omitted a CLI header"); + const size = pe.cli.metadata.size; + const start = pe.rvaToOffset(pe.cli.metadata.rva, size, "fixture metadata"); + const original = Buffer.from(bytes.subarray(start, start + size)); + const streamCount = original.readUInt16LE(30); + const headers: number[] = []; + let heapHeader = -1; + let end = 32; + for (let index = 0; index < streamCount; index += 1) { + headers.push(end); + const terminator = original.indexOf(0, end + 8); + if (original.toString("ascii", end + 8, terminator) === name) + heapHeader = end; + end = Math.ceil((terminator + 1) / 4) * 4; + } + if (heapHeader < 0) throw new Error(`Fixture omitted ${name}`); + const headerSize = 8 + Math.ceil((name.length + 1) / 4) * 4; + const alternate = size + headerSize; + const extended = Buffer.alloc( + variant === "alias" ? alternate : alternate + alternateHeap.byteLength, + ); + original.copy(extended, 0, 0, end); + original.copy(extended, end + headerSize, end); + extended.writeUInt16LE(streamCount + 1, 30); + for (const header of headers) + extended.writeUInt32LE(original.readUInt32LE(header) + headerSize, header); + const originalHeap = extended.readUInt32LE(heapHeader); + const [first, second] = ( + { + forward: [originalHeap, alternate], + reverse: [alternate, originalHeap], + alias: [originalHeap, originalHeap], + "outside-root": [originalHeap, extended.length], + } as const + )[variant]; + if (variant !== "alias") alternateHeap.copy(extended, alternate); + extended.writeUInt32LE(first, heapHeader); + extended.writeUInt32LE(second, end); + const originalSize = original.readUInt32LE(heapHeader + 4); + extended.writeUInt32LE( + variant === "reverse" ? alternateHeap.byteLength : originalSize, + heapHeader + 4, + ); + extended.writeUInt32LE( + variant === "alias" || variant === "reverse" + ? originalSize + : alternateHeap.byteLength, + end + 4, + ); + extended.write(name, end + 8); + bytes.set(extended, start); + bytes.writeUInt32LE(extended.length, pe.cli.headerOffset + 12); + + return { + bytes, + duplicateHeader: start + end, + firstHeader: start + heapHeader, + firstRange: start + first, + secondRange: start + second, + metadataEnd: start + extended.length, + }; +}; + +const duplicateGuidFixture = (variant: Variant) => + duplicateHeap( + buildManagedPeFixture({ mvid: Buffer.alloc(16, 0x11) }), + "#GUID", + Buffer.alloc(16, 0x22), + variant, + ); + +it.each([ + ["forward", null], + ["reverse", null], + ["alias", FIRST_MVID], +] as const)( + "preserves independent metadata with %s #GUID declarations", + (variant, mvid) => { + const { bytes, duplicateHeader, firstHeader, firstRange, secondRange } = + duplicateGuidFixture(variant); + const target = managedPeFixtureTarget(bytes); + const artifact = inspectManagedArtifactBytes(bytes, target); + const members = inspectManagedMembersBytes(bytes, target); + const boundaries = inspectManagedNativeBoundariesBytes(bytes, target); + const scope = "metadata.#GUID"; + const issue = { + code: "invalid-stream", + scope, + offset: duplicateHeader, + detail: expect.stringContaining( + variant === "alias" ? "same byte range" : "distinct byte ranges", + ), + }; + + for (const report of [artifact, members, boundaries]) { + expect(report).toMatchObject({ + artifact: { sha256: target.sha256 }, + metadata: { + status: "partial", + table_row_counts: { Module: 1, TypeDef: 1, MethodDef: 1 }, + }, + module: { + name: "Fixture.dll", + token: "0x00000001", + generation: 0, + mvid, + enc_id: null, + enc_base_id: null, + }, + }); + const issues = report.coverage.issues.filter( + (item) => item.scope === scope, + ); + expect(issues).toEqual([issue]); + expect(issues[0]?.detail).toContain( + `header ${firstHeader} (offset ${firstRange}, size 16), header ${duplicateHeader} (offset ${secondRange}, size 16)`, + ); + } + expect(members.identity_scope.requires_mvid).toBe(mvid); + expect(boundaries.identity_scope.requires_mvid).toBe(mvid); + expect( + members.types.some(({ full_name }) => full_name === "Fixture.Program"), + ).toBe(true); + expect(members.methods).toHaveLength(1); + if (variant !== "alias") { + for (const report of [artifact, members, boundaries]) { + expect(JSON.stringify(report)).not.toContain(FIRST_MVID); + expect(JSON.stringify(report)).not.toContain(SECOND_MVID); + } + } + }, +); + +it("rejects a duplicate #GUID declaration whose extent leaves the metadata root", () => { + const { bytes, metadataEnd } = duplicateGuidFixture("outside-root"); + const target = managedPeFixtureTarget(bytes); + + for (const inspect of [ + inspectManagedArtifactBytes, + inspectManagedMembersBytes, + inspectManagedNativeBoundariesBytes, + ]) { + const report = inspect(bytes, target); + expect(report.metadata.status).toBe("malformed"); + expect(report.module).toBeNull(); + expect(report.coverage.issues).toContainEqual({ + code: "invalid-metadata-root", + scope: "metadata.#GUID", + detail: "metadata.#GUID leaves the metadata byte range", + offset: metadataEnd, + }); + } +}); + +const duplicateUserStringFixture = (variant: Variant) => { + const { + heap, + tokens: [token], + } = fixtureUserStringHeap(["one"]); + if (token === undefined) throw new Error("Fixture omitted a user string"); + // ldstr, pop, ret in a tiny method body. + const ilBody = Buffer.from([0x1e, 0x72, 0, 0, 0, 0, 0x26, 0x2a]); + ilBody.writeUInt32LE(Number.parseInt(token, 16), 2); + + return { + token, + ...duplicateHeap( + buildManagedPeFixture({ ilBody, userStringHeap: heap }), + "#US", + fixtureUserStringHeap(["two"]).heap, + variant, + ), + }; +}; + +it.each(["forward", "reverse", "alias"] as const)( + "preserves member facts with %s #US declarations", + (variant) => { + const { bytes, token, duplicateHeader, firstRange } = + duplicateUserStringFixture(variant); + const target = managedPeFixtureTarget(bytes); + const members = inspectManagedMembersBytes(bytes, target); + expect(members.metadata.status).toBe("complete"); + expect(members.coverage).toEqual({ state: "complete", issues: [] }); + expect(members.methods[0]?.body.anchors[0]).toMatchObject({ + opcode: "ldstr", + operand: token, + }); + expect(members.user_strings).toEqual({ + state: "partial", + literals: + variant === "alias" + ? [{ token, offset: firstRange + 1, value: "one" }] + : [], + issues: [ + { + code: "invalid-stream", + scope: "metadata.#US", + offset: duplicateHeader, + detail: expect.stringContaining( + variant === "alias" ? "same byte range" : "distinct byte ranges", + ), + }, + ], + }); + expect(inspectManagedArtifactBytes(bytes, target).metadata.status).toBe( + "complete", + ); + expect( + inspectManagedNativeBoundariesBytes(bytes, target).metadata.status, + ).toBe("complete"); + }, +); + +it("rejects a duplicate #US declaration outside the metadata root", () => { + const { bytes, metadataEnd } = duplicateUserStringFixture("outside-root"); + const members = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + expect(members.metadata.status).toBe("malformed"); + expect(members.coverage.issues).toContainEqual({ + code: "invalid-metadata-root", + scope: "metadata.#US", + detail: "metadata.#US leaves the metadata byte range", + offset: metadataEnd, + }); + expect(members.user_strings).toBeNull(); +}); + +const duplicateBlobFixture = ( + variant: Variant, + options: Parameters[0] = {}, + nonzeroPublicKey = false, +) => { + const bytes = buildManagedPeFixture({ + fieldRowCount: 1, + references: ["UnityEngine.CoreModule"], + pinvoke: { importName: "get_value", moduleName: "fixture.dll" }, + ...options, + }); + if (nonzeroPublicKey) { + const pe = readManagedPeLayout(bytes); + if (pe.cli === null) throw new Error("Fixture omitted CLI metadata"); + const { rva, size } = pe.cli.metadata; + const offset = pe.rvaToOffset(rva, size, "fixture metadata"); + const layout = readManagedMetadataLayout(bytes, offset, size); + const assembly = layout.table(32); + if (assembly === undefined) + throw new Error("Fixture omitted Assembly metadata"); + // Assembly.PublicKey follows the hash algorithm, version, and flags. + bytes.writeUInt16LE(1, assembly.offset + 16); + } + return duplicateHeap(bytes, "#Blob", Buffer.from([0, 3, 0, 0, 1]), variant); +}; + +it.each(["forward", "reverse", "alias"] as const)( + "preserves independent metadata with %s #Blob declarations", + (variant) => { + const { bytes, duplicateHeader } = duplicateBlobFixture(variant); + const target = managedPeFixtureTarget(bytes); + const artifact = inspectManagedArtifactBytes(bytes, target); + const members = inspectManagedMembersBytes(bytes, target); + const boundaries = inspectManagedNativeBoundariesBytes(bytes, target); + for (const report of [artifact, members, boundaries]) { + expect(report).toMatchObject({ + metadata: { + status: "partial", + table_row_counts: { Module: 1, Assembly: 1, MethodDef: 1, Field: 1 }, + }, + module: { name: "Fixture.dll", mvid: expect.any(String) }, + coverage: { state: "partial" }, + }); + expect( + report.coverage.issues.filter( + ({ scope }) => scope === "metadata.#Blob", + ), + ).toEqual([ + { + code: "invalid-stream", + scope: "metadata.#Blob", + offset: duplicateHeader, + detail: expect.stringContaining( + variant === "alias" ? "same byte range" : "distinct byte ranges", + ), + }, + ]); + } + expect(artifact.classification.evidence).toContainEqual( + expect.objectContaining({ code: "unity-reference-observed" }), + ); + expect(artifact.assembly).toMatchObject({ + name: "Fixture.Managed", + public_key: { kind: "none", byte_length: 0, sha256: null, token: null }, + }); + expect(members.types).toContainEqual( + expect.objectContaining({ full_name: "Fixture.Program" }), + ); + expect(boundaries.pinvoke_imports).toContainEqual( + expect.objectContaining({ import_name: "get_value" }), + ); + if (variant === "alias") { + expect(artifact.references).toContainEqual( + expect.objectContaining({ name: "UnityEngine.CoreModule" }), + ); + expect(members.methods[0]?.signature.parse_status).toBe("decoded"); + expect(members.fields[0]?.signature.parse_status).toBe("decoded"); + } else { + expect(artifact.references).toEqual([]); + expect(artifact.attributes).toEqual([]); + expect(members.fields).toEqual([]); + expect(members.methods).toEqual([]); + expect(members.member_refs).toEqual([]); + expect(members.method_rows).toEqual({ offset: 0, count: 1, total: 1 }); + } + }, +); + +it("withholds a nonzero Assembly public key without claiming the key is absent", () => { + const { bytes } = duplicateBlobFixture("forward", {}, true); + const report = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + expect(report.module?.mvid).toBeTruthy(); + expect(report.assembly).toBeNull(); + expect(report.metadata.status).toBe("partial"); + expect(report.metadata.table_row_counts.Assembly).toBe(1); +}); + +it("preserves a decoded body while its nonzero locals blob is unresolved", () => { + const original = buildManagedPeFixture({ + methodSignature: Buffer.alloc(0), + // Fat header with a one-byte ret body and a StandAloneSig row-1 token. + ilBody: Buffer.from([0x03, 0x30, 8, 0, 1, 0, 0, 0, 1, 0, 0, 0x11, 0x2a]), + standaloneSignatures: [Buffer.from([0x07, 0x01, 0x08])], + }); + const { bytes } = duplicateHeap( + original, + "#Blob", + Buffer.from([0, 1, 0]), + "forward", + ); + const report = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + expect(report.methods[0]).toMatchObject({ + signature: { raw_length: 0 }, + body: { + status: "partial", + local_var_sig_token: "0x11000001", + local_var_signature: null, + anchors: [{ opcode: "ret" }], + }, + }); +}); + +it("validates the extent of a later #Blob declaration before observing metadata", () => { + const { bytes, metadataEnd } = duplicateBlobFixture("outside-root"); + const report = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + expect(report.metadata.status).toBe("malformed"); + expect(report.coverage.issues).toContainEqual({ + code: "invalid-metadata-root", + scope: "metadata.#Blob", + detail: "metadata.#Blob leaves the metadata byte range", + offset: metadataEnd, + }); +}); diff --git a/src/dotnet/ManagedMemberInspector.test.ts b/src/dotnet/ManagedMemberInspector.test.ts index 9808ee0c73..4b99fba28d 100644 --- a/src/dotnet/ManagedMemberInspector.test.ts +++ b/src/dotnet/ManagedMemberInspector.test.ts @@ -403,7 +403,6 @@ it.each([ bytes, layout, row.readIndex(layout.blobIndexSize), - layout.blob?.size ?? 0, ); const blobOffset = blob.byteOffset - bytes.byteOffset; const result = inspectManagedMembersBytes( @@ -493,7 +492,6 @@ const memberSignatureBlobOffset = ( bytes, layout, row.readIndex(layout.blobIndexSize), - layout.blob?.size ?? 0, ); return blob.byteOffset - bytes.byteOffset; }; @@ -1812,7 +1810,11 @@ describe("managed user string issues", () => { ...Array.from({ length: 64 }, () => Buffer.from([0x80, 0x00])), ]); const layout = { - userStrings: { name: "#US", offset: 0, size: heap.length }, + userStrings: { + kind: "available", + stream: { name: "#US", offset: 0, size: heap.length }, + issue: null, + }, } as unknown as ManagedMetadataLayout; // Leave 24 KiB: enough for the failures, not for 512 bytes leaked each. managedDecodeBudget(layout).reserve(64 * 1024 * 1024 - 24 * 1024, null); @@ -1963,7 +1965,11 @@ describe("managed user string scan completeness", () => { it("marks user strings partial when selected type identity is unreadable", () => { const layout = { - userStrings: { name: "#US", offset: 0, size: 1 }, + userStrings: { + kind: "available", + stream: { name: "#US", offset: 0, size: 1 }, + issue: null, + }, } as unknown as ManagedMetadataLayout; expect( parseUserStrings(Buffer.from([0]), layout, { @@ -1976,7 +1982,11 @@ describe("managed user string scan completeness", () => { it("marks user strings partial when an inspected method row was not read", () => { const layout = { - userStrings: { name: "#US", offset: 0, size: 1 }, + userStrings: { + kind: "available", + stream: { name: "#US", offset: 0, size: 1 }, + issue: null, + }, } as unknown as ManagedMetadataLayout; // The omitted row's body, and any ldstr in it, was never scanned. expect( diff --git a/src/dotnet/ManagedMemberRows.ts b/src/dotnet/ManagedMemberRows.ts index 1197dcd1d3..68938aad01 100644 --- a/src/dotnet/ManagedMemberRows.ts +++ b/src/dotnet/ManagedMemberRows.ts @@ -38,6 +38,7 @@ import { metadataRowCursor, decodeMetadataCodedIndex, readMetadataBlob, + metadataBlobOffset, readMetadataString, } from "../domain/managed/managedMetadataHeaps.js"; import { @@ -123,12 +124,7 @@ export const parseFields = ( kind: "field", }); if (!isSelected(selected, declared)) return; - const sig = readMetadataBlob( - bytes, - layout, - signatureIndex, - layout.blob?.size ?? 0, - ); + const sig = readMetadataBlob(bytes, layout, signatureIndex); const decoded = budgetedSignature( sig, layout, @@ -270,12 +266,7 @@ export const parseMemberRefs = ( nameIndex, layout.strings.size, ); - const sig = readMetadataBlob( - bytes, - layout, - signatureIndex, - layout.blob?.size ?? 0, - ); + const sig = readMetadataBlob(bytes, layout, signatureIndex); if (parent.kind !== "valid") issues.push({ code: "invalid-row", @@ -431,12 +422,7 @@ export const parseMethods = ({ core.set(token, member); if (!inPage) return; if (!isSelected(selected, declared)) return; - const sig = readMetadataBlob( - bytes, - layout, - signatureIndex, - layout.blob?.size ?? 0, - ); + const sig = readMetadataBlob(bytes, layout, signatureIndex); const decoded = budgetedSignature( sig, layout, @@ -690,10 +676,7 @@ const budgetedSignature = ( }, ): ReturnType => { const budget = managedDecodeBudget(layout); - budget.admitTransient( - bytes.length * 256, - layout.blob?.offset ?? layout.rootOffset, - ); + budget.admitTransient(bytes.length * 256, metadataBlobOffset(layout)); try { return admitManagedProjection( metadataSignature(bytes, { @@ -704,7 +687,7 @@ const budgetedSignature = ( if (name !== null) budget.admitTransient( (name.length * 6 + 2) * copies, - layout.blob?.offset ?? layout.rootOffset, + metadataBlobOffset(layout), ); return name; }, diff --git a/src/dotnet/ManagedMethodBodyReader.ts b/src/dotnet/ManagedMethodBodyReader.ts index f51972c49c..9ae9721937 100644 --- a/src/dotnet/ManagedMethodBodyReader.ts +++ b/src/dotnet/ManagedMethodBodyReader.ts @@ -29,7 +29,10 @@ import { readLocalSignatureFacts, isParsedTokenInstruction, } from "../domain/managed/managedMemberInspectorCore.js"; -import { ManagedReaderFailure } from "../domain/managed/managedReaderFailure.js"; +import { + ManagedHeapAmbiguityFailure, + ManagedReaderFailure, +} from "../domain/managed/managedReaderFailure.js"; import type { ManagedMetadataLayout } from "../domain/managed/managedMetadataLayout.js"; import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; import { sha256Hex } from "../kernel/sha256Hex.js"; @@ -253,7 +256,6 @@ export const methodBody = ( bytes, context.metadataLayout, cursor.readIndex(context.metadataLayout.blobIndexSize), - context.metadataLayout.blob?.size ?? 0, ); const signatureFacts = readLocalSignatureFacts( context.metadataLayout, @@ -282,8 +284,12 @@ export const methodBody = ( throw cause; if (!(cause instanceof ManagedReaderFailure)) throw cause; localSigIssue = cause.issue.detail; - localSigFailure = "malformed"; - issues.push({ ...cause.issue, scope: signatureScope }); + if (cause instanceof ManagedHeapAmbiguityFailure) + localSigFailure = "partial"; + else { + localSigFailure = "malformed"; + issues.push({ ...cause.issue, scope: signatureScope }); + } } } } diff --git a/src/dotnet/ManagedMethodSpecs.ts b/src/dotnet/ManagedMethodSpecs.ts index 9ec3a35e40..5408616493 100644 --- a/src/dotnet/ManagedMethodSpecs.ts +++ b/src/dotnet/ManagedMethodSpecs.ts @@ -146,7 +146,7 @@ const readMethodSpec = ( }); const instantiationFacts = specSignatureFacts( layout, - readMetadataBlob(bytes, layout, instantiation, layout.blob?.size ?? 0), + readMetadataBlob(bytes, layout, instantiation), { form: "instantiation", scope, offset }, signatureTypeName, ); @@ -213,10 +213,5 @@ const methodSignatureBlob = ( row, ).signatureIndex; } - return readMetadataBlob( - bytes, - layout, - signatureIndex, - layout.blob?.size ?? 0, - ); + return readMetadataBlob(bytes, layout, signatureIndex); }; diff --git a/src/dotnet/ManagedNativeBoundaryInspector.ts b/src/dotnet/ManagedNativeBoundaryInspector.ts index a0580e77a6..4dae550a32 100644 --- a/src/dotnet/ManagedNativeBoundaryInspector.ts +++ b/src/dotnet/ManagedNativeBoundaryInspector.ts @@ -166,7 +166,7 @@ export const inspectManagedNativeBoundariesBytes = ( }); } try { - const heapExtent = Math.max(layout.strings.size, layout.blob?.size ?? 0); + const heapExtent = layout.strings.size; appendManagedParseIssues(issues, inventory.issues); const moduleRefs = parseModuleRefs(bytes, layout, heapExtent, issues); const members = new Map([ diff --git a/src/dotnet/ManagedPe.fixture.ts b/src/dotnet/ManagedPe.fixture.ts index 64949cbff0..a1497647da 100644 --- a/src/dotnet/ManagedPe.fixture.ts +++ b/src/dotnet/ManagedPe.fixture.ts @@ -85,6 +85,8 @@ interface ManagedPeFixtureOptions { }[]; /** Raw #US stream bytes; omitted means the metadata has no #US stream. */ readonly userStringHeap?: Buffer; + readonly tableStreamName?: "#~" | "#-"; + readonly competingTableStream?: "distinct" | "alias" | "outside-root"; readonly virtualSectionSize?: number; } @@ -530,9 +532,14 @@ const metadataRoot = ( readonly userStrings?: Buffer | undefined; }, tableStreamName: "#~" | "#-" = "#~", + competingTableStream?: ManagedPeFixtureOptions["competingTableStream"], ): Buffer => { + const alternateName = tableStreamName === "#~" ? "#-" : "#~"; const streams: readonly (readonly [string, Buffer])[] = [ [tableStreamName, tables], + ...(competingTableStream === undefined + ? [] + : [[alternateName, Buffer.alloc(24)] as const]), ["#Strings", strings], ...(userStrings === undefined ? [] : [["#US", userStrings] as const]), ["#GUID", guid], @@ -560,9 +567,13 @@ const metadataRoot = ( return { name, data, at }; }); const headers = Buffer.concat( - placed.map(({ name, data, at }) => - metadataStreamHeader(at, data.length, name), - ), + placed.map(({ name, data, at }, index) => { + if (index === 1 && competingTableStream === "alias") + return metadataStreamHeader(placed[0]!.at, tables.length, name); + if (index === 1 && competingTableStream === "outside-root") + return metadataStreamHeader(offset, data.length, name); + return metadataStreamHeader(at, data.length, name); + }), ); const root = Buffer.alloc(offset); header.copy(root, 0); @@ -1169,18 +1180,22 @@ const buildManagedFixtureMetadata = ( (stringSize === 4 ? 0x01 : 0) | (guidSize === 4 ? 0x02 : 0) | (blobSize === 4 ? 0x04 : 0); - const metadata = metadataRoot({ - tables: tablesStream( - rows, - options.metadataValidMaskExtra ?? 0n, - heapFlags, - options.metadataExtraData, - ), - strings: strings.toBuffer(), - guid: guidHeap(options.mvid), - blob: blobs.toBuffer(), - userStrings: options.userStringHeap, - }); + const metadata = metadataRoot( + { + tables: tablesStream( + rows, + options.metadataValidMaskExtra ?? 0n, + heapFlags, + options.metadataExtraData, + ), + strings: strings.toBuffer(), + guid: guidHeap(options.mvid), + blob: blobs.toBuffer(), + userStrings: options.userStringHeap, + }, + options.tableStreamName, + options.competingTableStream, + ); if (options.corruptMetadataSignature === true) metadata.writeUInt32LE(0, 0); return metadata; }; diff --git a/src/dotnet/ManagedTableStreamConflict.test.ts b/src/dotnet/ManagedTableStreamConflict.test.ts new file mode 100644 index 0000000000..1fe3d13046 --- /dev/null +++ b/src/dotnet/ManagedTableStreamConflict.test.ts @@ -0,0 +1,121 @@ +import { expect, it } from "vitest"; + +import { readManagedPeLayout } from "../domain/managed/managedPeReader.js"; +import { inspectManagedArtifactBytes } from "./ManagedArtifactInspector.js"; +import { inspectManagedMembersBytes } from "./ManagedMemberInspector.js"; +import { inspectManagedNativeBoundariesBytes } from "./ManagedNativeBoundaryInspector.js"; +import { + buildManagedPeFixture, + managedPeFixtureTarget, +} from "./ManagedPe.fixture.js"; + +const tableHeaderLocations = (bytes: Buffer) => { + const pe = readManagedPeLayout(bytes); + if (pe.cli === null) throw new Error("Fixture omitted CLI metadata"); + const root = pe.rvaToOffset( + pe.cli.metadata.rva, + pe.cli.metadata.size, + "fixture metadata", + ); + // The fixture emits the two four-byte table names before the other streams. + return { + firstHeader: root + 32, + secondHeader: root + 44, + metadataEnd: root + pe.cli.metadata.size, + }; +}; + +it.each(["#~", "#-"] as const)( + "reads complete rows from a single %s table stream", + (tableStreamName) => { + const bytes = buildManagedPeFixture({ tableStreamName }); + const target = managedPeFixtureTarget(bytes); + const artifact = inspectManagedArtifactBytes(bytes, target); + const members = inspectManagedMembersBytes(bytes, target); + expect(artifact.metadata).toMatchObject({ + status: "complete", + stream_names: expect.arrayContaining([tableStreamName]), + table_row_counts: { Module: 1, MethodDef: 1 }, + }); + expect(members.coverage.state).toBe("complete"); + expect(members.methods).toHaveLength(1); + expect(members.module?.mvid).toBeTruthy(); + }, +); + +it.each([ + ["#~", "distinct"], + ["#-", "distinct"], + ["#~", "alias"], +] as const)( + "preserves only independent facts with %s first and a %s table declaration", + (tableStreamName, competingTableStream) => { + const bytes = buildManagedPeFixture({ + tableStreamName, + competingTableStream, + }); + const { firstHeader, secondHeader } = tableHeaderLocations(bytes); + const target = managedPeFixtureTarget(bytes); + const artifact = inspectManagedArtifactBytes(bytes, target); + const members = inspectManagedMembersBytes(bytes, target, { + typeNames: ["Fixture.Program"], + }); + const boundaries = inspectManagedNativeBoundariesBytes(bytes, target); + const issue = { + code: "unsupported-metadata", + scope: "metadata.tables", + detail: expect.stringContaining("Metadata row layout is unknown."), + offset: secondHeader, + }; + expect(artifact).toMatchObject({ + classification: { status: "managed" }, + metadata: { stream_names: expect.arrayContaining(["#~", "#-"]) }, + assembly: null, + coverage: { state: "partial", issues: [issue] }, + }); + const observed = artifact.coverage.issues[0]?.detail; + expect(observed).toContain(`${tableStreamName} at header ${firstHeader}`); + expect(observed).toContain( + `${tableStreamName === "#~" ? "#-" : "#~"} at header ${secondHeader}`, + ); + for (const report of [artifact, members, boundaries]) { + expect(report).toMatchObject({ + artifact: { sha256: target.sha256 }, + metadata: { + status: "partial", + version: "v4.0.30319", + table_row_counts: {}, + }, + module: null, + }); + } + for (const report of [members, boundaries]) { + expect(report).toMatchObject({ + identity_scope: { requires_mvid: null }, + coverage: { state: "unavailable", issues: [issue] }, + }); + } + expect(members).toMatchObject({ + methods: [], + fields: [], + method_rows: null, + selection: { type_names: ["Fixture.Program"], unmatched_type_names: [] }, + }); + expect(boundaries.pinvoke_imports).toEqual([]); + }, +); + +it("validates a competing declaration's extent before resolving table ambiguity", () => { + const bytes = buildManagedPeFixture({ competingTableStream: "outside-root" }); + const artifact = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + expect(artifact.metadata.status).toBe("malformed"); + expect(artifact.coverage.issues).toContainEqual({ + code: "invalid-metadata-root", + scope: "metadata.#-", + detail: "metadata.#- leaves the metadata byte range", + offset: tableHeaderLocations(bytes).metadataEnd, + }); +}); diff --git a/src/dotnet/ManagedTypeNames.ts b/src/dotnet/ManagedTypeNames.ts index 8b241f4ac0..c77296df28 100644 --- a/src/dotnet/ManagedTypeNames.ts +++ b/src/dotnet/ManagedTypeNames.ts @@ -56,7 +56,6 @@ export const typeNameLookup = ( bytes, layout, cursor.readIndex(layout.blobIndexSize), - layout.blob?.size ?? 0, ); const owner = { form: "type", diff --git a/src/dotnet/ManagedUserStrings.ts b/src/dotnet/ManagedUserStrings.ts index 59bf59b8e7..b6a2ab90d4 100644 --- a/src/dotnet/ManagedUserStrings.ts +++ b/src/dotnet/ManagedUserStrings.ts @@ -57,6 +57,9 @@ export const parseUserStrings = ( (parsed.unreadableSelectedTypes ?? 0) === 0; const userStrings: ManagedUserString[] = []; const issues: ManagedParseIssue[] = []; + const declaration = layout.userStrings; + if (declaration?.issue) + issues.push(admitManagedProjection(declaration.issue, budget)); const tokens = new Map(); try { // Only ldstr loads a literal; a malformed ldtoken can carry a 0x70 token. @@ -85,7 +88,7 @@ export const parseUserStrings = ( ), ); } - if (references.length > 0 && layout.userStrings === null) { + if (references.length > 0 && declaration === null) { // A missing stream is one fact about the metadata, not one per token. issues.push( admitManagedProjection( @@ -100,6 +103,9 @@ export const parseUserStrings = ( ); return inventory(userStrings, issues, scanned); } + if (declaration?.kind === "ambiguous") + return inventory(userStrings, issues, scanned); + for (const reference of references) { const failures: ManagedParseIssue[] = []; readManagedValue(() => { diff --git a/tests/acceptance/managedCliOutput.test.ts b/tests/acceptance/managedCliOutput.test.ts index e5f4e5efa9..490645dd22 100644 --- a/tests/acceptance/managedCliOutput.test.ts +++ b/tests/acceptance/managedCliOutput.test.ts @@ -9,6 +9,43 @@ import { buildManagedPeFixture } from "../../src/dotnet/ManagedPe.fixture.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { cliTest } from "../support/cli/cliFixture.js"; +cliTest( + "returns partial Evidence when managed table declarations conflict", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-managed-cli-table-conflict-", + ); + const path = join(root, "fixture.dll"); + await writeFile( + path, + buildManagedPeFixture({ competingTableStream: "distinct" }), + ); + const result = await cli.run({ + arguments: ["inspect-managed-members", path, "--full-output", "--json"], + }); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.json).toMatchObject({ + data: { + normalized_result: { + metadata: { + status: "partial", + version: "v4.0.30319", + table_row_counts: {}, + }, + methods: [], + method_rows: null, + coverage: { + state: "unavailable", + issues: [ + { code: "unsupported-metadata", scope: "metadata.tables" }, + ], + }, + }, + }, + }); + }, +); + cliTest( "delivers a managed artifact summary beside its complete Evidence bundle", async ({ cli }) => { diff --git a/tests/boundary/filesystem/desktopApplication.test.ts b/tests/boundary/filesystem/desktopApplication.test.ts index df66343d8b..8525ecad3c 100644 --- a/tests/boundary/filesystem/desktopApplication.test.ts +++ b/tests/boundary/filesystem/desktopApplication.test.ts @@ -502,6 +502,30 @@ it.each(["Game_Data/Managed", "Contents/Resources/Data/Managed"])( }, ); +it("retains PE/CLI evidence without inferring Unity from competing managed table streams", async () => { + const root = await createTestTempDirectory("rea-desktop-table-conflict-"); + const managedPath = "Game_Data/Managed/Assembly-CSharp.dll"; + await put( + root, + managedPath, + buildManagedPeFixture({ + references: ["UnityEngine.CoreModule"], + competingTableStream: "distinct", + }), + ); + const { report } = await inspect(root); + expect(report.coverage.status).toBe("partial"); + expect(report.candidates.map(({ family }) => family)).toEqual(["dotnet"]); + const file = report.files.find(({ path }) => path === managedPath); + expect(file?.facts.map(({ kind }) => kind)).toContain("pe-cli-directory"); + expect(file?.facts.map(({ kind }) => kind)).not.toContain( + "unity-assembly-reference", + ); + expect(file?.issues).toContainEqual( + expect.stringContaining("Metadata row layout is unknown."), + ); +}); + it("orders member probes for Unity predefined script assemblies before every managed artifact probe", async () => { const root = await createTestTempDirectory("rea-desktop-mono-members-"); const unity = buildManagedPeFixture({ @@ -1380,9 +1404,9 @@ it.each([ metadataOffset, image.length - metadataOffset, ); - if (layout.blob === null) + if (layout.blob?.kind !== "available") throw new Error("Fixture requires a #Blob stream"); - const blobSize = layout.blob.size; + const blobSize = layout.blob.stream.size; const reference = layout.table(35)!; // ECMA-335 AssemblyRef: 8 version bytes, 4 flag bytes, then PublicKeyOrToken. // Keep its name valid while making that unrelated blob index invalid. diff --git a/tests/fixtures/managedSelectedScope.ts b/tests/fixtures/managedSelectedScope.ts index d45512863d..aeb2d21ce5 100644 --- a/tests/fixtures/managedSelectedScope.ts +++ b/tests/fixtures/managedSelectedScope.ts @@ -43,12 +43,7 @@ export const distinctSelectedNestedOwners = () => { const method = readMethodDefinitionRow(bytes, layout, 1); if (index === 1) { bytes.writeUInt16LE(3, metadataRowOffset(layout, 41, 1) + 2); - readMetadataBlob( - bytes, - layout, - method.signatureIndex, - layout.blob!.size, - )[2] = 0x0a; + readMetadataBlob(bytes, layout, method.signatureIndex)[2] = 0x0a; const field = metadataRowCursor(bytes, layout, 4, 1); field.u16(); field.readIndex(layout.stringIndexSize); @@ -56,7 +51,6 @@ export const distinctSelectedNestedOwners = () => { bytes, layout, field.readIndex(layout.blobIndexSize), - layout.blob!.size, )[1] = 0x0a; } // All three MethodDefs share this fixture body and signature. Tiny bodies