From 4e11582923fb43e2d84331ded6b60342671cfdb9 Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 12 Oct 2026 08:01:55 -0400 Subject: [PATCH] feat(managed): decode ldstr user strings in member inspection inspect_managed_members reported every ldstr operand as a bare 0x70 token, so an analyst could see that a method loads a literal but not which one. Answering "where is this path or key used" required a second tool. Read the optional #US stream and add a top-level user_strings inventory: each distinct token loaded by an inspected method body, once, in token order, with its #US file offset and decoded UTF-16 value. Instruction anchors and normalized CIL digests keep their build-local tokens. A missing #US stream or malformed item is reported against its token as partial coverage. Evidence recorded before this field reads user_strings as null. Co-Authored-By: Claude Opus 5.5 --- docs/managed-code-analysis.md | 14 ++- .../managed/managedWorkflowExamples.ts | 1 + src/domain/managed/managedArtifact.ts | 14 +++ src/dotnet/ManagedMemberInspector.test.ts | 104 +++++++++++++++- src/dotnet/ManagedMemberInspector.ts | 8 +- src/dotnet/ManagedMemberInspectorCore.ts | 3 + src/dotnet/ManagedMemberRows.ts | 32 +++++ src/dotnet/ManagedMetadataHeaps.test.ts | 1 + src/dotnet/ManagedMetadataHeaps.ts | 52 ++++++++ src/dotnet/ManagedMetadataLayout.ts | 3 + src/dotnet/ManagedPe.fixture.ts | 117 ++++++++++++------ 11 files changed, 307 insertions(+), 42 deletions(-) diff --git a/docs/managed-code-analysis.md b/docs/managed-code-analysis.md index 386b7b6d59..e15681abfa 100644 --- a/docs/managed-code-analysis.md +++ b/docs/managed-code-analysis.md @@ -84,14 +84,24 @@ The result preserves the requested names in `selection.type_names` and reports unmatched names separately. If metadata names are unreadable, limitations retain that uncertainty. With no selection, `selection` is null and whole-assembly inspection keeps its existing resource policy. Selection restricts types, -fields, methods, call edges and field accesses; member references remain an -assembly-wide inventory, so output size is not guaranteed to be small. +fields, methods, call edges, field accesses and user strings; member references +remain an assembly-wide inventory, so output size is not guaranteed to be small. Coverage describes the selected members. Comparisons leave members outside the opposite observation's selection unknown, and application graphs report their selected scope as partial. Neither makes an absence claim about the whole assembly. Older member Evidence without `selection` remains readable. +### User strings + +`ldstr` anchors keep their build-local `0x70` tokens. `user_strings` lists each +distinct token loaded by an inspected method body once, in token order, with +its `#US` file offset and decoded value. Values keep their stored UTF-16 code +units, including unpaired surrogates. A missing `#US` stream or a malformed +item is reported against that token and makes coverage partial; readable +literals remain. Normalized CIL digests do not include decoded values. Older +member Evidence without `user_strings` reads as `null`: not examined. + ## Analysis objective REA's managed-code track is intended to answer five different questions without diff --git a/src/contracts/managed/managedWorkflowExamples.ts b/src/contracts/managed/managedWorkflowExamples.ts index 85297d671d..dc89df9932 100644 --- a/src/contracts/managed/managedWorkflowExamples.ts +++ b/src/contracts/managed/managedWorkflowExamples.ts @@ -53,6 +53,7 @@ const emptyManagedMembers = ( member_refs: [], call_edges: [], field_accesses: [], + user_strings: [], selection: null, coverage: { state: "complete", issues: [] }, limitations: [], diff --git a/src/domain/managed/managedArtifact.ts b/src/domain/managed/managedArtifact.ts index 297b60a4eb..2afec74f4f 100644 --- a/src/domain/managed/managedArtifact.ts +++ b/src/domain/managed/managedArtifact.ts @@ -211,6 +211,16 @@ const managedMemberReferenceSchema = z.object({ signature: managedSignatureSchema, }); +const managedUserStringSchema = z.object({ + token: tokenSchema, + offset: offsetSchema.describe("File offset of the #US item"), + value: z + .string() + .describe( + "Decoded UTF-16 code units, excluding the terminal flag byte; unpaired surrogates are kept", + ), +}); + const managedModuleReferenceSchema = z.object({ token: tokenSchema, row_offset: offsetSchema, @@ -440,6 +450,10 @@ export const managedMemberInspectionSchema = z.object({ member_refs: z.array(managedMemberReferenceSchema), call_edges: z.array(managedCallEdgeSchema), field_accesses: z.array(managedFieldAccessSchema), + // Distinct #US literals loaded by ldstr in the inspected method bodies, in + // token order. Evidence recorded before user strings were read omits this + // field; null means they were not examined, not that there were none. + user_strings: z.array(managedUserStringSchema).nullable().default(null), /** Caller-selected declaring types; null when every type was inspected. */ selection: z .object({ diff --git a/src/dotnet/ManagedMemberInspector.test.ts b/src/dotnet/ManagedMemberInspector.test.ts index 81d64f4a17..a352589434 100644 --- a/src/dotnet/ManagedMemberInspector.test.ts +++ b/src/dotnet/ManagedMemberInspector.test.ts @@ -11,9 +11,23 @@ import { comparisonLimitations } from "../domain/managed/managedMemberComparison import { buildManagedPeFixture, buildNativePeFixture, + fixtureUserStringHeap, managedPeFixtureTarget, } from "./ManagedPe.fixture.js"; +/** A tiny-header body that loads each `ldstr` token, discards it, and returns. */ +const ldstrBody = (tokens: readonly string[]): Buffer => { + const il = Buffer.concat([ + ...tokens.map((token) => { + const instruction = Buffer.from([0x72, 0, 0, 0, 0, 0x26]); + instruction.writeUInt32LE(Number.parseInt(token, 16), 1); + return instruction; + }), + Buffer.from([0x2a]), + ]); + return Buffer.concat([Buffer.from([(il.length << 2) | 2]), il]); +}; + describe("managed member inspection", () => { it("keeps metadata handles distinct from calls and field accesses", () => { const il = Buffer.from([ @@ -247,6 +261,94 @@ describe("managed exported types", () => { }); }); +describe("managed user strings", () => { + const inspect = (options: Parameters[0]) => { + const bytes = buildManagedPeFixture(options); + return inspectManagedMembersBytes(bytes, managedPeFixtureTarget(bytes)); + }; + + it("decodes each loaded literal once, in token order, without changing CIL tokens", () => { + const { heap, tokens } = fixtureUserStringHeap([ + "quicksave.peak", + "unused", + "Ünïcødé 𝄞", + "\ud800", + ]); + const [path, , unicode, lone] = tokens as [string, string, string, string]; + const ilBody = ldstrBody([unicode, path, lone, path]); + const result = inspect({ ilBody, userStringHeap: heap }); + + expect(result.user_strings).toEqual([ + { token: path, offset: expect.any(Number), value: "quicksave.peak" }, + { token: unicode, offset: expect.any(Number), value: "Ünïcødé 𝄞" }, + { token: lone, offset: expect.any(Number), value: "\ud800" }, + ]); + expect(result.coverage).toEqual({ state: "complete", issues: [] }); + const body = result.methods[0]?.body; + expect(body?.anchors.map(({ operand }) => operand)).toEqual([ + unicode, + path, + lone, + path, + ]); + expect(body?.normalized_il_sha256).toBe( + inspect({ ilBody }).methods[0]?.body.normalized_il_sha256, + ); + }); + + it("keeps the method and reports partial coverage when #US is absent", () => { + const result = inspect({ ilBody: ldstrBody(["0x70000001"]) }); + + expect(result.methods).toHaveLength(1); + expect(result.user_strings).toEqual([]); + expect(result.coverage).toEqual({ + state: "partial", + issues: [ + { + code: "missing-stream", + scope: "metadata.#US:0x70000001", + offset: null, + detail: "ldstr references #US, but the metadata has no #US stream", + }, + ], + }); + }); + + it("retains readable literals beside malformed #US items", () => { + const { heap, tokens } = fixtureUserStringHeap(["ok", "bad"]); + const [ok, bad] = tokens as [string, string]; + // "bad" encodes 7 bytes; an even length cannot end in the flag byte. + const badOffset = Number.parseInt(bad.slice(4), 16); + heap.writeUInt8(6, badOffset); + const result = inspect({ + ilBody: ldstrBody([ok, bad, "0x70ffffff"]), + userStringHeap: heap, + }); + + expect(result.user_strings).toEqual([ + { token: ok, offset: expect.any(Number), value: "ok" }, + ]); + expect(result.coverage.state).toBe("partial"); + expect( + result.coverage.issues.map(({ code, scope }) => [code, scope]), + ).toEqual([ + ["invalid-string", `metadata.#US:${bad}`], + ["invalid-heap-index", "metadata.#US:0x70ffffff"], + ]); + }); + + it("leaves user strings unexamined when CLI metadata is unavailable or predates them", () => { + const native = buildNativePeFixture(); + expect( + inspectManagedMembersBytes(native, managedPeFixtureTarget(native)) + .user_strings, + ).toBeNull(); + + const { user_strings: _omitted, ...legacy } = inspect({}); + expect(managedMemberInspectionSchema.parse(legacy).user_strings).toBeNull(); + }); +}); + describe("managed member decoding budget", () => { it("admits a wide-instruction method whose actual records fit the budget", () => { // Five-byte branches all target the next instruction. A final ret leaves @@ -885,7 +987,7 @@ describe("managed member type selection", () => { expect(result.field_accesses).toEqual(everything.field_accesses); expect(result.coverage).toEqual(everything.coverage); expect(result.limitations).toContain( - "Types, fields, methods, call edges, and field accesses cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", + "Types, fields, methods, call edges, field accesses, and user strings cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", ); }); diff --git a/src/dotnet/ManagedMemberInspector.ts b/src/dotnet/ManagedMemberInspector.ts index 24b97f9ebe..7e6c546e43 100644 --- a/src/dotnet/ManagedMemberInspector.ts +++ b/src/dotnet/ManagedMemberInspector.ts @@ -33,6 +33,7 @@ import { parseFields, parseMemberRefs, parseMethods, + parseUserStrings, type SelectedTypeTokens, } from "./ManagedMemberRows.js"; @@ -68,6 +69,7 @@ const unavailable = ( }, types: [], exported_types: null, + user_strings: null, fields: [], methods: [], member_refs: [], @@ -181,6 +183,8 @@ export const inspectManagedMembersBytes = ( selected, }); issues.push(...fields.issues, ...methods.issues); + const userStrings = parseUserStrings(bytes, layout, methods.methods); + issues.push(...userStrings.issues); const coverageIssues = uniqueManagedParseIssues(issues); const incompleteMembers = [...fields.fields, ...memberRefs.refs, ...methods.methods].some( @@ -228,6 +232,7 @@ export const inspectManagedMembersBytes = ( member_refs: memberRefs.refs, call_edges: related.callEdges, field_accesses: related.fieldAccesses, + user_strings: userStrings.userStrings, selection, coverage: { state: @@ -240,11 +245,12 @@ export const inspectManagedMembersBytes = ( "Metadata tokens are build-local coordinates and are only meaningful with the reported artifact SHA-256 and MVID.", "CIL instruction anchors are decoded from file-backed method bodies only; no target assembly is loaded or executed.", "Exported types are reported from ExportedType rows as declared; forwarded types are not resolved in their implementing assemblies.", + "User strings are the #US literals loaded by ldstr in the inspected method bodies; instruction anchors and normalized CIL digests keep their build-local tokens.", "Member signatures decode ECMA-335 primitive, class, valuetype, generic instantiation, generic variable, pointer, byref, single-dimension and general array, function pointer, custom modifier, and typed reference forms; class and valuetype operands keep their encoded TypeDefOrRefOrSpec values, and unsupported forms such as vararg sentinels retain raw signature hashes.", ...(selection === null ? [] : [ - "Types, fields, methods, call edges, and field accesses cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", + "Types, fields, methods, call edges, field accesses, and user strings cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", ]), ...(unreadableNames === 0 ? [] diff --git a/src/dotnet/ManagedMemberInspectorCore.ts b/src/dotnet/ManagedMemberInspectorCore.ts index f54a117699..b5d716b47d 100644 --- a/src/dotnet/ManagedMemberInspectorCore.ts +++ b/src/dotnet/ManagedMemberInspectorCore.ts @@ -34,6 +34,9 @@ export type ManagedMemberRef = ManagedMemberInspection["member_refs"][number]; export type ManagedCallEdge = ManagedMemberInspection["call_edges"][number]; export type ManagedFieldAccess = ManagedMemberInspection["field_accesses"][number]; +export type ManagedUserString = NonNullable< + ManagedMemberInspection["user_strings"] +>[number]; export type ManagedSignature = ManagedMethod["signature"]; export type ManagedMethodBody = ManagedMethod["body"]; export type ManagedInstructionAnchor = ManagedMethodBody["anchors"][number]; diff --git a/src/dotnet/ManagedMemberRows.ts b/src/dotnet/ManagedMemberRows.ts index f4a63db5a7..94af79c1ed 100644 --- a/src/dotnet/ManagedMemberRows.ts +++ b/src/dotnet/ManagedMemberRows.ts @@ -16,6 +16,7 @@ import { type ManagedFieldAccess, type ManagedMemberRef, type ManagedMethod, + type ManagedUserString, type MemberRefCore, type MethodCore, type TypeRange, @@ -27,6 +28,7 @@ import { metadataCodedTokenInvalidReason, readMetadataBlob, readMetadataString, + readMetadataUserString, } from "./ManagedMetadataHeaps.js"; import { readManagedValue } from "./ManagedReaderFailure.js"; import { methodBody } from "./ManagedMethodBodyReader.js"; @@ -317,6 +319,36 @@ export const parseMethods = ({ return { methods, core, issues }; }; +/** Decode each distinct #US literal an inspected method loads, in token order. */ +export const parseUserStrings = ( + bytes: Buffer, + layout: ManagedMetadataLayout, + methods: readonly ManagedMethod[], +): { + readonly userStrings: readonly ManagedUserString[]; + readonly issues: readonly ManagedParseIssue[]; +} => { + const tokens = new Set(); + for (const method of methods) + for (const anchor of method.body.anchors) + if (anchor.operand_kind === "string" && anchor.operand !== null) + tokens.add(anchor.operand); + const userStrings: ManagedUserString[] = []; + const issues: ManagedParseIssue[] = []; + // Tokens are fixed-width lowercase hex, so string order is token order. + for (const token of [...tokens].sort()) + readManagedValue(() => { + const { offset, value } = readMetadataUserString(bytes, layout, token); + userStrings.push( + admitManagedProjection( + { token, offset, value }, + managedDecodeBudget(layout), + ), + ); + }, issues); + return { userStrings, issues }; +}; + const targetKind = (token: string): ManagedCallEdge["target_kind"] => token.startsWith("0x06") ? "method-def" diff --git a/src/dotnet/ManagedMetadataHeaps.test.ts b/src/dotnet/ManagedMetadataHeaps.test.ts index c94ae40553..f72f915e74 100644 --- a/src/dotnet/ManagedMetadataHeaps.test.ts +++ b/src/dotnet/ManagedMetadataHeaps.test.ts @@ -16,6 +16,7 @@ describe("managed metadata heaps", () => { strings: { name: "#Strings", offset: 0, size: 0 }, guid: { name: "#GUID", offset: 0, size: 0 }, blob: { name: "#Blob", offset: 0, size: bytes.length }, + userStrings: null, tables: new Map(), rowCounts: [], stringIndexSize: 2, diff --git a/src/dotnet/ManagedMetadataHeaps.ts b/src/dotnet/ManagedMetadataHeaps.ts index 51b85576c5..3f46587876 100644 --- a/src/dotnet/ManagedMetadataHeaps.ts +++ b/src/dotnet/ManagedMetadataHeaps.ts @@ -191,6 +191,58 @@ export const readMetadataBlob = ( return bytes.subarray(content, content + decoded.length); }; +/** + * Read the #US literal named by an `ldstr` token: a compressed byte length, + * UTF-16LE code units, then one ECMA-335 terminal flag byte. Code units are + * kept as stored, including unpaired surrogates. + */ +export const readMetadataUserString = ( + bytes: Buffer, + layout: ManagedMetadataLayout, + token: string, +): { readonly offset: number; readonly value: string } => { + const heap = layout.userStrings; + const scope = `metadata.#US:${token}`; + if (heap === null) + throw managedFailure( + "missing-stream", + scope, + "ldstr references #US, but the metadata has no #US stream", + ); + const index = Number.parseInt(token.slice(4), 16); + if (!token.startsWith("0x70") || index >= heap.size) + throw managedFailure( + "invalid-heap-index", + scope, + "ldstr operand does not name an item inside #US", + token.startsWith("0x70") ? heap.offset + index : null, + ); + const start = heap.offset + index; + const decoded = compressedLength(bytes, start, scope); + const content = start + decoded.prefix; + if (content > heap.offset + heap.size - decoded.length) + throw managedFailure( + "invalid-string", + scope, + "User string leaves #US", + start, + ); + if (decoded.length % 2 !== 1) + throw managedFailure( + "invalid-string", + scope, + "User string length is not UTF-16 code units plus one terminal flag byte", + start, + ); + const units = (decoded.length - 1) / 2; + // JSON spends at most six bytes per UTF-16 code unit, the same bound as UTF-8 bytes. + managedDecodeBudget(layout).reserveString(units, start); + return { + offset: start, + value: bytes.subarray(content, content + units * 2).toString("utf16le"), + }; +}; + /** Read one conventional GUID value from the one-based #GUID heap. */ export const readMetadataGuid = ( bytes: Buffer, diff --git a/src/dotnet/ManagedMetadataLayout.ts b/src/dotnet/ManagedMetadataLayout.ts index b7dc0a7aeb..b388546453 100644 --- a/src/dotnet/ManagedMetadataLayout.ts +++ b/src/dotnet/ManagedMetadataLayout.ts @@ -78,6 +78,8 @@ export interface ManagedMetadataLayout { readonly strings: MetadataStream; readonly guid: MetadataStream; readonly blob: MetadataStream; + /** Optional: assemblies without `ldstr` literals may omit #US. */ + readonly userStrings: MetadataStream | null; readonly tables: ReadonlyMap; readonly rowCounts: readonly number[]; readonly stringIndexSize: 2 | 4; @@ -431,6 +433,7 @@ export const readManagedMetadataLayout = ( strings: requireStream(streams, "#Strings"), guid: requireStream(streams, "#GUID"), blob: requireStream(streams, "#Blob"), + userStrings: streams.get("#US") ?? null, tables, rowCounts, stringIndexSize: heaps.string, diff --git a/src/dotnet/ManagedPe.fixture.ts b/src/dotnet/ManagedPe.fixture.ts index 25f8199259..5406c7bd2f 100644 --- a/src/dotnet/ManagedPe.fixture.ts +++ b/src/dotnet/ManagedPe.fixture.ts @@ -43,6 +43,8 @@ interface ManagedPeFixtureOptions { readonly targetFramework?: string | null; readonly typeName?: string; readonly typeNamespace?: string; + /** Raw #US stream bytes; omitted means the metadata has no #US stream. */ + readonly userStringHeap?: Buffer; readonly virtualSectionSize?: number; } @@ -406,6 +408,33 @@ const fixedStringAttributeBlob = (value: string): Buffer => { return Buffer.concat([Buffer.from([1, 0, bytes.length]), bytes, u16(0)]); }; +/** + * Lay out ECMA-335 #US items (compressed length, UTF-16LE, terminal flag byte) + * and return the `ldstr` token that names each value. + */ +export const fixtureUserStringHeap = ( + values: readonly string[], +): { readonly heap: Buffer; readonly tokens: readonly string[] } => { + const items: Buffer[] = [Buffer.from([0])]; + const tokens: string[] = []; + let size = 1; + for (const value of values) { + const text = Buffer.from(value, "utf16le"); + const length = text.length + 1; + if (length > 0x3fff) + throw new RangeError("Managed PE fixture user string is too long"); + const prefix = + length < 0x80 + ? Buffer.from([length]) + : Buffer.from([0x80 | (length >> 8), length & 0xff]); + tokens.push(`0x70${size.toString(16).padStart(6, "0")}`); + const item = Buffer.concat([prefix, text, Buffer.from([0])]); + items.push(item); + size += item.length; + } + return { heap: Buffer.concat(items), tokens }; +}; + const metadataStreamHeader = ( relativeOffset: number, size: number, @@ -418,12 +447,28 @@ const metadataStreamHeader = ( }; const metadataRoot = ( - tables: Buffer, - strings: Buffer, - guid: Buffer, - blob: Buffer, + { + tables, + strings, + guid, + blob, + userStrings, + }: { + readonly tables: Buffer; + readonly strings: Buffer; + readonly guid: Buffer; + readonly blob: Buffer; + readonly userStrings?: Buffer | undefined; + }, tableStreamName: "#~" | "#-" = "#~", ): Buffer => { + const streams: readonly (readonly [string, Buffer])[] = [ + [tableStreamName, tables], + ["#Strings", strings], + ...(userStrings === undefined ? [] : [["#US", userStrings] as const]), + ["#GUID", guid], + ["#Blob", blob], + ]; const version = Buffer.from("v4.0.30319\0\0", "utf8"); const header = Buffer.concat([ u32(0x424a_5342), @@ -433,34 +478,27 @@ const metadataRoot = ( u32(version.length), version, u16(0), - u16(4), + u16(streams.length), ]); - const streamHeaderSize = - metadataStreamHeader(0, 0, tableStreamName).length + - metadataStreamHeader(0, 0, "#Strings").length + - metadataStreamHeader(0, 0, "#GUID").length + - metadataStreamHeader(0, 0, "#Blob").length; + const streamHeaderSize = streams.reduce( + (total, [name]) => total + metadataStreamHeader(0, 0, name).length, + 0, + ); let offset = align4(header.length + streamHeaderSize); - const tablesOffset = offset; - offset = align4(offset + tables.length); - const stringsOffset = offset; - offset = align4(offset + strings.length); - const guidOffset = offset; - offset = align4(offset + guid.length); - const blobOffset = offset; - const headers = Buffer.concat([ - metadataStreamHeader(tablesOffset, tables.length, tableStreamName), - metadataStreamHeader(stringsOffset, strings.length, "#Strings"), - metadataStreamHeader(guidOffset, guid.length, "#GUID"), - metadataStreamHeader(blobOffset, blob.length, "#Blob"), - ]); - const root = Buffer.alloc(align4(blobOffset + blob.length)); + const placed = streams.map(([name, data]) => { + const at = offset; + offset = align4(offset + data.length); + return { name, data, at }; + }); + const headers = Buffer.concat( + placed.map(({ name, data, at }) => + metadataStreamHeader(at, data.length, name), + ), + ); + const root = Buffer.alloc(offset); header.copy(root, 0); headers.copy(root, header.length); - tables.copy(root, tablesOffset); - strings.copy(root, stringsOffset); - guid.copy(root, guidOffset); - blob.copy(root, blobOffset); + for (const { data, at } of placed) data.copy(root, at); return root; }; @@ -882,17 +920,18 @@ const buildManagedFixtureMetadata = ( (stringSize === 4 ? 0x01 : 0) | (guidSize === 4 ? 0x02 : 0) | (blobSize === 4 ? 0x04 : 0); - const metadata = metadataRoot( - tablesStream( + const metadata = metadataRoot({ + tables: tablesStream( rows, options.metadataValidMaskExtra ?? 0n, heapFlags, options.metadataExtraData, ), - strings.toBuffer(), - guidHeap(options.mvid), - blobs.toBuffer(), - ); + strings: strings.toBuffer(), + guid: guidHeap(options.mvid), + blob: blobs.toBuffer(), + userStrings: options.userStringHeap, + }); if (options.corruptMetadataSignature === true) metadata.writeUInt32LE(0, 0); return metadata; }; @@ -1120,10 +1159,12 @@ export const buildManagedMemberPointerFixture = ( if (fieldPointers !== null) rows.set(3, fieldPointers.map(u16)); if (methodPointers !== null) rows.set(5, methodPointers.map(u16)); return metadataRoot( - tablesStream(rows, 0n, 0), - strings.toBuffer(), - guidHeap(), - blobs.toBuffer(), + { + tables: tablesStream(rows, 0n, 0), + strings: strings.toBuffer(), + guid: guidHeap(), + blob: blobs.toBuffer(), + }, options.streamName ?? "#-", ); });