diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index fa2f207ea3..e79295aaab 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -95,46 +95,6 @@ export const analyzeJavaScriptArtifactFiles = ( return ir; }); -/** - * Project each file's full IR immediately, retaining only later module facts. - * Parsing, analysis and each projection pass run as separate steps, so control - * messages and cancellation are handled within one large file as well. - */ -export const analyzeAndProjectJavaScriptArtifactFiles = async ( - fileSet: JavaScriptArtifactFileSet, - projectSemantics: ( - file: JavaScriptArtifactFile, - ir: JavaScriptSemanticIr, - ) => Iterator | void, - beforeFile?: ( - file: JavaScriptArtifactFile, - completed: number, - total: number, - ) => Promise, - signal?: AbortSignal, -): Promise => { - const state = emptyArtifactAnalysis(); - const projectFileSemantics = function* ( - file: JavaScriptArtifactFile, - ir: JavaScriptSemanticIr, - ): Generator { - const projection = projectSemantics(file, ir); - if (projection !== undefined) while (projection.next().done !== true) yield; - return projectJavaScriptModuleSemantics(ir); - }; - const context = { state, projectSemantics: projectFileSemantics }; - for (const [index, file] of fileSet.files.entries()) { - // Let progress, cancellation and garbage collection run between files. - await setImmediate(); - await beforeFile?.(file, index, fileSet.files.length); - await completeJavaScriptAnalysisSteps( - analyzeArtifactFileSteps(file, context), - signal, - ); - } - return finalizeArtifactAnalysis(state); -}; - /** Analyze sources behind a hard heap boundary, retaining collected facts on interruption. */ export const analyzeAndAdoptJavaScriptArtifactFiles = async ( fileSet: JavaScriptArtifactFileSet, diff --git a/src/domain/webInventory.test.ts b/src/domain/webInventory.test.ts index 692a8ab2b9..e4ef77472b 100644 --- a/src/domain/webInventory.test.ts +++ b/src/domain/webInventory.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest"; import { createCapturedWebScriptReconciler, - reconcileWebScript, stableWebResources, stableWebScriptKey, } from "./webInventory.js"; @@ -57,7 +56,7 @@ describe("stable web inventory", () => { ); }); - it("sorts resources deterministically and reconciles only exact URLs", () => { + it("sorts resources deterministically and reconciles observed raw URLs", () => { const inputs = [ { url: "https://app.example.test/assets/vendor.js", @@ -76,31 +75,45 @@ describe("stable web inventory", () => { ]; const forward = stableWebResources([...inputs, ...inputs.slice(0, 1)]); const reverse = stableWebResources([...inputs].reverse()); + const rawResources = [...inputs, ...inputs.slice(0, 1)].map((resource) => ({ + ...resource, + rawUrl: resource.url, + })); + const reconcile = createCapturedWebScriptReconciler(rawResources, forward); expect(reverse).toEqual(forward); - expect(reconcileWebScript(script, forward)).toEqual({ + expect(reconcile({ ...script, rawUrl: script.url })).toEqual({ status: "exact", resource_key: forward.find(({ url }) => url === script.url)?.resource_key, }); expect( - reconcileWebScript({ ...script, url: `${script.url}?v=2` }, forward), + reconcile({ + ...script, + url: `${script.url}?v=2`, + rawUrl: `${script.url}?v=2`, + }), ).toEqual({ status: "unmatched", reason: "no_exact_sanitized_url" }); }); it("reports duplicate resource observations as ambiguous", () => { const duplicate = { url: script.url, + rawUrl: script.url, origin: "https://app.example.test", type: "Script", mime_type: "text/javascript", content_size: 1_024, }; - const resources = stableWebResources([ - duplicate, - { ...duplicate, content_size: 2_048 }, - ]); + const rawResources = [duplicate, { ...duplicate, content_size: 2_048 }]; + const resources = stableWebResources( + rawResources.map(({ rawUrl: _rawUrl, ...resource }) => resource), + ); + const reconcile = createCapturedWebScriptReconciler( + rawResources, + resources, + ); - expect(reconcileWebScript(script, resources)).toEqual({ + expect(reconcile({ ...script, rawUrl: script.url })).toEqual({ status: "ambiguous", candidate_resource_keys: resources .map(({ resource_key }) => resource_key) diff --git a/src/domain/webInventory.ts b/src/domain/webInventory.ts index 92ee43ff2f..b98efb56ea 100644 --- a/src/domain/webInventory.ts +++ b/src/domain/webInventory.ts @@ -64,24 +64,6 @@ export const stableWebResources = ( ); }; -/** Reconcile one script with resource-tree observations without fuzzy matching. */ -export const reconcileWebScript = ( - script: WebInventoryScriptInput, - resources: readonly StableWebResource[], -): WebResourceReconciliation => { - const candidates = resources.filter( - (resource) => resource.url === script.url, - ); - if (candidates.length === 1) - return { status: "exact", resource_key: candidates[0]?.resource_key ?? "" }; - if (candidates.length > 1) - return { - status: "ambiguous", - candidate_resource_keys: uniqueResourceKeys(candidates), - }; - return { status: "unmatched", reason: "no_exact_sanitized_url" }; -}; - /** Index one capture's observations without merging duplicate raw URLs. */ export const createCapturedWebScriptReconciler = ( rawResources: readonly (WebInventoryResourceInput & { @@ -140,8 +122,3 @@ export const stableWebScriptKey = (script: WebInventoryScriptInput): string => }, "Web inventory", )}`; - -const uniqueResourceKeys = ( - resources: readonly StableWebResource[], -): string[] => - [...new Set(resources.map(({ resource_key }) => resource_key))].sort(); diff --git a/src/dotnet/ManagedMemberInspector.ts b/src/dotnet/ManagedMemberInspector.ts index 6a5d221b8a..00d92a13f6 100644 --- a/src/dotnet/ManagedMemberInspector.ts +++ b/src/dotnet/ManagedMemberInspector.ts @@ -210,16 +210,12 @@ export const inspectManagedMembersBytes = ( const { operand: signatureTypeName, declaration: declarationTypeName } = managedTypeNameDigests(layout, issues, parsedTypes, typeReferences); const types = parsedTypes.types - .filter((type) => selected === null || selected.has(type.token)) - .map((type) => { + .filter(({ type }) => selected === null || selected.has(type.token)) + .map(({ row, type }) => { managedDecodeBudget(layout).reserve(512, type.row_offset); return { ...type, - name_sha256: - declarationTypeName( - 2, - Number.parseInt(type.token.slice(4), 16), - )?.toString("hex") ?? null, + name_sha256: declarationTypeName(2, row)?.toString("hex") ?? null, }; }); const envelope = managedPeInspectionEnvelope( diff --git a/src/dotnet/ManagedMemberInspectorCore.ts b/src/dotnet/ManagedMemberInspectorCore.ts index 3f3105b209..1027f793b3 100644 --- a/src/dotnet/ManagedMemberInspectorCore.ts +++ b/src/dotnet/ManagedMemberInspectorCore.ts @@ -37,6 +37,12 @@ import { memberListTable } from "./ManagedMemberPointers.js"; import type { TypeMemberListFacts } from "./ManagedTypeMemberLists.js"; export type ManagedType = ManagedMemberInspection["types"][number]; +/** A renderable TypeDef projection paired with its original source row. */ +export interface DecodedTypeDefinition { + readonly row: number; + readonly type: ManagedType; +} + export type ManagedExportedType = NonNullable< ManagedMemberInspection["exported_types"] >[number]; @@ -226,13 +232,13 @@ export const parseTypes = ( layout: ManagedMetadataLayout, definitions: readonly TypeDefinition[], ): { - readonly types: readonly ManagedType[]; + readonly types: readonly DecodedTypeDefinition[]; readonly issues: readonly ManagedParseIssue[]; readonly nesting: ManagedTypeNesting; } => { const fields = memberListTable(layout, 4); const methods = memberListTable(layout, 6); - const items: ManagedType[] = []; + const items: DecodedTypeDefinition[] = []; const issues: ManagedParseIssue[] = []; const nesting = readManagedTypeNesting(bytes, layout, issues); for (const definition of definitions) { @@ -273,26 +279,31 @@ export const parseTypes = ( detail: `TypeDef Extends coded index 0x${extendsRaw.toString(16)} is invalid: ${extendsIndex.reason}`, }); const enclosingRow = nesting.enclosing.get(row); + managedDecodeBudget(layout).reserve(128, cursor.start); items.push({ - token, - row_offset: cursor.start, - namespace, - name, - full_name: definition.fullName, - flags, - extends_token: extendsIndex.kind === "valid" ? extendsIndex.token : null, - enclosing_type_token: - enclosingRow === undefined ? null : metadataToken(2, enclosingRow), - name_sha256: null, - field_list: { - first_row: fieldRange.first, - last_row: fieldRange.last, - count: fieldRange.count, - }, - method_list: { - first_row: methodRange.first, - last_row: methodRange.last, - count: methodRange.count, + row, + type: { + token, + row_offset: cursor.start, + namespace, + name, + full_name: definition.fullName, + flags, + extends_token: + extendsIndex.kind === "valid" ? extendsIndex.token : null, + enclosing_type_token: + enclosingRow === undefined ? null : metadataToken(2, enclosingRow), + name_sha256: null, + field_list: { + first_row: fieldRange.first, + last_row: fieldRange.last, + count: fieldRange.count, + }, + method_list: { + first_row: methodRange.first, + last_row: methodRange.last, + count: methodRange.count, + }, }, }); } diff --git a/src/dotnet/ManagedMemberTypeInventory.test.ts b/src/dotnet/ManagedMemberTypeInventory.test.ts index f4eae02293..4fd7278c73 100644 --- a/src/dotnet/ManagedMemberTypeInventory.test.ts +++ b/src/dotnet/ManagedMemberTypeInventory.test.ts @@ -7,6 +7,17 @@ import { } from "./ManagedPe.fixture.js"; import { buildManagedMemberCapacityFixture } from "../../tests/fixtures/managedMemberCapacity.js"; import { readManagedPeLayout } from "./ManagedPeReader.js"; +import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; +import { + parseTypes, + readTypeDefinitions, +} from "./ManagedMemberInspectorCore.js"; +import { + readManagedMetadataInventory, + readManagedResourceDirectory, +} from "./ManagedMetadataInventory.js"; +import { managedMetadataTypeNames } from "./ManagedMetadataInventoryRows.js"; +import { readTypeMemberListFacts } from "./ManagedTypeMemberLists.js"; import { metadataRowOffset, readManagedMetadataLayout, @@ -206,3 +217,81 @@ it("does not turn an unreadable requested type name into observed absence during ]); expect(result.limitations.join(" ")).toContain("absence is not observed"); }); + +it("keeps later type identity, member ownership and constructor names at their original row", () => { + const bytes = buildManagedPeFixture({ + additionalEmptyTypes: 1, + customAttributeTypeRaw: (1 << 3) | 2, + typeName: "LookupAttribute", + extendsRaw: 0, + methodName: ".ctor", + methodFlags: 0x1886, + methodSignature: Buffer.from([0x20, 0x01, 0x01, 0x0e]), + }); + const healthy = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + const owner = healthy.types.find(({ token }) => token === "0x02000002"); + expect(owner?.full_name).toBe("Fixture.LookupAttribute"); + expect(owner?.name_sha256).toMatch(/^[0-9a-f]{64}$/u); + expect(healthy.fields).toHaveLength(1); + expect(healthy.methods).toHaveLength(1); + const pe = readManagedPeLayout(bytes); + if (pe.cli === null) throw new Error("Expected CLI fixture"); + const layout = readManagedMetadataLayout( + bytes, + pe.rvaToOffset(pe.cli.metadata.rva, pe.cli.metadata.size, "cli.metadata"), + pe.cli.metadata.size, + ); + expect(layout.stringIndexSize).toBe(2); + // Only the preceding empty type's name is unreadable; row 2 still owns + // FieldDef 1 and MethodDef 1, including the local attribute constructor. + bytes.writeUInt16LE(0xffff, metadataRowOffset(layout, 2, 1) + 4); + const target = managedPeFixtureTarget(bytes); + const result = inspectManagedMembersBytes(bytes, target, { + typeNames: ["Fixture.LookupAttribute", "Fixture.Empty0"], + }); + + expect(result.types).toEqual([owner]); + expect(result.types[0]).toMatchObject({ + token: "0x02000002", + row_offset: metadataRowOffset(layout, 2, 2), + name_sha256: owner?.name_sha256, + field_list: { first_row: 1, last_row: 1, count: 1 }, + method_list: { first_row: 1, last_row: 1, count: 1 }, + }); + expect(result.fields).toEqual(healthy.fields); + expect(result.methods).toEqual(healthy.methods); + for (const member of [...result.fields, ...result.methods]) + expect(member).toMatchObject({ + declaring_type_token: "0x02000002", + declaring_type: "Fixture.LookupAttribute", + }); + expect(result.artifact.sha256).toBe(target.sha256); + expect(result.coverage.state).toBe("partial"); + expect(result.selection?.unmatched_type_names).toEqual(["Fixture.Empty0"]); + expect(result.limitations.join(" ")).toContain("absence is not observed"); + + // Member inspection uses the parsed-type name owner for its metadata + // inventory, whose attribute names are not included in the member report. + const issues: ManagedParseIssue[] = []; + const memberLists = readTypeMemberListFacts(bytes, layout); + const definitions = readTypeDefinitions(bytes, layout, issues, memberLists); + const parsedTypes = parseTypes(bytes, layout, definitions); + const names = managedMetadataTypeNames(bytes, layout, issues, parsedTypes); + const inventory = readManagedMetadataInventory( + bytes, + layout, + readManagedResourceDirectory(pe, issues), + memberLists, + undefined, + names, + ); + expect(inventory.attributes).toMatchObject([ + { + constructor_token: "0x06000001", + type_name: "Fixture.LookupAttribute", + }, + ]); +}); diff --git a/src/dotnet/ManagedMetadataInventoryRows.ts b/src/dotnet/ManagedMetadataInventoryRows.ts index 0fba855438..3638aa89c2 100644 --- a/src/dotnet/ManagedMetadataInventoryRows.ts +++ b/src/dotnet/ManagedMetadataInventoryRows.ts @@ -2,7 +2,6 @@ import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; import { managedTypeReferences } from "./ManagedTypeReferences.js"; import { MAX_TYPE_NESTING_DEPTH } from "./ManagedTypeNameDigests.js"; import { readManagedTypeNesting } from "./ManagedTypeNesting.js"; -import type { ManagedTypeNesting } from "./ManagedTypeNesting.js"; import { memberDeclaringTypeRowLookup } from "./ManagedMemberOwnership.js"; import type { TypeMemberListFacts } from "./ManagedTypeMemberLists.js"; @@ -27,7 +26,7 @@ import { import { managedFailure, readManagedValue } from "./ManagedReaderFailure.js"; import type { ManagedTypeReferenceLookup } from "./ManagedTypeReferences.js"; import type { ManagedResourceDirectory } from "./ManagedMetadataInventory.js"; -import type { ManagedType } from "./ManagedMemberInspectorCore.js"; +import type { ManagedType, parseTypes } from "./ManagedMemberInspectorCore.js"; type ModuleIdentity = NonNullable; type AssemblyIdentity = NonNullable; @@ -246,10 +245,7 @@ export const managedMetadataTypeNames = ( bytes: Buffer, layout: ManagedMetadataLayout, issues: ManagedParseIssue[], - parsedTypes?: { - readonly types: readonly ManagedType[]; - readonly nesting: ManagedTypeNesting; - }, + parsedTypes?: Pick, "types" | "nesting">, parsedReferences?: ManagedTypeReferenceLookup, ): ManagedMetadataTypeNameLookup => { const nesting = @@ -259,8 +255,7 @@ export const managedMetadataTypeNames = ( const definitions = new Map(); if (parsedTypes !== undefined) { managedDecodeBudget(layout).reserve(parsedTypes.types.length * 128, null); - for (const type of parsedTypes.types) - definitions.set(Number.parseInt(type.token.slice(4), 16), type); + for (const { row, type } of parsedTypes.types) definitions.set(row, type); } const cache = new Map< string, diff --git a/src/dotnet/ManagedTypeNameDigests.ts b/src/dotnet/ManagedTypeNameDigests.ts index c470a5a86f..9c07a82eb1 100644 --- a/src/dotnet/ManagedTypeNameDigests.ts +++ b/src/dotnet/ManagedTypeNameDigests.ts @@ -4,10 +4,10 @@ import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; import type { parseTypes, + ManagedType, SignatureTypeName, DeclarationTypeName, } from "./ManagedMemberInspectorCore.js"; -import { metadataToken } from "./ManagedMetadataHeaps.js"; import type { ManagedTypeReferenceLookup } from "./ManagedTypeReferences.js"; import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js"; import { managedFailure, readManagedValue } from "./ManagedReaderFailure.js"; @@ -153,11 +153,11 @@ export const managedTypeNameDigests = ( } => { const retain = () => managedDecodeBudget(layout).reserve(128, null); managedDecodeBudget(layout).reserve(parsedTypes.types.length * 128, null); - const types = new Map(); - for (const type of parsedTypes.types) types.set(type.token, type); + const types = new Map(); + for (const { row, type } of parsedTypes.types) types.set(row, type); const definitions = nestedTypeDigests((row) => { if (!parsedTypes.nesting.complete) return null; - const type = types.get(metadataToken(TYPE_DEF, row)); + const type = types.get(row); if ( type === undefined && (row < 1 || row > (layout.table(TYPE_DEF)?.rowCount ?? 0)) diff --git a/tests/boundary/filesystem/javascript-applications/javascriptApplicationService.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptApplicationService.test.ts index 7bec8204a8..9bf2f1eecf 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptApplicationService.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptApplicationService.test.ts @@ -181,21 +181,25 @@ it("resolves lazy package and JSON facts without completing their skipped observ outcome.analysis, ); expect(graph.coverage.status).toBe("partial"); - expect( - graph.edges.find( - ({ relation, properties }) => - relation === "imports" && properties.specifier === "late-pkg", - )?.properties, - ).toMatchObject({ + const packageRelationships = graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && + properties.module_link_kind === "require" && + properties.specifier === "late-pkg", + ); + expect(packageRelationships).toHaveLength(1); + expect(packageRelationships[0]?.properties).toMatchObject({ resolution_status: "resolved", resolved_path: "app/node_modules/late-pkg/index.cjs", }); - expect( - graph.edges.find( - ({ relation, properties }) => - relation === "imports" && properties.specifier === "./config.json", - )?.properties, - ).toMatchObject({ + const jsonRelationships = graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && + properties.module_link_kind === "require" && + properties.specifier === "./config.json", + ); + expect(jsonRelationships).toHaveLength(1); + expect(jsonRelationships[0]?.properties).toMatchObject({ target_file_kind: "json", target_json_status: "included", }); diff --git a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts index fc40ec29da..714d5b8e5b 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts @@ -19,7 +19,7 @@ import { } from "../../../support/javascriptApplicationScope.js"; import { analyzeJavaScriptArtifactFiles, - analyzeAndProjectJavaScriptArtifactFiles, + analyzeAndAdoptJavaScriptArtifactFiles, } from "../../../../src/application/javascript/JavaScriptArtifactAnalysis.js"; import { buildJavaScriptArtifactGraph } from "../../../../src/application/javascript/JavaScriptArtifactGraphBuilder.js"; import { @@ -40,6 +40,7 @@ import { } from "../../../../src/javascript/analysis/JavaScriptAnalysisTransfer.js"; import { javaScriptWorkerStaticAnalysisSchema } from "../../../../src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.js"; import { createJavaScriptArtifactReader } from "../../../../src/artifacts/javascript/JavaScriptArtifactReader.js"; +import { createJavaScriptSourceAnalysis } from "../../../../src/artifacts/javascript/JavaScriptSourceAnalysis.js"; import { scanCanonicalArtifactInventory, scanArtifactInventory, @@ -556,7 +557,7 @@ it("reports file progress and honors cancellation before the next source", async expect(progress[0]).not.toBe(progress[1]); }); -it("preserves graph commitments and export shapes when consuming file-local IR", async () => { +it("preserves graph commitments and export shapes through worker adoption", async () => { const root = await fixtureDirectory(); await writeFile( join(root, "exports.mjs"), @@ -574,10 +575,27 @@ it("preserves graph commitments and export shapes when consuming file-local IR", const files = await readJavaScriptArtifactFiles(reader, snapshot); const full = analyzeJavaScriptArtifactFiles(files); const projection = createJavaScriptSemanticGraphProjection(); - const compact = await analyzeAndProjectJavaScriptArtifactFiles( - files, - projection.projectFile, + const worker = await createJavaScriptSourceAnalysis( + { heapMb: 512, timeoutMs: 30_000 }, + files.files.filter( + (file) => file.kind === "javascript" && file.text.included, + ).length, ); + let outcome: Awaited< + ReturnType + >; + try { + outcome = await analyzeAndAdoptJavaScriptArtifactFiles( + files, + worker, + projection, + async () => {}, + ); + } finally { + await worker.close(); + } + if (outcome.interruption !== null) throw outcome.interruption; + const compact = outcome.analysis; const graph = buildJavaScriptArtifactGraph(snapshot, files, full); expect(buildJavaScriptArtifactGraph(snapshot, files, compact)).toEqual( graph,